Gradiant · nvtkaszpir · Apr 26, 2020 · Apr 27, 2020 · May 27, 2020
diff --git a/build-docker.sh b/build-docker.sh
@@ -1,11 +1,35 @@
 #!/bin/bash
+set -x
+set -e
+# to build your own image:
+# export HBASE_VERSION=2.2.4
+# export DOCKER_REGISTRY=quay.io/kaszpir
+# export HADOOP_IMAGE=quay.io/kaszpir/dockerized-hadoop:3.1.3
+# ./build-docker.sh
 
-VERSION=2.0.1
+#  hbase version
+: ${HBASE_VERSION:='2.2.4'}
 
-docker build --build-arg VERSION=$VERSION -t gradiant/hbase-base:$VERSION hbase-base
-docker build --build-arg VERSION=$VERSION -t gradiant/hbase-master:$VERSION hbase-master
-docker build --build-arg VERSION=$VERSION -t gradiant/hbase-region:$VERSION hbase-region
+# docker registry for destination images
+: ${DOCKER_REGISTRY:='quay.io/kaszpir'}
 
-docker tag gradiant/hbase-base:$VERSION gradiant/hbase-base:latest
-docker tag gradiant/hbase-master:$VERSION gradiant/hbase-master:latest
-docker tag gradiant/hbase-region:$VERSION gradiant/hbase-region:latest
+# docker image for hbase-* image
+: ${HADOOP_IMAGE:='quay.io/kaszpir/dockerized-hadoop:3.1.3'}
+
+docker build --build-arg version=$HBASE_VERSION --build-arg HADOOP_IMAGE=$HADOOP_IMAGE -t ${DOCKER_REGISTRY}/hbase-base:$HBASE_VERSION hbase-base
+
+HBASE_IMAGE="${DOCKER_REGISTRY}/hbase-base:$HBASE_VERSION"
+docker build --build-arg version=$HBASE_VERSION --build-arg HBASE_IMAGE=$HBASE_IMAGE -t ${DOCKER_REGISTRY}/hbase-master:$HBASE_VERSION hbase-master
+docker build --build-arg version=$HBASE_VERSION --build-arg HBASE_IMAGE=$HBASE_IMAGE -t ${DOCKER_REGISTRY}/hbase-region:$HBASE_VERSION hbase-region
+
+docker tag ${DOCKER_REGISTRY}/hbase-base:$HBASE_VERSION ${DOCKER_REGISTRY}/hbase-base:latest
+docker tag ${DOCKER_REGISTRY}/hbase-master:$HBASE_VERSION ${DOCKER_REGISTRY}/hbase-master:latest
+docker tag ${DOCKER_REGISTRY}/hbase-region:$HBASE_VERSION ${DOCKER_REGISTRY}/hbase-region:latest
+
+
+docker push ${DOCKER_REGISTRY}/hbase-base:$HBASE_VERSION
+docker push ${DOCKER_REGISTRY}/hbase-base:latest
+docker push ${DOCKER_REGISTRY}/hbase-master:$HBASE_VERSION
+docker push ${DOCKER_REGISTRY}/hbase-master:latest
+docker push ${DOCKER_REGISTRY}/hbase-region:$HBASE_VERSION
+docker push ${DOCKER_REGISTRY}/hbase-region:latest
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,50 +1,69 @@
-version: '3'
+# env vars to set before start
+# VERSION=2.1.10 - hbase version
+# DOCKER_REGISTRY=gradiant - docker registry
+# HADOOP_VERSION=2.7.7 - hadoop version tag
+# HDFS_IMAGE_NAMENODE=hdfs-namenode - docker image to use for hdfs namenode
+# HDFS_IMAGE_DATANODE=hdfs-datanode - docker image to use for hdfs namenode
+---
+version: "3"
 services:
   hbase-master:
-    image: gradiant/hbase-master
+    image: "${DOCKER_REGISTRY}/hbase-master:${VERSION}"
     environment:
-      - HBASE_CONF_hbase_rootdir=hdfs://hdfs-namenode:8020/hbase
-      - HBASE_CONF_hbase_master_port=60000
-      - HBASE_CONF_hbase_master_info_port=60010
-      - HBASE_CONF_hbase_zookeeper_quorum=zookeeper
+      HBASE_CONF_hbase_rootdir: "hdfs://hdfs-namenode:8020/hbase"
+      HBASE_CONF_hbase_master_port: "60000"
+      HBASE_CONF_hbase_master_info_port: "60010"
+      HBASE_CONF_hbase_zookeeper_quorum: "zookeeper"
+      HBASE_OPTS: "-XX:+PrintFlagsFinal -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XshowSettings:vm"
     ports:
-      - 60010:60010
+      - "60010:60010"
     depends_on:
-      - zookeeper
-      - hdfs-namenode
+      - "zookeeper"
+      - "hdfs-namenode"
   hbase-region:
-    image: gradiant/hbase-region
+    image: "${DOCKER_REGISTRY}/hbase-region:${VERSION}"
     environment:
-      - HBASE_CONF_hbase_rootdir=hdfs://hdfs-namenode:8020/hbase
-      - HBASE_CONF_hbase_master=hbase-master:60000
-      - HBASE_CONF_hbase_regionserver_info_port=60030
-      - HBASE_CONF_hbase_zookeeper_quorum=zookeeper
+      HBASE_CONF_hbase_rootdir: "hdfs://hdfs-namenode:8020/hbase"
+      HBASE_CONF_hbase_master: "hbase-master:60000"
+      HBASE_CONF_hbase_regionserver_info_port: "60030"
+      HBASE_CONF_hbase_zookeeper_quorum: "zookeeper"
+      HBASE_OPTS: "-XX:+PrintFlagsFinal -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XshowSettings:vm"
     ports:
-      - 60030:60030
+      - "60030:60030"
     depends_on:
-      - hbase-master
+      - "hbase-master"
 
   zookeeper:
-    image: zookeeper:3.4
+    image: "zookeeper:3.4"
 
   hdfs-namenode:
-    image: gradiant/hdfs-namenode:2.7.7
+    image: "${DOCKER_REGISTRY}/${HDFS_IMAGE_NAMENODE}:${HADOOP_VERSION}"
+    command: "namenode"
     environment:
-    - CLUSTER_NAME=hdfs-cluster
-    - HDFS_CONF_dfs_replication=1
+      CLUSTER_NAME: "hdfs-cluster"
+      HDFS_CONF_dfs_replication: "1"
+      HADOOP_SHELL_SCRIPT_DEBUG: "true"
+      HADOOP_OPTS: "-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XshowSettings:vm"
+    ports:
+      - "50070:50070"  # hdfs 2.x
+      - "19870:9870"  # hdfs 3.x
     volumes:
-    - name:/hadoop/dfs/name
+      - "name:/hadoop/dfs/name"
+
   hdfs-datanode:
-    image: gradiant/hdfs-datanode:2.7.7
+    image: "${DOCKER_REGISTRY}/${HDFS_IMAGE_DATANODE}:${HADOOP_VERSION}"
+    command: "datanode"
     environment:
-    - CORE_CONF_fs_defaultFS=hdfs://hdfs-namenode:8020
-    - HDFS_CONF_dfs_replication=1
-    - HDFS_CONF_dfs_datanode_data_dir=file:///hadoop/dfs/data
+      CORE_CONF_fs_defaultFS: "hdfs://hdfs-namenode:8020"
+      HDFS_CONF_dfs_replication: "1"
+      HDFS_CONF_dfs_datanode_data_dir: "file:///dfs/data"
+      HADOOP_SHELL_SCRIPT_DEBUG: "true"
+      HDFS_DATANODE_OPTS: "-XX:+PrintFlagsFinal -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XshowSettings:vm"
     volumes:
-    - data-0:/hadoop/dfs/data
+      - "data-0:/hadoop/dfs/data"
     depends_on:
-    - hdfs-namenode
+      - "hdfs-namenode"
 
 volumes:
   data-0:
-  name:
+  name:
diff --git a/hbase-base/Dockerfile b/hbase-base/Dockerfile
@@ -1,29 +1,59 @@
-FROM gradiant/hadoop-base:2.7.7 as hadoop
+ARG HADOOP_IMAGE=quay.io/kaszpir/hadoop-base:3.1.3
+FROM $HADOOP_IMAGE AS hadoop
 
-FROM openjdk:8u171-jre-alpine3.8
+FROM openjdk:8-jre-slim
 LABEL maintainer="cgiraldo@gradiant.org"
 LABEL organization="gradiant.org"
-ARG VERSION=2.0.1
-ENV HBASE_VERSION=$VERSION
+ARG version=2.2.4
+ENV HBASE_VERSION=$version
+# notice to set valid HADOOP_VERSION as stated in base container
+ENV HADOOP_VERSION=3.1.3 \
+    HADOOP_USER_NAME=hdfs \
+    USER=hbase
+
+# buster uses jruby wich requires openjdk-11, so we add stretch and instal jruby from it
+# need to add /usr/share/man/man1 dit to avoid issues during install
+RUN echo 'deb http://deb.debian.org/debian stretch main' >> /etc/apt/sources.list && \
+    DEBIAN_FRONTEND=noninteractive apt-get update && \
+    mkdir -p /usr/share/man/man1 && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+      bash \
+      curl \
+      default-jre-headless=2:1.8-58+deb9u1 \
+      default-jre=2:1.8-58+deb9u1 \
+      dnsutils \
+      jruby=1.7.26-1+deb9u1 \
+      net-tools \
+      perl \
+      procps \
+      psmisc \
+      sed \
+      wget \
+    && \
+    rm -rf /var/lib/apt/lists/* && \
+    mkdir -p /opt && \
+    wget -qO- "http://archive.apache.org/dist/hbase/$HBASE_VERSION/hbase-$HBASE_VERSION-bin.tar.gz" | tar xvz -C /opt && \
+    ln -s "/opt/hbase-$HBASE_VERSION" /opt/hbase && \
+    ln -s "/opt/hbase-$HBASE_VERSION/conf" /etc/hbase  && \
+    groupadd -g 114 -r hadoop && \
+    useradd --comment "HBase" -u 202 --shell /bin/bash -M -r --groups hadoop --home /var/lib/hbase/hbase hbase && \
+    mkdir -p "/opt/hbase-$HBASE_VERSION/logs" && \
+    chown -LR hbase:hadoop "/opt/hbase/" && \
+    find / -name "java.security" 2>/dev/null | xargs -n1 sed -i 's/.*networkaddress.cache.ttl.*/networkaddress.cache.ttl=30/g'
 
-RUN apk add --no-cache bash perl jruby curl && mkdir /opt && \
-wget -qO- http://archive.apache.org/dist/hbase/$HBASE_VERSION/hbase-$HBASE_VERSION-bin.tar.gz | tar xvz -C /opt && \
-ln -s /opt/hbase-$HBASE_VERSION /opt/hbase && \
-ln -s /opt/hbase-$HBASE_VERSION/conf /etc/hbase  && \
-mkdir /opt/hbase-$HBASE_VERSION/logs
 
 ENV HBASE_PREFIX=/opt/hbase-$HBASE_VERSION
 ENV HBASE_CONF_DIR=/etc/hbase
 ENV HBASE_CONF_hbase_cluster_distributed=true
 ENV PATH $HBASE_PREFIX/bin/:$PATH
+USER hbase
+WORKDIR /opt/hbase/
 
-# Add hadoop native libraries for alpine
-COPY --from=hadoop /opt/hadoop/lib/native/* /lib/
-
-
+# Add hadoop native libraries
+# notice that COPY --from does not follow symlinks
+COPY --from=hadoop /opt/hadoop-$HADOOP_VERSION/lib/native/* /lib/
 
 COPY entrypoint.sh /entrypoint.sh
 
-
 ENTRYPOINT ["/entrypoint.sh"]
 
diff --git a/hbase-master/Dockerfile b/hbase-master/Dockerfile
@@ -1,5 +1,6 @@
-ARG VERSION=2.0.1
-FROM gradiant/hbase-base:$VERSION
+ARG VERSION=2.1.10
+ARG HBASE_IMAGE=gradiant/hbase-base:$VERSION
+FROM $HBASE_IMAGE
 
 LABEL maintainer="cgiraldo@gradiant.org"
 LABEL organization="gradiant.org"
@@ -11,4 +12,4 @@ ENV HBASE_CONF_hbase_master_info_port=60010
 EXPOSE 16000 16010
 EXPOSE 60010
 
-CMD ["/run.sh"]
+CMD ["/run.sh"]
diff --git a/hbase-master/run.sh b/hbase-master/run.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 
-/opt/hbase-$HBASE_VERSION/bin/hbase master start
+exec /opt/hbase-$HBASE_VERSION/bin/hbase master start
diff --git a/hbase-region/Dockerfile b/hbase-region/Dockerfile
@@ -1,5 +1,6 @@
-ARG VERSION=2.0.1
-FROM gradiant/hbase-base:$VERSION
+ARG VERSION=2.1.10
+ARG HBASE_IMAGE=gradiant/hbase-base:$VERSION
+FROM $HBASE_IMAGE
 
 LABEL maintainer="cgiraldo@gradiant.org"
 LABEL organization="gradiant.org"
@@ -8,4 +9,4 @@ COPY run.sh /run.sh
 
 EXPOSE 16020 16030
 
-CMD ["/run.sh"]
+CMD ["/run.sh"]
diff --git a/hbase-region/run.sh b/hbase-region/run.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 
-/opt/hbase-$HBASE_VERSION/bin/hbase regionserver start
+exec /opt/hbase-$HBASE_VERSION/bin/hbase regionserver start