diff --git a/build-docker.sh b/build-docker.sh index 669ec48..aeef9ca 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -1,11 +1,35 @@ #!/bin/bash +set -x +set -e +# to build your own image: +# export HBASE_VERSION=2.2.4 +# export DOCKER_REGISTRY=quay.io/kaszpir +# export HADOOP_IMAGE=quay.io/kaszpir/dockerized-hadoop:3.1.3 +# ./build-docker.sh -VERSION=2.0.1 +# hbase version +: ${HBASE_VERSION:='2.2.4'} -docker build --build-arg VERSION=$VERSION -t gradiant/hbase-base:$VERSION hbase-base -docker build --build-arg VERSION=$VERSION -t gradiant/hbase-master:$VERSION hbase-master -docker build --build-arg VERSION=$VERSION -t gradiant/hbase-region:$VERSION hbase-region +# docker registry for destination images +: ${DOCKER_REGISTRY:='quay.io/kaszpir'} -docker tag gradiant/hbase-base:$VERSION gradiant/hbase-base:latest -docker tag gradiant/hbase-master:$VERSION gradiant/hbase-master:latest -docker tag gradiant/hbase-region:$VERSION gradiant/hbase-region:latest +# docker image for hbase-* image +: ${HADOOP_IMAGE:='quay.io/kaszpir/dockerized-hadoop:3.1.3'} + +docker build --build-arg version=$HBASE_VERSION --build-arg HADOOP_IMAGE=$HADOOP_IMAGE -t ${DOCKER_REGISTRY}/hbase-base:$HBASE_VERSION hbase-base + +HBASE_IMAGE="${DOCKER_REGISTRY}/hbase-base:$HBASE_VERSION" +docker build --build-arg version=$HBASE_VERSION --build-arg HBASE_IMAGE=$HBASE_IMAGE -t ${DOCKER_REGISTRY}/hbase-master:$HBASE_VERSION hbase-master +docker build --build-arg version=$HBASE_VERSION --build-arg HBASE_IMAGE=$HBASE_IMAGE -t ${DOCKER_REGISTRY}/hbase-region:$HBASE_VERSION hbase-region + +docker tag ${DOCKER_REGISTRY}/hbase-base:$HBASE_VERSION ${DOCKER_REGISTRY}/hbase-base:latest +docker tag ${DOCKER_REGISTRY}/hbase-master:$HBASE_VERSION ${DOCKER_REGISTRY}/hbase-master:latest +docker tag ${DOCKER_REGISTRY}/hbase-region:$HBASE_VERSION ${DOCKER_REGISTRY}/hbase-region:latest + + +docker push ${DOCKER_REGISTRY}/hbase-base:$HBASE_VERSION +docker push ${DOCKER_REGISTRY}/hbase-base:latest +docker push ${DOCKER_REGISTRY}/hbase-master:$HBASE_VERSION +docker push ${DOCKER_REGISTRY}/hbase-master:latest +docker push ${DOCKER_REGISTRY}/hbase-region:$HBASE_VERSION +docker push ${DOCKER_REGISTRY}/hbase-region:latest diff --git a/docker-compose.yml b/docker-compose.yml index 92dd26a..0587736 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,50 +1,69 @@ -version: '3' +# env vars to set before start +# VERSION=2.1.10 - hbase version +# DOCKER_REGISTRY=gradiant - docker registry +# HADOOP_VERSION=2.7.7 - hadoop version tag +# HDFS_IMAGE_NAMENODE=hdfs-namenode - docker image to use for hdfs namenode +# HDFS_IMAGE_DATANODE=hdfs-datanode - docker image to use for hdfs namenode +--- +version: "3" services: hbase-master: - image: gradiant/hbase-master + image: "${DOCKER_REGISTRY}/hbase-master:${VERSION}" environment: - - HBASE_CONF_hbase_rootdir=hdfs://hdfs-namenode:8020/hbase - - HBASE_CONF_hbase_master_port=60000 - - HBASE_CONF_hbase_master_info_port=60010 - - HBASE_CONF_hbase_zookeeper_quorum=zookeeper + HBASE_CONF_hbase_rootdir: "hdfs://hdfs-namenode:8020/hbase" + HBASE_CONF_hbase_master_port: "60000" + HBASE_CONF_hbase_master_info_port: "60010" + HBASE_CONF_hbase_zookeeper_quorum: "zookeeper" + HBASE_OPTS: "-XX:+PrintFlagsFinal -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XshowSettings:vm" ports: - - 60010:60010 + - "60010:60010" depends_on: - - zookeeper - - hdfs-namenode + - "zookeeper" + - "hdfs-namenode" hbase-region: - image: gradiant/hbase-region + image: "${DOCKER_REGISTRY}/hbase-region:${VERSION}" environment: - - HBASE_CONF_hbase_rootdir=hdfs://hdfs-namenode:8020/hbase - - HBASE_CONF_hbase_master=hbase-master:60000 - - HBASE_CONF_hbase_regionserver_info_port=60030 - - HBASE_CONF_hbase_zookeeper_quorum=zookeeper + HBASE_CONF_hbase_rootdir: "hdfs://hdfs-namenode:8020/hbase" + HBASE_CONF_hbase_master: "hbase-master:60000" + HBASE_CONF_hbase_regionserver_info_port: "60030" + HBASE_CONF_hbase_zookeeper_quorum: "zookeeper" + HBASE_OPTS: "-XX:+PrintFlagsFinal -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XshowSettings:vm" ports: - - 60030:60030 + - "60030:60030" depends_on: - - hbase-master + - "hbase-master" zookeeper: - image: zookeeper:3.4 + image: "zookeeper:3.4" hdfs-namenode: - image: gradiant/hdfs-namenode:2.7.7 + image: "${DOCKER_REGISTRY}/${HDFS_IMAGE_NAMENODE}:${HADOOP_VERSION}" + command: "namenode" environment: - - CLUSTER_NAME=hdfs-cluster - - HDFS_CONF_dfs_replication=1 + CLUSTER_NAME: "hdfs-cluster" + HDFS_CONF_dfs_replication: "1" + HADOOP_SHELL_SCRIPT_DEBUG: "true" + HADOOP_OPTS: "-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XshowSettings:vm" + ports: + - "50070:50070" # hdfs 2.x + - "19870:9870" # hdfs 3.x volumes: - - name:/hadoop/dfs/name + - "name:/hadoop/dfs/name" + hdfs-datanode: - image: gradiant/hdfs-datanode:2.7.7 + image: "${DOCKER_REGISTRY}/${HDFS_IMAGE_DATANODE}:${HADOOP_VERSION}" + command: "datanode" environment: - - CORE_CONF_fs_defaultFS=hdfs://hdfs-namenode:8020 - - HDFS_CONF_dfs_replication=1 - - HDFS_CONF_dfs_datanode_data_dir=file:///hadoop/dfs/data + CORE_CONF_fs_defaultFS: "hdfs://hdfs-namenode:8020" + HDFS_CONF_dfs_replication: "1" + HDFS_CONF_dfs_datanode_data_dir: "file:///dfs/data" + HADOOP_SHELL_SCRIPT_DEBUG: "true" + HDFS_DATANODE_OPTS: "-XX:+PrintFlagsFinal -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XshowSettings:vm" volumes: - - data-0:/hadoop/dfs/data + - "data-0:/hadoop/dfs/data" depends_on: - - hdfs-namenode + - "hdfs-namenode" volumes: data-0: - name: \ No newline at end of file + name: diff --git a/hbase-base/Dockerfile b/hbase-base/Dockerfile index 9db6225..a45adca 100644 --- a/hbase-base/Dockerfile +++ b/hbase-base/Dockerfile @@ -1,29 +1,59 @@ -FROM gradiant/hadoop-base:2.7.7 as hadoop +ARG HADOOP_IMAGE=quay.io/kaszpir/hadoop-base:3.1.3 +FROM $HADOOP_IMAGE AS hadoop -FROM openjdk:8u171-jre-alpine3.8 +FROM openjdk:8-jre-slim LABEL maintainer="cgiraldo@gradiant.org" LABEL organization="gradiant.org" -ARG VERSION=2.0.1 -ENV HBASE_VERSION=$VERSION +ARG version=2.2.4 +ENV HBASE_VERSION=$version +# notice to set valid HADOOP_VERSION as stated in base container +ENV HADOOP_VERSION=3.1.3 \ + HADOOP_USER_NAME=hdfs \ + USER=hbase + +# buster uses jruby wich requires openjdk-11, so we add stretch and instal jruby from it +# need to add /usr/share/man/man1 dit to avoid issues during install +RUN echo 'deb http://deb.debian.org/debian stretch main' >> /etc/apt/sources.list && \ + DEBIAN_FRONTEND=noninteractive apt-get update && \ + mkdir -p /usr/share/man/man1 && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + bash \ + curl \ + default-jre-headless=2:1.8-58+deb9u1 \ + default-jre=2:1.8-58+deb9u1 \ + dnsutils \ + jruby=1.7.26-1+deb9u1 \ + net-tools \ + perl \ + procps \ + psmisc \ + sed \ + wget \ + && \ + rm -rf /var/lib/apt/lists/* && \ + mkdir -p /opt && \ + wget -qO- "http://archive.apache.org/dist/hbase/$HBASE_VERSION/hbase-$HBASE_VERSION-bin.tar.gz" | tar xvz -C /opt && \ + ln -s "/opt/hbase-$HBASE_VERSION" /opt/hbase && \ + ln -s "/opt/hbase-$HBASE_VERSION/conf" /etc/hbase && \ + groupadd -g 114 -r hadoop && \ + useradd --comment "HBase" -u 202 --shell /bin/bash -M -r --groups hadoop --home /var/lib/hbase/hbase hbase && \ + mkdir -p "/opt/hbase-$HBASE_VERSION/logs" && \ + chown -LR hbase:hadoop "/opt/hbase/" && \ + find / -name "java.security" 2>/dev/null | xargs -n1 sed -i 's/.*networkaddress.cache.ttl.*/networkaddress.cache.ttl=30/g' -RUN apk add --no-cache bash perl jruby curl && mkdir /opt && \ -wget -qO- http://archive.apache.org/dist/hbase/$HBASE_VERSION/hbase-$HBASE_VERSION-bin.tar.gz | tar xvz -C /opt && \ -ln -s /opt/hbase-$HBASE_VERSION /opt/hbase && \ -ln -s /opt/hbase-$HBASE_VERSION/conf /etc/hbase && \ -mkdir /opt/hbase-$HBASE_VERSION/logs ENV HBASE_PREFIX=/opt/hbase-$HBASE_VERSION ENV HBASE_CONF_DIR=/etc/hbase ENV HBASE_CONF_hbase_cluster_distributed=true ENV PATH $HBASE_PREFIX/bin/:$PATH +USER hbase +WORKDIR /opt/hbase/ -# Add hadoop native libraries for alpine -COPY --from=hadoop /opt/hadoop/lib/native/* /lib/ - - +# Add hadoop native libraries +# notice that COPY --from does not follow symlinks +COPY --from=hadoop /opt/hadoop-$HADOOP_VERSION/lib/native/* /lib/ COPY entrypoint.sh /entrypoint.sh - ENTRYPOINT ["/entrypoint.sh"] diff --git a/hbase-master/Dockerfile b/hbase-master/Dockerfile index e03be6e..000147b 100644 --- a/hbase-master/Dockerfile +++ b/hbase-master/Dockerfile @@ -1,5 +1,6 @@ -ARG VERSION=2.0.1 -FROM gradiant/hbase-base:$VERSION +ARG VERSION=2.1.10 +ARG HBASE_IMAGE=gradiant/hbase-base:$VERSION +FROM $HBASE_IMAGE LABEL maintainer="cgiraldo@gradiant.org" LABEL organization="gradiant.org" @@ -11,4 +12,4 @@ ENV HBASE_CONF_hbase_master_info_port=60010 EXPOSE 16000 16010 EXPOSE 60010 -CMD ["/run.sh"] \ No newline at end of file +CMD ["/run.sh"] diff --git a/hbase-master/run.sh b/hbase-master/run.sh index ee21ee9..3330c89 100755 --- a/hbase-master/run.sh +++ b/hbase-master/run.sh @@ -1,3 +1,3 @@ #!/bin/bash -/opt/hbase-$HBASE_VERSION/bin/hbase master start +exec /opt/hbase-$HBASE_VERSION/bin/hbase master start diff --git a/hbase-region/Dockerfile b/hbase-region/Dockerfile index 8d4bb95..a158cad 100644 --- a/hbase-region/Dockerfile +++ b/hbase-region/Dockerfile @@ -1,5 +1,6 @@ -ARG VERSION=2.0.1 -FROM gradiant/hbase-base:$VERSION +ARG VERSION=2.1.10 +ARG HBASE_IMAGE=gradiant/hbase-base:$VERSION +FROM $HBASE_IMAGE LABEL maintainer="cgiraldo@gradiant.org" LABEL organization="gradiant.org" @@ -8,4 +9,4 @@ COPY run.sh /run.sh EXPOSE 16020 16030 -CMD ["/run.sh"] \ No newline at end of file +CMD ["/run.sh"] diff --git a/hbase-region/run.sh b/hbase-region/run.sh index fe94ec0..4854733 100755 --- a/hbase-region/run.sh +++ b/hbase-region/run.sh @@ -1,3 +1,3 @@ #!/bin/bash -/opt/hbase-$HBASE_VERSION/bin/hbase regionserver start \ No newline at end of file +exec /opt/hbase-$HBASE_VERSION/bin/hbase regionserver start