From bf36aea7b39423bbfe0baa1aa61c02919581c17c Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Thu, 30 Mar 2017 11:13:48 -0400 Subject: [PATCH 01/11] Adding MapR52-Base docker image This image contains installation of all mapr related softwares and there dependencies --- teradatalabs/mapr52-base/Dockerfile | 69 ++++++++++++++++++++ teradatalabs/mapr52-base/README.md | 9 +++ teradatalabs/mapr52-base/files/id_rsa.pub | 1 + teradatalabs/mapr52-base/files/maprtech.repo | 13 ++++ 4 files changed, 92 insertions(+) create mode 100644 teradatalabs/mapr52-base/Dockerfile create mode 100644 teradatalabs/mapr52-base/README.md create mode 100644 teradatalabs/mapr52-base/files/id_rsa.pub create mode 100644 teradatalabs/mapr52-base/files/maprtech.repo diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr52-base/Dockerfile new file mode 100644 index 0000000..25edeac --- /dev/null +++ b/teradatalabs/mapr52-base/Dockerfile @@ -0,0 +1,69 @@ +# Copyright 2017 Teradata +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM teradatalabs/centos6-java8-oracle +MAINTAINER Teradata Docker Team + +# ADD REPO FOR MAPR +ADD files/maprtech.repo /etc/yum.repos.d/maprtech.repo +COPY files/id_rsa.pub /root/ +RUN yum update -y \ +# ... GET MapRGPG KEY + && rpm --import http://package.mapr.com/releases/pub/maprgpg.key \ + +# INSTALL UTILITY SOFTWARE + && yum install -y iputils vim openssh-server openssh-clients sudo lsof \ +# CONFIGURE SSH + && chkconfig sshd on \ + && grep -rl '#Port 22' /etc/ssh/sshd_config | xargs sed -i 's/#Port 22/Port 22/g' \ + +# INSTALL MAPR + && yum install -y mapr-fileserver mapr-nfs mapr-nodemanager mapr-cldb \ + && yum install -y mapr-zookeeper mapr-resourcemanager mapr-historyserver \ + && yum install -y mapr-webserver mapr-gateway mapr-httpfs \ + +# ADD USERS AND CHANGE OWNERSHIPS + && adduser mapr \ + && adduser hive \ + && adduser hdfs \ + && touch /home/mapr /home/hive /home/hdfs \ + && echo "cd /home/mapr" >> /home/mapr/.bashrc \ + && echo "cd /home/hive" >> /home/hive/.bashrc \ + && echo "cd /home/hdfs" >> /home/hdfs/.bashrc \ + && chown -R mapr:mapr /home/mapr /opt/mapr/httpfs \ + && chown hive:hive /home/hive \ + && chown hdfs:hdfs /home/hdfs \ +# CONFIGURE ZOOKEEPER'S DATA DIRECTORY + && rm -rf /opt/mapr/zkdata \ + && mkdir /opt/mapr/zkdata \ + && chmod 777 /opt/mapr/zkdata \ + && mkdir -p /mapr \ + +# INSTALL PYTHON AND SUPERVISORD + && yum install -y python-setuptools \ + && easy_install pip \ + && pip install supervisor \ + && mkdir /etc/supervisord.d/ \ +# ... AND ITS MISSING DEPENDENCY + && wget http://dl.fedoraproject.org/pub/epel/6/x86_64/python-meld3-0.6.7-1.el6.x86_64.rpm \ + && rpm -ihv python-meld3-0.6.7-1.el6.x86_64.rpm \ + && rm python-meld3-0.6.7-1.el6.x86_64.rpm \ + +# CLEANUP + && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ + +# GENERATE SSH KEYS + && ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa \ + && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ + && cat /root/id_rsa.pub | cat >> ~/.ssh/authorized_keys diff --git a/teradatalabs/mapr52-base/README.md b/teradatalabs/mapr52-base/README.md new file mode 100644 index 0000000..3f5439e --- /dev/null +++ b/teradatalabs/mapr52-base/README.md @@ -0,0 +1,9 @@ +# mapr52-base + + +Docker image with all MapR related softwares installed and there dependencies. + +## Oracle license + +By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: +[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-base/files/id_rsa.pub b/teradatalabs/mapr52-base/files/id_rsa.pub new file mode 100644 index 0000000..cdd0869 --- /dev/null +++ b/teradatalabs/mapr52-base/files/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA1PL4EwRZFy1ewBTa4a1TK+mQ4rAupOeZsiqir/su61dAGvC6pEFAa+Litj6ub6NvcBRMAdXeBtbOnQpInE7BFwKVhwU3n60Mc69SjLiozK3Oxh9sfmbJv/JdELRS5aB9x82Y0bO5fZFPFj7SxPNMugQQMEMQHW01wsa5nJR2pYLwCtu7yoD6fQ0TJEsRqWwyQTNoR19yzL6h7p/hq9SqiqCKfsHWK4+Tj0IgF7Nwz8i+BqqOq2kC9lTRuT8HalNbqVVQ6iI+ER7FgdfSZtKKX6R9SOaKQ7p0Dt6JLFibMNhjwt5EKHsgfMOsl1G8SEncDREtTng8/JLlvIhiqmWzwQ== root@d57cdb1934d1 diff --git a/teradatalabs/mapr52-base/files/maprtech.repo b/teradatalabs/mapr52-base/files/maprtech.repo new file mode 100644 index 0000000..bbb4997 --- /dev/null +++ b/teradatalabs/mapr52-base/files/maprtech.repo @@ -0,0 +1,13 @@ +[maprtech] +name=MapR Technologies +baseurl=http://package.mapr.com/releases/v5.2.0/redhat/ +enabled=1 +gpgcheck=0 +protect=1 + +[maprecosystem] +name=MapR Technologies +baseurl=http://package.mapr.com/releases/MEP/MEP-1.0/redhat +enabled=1 +gpgcheck=0 +protect=1 From a95013cda3b4c6894af3902a0ba2c27aea1ec2fa Mon Sep 17 00:00:00 2001 From: root Date: Thu, 30 Mar 2017 15:51:27 +0000 Subject: [PATCH 02/11] Add docker container for MapR with hive --- teradatalabs/mapr52-hive/Dockerfile | 51 ++++++++++++++ teradatalabs/mapr52-hive/README.md | 17 +++++ teradatalabs/mapr52-hive/capabilities.txt | 4 ++ teradatalabs/mapr52-hive/files/bootstrap.sh | 31 +++++++++ .../mapr52-hive/files/conf/core-site.xml | 33 +++++++++ .../mapr52-hive/files/conf/hive-site.xml | 67 +++++++++++++++++++ teradatalabs/mapr52-hive/files/setup.sh | 16 +++++ teradatalabs/mapr52-hive/files/socks-proxy.sh | 1 + teradatalabs/mapr52-hive/files/startup.sh | 3 + .../mapr52-hive/files/supervisord.conf | 1 + .../files/supervisord.d/bootstrap.conf | 1 + .../files/supervisord.d/mysql-metastore.conf | 1 + .../files/supervisord.d/socks-proxy.conf | 1 + .../mapr52-hive/files/supervisord.d/sshd.conf | 1 + .../mapr52-hive/files/wardenTracker.sh | 54 +++++++++++++++ 15 files changed, 282 insertions(+) create mode 100644 teradatalabs/mapr52-hive/Dockerfile create mode 100644 teradatalabs/mapr52-hive/README.md create mode 100644 teradatalabs/mapr52-hive/capabilities.txt create mode 100644 teradatalabs/mapr52-hive/files/bootstrap.sh create mode 100644 teradatalabs/mapr52-hive/files/conf/core-site.xml create mode 100644 teradatalabs/mapr52-hive/files/conf/hive-site.xml create mode 100644 teradatalabs/mapr52-hive/files/setup.sh create mode 120000 teradatalabs/mapr52-hive/files/socks-proxy.sh create mode 100644 teradatalabs/mapr52-hive/files/startup.sh create mode 120000 teradatalabs/mapr52-hive/files/supervisord.conf create mode 120000 teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf create mode 120000 teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf create mode 120000 teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf create mode 120000 teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf create mode 100644 teradatalabs/mapr52-hive/files/wardenTracker.sh diff --git a/teradatalabs/mapr52-hive/Dockerfile b/teradatalabs/mapr52-hive/Dockerfile new file mode 100644 index 0000000..9d3f68f --- /dev/null +++ b/teradatalabs/mapr52-hive/Dockerfile @@ -0,0 +1,51 @@ +# Copyright 2017 Teradata +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM teradatalabs/mapr52-base +MAINTAINER Teradata Docker Team + +# ADD ALL REQUIRED SCRIPTS AND FILES TO ROOT DIRECTORY +ADD files/*.sh /root/ +ADD files/conf/hive-site.xml /opt/mapr/hive/hive-1.2/conf/hive-site.xml +ADD files/conf/core-site.xml /opt/mapr/hadoop/hadoop-2.7.0/etc/hadoop/core-site.xml +ADD files/supervisord.conf /etc/supervisord.conf +COPY files/supervisord.d/* /etc/supervisord.d/ +RUN chmod 777 /root/*.sh \ + +# INSTALL MYSQL SERVER + && yum install -y mysql-server mysql-connector-java \ + && /root/setup.sh \ + +# INSTALL MAPR HIVE + && yum install -y mapr-hive mapr-hiveserver2 mapr-hivemetastore \ + && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ + +# SETUP FLAT FILE /home/mapr/storagefile + && dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 \ +# THE /root/disk.txt IS READ BY MapR TO DETERMINE WHICH DISKS IT CAN USE AND THE /home/mapr/storagefile IS A FLAT FILE THAT ACTS AS A DISK + && echo "/home/mapr/storagefile" > /root/disk.txt + +# HDFS PORTS +EXPOSE 5660 6660 5692 5724 5756 6692 6724 6756 + +# YARN PORTS +EXPOSE 8030 8020 7222 8088 7220 7221 8443 8031 8032 8033 8040 8041 8042 8088 10020 19888 + +# HIVE PORT +EXPOSE 9083 10000 22 + +# SOCKS PORT +EXPOSE 1180 + +CMD /root/startup.sh diff --git a/teradatalabs/mapr52-hive/README.md b/teradatalabs/mapr52-hive/README.md new file mode 100644 index 0000000..ebf092a --- /dev/null +++ b/teradatalabs/mapr52-hive/README.md @@ -0,0 +1,17 @@ +# mapr52-hive + +Docker image with HDFS, YARN and HIVE installed. Please note that running services have lower memory heap size set. +For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file. +If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal +for general correctness testing. + +## Run + +``` +$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr52-hive +``` + +## Oracle license + +By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: +[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-hive/capabilities.txt b/teradatalabs/mapr52-hive/capabilities.txt new file mode 100644 index 0000000..c1da436 --- /dev/null +++ b/teradatalabs/mapr52-hive/capabilities.txt @@ -0,0 +1,4 @@ +exposes_mapr +allows_creating_a_table_in_hive +allows_selecting_from_the_table +exposes_socks_proxy \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/bootstrap.sh b/teradatalabs/mapr52-hive/files/bootstrap.sh new file mode 100644 index 0000000..384ad26 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/bootstrap.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +# START SSHD AND THE SOCKS PROXY FOR THE HIVE METASTORE +supervisorctl start sshd +supervisorctl start socks-proxy + +# CONFIGURE MAPR +/opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart + +# SETUP DISK FOR MAPR BY RUNNING DISKSETUP +/opt/mapr/server/disksetup -M -F /root/disk.txt + +# CREATE HIVE PROXY USERS +chmod 755 /opt/mapr/conf/proxy + +# START SERVICES +service mapr-zookeeper start +service mapr-warden start + +# CONFIGURE HIVE +/opt/mapr/server/configure.sh -R + +# WAIT FOR WARDEN TO START ALL THE SERVICES +sh /root/wardenTracker.sh + +# START HTTPFS SERVICES +maprcli node services -name httpfs -action start -nodes $(hostname) + +# RUN HDFS COMMANDS +hadoop fs -mkdir /user/root /user/hive /user/hdfs /user/hive/warehouse /var /var/mapr /var/mapr/cluster /var/mapr/cluster/yarn /var/mapr/cluster/yarn/rm /var/mapr/cluster/yarn/rm/staging /var/mapr/cluster/yarn/rm/staging/hive +hadoop fs -chmod 777 /user/hive /user/hdfs /user/hive/warehouse /var/mapr /var/mapr/cluster/yarn/rm/staging/hive diff --git a/teradatalabs/mapr52-hive/files/conf/core-site.xml b/teradatalabs/mapr52-hive/files/conf/core-site.xml new file mode 100644 index 0000000..6549e0b --- /dev/null +++ b/teradatalabs/mapr52-hive/files/conf/core-site.xml @@ -0,0 +1,33 @@ + + + + + + + + + hadoop.proxyuser.mapr.groups + * + Allow the superuser mapr to impersonate any member of any group + + + + hadoop.proxyuser.mapr.hosts + * + The superuser can connect from any host to impersonate a user + + + + diff --git a/teradatalabs/mapr52-hive/files/conf/hive-site.xml b/teradatalabs/mapr52-hive/files/conf/hive-site.xml new file mode 100644 index 0000000..3518b61 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/conf/hive-site.xml @@ -0,0 +1,67 @@ + + + + + + + datanucleus.schema.autoCreateAll + true + creates necessary schema on a startup if one doesn't exist. set + this to false, after creating it once + + + + hive.server2.enable.doAs + true + Set this property to enable impersonation in Hive Server 2 + + + + hive.metastore.execute.setugi + true + Set this property to enable Hive Metastore service impersonation in unsecure mode. In unsecure mode, setting this property to true will cause the metastore to execute DFS operations using the client's reported user and group permissions. Note that this property must be set on both the client and server sides. If the client sets it to true and the server sets it to false, the client setting will be ignored. + + + + javax.jdo.option.ConnectionURL + jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true + JDBC connect string for a JDBC metastore + + + + javax.jdo.option.ConnectionDriverName + com.mysql.jdbc.Driver + Driver class name for a JDBC metastore + + + + javax.jdo.option.ConnectionUserName + root + username to use against metastore database + + + + javax.jdo.option.ConnectionPassword + root + password to use against metastore database + + + + hive.metastore.uris + thrift://localhost:9083 + + + diff --git a/teradatalabs/mapr52-hive/files/setup.sh b/teradatalabs/mapr52-hive/files/setup.sh new file mode 100644 index 0000000..2781708 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/setup.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# SETUP METASTORE +mysql_install_db + +/usr/bin/mysqld_safe & +sleep 10s + +echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql +echo "CREATE DATABASE metastore; USE metastore; SOURCE /usr/lib/hive/scripts/metastore/upgrade/mysql/hive-schema-0.13.0.mysql.sql;" | mysql +/usr/bin/mysqladmin -u root password 'root' + +killall mysqld +sleep 10s +mkdir /var/log/mysql/ +chown mysql:mysql /var/log/mysql/ diff --git a/teradatalabs/mapr52-hive/files/socks-proxy.sh b/teradatalabs/mapr52-hive/files/socks-proxy.sh new file mode 120000 index 0000000..852d770 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/socks-proxy.sh @@ -0,0 +1 @@ +../../../commons/socks-proxy.sh \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/startup.sh b/teradatalabs/mapr52-hive/files/startup.sh new file mode 100644 index 0000000..afd8798 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/startup.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +exec supervisord -c /etc/supervisord.conf diff --git a/teradatalabs/mapr52-hive/files/supervisord.conf b/teradatalabs/mapr52-hive/files/supervisord.conf new file mode 120000 index 0000000..c821a2d --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.conf @@ -0,0 +1 @@ +../../../commons/supervisord.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf b/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf new file mode 120000 index 0000000..16a8759 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf @@ -0,0 +1 @@ +../../../../commons/supervisord.d/bootstrap.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf b/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf new file mode 120000 index 0000000..b48213c --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf @@ -0,0 +1 @@ +../../../../commons/supervisord.d/mysql-metastore.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf b/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf new file mode 120000 index 0000000..905e977 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf @@ -0,0 +1 @@ +../../../../commons/supervisord.d/socks-proxy.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf b/teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf new file mode 120000 index 0000000..31d04a8 --- /dev/null +++ b/teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf @@ -0,0 +1 @@ +../../../../commons/supervisord.d/sshd.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/wardenTracker.sh b/teradatalabs/mapr52-hive/files/wardenTracker.sh new file mode 100644 index 0000000..5124a4d --- /dev/null +++ b/teradatalabs/mapr52-hive/files/wardenTracker.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +hname=$(hostname) + +Services=0 + +maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) + +# WAIT FOR CLDB TO START +while [ $Services -ne 2 ] +do +if [ $maprcliReady == 1 ] +then + maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) + Services=0 +else + Services=$(maprcli service list -node $hname | grep CLDB |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +fi +done + +# WAIT FOR NODEMANAGER TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep NodeManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +# WAIT FOR RESOURCEMANAGER TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep ResourceManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +# WAIT FOR HIVE METASTORE TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep HiveMetastore |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +# WAIT FOR HIVESERVER2 TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep HiveServer2 |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +# WAIT FOR JOBHISTORYSERVER TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep JobHistoryServer |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done From bdc12e6df433ae875c827dbcea6ccab4a07f3dc3 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Thu, 30 Mar 2017 13:27:05 -0400 Subject: [PATCH 03/11] Add docker container for security/kerberos enabled MapR with hive This contaner is enabled with security on both MapR cluster and Hive --- .../mapr52-hive-kerberized/Dockerfile | 87 ++++++++++++++++ teradatalabs/mapr52-hive-kerberized/README.md | 17 ++++ .../mapr52-hive-kerberized/files/bootstrap.sh | 49 +++++++++ .../files/conf/hive-site.xml | 94 ++++++++++++++++++ .../files/jceJars/US_export_policy.jar | Bin 0 -> 3023 bytes .../files/jceJars/local_policy.jar | Bin 0 -> 3035 bytes .../files/kerberos/kadm5.acl | 1 + .../files/kerberos/kdc.conf | 12 +++ .../files/kerberos/krb5.conf | 18 ++++ 9 files changed, 278 insertions(+) create mode 100644 teradatalabs/mapr52-hive-kerberized/Dockerfile create mode 100644 teradatalabs/mapr52-hive-kerberized/README.md create mode 100644 teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh create mode 100644 teradatalabs/mapr52-hive-kerberized/files/conf/hive-site.xml create mode 100644 teradatalabs/mapr52-hive-kerberized/files/jceJars/US_export_policy.jar create mode 100644 teradatalabs/mapr52-hive-kerberized/files/jceJars/local_policy.jar create mode 100644 teradatalabs/mapr52-hive-kerberized/files/kerberos/kadm5.acl create mode 100644 teradatalabs/mapr52-hive-kerberized/files/kerberos/kdc.conf create mode 100644 teradatalabs/mapr52-hive-kerberized/files/kerberos/krb5.conf diff --git a/teradatalabs/mapr52-hive-kerberized/Dockerfile b/teradatalabs/mapr52-hive-kerberized/Dockerfile new file mode 100644 index 0000000..24956c2 --- /dev/null +++ b/teradatalabs/mapr52-hive-kerberized/Dockerfile @@ -0,0 +1,87 @@ +# Copyright 2017 Teradata +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM teradatalabs/mapr52-hive +MAINTAINER Teradata Docker Team + +# REMOVE UNNECESSARY FILES +RUN rm -rf /opt/mapr/conf/ssl_truststore \ + && rm -rf /opt/mapr/conf/maprserverticket \ + && rm -rf /opt/mapr/conf/cldb.key \ + && rm -rf /opt/mapr/conf/ssl_keystore \ + && rm -rf /root/bootstrap.sh \ + +# INSTALL KERBEROS + && yum install -y krb5-libs krb5-server krb5-workstation + +# ADD KERBEROS CONFIGURATION +ADD files/bootstrap.sh /root/ +ADD files/kerberos/krb5.conf /etc/krb5.conf +ADD files/kerberos/kdc.conf /var/kerberos/krb5kdc/kdc.conf +ADD files/kerberos/kadm5.acl /var/kerberos/krb5kdc/kadm5.acl +ADD files/jceJars/local_policy.jar /usr/java/jdk1.8.0_102/jre/lib/security/local_policy.jar +ADD files/jceJars/US_export_policy.jar /usr/java/jdk1.8.0_102/jre/lib/security/US_export_policy.jar + +# ENABLE HIVE SECURITY +ADD files/conf/hive-site.xml /opt/mapr/hive/hive-1.2/conf/hive-site.xml + +# CREATE KERBEROS DATABASE +RUN /usr/sbin/kdb5_util create -s -P password \ + && usermod -g root hdfs \ + && usermod -g mapr hdfs \ +# CREATE MAPR AND HIVE PRINCIPALS AND KEYTABS + && /usr/sbin/kadmin.local -q "addprinc -randkey mapr/mycluster@LABS.TERADATA.COM" \ + && /usr/sbin/kadmin.local -q "xst -norandkey -k /opt/mapr/conf/mapr.keytab mapr/mycluster@LABS.TERADATA.COM" \ + && /usr/sbin/kadmin.local -q "addprinc -randkey hive/mycluster@LABS.TERADATA.COM" \ + && /usr/sbin/kadmin.local -q "xst -norandkey -k /opt/mapr/conf/hive.keytab hive/mycluster@LABS.TERADATA.COM" \ + +# CREATE HDFS USER + && /usr/sbin/kadmin.local -q "addprinc -randkey hdfs/mycluster@LABS.TERADATA.COM" \ + && /usr/sbin/kadmin.local -q "xst -norandkey -k /opt/mapr/conf/hdfs.keytab hdfs/mycluster@LABS.TERADATA.COM" \ + +# CHANGE THE PERMISSIONS AND OWNERSHIPS FOR KEYTABS + && chmod 644 /opt/mapr/conf/hive.keytab /opt/mapr/conf/mapr.keytab /opt/mapr/conf/hdfs.keytab \ + && chmod 777 /root/bootstrap.sh \ + && chown mapr:mapr /opt/mapr/conf/mapr.keytab \ + && chown hive:hive /opt/mapr/conf/hive.keytab \ + && chown hdfs:hdfs /opt/mapr/conf/hdfs.keytab \ + +# CREATE PRESTO PRINCIPAL AND KEYTAB + && /usr/sbin/kadmin.local -q "addprinc -randkey presto-server/presto-master.docker.cluster@LABS.TERADATA.COM" \ + && /usr/sbin/kadmin.local -q "addprinc -randkey presto-client/presto-master.docker.cluster@LABS.TERADATA.COM" \ + && /usr/sbin/kadmin.local -q "addprinc -randkey hive/presto-master.docker.cluster@LABS.TERADATA.COM" \ + && mkdir -p /etc/presto/conf \ + && /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/presto/conf/presto-server.keytab presto-server/presto-master.docker.cluster" \ + && /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/presto/conf/presto-client.keytab presto-client/presto-master.docker.cluster" \ + && /usr/sbin/kadmin.local -q "xst -norandkey -k /etc/presto/conf/hive-presto-master.keytab hive/presto-master.docker.cluster" \ + && chmod 644 /etc/presto/conf/*.keytab \ + && cat /opt/mapr/conf/env.sh | sed -e '0,/MAPR_HIVE_SERVER_LOGIN_OPTS="-Dhadoop.login=maprsasl_keytab"/ s/MAPR_HIVE_SERVER_LOGIN_OPTS="-Dhadoop.login=maprsasl_keytab"/MAPR_HIVE_SERVER_LOGIN_OPTS="-Dhadoop.login=hybrid"/' > env_new.sh \ + && cat env_new.sh | sed -e '0,/MAPR_HIVE_LOGIN_OPTS="-Dhadoop.login=maprsasl"/ s/MAPR_HIVE_LOGIN_OPTS="-Dhadoop.login=maprsasl"/MAPR_HIVE_LOGIN_OPTS="-Dhadoop.login=hybrid"/' > /opt/mapr/conf/env.sh \ + && rm -rf env_new.sh + +# CREATE SSL KEYSTORE +RUN keytool -genkeypair \ + -alias presto \ + -keyalg RSA \ + -keystore /etc/presto/conf/keystore.jks \ + -keypass password \ + -storepass password \ + -dname "CN=presto-master, OU=, O=, L=, S=, C=" +RUN chmod 644 /etc/presto/conf/keystore.jks + +# EXPOSE KERBEROS PORTS +EXPOSE 88 +EXPOSE 749 + +CMD /root/startup.sh diff --git a/teradatalabs/mapr52-hive-kerberized/README.md b/teradatalabs/mapr52-hive-kerberized/README.md new file mode 100644 index 0000000..4949497 --- /dev/null +++ b/teradatalabs/mapr52-hive-kerberized/README.md @@ -0,0 +1,17 @@ +# mapr52-hive-kerberized + +Docker image with kerberos enabled for MapR. Please note that running services have lower memory heap size set. +For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file. +If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal +for general correctness testing. + +## Run + +``` +$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr52-hive-kerberized +``` + +## Oracle license + +By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: +[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh b/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh new file mode 100644 index 0000000..240ceab --- /dev/null +++ b/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh @@ -0,0 +1,49 @@ +#!/bin/sh + + +# START SSHD AND THE SOCKS PROXY FOR THE HIVE METASTORE +supervisorctl start sshd +supervisorctl start socks-proxy + +# CONFIGURE MAPR +/opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart + +# SETUP DISK FOR MAPR BY RUNNING disksetup +/opt/mapr/server/disksetup -M -F /root/disk.txt + +# CREATE HIVE PROXY USERS +chmod 755 /opt/mapr/conf/proxy + +# CONFIGURE HIVE +/opt/mapr/server/configure.sh -R + +# ENABLE SECURITY IN MAPR +/opt/mapr/server/configure.sh -secure -genkeys -C localhost -Z localhost -N mycluster -no-autostart + +# START KERBEROS SERVICES +/sbin/service krb5kdc start +/sbin/service kadmin start + +# START MAPR SERVICES +service mapr-zookeeper start +service mapr-warden start + +# WAIT FOR WARDEN TO START ALL THE SERVICES +sh /root/wardenTracker.sh + +# START HTTPFS SERVICES +maprcli node services -name httpfs -action start -nodes $(hostname) + +# CREATE KERBEROS TICKET +kinit -kt /opt/mapr/conf/mapr.keytab mapr/mycluster@LABS.TERADATA.COM + +# CREATE MAPR TICKET +maprlogin kerberos -user mapr/mycluster@LABS.TERADATA.COM + +# RUN HDFS COMMANDS +hadoop fs -mkdir /user/root /user/hive /user/hdfs /user/hive/warehouse /var /var/mapr /var/mapr/cluster /var/mapr/cluster/yarn /var/mapr/cluster/yarn/rm /var/mapr/cluster/yarn/rm/staging /var/mapr/cluster/yarn/rm/staging/hive +hadoop fs -chmod 777 /user/hive /user/hdfs /user/hive/warehouse /var/mapr /var/mapr/cluster/yarn/rm/staging/hive + +# REMOVE MAPR TICKET AND KERBEROS TICKET +kdestroy +rm -rf /tmp/* diff --git a/teradatalabs/mapr52-hive-kerberized/files/conf/hive-site.xml b/teradatalabs/mapr52-hive-kerberized/files/conf/hive-site.xml new file mode 100644 index 0000000..beeeebd --- /dev/null +++ b/teradatalabs/mapr52-hive-kerberized/files/conf/hive-site.xml @@ -0,0 +1,94 @@ + + + + + + + datanucleus.schema.autoCreateAll + true + creates necessary schema on a startup if one doesn't exist. set + this to false, after creating it once + + + + hive.server2.enable.doAs + true + Set this property to enable impersonation in Hive Server 2 + + + + hive.metastore.execute.setugi + true + Set this property to enable Hive Metastore service impersonation in unsecure mode. In unsecure mode, setting this property to true will cause the metastore to execute DFS operations using the client's reported user and group permissions. Note that this property must be set on both the client and server sides. If the client sets it to true and the server sets it to false, the client setting will be ignored. + + + + javax.jdo.option.ConnectionURL + jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true + JDBC connect string for a JDBC metastore + + + + javax.jdo.option.ConnectionDriverName + com.mysql.jdbc.Driver + Driver class name for a JDBC metastore + + + + javax.jdo.option.ConnectionUserName + root + username to use against metastore database + + + + javax.jdo.option.ConnectionPassword + root + password to use against metastore database + + + + hive.metastore.uris + thrift://localhost:9083 + + + + + + hive.metastore.kerberos.keytab.file + /opt/mapr/conf/hive.keytab + The path to the Kerberos Keytab file containing the metastore thrift server's service principal. + + + hive.metastore.kerberos.principal + hive/mycluster@LABS.TERADATA.COM + The service principal for the metastore thrift server. The special string _HOST will be replaced automatically with the correct hostname. + + + hive.server2.authentication + KERBEROS + authenticationtype + + + hive.server2.authentication.kerberos.principal + hive/mycluster@LABS.TERADATA.COM + HiveServer2 principal. If _HOST is used as the FQDN portion, it will be replaced with the actual hostname of the running instance. + + + hive.server2.authentication.kerberos.keytab + /opt/mapr/conf/hive.keytab + Keytab file for HiveServer2 principal + + diff --git a/teradatalabs/mapr52-hive-kerberized/files/jceJars/US_export_policy.jar b/teradatalabs/mapr52-hive-kerberized/files/jceJars/US_export_policy.jar new file mode 100644 index 0000000000000000000000000000000000000000..251b102c57c076504ba818330e4fceb4509753d8 GIT binary patch literal 3023 zcma)82Q-^|AC9d?+fPt?)UHutUz8wm36X=Jzvp*;&-**idH&~l;4nIRP5?7AGoaZf)c|m! zIH_kdh!t4g_%igO8Thg>6k=(mU63ZbT|Xr|6!~>UD!I2VF6jX z$GM}7FXJ9BPm2h}ojWt>yzYF*AENmL?3EQ|Wx57%(kk_(%)j)Cic4=M)t!R(J5+n{ z6$dc|v+dl`NQ~&BI0-uuCFC9Sltr~f2|@(eczH?8s;HjHm+~0Lipyi%ar)Y|xLM8d zgf5x~dtR1v-C-vN!j_A zBA~o%?BN|j-OQ?+M_GtCHZ)<>9(L9HX^LET3f(5v1J|pC_m~=Ion4DznxFKYNIyXtR_FP>=OeFUeypaYVAgaACTrFc1u(snGASKw~TE*&3zwyjZ;=B%rw!d*yxp^U4g=c3R-(0 zzeAyJuJ94(vP#zsK~%nR|C4jBlhXXoxrHTI+0!+$uahveIIxwCP0IEH z1HZCIOK!=UVlemj8S8zrPWcL92xK`^43Uj-g83eX6+a4M$7gWU2JF+iOF|_jCEZ~U zS#CUh5X3D<)?0`NvemL{jAa^=LbNCqiBG|Y z_4vd)CFD39@kP^_ILB0kqrGD?-hnuc!}>h-dB~i+$eq4!A_uc#?lYH_F_$y3M)jE+ zTQm2W_VZwY<)5^GVb8iLSfFxO_XxsaJ#n|<2S%9EbKkz%uT$@BJdz3X>p9{1^)q6Jupn2BLqZ z5q@b~zK_8ig2RWI7|TU238Q(?FJ5(y$g@oq>;VO&__z!e!h*mv#3xbOKVQW`TJ?xv z@y!8jbqe;+OQv7YcBz}2P{?aybT8X$Jr;WVd3HDcM=1|DEC<)o7GHWSVSyWH3&Hgk za1k~u7oA+3vB}F8oMEVcF7Rq7ue<>KQ!#O;7^d*m zV?64S?oVx0ft&#Z2#P0xSlI5V`~om7fEF??GU^>27=NB6qO^~i-?+c(!QkWH=;1!$w&I$ShnSgc-Z{Mf|?GzdYV3#ep@rAehfKf zFRXhQp;@->(khqcTuq>;MX{V=?{kPzmruDct8)`)oSFw2+MFrPmQAs-ZTwAbbO0y+ zXHy;LeC3c7tQa=g6!Zqqwc;Idlh)x%r{SGd2|O}1Z`gKZx3KfImwlNW3~?h5H8i+9 zE-WZF>l7G}Hu1+?`539adOMfSg%jYm6wzlkE6rl`^4T6&Kfj%h;5lG`H87 z_AZA?Ea)#uG&Fw5(Qrz=)qe31`@Evz#VR69d{E`|Thryz!6DsL#cqdrL`=481az=b z;7ormYyI5SB@>o?&Jgxgh5GgEumC)m1#07QO?{dPY32gA@ld_SXc42VoUhn?AztE) zcZ9CLvd)%Ahg!mNf`i|B-JbRsDAu@r)5l9PclP3#cc!X(!@i%K_x^TbkqfW@sAx7u z<1G>0`94~a=~r3Z$~PQUsUBWF83z#`2GZ{qt>`q&E&;s{#9!2!tiSOkXKcXj<{YP$ zi?E$NMv}R!w@PPyD{Iyd&Wl#OGwN~BkBC(5B4J0>i0+L#A#XPN{rc54e01TSV&%Tp z0nHS$7>D_^8T0uP66nTbHG6l06_O`_$VpR9=)h_?2dmQ=Am}oknS!a zKEViUO9b*3%0Dnz0p;)Gb^W#uTIqDFA{%5oYc^LzQ%|ZL)85QLl=h9c&NkaZ$BuuX)zs}6&dry{L*4SVuKuMd2Q0Yx7_F1tzMf*;`yCuh6s7YUc+ z^BFgXc;EA59pR$1UdR#5aL8(tefVH&OC7#F>+7ZAMi0W}vb|BgwM#jddX03tCya&! zlDB1Cy@7)UJed=kDDf=3KW1+_Oz^7{93w=p&+19}3fcno*Cw4Sl#aWqh+VH(6t~MW zv;fyDOo>HTv%vPK*33<`QG!0fc*BH?{COX=rxqC35)^2r z-QD}moPF9jJ7}oJT=31DX=phC$I0Q6xrftNh(6rP{{me?m0K~zvuqGbH4L_=RNQ9zTb16-}^jhBoD6`00aU7o;Z;# z06R*IJwA?bfNNQwu+%;dKVfZ&uy=qSx5V|rZDV0#ypz8Qu&}Ap596*oh>D8%{Y10# zO@f=G;lgjrRBm*Xt9_hD*8lm@uoKn5xUL=)H}p*StOlCTYqq-b(U67bwIa*d5{9~B zPWrT_Fll(th}gS7s#Bucd))HIKN!3g?T`+QrBV`WA8Cz6;P9Yh0#-xT8ORybFJPNy z{xO@^$~oGLK58ZJ;jmm7k_8=;lu!M=dRfqkKI^|PwdDY|ZUS*Cc`V}cl)mZas8$%& z!vE;Nmg;N!ymGosjHxdsT*Z{oP*?6>xU~_L@nLWZ+-z$qxP0e&;|w1f$q#Z&Y`{;k z2@v4|0MOqlYWXjUZ0+C{CV@qs=20OZeX}FNyV@B*Sp#*=Se!y6@Ba8PdM_66pxCt&ZocdEOb^Fh~TdTY4IL1 zjTUkB<>^&71*MP+y)IYjamSVlXtX^t8Y3gx!!$AiiUU-g?F{{i2eONoq1INhTaD#~ zb%s?LbBog0!1FS*(Z(7PZ~dtvX##EC*Bq9DT$tB~un%kXp@<8G4v{s{5@T9!7O-%# z9RPW&I?gbTJDT-L4OuRBQSh+M)f5=%JVq4MS=$OBEFrZ)Cn; zSpL-z2~7sY1E|+lL)})>#xbzh*4h!& zi|XrUAdN`6*SOU^BqUaY!w_k9fnf&MAa4&D80c_$==1PEUaKh%3wIp47z`GrO#^*+ ztgpz#Z6=9kguu!Xd|wfX%9O9+z~oQxWd*IAdEI;Dc&od0-C4;`s8cEWy|3twDlf$Q)SMRSo6^F-krJsxH z^Y)2piinU|YpF`eF=Na)Yj&?(n!2+HVfrG|bIUq|6Bbcb<9SXWOMZXXn=(3AIn~}1 z>e!MMl;?NGaXiX^b>d*(T4?=gqsJ7ol+->+qg5$`Fd2~fbi+ad^F1ulhW zC(Ad9L>Bz~5I?Y7(NrO6UsA_F+pw1KMBzF_R#jU3C^RwzK1IKK@yM~$6vPu49j?TD zo>iHV_01IZIq4LM$+SegpvQI#zkmtyI?sT;WNv3%6}9hQNwWLgWseO0lw>Ej(hM)o z%FatlFCjCfgwK+Xno9=XDWwD!IR}2a+?szb8NZS&ujM7WO!d=}MK6`mzmy=M8~Cw{ zx6OWOVGBeA5G1FRrY9&IZ}0~IRcKiRoCU%-1K_do=65qHZ3{3ukI*SgMJstyF6O=w zKJ8a=wJfQ}Q}(Q?d$}v^*BM^*r8SLSkwYgtWrA&Q#RTg1QkNUunFsGbsLY=1iILAe z^FenraOTtHMLZuNpaJha?!T(u|KWuJB(-eS1C!r$>y4oax^j{?fp_h2Zrvzu!dbzL z6?M351^YxZ^Y=&(kL7DAuH#h(YF5X^uvQ)5(S_h9l1@THhLL z#bzo7RgopgSB)Vrsp21eqt0=={ML3fdQq8*i?}&-a`}KF#|NNMO zoaVG!a8Ty>t-T6-z+Qxmv7BP?F_B9B27CTO&6oBMnvrxmU1l(GIUZ4#{SC4r zeR)E4uTQ_V5!@6D1(Tq4D>;!tRJfp}6W-luk{@>*i*~{vatGSQ>FXEhH2s{cEaDqw z7NBpkihpXDGN0n&zf!w?WK=J~x|K=rQOTRu9`(I($f$nPU(r<^=pz!|htL65TmL$IRm=E2wu*1y-hioed(m)%XFW7#jbi;S6C%=g z{BVe$?V@x@^5(07HhEo~tAcl^S(N1YiT9_&ADHL%Ik8mpzM@d2-)ysDyf+D3cjq7N zM?8$Swz@rd52}`zTjRY^P-#?yAI`m-vlhLYRxm9SLmrTu+wu>fnlarM^2D1<>?XK4 zw=MkWs}&7C0AQYfr@bAL+MNId0KD4?V5`)SNBV$B-yc-Zl*sKoXb{R zhA!#p?{}xKd6ef={5G8kIo*?Oh{_hQ7r({hZg*rBq2lLR`b~aZR-@=5nvS#_Jbb%lFUL*rb-ui4>;M>9vrmFITi#^Bc?6A-!2h;y6NfxOkp4Q1)c{rgG{9cpY@&VC=(Y`KO8w41`%ZPf_dDS3_sz zw&J-TRDlFr&&S%PB_54zW084QD%|bQBT971w}~MW*9kd7 zZ=P`p-C!nJrI^#KKiP;gmN)g*4TBS#LxPVV>FieE2!YTd_LPh5*M)z3HW#-TU^_|f z3{V^??5=;ym1yKP0w-wgOa!|T4zlO%pWlqV^xs-zCjv)*JNf14Z@cyg>{25d$p_?20@+U# Lo19~UoUi`@yDZ_Z literal 0 HcmV?d00001 diff --git a/teradatalabs/mapr52-hive-kerberized/files/kerberos/kadm5.acl b/teradatalabs/mapr52-hive-kerberized/files/kerberos/kadm5.acl new file mode 100644 index 0000000..0530526 --- /dev/null +++ b/teradatalabs/mapr52-hive-kerberized/files/kerberos/kadm5.acl @@ -0,0 +1 @@ +*/admin@LABS.TERADATA.COM * diff --git a/teradatalabs/mapr52-hive-kerberized/files/kerberos/kdc.conf b/teradatalabs/mapr52-hive-kerberized/files/kerberos/kdc.conf new file mode 100644 index 0000000..c864897 --- /dev/null +++ b/teradatalabs/mapr52-hive-kerberized/files/kerberos/kdc.conf @@ -0,0 +1,12 @@ +[kdcdefaults] + kdc_ports = 88 + kdc_tcp_ports = 88 + +[realms] + LABS.TERADATA.COM = { + #master_key_type = aes256-cts + acl_file = /var/kerberos/krb5kdc/kadm5.acl + dict_file = /usr/share/dict/words + admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab + supported_enctypes = aes256-cts:normal aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal + } diff --git a/teradatalabs/mapr52-hive-kerberized/files/kerberos/krb5.conf b/teradatalabs/mapr52-hive-kerberized/files/kerberos/krb5.conf new file mode 100644 index 0000000..344e348 --- /dev/null +++ b/teradatalabs/mapr52-hive-kerberized/files/kerberos/krb5.conf @@ -0,0 +1,18 @@ +[logging] +default = FILE:/var/log/krb5libs.log +kdc = FILE:/var/log/krb5kdc.log +admin_server = FILE:/var/log/kadmind.log + +[libdefaults] +default_realm = LABS.TERADATA.COM +dns_lookup_realm = false +dns_lookup_kdc = false +ticket_lifetime = 24h +renew_lifetime = 7d +forwardable = true + +[realms] +LABS.TERADATA.COM = { +kdc = hadoop-master +admin_server = hadoop-master +} From be1582fdb7e2f7a48adc5a4c8f8bcbd7b80f816b Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Thu, 30 Mar 2017 13:46:08 -0400 Subject: [PATCH 04/11] Adding changes inside test container to include MapR The changes includes a new function expose_mapr and the shell script to wait for MapR , also this changes will enable tests to switch user when the container is not MapR. --- Makefile | 2 +- test/Dockerfile | 16 ++++++++++++++-- test/files/conf/id_rsa | 27 +++++++++++++++++++++++++++ test/files/conf/id_rsa.pub | 1 + test/files/expose_mapr_hive.sh | 22 ++++++++++++++++++++++ test/image_tests/image_tests.bats.sh | 19 +++++++++++++++++-- 6 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 test/files/conf/id_rsa create mode 100644 test/files/conf/id_rsa.pub create mode 100644 test/files/expose_mapr_hive.sh diff --git a/Makefile b/Makefile index 1024abe..97bcaa0 100644 --- a/Makefile +++ b/Makefile @@ -155,7 +155,7 @@ $(IMAGE_DIRS): %: %/Dockerfile | check-links export TESTED_IMAGE=$* && \ cd test && \ docker-compose up -t 0 -d hadoop-master && \ - time docker-compose run -e EXPECTED_CAPABILITIES="`cat ../$*/capabilities.txt | tr '\n' ' '`" --rm test-runner + time docker-compose run -e EXPECTED_CAPABILITIES="`cat ../$*/capabilities.txt | tr '\n' ' '`" -e IMAGE=$* --rm test-runner # # Static pattern rule to pull docker images that are external dependencies of diff --git a/test/Dockerfile b/test/Dockerfile index 17dfffe..89a0168 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -15,12 +15,22 @@ FROM teradatalabs/centos6-java8-oracle ENV DOCKERIZE_VERSION v0.3.0 - +RUN mkdir /root/.ssh +COPY /files/conf/* /root/.ssh/ +COPY files/* /root/ RUN \ yum install -y \ wget \ unzip \ +# INSTALL UTILITY SOFTWARE + && yum install -y openssh-server openssh-clients \ + +# CONFIGURE SSH + && chkconfig sshd on \ + && grep -rl '#Port 22' /etc/ssh/sshd_config | xargs sed -i 's/#Port 22/Port 22/g' \ + && service sshd start \ + # setup CDH repo, pin the CDH distribution to a concrete version && wget -nv http://archive.cloudera.com/cdh5/one-click-install/redhat/6/x86_64/cloudera-cdh-5-0.x86_64.rpm \ && yum --nogpgcheck localinstall -y cloudera-cdh-5-0.x86_64.rpm \ @@ -45,7 +55,9 @@ RUN \ && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ # create 'image_tests' volume mount path - && mkdir /image_tests + && mkdir /image_tests \ + && chmod 777 /root/*.sh \ + && chmod 600 /root/.ssh/id_rsa.pub /root/.ssh/id_rsa VOLUME /image_tests diff --git a/test/files/conf/id_rsa b/test/files/conf/id_rsa new file mode 100644 index 0000000..2d01c1c --- /dev/null +++ b/test/files/conf/id_rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA1PL4EwRZFy1ewBTa4a1TK+mQ4rAupOeZsiqir/su61dAGvC6 +pEFAa+Litj6ub6NvcBRMAdXeBtbOnQpInE7BFwKVhwU3n60Mc69SjLiozK3Oxh9s +fmbJv/JdELRS5aB9x82Y0bO5fZFPFj7SxPNMugQQMEMQHW01wsa5nJR2pYLwCtu7 +yoD6fQ0TJEsRqWwyQTNoR19yzL6h7p/hq9SqiqCKfsHWK4+Tj0IgF7Nwz8i+BqqO +q2kC9lTRuT8HalNbqVVQ6iI+ER7FgdfSZtKKX6R9SOaKQ7p0Dt6JLFibMNhjwt5E +KHsgfMOsl1G8SEncDREtTng8/JLlvIhiqmWzwQIBIwKCAQAYVkg8sAouiNeg7HDJ +VaMa9h8v2Z7uRlq1RrOB05BVaQ6j/kE3V+oa9Vu9DnpV5sroPNV8jXh1z2gR8ov0 +m0lEdVLq3AZbZDvv9sebDcomBTwz5lWKyesr4S81G+w3gA5f+j1Zy2Wn81l+4pu+ +vLhPxfM4tzUKrWU61OIDRCrRFhHaFiDxBMsMhOSBfGT5OuIIHOAcjvB+YxKgB644 +ANoOnnExaukOOX0ZLgnwIG7YG0Cf6MVJpbK+gx7ER85SnMV/wjzsAedOfF+tD8Z1 +P0nfRXKr9X+CE2m3JMAb4QxXphsVHLAtGjPmzEz/2JNu7mnevJW1iQ0X19giSdyE +0+U7AoGBAPgsgpFrGKf0vy3XKMkIl8uwn+GGKob3/Fc6sGYaGu2fgR/BeMnKf4IT +BsDBgPJPz/ozQaWXWVBCZc5Wqm7CPCiccHzUF8G0PZFMNkR2ayO854co82Sj+sWW +XwS/kdTRRQ6HHr67KKSZFZO60HaWWSJ9Z0IlrVe+k3s0FtMV72pXAoGBANuqF4oi +fmHawxXgIdUcESzJlt3HUjOC32e7rEcnskjrNQxM4ZUkmQrf6IhhFrSdCk5cNCrP +pdaw87YtIEtVO3LvAT8E+dRuIO8LV4VufaBBHUq/TwntldF1/fN41ZpgpehDBH/i +rTXAMncd2junSpsyt800W7KtXsdca0YKODOnAoGAHFzbuNkKIdLUBT0pO4v0F0di +urA/X+HTsjKX0SeODIdB9QArD79fB40lV9uZtUrzMomD2GkRhYPuYLlykFC2awqQ +g0twbei2lEM5Zuj2TTopFsLZ/N+KX7lp8eoCCbGEPCyzDnvKIXCUwG0fI37lnYqs +txo/skGqdHr7S1L2ylMCgYEA1WNnUvzvzMXiFUAg3aY8kehmq45tHBi7w9rh3rjZ +E6Koj5skR70m9J7/JWWhBzl3udXpiKx1NvUKAW2qV88jzrxKWnncv7twcy+W2WQF +AhNeSJyV7GMj0spkaNvIL5EWMhU3kiyoQtf2gleDimf/VO9wuLaE9rcLnRfzLhiO +Xg8CgYEAk6QP8i01oPfY4IzE9JTHbVIglllbygxL7CKid6kH2GtbcnT8NU9aj4YY +XdGRiDRw7/O6sHSKbLkSQ1TUAVil8Xw6tLQddbRI+p93RVw9haa8bCBtzDV7eB4m +M5/MrINmIquLaSBd+l3HA36GCxLzBUsT87UZIQnJu7YauFVJYsU= +-----END RSA PRIVATE KEY----- diff --git a/test/files/conf/id_rsa.pub b/test/files/conf/id_rsa.pub new file mode 100644 index 0000000..cdd0869 --- /dev/null +++ b/test/files/conf/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA1PL4EwRZFy1ewBTa4a1TK+mQ4rAupOeZsiqir/su61dAGvC6pEFAa+Litj6ub6NvcBRMAdXeBtbOnQpInE7BFwKVhwU3n60Mc69SjLiozK3Oxh9sfmbJv/JdELRS5aB9x82Y0bO5fZFPFj7SxPNMugQQMEMQHW01wsa5nJR2pYLwCtu7yoD6fQ0TJEsRqWwyQTNoR19yzL6h7p/hq9SqiqCKfsHWK4+Tj0IgF7Nwz8i+BqqOq2kC9lTRuT8HalNbqVVQ6iI+ER7FgdfSZtKKX6R9SOaKQ7p0Dt6JLFibMNhjwt5EKHsgfMOsl1G8SEncDREtTng8/JLlvIhiqmWzwQ== root@d57cdb1934d1 diff --git a/test/files/expose_mapr_hive.sh b/test/files/expose_mapr_hive.sh new file mode 100644 index 0000000..5fbf875 --- /dev/null +++ b/test/files/expose_mapr_hive.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +#Wait for hadoop-master to be available +conn=0 +while [ $conn -ne 1 ] +do + ping -c 4 -q hadoop-master + if [ "$?" -eq 0 ]; then + conn=1 + else + conn=0 + fi +done + +# Wait for hive and warehouse directories in HDFS +hivecliReady=1 +while [ $hivecliReady -ne 0 ] +do + sleep 5s + ssh -o StrictHostKeyChecking=no root@hadoop-master 'hadoop fs -ls /user/hive/warehouse' + hivecliReady=$? +done diff --git a/test/image_tests/image_tests.bats.sh b/test/image_tests/image_tests.bats.sh index 4f0af22..7c1abe1 100755 --- a/test/image_tests/image_tests.bats.sh +++ b/test/image_tests/image_tests.bats.sh @@ -1,5 +1,15 @@ #!/usr/bin/env bash +get_user() { + skip_if_needed + if [ "${IMAGE}" == "teradatalabs/mapr52-hive" ] + then + echo "hive" + else + echo "hdfs" + fi +} + skip_if_needed() { SHOULD_RUN=true # Can't rely on exit codes here, as BATS will fail the test if any of the statements @@ -22,6 +32,11 @@ assert_output_contains() { printf '%s\n' "${lines[@]}" | grep -q $1 } +function exposes_mapr { + skip_if_needed + assert_run sh /root/expose_mapr_hive.sh +} + function exposes_hive { skip_if_needed assert_run dockerize -wait tcp://hadoop-master:10000 -timeout 90s @@ -29,12 +44,12 @@ function exposes_hive { function allows_creating_a_table_in_hive { skip_if_needed - assert_run beeline -n hdfs -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' + assert_run beeline -n $(get_user) -u jdbc:hive2://hadoop-master:10000 -e 'create table test as select 42 id' } function allows_selecting_from_the_table { skip_if_needed - assert_run beeline -n hdfs -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' + assert_run beeline -n $(get_user) -u jdbc:hive2://hadoop-master:10000 -e 'select * from test' assert_output_contains 'test.id' assert_output_contains '42' } From 3c9fe45ac9aeb20603ccd8b9480ddc60a8860d39 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Thu, 30 Mar 2017 13:58:31 -0400 Subject: [PATCH 05/11] Adding privileged inside docker-compose for MapR --- test/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 2e04ebe..1908301 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -4,6 +4,7 @@ services: hadoop-master: image: ${TESTED_IMAGE} hostname: hadoop-master + privileged: true ports: - '1180' From f5602e05eb23738c9b4dd5d6c7cb9e29aded9448 Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Fri, 31 Mar 2017 11:37:36 -0400 Subject: [PATCH 06/11] Adding missing sshd start in Docker-base Adding missing sshd start in Docker-base --- teradatalabs/mapr52-base/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr52-base/Dockerfile index 25edeac..9c2ed2f 100644 --- a/teradatalabs/mapr52-base/Dockerfile +++ b/teradatalabs/mapr52-base/Dockerfile @@ -27,6 +27,7 @@ RUN yum update -y \ # CONFIGURE SSH && chkconfig sshd on \ && grep -rl '#Port 22' /etc/ssh/sshd_config | xargs sed -i 's/#Port 22/Port 22/g' \ + && service sshd start \ # INSTALL MAPR && yum install -y mapr-fileserver mapr-nfs mapr-nodemanager mapr-cldb \ From e35f69280915fc2759a6e78fdf77bf5f264b976d Mon Sep 17 00:00:00 2001 From: "Singh, Ashish" Date: Thu, 6 Apr 2017 10:27:08 -0400 Subject: [PATCH 07/11] Adding Logic to wait for MapR ticket in Warden This changes will make maprcli wait for MapR ticket in wardenTracker --- .../mapr52-hive-kerberized/Dockerfile | 5 +- .../mapr52-hive-kerberized/files/bootstrap.sh | 12 +--- .../files/wardenTracker.sh | 64 +++++++++++++++++++ 3 files changed, 68 insertions(+), 13 deletions(-) create mode 100644 teradatalabs/mapr52-hive-kerberized/files/wardenTracker.sh diff --git a/teradatalabs/mapr52-hive-kerberized/Dockerfile b/teradatalabs/mapr52-hive-kerberized/Dockerfile index 24956c2..0a8b477 100644 --- a/teradatalabs/mapr52-hive-kerberized/Dockerfile +++ b/teradatalabs/mapr52-hive-kerberized/Dockerfile @@ -20,13 +20,14 @@ RUN rm -rf /opt/mapr/conf/ssl_truststore \ && rm -rf /opt/mapr/conf/maprserverticket \ && rm -rf /opt/mapr/conf/cldb.key \ && rm -rf /opt/mapr/conf/ssl_keystore \ - && rm -rf /root/bootstrap.sh \ + && rm -rf /root/bootstrap.sh /root/wardenTracker.sh \ # INSTALL KERBEROS && yum install -y krb5-libs krb5-server krb5-workstation # ADD KERBEROS CONFIGURATION ADD files/bootstrap.sh /root/ +ADD files/wardenTracker.sh /root/ ADD files/kerberos/krb5.conf /etc/krb5.conf ADD files/kerberos/kdc.conf /var/kerberos/krb5kdc/kdc.conf ADD files/kerberos/kadm5.acl /var/kerberos/krb5kdc/kadm5.acl @@ -52,7 +53,7 @@ RUN /usr/sbin/kdb5_util create -s -P password \ # CHANGE THE PERMISSIONS AND OWNERSHIPS FOR KEYTABS && chmod 644 /opt/mapr/conf/hive.keytab /opt/mapr/conf/mapr.keytab /opt/mapr/conf/hdfs.keytab \ - && chmod 777 /root/bootstrap.sh \ + && chmod 777 /root/bootstrap.sh /root/wardenTracker.sh \ && chown mapr:mapr /opt/mapr/conf/mapr.keytab \ && chown hive:hive /opt/mapr/conf/hive.keytab \ && chown hdfs:hdfs /opt/mapr/conf/hdfs.keytab \ diff --git a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh b/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh index 240ceab..9136588 100644 --- a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh +++ b/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh @@ -34,16 +34,6 @@ sh /root/wardenTracker.sh # START HTTPFS SERVICES maprcli node services -name httpfs -action start -nodes $(hostname) -# CREATE KERBEROS TICKET -kinit -kt /opt/mapr/conf/mapr.keytab mapr/mycluster@LABS.TERADATA.COM - -# CREATE MAPR TICKET -maprlogin kerberos -user mapr/mycluster@LABS.TERADATA.COM - # RUN HDFS COMMANDS -hadoop fs -mkdir /user/root /user/hive /user/hdfs /user/hive/warehouse /var /var/mapr /var/mapr/cluster /var/mapr/cluster/yarn /var/mapr/cluster/yarn/rm /var/mapr/cluster/yarn/rm/staging /var/mapr/cluster/yarn/rm/staging/hive +hadoop fs -mkdir /user /user/root /user/hive /user/hdfs /user/hive/warehouse /var /var/mapr /var/mapr/cluster /var/mapr/cluster/yarn /var/mapr/cluster/yarn/rm /var/mapr/cluster/yarn/rm/staging /var/mapr/cluster/yarn/rm/staging/hive hadoop fs -chmod 777 /user/hive /user/hdfs /user/hive/warehouse /var/mapr /var/mapr/cluster/yarn/rm/staging/hive - -# REMOVE MAPR TICKET AND KERBEROS TICKET -kdestroy -rm -rf /tmp/* diff --git a/teradatalabs/mapr52-hive-kerberized/files/wardenTracker.sh b/teradatalabs/mapr52-hive-kerberized/files/wardenTracker.sh new file mode 100644 index 0000000..802c8ef --- /dev/null +++ b/teradatalabs/mapr52-hive-kerberized/files/wardenTracker.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +hname=$(hostname) + +Services=0 +maprloginReady=1 +maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) + +# WAIT FOR CLDB TO START +while [ $Services -ne 2 ] +do +if [ $maprcliReady == 1 ] +then + maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) + Services=0 + # Get MapR ticket + if [ $maprloginReady -ne 0 ] + then + # CREATE KERBEROS TICKET + kinit -kt /opt/mapr/conf/mapr.keytab mapr/mycluster@LABS.TERADATA.COM + + # CREATE MAPR TICKET + maprlogin kerberos -user mapr/mycluster@LABS.TERADATA.COM + maprloginReady=$? + fi +else + Services=$(maprcli service list -node $hname | grep CLDB |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +fi +done + +# WAIT FOR NODEMANAGER TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep NodeManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +# WAIT FOR RESOURCEMANAGER TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep ResourceManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +# WAIT FOR HIVE METASTORE TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep HiveMetastore |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +# WAIT FOR HIVESERVER2 TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep HiveServer2 |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done + +# WAIT FOR JOBHISTORYSERVER TO START +Services=0 +while [ $Services -ne 2 ] +do + Services=$(maprcli service list -node $hname | grep JobHistoryServer |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) +done From 3004ec0799bdc10c96c1f505ac9b784d20d601cb Mon Sep 17 00:00:00 2001 From: Artur Gajowy Date: Mon, 10 Apr 2017 20:27:50 +0200 Subject: [PATCH 08/11] Split the travis jobs so that the build finishes under the 50 minutes mark --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e501d61..72f0992 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,11 @@ dist: trusty env: matrix: - - IMAGES='teradatalabs/centos6-java8-oracle.dependants' + - IMAGES='teradatalabs/cdh5-base.dependants' + - IMAGES='teradatalabs/iop-base.dependants' + - IMAGES='teradatalabs/mapr-base.dependants' - IMAGES='teradatalabs/centos6-ssh-oj8.dependants' - - IMAGES=$( echo teradatalabs/{dns,ubuntu-trusty-python2.6,presto-admin-devenv} ) + - IMAGES=$( echo teradatalabs/{dns,ubuntu-trusty-python2.6,presto-admin-devenv,centos6-java8-oracle-ldap,centos6-java8-oracle-openldap} ) # Prevent duplicate builds on tag pushes. # See https://github.com/mockito/mockito/commit/500519aac4cc13770aa47c1eb7d28e905c27e7aa From 702f0af58aacb937434b35591d5a7b49ecdc4033 Mon Sep 17 00:00:00 2001 From: Artur Gajowy Date: Mon, 10 Apr 2017 20:42:20 +0200 Subject: [PATCH 09/11] Wrap `make ${IMAGES[@]}` in travis_wait so that the build isn't terminated after long time without output --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 72f0992..0168e21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ env: matrix: - IMAGES='teradatalabs/cdh5-base.dependants' - IMAGES='teradatalabs/iop-base.dependants' - - IMAGES='teradatalabs/mapr-base.dependants' + - IMAGES='teradatalabs/mapr52-base.dependants' - IMAGES='teradatalabs/centos6-ssh-oj8.dependants' - IMAGES=$( echo teradatalabs/{dns,ubuntu-trusty-python2.6,presto-admin-devenv,centos6-java8-oracle-ldap,centos6-java8-oracle-openldap} ) @@ -24,7 +24,7 @@ before_install: - sudo pip install docker-release - docker-release --version -install: make ${IMAGES[@]} +install: travis_wait 30 make ${IMAGES[@]} before_deploy: # decrypt the github deploy key From c2ddb51ea45461a0bad071df4ede8867962f3d17 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 18 Apr 2017 19:41:47 +0000 Subject: [PATCH 10/11] Adding conf files from mapr-hive to mapr-base --- teradatalabs/mapr52-base/files/startup.sh | 1 + teradatalabs/mapr52-base/files/supervisord.conf | 1 + teradatalabs/mapr52-base/files/supervisord.d/bootstrap.conf | 1 + 3 files changed, 3 insertions(+) create mode 120000 teradatalabs/mapr52-base/files/startup.sh create mode 120000 teradatalabs/mapr52-base/files/supervisord.conf create mode 120000 teradatalabs/mapr52-base/files/supervisord.d/bootstrap.conf diff --git a/teradatalabs/mapr52-base/files/startup.sh b/teradatalabs/mapr52-base/files/startup.sh new file mode 120000 index 0000000..9a9f191 --- /dev/null +++ b/teradatalabs/mapr52-base/files/startup.sh @@ -0,0 +1 @@ +../../../commons/startup.sh \ No newline at end of file diff --git a/teradatalabs/mapr52-base/files/supervisord.conf b/teradatalabs/mapr52-base/files/supervisord.conf new file mode 120000 index 0000000..c821a2d --- /dev/null +++ b/teradatalabs/mapr52-base/files/supervisord.conf @@ -0,0 +1 @@ +../../../commons/supervisord.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-base/files/supervisord.d/bootstrap.conf b/teradatalabs/mapr52-base/files/supervisord.d/bootstrap.conf new file mode 120000 index 0000000..16a8759 --- /dev/null +++ b/teradatalabs/mapr52-base/files/supervisord.d/bootstrap.conf @@ -0,0 +1 @@ +../../../../commons/supervisord.d/bootstrap.conf \ No newline at end of file From e2852a7989858cf33dbe6cb7e9c836a303da6e9a Mon Sep 17 00:00:00 2001 From: AS186170 Date: Mon, 24 Apr 2017 19:25:16 +0000 Subject: [PATCH 11/11] fixup! Add docker image for MapR --- .travis.yml | 8 +-- .../{mapr52-base => mapr-base}/Dockerfile | 14 +++- teradatalabs/mapr-base/README.md | 13 ++++ .../files/id_rsa.pub | 0 .../files/maprtech.repo | 0 .../files/startup.sh | 0 .../files/supervisord.conf | 0 .../files/supervisord.d/bootstrap.conf | 0 .../Dockerfile | 9 +-- teradatalabs/mapr-hive-kerberized/README.md | 23 +++++++ .../files/bootstrap.sh | 2 +- .../files/conf/hive-site.xml | 0 .../files/jceJars/US_export_policy.jar | Bin .../files/jceJars/local_policy.jar | Bin .../files/kerberos/kadm5.acl | 0 .../files/kerberos/kdc.conf | 0 .../files/kerberos/krb5.conf | 0 .../files/warden_tracker.sh | 37 ++++++++++ .../{mapr52-hive => mapr-hive}/Dockerfile | 17 +++-- teradatalabs/mapr-hive/README.md | 22 ++++++ .../capabilities.txt | 2 +- .../files/bootstrap.sh | 4 +- .../files/conf/core-site.xml | 0 .../files/conf/hive-site.xml | 0 .../{mapr52-hive => mapr-hive}/files/setup.sh | 0 .../files/socks-proxy.sh | 0 .../files/supervisord.d/mysql-metastore.conf | 0 .../files/supervisord.d/socks-proxy.conf | 0 .../files/supervisord.d/sshd.conf | 0 .../mapr-hive/files/warden_tracker.sh | 28 ++++++++ teradatalabs/mapr52-base/README.md | 9 --- teradatalabs/mapr52-hive-kerberized/README.md | 17 ----- .../files/wardenTracker.sh | 64 ------------------ teradatalabs/mapr52-hive/README.md | 17 ----- teradatalabs/mapr52-hive/files/startup.sh | 3 - .../mapr52-hive/files/supervisord.conf | 1 - .../files/supervisord.d/bootstrap.conf | 1 - .../mapr52-hive/files/wardenTracker.sh | 54 --------------- test/image_tests/image_tests.bats.sh | 2 +- 39 files changed, 158 insertions(+), 189 deletions(-) rename teradatalabs/{mapr52-base => mapr-base}/Dockerfile (86%) create mode 100644 teradatalabs/mapr-base/README.md rename teradatalabs/{mapr52-base => mapr-base}/files/id_rsa.pub (100%) rename teradatalabs/{mapr52-base => mapr-base}/files/maprtech.repo (100%) rename teradatalabs/{mapr52-base => mapr-base}/files/startup.sh (100%) rename teradatalabs/{mapr52-base => mapr-base}/files/supervisord.conf (100%) rename teradatalabs/{mapr52-base => mapr-base}/files/supervisord.d/bootstrap.conf (100%) rename teradatalabs/{mapr52-hive-kerberized => mapr-hive-kerberized}/Dockerfile (95%) create mode 100644 teradatalabs/mapr-hive-kerberized/README.md rename teradatalabs/{mapr52-hive-kerberized => mapr-hive-kerberized}/files/bootstrap.sh (97%) rename teradatalabs/{mapr52-hive-kerberized => mapr-hive-kerberized}/files/conf/hive-site.xml (100%) rename teradatalabs/{mapr52-hive-kerberized => mapr-hive-kerberized}/files/jceJars/US_export_policy.jar (100%) rename teradatalabs/{mapr52-hive-kerberized => mapr-hive-kerberized}/files/jceJars/local_policy.jar (100%) rename teradatalabs/{mapr52-hive-kerberized => mapr-hive-kerberized}/files/kerberos/kadm5.acl (100%) rename teradatalabs/{mapr52-hive-kerberized => mapr-hive-kerberized}/files/kerberos/kdc.conf (100%) rename teradatalabs/{mapr52-hive-kerberized => mapr-hive-kerberized}/files/kerberos/krb5.conf (100%) create mode 100644 teradatalabs/mapr-hive-kerberized/files/warden_tracker.sh rename teradatalabs/{mapr52-hive => mapr-hive}/Dockerfile (75%) create mode 100644 teradatalabs/mapr-hive/README.md rename teradatalabs/{mapr52-hive => mapr-hive}/capabilities.txt (79%) rename teradatalabs/{mapr52-hive => mapr-hive}/files/bootstrap.sh (88%) rename teradatalabs/{mapr52-hive => mapr-hive}/files/conf/core-site.xml (100%) rename teradatalabs/{mapr52-hive => mapr-hive}/files/conf/hive-site.xml (100%) rename teradatalabs/{mapr52-hive => mapr-hive}/files/setup.sh (100%) rename teradatalabs/{mapr52-hive => mapr-hive}/files/socks-proxy.sh (100%) rename teradatalabs/{mapr52-hive => mapr-hive}/files/supervisord.d/mysql-metastore.conf (100%) rename teradatalabs/{mapr52-hive => mapr-hive}/files/supervisord.d/socks-proxy.conf (100%) rename teradatalabs/{mapr52-hive => mapr-hive}/files/supervisord.d/sshd.conf (100%) create mode 100644 teradatalabs/mapr-hive/files/warden_tracker.sh delete mode 100644 teradatalabs/mapr52-base/README.md delete mode 100644 teradatalabs/mapr52-hive-kerberized/README.md delete mode 100644 teradatalabs/mapr52-hive-kerberized/files/wardenTracker.sh delete mode 100644 teradatalabs/mapr52-hive/README.md delete mode 100644 teradatalabs/mapr52-hive/files/startup.sh delete mode 120000 teradatalabs/mapr52-hive/files/supervisord.conf delete mode 120000 teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf delete mode 100644 teradatalabs/mapr52-hive/files/wardenTracker.sh diff --git a/.travis.yml b/.travis.yml index 0168e21..902f0c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,11 +3,7 @@ dist: trusty env: matrix: - - IMAGES='teradatalabs/cdh5-base.dependants' - - IMAGES='teradatalabs/iop-base.dependants' - - IMAGES='teradatalabs/mapr52-base.dependants' - - IMAGES='teradatalabs/centos6-ssh-oj8.dependants' - - IMAGES=$( echo teradatalabs/{dns,ubuntu-trusty-python2.6,presto-admin-devenv,centos6-java8-oracle-ldap,centos6-java8-oracle-openldap} ) + - IMAGES=$( echo teradatalabs/{dns,ubuntu-trusty-python2.6,presto-admin-devenv} ) # Prevent duplicate builds on tag pushes. # See https://github.com/mockito/mockito/commit/500519aac4cc13770aa47c1eb7d28e905c27e7aa @@ -24,7 +20,7 @@ before_install: - sudo pip install docker-release - docker-release --version -install: travis_wait 30 make ${IMAGES[@]} +install: travis_wait 48 make teradatalabs/mapr-hive-kerberized before_deploy: # decrypt the github deploy key diff --git a/teradatalabs/mapr52-base/Dockerfile b/teradatalabs/mapr-base/Dockerfile similarity index 86% rename from teradatalabs/mapr52-base/Dockerfile rename to teradatalabs/mapr-base/Dockerfile index 9c2ed2f..de73416 100644 --- a/teradatalabs/mapr52-base/Dockerfile +++ b/teradatalabs/mapr-base/Dockerfile @@ -15,15 +15,17 @@ FROM teradatalabs/centos6-java8-oracle MAINTAINER Teradata Docker Team -# ADD REPO FOR MAPR +# ADD MAPR REPO ADD files/maprtech.repo /etc/yum.repos.d/maprtech.repo COPY files/id_rsa.pub /root/ + RUN yum update -y \ + # ... GET MapRGPG KEY && rpm --import http://package.mapr.com/releases/pub/maprgpg.key \ # INSTALL UTILITY SOFTWARE - && yum install -y iputils vim openssh-server openssh-clients sudo lsof \ + && yum install -y iputils openssh-server openssh-clients sudo lsof \ # CONFIGURE SSH && chkconfig sshd on \ && grep -rl '#Port 22' /etc/ssh/sshd_config | xargs sed -i 's/#Port 22/Port 22/g' \ @@ -68,3 +70,11 @@ RUN yum update -y \ && ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa \ && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ && cat /root/id_rsa.pub | cat >> ~/.ssh/authorized_keys + +# Copy supervisord startup script and base configs +COPY files/startup.sh /root/startup.sh +COPY files/supervisord.conf /etc/supervisord.conf +COPY files/supervisord.d/bootstrap.conf /etc/supervisord.d/bootstrap.conf + +# Add supervisord configs in child images +ONBUILD COPY files/supervisord.d/* /etc/supervisord.d/ diff --git a/teradatalabs/mapr-base/README.md b/teradatalabs/mapr-base/README.md new file mode 100644 index 0000000..d0cae4a --- /dev/null +++ b/teradatalabs/mapr-base/README.md @@ -0,0 +1,13 @@ +# mapr-base [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] + +[layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/mapr-base.svg +[layers-link]: https://microbadger.com/images/teradatalabs/mapr-base +[version-badge]: https://images.microbadger.com/badges/version/teradatalabs/mapr-base.svg +[dockerhub-link]: https://hub.docker.com/r/teradatalabs/mapr-base + +A Hadoop base image for creating images testing Presto, based on the MapR distro. + +## Oracle license + +By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: +[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) \ No newline at end of file diff --git a/teradatalabs/mapr52-base/files/id_rsa.pub b/teradatalabs/mapr-base/files/id_rsa.pub similarity index 100% rename from teradatalabs/mapr52-base/files/id_rsa.pub rename to teradatalabs/mapr-base/files/id_rsa.pub diff --git a/teradatalabs/mapr52-base/files/maprtech.repo b/teradatalabs/mapr-base/files/maprtech.repo similarity index 100% rename from teradatalabs/mapr52-base/files/maprtech.repo rename to teradatalabs/mapr-base/files/maprtech.repo diff --git a/teradatalabs/mapr52-base/files/startup.sh b/teradatalabs/mapr-base/files/startup.sh similarity index 100% rename from teradatalabs/mapr52-base/files/startup.sh rename to teradatalabs/mapr-base/files/startup.sh diff --git a/teradatalabs/mapr52-base/files/supervisord.conf b/teradatalabs/mapr-base/files/supervisord.conf similarity index 100% rename from teradatalabs/mapr52-base/files/supervisord.conf rename to teradatalabs/mapr-base/files/supervisord.conf diff --git a/teradatalabs/mapr52-base/files/supervisord.d/bootstrap.conf b/teradatalabs/mapr-base/files/supervisord.d/bootstrap.conf similarity index 100% rename from teradatalabs/mapr52-base/files/supervisord.d/bootstrap.conf rename to teradatalabs/mapr-base/files/supervisord.d/bootstrap.conf diff --git a/teradatalabs/mapr52-hive-kerberized/Dockerfile b/teradatalabs/mapr-hive-kerberized/Dockerfile similarity index 95% rename from teradatalabs/mapr52-hive-kerberized/Dockerfile rename to teradatalabs/mapr-hive-kerberized/Dockerfile index 0a8b477..fc2fa5e 100644 --- a/teradatalabs/mapr52-hive-kerberized/Dockerfile +++ b/teradatalabs/mapr-hive-kerberized/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM teradatalabs/mapr52-hive +FROM teradatalabs/mapr-hive MAINTAINER Teradata Docker Team # REMOVE UNNECESSARY FILES @@ -20,14 +20,14 @@ RUN rm -rf /opt/mapr/conf/ssl_truststore \ && rm -rf /opt/mapr/conf/maprserverticket \ && rm -rf /opt/mapr/conf/cldb.key \ && rm -rf /opt/mapr/conf/ssl_keystore \ - && rm -rf /root/bootstrap.sh /root/wardenTracker.sh \ + && rm -rf /root/bootstrap.sh /root/warden_tracker.sh \ # INSTALL KERBEROS && yum install -y krb5-libs krb5-server krb5-workstation # ADD KERBEROS CONFIGURATION ADD files/bootstrap.sh /root/ -ADD files/wardenTracker.sh /root/ +ADD files/warden_tracker.sh /root/ ADD files/kerberos/krb5.conf /etc/krb5.conf ADD files/kerberos/kdc.conf /var/kerberos/krb5kdc/kdc.conf ADD files/kerberos/kadm5.acl /var/kerberos/krb5kdc/kadm5.acl @@ -41,6 +41,7 @@ ADD files/conf/hive-site.xml /opt/mapr/hive/hive-1.2/conf/hive-site.xml RUN /usr/sbin/kdb5_util create -s -P password \ && usermod -g root hdfs \ && usermod -g mapr hdfs \ + # CREATE MAPR AND HIVE PRINCIPALS AND KEYTABS && /usr/sbin/kadmin.local -q "addprinc -randkey mapr/mycluster@LABS.TERADATA.COM" \ && /usr/sbin/kadmin.local -q "xst -norandkey -k /opt/mapr/conf/mapr.keytab mapr/mycluster@LABS.TERADATA.COM" \ @@ -53,7 +54,7 @@ RUN /usr/sbin/kdb5_util create -s -P password \ # CHANGE THE PERMISSIONS AND OWNERSHIPS FOR KEYTABS && chmod 644 /opt/mapr/conf/hive.keytab /opt/mapr/conf/mapr.keytab /opt/mapr/conf/hdfs.keytab \ - && chmod 777 /root/bootstrap.sh /root/wardenTracker.sh \ + && chmod 777 /root/bootstrap.sh /root/warden_tracker.sh \ && chown mapr:mapr /opt/mapr/conf/mapr.keytab \ && chown hive:hive /opt/mapr/conf/hive.keytab \ && chown hdfs:hdfs /opt/mapr/conf/hdfs.keytab \ diff --git a/teradatalabs/mapr-hive-kerberized/README.md b/teradatalabs/mapr-hive-kerberized/README.md new file mode 100644 index 0000000..d38d024 --- /dev/null +++ b/teradatalabs/mapr-hive-kerberized/README.md @@ -0,0 +1,23 @@ +# mapr-hive-kerberized [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] + +[layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/mapr-hive-kerberized.svg +[layers-link]: https://microbadger.com/images/teradatalabs/mapr-hive-kerberized +[version-badge]: https://images.microbadger.com/badges/version/teradatalabs/mapr-hive-kerberized.svg +[dockerhub-link]: https://hub.docker.com/r/teradatalabs/mapr-hive-kerberized + + +Docker image with MapR FS, YARN and HIVE installed in a kerberized environment. Please note that running services have lower memory heap size set. +For more details please check hadoop-env.sh(configuration) file. +If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal +for general correctness testing. + +## Run + +``` +$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr-hive-kerberized +``` + +## Oracle license + +By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: +[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh b/teradatalabs/mapr-hive-kerberized/files/bootstrap.sh similarity index 97% rename from teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh rename to teradatalabs/mapr-hive-kerberized/files/bootstrap.sh index 9136588..5553f30 100644 --- a/teradatalabs/mapr52-hive-kerberized/files/bootstrap.sh +++ b/teradatalabs/mapr-hive-kerberized/files/bootstrap.sh @@ -29,7 +29,7 @@ service mapr-zookeeper start service mapr-warden start # WAIT FOR WARDEN TO START ALL THE SERVICES -sh /root/wardenTracker.sh +/root/warden_tracker.sh # START HTTPFS SERVICES maprcli node services -name httpfs -action start -nodes $(hostname) diff --git a/teradatalabs/mapr52-hive-kerberized/files/conf/hive-site.xml b/teradatalabs/mapr-hive-kerberized/files/conf/hive-site.xml similarity index 100% rename from teradatalabs/mapr52-hive-kerberized/files/conf/hive-site.xml rename to teradatalabs/mapr-hive-kerberized/files/conf/hive-site.xml diff --git a/teradatalabs/mapr52-hive-kerberized/files/jceJars/US_export_policy.jar b/teradatalabs/mapr-hive-kerberized/files/jceJars/US_export_policy.jar similarity index 100% rename from teradatalabs/mapr52-hive-kerberized/files/jceJars/US_export_policy.jar rename to teradatalabs/mapr-hive-kerberized/files/jceJars/US_export_policy.jar diff --git a/teradatalabs/mapr52-hive-kerberized/files/jceJars/local_policy.jar b/teradatalabs/mapr-hive-kerberized/files/jceJars/local_policy.jar similarity index 100% rename from teradatalabs/mapr52-hive-kerberized/files/jceJars/local_policy.jar rename to teradatalabs/mapr-hive-kerberized/files/jceJars/local_policy.jar diff --git a/teradatalabs/mapr52-hive-kerberized/files/kerberos/kadm5.acl b/teradatalabs/mapr-hive-kerberized/files/kerberos/kadm5.acl similarity index 100% rename from teradatalabs/mapr52-hive-kerberized/files/kerberos/kadm5.acl rename to teradatalabs/mapr-hive-kerberized/files/kerberos/kadm5.acl diff --git a/teradatalabs/mapr52-hive-kerberized/files/kerberos/kdc.conf b/teradatalabs/mapr-hive-kerberized/files/kerberos/kdc.conf similarity index 100% rename from teradatalabs/mapr52-hive-kerberized/files/kerberos/kdc.conf rename to teradatalabs/mapr-hive-kerberized/files/kerberos/kdc.conf diff --git a/teradatalabs/mapr52-hive-kerberized/files/kerberos/krb5.conf b/teradatalabs/mapr-hive-kerberized/files/kerberos/krb5.conf similarity index 100% rename from teradatalabs/mapr52-hive-kerberized/files/kerberos/krb5.conf rename to teradatalabs/mapr-hive-kerberized/files/kerberos/krb5.conf diff --git a/teradatalabs/mapr-hive-kerberized/files/warden_tracker.sh b/teradatalabs/mapr-hive-kerberized/files/warden_tracker.sh new file mode 100644 index 0000000..78e1ef2 --- /dev/null +++ b/teradatalabs/mapr-hive-kerberized/files/warden_tracker.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +HOSTNAME=$(hostname) +MAPR_LOGIN_READY=1 +MAPR_CLI_READY=$(maprcli service list -node $HOSTNAME | grep 'ERROR (10009)' | wc -l) + +# CHECK RUNNING STATUS OF GIVEN SERVICES USING MAPRCLI +function exposes_mapr_services { + SERVICES=0 + while [ $SERVICES -ne 2 ] + do + SERVICES=$(maprcli service list -node $HOSTNAME | grep $1 | awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) + done +} + +# WAIT FOR MAPRCLI +while [ $MAPR_CLI_READY == 1 ] +do + MAPR_CLI_READY=$(maprcli service list -node $HOSTNAME | grep 'ERROR (10009)' | wc -l) + if [ $MAPR_LOGIN_READY -ne 0 ] + then + # CREATE KERBEROS TICKET + kinit -kt /opt/mapr/conf/mapr.keytab mapr/mycluster@LABS.TERADATA.COM + + # CREATE MAPR TICKET + maprlogin kerberos -user mapr/mycluster@LABS.TERADATA.COM + MAPR_LOGIN_READY=$? + fi +done + +# WAIT FOR MAPR SERVICES TO START +exposes_mapr_services CLDB +exposes_mapr_services NodeManager +exposes_mapr_services ResourceManager +exposes_mapr_services HiveMetastore +exposes_mapr_services HiveServer2 +exposes_mapr_services JobHistoryServer diff --git a/teradatalabs/mapr52-hive/Dockerfile b/teradatalabs/mapr-hive/Dockerfile similarity index 75% rename from teradatalabs/mapr52-hive/Dockerfile rename to teradatalabs/mapr-hive/Dockerfile index 9d3f68f..b29a83d 100644 --- a/teradatalabs/mapr52-hive/Dockerfile +++ b/teradatalabs/mapr-hive/Dockerfile @@ -12,16 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM teradatalabs/mapr52-base +FROM teradatalabs/mapr-base MAINTAINER Teradata Docker Team # ADD ALL REQUIRED SCRIPTS AND FILES TO ROOT DIRECTORY -ADD files/*.sh /root/ +ADD files/setup.sh /root/ +ADD files/warden_tracker.sh /root/ +ADD files/bootstrap.sh /root/ +ADD files/socks-proxy.sh /root/ ADD files/conf/hive-site.xml /opt/mapr/hive/hive-1.2/conf/hive-site.xml ADD files/conf/core-site.xml /opt/mapr/hadoop/hadoop-2.7.0/etc/hadoop/core-site.xml -ADD files/supervisord.conf /etc/supervisord.conf -COPY files/supervisord.d/* /etc/supervisord.d/ -RUN chmod 777 /root/*.sh \ +COPY files/supervisord.d/mysql-metastore.conf /etc/supervisord.d/ +COPY files/supervisord.d/socks-proxy.conf /etc/supervisord.d/ +COPY files/supervisord.d/sshd.conf /etc/supervisord.d/ + +RUN chmod 777 /root/setup.sh /root/warden_tracker.sh /root/bootstrap.sh /root/socks-proxy.sh /root/startup.sh \ # INSTALL MYSQL SERVER && yum install -y mysql-server mysql-connector-java \ @@ -32,7 +37,7 @@ RUN chmod 777 /root/*.sh \ && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ # SETUP FLAT FILE /home/mapr/storagefile - && dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 \ + && dd if=/dev/zero of=/home/mapr/storagefile bs=250M count=30 \ # THE /root/disk.txt IS READ BY MapR TO DETERMINE WHICH DISKS IT CAN USE AND THE /home/mapr/storagefile IS A FLAT FILE THAT ACTS AS A DISK && echo "/home/mapr/storagefile" > /root/disk.txt diff --git a/teradatalabs/mapr-hive/README.md b/teradatalabs/mapr-hive/README.md new file mode 100644 index 0000000..f82e4da --- /dev/null +++ b/teradatalabs/mapr-hive/README.md @@ -0,0 +1,22 @@ +# mapr-hive [![][layers-badge]][layers-link] [![][version-badge]][dockerhub-link] + +[layers-badge]: https://images.microbadger.com/badges/image/teradatalabs/mapr-hive.svg +[layers-link]: https://microbadger.com/images/teradatalabs/mapr-hive +[version-badge]: https://images.microbadger.com/badges/version/teradatalabs/mapr-hive.svg +[dockerhub-link]: https://hub.docker.com/r/teradatalabs/mapr-hive + +Docker image with MapR FS, YARN and HIVE installed. Please note that running services have lower memory heap size set. +For more details please check hadoop-env.sh(configuration) file. +If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal +for general correctness testing. + +## Run + +``` +$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr-hive +``` + +## Oracle license + +By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: +[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-hive/capabilities.txt b/teradatalabs/mapr-hive/capabilities.txt similarity index 79% rename from teradatalabs/mapr52-hive/capabilities.txt rename to teradatalabs/mapr-hive/capabilities.txt index c1da436..9d44a71 100644 --- a/teradatalabs/mapr52-hive/capabilities.txt +++ b/teradatalabs/mapr-hive/capabilities.txt @@ -1,4 +1,4 @@ exposes_mapr allows_creating_a_table_in_hive allows_selecting_from_the_table -exposes_socks_proxy \ No newline at end of file +exposes_socks_proxy diff --git a/teradatalabs/mapr52-hive/files/bootstrap.sh b/teradatalabs/mapr-hive/files/bootstrap.sh similarity index 88% rename from teradatalabs/mapr52-hive/files/bootstrap.sh rename to teradatalabs/mapr-hive/files/bootstrap.sh index 384ad26..b460bd7 100644 --- a/teradatalabs/mapr52-hive/files/bootstrap.sh +++ b/teradatalabs/mapr-hive/files/bootstrap.sh @@ -5,7 +5,7 @@ supervisorctl start sshd supervisorctl start socks-proxy # CONFIGURE MAPR -/opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart +/opt/mapr/server/configure.sh -N mycluster -Z localhost -C localhost -HS localhost -no-autostart # SETUP DISK FOR MAPR BY RUNNING DISKSETUP /opt/mapr/server/disksetup -M -F /root/disk.txt @@ -21,7 +21,7 @@ service mapr-warden start /opt/mapr/server/configure.sh -R # WAIT FOR WARDEN TO START ALL THE SERVICES -sh /root/wardenTracker.sh +/root/warden_tracker.sh # START HTTPFS SERVICES maprcli node services -name httpfs -action start -nodes $(hostname) diff --git a/teradatalabs/mapr52-hive/files/conf/core-site.xml b/teradatalabs/mapr-hive/files/conf/core-site.xml similarity index 100% rename from teradatalabs/mapr52-hive/files/conf/core-site.xml rename to teradatalabs/mapr-hive/files/conf/core-site.xml diff --git a/teradatalabs/mapr52-hive/files/conf/hive-site.xml b/teradatalabs/mapr-hive/files/conf/hive-site.xml similarity index 100% rename from teradatalabs/mapr52-hive/files/conf/hive-site.xml rename to teradatalabs/mapr-hive/files/conf/hive-site.xml diff --git a/teradatalabs/mapr52-hive/files/setup.sh b/teradatalabs/mapr-hive/files/setup.sh similarity index 100% rename from teradatalabs/mapr52-hive/files/setup.sh rename to teradatalabs/mapr-hive/files/setup.sh diff --git a/teradatalabs/mapr52-hive/files/socks-proxy.sh b/teradatalabs/mapr-hive/files/socks-proxy.sh similarity index 100% rename from teradatalabs/mapr52-hive/files/socks-proxy.sh rename to teradatalabs/mapr-hive/files/socks-proxy.sh diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf b/teradatalabs/mapr-hive/files/supervisord.d/mysql-metastore.conf similarity index 100% rename from teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf rename to teradatalabs/mapr-hive/files/supervisord.d/mysql-metastore.conf diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf b/teradatalabs/mapr-hive/files/supervisord.d/socks-proxy.conf similarity index 100% rename from teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf rename to teradatalabs/mapr-hive/files/supervisord.d/socks-proxy.conf diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf b/teradatalabs/mapr-hive/files/supervisord.d/sshd.conf similarity index 100% rename from teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf rename to teradatalabs/mapr-hive/files/supervisord.d/sshd.conf diff --git a/teradatalabs/mapr-hive/files/warden_tracker.sh b/teradatalabs/mapr-hive/files/warden_tracker.sh new file mode 100644 index 0000000..4cab14b --- /dev/null +++ b/teradatalabs/mapr-hive/files/warden_tracker.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +HOSTNAME=$(hostname) +MAPR_CLI_READY=$(maprcli service list -node $HOSTNAME | grep 'ERROR (10009)' | wc -l) + +# CHECK RUNNING STATUS OF GIVEN SERVICES USING MAPRCLI +function exposes_mapr_services { + SERVICES=0 + while [ $SERVICES -ne 2 ] + do + SERVICES=$(maprcli service list -node $HOSTNAME | grep $1 | awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) + done +} + +# WAIT FOR MAPRCLI +while [ $MAPR_CLI_READY == 1 ] +do + MAPR_CLI_READY=$(maprcli service list -node $HOSTNAME | grep 'ERROR (10009)' | wc -l) +done + + +# WAIT FOR MAPR SERVICES TO START +exposes_mapr_services CLDB +exposes_mapr_services NodeManager +exposes_mapr_services ResourceManager +exposes_mapr_services HiveMetastore +exposes_mapr_services HiveServer2 +exposes_mapr_services JobHistoryServer \ No newline at end of file diff --git a/teradatalabs/mapr52-base/README.md b/teradatalabs/mapr52-base/README.md deleted file mode 100644 index 3f5439e..0000000 --- a/teradatalabs/mapr52-base/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# mapr52-base - - -Docker image with all MapR related softwares installed and there dependencies. - -## Oracle license - -By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: -[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-hive-kerberized/README.md b/teradatalabs/mapr52-hive-kerberized/README.md deleted file mode 100644 index 4949497..0000000 --- a/teradatalabs/mapr52-hive-kerberized/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# mapr52-hive-kerberized - -Docker image with kerberos enabled for MapR. Please note that running services have lower memory heap size set. -For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file. -If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal -for general correctness testing. - -## Run - -``` -$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr52-hive-kerberized -``` - -## Oracle license - -By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: -[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-hive-kerberized/files/wardenTracker.sh b/teradatalabs/mapr52-hive-kerberized/files/wardenTracker.sh deleted file mode 100644 index 802c8ef..0000000 --- a/teradatalabs/mapr52-hive-kerberized/files/wardenTracker.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -hname=$(hostname) - -Services=0 -maprloginReady=1 -maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) - -# WAIT FOR CLDB TO START -while [ $Services -ne 2 ] -do -if [ $maprcliReady == 1 ] -then - maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) - Services=0 - # Get MapR ticket - if [ $maprloginReady -ne 0 ] - then - # CREATE KERBEROS TICKET - kinit -kt /opt/mapr/conf/mapr.keytab mapr/mycluster@LABS.TERADATA.COM - - # CREATE MAPR TICKET - maprlogin kerberos -user mapr/mycluster@LABS.TERADATA.COM - maprloginReady=$? - fi -else - Services=$(maprcli service list -node $hname | grep CLDB |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -fi -done - -# WAIT FOR NODEMANAGER TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep NodeManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR RESOURCEMANAGER TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep ResourceManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR HIVE METASTORE TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep HiveMetastore |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR HIVESERVER2 TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep HiveServer2 |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR JOBHISTORYSERVER TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep JobHistoryServer |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done diff --git a/teradatalabs/mapr52-hive/README.md b/teradatalabs/mapr52-hive/README.md deleted file mode 100644 index ebf092a..0000000 --- a/teradatalabs/mapr52-hive/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# mapr52-hive - -Docker image with HDFS, YARN and HIVE installed. Please note that running services have lower memory heap size set. -For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file. -If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal -for general correctness testing. - -## Run - -``` -$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr52-hive -``` - -## Oracle license - -By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here: -[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html) diff --git a/teradatalabs/mapr52-hive/files/startup.sh b/teradatalabs/mapr52-hive/files/startup.sh deleted file mode 100644 index afd8798..0000000 --- a/teradatalabs/mapr52-hive/files/startup.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -exec supervisord -c /etc/supervisord.conf diff --git a/teradatalabs/mapr52-hive/files/supervisord.conf b/teradatalabs/mapr52-hive/files/supervisord.conf deleted file mode 120000 index c821a2d..0000000 --- a/teradatalabs/mapr52-hive/files/supervisord.conf +++ /dev/null @@ -1 +0,0 @@ -../../../commons/supervisord.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf b/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf deleted file mode 120000 index 16a8759..0000000 --- a/teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf +++ /dev/null @@ -1 +0,0 @@ -../../../../commons/supervisord.d/bootstrap.conf \ No newline at end of file diff --git a/teradatalabs/mapr52-hive/files/wardenTracker.sh b/teradatalabs/mapr52-hive/files/wardenTracker.sh deleted file mode 100644 index 5124a4d..0000000 --- a/teradatalabs/mapr52-hive/files/wardenTracker.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -hname=$(hostname) - -Services=0 - -maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) - -# WAIT FOR CLDB TO START -while [ $Services -ne 2 ] -do -if [ $maprcliReady == 1 ] -then - maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l) - Services=0 -else - Services=$(maprcli service list -node $hname | grep CLDB |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -fi -done - -# WAIT FOR NODEMANAGER TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep NodeManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR RESOURCEMANAGER TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep ResourceManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR HIVE METASTORE TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep HiveMetastore |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR HIVESERVER2 TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep HiveServer2 |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done - -# WAIT FOR JOBHISTORYSERVER TO START -Services=0 -while [ $Services -ne 2 ] -do - Services=$(maprcli service list -node $hname | grep JobHistoryServer |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f) -done diff --git a/test/image_tests/image_tests.bats.sh b/test/image_tests/image_tests.bats.sh index 7c1abe1..aee8bb4 100755 --- a/test/image_tests/image_tests.bats.sh +++ b/test/image_tests/image_tests.bats.sh @@ -2,7 +2,7 @@ get_user() { skip_if_needed - if [ "${IMAGE}" == "teradatalabs/mapr52-hive" ] + if [ "${IMAGE}" == "teradatalabs/mapr-hive" ] then echo "hive" else