From 4394f38d48ceeab7c660caf128d35a003e612f34 Mon Sep 17 00:00:00 2001 From: Hanisha Koneru Date: Sun, 13 Oct 2019 14:07:29 -0700 Subject: [PATCH 1/2] POC for OM HA robot tests --- .../src/main/compose/ozone-om-ha/Dockerfile | 47 +++++ .../compose/ozone-om-ha/docker-compose.yaml | 20 +- .../main/compose/ozone-om-ha/docker-config | 57 +++++- .../src/main/compose/ozone-om-ha/startOM.sh | 28 +++ .../src/main/compose/ozone-om-ha/stopOM.sh | 20 ++ .../dist/src/main/compose/ozone-om-ha/test.sh | 40 ++++ .../src/main/smoketest/omha/testOMHA.robot | 193 ++++++++++++++++++ 7 files changed, 389 insertions(+), 16 deletions(-) create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-om-ha/Dockerfile create mode 100755 hadoop-ozone/dist/src/main/compose/ozone-om-ha/startOM.sh create mode 100755 hadoop-ozone/dist/src/main/compose/ozone-om-ha/stopOM.sh create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-om-ha/test.sh create mode 100644 hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/Dockerfile b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/Dockerfile new file mode 100644 index 000000000000..861b4699ed0b --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/Dockerfile @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM apache/ozone-runner + +# Install ssh +RUN sudo yum install -y openssh-clients openssh-server + +RUN sudo ssh-keygen -A +RUN sudo mkdir -p /run/sshd +RUN sudo sed -i "s/.*UsePrivilegeSeparation.*/UsePrivilegeSeparation no/g" /etc/ssh/sshd_config +RUN sudo sed -i "s/.*PermitUserEnvironment.*/PermitUserEnvironment yes/g" /etc/ssh/sshd_config +RUN sudo sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +#/opt/hadoop is mounted, we can't use it as a home +RUN sudo usermod -d /opt hadoop +ADD .ssh /opt/.ssh +RUN sudo chown -R hadoop /opt/.ssh +RUN sudo chown hadoop /opt +RUN sudo chmod 600 /opt/.ssh/* +RUN sudo chmod 700 /opt/.ssh + +RUN sudo sh -c 'echo "export JAVA_HOME=/usr/lib/jvm/jre/" >> /etc/profile' + +# Install required robot framework libraries +RUN sudo pip install robotframework-sshlibrary + +# Add start script to start OMs +ADD startOM.sh /opt/startOM.sh +ADD stopOM.sh /opt/stopOM.sh +RUN sudo chown -R hadoop /opt/startOM.sh +RUN sudo chown -R hadoop /opt/stopOM.sh +RUN sudo chmod 700 /opt/startOM.sh +RUN sudo chmod 700 /opt/stopOM.sh diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-compose.yaml index 2cd2ce80c16a..1718430662c8 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-compose.yaml @@ -17,17 +17,17 @@ version: "3" services: datanode: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} + build: . privileged: true #required by the profiler volumes: - ../..:/opt/hadoop ports: - 9864 - command: ["/opt/hadoop/bin/ozone","datanode"] + command: sh -c "sudo /usr/sbin/sshd -E /tmp/sshd.log && /opt/hadoop/bin/ozone datanode" env_file: - ./docker-config om1: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} + build: . privileged: true #required by the profiler volumes: - ../..:/opt/hadoop @@ -38,9 +38,9 @@ services: ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION env_file: - ./docker-config - command: ["/opt/hadoop/bin/ozone","om"] + command: sleep 1d om2: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} + build: . privileged: true #required by the profiler volumes: - ../..:/opt/hadoop @@ -51,9 +51,9 @@ services: ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION env_file: - ./docker-config - command: ["/opt/hadoop/bin/ozone","om"] + command: sleep 1d om3: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} + build: . privileged: true #required by the profiler volumes: - ../..:/opt/hadoop @@ -64,9 +64,9 @@ services: ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION env_file: - ./docker-config - command: ["/opt/hadoop/bin/ozone","om"] + command: sleep 1d scm: - image: apache/ozone-runner:${OZONE_RUNNER_VERSION} + build: . privileged: true #required by the profiler volumes: - ../..:/opt/hadoop @@ -76,4 +76,4 @@ services: - ./docker-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - command: ["/opt/hadoop/bin/ozone","scm"] + command: sh -c "sudo /usr/sbin/sshd -E /tmp/sshd.log && /opt/hadoop/bin/ozone scm" diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config index a727e3fe24ce..a9e473f3564a 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config @@ -15,12 +15,12 @@ # limitations under the License. CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem -CORE-SITE.XML_fs.defaultFS=o3fs://bucket.volume.id1 -OZONE-SITE.XML_ozone.om.service.ids=id1 -OZONE-SITE.XML_ozone.om.nodes.id1=om1,om2,om3 -OZONE-SITE.XML_ozone.om.address.id1.om1=om1 -OZONE-SITE.XML_ozone.om.address.id1.om2=om2 -OZONE-SITE.XML_ozone.om.address.id1.om3=om3 +CORE-SITE.XML_fs.defaultFS=o3fs://bucket.volume.omservice +OZONE-SITE.XML_ozone.om.service.ids=omservice +OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3 +OZONE-SITE.XML_ozone.om.address.omservice.om1=om1 +OZONE-SITE.XML_ozone.om.address.omservice.om2=om2 +OZONE-SITE.XML_ozone.om.address.omservice.om3=om3 OZONE-SITE.XML_ozone.om.ratis.enable=true OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data @@ -34,6 +34,51 @@ OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 ASYNC_PROFILER_HOME=/opt/profiler +LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout +LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender +LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR +LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN +LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR +LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.grpc.client.GrpcClientProtocolClient=WARN #Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. #BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm + +#LOG4J2.PROPERTIES_* are for Ozone Audit Logging +LOG4J2.PROPERTIES_monitorInterval=30 +LOG4J2.PROPERTIES_filter=read,write +LOG4J2.PROPERTIES_filter.read.type=MarkerFilter +LOG4J2.PROPERTIES_filter.read.marker=READ +LOG4J2.PROPERTIES_filter.read.onMatch=DENY +LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL +LOG4J2.PROPERTIES_filter.write.type=MarkerFilter +LOG4J2.PROPERTIES_filter.write.marker=WRITE +LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL +LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL +LOG4J2.PROPERTIES_appenders=console, rolling +LOG4J2.PROPERTIES_appender.console.type=Console +LOG4J2.PROPERTIES_appender.console.name=STDOUT +LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout +LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n +LOG4J2.PROPERTIES_appender.rolling.type=RollingFile +LOG4J2.PROPERTIES_appender.rolling.name=RollingFile +LOG4J2.PROPERTIES_appender.rolling.fileName=${sys:hadoop.log.dir}/om-audit-${hostName}.log +LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz +LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout +LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n +LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies +LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy +LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400 +LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy +LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB +LOG4J2.PROPERTIES_loggers=audit +LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger +LOG4J2.PROPERTIES_logger.audit.name=OMAudit +LOG4J2.PROPERTIES_logger.audit.level=INFO +LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling +LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile +LOG4J2.PROPERTIES_rootLogger.level=INFO +LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout +LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/startOM.sh b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/startOM.sh new file mode 100755 index 000000000000..e3a439914b69 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/startOM.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## + +ARG=$1 + +if [ "$ARG" != "--restart" ]; then + sudo /usr/sbin/sshd -E /tmp/sshd.log +fi + +export HADOOP_LOG_DIR=/var/log/hadoop +export HADOOP_CONF_DIR=/etc/hadoop +/opt/hadoop/bin/ozone --daemon start om \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/stopOM.sh b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/stopOM.sh new file mode 100755 index 000000000000..f8094380825a --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/stopOM.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## + +/opt/hadoop/bin/ozone --daemon stop om \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/test.sh new file mode 100644 index 000000000000..57f334e8575b --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/test.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR + +# shellcheck source=/dev/null +source "$COMPOSE_DIR/../testlib.sh" + +start_docker_env + +# Start OMs separately. In this test, the OMs will be stopped and restarted multiple times. +# So we do not want the container to be tied to the OM process. +startOMs + +execute_robot_test scm omha/testOMHA.robot + +stop_docker_env + +generate_report + +startOMs(){ + docker-compose exec -T om1 /opt/startOM.sh + docker-compose exec -T om2 /opt/startOM.sh + docker-compose exec -T om3 /opt/startOM.sh +} \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot b/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot new file mode 100644 index 000000000000..66484ac5d0b2 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot @@ -0,0 +1,193 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Smoketest ozone cluster startup +Library OperatingSystem +Library SSHLibrary +Library Collections +Resource ../commonlib.robot +Test Timeout 5 minutes + +*** Variables *** +${SECURITY_ENABLED} false +${HOST} om1 +${USERNAME} hadoop +${PUBLIC_KEY} /opt/.ssh/id_rsa +${OM_SERVICE_ID} omservice +${OZONE_LOG_DIR} /ozone/logs/ +${RATIS_DIR} /data/metadata/ratis +${VOLUME} volume1 +${BUCKET} bucket1 +${TEST_FILE} NOTICE.txt +${WRITE_FILE_COUNT} 0 + +** Keywords *** +Open Connection And Log In + Open Connection ${HOST} + Login With Public Key ${USERNAME} ${PUBLIC_KEY} + +Start OM + [arguments] ${OM_HOST} + Set Global Variable ${HOST} ${OM_HOST} + Open Connection And Log In + Execute Command /opt/startOM.sh --restart + ${startupMsg} = Execute Command sudo ps aux | grep om + Close Connection + Should Contain ${startupMsg} OzoneManagerStarter + +Stop OM + [arguments] ${OM_HOST} + Set Global Variable ${HOST} ${OM_HOST} + Open Connection And Log In + Execute Command /opt/stopOM.sh + ${shutdownMsg} = Execute Command sudo ps aux | grep om + Close Connection + Should Not Contain ${shutdownMsg} OzoneManagerStarter + +Create volume and bucket + Execute ozone sh volume create o3://${OM_SERVICE_ID}/${VOLUME} + Execute ozone sh bucket create o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET} + +Write Test File + ${writeFileCount} = Evaluate ${WRITE_FILE_COUNT}+1 + Set Global Variable ${WRITE_FILE_COUNT} ${writeFileCount} + ${fileName} = Catenate SEPARATOR= ${WRITE_FILE_COUNT} .txt + Copy File ${TEST_FILE} ${fileName} + Execute ozone fs -copyFromLocal ${fileName} o3fs://${BUCKET}.${VOLUME}.${OM_SERVICE_ID}/ + ${result} = Execute ozone sh key list o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET} | grep -v WARN | jq -r '.name' + Should contain ${result} ${fileName} + Remove File ${fileName} + +Put Key + [arguments] ${FILE} ${KEY} + Execute ozone sh key put o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${KEY} ${FILE} + ${result} = Execute ozone sh key info o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${KEY} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.name=="${KEY}")' + Should contain ${result} creationTime + +Put Multiple Keys + [arguments] ${NUM_KEYS} ${KEY_PREFIX} ${FILE} + FOR ${INDEX} IN RANGE ${NUM_KEYS} + ${tmpKey} = Catenate SEPARATOR= ${KEY_PREFIX} _ ${INDEX} + LOG ${tmpKey} + Put Key ${FILE} ${tmpKey} + END + +Put Key With Timeout + [arguments] ${KEY} + [Timeout] 3 minutes + Put Key ${TEST_FILE} ${KEY} + +Get OM Leader Node + ${result} = Execute ozone omha --getservicestate --serviceId=omservice + LOG ${result} + Should Contain ${result} LEADER 1 + Should Contain ${result} FOLLOWER 2 + ${omLine} = Get Lines Containing String ${result} LEADER + ${split1} ${split2} Split String ${omLine} : + ${leaderOM} = Strip String ${split1} + LOG Leader OM: ${leaderOM} + [return] ${leaderOM} + +Get Ratis Logs + [arguments] ${OM_HOST} + Set Global Variable ${HOST} ${OM_HOST} + Open Connection And Log In + ${gorupId} = Execute Command ls ${RATIS_DIR} | grep -v 'snapshot' + LOG Ratis GroupId: ${gorupId} + ${currDir} = Catenate SEPARATOR= ${RATIS_DIR} / ${gorupId} /current/ + @{logs} = SSHLibrary.List Files In Directory ${currDir} log_* + ${numLogs} = Get Length ${logs} + [return] ${numLogs} ${logs} + +** Test Cases *** +Stop Leader OM and Verify Failover + # Check OM write operation before failover + Create volume and bucket + Write Test File + + # Find Leader OM and stop it + ${leaderOM} = Get OM Leader Node + ${stopOMResult} = Stop OM ${leaderOM} + + # Verify that new Leader OM is elected + ${newLeaderOM} = Get OM Leader Node + Should Not be Equal ${leaderOM} ${newLeaderOM} OMs did not failover + + # Verify write succeeds after failover + Write Test File + + # Restart stopped OM + Start OM ${leaderOM} + +Restart OM and Verify Ratis Logs + Set Test Variable ${OM_HOST} om2 + Set Test Variable ${keyBase} testOMRestart_ + + # Stop 1 OM and get the Logs present in its Ratis Dir + Stop OM ${OM_HOST} + ${numLogsBefore} @{logsBefore} = Get Ratis Logs ${OM_HOST} + ${leaderOM} = Get OM Leader Node + + # Perform write operations to advance the Ratis log index till a new Log segment is created + FOR ${INDEX} IN RANGE 20 + Set Test Variable ${keyPrefix} ${keyBase}${INDEX} + Put Multiple Keys 5 ${keyPrefix} ${TEST_FILE} + ${numLogsLeader} @{logsLeader} = Get Ratis Logs ${leaderOM} + EXIT FOR LOOP IF ${numLogsLeader} > ${numLogsBefore} + END + Should Be True ${numLogsLeader} > ${numLogsBefore} Cannot test OM Restart as Ratis did not start new log segment. + + # Restart the stopped OM and wait for Ratis to catch up with Leader OM + Start OM ${OM_HOST} + FOR ${INDEX} IN RANGE 300 + ${numLogsAfter} @{logsAfter} = Get Ratis Logs ${OM_HOST} + EXIT FOR LOOP IF ${numLogsAfter} >= ${numLogsLeader} + Sleep 1s + END + Should Be True ${numLogsAfter} >= ${numLogsLeader} Restarted OM did not catch up with Leader OM + + # Verify that the logs match with the Leader OMs logs + List Should Contain Sub List ${logsAfter} ${logsLeader} + +Test Multiple Failovers + FOR ${INDEX} IN RANGE 5 + # Find Leader OM and stop it + ${leaderOM} = Get OM Leader Node + ${stopOMResult} = Stop OM ${leaderOM} + + # Verify that new Leader OM is elected + ${newLeaderOM} = Get OM Leader Node + Should Not be Equal ${leaderOM} ${newLeaderOM} OMs did not failover + + # Verify write succeeds after failover + Write Test File + + # Restart OM + Start OM ${leaderOM} +# END + +Test Two OMs Down + # Stop 2 OMs + Set Test Variable ${keyName} put_key_with_timeout + STOP OM om1 + STOP OM om2 + ${leaderOM} = Get OM Leader Node + Should Be Equal ${leaderOM} om3 + Put Key With Timeout ${keyName} + ${result} = Execute ozone sh key info o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${keyName} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' + Should contain ${result} KEY_NOT_FOUND + + From 615ece09edec710d7ee8f9ed512a4f6214b2d71d Mon Sep 17 00:00:00 2001 From: Hanisha Koneru Date: Wed, 30 Oct 2019 18:41:13 -0700 Subject: [PATCH 2/2] fixes after HDDS-2240 --- .../compose/ozone-om-ha/.ssh/authorized_keys | 16 +++++++ .../src/main/compose/ozone-om-ha/.ssh/config | 18 ++++++++ .../main/compose/ozone-om-ha/.ssh/environment | 16 +++++++ .../src/main/compose/ozone-om-ha/.ssh/id_rsa | 42 +++++++++++++++++++ .../main/compose/ozone-om-ha/.ssh/id_rsa.pub | 16 +++++++ .../main/compose/ozone-om-ha/docker-config | 1 + .../dist/src/main/compose/ozone-om-ha/test.sh | 12 +++--- .../src/main/smoketest/omha/testOMHA.robot | 21 +--------- 8 files changed, 117 insertions(+), 25 deletions(-) create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/authorized_keys create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/config create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/environment create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/id_rsa create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/id_rsa.pub diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/authorized_keys b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/authorized_keys new file mode 100644 index 000000000000..ae390529c7eb --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/authorized_keys @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDgEmLpYm4BrWtq1KG9hhZXCZgGrETntu0eNTo21U3VKc9nH9/ot7M6lAawsFcT9uXu4b58PTlnfvwH/TATlCFjC8n0Z7SOx+FU6L3Sn8URh9HaX4L0tF8u87oCAD4dBrUGhhB36eiuH9dBBWly6RKffYJvrjatbc7GxBO/e5OSUMtqk/DSVKksmBhZxutrKivCNjDish9ViGIf8b5yS/MlEGmaVKApik1fJ5iOlloM/GgpB60YV/hbqfCecbWgeiM1gK92gdOcA/Wx1C7fj8BSI5iDSE6eZeF80gM3421lvyPDWyVhFaGbka4rXBX/fb9QSRBA9RTqhRKAEmAIf49H hadoop@cdae967fa87a diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/config new file mode 100644 index 000000000000..6506916ded0f --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/config @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +Host * + UserKnownHostsFile /dev/null + StrictHostKeyChecking no diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/environment b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/environment new file mode 100644 index 000000000000..cbde0f2078df --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/environment @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +JAVA_HOME=/usr/lib/jvm/jre diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/id_rsa b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/id_rsa new file mode 100644 index 000000000000..6632ce51c54a --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/id_rsa @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA4BJi6WJuAa1ratShvYYWVwmYBqxE57btHjU6NtVN1SnPZx/f +6LezOpQGsLBXE/bl7uG+fD05Z378B/0wE5QhYwvJ9Ge0jsfhVOi90p/FEYfR2l+C +9LRfLvO6AgA+HQa1BoYQd+norh/XQQVpcukSn32Cb642rW3OxsQTv3uTklDLapPw +0lSpLJgYWcbrayorwjYw4rIfVYhiH/G+ckvzJRBpmlSgKYpNXyeYjpZaDPxoKQet +GFf4W6nwnnG1oHojNYCvdoHTnAP1sdQu34/AUiOYg0hOnmXhfNIDN+NtZb8jw1sl +YRWhm5GuK1wV/32/UEkQQPUU6oUSgBJgCH+PRwIDAQABAoIBAQDI1TH6ZNKchkck +9XgSWsBjOqIcOQN5fCeDT8nho8WjLVpL3/Hcr+ngsxRcAXHK3xyvw33r9SQic1qJ +/pC8u6RBFivo95qJ7vU0GXcp9TG4yLd6tui1U4WMm784U+dYNM7EDh1snSaECt3v +1V3yNJ0QfnoOh2NShn0zAkOA+M4H8Nx2SudMCsjcbK9+fYxzW3hX+sJpMKdjG1HW +DUz+I7cW7t0EGaVrgVSV+eR58LiXu+14YDNMrySiejB4nD2sKrx93XgiCBECCsBN +GLQGJCztaXoAY+5Kf/aJ9EEf2wBF3GecRk+XIAd87PeDmeahLQAVkAJ/rD1vsKFs +8kWi6CrhAoGBAP7leG/dMBhlfvROpBddIfWm2i8B+oZiAlSjdYGz+/ZhUaByXk18 +pupMGiiMQR1ziPnEg0gNgR2ZkH54qrXPn5WcQa4rlSEtUsZEp5v5WblhfX2QwKzY +G/uhA+mB7wXpQkSmXo0LclfPF2teROQrG1OyfWkWbxFH4i3+em7sL95jAoGBAOEK +v+wscqkMLW7Q8ONbWMCCBlmMHr6baB3VDCYZx25lr+GIF5zmJJFTmF2rq2VSAlts +qx1AGmaUSo78kC5FuJvSNTL6a1Us5ucdthQZM3N8pAz+OAE+QEU+BsdA27yAh3tO +yKDsMFNHKtXcgy5LeB5gzENLlNyw2jgkRv2Ef77NAoGAVH8DHqoHEH9Mx3XuRWR1 +JnaqKx0PzE5fEWmiQV3Fr5XxNivTgQJKXq7dmQVtbHLpPErdbhwz6fkHAjXD+UMb +VsAWscL2y6m3n8wQd87/5EkiDWbXyDRXimGE53pQHviFJDa2bzEVNXCMBeaZFb4I +cAViN1zdcrAOlUqfkXewIpsCgYB8wsXl/DpRB+RENGfn0+OfTjaQ/IKq72NIbq1+ +jfondQ6N/TICFQEe5HZrL9okoNOXteYjoD9CsWGoZdLVJGgVUvOVYImSvgMBDFK+ +T75bfzU/0sxfvBBLkviVDJsFpUf3D5VgybB86s6Po+HCD6r3RHjZshRESXOhflMx +B3z+3QKBgE2Lwo0DuwUGkm6k8psyn3x8EiXNsiNw12cojicFTyKUYLHxMBeVbCLW +3j3pxSggJgRuBLLzixUHbHp91r2ARTy28naK7R/la8yKVqK6ojcikN2mQsCHYtwB +nuFwXr42ytn6G+9Wn4xT64tGjRCqyZn0/v0XsPjVCyrZ6G7EtNHP +-----END RSA PRIVATE KEY----- diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/id_rsa.pub b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/id_rsa.pub new file mode 100644 index 000000000000..ae390529c7eb --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/.ssh/id_rsa.pub @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDgEmLpYm4BrWtq1KG9hhZXCZgGrETntu0eNTo21U3VKc9nH9/ot7M6lAawsFcT9uXu4b58PTlnfvwH/TATlCFjC8n0Z7SOx+FU6L3Sn8URh9HaX4L0tF8u87oCAD4dBrUGhhB36eiuH9dBBWly6RKffYJvrjatbc7GxBO/e5OSUMtqk/DSVKksmBhZxutrKivCNjDish9ViGIf8b5yS/MlEGmaVKApik1fJ5iOlloM/GgpB60YV/hbqfCecbWgeiM1gK92gdOcA/Wx1C7fj8BSI5iDSE6eZeF80gM3421lvyPDWyVhFaGbka4rXBX/fb9QSRBA9RTqhRKAEmAIf49H hadoop@cdae967fa87a diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config index a9e473f3564a..fbf569dbb896 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config @@ -29,6 +29,7 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.handler.type=distributed OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.replication=1 +OZONE-SITE.XML_ozone.client.failover.max.attempts=6 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true HDFS-SITE.XML_rpc.metrics.quantile.enable=true diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/test.sh index 57f334e8575b..3146ae7ae711 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/test.sh @@ -15,6 +15,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +startOMs(){ + docker-compose exec -T om1 /opt/startOM.sh + docker-compose exec -T om2 /opt/startOM.sh + docker-compose exec -T om3 /opt/startOM.sh +} + COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR @@ -32,9 +38,3 @@ execute_robot_test scm omha/testOMHA.robot stop_docker_env generate_report - -startOMs(){ - docker-compose exec -T om1 /opt/startOM.sh - docker-compose exec -T om2 /opt/startOM.sh - docker-compose exec -T om3 /opt/startOM.sh -} \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot b/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot index 66484ac5d0b2..2b63b57bf3e3 100644 --- a/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot +++ b/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot @@ -19,7 +19,6 @@ Library OperatingSystem Library SSHLibrary Library Collections Resource ../commonlib.robot -Test Timeout 5 minutes *** Variables *** ${SECURITY_ENABLED} false @@ -85,13 +84,8 @@ Put Multiple Keys Put Key ${FILE} ${tmpKey} END -Put Key With Timeout - [arguments] ${KEY} - [Timeout] 3 minutes - Put Key ${TEST_FILE} ${KEY} - Get OM Leader Node - ${result} = Execute ozone omha --getservicestate --serviceId=omservice + ${result} = Execute ozone admin om getserviceroles --service-id=omservice LOG ${result} Should Contain ${result} LEADER 1 Should Contain ${result} FOLLOWER 2 @@ -177,17 +171,6 @@ Test Multiple Failovers # Restart OM Start OM ${leaderOM} -# END - -Test Two OMs Down - # Stop 2 OMs - Set Test Variable ${keyName} put_key_with_timeout - STOP OM om1 - STOP OM om2 - ${leaderOM} = Get OM Leader Node - Should Be Equal ${leaderOM} om3 - Put Key With Timeout ${keyName} - ${result} = Execute ozone sh key info o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${keyName} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' - Should contain ${result} KEY_NOT_FOUND + END