From 7e12cade95c9d2e34c4fee197a35323082213aa3 Mon Sep 17 00:00:00 2001 From: daniilchik Date: Thu, 10 Oct 2024 12:33:51 +0300 Subject: [PATCH 1/8] HDDS-11503. Add Robot test to verify Container Balancer for EC containers --- .../src/main/compose/ozone-balancer-ec/.env | 20 ++ .../ozone-balancer-ec/docker-compose.yaml | 211 ++++++++++++++++++ .../compose/ozone-balancer-ec/docker-config | 59 +++++ .../main/compose/ozone-balancer-ec/test.sh | 31 +++ .../smoketest/balancer/testBalancerEC.robot | 180 +++++++++++++++ 5 files changed, 501 insertions(+) create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/.env create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-compose.yaml create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-config create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh create mode 100644 hadoop-ozone/dist/src/main/smoketest/balancer/testBalancerEC.robot diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/.env b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/.env new file mode 100644 index 000000000000..0e99fab82fd0 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/.env @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HDDS_VERSION=${hdds.version} +OZONE_RUNNER_VERSION=${docker.ozone-runner.version} +OZONE_RUNNER_IMAGE=apache/ozone-runner +OZONE_OPTS= \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-compose.yaml new file mode 100644 index 000000000000..eeaf1a794649 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-compose.yaml @@ -0,0 +1,211 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# reusable fragments (see https://docs.docker.com/compose/compose-file/#extension-fields) +x-common-config: + &common-config + image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} + volumes: + - ../..:/opt/hadoop + env_file: + - docker-config + +x-replication: + &replication + OZONE-SITE.XML_ozone.server.default.replication: ${OZONE_REPLICATION_FACTOR:-3} + +services: + datanode1: + <<: *common-config + ports: + - 19864 + - 9882 + environment: + <<: *replication + command: ["ozone","datanode"] + volumes: + - tmpfs1:/data + - ../..:/opt/hadoop + datanode2: + <<: *common-config + ports: + - 19864 + - 9882 + environment: + <<: *replication + command: [ "ozone","datanode" ] + volumes: + - tmpfs2:/data + - ../..:/opt/hadoop + datanode3: + <<: *common-config + ports: + - 19864 + - 9882 + environment: + <<: *replication + command: [ "ozone","datanode" ] + volumes: + - tmpfs3:/data + - ../..:/opt/hadoop + datanode4: + <<: *common-config + ports: + - 19864 + - 9882 + environment: + <<: *replication + command: [ "ozone","datanode" ] + volumes: + - tmpfs4:/data + - ../..:/opt/hadoop + datanode5: + <<: *common-config + ports: + - 19864 + - 9882 + environment: + <<: *replication + command: [ "ozone","datanode" ] + volumes: + - tmpfs5:/data + - ../..:/opt/hadoop + datanode6: + <<: *common-config + ports: + - 19864 + - 9882 + environment: + <<: *replication + command: [ "ozone","datanode" ] + volumes: + - tmpfs6:/data + - ../..:/opt/hadoop + om1: + <<: *common-config + environment: + WAITFOR: scm3:9894 + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + <<: *replication + ports: + - 9874:9874 + - 9862 + hostname: om1 + command: ["ozone","om"] + om2: + <<: *common-config + environment: + WAITFOR: scm3:9894 + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + <<: *replication + ports: + - 9874 + - 9862 + hostname: om2 + command: ["ozone","om"] + om3: + <<: *common-config + environment: + WAITFOR: scm3:9894 + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + <<: *replication + ports: + - 9874 + - 9862 + hostname: om3 + command: ["ozone","om"] + scm1: + <<: *common-config + ports: + - 9876:9876 + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1} + <<: *replication + command: ["ozone","scm"] + scm2: + <<: *common-config + ports: + - 9876 + environment: + WAITFOR: scm1:9894 + ENSURE_SCM_BOOTSTRAPPED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1} + <<: *replication + command: ["ozone","scm"] + scm3: + <<: *common-config + ports: + - 9876 + environment: + WAITFOR: scm2:9894 + ENSURE_SCM_BOOTSTRAPPED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1} + <<: *replication + command: ["ozone","scm"] + httpfs: + <<: *common-config + environment: + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1} + <<: *replication + ports: + - 14000:14000 + command: [ "ozone","httpfs" ] + s3g: + <<: *common-config + environment: + OZONE_OPTS: + <<: *replication + ports: + - 9878:9878 + command: ["ozone","s3g"] +volumes: + tmpfs1: + driver: local + driver_opts: + o: "size=1g,uid=1000" + device: tmpfs + type: tmpfs + tmpfs2: + driver: local + driver_opts: + o: "size=1g,uid=2000" + device: tmpfs + type: tmpfs + tmpfs3: + driver: local + driver_opts: + o: "size=1g,uid=3000" + device: tmpfs + type: tmpfs + tmpfs4: + driver: local + driver_opts: + o: "size=1g,uid=4000" + device: tmpfs + type: tmpfs + tmpfs5: + driver: local + driver_opts: + o: "size=1g,uid=5000" + device: tmpfs + type: tmpfs + tmpfs6: + driver: local + driver_opts: + o: "size=1g,uid=6000" + device: tmpfs + type: tmpfs diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-config new file mode 100644 index 000000000000..e2c288b16f18 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-config @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For HttpFS service it is required to enable proxying users. +CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts=* +CORE-SITE.XML_hadoop.proxyuser.hadoop.groups=* + +CORE-SITE.XML_fs.defaultFS=ofs://om/ +CORE-SITE.XML_fs.trash.interval=1 + +OZONE-SITE.XML_ozone.om.service.ids=om +OZONE-SITE.XML_ozone.om.nodes.om=om1,om2,om3 +OZONE-SITE.XML_ozone.om.address.om.om1=om1 +OZONE-SITE.XML_ozone.om.address.om.om2=om2 +OZONE-SITE.XML_ozone.om.address.om.om3=om3 + +OZONE-SITE.XML_ozone.scm.service.ids=scmservice +OZONE-SITE.XML_ozone.scm.nodes.scmservice=scm1,scm2,scm3 +OZONE-SITE.XML_ozone.scm.address.scmservice.scm1=scm1 +OZONE-SITE.XML_ozone.scm.address.scmservice.scm2=scm2 +OZONE-SITE.XML_ozone.scm.address.scmservice.scm3=scm3 +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.container.size=100MB +OZONE-SITE.XML_ozone.scm.block.size=20MB +OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB +OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata +OZONE-SITE.XML_hdds.node.report.interval=20s +OZONE-SITE.XML_hdds.heartbeat.interval=20s +OZONE-SITE.XML_hdds.datanode.du.refresh.period=20s +OZONE-SITE.XML_hdds.container.balancer.balancing.iteration.interval=2m +OZONE-SITE.XML_hdds.datanode.dir=/data/hdds +OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB +OZONE-SITE.XML_ozone.scm.pipeline.creation.auto.factor.one=false +OZONE-SITE.XML_ozone.datanode.pipeline.limit=1 +OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s +OZONE-SITE.XML_ozone.scm.primordial.node.id=scm1 +OZONE-SITE.XML_hdds.container.report.interval=30s +OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true +OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon +OZONE-SITE.XML_dfs.container.ratis.datastream.enabled=true +OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http + +OZONE_CONF_DIR=/etc/hadoop +OZONE_LOG_DIR=/var/log/hadoop + +no_proxy=om1,om2,om3,scm,s3g,recon,kdc,localhost,127.0.0.1 diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh new file mode 100644 index 000000000000..5f7a3f89dfb5 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#suite:balancer + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR +export OM_SERVICE_ID="om" +export OM=om1 +export SCM=scm1 +export OZONE_REPLICATION_FACTOR=3 + +source "$COMPOSE_DIR/../testlib.sh" + +start_docker_env 5 + +execute_robot_test ${OM} balancer/testBalancerEC.robot diff --git a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancerEC.robot b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancerEC.robot new file mode 100644 index 000000000000..ea64c6ba349d --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancerEC.robot @@ -0,0 +1,180 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*** Settings *** +Documentation Smoketest ozone cluster startup +Library OperatingSystem +Library Collections +Resource ../commonlib.robot +Resource ../ozone-lib/shell.robot + +Test Timeout 20 minutes + +*** Variables *** +${SECURITY_ENABLED} false +${HOST} datanode1 +${VOLUME} volume1 +${BUCKET} bucket1 +${SIZE} 104857600 + + +** Keywords *** +Prepare For Tests + Execute dd if=/dev/urandom of=/tmp/100mb bs=1048576 count=100 + Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab + Execute ozone sh volume create ${VOLUME} + Execute ozone sh bucket create --replication rs-3-2-1024k --type EC /${VOLUME}/${BUCKET} + + +Datanode In Maintenance Mode + ${result} = Execute ozone admin datanode maintenance ${HOST} + Should Contain ${result} Entering maintenance mode on datanode + ${result} = Execute ozone admin datanode list | grep "Operational State:*" + Wait Until Keyword Succeeds 30sec 5sec Should contain ${result} ENTERING_MAINTENANCE + Wait Until Keyword Succeeds 3min 10sec Related pipelines are closed + Sleep 60000ms + +Related pipelines are closed + ${result} = Execute ozone admin datanode list | awk -v RS= '{$1=$1}1'|grep MAINT | sed -e 's/^.*pipelines: \\(.*\\)$/\\1/' + Should Contain Any ${result} CLOSED No related pipelines or the node is not in Healthy state. + +Datanode Recommission + ${result} = Execute ozone admin datanode recommission ${HOST} + Should Contain ${result} Started recommissioning datanode + Wait Until Keyword Succeeds 1min 10sec Datanode Recommission is Finished + Sleep 300000ms + +Datanode Recommission is Finished + ${result} = Execute ozone admin datanode list | grep "Operational State:*" + Should Not Contain ${result} ENTERING_MAINTENANCE + +Run Container Balancer + ${result} = Execute ozone admin containerbalancer start -t 0.1 -d 100 -i 1 + Should Contain ${result} Container Balancer started successfully. + +Wait Finish Of Balancing + ${result} = Execute ozone admin containerbalancer status -v --history + Should Contain ${result} ContainerBalancer is Running. + Wait Until Keyword Succeeds 10min 10sec ContainerBalancer is Not Running + Sleep 60000ms + +Verify Verbose Balancer Status + [arguments] ${output} + + Should Contain ${output} ContainerBalancer is Running. + Should Contain ${output} Started at: + Should Contain ${output} Container Balancer Configuration values: + +Verify Balancer Iteration + [arguments] ${output} ${number} ${status} ${containers} + + Should Contain ${output} Iteration number ${number} + Should Contain ${output} Iteration result ${status} + Should Contain ${output} Scheduled to move containers ${containers} + +Run Balancer Status + ${result} = Execute ozone admin containerbalancer status + Should Contain ${result} ContainerBalancer is Running. + +Run Balancer Verbose Status + ${result} = Execute ozone admin containerbalancer status -v + Verify Verbose Balancer Status ${result} + Verify Balancer Iteration ${result} 1 IN_PROGRESS 3 + Should Contain ${result} Current iteration info: + +Run Balancer Verbose History Status + ${result} = Execute ozone admin containerbalancer status -v --history + Verify Verbose Balancer Status ${result} + Verify Balancer Iteration ${result} 1 IN_PROGRESS 3 + Should Contain ${result} Iteration history list: + +ContainerBalancer is Not Running + ${result} = Execute ozone admin containerbalancer status + Should contain ${result} ContainerBalancer is Not Running. + +Create Multiple Keys + [arguments] ${NUM_KEYS} + ${file} = Set Variable /tmp/100mb + FOR ${INDEX} IN RANGE ${NUM_KEYS} + ${fileName} = Set Variable file-${INDEX}.txt + ${key} = Set Variable /${VOLUME}/${BUCKET}/${fileName} + LOG ${fileName} + Create Key ${key} ${file} --replication=rs-3-2-1024k --type=EC + Key Should Match Local File ${key} ${file} + END + +Datanode Usageinfo + [arguments] ${uuid} + ${result} = Execute ozone admin datanode usageinfo --uuid=${uuid} + Should Contain ${result} Ozone Used + +Get Uuid + ${result} = Execute ozone admin datanode list | awk -v RS= '{$1=$1}1'| grep ${HOST} | sed -e 's/Datanode: //'|sed -e 's/ .*$//' + [return] ${result} + + +Close All Containers + FOR ${INDEX} IN RANGE 15 + ${container} = Execute ozone admin container list --state OPEN | jq -r 'select(.replicationConfig.data == 3) | .containerID' | head -1 + EXIT FOR LOOP IF "${container}" == "${EMPTY}" + ${message} = Execute And Ignore Error ozone admin container close "${container}" + Run Keyword If '${message}' != '${EMPTY}' Should Contain ${message} is in closing state + ${output} = Execute ozone admin container info "${container}" + Should contain ${output} CLOS + END + Wait until keyword succeeds 3min 10sec All container is closed + +All container is closed + ${output} = Execute ozone admin container list --state OPEN + Should Be Empty ${output} + +Get Datanode Ozone Used Bytes Info + [arguments] ${uuid} + ${output} = Execute export DATANODES=$(ozone admin datanode list --json) && for datanode in $(echo "$\{DATANODES\}" | jq -r '.[].datanodeDetails.uuid'); do ozone admin datanode usageinfo --uuid=$\{datanode\} --json | jq '{(.[0].datanodeDetails.uuid) : .[0].ozoneUsed}'; done | jq -s add + ${result} = Execute echo '${output}' | jq '. | to_entries | .[] | select(.key == "${uuid}") | .value' + [return] ${result} + +** Test Cases *** +Verify Container Balancer for EC containers + Prepare For Tests + + Datanode In Maintenance Mode + + ${uuid} = Get Uuid + Datanode Usageinfo ${uuid} + + Create Multiple Keys 3 + + Close All Containers + + ${datanodeOzoneUsedBytesInfo} = Get Datanode Ozone Used Bytes Info ${uuid} + Should Be True ${datanodeOzoneUsedBytesInfo} < ${SIZE} + + Datanode Recommission + + Run Container Balancer + + Run Balancer Status + + Run Balancer Verbose Status + + Run Balancer Verbose History Status + + Wait Finish Of Balancing + + + ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} = Get Datanode Ozone Used Bytes Info ${uuid} + Should Not Be Equal As Integers ${datanodeOzoneUsedBytesInfo} ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} + Should Be True ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} < ${SIZE} * 2.5 + Should Be True ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} > ${SIZE} * 2 \ No newline at end of file From 475f28b08f752182b71f6c8b15ad9fff5e396a55 Mon Sep 17 00:00:00 2001 From: daniilchik Date: Fri, 11 Oct 2024 01:35:41 +0300 Subject: [PATCH 2/8] HDDS-11503. Add Robot test to verify Container Balancer for EC containers --- .../compose/ozone-balancer-ec/docker-config | 1 + .../main/compose/ozone-balancer-ec/test.sh | 7 +- .../src/main/compose/ozone-balancer/test.sh | 2 +- .../smoketest/balancer/testBalancer.robot | 11 +- .../smoketest/balancer/testBalancerEC.robot | 180 ------------------ 5 files changed, 12 insertions(+), 189 deletions(-) delete mode 100644 hadoop-ozone/dist/src/main/smoketest/balancer/testBalancerEC.robot diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-config index e2c288b16f18..d0e0ab7dcfb5 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-config @@ -42,6 +42,7 @@ OZONE-SITE.XML_hdds.heartbeat.interval=20s OZONE-SITE.XML_hdds.datanode.du.refresh.period=20s OZONE-SITE.XML_hdds.container.balancer.balancing.iteration.interval=2m OZONE-SITE.XML_hdds.datanode.dir=/data/hdds +OZONE-SITE.XML_hdds.datanode.container.db.dir=/data/metadata OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB OZONE-SITE.XML_ozone.scm.pipeline.creation.auto.factor.one=false OZONE-SITE.XML_ozone.datanode.pipeline.limit=1 diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh index 5f7a3f89dfb5..bb370bce0da8 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh @@ -17,7 +17,7 @@ #suite:balancer -COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE0}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR export OM_SERVICE_ID="om" export OM=om1 @@ -26,6 +26,7 @@ export OZONE_REPLICATION_FACTOR=3 source "$COMPOSE_DIR/../testlib.sh" -start_docker_env 5 +start_docker_env 6 +execute_robot_test ${OM} -v REPLICATION:rs-3-2-1024k -v REPLICATION_TYPE:EC balancer/testBalancerEC.robot + -execute_robot_test ${OM} balancer/testBalancerEC.robot diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh index e79979877ba3..0e5baaff2f58 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh @@ -30,4 +30,4 @@ source "$COMPOSE_DIR/../testlib.sh" # We need 4 dataNodes in this tests start_docker_env 4 -execute_robot_test ${OM} balancer/testBalancer.robot +execute_robot_test ${OM} -v REPLICATION:THREE -v REPLICATION_TYPE:RATIS balancer/testBalancer.robot diff --git a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot index 4299afe5f2d1..6722cf919dac 100644 --- a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot +++ b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot @@ -16,6 +16,7 @@ *** Settings *** Documentation Smoketest ozone cluster startup Library OperatingSystem +Library String Library Collections Resource ../commonlib.robot Resource ../ozone-lib/shell.robot @@ -35,7 +36,7 @@ Prepare For Tests Execute dd if=/dev/urandom of=/tmp/100mb bs=1048576 count=100 Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab Execute ozone sh volume create /${VOLUME} - Execute ozone sh bucket create /${VOLUME}/${BUCKET} + Execute ozone sh bucket create --replication rs-3-2-1024k --type EC /${VOLUME}/${BUCKET} Datanode In Maintenance Mode @@ -67,7 +68,7 @@ Run Container Balancer Wait Finish Of Balancing ${result} = Execute ozone admin containerbalancer status Should Contain ${result} ContainerBalancer is Running. - Wait Until Keyword Succeeds 3min 10sec ContainerBalancer is Not Running + Wait Until Keyword Succeeds 10min 10sec ContainerBalancer is Not Running Sleep 60000ms Verify Verbose Balancer Status @@ -111,7 +112,7 @@ Create Multiple Keys ${fileName} = Set Variable file-${INDEX}.txt ${key} = Set Variable /${VOLUME}/${BUCKET}/${fileName} LOG ${fileName} - Create Key ${key} ${file} + Create Key ${key} ${file} --replication=rs-3-2-1024k --type=EC Key Should Match Local File ${key} ${file} END @@ -126,14 +127,14 @@ Get Uuid Close All Containers FOR ${INDEX} IN RANGE 15 - ${container} = Execute ozone admin container list --state OPEN | jq -r 'select(.replicationConfig.replicationFactor == "THREE") | .containerID' | head -1 + ${container} = Execute ozone admin container list --state OPEN | jq -r 'select(.replicationConfig.data == 3) | .containerID' | head -1 EXIT FOR LOOP IF "${container}" == "${EMPTY}" ${message} = Execute And Ignore Error ozone admin container close "${container}" Run Keyword If '${message}' != '${EMPTY}' Should Contain ${message} is in closing state ${output} = Execute ozone admin container info "${container}" Should contain ${output} CLOS END - Wait until keyword succeeds 3min 10sec All container is closed + Wait until keyword succeeds 4min 10sec All container is closed All container is closed ${output} = Execute ozone admin container list --state OPEN diff --git a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancerEC.robot b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancerEC.robot deleted file mode 100644 index ea64c6ba349d..000000000000 --- a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancerEC.robot +++ /dev/null @@ -1,180 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*** Settings *** -Documentation Smoketest ozone cluster startup -Library OperatingSystem -Library Collections -Resource ../commonlib.robot -Resource ../ozone-lib/shell.robot - -Test Timeout 20 minutes - -*** Variables *** -${SECURITY_ENABLED} false -${HOST} datanode1 -${VOLUME} volume1 -${BUCKET} bucket1 -${SIZE} 104857600 - - -** Keywords *** -Prepare For Tests - Execute dd if=/dev/urandom of=/tmp/100mb bs=1048576 count=100 - Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab - Execute ozone sh volume create ${VOLUME} - Execute ozone sh bucket create --replication rs-3-2-1024k --type EC /${VOLUME}/${BUCKET} - - -Datanode In Maintenance Mode - ${result} = Execute ozone admin datanode maintenance ${HOST} - Should Contain ${result} Entering maintenance mode on datanode - ${result} = Execute ozone admin datanode list | grep "Operational State:*" - Wait Until Keyword Succeeds 30sec 5sec Should contain ${result} ENTERING_MAINTENANCE - Wait Until Keyword Succeeds 3min 10sec Related pipelines are closed - Sleep 60000ms - -Related pipelines are closed - ${result} = Execute ozone admin datanode list | awk -v RS= '{$1=$1}1'|grep MAINT | sed -e 's/^.*pipelines: \\(.*\\)$/\\1/' - Should Contain Any ${result} CLOSED No related pipelines or the node is not in Healthy state. - -Datanode Recommission - ${result} = Execute ozone admin datanode recommission ${HOST} - Should Contain ${result} Started recommissioning datanode - Wait Until Keyword Succeeds 1min 10sec Datanode Recommission is Finished - Sleep 300000ms - -Datanode Recommission is Finished - ${result} = Execute ozone admin datanode list | grep "Operational State:*" - Should Not Contain ${result} ENTERING_MAINTENANCE - -Run Container Balancer - ${result} = Execute ozone admin containerbalancer start -t 0.1 -d 100 -i 1 - Should Contain ${result} Container Balancer started successfully. - -Wait Finish Of Balancing - ${result} = Execute ozone admin containerbalancer status -v --history - Should Contain ${result} ContainerBalancer is Running. - Wait Until Keyword Succeeds 10min 10sec ContainerBalancer is Not Running - Sleep 60000ms - -Verify Verbose Balancer Status - [arguments] ${output} - - Should Contain ${output} ContainerBalancer is Running. - Should Contain ${output} Started at: - Should Contain ${output} Container Balancer Configuration values: - -Verify Balancer Iteration - [arguments] ${output} ${number} ${status} ${containers} - - Should Contain ${output} Iteration number ${number} - Should Contain ${output} Iteration result ${status} - Should Contain ${output} Scheduled to move containers ${containers} - -Run Balancer Status - ${result} = Execute ozone admin containerbalancer status - Should Contain ${result} ContainerBalancer is Running. - -Run Balancer Verbose Status - ${result} = Execute ozone admin containerbalancer status -v - Verify Verbose Balancer Status ${result} - Verify Balancer Iteration ${result} 1 IN_PROGRESS 3 - Should Contain ${result} Current iteration info: - -Run Balancer Verbose History Status - ${result} = Execute ozone admin containerbalancer status -v --history - Verify Verbose Balancer Status ${result} - Verify Balancer Iteration ${result} 1 IN_PROGRESS 3 - Should Contain ${result} Iteration history list: - -ContainerBalancer is Not Running - ${result} = Execute ozone admin containerbalancer status - Should contain ${result} ContainerBalancer is Not Running. - -Create Multiple Keys - [arguments] ${NUM_KEYS} - ${file} = Set Variable /tmp/100mb - FOR ${INDEX} IN RANGE ${NUM_KEYS} - ${fileName} = Set Variable file-${INDEX}.txt - ${key} = Set Variable /${VOLUME}/${BUCKET}/${fileName} - LOG ${fileName} - Create Key ${key} ${file} --replication=rs-3-2-1024k --type=EC - Key Should Match Local File ${key} ${file} - END - -Datanode Usageinfo - [arguments] ${uuid} - ${result} = Execute ozone admin datanode usageinfo --uuid=${uuid} - Should Contain ${result} Ozone Used - -Get Uuid - ${result} = Execute ozone admin datanode list | awk -v RS= '{$1=$1}1'| grep ${HOST} | sed -e 's/Datanode: //'|sed -e 's/ .*$//' - [return] ${result} - - -Close All Containers - FOR ${INDEX} IN RANGE 15 - ${container} = Execute ozone admin container list --state OPEN | jq -r 'select(.replicationConfig.data == 3) | .containerID' | head -1 - EXIT FOR LOOP IF "${container}" == "${EMPTY}" - ${message} = Execute And Ignore Error ozone admin container close "${container}" - Run Keyword If '${message}' != '${EMPTY}' Should Contain ${message} is in closing state - ${output} = Execute ozone admin container info "${container}" - Should contain ${output} CLOS - END - Wait until keyword succeeds 3min 10sec All container is closed - -All container is closed - ${output} = Execute ozone admin container list --state OPEN - Should Be Empty ${output} - -Get Datanode Ozone Used Bytes Info - [arguments] ${uuid} - ${output} = Execute export DATANODES=$(ozone admin datanode list --json) && for datanode in $(echo "$\{DATANODES\}" | jq -r '.[].datanodeDetails.uuid'); do ozone admin datanode usageinfo --uuid=$\{datanode\} --json | jq '{(.[0].datanodeDetails.uuid) : .[0].ozoneUsed}'; done | jq -s add - ${result} = Execute echo '${output}' | jq '. | to_entries | .[] | select(.key == "${uuid}") | .value' - [return] ${result} - -** Test Cases *** -Verify Container Balancer for EC containers - Prepare For Tests - - Datanode In Maintenance Mode - - ${uuid} = Get Uuid - Datanode Usageinfo ${uuid} - - Create Multiple Keys 3 - - Close All Containers - - ${datanodeOzoneUsedBytesInfo} = Get Datanode Ozone Used Bytes Info ${uuid} - Should Be True ${datanodeOzoneUsedBytesInfo} < ${SIZE} - - Datanode Recommission - - Run Container Balancer - - Run Balancer Status - - Run Balancer Verbose Status - - Run Balancer Verbose History Status - - Wait Finish Of Balancing - - - ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} = Get Datanode Ozone Used Bytes Info ${uuid} - Should Not Be Equal As Integers ${datanodeOzoneUsedBytesInfo} ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} - Should Be True ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} < ${SIZE} * 2.5 - Should Be True ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} > ${SIZE} * 2 \ No newline at end of file From efd7df15e279e4b7b13ddb2f19b94274ca2e2993 Mon Sep 17 00:00:00 2001 From: daniilchik Date: Fri, 11 Oct 2024 13:37:35 +0300 Subject: [PATCH 3/8] HDDS-11503. Add Robot test to verify Container Balancer for EC containers --- .../dist/src/main/compose/ozone-balancer-ec/test.sh | 2 +- .../dist/src/main/compose/ozone-balancer/test.sh | 2 +- .../src/main/smoketest/balancer/testBalancer.robot | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh index bb370bce0da8..df86fb7c0635 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh @@ -27,6 +27,6 @@ export OZONE_REPLICATION_FACTOR=3 source "$COMPOSE_DIR/../testlib.sh" start_docker_env 6 -execute_robot_test ${OM} -v REPLICATION:rs-3-2-1024k -v REPLICATION_TYPE:EC balancer/testBalancerEC.robot +execute_robot_test ${OM} -v REPLICATION:rs-3-2-1024k -v TYPE:EC -v KEYS:7 -v LOWER_LIMIT:1.5 -v UPPER_LIMIT:2.5 balancer/testBalancerEC.robot diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh index 0e5baaff2f58..3fbbbea95fa7 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh @@ -30,4 +30,4 @@ source "$COMPOSE_DIR/../testlib.sh" # We need 4 dataNodes in this tests start_docker_env 4 -execute_robot_test ${OM} -v REPLICATION:THREE -v REPLICATION_TYPE:RATIS balancer/testBalancer.robot +execute_robot_test ${OM} -v REPLICATION:THREE -v TYPE:RATIS -v KEYS:3 -v LOWER_LIMIT:3 -v UPPER_LIMIT:3.5 balancer/testBalancer.robot diff --git a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot index 6722cf919dac..a56ddd2bea47 100644 --- a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot +++ b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot @@ -36,7 +36,7 @@ Prepare For Tests Execute dd if=/dev/urandom of=/tmp/100mb bs=1048576 count=100 Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab Execute ozone sh volume create /${VOLUME} - Execute ozone sh bucket create --replication rs-3-2-1024k --type EC /${VOLUME}/${BUCKET} + Execute ozone sh bucket create --replication ${REPLICATION} --type ${TYPE} /${VOLUME}/${BUCKET} Datanode In Maintenance Mode @@ -112,7 +112,7 @@ Create Multiple Keys ${fileName} = Set Variable file-${INDEX}.txt ${key} = Set Variable /${VOLUME}/${BUCKET}/${fileName} LOG ${fileName} - Create Key ${key} ${file} --replication=rs-3-2-1024k --type=EC + Create Key ${key} ${file} --replication=${REPLICATION} --type=${TYPE} Key Should Match Local File ${key} ${file} END @@ -147,7 +147,7 @@ Get Datanode Ozone Used Bytes Info [return] ${result} ** Test Cases *** -Verify Container Balancer for RATIS containers +Verify Container Balancer for RATIS/EC containers Prepare For Tests Datanode In Maintenance Mode @@ -155,7 +155,7 @@ Verify Container Balancer for RATIS containers ${uuid} = Get Uuid Datanode Usageinfo ${uuid} - Create Multiple Keys 3 + Create Multiple Keys ${KEYS} Close All Containers @@ -176,8 +176,8 @@ Verify Container Balancer for RATIS containers ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} = Get Datanode Ozone Used Bytes Info ${uuid} Should Not Be Equal As Integers ${datanodeOzoneUsedBytesInfo} ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} - Should Be True ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} < ${SIZE} * 3.5 - Should Be True ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} > ${SIZE} * 3 + Should Be True ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} < ${SIZE} * ${UPPER_LIMIT} + Should Be True ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} > ${SIZE} * ${LOWER_LIMIT} From ff648062af62b485a4fd92626dc218f81147cac3 Mon Sep 17 00:00:00 2001 From: daniilchik Date: Fri, 11 Oct 2024 14:10:36 +0300 Subject: [PATCH 4/8] HDDS-11503. Add Robot test to verify Container Balancer for EC containers --- hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh index df86fb7c0635..fb7ad009261f 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh @@ -27,6 +27,6 @@ export OZONE_REPLICATION_FACTOR=3 source "$COMPOSE_DIR/../testlib.sh" start_docker_env 6 -execute_robot_test ${OM} -v REPLICATION:rs-3-2-1024k -v TYPE:EC -v KEYS:7 -v LOWER_LIMIT:1.5 -v UPPER_LIMIT:2.5 balancer/testBalancerEC.robot +execute_robot_test ${OM} -v REPLICATION:rs-3-2-1024k -v TYPE:EC -v KEYS:7 -v LOWER_LIMIT:1.5 -v UPPER_LIMIT:2.5 balancer/testBalancer.robot From 91e84c964965c6c19d9401d39f9b06c7755d358b Mon Sep 17 00:00:00 2001 From: daniilchik Date: Tue, 15 Oct 2024 17:44:06 +0300 Subject: [PATCH 5/8] HDDS-11503. Add Robot test to verify Container Balancer for EC containers. --- .../src/main/compose/ozone-balancer-ec/.env | 20 -- .../ozone-balancer-ec/docker-compose.yaml | 211 ------------------ .../compose/ozone-balancer-ec/docker-config | 60 ----- .../ozone-balancer/docker-compose.yaml | 46 ++++ .../src/main/compose/ozone-balancer/test.sh | 3 + .../test.sh => ozone-balancer/test_ec.sh} | 2 - 6 files changed, 49 insertions(+), 293 deletions(-) delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/.env delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-compose.yaml delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-config rename hadoop-ozone/dist/src/main/compose/{ozone-balancer-ec/test.sh => ozone-balancer/test_ec.sh} (99%) diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/.env b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/.env deleted file mode 100644 index 0e99fab82fd0..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/.env +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HDDS_VERSION=${hdds.version} -OZONE_RUNNER_VERSION=${docker.ozone-runner.version} -OZONE_RUNNER_IMAGE=apache/ozone-runner -OZONE_OPTS= \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-compose.yaml deleted file mode 100644 index eeaf1a794649..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-compose.yaml +++ /dev/null @@ -1,211 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# reusable fragments (see https://docs.docker.com/compose/compose-file/#extension-fields) -x-common-config: - &common-config - image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - env_file: - - docker-config - -x-replication: - &replication - OZONE-SITE.XML_ozone.server.default.replication: ${OZONE_REPLICATION_FACTOR:-3} - -services: - datanode1: - <<: *common-config - ports: - - 19864 - - 9882 - environment: - <<: *replication - command: ["ozone","datanode"] - volumes: - - tmpfs1:/data - - ../..:/opt/hadoop - datanode2: - <<: *common-config - ports: - - 19864 - - 9882 - environment: - <<: *replication - command: [ "ozone","datanode" ] - volumes: - - tmpfs2:/data - - ../..:/opt/hadoop - datanode3: - <<: *common-config - ports: - - 19864 - - 9882 - environment: - <<: *replication - command: [ "ozone","datanode" ] - volumes: - - tmpfs3:/data - - ../..:/opt/hadoop - datanode4: - <<: *common-config - ports: - - 19864 - - 9882 - environment: - <<: *replication - command: [ "ozone","datanode" ] - volumes: - - tmpfs4:/data - - ../..:/opt/hadoop - datanode5: - <<: *common-config - ports: - - 19864 - - 9882 - environment: - <<: *replication - command: [ "ozone","datanode" ] - volumes: - - tmpfs5:/data - - ../..:/opt/hadoop - datanode6: - <<: *common-config - ports: - - 19864 - - 9882 - environment: - <<: *replication - command: [ "ozone","datanode" ] - volumes: - - tmpfs6:/data - - ../..:/opt/hadoop - om1: - <<: *common-config - environment: - WAITFOR: scm3:9894 - ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - <<: *replication - ports: - - 9874:9874 - - 9862 - hostname: om1 - command: ["ozone","om"] - om2: - <<: *common-config - environment: - WAITFOR: scm3:9894 - ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - <<: *replication - ports: - - 9874 - - 9862 - hostname: om2 - command: ["ozone","om"] - om3: - <<: *common-config - environment: - WAITFOR: scm3:9894 - ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - <<: *replication - ports: - - 9874 - - 9862 - hostname: om3 - command: ["ozone","om"] - scm1: - <<: *common-config - ports: - - 9876:9876 - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1} - <<: *replication - command: ["ozone","scm"] - scm2: - <<: *common-config - ports: - - 9876 - environment: - WAITFOR: scm1:9894 - ENSURE_SCM_BOOTSTRAPPED: /data/metadata/scm/current/VERSION - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1} - <<: *replication - command: ["ozone","scm"] - scm3: - <<: *common-config - ports: - - 9876 - environment: - WAITFOR: scm2:9894 - ENSURE_SCM_BOOTSTRAPPED: /data/metadata/scm/current/VERSION - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1} - <<: *replication - command: ["ozone","scm"] - httpfs: - <<: *common-config - environment: - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1} - <<: *replication - ports: - - 14000:14000 - command: [ "ozone","httpfs" ] - s3g: - <<: *common-config - environment: - OZONE_OPTS: - <<: *replication - ports: - - 9878:9878 - command: ["ozone","s3g"] -volumes: - tmpfs1: - driver: local - driver_opts: - o: "size=1g,uid=1000" - device: tmpfs - type: tmpfs - tmpfs2: - driver: local - driver_opts: - o: "size=1g,uid=2000" - device: tmpfs - type: tmpfs - tmpfs3: - driver: local - driver_opts: - o: "size=1g,uid=3000" - device: tmpfs - type: tmpfs - tmpfs4: - driver: local - driver_opts: - o: "size=1g,uid=4000" - device: tmpfs - type: tmpfs - tmpfs5: - driver: local - driver_opts: - o: "size=1g,uid=5000" - device: tmpfs - type: tmpfs - tmpfs6: - driver: local - driver_opts: - o: "size=1g,uid=6000" - device: tmpfs - type: tmpfs diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-config deleted file mode 100644 index d0e0ab7dcfb5..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/docker-config +++ /dev/null @@ -1,60 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# For HttpFS service it is required to enable proxying users. -CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts=* -CORE-SITE.XML_hadoop.proxyuser.hadoop.groups=* - -CORE-SITE.XML_fs.defaultFS=ofs://om/ -CORE-SITE.XML_fs.trash.interval=1 - -OZONE-SITE.XML_ozone.om.service.ids=om -OZONE-SITE.XML_ozone.om.nodes.om=om1,om2,om3 -OZONE-SITE.XML_ozone.om.address.om.om1=om1 -OZONE-SITE.XML_ozone.om.address.om.om2=om2 -OZONE-SITE.XML_ozone.om.address.om.om3=om3 - -OZONE-SITE.XML_ozone.scm.service.ids=scmservice -OZONE-SITE.XML_ozone.scm.nodes.scmservice=scm1,scm2,scm3 -OZONE-SITE.XML_ozone.scm.address.scmservice.scm1=scm1 -OZONE-SITE.XML_ozone.scm.address.scmservice.scm2=scm2 -OZONE-SITE.XML_ozone.scm.address.scmservice.scm3=scm3 -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data -OZONE-SITE.XML_ozone.scm.container.size=100MB -OZONE-SITE.XML_ozone.scm.block.size=20MB -OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB -OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_hdds.node.report.interval=20s -OZONE-SITE.XML_hdds.heartbeat.interval=20s -OZONE-SITE.XML_hdds.datanode.du.refresh.period=20s -OZONE-SITE.XML_hdds.container.balancer.balancing.iteration.interval=2m -OZONE-SITE.XML_hdds.datanode.dir=/data/hdds -OZONE-SITE.XML_hdds.datanode.container.db.dir=/data/metadata -OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB -OZONE-SITE.XML_ozone.scm.pipeline.creation.auto.factor.one=false -OZONE-SITE.XML_ozone.datanode.pipeline.limit=1 -OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s -OZONE-SITE.XML_ozone.scm.primordial.node.id=scm1 -OZONE-SITE.XML_hdds.container.report.interval=30s -OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true -OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon -OZONE-SITE.XML_dfs.container.ratis.datastream.enabled=true -OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http - -OZONE_CONF_DIR=/etc/hadoop -OZONE_LOG_DIR=/var/log/hadoop - -no_proxy=om1,om2,om3,scm,s3g,recon,kdc,localhost,127.0.0.1 diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-balancer/docker-compose.yaml index 5220d71669df..e2d7272b0300 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-balancer/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer/docker-compose.yaml @@ -39,6 +39,8 @@ services: volumes: - tmpfs1:/data - ../..:/opt/hadoop + deploy: + replicas: ${DATANODE1_REPLICA:-1} datanode2: <<: *common-config ports: @@ -50,6 +52,8 @@ services: volumes: - tmpfs2:/data - ../..:/opt/hadoop + deploy: + replicas: ${DATANODE2_REPLICA:-1} datanode3: <<: *common-config ports: @@ -61,6 +65,8 @@ services: volumes: - tmpfs3:/data - ../..:/opt/hadoop + deploy: + replicas: ${DATANODE3_REPLICA:-1} datanode4: <<: *common-config ports: @@ -72,6 +78,34 @@ services: volumes: - tmpfs4:/data - ../..:/opt/hadoop + deploy: + replicas: ${DATANODE4_REPLICA:-1} + datanode5: + <<: *common-config + ports: + - 19864 + - 9882 + environment: + <<: *replication + command: [ "ozone","datanode" ] + volumes: + - tmpfs5:/data + - ../..:/opt/hadoop + deploy: + replicas: ${DATANODE5_REPLICA:-1} + datanode6: + <<: *common-config + ports: + - 19864 + - 9882 + environment: + <<: *replication + command: [ "ozone","datanode" ] + volumes: + - tmpfs6:/data + - ../..:/opt/hadoop + deploy: + replicas: ${DATANODE6_REPLICA:-1} om1: <<: *common-config environment: @@ -175,3 +209,15 @@ volumes: o: "size=1g,uid=4000" device: tmpfs type: tmpfs + tmpfs5: + driver: local + driver_opts: + o: "size=1g,uid=5000" + device: tmpfs + type: tmpfs + tmpfs6: + driver: local + driver_opts: + o: "size=1g,uid=6000" + device: tmpfs + type: tmpfs \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh index 3fbbbea95fa7..1619f5f34367 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh @@ -24,6 +24,9 @@ export OM=om1 export SCM=scm1 export OZONE_REPLICATION_FACTOR=3 +export DATANODE2_REPLICA=0 +export DATANODE5_REPLICA=0 + # shellcheck source=/dev/null source "$COMPOSE_DIR/../testlib.sh" diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-balancer/test_ec.sh similarity index 99% rename from hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh rename to hadoop-ozone/dist/src/main/compose/ozone-balancer/test_ec.sh index fb7ad009261f..d43dfcb42b91 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-balancer-ec/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer/test_ec.sh @@ -28,5 +28,3 @@ source "$COMPOSE_DIR/../testlib.sh" start_docker_env 6 execute_robot_test ${OM} -v REPLICATION:rs-3-2-1024k -v TYPE:EC -v KEYS:7 -v LOWER_LIMIT:1.5 -v UPPER_LIMIT:2.5 balancer/testBalancer.robot - - From f5130118a37ad8b1a8783969ada8957c58440264 Mon Sep 17 00:00:00 2001 From: daniilchik Date: Wed, 16 Oct 2024 13:44:46 +0300 Subject: [PATCH 6/8] HDDS-11503. Add Robot test to verify Container Balancer for EC containers --- hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh | 3 +-- hadoop-ozone/dist/src/main/compose/ozone-balancer/test_ec.sh | 3 ++- .../dist/src/main/smoketest/balancer/testBalancer.robot | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh index 1619f5f34367..34ffb9e02cd5 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh @@ -30,7 +30,6 @@ export DATANODE5_REPLICA=0 # shellcheck source=/dev/null source "$COMPOSE_DIR/../testlib.sh" -# We need 4 dataNodes in this tests -start_docker_env 4 +start_docker_env execute_robot_test ${OM} -v REPLICATION:THREE -v TYPE:RATIS -v KEYS:3 -v LOWER_LIMIT:3 -v UPPER_LIMIT:3.5 balancer/testBalancer.robot diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer/test_ec.sh b/hadoop-ozone/dist/src/main/compose/ozone-balancer/test_ec.sh index d43dfcb42b91..fd8fe1c8bea4 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-balancer/test_ec.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer/test_ec.sh @@ -24,7 +24,8 @@ export OM=om1 export SCM=scm1 export OZONE_REPLICATION_FACTOR=3 +# shellcheck source=/dev/null source "$COMPOSE_DIR/../testlib.sh" -start_docker_env 6 +start_docker_env execute_robot_test ${OM} -v REPLICATION:rs-3-2-1024k -v TYPE:EC -v KEYS:7 -v LOWER_LIMIT:1.5 -v UPPER_LIMIT:2.5 balancer/testBalancer.robot diff --git a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot index a56ddd2bea47..06d8a3416f00 100644 --- a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot +++ b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot @@ -68,7 +68,7 @@ Run Container Balancer Wait Finish Of Balancing ${result} = Execute ozone admin containerbalancer status Should Contain ${result} ContainerBalancer is Running. - Wait Until Keyword Succeeds 10min 10sec ContainerBalancer is Not Running + Wait Until Keyword Succeeds 6min 10sec ContainerBalancer is Not Running Sleep 60000ms Verify Verbose Balancer Status @@ -176,6 +176,8 @@ Verify Container Balancer for RATIS/EC containers ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} = Get Datanode Ozone Used Bytes Info ${uuid} Should Not Be Equal As Integers ${datanodeOzoneUsedBytesInfo} ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} + #We need to ensure that after balancing, the amount of data recorded on each datanode falls within the following ranges: + #{SIZE}*3 < used < {SIZE}*3.5 for RATIS containers, and {SIZE}*1.5 < used < {SIZE}*2.5 for EC containers. Should Be True ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} < ${SIZE} * ${UPPER_LIMIT} Should Be True ${datanodeOzoneUsedBytesInfoAfterContainerBalancing} > ${SIZE} * ${LOWER_LIMIT} From 9a80e03a77af9accdb67eceb6a2f11280e6c8ea2 Mon Sep 17 00:00:00 2001 From: daniilchik Date: Wed, 16 Oct 2024 16:27:28 +0300 Subject: [PATCH 7/8] HDDS-11503. Add Robot test to verify Container Balancer for EC containers --- .../src/main/compose/ozone-balancer/{test_ec.sh => test-ec.sh} | 2 +- .../src/main/compose/ozone-balancer/{test.sh => test-ratish.sh} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename hadoop-ozone/dist/src/main/compose/ozone-balancer/{test_ec.sh => test-ec.sh} (92%) rename hadoop-ozone/dist/src/main/compose/ozone-balancer/{test.sh => test-ratish.sh} (92%) diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer/test_ec.sh b/hadoop-ozone/dist/src/main/compose/ozone-balancer/test-ec.sh similarity index 92% rename from hadoop-ozone/dist/src/main/compose/ozone-balancer/test_ec.sh rename to hadoop-ozone/dist/src/main/compose/ozone-balancer/test-ec.sh index fd8fe1c8bea4..7365d104fab1 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-balancer/test_ec.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer/test-ec.sh @@ -28,4 +28,4 @@ export OZONE_REPLICATION_FACTOR=3 source "$COMPOSE_DIR/../testlib.sh" start_docker_env -execute_robot_test ${OM} -v REPLICATION:rs-3-2-1024k -v TYPE:EC -v KEYS:7 -v LOWER_LIMIT:1.5 -v UPPER_LIMIT:2.5 balancer/testBalancer.robot +execute_robot_test ${OM} -v REPLICATION:rs-3-2-1024k -v TYPE:EC -v KEYS:7 -v LOWER_LIMIT:1.5 -v UPPER_LIMIT:2.5 -N ozone-balancer-EC balancer/testBalancer.robot diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-balancer/test-ratish.sh similarity index 92% rename from hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh rename to hadoop-ozone/dist/src/main/compose/ozone-balancer/test-ratish.sh index 34ffb9e02cd5..a358aea79ad2 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-balancer/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-balancer/test-ratish.sh @@ -32,4 +32,4 @@ source "$COMPOSE_DIR/../testlib.sh" start_docker_env -execute_robot_test ${OM} -v REPLICATION:THREE -v TYPE:RATIS -v KEYS:3 -v LOWER_LIMIT:3 -v UPPER_LIMIT:3.5 balancer/testBalancer.robot +execute_robot_test ${OM} -v REPLICATION:THREE -v TYPE:RATIS -v KEYS:3 -v LOWER_LIMIT:3 -v UPPER_LIMIT:3.5 -N ozone-balancer-RATIS balancer/testBalancer.robot From a171ab83a1c4c3c0c635d028e8d5e316c877a42d Mon Sep 17 00:00:00 2001 From: daniilchik Date: Wed, 16 Oct 2024 18:52:04 +0300 Subject: [PATCH 8/8] HDDS-11503. Add Robot test to verify Container Balancer for EC containers --- .../main/compose/ozone-balancer/{test-ratish.sh => test-ratis.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename hadoop-ozone/dist/src/main/compose/ozone-balancer/{test-ratish.sh => test-ratis.sh} (100%) diff --git a/hadoop-ozone/dist/src/main/compose/ozone-balancer/test-ratish.sh b/hadoop-ozone/dist/src/main/compose/ozone-balancer/test-ratis.sh similarity index 100% rename from hadoop-ozone/dist/src/main/compose/ozone-balancer/test-ratish.sh rename to hadoop-ozone/dist/src/main/compose/ozone-balancer/test-ratis.sh