From abd7e613e80d3698df019873b35dd707f880eb0c Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 20 Mar 2023 08:21:15 +0100 Subject: [PATCH 1/4] HDDS-7975. Acceptance tests need rebalancing --- .github/workflows/ci.yml | 3 ++- .../dist/src/main/compose/ozone-ha/docker-config | 2 ++ .../dist/src/main/compose/ozone-ha/test.sh | 16 +++++++++++++--- hadoop-ozone/dist/src/main/compose/ozone/test.sh | 10 ---------- .../main/compose/ozonesecure-ha/docker-config | 5 +++++ .../dist/src/main/compose/ozonesecure-ha/test.sh | 16 ++++++++++++++-- .../dist/src/main/compose/ozonesecure/test.sh | 10 ---------- 7 files changed, 36 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f540c1df77a3..2697a6e3f8ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -248,7 +248,8 @@ jobs: - secure - unsecure - compat - - HA + - HA-secure + - HA-unsecure - MR - misc fail-fast: false diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config index 6fb4003b6b3c..bfe0c03f77cf 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config @@ -19,6 +19,7 @@ CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts=* CORE-SITE.XML_hadoop.proxyuser.hadoop.groups=* CORE-SITE.XML_fs.defaultFS=ofs://omservice/ +CORE-SITE.XML_fs.trash.interval=1 OZONE-SITE.XML_ozone.om.service.ids=omservice OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3 @@ -47,6 +48,7 @@ OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon OZONE-SITE.XML_ozone.recon.address=recon:9891 OZONE-SITE.XML_ozone.recon.http-address=0.0.0.0:9888 OZONE-SITE.XML_ozone.recon.https-address=0.0.0.0:9889 +OZONE-SITE.XML_dfs.container.ratis.datastream.enabled=true OZONE_CONF_DIR=/etc/hadoop OZONE_LOG_DIR=/var/log/hadoop diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh index 867c6923f5dd..7d108ba5f43d 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -#suite:HA +#suite:HA-unsecure COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR @@ -28,11 +28,21 @@ export OM_SERVICE_ID=omservice # shellcheck source=/dev/null source "$COMPOSE_DIR/../testlib.sh" -start_docker_env +start_docker_env 5 execute_robot_test ${SCM} basic/ozone-shell-single.robot execute_robot_test ${SCM} basic/links.robot -execute_robot_test ${SCM} s3 + +execute_robot_test ${SCM} -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-ofs-link ozonefs/ozonefs.robot +execute_robot_test ${SCM} -v SCHEME:o3fs -v BUCKET_TYPE:bucket -N ozonefs-o3fs-bucket ozonefs/ozonefs.robot + +exclude="" +for bucket in erasure link generated; do + execute_robot_test ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 + # some tests are independent of the bucket type, only need to be run once + exclude="--exclude no-bucket-type" +done + execute_robot_test ${SCM} freon execute_robot_test ${SCM} -v USERNAME:httpfs httpfs diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/test.sh index b61c6277c9f2..290056903d72 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh @@ -39,13 +39,6 @@ execute_robot_test scm gdpr execute_robot_test scm security/ozone-secure-token.robot -exclude="" -for bucket in erasure link generated; do - execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 - # some tests are independent of the bucket type, only need to be run once - exclude="--exclude no-bucket-type" -done - execute_robot_test scm recon execute_robot_test scm om-ratis @@ -58,9 +51,6 @@ execute_robot_test scm admincli execute_robot_test scm -v USERNAME:httpfs httpfs execute_debug_tests -execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-ofs-link ozonefs/ozonefs.robot -execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:bucket -N ozonefs-o3fs-bucket ozonefs/ozonefs.robot - prefix=${RANDOM} execute_robot_test scm -v PREFIX:${prefix} ec/basic.robot docker-compose up -d --no-recreate --scale datanode=4 diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config index ed82642df851..dce3cf4460c2 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config @@ -19,6 +19,7 @@ CORE-SITE.XML_hadoop.proxyuser.httpfs.hosts=* CORE-SITE.XML_hadoop.proxyuser.httpfs.groups=* CORE-SITE.XML_fs.defaultFS=ofs://id1 +CORE-SITE.XML_fs.trash.interval=1 OZONE-SITE.XML_ozone.om.service.ids=id1 OZONE-SITE.XML_ozone.om.internal.service.id=id1 @@ -56,6 +57,7 @@ OZONE-SITE.XML_ozone.replication=3 OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s OZONE-SITE.XML_hdds.container.report.interval=60s OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true +OZONE-SITE.XML_dfs.container.ratis.datastream.enabled=true OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon @@ -142,6 +144,9 @@ HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/httpfs@EXAMPL HTTPFS-SITE.XML_httpfs.hadoop.authentication.type=kerberos HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.keytab=/etc/security/keytabs/httpfs.keytab HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.principal=httpfs/httpfs@EXAMPLE.COM +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users=* +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups=* +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts=* #Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. #BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh index dcd16a802b68..0193463efc91 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -#suite:HA +#suite:HA-secure COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR @@ -24,18 +24,30 @@ export SECURITY_ENABLED=true export OM_SERVICE_ID="id1" export SCM=scm1.org +: ${OZONE_BUCKET_KEY_NAME:=key1} + # shellcheck source=/dev/null source "$COMPOSE_DIR/../testlib.sh" start_docker_env +execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME} + execute_robot_test ${SCM} kinit.robot execute_robot_test ${SCM} freon +execute_robot_test ${SCM} -v SCHEME:ofs -v BUCKET_TYPE:bucket -N ozonefs-ofs-bucket ozonefs/ozonefs.robot +execute_robot_test ${SCM} -v SCHEME:o3fs -v BUCKET_TYPE:link -N ozonefs-o3fs-link ozonefs/ozonefs.robot + execute_robot_test ${SCM} basic/links.robot -execute_robot_test ${SCM} s3 +exclude="" +for bucket in encrypted link generated; do + execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 + # some tests are independent of the bucket type, only need to be run once + exclude="--exclude no-bucket-type" +done execute_robot_test ${SCM} admincli diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh index df96734ed97c..b367c32a02cd 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh @@ -37,16 +37,6 @@ execute_robot_test scm basic execute_robot_test scm security -execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:bucket -N ozonefs-ofs-bucket ozonefs/ozonefs.robot -execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:link -N ozonefs-o3fs-link ozonefs/ozonefs.robot - -exclude="" -for bucket in encrypted link generated; do - execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 - # some tests are independent of the bucket type, only need to be run once - exclude="--exclude no-bucket-type" -done - #expects 4 pipelines, should be run before #admincli which creates STANDALONE pipeline execute_robot_test scm recon From a36a25b7e12aa475ec3550fd45fdc305d0bfdde8 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 20 Mar 2023 09:20:28 +0100 Subject: [PATCH 2/4] HDDS-8219. Run HA secure tests from s3g container --- .../src/main/compose/ozonesecure-ha/test.sh | 17 +++++++++-------- .../dist/src/main/smoketest/basic/links.robot | 11 ++++++----- .../dist/src/main/smoketest/commonlib.robot | 15 ++++++++++----- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh index 0193463efc91..a47b5061ec3a 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh @@ -33,14 +33,14 @@ start_docker_env execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME} -execute_robot_test ${SCM} kinit.robot +execute_robot_test s3g kinit.robot -execute_robot_test ${SCM} freon +execute_robot_test s3g freon -execute_robot_test ${SCM} -v SCHEME:ofs -v BUCKET_TYPE:bucket -N ozonefs-ofs-bucket ozonefs/ozonefs.robot -execute_robot_test ${SCM} -v SCHEME:o3fs -v BUCKET_TYPE:link -N ozonefs-o3fs-link ozonefs/ozonefs.robot +execute_robot_test s3g -v SCHEME:ofs -v BUCKET_TYPE:bucket -N ozonefs-ofs-bucket ozonefs/ozonefs.robot +execute_robot_test s3g -v SCHEME:o3fs -v BUCKET_TYPE:link -N ozonefs-o3fs-link ozonefs/ozonefs.robot -execute_robot_test ${SCM} basic/links.robot +execute_robot_test s3g basic/links.robot exclude="" for bucket in encrypted link generated; do @@ -49,12 +49,13 @@ for bucket in encrypted link generated; do exclude="--exclude no-bucket-type" done -execute_robot_test ${SCM} admincli +execute_robot_test s3g admincli -execute_robot_test ${SCM} httpfs +execute_robot_test s3g httpfs export SCM=scm2.org -execute_robot_test ${SCM} admincli +execute_robot_test s3g admincli + stop_docker_env generate_report diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/links.robot b/hadoop-ozone/dist/src/main/smoketest/basic/links.robot index 1057a7536574..dc46e76e6d4c 100644 --- a/hadoop-ozone/dist/src/main/smoketest/basic/links.robot +++ b/hadoop-ozone/dist/src/main/smoketest/basic/links.robot @@ -42,11 +42,12 @@ Setup ACL tests Execute ozone sh bucket link ${source}/readable-bucket ${target}/readable-link Execute ozone sh bucket link ${source}/readable-bucket ${target}/unreadable-link Execute ozone sh bucket link ${source}/unreadable-bucket ${target}/link-to-unreadable-bucket - Execute ozone sh volume addacl --acl user:testuser2/scm@EXAMPLE.COM:r ${target} - Execute ozone sh volume addacl --acl user:testuser2/scm@EXAMPLE.COM:rl ${source} - Execute ozone sh bucket addacl --acl user:testuser2/scm@EXAMPLE.COM:rl ${source}/readable-bucket - Execute ozone sh bucket addacl --acl user:testuser2/scm@EXAMPLE.COM:r ${target}/readable-link - Execute ozone sh bucket addacl --acl user:testuser2/scm@EXAMPLE.COM:r ${target}/link-to-unreadable-bucket + ${principal} = Get test user principal testuser2 + Execute ozone sh volume addacl --acl user:${principal}:r ${target} + Execute ozone sh volume addacl --acl user:${principal}:rl ${source} + Execute ozone sh bucket addacl --acl user:${principal}:rl ${source}/readable-bucket + Execute ozone sh bucket addacl --acl user:${principal}:r ${target}/readable-link + Execute ozone sh bucket addacl --acl user:${principal}:r ${target}/link-to-unreadable-bucket Can follow link with read access Execute kdestroy diff --git a/hadoop-ozone/dist/src/main/smoketest/commonlib.robot b/hadoop-ozone/dist/src/main/smoketest/commonlib.robot index 41a1e284677d..62ea760efcfd 100644 --- a/hadoop-ozone/dist/src/main/smoketest/commonlib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/commonlib.robot @@ -26,15 +26,20 @@ ${OM_HA_PARAM} ${EMPTY} ${OM_SERVICE_ID} om *** Keywords *** +Get test user principal + [arguments] ${user} + ${instance} = Execute hostname | sed 's/scm[0-9].org/scm/' + [return] ${user}/${instance}@EXAMPLE.COM + Kinit HTTP user - ${hostname} = Execute hostname | sed 's/scm[0-9].org/scm/' - Wait Until Keyword Succeeds 2min 10sec Execute kinit -k HTTP/${hostname}@EXAMPLE.COM -t /etc/security/keytabs/HTTP.keytab + ${principal} = Get test user principal HTTP + Wait Until Keyword Succeeds 2min 10sec Execute kinit -k -t /etc/security/keytabs/HTTP.keytab ${principal} Kinit test user [arguments] ${user} ${keytab} - ${instance} = Execute hostname | sed 's/scm[0-9].org/scm/' - Set Suite Variable ${TEST_USER} ${user}/${instance}@EXAMPLE.COM - Wait Until Keyword Succeeds 2min 10sec Execute kinit -k ${user}/${instance}@EXAMPLE.COM -t /etc/security/keytabs/${keytab} + ${TEST_USER} = Get test user principal ${user} + Set Suite Variable ${TEST_USER} + Wait Until Keyword Succeeds 2min 10sec Execute kinit -k -t /etc/security/keytabs/${keytab} ${TEST_USER} Access should be denied [arguments] ${command} From b9871f1062c2124833459bf5fbd29e5da4534a5e Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 20 Mar 2023 12:52:23 +0100 Subject: [PATCH 3/4] Distribute FS and S3 tests --- hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh | 3 +-- hadoop-ozone/dist/src/main/compose/ozone/test.sh | 9 +++++++++ .../dist/src/main/compose/ozonesecure-ha/test.sh | 3 +-- hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh | 9 +++++++++ 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh index 7d108ba5f43d..2c2834cc3ba5 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh @@ -34,10 +34,9 @@ execute_robot_test ${SCM} basic/ozone-shell-single.robot execute_robot_test ${SCM} basic/links.robot execute_robot_test ${SCM} -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-ofs-link ozonefs/ozonefs.robot -execute_robot_test ${SCM} -v SCHEME:o3fs -v BUCKET_TYPE:bucket -N ozonefs-o3fs-bucket ozonefs/ozonefs.robot exclude="" -for bucket in erasure link generated; do +for bucket in generated; do execute_robot_test ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 # some tests are independent of the bucket type, only need to be run once exclude="--exclude no-bucket-type" diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/test.sh index 290056903d72..6c84c3054358 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh @@ -39,6 +39,13 @@ execute_robot_test scm gdpr execute_robot_test scm security/ozone-secure-token.robot +exclude="" +for bucket in erasure; do + execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 + # some tests are independent of the bucket type, only need to be run once + exclude="--exclude no-bucket-type" +done + execute_robot_test scm recon execute_robot_test scm om-ratis @@ -51,6 +58,8 @@ execute_robot_test scm admincli execute_robot_test scm -v USERNAME:httpfs httpfs execute_debug_tests +execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:bucket -N ozonefs-o3fs-bucket ozonefs/ozonefs.robot + prefix=${RANDOM} execute_robot_test scm -v PREFIX:${prefix} ec/basic.robot docker-compose up -d --no-recreate --scale datanode=4 diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh index 0193463efc91..a6d5abfaae8a 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh @@ -37,13 +37,12 @@ execute_robot_test ${SCM} kinit.robot execute_robot_test ${SCM} freon -execute_robot_test ${SCM} -v SCHEME:ofs -v BUCKET_TYPE:bucket -N ozonefs-ofs-bucket ozonefs/ozonefs.robot execute_robot_test ${SCM} -v SCHEME:o3fs -v BUCKET_TYPE:link -N ozonefs-o3fs-link ozonefs/ozonefs.robot execute_robot_test ${SCM} basic/links.robot exclude="" -for bucket in encrypted link generated; do +for bucket in encrypted; do execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 # some tests are independent of the bucket type, only need to be run once exclude="--exclude no-bucket-type" diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh index b367c32a02cd..4b2f45a3de35 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh @@ -37,6 +37,15 @@ execute_robot_test scm basic execute_robot_test scm security +execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:bucket -N ozonefs-ofs-bucket ozonefs/ozonefs.robot + +exclude="" +for bucket in link; do + execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 + # some tests are independent of the bucket type, only need to be run once + exclude="--exclude no-bucket-type" +done + #expects 4 pipelines, should be run before #admincli which creates STANDALONE pipeline execute_robot_test scm recon From 1f22831d1185e47e6ef982c31755b42fd823063d Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 20 Mar 2023 14:54:46 +0100 Subject: [PATCH 4/4] tweak --- hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh | 2 +- hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh index a6d5abfaae8a..c87e6dce23e6 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh @@ -42,7 +42,7 @@ execute_robot_test ${SCM} -v SCHEME:o3fs -v BUCKET_TYPE:link -N ozonefs-o3fs-lin execute_robot_test ${SCM} basic/links.robot exclude="" -for bucket in encrypted; do +for bucket in encrypted link; do execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 # some tests are independent of the bucket type, only need to be run once exclude="--exclude no-bucket-type" diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh index 4b2f45a3de35..a1e31830a151 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh @@ -39,13 +39,6 @@ execute_robot_test scm security execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:bucket -N ozonefs-ofs-bucket ozonefs/ozonefs.robot -exclude="" -for bucket in link; do - execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 - # some tests are independent of the bucket type, only need to be run once - exclude="--exclude no-bucket-type" -done - #expects 4 pipelines, should be run before #admincli which creates STANDALONE pipeline execute_robot_test scm recon