From 4e645cd136f2c7faa2b3bbc65c39d99b6cd2aefa Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Fri, 29 Mar 2024 12:57:32 +0100
Subject: [PATCH 1/8] HDDS-8450. Dedicated acceptance test suite for s3a

---
 hadoop-ozone/dev-support/checks/_lib.sh       |  9 ++++
 .../dev-support/checks/_mvn_unit_report.sh    |  4 +-
 hadoop-ozone/dev-support/checks/acceptance.sh | 45 ++++++++++++++----
 .../dist/src/main/compose/common/s3a-test.sh  | 41 +++++++++++++++++
 .../dist/src/main/compose/ozone/test-s3a.sh   | 25 ++++++++++
 .../dist/src/main/compose/test-all.sh         |  5 +-
 hadoop-ozone/dist/src/main/compose/testlib.sh | 36 ++++++++++++++-
 .../dist/src/test/resources/auth-keys.xml     | 46 +++++++++++++++++++
 8 files changed, 196 insertions(+), 15 deletions(-)
 create mode 100644 hadoop-ozone/dist/src/main/compose/common/s3a-test.sh
 create mode 100644 hadoop-ozone/dist/src/main/compose/ozone/test-s3a.sh
 create mode 100644 hadoop-ozone/dist/src/test/resources/auth-keys.xml

diff --git a/hadoop-ozone/dev-support/checks/_lib.sh b/hadoop-ozone/dev-support/checks/_lib.sh
index b81acf989930..7aafd38e52e3 100644
--- a/hadoop-ozone/dev-support/checks/_lib.sh
+++ b/hadoop-ozone/dev-support/checks/_lib.sh
@@ -149,3 +149,12 @@ install_spotbugs() {
 _install_spotbugs() {
   curl -LSs https://repo.maven.apache.org/maven2/com/github/spotbugs/spotbugs/3.1.12/spotbugs-3.1.12.tgz | tar -xz -f -
 }
+
+download_hadoop_aws() {
+  local dir="$1"
+  if [[ -z ${dir} ]] || [[ ! -e ${dir} ]] || [[ ! -d ${dir}/src/test/resources ]]; then
+    mkdir -p ${dir}
+    [[ -f ${dir}.tar.gz ]] || curl -LSs -o ${dir}.tar.gz https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}-src.tar.gz
+    tar -x -z -C ${dir} --strip-components=3 -f ${dir}.tar.gz --wildcards 'hadoop-*-src/hadoop-tools/hadoop-aws' || exit 1
+  fi
+}
diff --git a/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh b/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
index 4fca7bb6aaee..36205c69bb64 100755
--- a/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
+++ b/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
@@ -81,8 +81,8 @@ for failed_test in $(< ${REPORT_DIR}/summary.txt); do
       \( -name "${failed_test}.txt" -or -name "${failed_test}-output.txt" -or -name "TEST-${failed_test}.xml" \)); do
     dir=$(dirname "${file}")
     dest_dir=$(_realpath --relative-to="${PWD}" "${dir}/../..") || continue
-    mkdir -p "${REPORT_DIR}/${dest_dir}"
-    mv "${file}" "${REPORT_DIR}/${dest_dir}"/
+    mkdir -pv "${REPORT_DIR}/${dest_dir}"
+    mv -v "${file}" "${REPORT_DIR}/${dest_dir}"/
   done
 done
 
diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh b/hadoop-ozone/dev-support/checks/acceptance.sh
index 0489fa24384a..ec1122d6db5a 100755
--- a/hadoop-ozone/dev-support/checks/acceptance.sh
+++ b/hadoop-ozone/dev-support/checks/acceptance.sh
@@ -19,15 +19,19 @@ set -u -o pipefail
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "$DIR/../../.." || exit 1
 
-source "${DIR}/_lib.sh"
+OZONE_ROOT=$(pwd -P)
+
+: ${HADOOP_AWS_DIR:=""}
+: ${OZONE_ACCEPTANCE_SUITE:=""}
+: ${OZONE_TEST_SELECTOR:=""}
+: ${OZONE_WITH_COVERAGE:="false"}
 
-install_virtualenv
-install_robot
+source "${DIR}/_lib.sh"
 
-REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/acceptance"}
+REPORT_DIR=${OUTPUT_DIR:-"${OZONE_ROOT}/target/acceptance"}
 
 OZONE_VERSION=$(mvn help:evaluate -Dexpression=ozone.version -q -DforceStdout)
-DIST_DIR="$DIR/../../dist/target/ozone-$OZONE_VERSION"
+DIST_DIR="${OZONE_ROOT}/hadoop-ozone/dist/target/ozone-$OZONE_VERSION"
 
 if [ ! -d "$DIST_DIR" ]; then
     echo "Distribution dir is missing. Doing a full build"
@@ -36,15 +40,38 @@ fi
 
 mkdir -p "$REPORT_DIR"
 
+if [[ "${OZONE_ACCEPTANCE_SUITE}" == "s3a" ]]; then
+  if [[ -z "${HADOOP_AWS_DIR}" ]]; then
+    HADOOP_VERSION=$(mvn help:evaluate -Dexpression=hadoop.version -q -DforceStdout)
+    export HADOOP_AWS_DIR=${OZONE_ROOT}/target/hadoop-src
+  fi
+  download_hadoop_aws "${HADOOP_AWS_DIR}"
+  cp -fv ${OZONE_ROOT}/hadoop-ozone/dist/src/test/resources/auth-keys.xml "${HADOOP_AWS_DIR}"/src/test/resources/
+else
+  install_virtualenv
+  install_robot
+fi
+
 export OZONE_ACCEPTANCE_SUITE
 
 cd "$DIST_DIR/compose" || exit 1
 ./test-all.sh 2>&1 | tee "${REPORT_DIR}/output.log"
 RES=$?
-cp -rv result/* "$REPORT_DIR/"
-cp "$REPORT_DIR/log.html" "$REPORT_DIR/summary.html"
-find "$REPORT_DIR" -type f -empty -print0 | xargs -0 rm -v
 
-grep -A1 FAIL "${REPORT_DIR}/output.log" | grep -v '^Output' > "${REPORT_DIR}/summary.txt"
+if [[ "${OZONE_ACCEPTANCE_SUITE}" == "s3a" ]]; then
+  pushd result
+  source "${DIR}/_mvn_unit_report.sh"
+  find . -name junit -print0 | xargs -r -0 rm -frv
+  cp -rv * "${REPORT_DIR}"/
+  popd
+else
+  cp -rv result/* "$REPORT_DIR/"
+  if [[ -f "${REPORT_DIR}/log.html" ]]; then
+    cp "$REPORT_DIR/log.html" "$REPORT_DIR/summary.html"
+  fi
+  grep -A1 FAIL "${REPORT_DIR}/output.log" | grep -v '^Output' > "${REPORT_DIR}/summary.txt"
+fi
+
+find "$REPORT_DIR" -type f -empty -not -name summary.txt -print0 | xargs -0 rm -v
 
 exit $RES
diff --git a/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh b/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh
new file mode 100644
index 000000000000..75f68da26729
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+start_docker_env
+
+if [[ ${SECURITY_ENABLED} == "true" ]]; then
+  execute_robot_test s3g kinit.robot
+  # TODO get secret
+else
+  AWS_ACCESS_KEY_ID=s3a-contract
+  AWS_SECRET_ACCESS_KEY=unsecure
+fi
+
+OZONE_S3G_ADDRESS=http://localhost:9878
+
+execute_command_in_container s3g ozone sh bucket create --layout OBJECT_STORE "/s3v/obs-bucket"
+execute_command_in_container s3g ozone sh bucket create --layout LEGACY "/s3v/leg-bucket"
+execute_command_in_container s3g ozone sh bucket create --layout FILE_SYSTEM_OPTIMIZED "/s3v/fso-bucket"
+
+export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY OZONE_S3G_ADDRESS
+
+for bucket in obs-bucket leg-bucket fso-bucket; do
+  execute_s3a_tests "$bucket"
+done
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-s3a.sh b/hadoop-ozone/dist/src/main/compose/ozone/test-s3a.sh
new file mode 100644
index 000000000000..c277e71a4bf0
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test-s3a.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#suite:s3a
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export SECURITY_ENABLED=false
+
+source "$COMPOSE_DIR/../common/s3a-test.sh"
diff --git a/hadoop-ozone/dist/src/main/compose/test-all.sh b/hadoop-ozone/dist/src/main/compose/test-all.sh
index a998690032a7..d07f7a09b335 100755
--- a/hadoop-ozone/dist/src/main/compose/test-all.sh
+++ b/hadoop-ozone/dist/src/main/compose/test-all.sh
@@ -46,7 +46,8 @@ if [[ "${OZONE_WITH_COVERAGE}" == "true" ]]; then
   cp /tmp/jacoco-combined.exec "$SCRIPT_DIR"/result
 fi
 
-generate_report "acceptance" "${ALL_RESULT_DIR}" "${XUNIT_RESULT_DIR}"
-
+if [[ "${OZONE_ACCEPTANCE_SUITE:-}" != "s3a" ]]; then
+  generate_report "acceptance" "${ALL_RESULT_DIR}" "${XUNIT_RESULT_DIR}"
+fi
 
 exit $RESULT
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 860c4f516402..6043386f8473 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -405,12 +405,12 @@ copy_results() {
     target_dir="${target_dir}/${test_script_name}"
   fi
 
-  if [[ -n "$(find "${result_dir}" -name "*.xml")" ]]; then
+  if command -v rebot > /dev/null 2>&1 && [[ -n "$(find "${result_dir}" -name "*.xml")" ]]; then
     rebot --nostatusrc -N "${test_name}" -l NONE -r NONE -o "${all_result_dir}/${test_name}.xml" "${result_dir}"/*.xml \
       && rm -fv "${result_dir}"/*.xml "${result_dir}"/log.html "${result_dir}"/report.html
   fi
 
-  mkdir -p "${target_dir}"
+  mkdir -pv "${target_dir}"
   mv -v "${result_dir}"/* "${target_dir}"/
 }
 
@@ -574,3 +574,35 @@ wait_for_root_certificate(){
   echo "Timed out waiting on $count root certificates. Current timestamp " $(date +"%T")
   return 1
 }
+
+execute_s3a_tests() {
+  local bucket="$1"
+
+  if [[ -z ${bucket} ]]; then
+    echo "Required argument: the S3 bucket to be tested" >&2
+    return 1
+  fi
+
+  if [[ -z ${HADOOP_AWS_DIR} ]] || [[ ! -e ${HADOOP_AWS_DIR} ]] || [[ ! -d ${HADOOP_AWS_DIR}/src/test/resources ]]; then
+    echo "Set HADOOP_AWS_DIR to the directory with hadoop-aws sources" >&2
+    return 1
+  fi
+
+  if [[ -z ${OZONE_S3G_ADDRESS} ]]; then
+    echo "Set OZONE_S3G_ADDRESS to the address of S3 Gateway" >&2
+    return 1
+  fi
+
+  pushd ${HADOOP_AWS_DIR}
+  mvn -B -V --no-transfer-progress \
+    -Dtest.fs.s3a.endpoint="${OZONE_S3G_ADDRESS}" \
+    -Dtest.fs.s3a.name="s3a://${bucket}/" \
+    -Dtest='ITestS3AContract*, !ITestS3AContractDistCp, !ITestS3AContractEtag, !ITestS3AContractGetFileStatusV1List, !ITestS3AContractMkdir, !ITestS3AContractRename, !ITestS3AContractVectoredRead' \
+    clean test
+  rc=$?
+  mkdir -p ${RESULT_DIR}/junit/${bucket}/target
+  mv -iv target/surefire-reports ${RESULT_DIR}/junit/${bucket}/target/
+  popd
+
+  return $rc
+}
diff --git a/hadoop-ozone/dist/src/test/resources/auth-keys.xml b/hadoop-ozone/dist/src/test/resources/auth-keys.xml
new file mode 100644
index 000000000000..43311b8331c5
--- /dev/null
+++ b/hadoop-ozone/dist/src/test/resources/auth-keys.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+  <property>
+    <name>fs.s3a.endpoint</name>
+    <value>${test.fs.s3a.endpoint}</value>
+  </property>
+
+  <property>
+    <name>fs.contract.test.fs.s3a</name>
+    <value>${test.fs.s3a.name}</value>
+  </property>
+
+  <property>
+    <name>test.fs.s3a.sts.enabled</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <name>fs.s3a.path.style.access</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.s3a.directory.marker.retention</name>
+    <value>keep</value>
+  </property>
+
+</configuration>

From f0fc477e0627422174f110257bb72a81183754af Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Sat, 30 Mar 2024 09:14:28 +0100
Subject: [PATCH 2/8] reject empty target dir in download_hadoop_aws

---
 hadoop-ozone/dev-support/checks/_lib.sh | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/hadoop-ozone/dev-support/checks/_lib.sh b/hadoop-ozone/dev-support/checks/_lib.sh
index 7aafd38e52e3..134c8f53c6e8 100644
--- a/hadoop-ozone/dev-support/checks/_lib.sh
+++ b/hadoop-ozone/dev-support/checks/_lib.sh
@@ -152,9 +152,15 @@ _install_spotbugs() {
 
 download_hadoop_aws() {
   local dir="$1"
-  if [[ -z ${dir} ]] || [[ ! -e ${dir} ]] || [[ ! -d ${dir}/src/test/resources ]]; then
-    mkdir -p ${dir}
-    [[ -f ${dir}.tar.gz ]] || curl -LSs -o ${dir}.tar.gz https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}-src.tar.gz
-    tar -x -z -C ${dir} --strip-components=3 -f ${dir}.tar.gz --wildcards 'hadoop-*-src/hadoop-tools/hadoop-aws' || exit 1
+
+  if [[ -z ${dir} ]]; then
+    echo "Required argument: target directory for Hadoop AWS sources" >&2
+    return 1
+  fi
+
+  if [[ ! -e "${dir}" ]] || [[ ! -d "${dir}"/src/test/resources ]]; then
+    mkdir -p "${dir}"
+    [[ -f "${dir}.tar.gz" ]] || curl -LSs -o "${dir}.tar.gz" https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}-src.tar.gz
+    tar -x -z -C "${dir}" --strip-components=3 -f "${dir}.tar.gz" --wildcards 'hadoop-*-src/hadoop-tools/hadoop-aws' || return 1
   fi
 }

From a34ecd97a33f5113d60f0045880533a763d02c27 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Tue, 2 Apr 2024 12:24:25 +0200
Subject: [PATCH 3/8] fix setup for ITestS3AContractVectoredRead (see
 HADOOP-19133)

---
 hadoop-ozone/dist/src/main/compose/testlib.sh      | 4 ++--
 hadoop-ozone/dist/src/test/resources/auth-keys.xml | 7 ++++++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 6043386f8473..207f55a29f30 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -596,8 +596,8 @@ execute_s3a_tests() {
   pushd ${HADOOP_AWS_DIR}
   mvn -B -V --no-transfer-progress \
     -Dtest.fs.s3a.endpoint="${OZONE_S3G_ADDRESS}" \
-    -Dtest.fs.s3a.name="s3a://${bucket}/" \
-    -Dtest='ITestS3AContract*, !ITestS3AContractDistCp, !ITestS3AContractEtag, !ITestS3AContractGetFileStatusV1List, !ITestS3AContractMkdir, !ITestS3AContractRename, !ITestS3AContractVectoredRead' \
+    -Dcustom.fs.s3a.name="s3a://${bucket}/" \
+    -Dtest='ITestS3AContract*, !ITestS3AContractDistCp, !ITestS3AContractEtag, !ITestS3AContractGetFileStatusV1List, !ITestS3AContractMkdir, !ITestS3AContractRename' \
     clean test
   rc=$?
   mkdir -p ${RESULT_DIR}/junit/${bucket}/target
diff --git a/hadoop-ozone/dist/src/test/resources/auth-keys.xml b/hadoop-ozone/dist/src/test/resources/auth-keys.xml
index 43311b8331c5..b2831d1eb2f9 100644
--- a/hadoop-ozone/dist/src/test/resources/auth-keys.xml
+++ b/hadoop-ozone/dist/src/test/resources/auth-keys.xml
@@ -25,7 +25,12 @@
 
   <property>
     <name>fs.contract.test.fs.s3a</name>
-    <value>${test.fs.s3a.name}</value>
+    <value>${custom.fs.s3a.name}</value>
+  </property>
+
+  <property>
+    <name>test.fs.s3a.name</name>
+    <value>${custom.fs.s3a.name}</value>
   </property>
 
   <property>

From 5e0146d5cab0234adfe8d075e3ef8229a62365a0 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Mon, 8 Apr 2024 09:02:55 +0200
Subject: [PATCH 4/8] move execute_s3a_tests(); add comments; implement secure
 test; create auth-keys.xml inline

---
 hadoop-ozone/dev-support/checks/acceptance.sh | 11 ++-
 .../dist/src/main/compose/common/s3a-test.sh  | 96 ++++++++++++++++---
 .../main/compose/ozonesecure-ha/test-s3a.sh   | 27 ++++++
 .../dist/src/main/compose/test-all.sh         |  4 +-
 hadoop-ozone/dist/src/main/compose/testlib.sh | 32 -------
 .../dist/src/test/resources/auth-keys.xml     | 51 ----------
 6 files changed, 123 insertions(+), 98 deletions(-)
 create mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-s3a.sh
 delete mode 100644 hadoop-ozone/dist/src/test/resources/auth-keys.xml

diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh b/hadoop-ozone/dev-support/checks/acceptance.sh
index ec1122d6db5a..f60942d50bca 100755
--- a/hadoop-ozone/dev-support/checks/acceptance.sh
+++ b/hadoop-ozone/dev-support/checks/acceptance.sh
@@ -24,6 +24,7 @@ OZONE_ROOT=$(pwd -P)
 : ${HADOOP_AWS_DIR:=""}
 : ${OZONE_ACCEPTANCE_SUITE:=""}
 : ${OZONE_TEST_SELECTOR:=""}
+: ${OZONE_ACCEPTANCE_TEST_TYPE:="robot"}
 : ${OZONE_WITH_COVERAGE:="false"}
 
 source "${DIR}/_lib.sh"
@@ -41,13 +42,17 @@ fi
 mkdir -p "$REPORT_DIR"
 
 if [[ "${OZONE_ACCEPTANCE_SUITE}" == "s3a" ]]; then
+  OZONE_ACCEPTANCE_TEST_TYPE="maven"
+
   if [[ -z "${HADOOP_AWS_DIR}" ]]; then
     HADOOP_VERSION=$(mvn help:evaluate -Dexpression=hadoop.version -q -DforceStdout)
     export HADOOP_AWS_DIR=${OZONE_ROOT}/target/hadoop-src
   fi
+
   download_hadoop_aws "${HADOOP_AWS_DIR}"
-  cp -fv ${OZONE_ROOT}/hadoop-ozone/dist/src/test/resources/auth-keys.xml "${HADOOP_AWS_DIR}"/src/test/resources/
-else
+fi
+
+if [[ "${OZONE_ACCEPTANCE_TEST_TYPE}" == "robot" ]]; then
   install_virtualenv
   install_robot
 fi
@@ -58,7 +63,7 @@ cd "$DIST_DIR/compose" || exit 1
 ./test-all.sh 2>&1 | tee "${REPORT_DIR}/output.log"
 RES=$?
 
-if [[ "${OZONE_ACCEPTANCE_SUITE}" == "s3a" ]]; then
+if [[ "${OZONE_ACCEPTANCE_TEST_TYPE}" == "maven" ]]; then
   pushd result
   source "${DIR}/_mvn_unit_report.sh"
   find . -name junit -print0 | xargs -r -0 rm -frv
diff --git a/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh b/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh
index 75f68da26729..e5de48d76b11 100644
--- a/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh
+++ b/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh
@@ -15,26 +15,100 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# This script runs S3A contract tests against various bucket types on
+# a Docker Compose-based Ozone cluster.
+# Requires HADOOP_AWS_DIR to point the directory containing hadoop-aws sources.
+
+if [[ -z ${HADOOP_AWS_DIR} ]] || [[ ! -e ${HADOOP_AWS_DIR} ]]; then
+  echo "Set HADOOP_AWS_DIR to the directory with hadoop-aws sources" >&2
+  exit 1
+fi
+
 # shellcheck source=/dev/null
 source "$COMPOSE_DIR/../testlib.sh"
 
+## @description Run S3A contract tests against Ozone.
+## @param       Ozone S3 bucket
+execute_s3a_tests() {
+  local bucket="$1"
+
+  pushd "${HADOOP_AWS_DIR}"
+
+  # S3A contract tests are enabled by presence of `auth-keys.xml`.
+  # https://hadoop.apache.org/docs/3.3.6/hadoop-aws/tools/hadoop-aws/testing.html#Setting_up_the_tests
+  cat > src/test/resources/auth-keys.xml <<-EOF
+  <configuration>
+
+    <property>
+      <name>fs.s3a.endpoint</name>
+      <value>http://localhost:9878</value>
+    </property>
+
+    <property>
+      <name>test.fs.s3a.endpoint</name>
+      <value>http://localhost:9878</value>
+    </property>
+
+    <property>
+      <name>fs.contract.test.fs.s3a</name>
+      <value>s3a://${bucket}/</value>
+    </property>
+
+    <property>
+      <name>test.fs.s3a.name</name>
+      <value>s3a://${bucket}/</value>
+    </property>
+
+    <property>
+      <name>test.fs.s3a.sts.enabled</name>
+      <value>false</value>
+    </property>
+
+    <property>
+      <name>fs.s3a.path.style.access</name>
+      <value>true</value>
+    </property>
+
+    <property>
+      <name>fs.s3a.directory.marker.retention</name>
+      <value>keep</value>
+    </property>
+
+  </configuration>
+EOF
+
+  # Some tests are skipped due to known issues.
+  # - ITestS3AContractDistCp: HDDS-10616
+  # - ITestS3AContractEtag, ITestS3AContractRename: HDDS-10615
+  # - ITestS3AContractGetFileStatusV1List: HDDS-10617
+  # - ITestS3AContractMkdir: HDDS-10572
+  mvn -B -V --no-transfer-progress \
+    -Dtest='ITestS3AContract*, !ITestS3AContractDistCp, !ITestS3AContractEtag, !ITestS3AContractGetFileStatusV1List, !ITestS3AContractMkdir, !ITestS3AContractRename' \
+    clean test
+  rc=$?
+
+  local target="${RESULT_DIR}/junit/${bucket}/target"
+  mkdir -p "${target}"
+  mv -iv target/surefire-reports "${target}"/
+  popd
+
+  return $rc
+}
+
 start_docker_env
 
 if [[ ${SECURITY_ENABLED} == "true" ]]; then
-  execute_robot_test s3g kinit.robot
-  # TODO get secret
+  execute_command_in_container s3g kinit -kt /etc/security/keytabs/testuser.keytab "testuser/s3g@EXAMPLE.COM"
+  access=$(execute_command_in_container s3g ozone s3 getsecret -e)
+  eval "$access"
 else
-  AWS_ACCESS_KEY_ID=s3a-contract
-  AWS_SECRET_ACCESS_KEY=unsecure
+  export AWS_ACCESS_KEY_ID="s3a-contract"
+  export AWS_SECRET_ACCESS_KEY="unsecure"
 fi
 
-OZONE_S3G_ADDRESS=http://localhost:9878
-
-execute_command_in_container s3g ozone sh bucket create --layout OBJECT_STORE "/s3v/obs-bucket"
-execute_command_in_container s3g ozone sh bucket create --layout LEGACY "/s3v/leg-bucket"
-execute_command_in_container s3g ozone sh bucket create --layout FILE_SYSTEM_OPTIMIZED "/s3v/fso-bucket"
-
-export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY OZONE_S3G_ADDRESS
+execute_command_in_container s3g ozone sh bucket create --layout OBJECT_STORE /s3v/obs-bucket
+execute_command_in_container s3g ozone sh bucket create --layout LEGACY /s3v/leg-bucket
+execute_command_in_container s3g ozone sh bucket create --layout FILE_SYSTEM_OPTIMIZED /s3v/fso-bucket
 
 for bucket in obs-bucket leg-bucket fso-bucket; do
   execute_s3a_tests "$bucket"
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-s3a.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-s3a.sh
new file mode 100644
index 000000000000..78b8b51d9d81
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-s3a.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#suite:s3a
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export SECURITY_ENABLED=true
+export OM_SERVICE_ID="omservice"
+export SCM=scm1.org
+
+source "$COMPOSE_DIR/../common/s3a-test.sh"
diff --git a/hadoop-ozone/dist/src/main/compose/test-all.sh b/hadoop-ozone/dist/src/main/compose/test-all.sh
index d07f7a09b335..85294b6b7938 100755
--- a/hadoop-ozone/dist/src/main/compose/test-all.sh
+++ b/hadoop-ozone/dist/src/main/compose/test-all.sh
@@ -27,6 +27,7 @@ rm "$ALL_RESULT_DIR"/* || true
 
 source "$SCRIPT_DIR"/testlib.sh
 
+: ${OZONE_ACCEPTANCE_TEST_TYPE:="robot"}
 : ${OZONE_WITH_COVERAGE:="false"}
 
 if [[ "${OZONE_WITH_COVERAGE}" == "true" ]]; then
@@ -46,7 +47,8 @@ if [[ "${OZONE_WITH_COVERAGE}" == "true" ]]; then
   cp /tmp/jacoco-combined.exec "$SCRIPT_DIR"/result
 fi
 
-if [[ "${OZONE_ACCEPTANCE_SUITE:-}" != "s3a" ]]; then
+if [[ "${OZONE_ACCEPTANCE_TEST_TYPE}" == "robot" ]]; then
+  # does not apply to JUnit tests run via Maven
   generate_report "acceptance" "${ALL_RESULT_DIR}" "${XUNIT_RESULT_DIR}"
 fi
 
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 207f55a29f30..5ac3d09d59ac 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -574,35 +574,3 @@ wait_for_root_certificate(){
   echo "Timed out waiting on $count root certificates. Current timestamp " $(date +"%T")
   return 1
 }
-
-execute_s3a_tests() {
-  local bucket="$1"
-
-  if [[ -z ${bucket} ]]; then
-    echo "Required argument: the S3 bucket to be tested" >&2
-    return 1
-  fi
-
-  if [[ -z ${HADOOP_AWS_DIR} ]] || [[ ! -e ${HADOOP_AWS_DIR} ]] || [[ ! -d ${HADOOP_AWS_DIR}/src/test/resources ]]; then
-    echo "Set HADOOP_AWS_DIR to the directory with hadoop-aws sources" >&2
-    return 1
-  fi
-
-  if [[ -z ${OZONE_S3G_ADDRESS} ]]; then
-    echo "Set OZONE_S3G_ADDRESS to the address of S3 Gateway" >&2
-    return 1
-  fi
-
-  pushd ${HADOOP_AWS_DIR}
-  mvn -B -V --no-transfer-progress \
-    -Dtest.fs.s3a.endpoint="${OZONE_S3G_ADDRESS}" \
-    -Dcustom.fs.s3a.name="s3a://${bucket}/" \
-    -Dtest='ITestS3AContract*, !ITestS3AContractDistCp, !ITestS3AContractEtag, !ITestS3AContractGetFileStatusV1List, !ITestS3AContractMkdir, !ITestS3AContractRename' \
-    clean test
-  rc=$?
-  mkdir -p ${RESULT_DIR}/junit/${bucket}/target
-  mv -iv target/surefire-reports ${RESULT_DIR}/junit/${bucket}/target/
-  popd
-
-  return $rc
-}
diff --git a/hadoop-ozone/dist/src/test/resources/auth-keys.xml b/hadoop-ozone/dist/src/test/resources/auth-keys.xml
deleted file mode 100644
index b2831d1eb2f9..000000000000
--- a/hadoop-ozone/dist/src/test/resources/auth-keys.xml
+++ /dev/null
@@ -1,51 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
-
-  <property>
-    <name>fs.s3a.endpoint</name>
-    <value>${test.fs.s3a.endpoint}</value>
-  </property>
-
-  <property>
-    <name>fs.contract.test.fs.s3a</name>
-    <value>${custom.fs.s3a.name}</value>
-  </property>
-
-  <property>
-    <name>test.fs.s3a.name</name>
-    <value>${custom.fs.s3a.name}</value>
-  </property>
-
-  <property>
-    <name>test.fs.s3a.sts.enabled</name>
-    <value>false</value>
-  </property>
-
-  <property>
-    <name>fs.s3a.path.style.access</name>
-    <value>true</value>
-  </property>
-
-  <property>
-    <name>fs.s3a.directory.marker.retention</name>
-    <value>keep</value>
-  </property>
-
-</configuration>

From add9ce20bdf1497bbf61bdfa7baa69ceaac33d54 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Tue, 9 Apr 2024 17:19:48 +0200
Subject: [PATCH 5/8] fix typo in link

---
 hadoop-ozone/dist/src/main/compose/common/s3a-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh b/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh
index e5de48d76b11..cd07982ad4d9 100644
--- a/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh
+++ b/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh
@@ -35,7 +35,7 @@ execute_s3a_tests() {
   pushd "${HADOOP_AWS_DIR}"
 
   # S3A contract tests are enabled by presence of `auth-keys.xml`.
-  # https://hadoop.apache.org/docs/3.3.6/hadoop-aws/tools/hadoop-aws/testing.html#Setting_up_the_tests
+  # https://hadoop.apache.org/docs/r3.3.6/hadoop-aws/tools/hadoop-aws/testing.html#Setting_up_the_tests
   cat > src/test/resources/auth-keys.xml <<-EOF
   <configuration>
 

From 3ffb95c5fd601a4b78cea560ccff41aa90f44f97 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Thu, 11 Apr 2024 08:43:31 +0200
Subject: [PATCH 6/8] export OZONE_ACCEPTANCE_TEST_TYPE

---
 hadoop-ozone/dev-support/checks/acceptance.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh b/hadoop-ozone/dev-support/checks/acceptance.sh
index f60942d50bca..5be3f7b5879a 100755
--- a/hadoop-ozone/dev-support/checks/acceptance.sh
+++ b/hadoop-ozone/dev-support/checks/acceptance.sh
@@ -57,7 +57,7 @@ if [[ "${OZONE_ACCEPTANCE_TEST_TYPE}" == "robot" ]]; then
   install_robot
 fi
 
-export OZONE_ACCEPTANCE_SUITE
+export OZONE_ACCEPTANCE_SUITE OZONE_ACCEPTANCE_TEST_TYPE
 
 cd "$DIST_DIR/compose" || exit 1
 ./test-all.sh 2>&1 | tee "${REPORT_DIR}/output.log"

From 288fcd2f54dfa47cb5781dca1105d0e4cff72312 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Thu, 11 Apr 2024 10:10:28 +0200
Subject: [PATCH 7/8] avoid early exit due to test failures

---
 hadoop-ozone/dist/src/main/compose/common/s3a-test.sh | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh b/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh
index cd07982ad4d9..aa107aa1edfe 100644
--- a/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh
+++ b/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh
@@ -82,17 +82,14 @@ EOF
   # - ITestS3AContractEtag, ITestS3AContractRename: HDDS-10615
   # - ITestS3AContractGetFileStatusV1List: HDDS-10617
   # - ITestS3AContractMkdir: HDDS-10572
-  mvn -B -V --no-transfer-progress \
+  mvn -B -V --fail-never --no-transfer-progress \
     -Dtest='ITestS3AContract*, !ITestS3AContractDistCp, !ITestS3AContractEtag, !ITestS3AContractGetFileStatusV1List, !ITestS3AContractMkdir, !ITestS3AContractRename' \
     clean test
-  rc=$?
 
   local target="${RESULT_DIR}/junit/${bucket}/target"
   mkdir -p "${target}"
   mv -iv target/surefire-reports "${target}"/
   popd
-
-  return $rc
 }
 
 start_docker_env

From 45be98e04f6cc0a0a9c9ae851640af25c2999e28 Mon Sep 17 00:00:00 2001
From: "Doroszlai, Attila" <adoroszlai@apache.org>
Date: Fri, 12 Apr 2024 08:59:32 +0200
Subject: [PATCH 8/8] skip instead of fail if HADOOP_AWS_DIR is missing

---
 hadoop-ozone/dist/src/main/compose/common/s3a-test.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh b/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh
index aa107aa1edfe..85dbc5feced2 100644
--- a/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh
+++ b/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh
@@ -20,8 +20,8 @@
 # Requires HADOOP_AWS_DIR to point the directory containing hadoop-aws sources.
 
 if [[ -z ${HADOOP_AWS_DIR} ]] || [[ ! -e ${HADOOP_AWS_DIR} ]]; then
-  echo "Set HADOOP_AWS_DIR to the directory with hadoop-aws sources" >&2
-  exit 1
+  echo "Skipping S3A tests due to missing HADOOP_AWS_DIR (directory with hadoop-aws sources)" >&2
+  exit
 fi
 
 # shellcheck source=/dev/null