From fa02111f71cd1188e47958e684a612d491d2974a Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Fri, 12 Jan 2024 19:09:21 +0100 Subject: [PATCH] HDDS-10123. InaccessibleObjectException in tests using ChecksumByteBufferImpl with Java 17 --- hadoop-ozone/dev-support/checks/junit.sh | 11 +++++-- hadoop-ozone/dist/src/shell/ozone/ozone | 16 ---------- .../dist/src/shell/ozone/ozone-functions.sh | 29 +++++++++++++++++++ pom.xml | 3 +- 4 files changed, 40 insertions(+), 19 deletions(-) diff --git a/hadoop-ozone/dev-support/checks/junit.sh b/hadoop-ozone/dev-support/checks/junit.sh index 6d6688c11536..417ae35e5e42 100755 --- a/hadoop-ozone/dev-support/checks/junit.sh +++ b/hadoop-ozone/dev-support/checks/junit.sh @@ -31,7 +31,7 @@ if [[ ${ITERATIONS} -le 0 ]]; then fi export MAVEN_OPTS="-Xmx4096m $MAVEN_OPTS" -MAVEN_OPTIONS='-B -Dskip.npx -Dskip.installnpx -Dnative.lib.tmp.dir=/tmp --no-transfer-progress' +MAVEN_OPTIONS="-B -V -Dskip.npx -Dskip.installnpx -Dnative.lib.tmp.dir=/tmp --no-transfer-progress" if [[ "${OZONE_WITH_COVERAGE}" != "true" ]]; then MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip" @@ -43,6 +43,13 @@ else MAVEN_OPTIONS="${MAVEN_OPTIONS} --fail-at-end" fi +# apply module access args (for Java 9+) +OZONE_MODULE_ACCESS_ARGS="" +if [[ -f hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh ]]; then + source hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh + ozone_java_setup +fi + if [[ "${CHECK}" == "integration" ]] || [[ ${ITERATIONS} -gt 1 ]]; then if [[ ${OZONE_REPO_CACHED} == "false" ]]; then mvn ${MAVEN_OPTIONS} -DskipTests clean install @@ -61,7 +68,7 @@ for i in $(seq 1 ${ITERATIONS}); do mkdir -p "${REPORT_DIR}" fi - mvn ${MAVEN_OPTIONS} "$@" test \ + mvn ${MAVEN_OPTIONS} -Dmaven-surefire-plugin.argLineAccessArgs="${OZONE_MODULE_ACCESS_ARGS}" "$@" test \ | tee "${REPORT_DIR}/output.log" irc=$? diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone b/hadoop-ozone/dist/src/shell/ozone/ozone index 752d31153506..8686f56c287a 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone +++ b/hadoop-ozone/dist/src/shell/ozone/ozone @@ -79,22 +79,6 @@ function ozonecmd_case # This parameter significantly reduces GC pressure for Datanode. # Corresponding Ratis issue https://issues.apache.org/jira/browse/RATIS-534. RATIS_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.allocator.useCacheForAllThreads=false ${RATIS_OPTS}" - # Add JVM parameter for Java 17 - OZONE_MODULE_ACCESS_ARGS="" - - # Get the version string - JAVA_VERSION_STRING=$(java -version 2>&1 | head -n 1) - - # Extract the major version number - JAVA_MAJOR_VERSION=$(echo "$JAVA_VERSION_STRING" | sed -E -n 's/.* version "([^.-]*).*"/\1/p' | cut -d' ' -f1) - - # populate JVM args based on java version - if [[ "${JAVA_MAJOR_VERSION}" == "17" ]]; then - OZONE_MODULE_ACCESS_ARGS="--add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED --add-exports java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED" - fi - if [[ "${JAVA_MAJOR_VERSION}" -ge 9 ]]; then - OZONE_MODULE_ACCESS_ARGS="${OZONE_MODULE_ACCESS_ARGS} --add-opens java.base/java.nio=ALL-UNNAMED" - fi case ${subcmd} in auditparser) diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh b/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh index 500d7d446c86..392372bfc121 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh +++ b/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh @@ -1406,6 +1406,35 @@ function ozone_java_setup ozone_error "ERROR: $JAVA is not executable." exit 1 fi + + # Get the version string + JAVA_VERSION_STRING=$(${JAVA} -version 2>&1 | head -n 1) + + # Extract the major version number + JAVA_MAJOR_VERSION=$(echo "$JAVA_VERSION_STRING" | sed -E -n 's/.* version "([^.-]*).*"/\1/p' | cut -d' ' -f1) + + ozone_set_module_access_args +} + +## @description Set OZONE_MODULE_ACCESS_ARGS based on Java version +## @audience private +## @stability evolving +## @replaceable yes +function ozone_set_module_access_args +{ + if [[ -z "${JAVA_MAJOR_VERSION}" ]]; then + return + fi + + # populate JVM args based on java version + if [[ "${JAVA_MAJOR_VERSION}" -ge 17 ]]; then + OZONE_MODULE_ACCESS_ARGS="--add-opens java.base/java.lang=ALL-UNNAMED" + OZONE_MODULE_ACCESS_ARGS="${OZONE_MODULE_ACCESS_ARGS} --add-opens java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED" + OZONE_MODULE_ACCESS_ARGS="${OZONE_MODULE_ACCESS_ARGS} --add-exports java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED" + fi + if [[ "${JAVA_MAJOR_VERSION}" -ge 9 ]]; then + OZONE_MODULE_ACCESS_ARGS="${OZONE_MODULE_ACCESS_ARGS} --add-opens java.base/java.nio=ALL-UNNAMED" + fi } ## @description Finish Java JNI paths prior to execution diff --git a/pom.xml b/pom.xml index 32884499c060..f36487be9f51 100644 --- a/pom.xml +++ b/pom.xml @@ -252,6 +252,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs -Xmx4096m -XX:+HeapDumpOnOutOfMemoryError + flaky | slow | unhealthy 3.2.2 ${maven-surefire-plugin.version} @@ -1973,7 +1974,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs false ${surefire.fork.timeout} - ${maven-surefire-plugin.argLine} @{argLine} + ${maven-surefire-plugin.argLine} ${maven-surefire-plugin.argLineAccessArgs} @{argLine} 4