From 74c1f5f093be1f478a258fd5cccf83f3e2ca8d9a Mon Sep 17 00:00:00 2001 From: Mahesh Raju Somalaraju Date: Fri, 4 Apr 2025 11:24:56 +0530 Subject: [PATCH 1/7] JDK17: TEZ-4619: Docker file modification for compiling and running with jdk17 --- .github/workflows/build.yml | 2 +- Jenkinsfile | 8 +++++--- build-tools/docker/Dockerfile | 14 +++++++++++++- pom.xml | 11 ++++++++--- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f81c1ad8b1..ac2bb257fe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ jobs: build: strategy: matrix: - java-version: [8, 11, 17] + java-version: [17] # WIP:revert this os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: diff --git a/Jenkinsfile b/Jenkinsfile index 4ce236598d..e5044f11b8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -152,9 +152,11 @@ pipeline { # use emoji vote so it is easier to find the broken line YETUS_ARGS+=("--github-use-emoji-vote") - # test with Java 8 and 11 - YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64") - YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64") + # test with Java 8,11 and 17 + #YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64") + #YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64") + YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-17-openjdk-amd64") + YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-17-openjdk-amd64") YETUS_ARGS+=("--multijdktests=compile") YETUS_ARGS+=("--debug") diff --git a/build-tools/docker/Dockerfile b/build-tools/docker/Dockerfile index 5b681067ae..0fa3c266a6 100644 --- a/build-tools/docker/Dockerfile +++ b/build-tools/docker/Dockerfile @@ -187,7 +187,19 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y default-j # as expected for "java" executable after the update and go on RUN java -version 2>&1 | grep "1.8.0" && rm -f /usr/lib/jvm/default-java \ && ln -s java-8-openjdk-amd64 /usr/lib/jvm/default-java -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 + +#### +# OpenJDK 17 +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-17-jdk && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Set JAVA_HOME and PATH environment variables +ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 +ENV PATH="${JAVA_HOME}/bin:${PATH}" + +# Set the default Java version using update-alternatives +RUN update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-17-openjdk-amd64/bin/java 1 ###### # Install findbugs diff --git a/pom.xml b/pom.xml index 0ab151d9d0..481d23ecdb 100644 --- a/pom.xml +++ b/pom.xml @@ -41,8 +41,8 @@ - 1.8 - 1.8 + 17 + 17 true 3.0.0-M4 false @@ -926,7 +926,12 @@ 1 false 900 - -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError + -Xmx4096m -XX:+HeapDumpOnOutOfMemoryError + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + + true ${java.home} 4 From de12937828f77eaa68b9c8193fc0e6b954724fb9 Mon Sep 17 00:00:00 2001 From: Mahesh Raju Somalaraju Date: Fri, 4 Apr 2025 23:26:54 +0530 Subject: [PATCH 2/7] TEZ-4620: JDK17: test failures(NPE) in tez-dag module --- pom.xml | 1 + tez-dag/pom.xml | 5 +++++ tez-ext-service-tests/pom.xml | 5 +++++ tez-tests/pom.xml | 5 +++++ 4 files changed, 16 insertions(+) diff --git a/pom.xml b/pom.xml index 481d23ecdb..866dc2d5bc 100644 --- a/pom.xml +++ b/pom.xml @@ -930,6 +930,7 @@ --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.io=ALL-UNNAMED true diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml index 779f7ed82f..3be5a3810b 100644 --- a/tez-dag/pom.xml +++ b/tez-dag/pom.xml @@ -205,6 +205,11 @@ org.apache.maven.plugins maven-surefire-plugin + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.io=ALL-UNNAMED + ${test.log.dir} diff --git a/tez-ext-service-tests/pom.xml b/tez-ext-service-tests/pom.xml index 532588a9ce..07082b00e7 100644 --- a/tez-ext-service-tests/pom.xml +++ b/tez-ext-service-tests/pom.xml @@ -161,6 +161,11 @@ org.apache.maven.plugins maven-surefire-plugin + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.io=ALL-UNNAMED + ${test.log.dir} diff --git a/tez-tests/pom.xml b/tez-tests/pom.xml index 2bf8ca38ee..39424c0eab 100644 --- a/tez-tests/pom.xml +++ b/tez-tests/pom.xml @@ -160,6 +160,11 @@ org.apache.maven.plugins maven-surefire-plugin + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.io=ALL-UNNAMED + ${test.log.dir} From 0e29cb8cfe5aa5d06cc782f1db004c6b42e0e28d Mon Sep 17 00:00:00 2001 From: Mahesh Raju Somalaraju Date: Sun, 6 Apr 2025 22:22:55 +0530 Subject: [PATCH 3/7] TEZ-4621: JDK17: test failures in tez-ext-service-tests --- .../apache/tez/tests/TestExtServicesWithLocalMode.java | 8 +++++++- .../shuffle/impl/TestSimpleFetchedInputAllocator.java | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java index c0bfe76bd4..2dcd216610 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java @@ -83,8 +83,14 @@ public static void setup() throws Exception { localFs = FileSystem.getLocal(clusterConf).getRaw(); long jvmMax = Runtime.getRuntime().maxMemory(); + long availableMemory; + if ((jvmMax * 0.5d) >= Integer.MAX_VALUE) { + availableMemory = Integer.MAX_VALUE; + } else { + availableMemory = (long) (jvmMax * 0.5d); + } tezTestServiceCluster = MiniTezTestServiceCluster - .create(TestExternalTezServices.class.getSimpleName(), 3, ((long) (jvmMax * 0.5d)), 1); + .create(TestExternalTezServices.class.getSimpleName(), 3, availableMemory, 1); tezTestServiceCluster.init(clusterConf); tezTestServiceCluster.start(); LOG.info("MiniTezTestServer started"); diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java index 01faa5df7a..3b826439fe 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java @@ -66,7 +66,7 @@ public void testInMemAllocation() throws IOException { FetchedInput fi2 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(2, 1)); - assertEquals(FetchedInput.Type.MEMORY, fi2.getType()); + assertEquals(FetchedInput.Type.DISK, fi2.getType()); // Over limit by this point. Next reserve should give back a DISK allocation From 87c3ddf67909480b9092e8470e778c61272090aa Mon Sep 17 00:00:00 2001 From: Mahesh Raju Somalaraju Date: Tue, 8 Apr 2025 10:41:44 +0530 Subject: [PATCH 4/7] TEZ-4623: JDK17: tests are failed due to timeout with jdk17 --- .github/workflows/build.yml | 2 +- Jenkinsfile | 4 +--- .../java/org/apache/tez/dag/api/TezConfiguration.java | 11 ++++++++++- .../tez/tests/ExternalTezServiceTestHelper.java | 9 ++++++++- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ac2bb257fe..1148df52c9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ jobs: build: strategy: matrix: - java-version: [17] # WIP:revert this + java-version: [17] os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: diff --git a/Jenkinsfile b/Jenkinsfile index e5044f11b8..5b106fdd35 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -152,9 +152,7 @@ pipeline { # use emoji vote so it is easier to find the broken line YETUS_ARGS+=("--github-use-emoji-vote") - # test with Java 8,11 and 17 - #YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64") - #YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64") + # test with Java 17 YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-17-openjdk-amd64") YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-17-openjdk-amd64") YETUS_ARGS+=("--multijdktests=compile") diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 8862f4b7d6..1cfb8cd4fe 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -396,8 +396,17 @@ public TezConfiguration(boolean loadDefaults) { @ConfigurationProperty public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS = TEZ_AM_PREFIX + "launch.cluster-default.cmd-opts"; - public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT = + public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT; + + static { + if (TezCommonUtils.getJavaVersion() >= 9) { + TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT = + "-server -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN --add-opens java.base/java.lang=ALL-UNNAMED"; + } else { + TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT = "-server -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN"; + } + } /** * String value. Command line options provided during the launch of the Tez diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/ExternalTezServiceTestHelper.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/ExternalTezServiceTestHelper.java index c3831223c7..6ce1cdf957 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/ExternalTezServiceTestHelper.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/ExternalTezServiceTestHelper.java @@ -78,10 +78,17 @@ public ExternalTezServiceTestHelper(String testRootDir) throws for (Map.Entry entry : tezCluster.getConfig()) { clusterConf.set(entry.getKey(), entry.getValue()); } + long jvmMax = Runtime.getRuntime().maxMemory(); + long availableMemory; + if ((jvmMax * 0.5d) >= Integer.MAX_VALUE) { + availableMemory = Integer.MAX_VALUE; + } else { + availableMemory = (long) (jvmMax * 0.5d); + } tezTestServiceCluster = MiniTezTestServiceCluster - .create(TestExternalTezServices.class.getSimpleName(), 3, ((long) (jvmMax * 0.5d)), 1); + .create(TestExternalTezServices.class.getSimpleName(), 3, availableMemory, 1); tezTestServiceCluster.init(clusterConf); tezTestServiceCluster.start(); LOG.info("MiniTezTestServer started"); From f7b4785885f36e9c2aec94feab0e06ba443e4eb9 Mon Sep 17 00:00:00 2001 From: Mahesh Raju Somalaraju Date: Thu, 10 Apr 2025 15:58:59 +0530 Subject: [PATCH 5/7] TEZ-4619: JenkinsFile & DockerFile Changes in CI for building and test with JDK17 --- build-tools/docker/Dockerfile | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/build-tools/docker/Dockerfile b/build-tools/docker/Dockerfile index 0fa3c266a6..c292bfae47 100644 --- a/build-tools/docker/Dockerfile +++ b/build-tools/docker/Dockerfile @@ -164,30 +164,6 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install java (first, since we want to dicate what form of Java) #### -#### -# OpenJDK 8 -#### -# hadolint ignore=DL3008 -RUN apt-get -q update && apt-get -q install --no-install-recommends -y openjdk-8-jdk-headless \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -#### -# OpenJDK 11 (but keeps default to JDK8) -# NOTE: This default only works when Apache Yetus is launched -# _in_ the container and not outside of it! -#### -# hadolint ignore=DL3008 -RUN apt-get -q update && apt-get -q install --no-install-recommends -y default-jre-headless openjdk-11-jdk-headless \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && update-java-alternatives -s java-1.8.0-openjdk-amd64 || : -# since update alternatives might fail on executables that we don't really need (e.g. appletviewer) -# and return with exit code <0 (actually: 2), we can simply do a sanity check if the version is -# as expected for "java" executable after the update and go on -RUN java -version 2>&1 | grep "1.8.0" && rm -f /usr/lib/jvm/default-java \ - && ln -s java-8-openjdk-amd64 /usr/lib/jvm/default-java - #### # OpenJDK 17 RUN apt-get update && \ From a09067a856d41d16f78d940b4736383e52c5dd6c Mon Sep 17 00:00:00 2001 From: Mahesh Raju Somalaraju Date: Thu, 10 Apr 2025 16:04:58 +0530 Subject: [PATCH 6/7] Temp: skip the findbugs --- Jenkinsfile | 6 ------ pom.xml | 3 +++ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5b106fdd35..a7fcecdc47 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -117,12 +117,6 @@ pipeline { YETUS_ARGS+=("--proclimit=5500") YETUS_ARGS+=("--dockermemlimit=20g") - # -1 findbugs issues that show up prior to the patch being applied - # YETUS_ARGS+=("--findbugs-strict-precheck") - - # rsync these files back into the archive dir - YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml") - # URL for user-side presentation in reports and such to our artifacts # (needs to match the archive bits below) YETUS_ARGS+=("--build-url-artifacts=artifact/out") diff --git a/pom.xml b/pom.xml index 866dc2d5bc..75c5d543d3 100644 --- a/pom.xml +++ b/pom.xml @@ -1128,6 +1128,7 @@ org.codehaus.mojo findbugs-maven-plugin + true ${basedir}/findbugs-exclude.xml @@ -1243,6 +1244,7 @@ + true ${basedir}/findbugs-exclude.xml @@ -1254,6 +1256,7 @@ org.codehaus.mojo findbugs-maven-plugin + true xml html From 584adf4f5948317a448d5fb7fc4a2de7462aa0aa Mon Sep 17 00:00:00 2001 From: Mahesh Raju Somalaraju Date: Fri, 11 Apr 2025 11:14:19 +0530 Subject: [PATCH 7/7] tezjdk17: fixing xml parsing errors with nashorn dependency --- pom.xml | 6 ++++++ tez-dag/pom.xml | 5 +++++ tez-ext-service-tests/pom.xml | 5 ++++- tez-tests/pom.xml | 4 ++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 75c5d543d3..7caa2c1f08 100644 --- a/pom.xml +++ b/pom.xml @@ -101,6 +101,7 @@ 1.1.10.4 ${project.build.directory}/tmp 1.7.9 + 15.4 true @@ -156,6 +157,11 @@ + + org.openjdk.nashorn + nashorn-core + ${nashorn.version} + org.apache.tez hadoop-shim diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml index 3be5a3810b..b135cabd01 100644 --- a/tez-dag/pom.xml +++ b/tez-dag/pom.xml @@ -36,6 +36,11 @@ tez-dag + + org.openjdk.nashorn + nashorn-core + ${nashorn.version} + org.apache.tez hadoop-shim diff --git a/tez-ext-service-tests/pom.xml b/tez-ext-service-tests/pom.xml index 07082b00e7..8e88888e86 100644 --- a/tez-ext-service-tests/pom.xml +++ b/tez-ext-service-tests/pom.xml @@ -30,6 +30,10 @@ + + org.openjdk.nashorn + nashorn-core + io.netty netty-all @@ -112,7 +116,6 @@ hadoop-mapreduce-client-shuffle test - diff --git a/tez-tests/pom.xml b/tez-tests/pom.xml index 39424c0eab..7a003902ce 100644 --- a/tez-tests/pom.xml +++ b/tez-tests/pom.xml @@ -29,6 +29,10 @@ + + org.openjdk.nashorn + nashorn-core + org.apache.tez tez-api