From 40ab653c5e8984533cfe01cfe58b535358ea62c1 Mon Sep 17 00:00:00 2001 From: Gonzalo Ortiz Date: Fri, 22 Sep 2023 10:36:45 +0200 Subject: [PATCH] Run tests with Corretto 21 (with unsafe and without unsafe). Remove tests with Temurin 20 --- .github/workflows/pinot_tests.yml | 89 ++++++++++++++++--- .github/workflows/scripts/.pinot_compile.sh | 28 ++++++ .github/workflows/scripts/.pinot_test.sh | 36 ++++---- .../segment/spi/memory/PinotDataBuffer.java | 20 +++-- 4 files changed, 136 insertions(+), 37 deletions(-) create mode 100755 .github/workflows/scripts/.pinot_compile.sh diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml index beb510bb5bdf..972e66636e23 100644 --- a/.github/workflows/pinot_tests.yml +++ b/.github/workflows/pinot_tests.yml @@ -76,16 +76,23 @@ jobs: fail-fast: false matrix: testset: [ 1, 2 ] - java: [ 11, 17, 20 ] + java: [ 11, 17 ] distribution: [ "temurin" ] + prioritize_bytebuffer: [false] + include: + - distribution: corretto + java: 21 + - distribution: corretto + java: 21 + prioritize_bytebuffer: true name: Pinot Unit Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}}) steps: - uses: actions/checkout@v3 - - name: Set up JDK ${{ matrix.java }} + - name: Set up JDK 11 uses: actions/setup-java@v3 with: - java-version: ${{ matrix.java }} - distribution: ${{ matrix.distribution }} + java-version: 11 + distribution: temurin cache: 'maven' # Step that does that actual cache save and restore - uses: actions/cache@v3 @@ -96,10 +103,31 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- - - name: Unit Test + - name: Compile Test + env: + RUN_INTEGRATION_TESTS: false + RUN_TEST_SET: ${{ matrix.testset }} + MAVEN_OPTS: > + -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 + -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false + -XX:+IgnoreUnrecognizedVMOptions + --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + run: .github/workflows/scripts/.pinot_compile.sh + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + java-version: ${{ matrix.java }} + distribution: ${{ matrix.distribution }} + cache: 'maven' + - name: Unit Test with ${{ matrix.distribution }} ${{ matrix.java }} env: RUN_INTEGRATION_TESTS: false RUN_TEST_SET: ${{ matrix.testset }} + PINOT_OFFHEAP_PRIORITIZE_BYTEBUFFER: ${{ matrix.prioritize_bytebuffer }} MAVEN_OPTS: > -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false @@ -128,16 +156,23 @@ jobs: fail-fast: false matrix: testset: [ 1, 2 ] - java: [ 11, 17, 20 ] + java: [ 11, 17 ] distribution: [ "temurin" ] + prioritize_bytebuffer: [false] + include: + - distribution: corretto + java: 21 + - distribution: corretto + java: 21 + prioritize_bytebuffer: true name: Pinot Integration Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}}) steps: - uses: actions/checkout@v3 - - name: Set up JDK ${{ matrix.java }} + - name: Set up JDK 11 uses: actions/setup-java@v3 with: - java-version: ${{ matrix.java }} - distribution: ${{ matrix.distribution }} + java-version: 11 + distribution: temurin cache: 'maven' # Step that does that actual cache save and restore - uses: actions/cache@v3 @@ -148,10 +183,31 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- + - name: Compile Test + env: + RUN_INTEGRATION_TESTS: false + RUN_TEST_SET: ${{ matrix.testset }} + MAVEN_OPTS: > + -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 + -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false + -XX:+IgnoreUnrecognizedVMOptions + --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + run: .github/workflows/scripts/.pinot_compile.sh + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + java-version: ${{ matrix.java }} + distribution: ${{ matrix.distribution }} + cache: 'maven' - name: Integration Test env: RUN_INTEGRATION_TESTS: true RUN_TEST_SET: ${{ matrix.testset }} + PINOT_OFFHEAP_PRIORITIZE_BYTEBUFFER: ${{ matrix.prioritize_bytebuffer }} MAVEN_OPTS: > -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false @@ -234,9 +290,16 @@ jobs: # Changed to false in order to improve coverage using unsafe buffers fail-fast: false matrix: - java: [ 11, 17, 20 ] + java: [ 11, 17 ] distribution: [ "temurin" ] - name: Pinot Quickstart on JDK ${{ matrix.java }} + prioritize_bytebuffer: [false] + include: + - distribution: corretto + java: 21 + - distribution: corretto + java: 21 + prioritize_bytebuffer: true + name: Pinot Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }} steps: - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.java }} @@ -254,5 +317,7 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- - - name: Quickstart on JDK ${{ matrix.java }} + - name: Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }} + env: + PINOT_OFFHEAP_PRIORITIZE_BYTEBUFFER: ${{ matrix.prioritize_bytebuffer }} run: .github/workflows/scripts/.pinot_quickstart.sh diff --git a/.github/workflows/scripts/.pinot_compile.sh b/.github/workflows/scripts/.pinot_compile.sh new file mode 100755 index 000000000000..050770c217eb --- /dev/null +++ b/.github/workflows/scripts/.pinot_compile.sh @@ -0,0 +1,28 @@ +#!/bin/bash -x +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Java version +java -version + +# Check network +ifconfig +netstat -i + +mvn clean verify -DskipTests -T 16 diff --git a/.github/workflows/scripts/.pinot_test.sh b/.github/workflows/scripts/.pinot_test.sh index 7179fa91a0b0..6def9a4cef0e 100755 --- a/.github/workflows/scripts/.pinot_test.sh +++ b/.github/workflows/scripts/.pinot_test.sh @@ -27,35 +27,35 @@ netstat -i if [ "$RUN_INTEGRATION_TESTS" != false ]; then # Integration Tests - mvn clean install -DskipTests -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip -Dlicense.skip -am -B \ - -pl 'pinot-integration-tests' -T 16 || exit 1 +# mvn clean install -DskipTests -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip -Dlicense.skip -am -B \ +# -pl 'pinot-integration-tests' -T 16 || exit 1 cd pinot-integration-tests || exit 1 if [ "$RUN_TEST_SET" == "1" ]; then mvn test \ - -P github-actions,custom-cluster-integration-test-suite || exit 1 + -P github-actions,custom-cluster-integration-test-suite -Dmaven.main.skip || exit 1 mvn test \ - -P github-actions,integration-tests-set-1 && exit 0 || exit 1 + -P github-actions,integration-tests-set-1 -Dmaven.main.skip && exit 0 || exit 1 fi if [ "$RUN_TEST_SET" == "2" ]; then mvn test \ - -P github-actions,integration-tests-set-2 && exit 0 || exit 1 + -P github-actions,integration-tests-set-2 -Dmaven.main.skip && exit 0 || exit 1 fi else # Unit Tests # - TEST_SET#1 runs install and test together so the module list must ensure no additional modules were tested # due to the -am flag (include dependency modules) if [ "$RUN_TEST_SET" == "1" ]; then - mvn clean install -DskipTests -am -B -T 16 \ - -pl 'pinot-spi' \ - -pl 'pinot-segment-spi' \ - -pl 'pinot-common' \ - -pl 'pinot-segment-local' \ - -pl 'pinot-core' \ - -pl 'pinot-query-planner' \ - -pl 'pinot-query-runtime' \ - -P github-actions,no-integration-tests \ - -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip -Dlicense.skip || exit 1 - mvn test -T 16 \ +# mvn clean install -DskipTests -am -B -T 16 \ +# -pl 'pinot-spi' \ +# -pl 'pinot-segment-spi' \ +# -pl 'pinot-common' \ +# -pl 'pinot-segment-local' \ +# -pl 'pinot-core' \ +# -pl 'pinot-query-planner' \ +# -pl 'pinot-query-runtime' \ +# -P github-actions,no-integration-tests \ +# -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip -Dlicense.skip || exit 1 + mvn test -T 16 -Dmaven.main.skip \ -pl 'pinot-spi' \ -pl 'pinot-segment-spi' \ -pl 'pinot-common' \ @@ -66,8 +66,8 @@ else -P github-actions,no-integration-tests && exit 0 || exit 1 fi if [ "$RUN_TEST_SET" == "2" ]; then - mvn clean install -DskipTests -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip -Dlicense.skip -T 16 || exit 1 - mvn test -am -B \ +# mvn clean install -DskipTests -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip -Dlicense.skip -T 16 || exit 1 + mvn test -am -B -Dmaven.main.skip \ -pl '!pinot-spi' \ -pl '!pinot-segment-spi' \ -pl '!pinot-common' \ diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java index 9bd36aeb38d4..002514f820d3 100644 --- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java +++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java @@ -67,6 +67,7 @@ public abstract class PinotDataBuffer implements Closeable { // With number of bytes less than this threshold, we get/put bytes one by one // With number of bytes more than this threshold, we create a ByteBuffer from the buffer and use bulk get/put method public static final int BULK_BYTES_PROCESSING_THRESHOLD = 10; + private static final String PRIORITIZE_BYTEBUFFER_ENV = "PINOT_OFFHEAP_PRIORITIZE_BYTEBUFFER"; private static class BufferContext { enum Type { @@ -152,17 +153,22 @@ public static PinotBufferFactory getFactory() { } public static PinotBufferFactory createDefaultFactory() { - return createDefaultFactory(true); + String prioritizeBbEnvValue = System.getenv(PRIORITIZE_BYTEBUFFER_ENV); + boolean prioritizeByteBuffer = prioritizeBbEnvValue == null || Boolean.parseBoolean(prioritizeBbEnvValue); + return createDefaultFactory(prioritizeByteBuffer); } public static PinotBufferFactory createDefaultFactory(boolean prioritizeByteBuffer) { String factoryClassName; - if (JavaVersion.VERSION < 16) { - LOGGER.info("Using LArray as buffer on JVM version {}", JavaVersion.VERSION); - factoryClassName = LArrayPinotBufferFactory.class.getCanonicalName(); - } else { - LOGGER.info("Using Unsafe as buffer on JVM version {}", JavaVersion.VERSION); - factoryClassName = UnsafePinotBufferFactory.class.getCanonicalName(); + factoryClassName = System.getenv("PINOT_BUFFER_LIBRARY"); + if (factoryClassName == null) { + if (JavaVersion.VERSION < 16) { + LOGGER.info("Using LArray as buffer on JVM version {}", JavaVersion.VERSION); + factoryClassName = LArrayPinotBufferFactory.class.getCanonicalName(); + } else { + LOGGER.info("Using Unsafe as buffer on JVM version {}", JavaVersion.VERSION); + factoryClassName = UnsafePinotBufferFactory.class.getCanonicalName(); + } } return createFactory(factoryClassName, prioritizeByteBuffer); }