[draft] Run tests with Corretto 21

.github/workflows/pinot_tests.yml

@@ -76,16 +76,23 @@ jobs:
       fail-fast: false
       matrix:
         testset: [ 1, 2 ]
-        java: [ 11, 17, 20 ]
+        java: [ 11, 17 ]
         distribution: [ "temurin" ]
+        prioritize_bytebuffer: [false]
+        include:
+          - distribution: corretto
+            java: 21
+          - distribution: corretto
+            java: 21
+            prioritize_bytebuffer: true
     name: Pinot Unit Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
     steps:
       - uses: actions/checkout@v3
-      - name: Set up JDK ${{ matrix.java }}
+      - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:
-          java-version: ${{ matrix.java }}
-          distribution: ${{ matrix.distribution }}
+          java-version: 11
+          distribution: temurin
           cache: 'maven'
       # Step that does that actual cache save and restore
       - uses: actions/cache@v3
@@ -96,10 +103,31 @@ jobs:
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
-      - name: Unit Test
+      - name: Compile Test
+        env:
+          RUN_INTEGRATION_TESTS: false
+          RUN_TEST_SET: ${{ matrix.testset }}
+          MAVEN_OPTS: >
+            -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+            -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -XX:+IgnoreUnrecognizedVMOptions
+            --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+        run: .github/workflows/scripts/.pinot_compile.sh
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.java }}
+          distribution: ${{ matrix.distribution }}
+          cache: 'maven'
+      - name: Unit Test with ${{ matrix.distribution }} ${{ matrix.java }}
         env:
           RUN_INTEGRATION_TESTS: false
           RUN_TEST_SET: ${{ matrix.testset }}
+          PINOT_OFFHEAP_PRIORITIZE_BYTEBUFFER: ${{ matrix.prioritize_bytebuffer }}
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
             -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
@@ -128,16 +156,23 @@ jobs:
       fail-fast: false
       matrix:
         testset: [ 1, 2 ]
-        java: [ 11, 17, 20 ]
+        java: [ 11, 17 ]
         distribution: [ "temurin" ]
+        prioritize_bytebuffer: [false]
+        include:
+          - distribution: corretto
+            java: 21
+          - distribution: corretto
+            java: 21
+            prioritize_bytebuffer: true
     name: Pinot Integration Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
     steps:
       - uses: actions/checkout@v3
-      - name: Set up JDK ${{ matrix.java }}
+      - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:
-          java-version: ${{ matrix.java }}
-          distribution: ${{ matrix.distribution }}
+          java-version: 11
+          distribution: temurin
           cache: 'maven'
       # Step that does that actual cache save and restore
       - uses: actions/cache@v3
@@ -148,10 +183,31 @@ jobs:
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
+      - name: Compile Test
+        env:
+          RUN_INTEGRATION_TESTS: false
+          RUN_TEST_SET: ${{ matrix.testset }}
+          MAVEN_OPTS: >
+            -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+            -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -XX:+IgnoreUnrecognizedVMOptions
+            --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+        run: .github/workflows/scripts/.pinot_compile.sh
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.java }}
+          distribution: ${{ matrix.distribution }}
+          cache: 'maven'
       - name: Integration Test
         env:
           RUN_INTEGRATION_TESTS: true
           RUN_TEST_SET: ${{ matrix.testset }}
+          PINOT_OFFHEAP_PRIORITIZE_BYTEBUFFER: ${{ matrix.prioritize_bytebuffer }}
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
             -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
@@ -234,9 +290,16 @@ jobs:
       # Changed to false in order to improve coverage using unsafe buffers
       fail-fast: false
       matrix:
-        java: [ 11, 17, 20 ]
+        java: [ 11, 17 ]
         distribution: [ "temurin" ]
-    name: Pinot Quickstart on JDK ${{ matrix.java }}
+        prioritize_bytebuffer: [false]
+        include:
+          - distribution: corretto
+            java: 21
+          - distribution: corretto
+            java: 21
+            prioritize_bytebuffer: true
+    name: Pinot Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }}
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK ${{ matrix.java }}
@@ -254,5 +317,7 @@ jobs:
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
-      - name: Quickstart on JDK ${{ matrix.java }}
+      - name: Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }}
+        env:
+          PINOT_OFFHEAP_PRIORITIZE_BYTEBUFFER: ${{ matrix.prioritize_bytebuffer }}
         run: .github/workflows/scripts/.pinot_quickstart.sh

.github/workflows/scripts/.pinot_compile.sh

@@ -0,0 +1,28 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Java version
+java -version
+
+# Check network
+ifconfig
+netstat -i
+
+mvn clean verify -DskipTests -T 16

.github/workflows/scripts/.pinot_test.sh

@@ -27,35 +27,35 @@ netstat -i
 
 if [ "$RUN_INTEGRATION_TESTS" != false ]; then
   # Integration Tests
-  mvn clean install -DskipTests -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip -Dlicense.skip -am -B \
-    -pl 'pinot-integration-tests' -T 16 || exit 1
+#  mvn clean install -DskipTests -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip -Dlicense.skip -am -B \
+#    -pl 'pinot-integration-tests' -T 16 || exit 1
   cd pinot-integration-tests || exit 1
   if [ "$RUN_TEST_SET" == "1" ]; then
     mvn test \
-        -P github-actions,custom-cluster-integration-test-suite || exit 1
+        -P github-actions,custom-cluster-integration-test-suite -Dmaven.main.skip || exit 1
     mvn test \
-        -P github-actions,integration-tests-set-1 && exit 0 || exit 1
+        -P github-actions,integration-tests-set-1 -Dmaven.main.skip && exit 0 || exit 1
   fi
   if [ "$RUN_TEST_SET" == "2" ]; then
     mvn test \
-        -P github-actions,integration-tests-set-2 && exit 0 || exit 1
+        -P github-actions,integration-tests-set-2 -Dmaven.main.skip && exit 0 || exit 1
   fi
 else
   # Unit Tests
   #   - TEST_SET#1 runs install and test together so the module list must ensure no additional modules were tested
   #     due to the -am flag (include dependency modules)
   if [ "$RUN_TEST_SET" == "1" ]; then
-    mvn clean install -DskipTests -am -B -T 16 \
-        -pl 'pinot-spi' \
-        -pl 'pinot-segment-spi' \
-        -pl 'pinot-common' \
-        -pl 'pinot-segment-local' \
-        -pl 'pinot-core' \
-        -pl 'pinot-query-planner' \
-        -pl 'pinot-query-runtime' \
-        -P github-actions,no-integration-tests \
-        -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip -Dlicense.skip || exit 1
-    mvn test -T 16 \
+#    mvn clean install -DskipTests -am -B -T 16 \
+#        -pl 'pinot-spi' \
+#        -pl 'pinot-segment-spi' \
+#        -pl 'pinot-common' \
+#        -pl 'pinot-segment-local' \
+#        -pl 'pinot-core' \
+#        -pl 'pinot-query-planner' \
+#        -pl 'pinot-query-runtime' \
+#        -P github-actions,no-integration-tests \
+#        -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip -Dlicense.skip || exit 1
+    mvn test -T 16 -Dmaven.main.skip \
         -pl 'pinot-spi' \
         -pl 'pinot-segment-spi' \
         -pl 'pinot-common' \
@@ -66,8 +66,8 @@ else
         -P github-actions,no-integration-tests && exit 0 || exit 1
   fi
   if [ "$RUN_TEST_SET" == "2" ]; then
-    mvn clean install -DskipTests -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip -Dlicense.skip -T 16 || exit 1
-    mvn test -am -B \
+#    mvn clean install -DskipTests -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip -Dlicense.skip -T 16 || exit 1
+    mvn test -am -B -Dmaven.main.skip \
         -pl '!pinot-spi' \
         -pl '!pinot-segment-spi' \
         -pl '!pinot-common' \

pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java

@@ -67,6 +67,7 @@ public abstract class PinotDataBuffer implements Closeable {
   // With number of bytes less than this threshold, we get/put bytes one by one
   // With number of bytes more than this threshold, we create a ByteBuffer from the buffer and use bulk get/put method
   public static final int BULK_BYTES_PROCESSING_THRESHOLD = 10;
+  private static final String PRIORITIZE_BYTEBUFFER_ENV = "PINOT_OFFHEAP_PRIORITIZE_BYTEBUFFER";
 
   private static class BufferContext {
     enum Type {
@@ -152,17 +153,22 @@ public static PinotBufferFactory getFactory() {
   }
 
   public static PinotBufferFactory createDefaultFactory() {
-    return createDefaultFactory(true);
+    String prioritizeBbEnvValue = System.getenv(PRIORITIZE_BYTEBUFFER_ENV);
+    boolean prioritizeByteBuffer = prioritizeBbEnvValue == null || Boolean.parseBoolean(prioritizeBbEnvValue);
+    return createDefaultFactory(prioritizeByteBuffer);
   }
 
   public static PinotBufferFactory createDefaultFactory(boolean prioritizeByteBuffer) {
     String factoryClassName;
-    if (JavaVersion.VERSION < 16) {
-      LOGGER.info("Using LArray as buffer on JVM version {}", JavaVersion.VERSION);
-      factoryClassName = LArrayPinotBufferFactory.class.getCanonicalName();
-    } else {
-      LOGGER.info("Using Unsafe as buffer on JVM version {}", JavaVersion.VERSION);
-      factoryClassName = UnsafePinotBufferFactory.class.getCanonicalName();
+    factoryClassName = System.getenv("PINOT_BUFFER_LIBRARY");
+    if (factoryClassName == null) {
+      if (JavaVersion.VERSION < 16) {
+        LOGGER.info("Using LArray as buffer on JVM version {}", JavaVersion.VERSION);
+        factoryClassName = LArrayPinotBufferFactory.class.getCanonicalName();
+      } else {
+        LOGGER.info("Using Unsafe as buffer on JVM version {}", JavaVersion.VERSION);
+        factoryClassName = UnsafePinotBufferFactory.class.getCanonicalName();
+      }
     }
     return createFactory(factoryClassName, prioritizeByteBuffer);
   }