From 9fac568e93b8605d0441fb814af1ddb3e15b991c Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Fri, 22 Sep 2023 12:31:34 +0200
Subject: [PATCH 01/18] Enable Java 21 in github test pipeline

---
 .github/workflows/pinot_tests.yml             | 37 ++++++++++++++++---
 .../segment/spi/memory/PinotDataBuffer.java   | 20 ++++++----
 2 files changed, 44 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml
index 727280bd2e3d..a4c1e027c25e 100644
--- a/.github/workflows/pinot_tests.yml
+++ b/.github/workflows/pinot_tests.yml
@@ -77,12 +77,19 @@ jobs:
       fail-fast: false
       matrix:
         testset: [ 1, 2 ]
-        java: [ 11, 17, 20 ]
+        java: [ 11, 17 ]
         distribution: [ "temurin" ]
+        prioritize_bytebuffer: [false]
+        include:
+          - distribution: corretto
+            java: 21
+          - distribution: corretto
+            java: 21
+            prioritize_bytebuffer: true
     name: Pinot Unit Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
     steps:
       - uses: actions/checkout@v3
-      - name: Set up JDK ${{ matrix.java }}
+      - name: Set up JDK ${{ matrix.java }}-${{ matrix.distribution }}
         uses: actions/setup-java@v3
         with:
           java-version: ${{ matrix.java }}
@@ -116,6 +123,7 @@ jobs:
         env:
           RUN_INTEGRATION_TESTS: false
           RUN_TEST_SET: ${{ matrix.testset }}
+          PINOT_OFFHEAP_PRIORITIZE_BYTEBUFFER: ${{ matrix.prioritize_bytebuffer }}
           GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
@@ -145,12 +153,19 @@ jobs:
       fail-fast: false
       matrix:
         testset: [ 1, 2 ]
-        java: [ 11, 17, 20 ]
+        java: [ 11, 17 ]
         distribution: [ "temurin" ]
+        prioritize_bytebuffer: [false]
+        include:
+          - distribution: corretto
+            java: 21
+          - distribution: corretto
+            java: 21
+            prioritize_bytebuffer: true
     name: Pinot Integration Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
     steps:
       - uses: actions/checkout@v3
-      - name: Set up JDK ${{ matrix.java }}
+      - name: Set up JDK ${{ matrix.java }}-${{ matrix.distribution }}
         uses: actions/setup-java@v3
         with:
           java-version: ${{ matrix.java }}
@@ -183,6 +198,7 @@ jobs:
         env:
           RUN_INTEGRATION_TESTS: true
           RUN_TEST_SET: ${{ matrix.testset }}
+          PINOT_OFFHEAP_PRIORITIZE_BYTEBUFFER: ${{ matrix.prioritize_bytebuffer }}
           GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
@@ -292,9 +308,16 @@ jobs:
       # Changed to false in order to improve coverage using unsafe buffers
       fail-fast: false
       matrix:
-        java: [ 11, 17, 20 ]
+        java: [ 11, 17 ]
         distribution: [ "temurin" ]
-    name: Pinot Quickstart on JDK ${{ matrix.java }}
+        prioritize_bytebuffer: [false]
+        include:
+          - distribution: corretto
+            java: 21
+          - distribution: corretto
+            java: 21
+            prioritize_bytebuffer: true
+    name: Pinot Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }}
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK ${{ matrix.java }}
@@ -314,4 +337,6 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-maven-
       - name: Quickstart on JDK ${{ matrix.java }}
+        env:
+          PINOT_OFFHEAP_PRIORITIZE_BYTEBUFFER: ${{ matrix.prioritize_bytebuffer }}
         run: .github/workflows/scripts/.pinot_quickstart.sh
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java
index 9bd36aeb38d4..002514f820d3 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java
@@ -67,6 +67,7 @@ public abstract class PinotDataBuffer implements Closeable {
   // With number of bytes less than this threshold, we get/put bytes one by one
   // With number of bytes more than this threshold, we create a ByteBuffer from the buffer and use bulk get/put method
   public static final int BULK_BYTES_PROCESSING_THRESHOLD = 10;
+  private static final String PRIORITIZE_BYTEBUFFER_ENV = "PINOT_OFFHEAP_PRIORITIZE_BYTEBUFFER";
 
   private static class BufferContext {
     enum Type {
@@ -152,17 +153,22 @@ public static PinotBufferFactory getFactory() {
   }
 
   public static PinotBufferFactory createDefaultFactory() {
-    return createDefaultFactory(true);
+    String prioritizeBbEnvValue = System.getenv(PRIORITIZE_BYTEBUFFER_ENV);
+    boolean prioritizeByteBuffer = prioritizeBbEnvValue == null || Boolean.parseBoolean(prioritizeBbEnvValue);
+    return createDefaultFactory(prioritizeByteBuffer);
   }
 
   public static PinotBufferFactory createDefaultFactory(boolean prioritizeByteBuffer) {
     String factoryClassName;
-    if (JavaVersion.VERSION < 16) {
-      LOGGER.info("Using LArray as buffer on JVM version {}", JavaVersion.VERSION);
-      factoryClassName = LArrayPinotBufferFactory.class.getCanonicalName();
-    } else {
-      LOGGER.info("Using Unsafe as buffer on JVM version {}", JavaVersion.VERSION);
-      factoryClassName = UnsafePinotBufferFactory.class.getCanonicalName();
+    factoryClassName = System.getenv("PINOT_BUFFER_LIBRARY");
+    if (factoryClassName == null) {
+      if (JavaVersion.VERSION < 16) {
+        LOGGER.info("Using LArray as buffer on JVM version {}", JavaVersion.VERSION);
+        factoryClassName = LArrayPinotBufferFactory.class.getCanonicalName();
+      } else {
+        LOGGER.info("Using Unsafe as buffer on JVM version {}", JavaVersion.VERSION);
+        factoryClassName = UnsafePinotBufferFactory.class.getCanonicalName();
+      }
     }
     return createFactory(factoryClassName, prioritizeByteBuffer);
   }

From 22dea3309556580b12b4e633adc000b5b76b50f1 Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Fri, 22 Sep 2023 12:51:17 +0200
Subject: [PATCH 02/18] Move scala dependencies to root pom

---
 pom.xml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/pom.xml b/pom.xml
index 4ab348ab1367..2b46c0aa200b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -229,6 +229,16 @@
         </plugins>
       </build>
     </profile>
+    <profile>
+      <id>scala-2.12</id>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+      </activation>
+      <properties>
+        <scala.version>2.12.18</scala.version>
+        <scala.compat.version>2.12</scala.compat.version>
+      </properties>
+    </profile>
 
     <profile>
       <id>scala-2.13</id>

From c9540a7761c05abd89b0f638ab6431b66fe6ad8f Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Mon, 25 Sep 2023 10:47:30 +0200
Subject: [PATCH 03/18] Move scala.version and scala.compat.version to default
 properties

---
 pom.xml | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/pom.xml b/pom.xml
index 2b46c0aa200b..011ebfaf0838 100644
--- a/pom.xml
+++ b/pom.xml
@@ -184,6 +184,10 @@
     <scala.version>2.12.18</scala.version>
     <scala.compat.version>2.12</scala.compat.version>
 
+    <!-- Configuration for Scala -->
+    <scala.version>2.12.18</scala.version>
+    <scala.compat.version>2.12</scala.compat.version>
+
     <flink.version>1.12.0</flink.version>
     <commons-configuration2.version>2.9.0</commons-configuration2.version>
   </properties>
@@ -229,16 +233,6 @@
         </plugins>
       </build>
     </profile>
-    <profile>
-      <id>scala-2.12</id>
-      <activation>
-        <activeByDefault>true</activeByDefault>
-      </activation>
-      <properties>
-        <scala.version>2.12.18</scala.version>
-        <scala.compat.version>2.12</scala.compat.version>
-      </properties>
-    </profile>
 
     <profile>
       <id>scala-2.13</id>

From 7abd34470d0d903d7ef97ca0efcbeebab0789d85 Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Mon, 25 Sep 2023 12:14:06 +0200
Subject: [PATCH 04/18] Run tests with corretto instead of temurin to simplify
 matrix

---
 .github/workflows/pinot_tests.yml | 34 ++++++++++++-------------------
 1 file changed, 13 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml
index a4c1e027c25e..b141043ac41d 100644
--- a/.github/workflows/pinot_tests.yml
+++ b/.github/workflows/pinot_tests.yml
@@ -77,15 +77,12 @@ jobs:
       fail-fast: false
       matrix:
         testset: [ 1, 2 ]
-        java: [ 11, 17 ]
-        distribution: [ "temurin" ]
+        java: [ 11, 17, 21 ]
+        distribution: [ "corretto" ]
         prioritize_bytebuffer: [false]
         include:
-          - distribution: corretto
-            java: 21
-          - distribution: corretto
-            java: 21
-            prioritize_bytebuffer: true
+          - java: 21
+          - prioritize_bytebuffer: true
     name: Pinot Unit Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
     steps:
       - uses: actions/checkout@v3
@@ -153,15 +150,12 @@ jobs:
       fail-fast: false
       matrix:
         testset: [ 1, 2 ]
-        java: [ 11, 17 ]
-        distribution: [ "temurin" ]
+        java: [ 11, 17, 21 ]
+        distribution: [ "corretto" ]
         prioritize_bytebuffer: [false]
         include:
-          - distribution: corretto
-            java: 21
-          - distribution: corretto
-            java: 21
-            prioritize_bytebuffer: true
+          - java: 21
+          - prioritize_bytebuffer: true
     name: Pinot Integration Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
     steps:
       - uses: actions/checkout@v3
@@ -308,15 +302,13 @@ jobs:
       # Changed to false in order to improve coverage using unsafe buffers
       fail-fast: false
       matrix:
-        java: [ 11, 17 ]
-        distribution: [ "temurin" ]
+        testset: [ 1, 2 ]
+        java: [ 11, 17, 21 ]
+        distribution: [ "corretto" ]
         prioritize_bytebuffer: [false]
         include:
-          - distribution: corretto
-            java: 21
-          - distribution: corretto
-            java: 21
-            prioritize_bytebuffer: true
+          - java: 21
+          - prioritize_bytebuffer: true
     name: Pinot Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }}
     steps:
       - uses: actions/checkout@v3

From 15a5fbd0258b88c92313be619eebe44be313a81c Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Mon, 25 Sep 2023 12:41:28 +0200
Subject: [PATCH 05/18] Increase Mockito version to be able to use it in Java
 21

---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 011ebfaf0838..a9558587e2d2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1142,13 +1142,13 @@
       <dependency>
         <groupId>org.mockito</groupId>
         <artifactId>mockito-core</artifactId>
-        <version>5.3.1</version>
+        <version>5.5.0</version>
         <scope>test</scope>
       </dependency>
       <dependency>
         <groupId>org.mockito</groupId>
         <artifactId>mockito-inline</artifactId>
-        <version>4.7.0</version>
+        <version>5.2.0</version>
         <scope>test</scope>
       </dependency>
       <dependency>

From c8822f117cf5a924c30d2dff1358d1788bd4617c Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Mon, 25 Sep 2023 14:14:33 +0200
Subject: [PATCH 06/18] Add -Dnet.bytebuddy.experimental=true on tests

---
 pom.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pom.xml b/pom.xml
index a9558587e2d2..1fd92be11047 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1499,6 +1499,7 @@
               --add-opens=java.base/java.util=ALL-UNNAMED
               --add-exports=java.base/jdk.internal.util.random=ALL-UNNAMED
               --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
+              -Dnet.bytebuddy.experimental=true
             </argLine>
           </configuration>
           <!-- Explicitly select the test provider, instead of relying on the classpath -->

From 847fda035e455b71951eee74bfa9ef7045a09d45 Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Mon, 25 Sep 2023 14:20:25 +0200
Subject: [PATCH 07/18] Correctly add prioritize_bytebuffer: true to the matrix

---
 .github/workflows/pinot_tests.yml | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml
index b141043ac41d..7cd93b6872d7 100644
--- a/.github/workflows/pinot_tests.yml
+++ b/.github/workflows/pinot_tests.yml
@@ -82,7 +82,13 @@ jobs:
         prioritize_bytebuffer: [false]
         include:
           - java: 21
-          - prioritize_bytebuffer: true
+            testset: 1
+            distribution: "corretto"
+            prioritize_bytebuffer: true
+          - java: 21
+            testset: 2
+            distribution: "corretto"
+            prioritize_bytebuffer: true
     name: Pinot Unit Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
     steps:
       - uses: actions/checkout@v3
@@ -155,7 +161,13 @@ jobs:
         prioritize_bytebuffer: [false]
         include:
           - java: 21
-          - prioritize_bytebuffer: true
+            testset: 1
+            distribution: "corretto"
+            prioritize_bytebuffer: true
+          - java: 21
+            testset: 2
+            distribution: "corretto"
+            prioritize_bytebuffer: true
     name: Pinot Integration Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
     steps:
       - uses: actions/checkout@v3
@@ -308,7 +320,13 @@ jobs:
         prioritize_bytebuffer: [false]
         include:
           - java: 21
-          - prioritize_bytebuffer: true
+            testset: 1
+            distribution: "corretto"
+            prioritize_bytebuffer: true
+          - java: 21
+            testset: 2
+            distribution: "corretto"
+            prioritize_bytebuffer: true
     name: Pinot Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }}
     steps:
       - uses: actions/checkout@v3

From 9d93b3970fd0aa7b9c3586fbc396bdf9430caa79 Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Mon, 25 Sep 2023 15:35:43 +0200
Subject: [PATCH 08/18] Use Spark 3.5.0 instead of 3.2.1 given that the former
 supports Java 21

---
 .../pinot-batch-ingestion-spark-3/pom.xml                   | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml
index 95f3fcdbc2bc..7f0e3c23f436 100644
--- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml
+++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml
@@ -36,9 +36,7 @@
   <properties>
     <pinot.root>${basedir}/../../..</pinot.root>
     <phase.prop>package</phase.prop>
-    <scala.major.version>2.12</scala.major.version>
     <spark.version>3.5.0</spark.version>
-    <scala.minor.version>2.12.15</scala.minor.version>
     <commons-lang3.version>3.11</commons-lang3.version>
   </properties>
 
@@ -50,7 +48,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
-      <artifactId>spark-core_${scala.major.version}</artifactId>
+      <artifactId>spark-core_${scala.compat.version}</artifactId>
       <version>${spark.version}</version>
       <scope>provided</scope>
       <exclusions>
@@ -99,7 +97,7 @@
     <dependency>
       <groupId>org.scala-lang</groupId>
       <artifactId>scala-library</artifactId>
-      <version>${scala.minor.version}</version>
+      <version>${scala.version}</version>
       <scope>provided</scope>
     </dependency>
     <dependency>

From 16a45c10ecc2c94f6b9010f2c5110b8c3bf00246 Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Tue, 26 Sep 2023 19:29:14 +0200
Subject: [PATCH 09/18] Go back to spark 3.2.1 instead of 3.5.0 but do not
 compile it with Java 21

---
 .../pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml
index 7f0e3c23f436..de00603b586f 100644
--- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml
+++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml
@@ -36,7 +36,7 @@
   <properties>
     <pinot.root>${basedir}/../../..</pinot.root>
     <phase.prop>package</phase.prop>
-    <spark.version>3.5.0</spark.version>
+    <spark.version>3.2.1</spark.version>
     <commons-lang3.version>3.11</commons-lang3.version>
   </properties>
 

From 027f8cd41ccac7903741559fc87ccd90799e7d27 Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Thu, 28 Sep 2023 10:08:29 +0200
Subject: [PATCH 10/18] use prioritize_bytebuffer in Java 11

---
 .github/workflows/pinot_tests.yml | 31 ++++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml
index 7cd93b6872d7..d3ed60d735fe 100644
--- a/.github/workflows/pinot_tests.yml
+++ b/.github/workflows/pinot_tests.yml
@@ -81,14 +81,22 @@ jobs:
         distribution: [ "corretto" ]
         prioritize_bytebuffer: [false]
         include:
-          - java: 21
+          - java: 11
             testset: 1
             distribution: "corretto"
             prioritize_bytebuffer: true
-          - java: 21
+          - java: 11
             testset: 2
             distribution: "corretto"
             prioritize_bytebuffer: true
+          - java: 21
+            testset: 1
+            distribution: "corretto"
+            prioritize_bytebuffer: false
+          - java: 21
+            testset: 2
+            distribution: "corretto"
+            prioritize_bytebuffer: false
     name: Pinot Unit Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
     steps:
       - uses: actions/checkout@v3
@@ -160,14 +168,22 @@ jobs:
         distribution: [ "corretto" ]
         prioritize_bytebuffer: [false]
         include:
-          - java: 21
+          - java: 11
             testset: 1
             distribution: "corretto"
             prioritize_bytebuffer: true
-          - java: 21
+          - java: 11
             testset: 2
             distribution: "corretto"
             prioritize_bytebuffer: true
+          - java: 21
+            testset: 1
+            distribution: "corretto"
+            prioritize_bytebuffer: false
+          - java: 21
+            testset: 2
+            distribution: "corretto"
+            prioritize_bytebuffer: false
     name: Pinot Integration Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
     steps:
       - uses: actions/checkout@v3
@@ -314,19 +330,16 @@ jobs:
       # Changed to false in order to improve coverage using unsafe buffers
       fail-fast: false
       matrix:
-        testset: [ 1, 2 ]
         java: [ 11, 17, 21 ]
         distribution: [ "corretto" ]
         prioritize_bytebuffer: [false]
         include:
-          - java: 21
-            testset: 1
+          - java: 11
             distribution: "corretto"
             prioritize_bytebuffer: true
           - java: 21
-            testset: 2
             distribution: "corretto"
-            prioritize_bytebuffer: true
+            prioritize_bytebuffer: false
     name: Pinot Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }}
     steps:
       - uses: actions/checkout@v3

From 8a820ce25e3586bfaea7cf0787bc64c6402b1a0c Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Thu, 28 Sep 2023 13:42:38 +0200
Subject: [PATCH 11/18] Improve job names

---
 .github/workflows/pinot_tests.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml
index d3ed60d735fe..3e6a69aab6c6 100644
--- a/.github/workflows/pinot_tests.yml
+++ b/.github/workflows/pinot_tests.yml
@@ -97,7 +97,7 @@ jobs:
             testset: 2
             distribution: "corretto"
             prioritize_bytebuffer: false
-    name: Pinot Unit Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
+    name: Pinot Unit Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}}) with bytebuffers:${{matrix.prioritize_bytebuffer}}
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK ${{ matrix.java }}-${{ matrix.distribution }}
@@ -184,7 +184,7 @@ jobs:
             testset: 2
             distribution: "corretto"
             prioritize_bytebuffer: false
-    name: Pinot Integration Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
+    name: Pinot Integration Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}}) with bytebuffers:${{matrix.prioritize_bytebuffer}}
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK ${{ matrix.java }}-${{ matrix.distribution }}
@@ -340,7 +340,7 @@ jobs:
           - java: 21
             distribution: "corretto"
             prioritize_bytebuffer: false
-    name: Pinot Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }}
+    name: Pinot Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }} with bytebuffers:${{matrix.prioritize_bytebuffer}}
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK ${{ matrix.java }}

From cdae774e97f8bfbe6ee8efe3c0a60dadc705545c Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Thu, 28 Sep 2023 17:55:31 +0200
Subject: [PATCH 12/18] Run without byte buffer in Java 11 (LArray) and 17
 (Unsafe buffers)

---
 .github/workflows/pinot_tests.yml | 31 ++++++-------------------------
 1 file changed, 6 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml
index 3e6a69aab6c6..8df3aac335c5 100644
--- a/.github/workflows/pinot_tests.yml
+++ b/.github/workflows/pinot_tests.yml
@@ -77,18 +77,10 @@ jobs:
       fail-fast: false
       matrix:
         testset: [ 1, 2 ]
-        java: [ 11, 17, 21 ]
+        java: [ 11, 17 ]
         distribution: [ "corretto" ]
-        prioritize_bytebuffer: [false]
+        prioritize_bytebuffer: [false, true]
         include:
-          - java: 11
-            testset: 1
-            distribution: "corretto"
-            prioritize_bytebuffer: true
-          - java: 11
-            testset: 2
-            distribution: "corretto"
-            prioritize_bytebuffer: true
           - java: 21
             testset: 1
             distribution: "corretto"
@@ -164,18 +156,10 @@ jobs:
       fail-fast: false
       matrix:
         testset: [ 1, 2 ]
-        java: [ 11, 17, 21 ]
+        java: [ 11, 17 ]
         distribution: [ "corretto" ]
-        prioritize_bytebuffer: [false]
+        prioritize_bytebuffer: [false, true]
         include:
-          - java: 11
-            testset: 1
-            distribution: "corretto"
-            prioritize_bytebuffer: true
-          - java: 11
-            testset: 2
-            distribution: "corretto"
-            prioritize_bytebuffer: true
           - java: 21
             testset: 1
             distribution: "corretto"
@@ -330,13 +314,10 @@ jobs:
       # Changed to false in order to improve coverage using unsafe buffers
       fail-fast: false
       matrix:
-        java: [ 11, 17, 21 ]
+        java: [ 11, 17 ]
         distribution: [ "corretto" ]
-        prioritize_bytebuffer: [false]
+        prioritize_bytebuffer: [false, true]
         include:
-          - java: 11
-            distribution: "corretto"
-            prioritize_bytebuffer: true
           - java: 21
             distribution: "corretto"
             prioritize_bytebuffer: false

From 261551acc68f91aaf2ac8deaa9b11249d9ab5dbf Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Thu, 28 Sep 2023 10:55:13 +0200
Subject: [PATCH 13/18] Upgrade spark from 3.2 to 3.5

---
 .../pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml
index de00603b586f..7f0e3c23f436 100644
--- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml
+++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml
@@ -36,7 +36,7 @@
   <properties>
     <pinot.root>${basedir}/../../..</pinot.root>
     <phase.prop>package</phase.prop>
-    <spark.version>3.2.1</spark.version>
+    <spark.version>3.5.0</spark.version>
     <commons-lang3.version>3.11</commons-lang3.version>
   </properties>
 

From 0cf0495d822176a1d94ebaf979f486361a0a5b3b Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Tue, 3 Oct 2023 09:23:16 +0200
Subject: [PATCH 14/18] Use skip_bytebuffer instead of prioritize_bytebuffer

---
 .github/workflows/pinot_tests.yml             | 28 +++++++++----------
 .../segment/spi/memory/PinotDataBuffer.java   |  6 ++--
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml
index 8df3aac335c5..16b2e533d4a0 100644
--- a/.github/workflows/pinot_tests.yml
+++ b/.github/workflows/pinot_tests.yml
@@ -79,17 +79,17 @@ jobs:
         testset: [ 1, 2 ]
         java: [ 11, 17 ]
         distribution: [ "corretto" ]
-        prioritize_bytebuffer: [false, true]
+        skip_bytebuffer: [false, true]
         include:
           - java: 21
             testset: 1
             distribution: "corretto"
-            prioritize_bytebuffer: false
+            skip_bytebuffer: true
           - java: 21
             testset: 2
             distribution: "corretto"
-            prioritize_bytebuffer: false
-    name: Pinot Unit Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}}) with bytebuffers:${{matrix.prioritize_bytebuffer}}
+            skip_bytebuffer: true
+    name: Pinot Unit Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}}) with skip bytebuffers:${{matrix.skip_bytebuffer}}
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK ${{ matrix.java }}-${{ matrix.distribution }}
@@ -126,7 +126,7 @@ jobs:
         env:
           RUN_INTEGRATION_TESTS: false
           RUN_TEST_SET: ${{ matrix.testset }}
-          PINOT_OFFHEAP_PRIORITIZE_BYTEBUFFER: ${{ matrix.prioritize_bytebuffer }}
+          PINOT_OFFHEAP_SKIP_BYTEBUFFER: ${{ matrix.skip_bytebuffer }}
           GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
@@ -158,17 +158,17 @@ jobs:
         testset: [ 1, 2 ]
         java: [ 11, 17 ]
         distribution: [ "corretto" ]
-        prioritize_bytebuffer: [false, true]
+        skip_bytebuffer: [false, true]
         include:
           - java: 21
             testset: 1
             distribution: "corretto"
-            prioritize_bytebuffer: false
+            skip_bytebuffer: true
           - java: 21
             testset: 2
             distribution: "corretto"
-            prioritize_bytebuffer: false
-    name: Pinot Integration Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}}) with bytebuffers:${{matrix.prioritize_bytebuffer}}
+            skip_bytebuffer: true
+    name: Pinot Integration Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}}) with skip bytebuffers:${{matrix.skip_bytebuffer}}
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK ${{ matrix.java }}-${{ matrix.distribution }}
@@ -204,7 +204,7 @@ jobs:
         env:
           RUN_INTEGRATION_TESTS: true
           RUN_TEST_SET: ${{ matrix.testset }}
-          PINOT_OFFHEAP_PRIORITIZE_BYTEBUFFER: ${{ matrix.prioritize_bytebuffer }}
+          PINOT_OFFHEAP_SKIP_BYTEBUFFER: ${{ matrix.skip_bytebuffer }}
           GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
@@ -316,12 +316,12 @@ jobs:
       matrix:
         java: [ 11, 17 ]
         distribution: [ "corretto" ]
-        prioritize_bytebuffer: [false, true]
+        skip_bytebuffer: [false, true]
         include:
           - java: 21
             distribution: "corretto"
-            prioritize_bytebuffer: false
-    name: Pinot Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }} with bytebuffers:${{matrix.prioritize_bytebuffer}}
+            skip_bytebuffer: false
+    name: Pinot Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }} with skip bytebuffers:${{matrix.skip_bytebuffer}}
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK ${{ matrix.java }}
@@ -342,5 +342,5 @@ jobs:
             ${{ runner.os }}-maven-
       - name: Quickstart on JDK ${{ matrix.java }}
         env:
-          PINOT_OFFHEAP_PRIORITIZE_BYTEBUFFER: ${{ matrix.prioritize_bytebuffer }}
+          PINOT_OFFHEAP_SKIP_BYTEBUFFER: ${{ matrix.skip_bytebuffer }}
         run: .github/workflows/scripts/.pinot_quickstart.sh
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java
index 002514f820d3..4b2a28519562 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java
@@ -67,7 +67,7 @@ public abstract class PinotDataBuffer implements Closeable {
   // With number of bytes less than this threshold, we get/put bytes one by one
   // With number of bytes more than this threshold, we create a ByteBuffer from the buffer and use bulk get/put method
   public static final int BULK_BYTES_PROCESSING_THRESHOLD = 10;
-  private static final String PRIORITIZE_BYTEBUFFER_ENV = "PINOT_OFFHEAP_PRIORITIZE_BYTEBUFFER";
+  private static final String SKIP_BYTEBUFFER_ENV = "PINOT_OFFHEAP_SKIP_BYTEBUFFER";
 
   private static class BufferContext {
     enum Type {
@@ -153,8 +153,8 @@ public static PinotBufferFactory getFactory() {
   }
 
   public static PinotBufferFactory createDefaultFactory() {
-    String prioritizeBbEnvValue = System.getenv(PRIORITIZE_BYTEBUFFER_ENV);
-    boolean prioritizeByteBuffer = prioritizeBbEnvValue == null || Boolean.parseBoolean(prioritizeBbEnvValue);
+    String skipBbEnvValue = System.getenv(SKIP_BYTEBUFFER_ENV);
+    boolean prioritizeByteBuffer = !Boolean.parseBoolean(skipBbEnvValue);
     return createDefaultFactory(prioritizeByteBuffer);
   }
 

From 266f071856d202bc9104bbbfe869b3c68ec6824c Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Wed, 4 Oct 2023 12:49:12 +0200
Subject: [PATCH 15/18] Run tests with Java 11, Java 21 and Java 21 without
 bytebuffers

---
 .github/workflows/pinot_tests.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml
index 16b2e533d4a0..d3ac0970ac5d 100644
--- a/.github/workflows/pinot_tests.yml
+++ b/.github/workflows/pinot_tests.yml
@@ -77,9 +77,9 @@ jobs:
       fail-fast: false
       matrix:
         testset: [ 1, 2 ]
-        java: [ 11, 17 ]
+        java: [ 11, 21 ]
         distribution: [ "corretto" ]
-        skip_bytebuffer: [false, true]
+        skip_bytebuffer: [false]
         include:
           - java: 21
             testset: 1
@@ -156,9 +156,9 @@ jobs:
       fail-fast: false
       matrix:
         testset: [ 1, 2 ]
-        java: [ 11, 17 ]
+        java: [ 11, 21 ]
         distribution: [ "corretto" ]
-        skip_bytebuffer: [false, true]
+        skip_bytebuffer: [false]
         include:
           - java: 21
             testset: 1
@@ -314,13 +314,13 @@ jobs:
       # Changed to false in order to improve coverage using unsafe buffers
       fail-fast: false
       matrix:
-        java: [ 11, 17 ]
+        java: [ 11, 21 ]
         distribution: [ "corretto" ]
-        skip_bytebuffer: [false, true]
+        skip_bytebuffer: [false]
         include:
           - java: 21
             distribution: "corretto"
-            skip_bytebuffer: false
+            skip_bytebuffer: true
     name: Pinot Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }} with skip bytebuffers:${{matrix.skip_bytebuffer}}
     steps:
       - uses: actions/checkout@v3

From 43bf59b4bcfb392144857ee805820cb581eabbde Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Mon, 16 Oct 2023 10:37:42 +0200
Subject: [PATCH 16/18] Read PINOT_OFFHEAP_SKIP_BYTEBUFFER env variable only
 once

---
 .../pinot/segment/spi/memory/PinotDataBuffer.java      | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java
index 4b2a28519562..0895a1782bc3 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java
@@ -68,6 +68,12 @@ public abstract class PinotDataBuffer implements Closeable {
   // With number of bytes more than this threshold, we create a ByteBuffer from the buffer and use bulk get/put method
   public static final int BULK_BYTES_PROCESSING_THRESHOLD = 10;
   private static final String SKIP_BYTEBUFFER_ENV = "PINOT_OFFHEAP_SKIP_BYTEBUFFER";
+  private static final boolean DEFAULT_PRIORITIZE_BYTE_BUFFER;
+
+  static {
+     String skipBbEnvValue = System.getenv(SKIP_BYTEBUFFER_ENV);
+     DEFAULT_PRIORITIZE_BYTE_BUFFER = !Boolean.parseBoolean(skipBbEnvValue);
+  }
 
   private static class BufferContext {
     enum Type {
@@ -153,9 +159,7 @@ public static PinotBufferFactory getFactory() {
   }
 
   public static PinotBufferFactory createDefaultFactory() {
-    String skipBbEnvValue = System.getenv(SKIP_BYTEBUFFER_ENV);
-    boolean prioritizeByteBuffer = !Boolean.parseBoolean(skipBbEnvValue);
-    return createDefaultFactory(prioritizeByteBuffer);
+    return createDefaultFactory(DEFAULT_PRIORITIZE_BYTE_BUFFER);
   }
 
   public static PinotBufferFactory createDefaultFactory(boolean prioritizeByteBuffer) {

From 7415f372fc81a63cf334a58447fa4f2dc7fee3eb Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Wed, 18 Oct 2023 08:36:18 +0200
Subject: [PATCH 17/18] Add a codecov flag to indicate which buffers are used

---
 .github/workflows/pinot_tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml
index d3ac0970ac5d..ae0432f96e5d 100644
--- a/.github/workflows/pinot_tests.yml
+++ b/.github/workflows/pinot_tests.yml
@@ -143,7 +143,7 @@ jobs:
         continue-on-error: true
         timeout-minutes: 5
         with:
-          flags: unittests,unittests${{ matrix.testset }},${{matrix.distribution}},java-${{matrix.java}}
+          flags: unittests,unittests${{ matrix.testset }},${{matrix.distribution}},java-${{matrix.java}},skip-bytebuffers-${{matrix.skip_bytebuffer}}
           name: codecov-unit-tests
           fail_ci_if_error: false
           verbose: true
@@ -221,7 +221,7 @@ jobs:
         continue-on-error: true
         timeout-minutes: 5
         with:
-          flags: integration,integration${{ matrix.testset }},${{matrix.distribution}},java-${{matrix.java}}
+          flags: integration,integration${{ matrix.testset }},${{matrix.distribution}},java-${{matrix.java}},skip-bytebuffers-${{matrix.skip_bytebuffer}}
           name: codecov-integration-tests
           fail_ci_if_error: false
           verbose: true

From 6b3b853dcd87402dd969e8f614b9bc3f9b7842e5 Mon Sep 17 00:00:00 2001
From: Gonzalo Ortiz <gortiz@users.noreply.github.com>
Date: Wed, 18 Oct 2023 08:46:44 +0200
Subject: [PATCH 18/18] Change from corretto to temurin

---
 .github/workflows/pinot_tests.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml
index ae0432f96e5d..0417d3f92727 100644
--- a/.github/workflows/pinot_tests.yml
+++ b/.github/workflows/pinot_tests.yml
@@ -78,16 +78,16 @@ jobs:
       matrix:
         testset: [ 1, 2 ]
         java: [ 11, 21 ]
-        distribution: [ "corretto" ]
+        distribution: [ "temurin" ]
         skip_bytebuffer: [false]
         include:
           - java: 21
             testset: 1
-            distribution: "corretto"
+            distribution: "temurin"
             skip_bytebuffer: true
           - java: 21
             testset: 2
-            distribution: "corretto"
+            distribution: "temurin"
             skip_bytebuffer: true
     name: Pinot Unit Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}}) with skip bytebuffers:${{matrix.skip_bytebuffer}}
     steps:
@@ -157,16 +157,16 @@ jobs:
       matrix:
         testset: [ 1, 2 ]
         java: [ 11, 21 ]
-        distribution: [ "corretto" ]
+        distribution: [ "temurin" ]
         skip_bytebuffer: [false]
         include:
           - java: 21
             testset: 1
-            distribution: "corretto"
+            distribution: "temurin"
             skip_bytebuffer: true
           - java: 21
             testset: 2
-            distribution: "corretto"
+            distribution: "temurin"
             skip_bytebuffer: true
     name: Pinot Integration Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}}) with skip bytebuffers:${{matrix.skip_bytebuffer}}
     steps:
@@ -315,11 +315,11 @@ jobs:
       fail-fast: false
       matrix:
         java: [ 11, 21 ]
-        distribution: [ "corretto" ]
+        distribution: [ "temurin" ]
         skip_bytebuffer: [false]
         include:
           - java: 21
-            distribution: "corretto"
+            distribution: "temurin"
             skip_bytebuffer: true
     name: Pinot Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }} with skip bytebuffers:${{matrix.skip_bytebuffer}}
     steps: