Skip to content

Java 21 pipeline #11672

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xiangfu0 merged 18 commits into from
Oct 19, 2023
Merged

Java 21 pipeline #11672

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
9fac568
Enable Java 21 in github test pipeline
gortiz Sep 22, 2023
22dea33
Move scala dependencies to root pom
gortiz Sep 22, 2023
c9540a7
Move scala.version and scala.compat.version to default properties
gortiz Sep 25, 2023
7abd344
Run tests with corretto instead of temurin to simplify matrix
gortiz Sep 25, 2023
15a5fbd
Increase Mockito version to be able to use it in Java 21
gortiz Sep 25, 2023
c8822f1
Add -Dnet.bytebuddy.experimental=true on tests
gortiz Sep 25, 2023
847fda0
Correctly add prioritize_bytebuffer: true to the matrix
gortiz Sep 25, 2023
9d93b39
Use Spark 3.5.0 instead of 3.2.1 given that the former supports Java 21
gortiz Sep 25, 2023
16a45c1
Go back to spark 3.2.1 instead of 3.5.0 but do not compile it with Ja…
gortiz Sep 26, 2023
027f8cd
use prioritize_bytebuffer in Java 11
gortiz Sep 28, 2023
8a820ce
Improve job names
gortiz Sep 28, 2023
cdae774
Run without byte buffer in Java 11 (LArray) and 17 (Unsafe buffers)
gortiz Sep 28, 2023
261551a
Upgrade spark from 3.2 to 3.5
gortiz Sep 28, 2023
0cf0495
Use skip_bytebuffer instead of prioritize_bytebuffer
gortiz Oct 3, 2023
266f071
Run tests with Java 11, Java 21 and Java 21 without bytebuffers
gortiz Oct 4, 2023
43bf59b
Read PINOT_OFFHEAP_SKIP_BYTEBUFFER env variable only once
gortiz Oct 16, 2023
7415f37
Add a codecov flag to indicate which buffers are used
gortiz Oct 18, 2023
6b3b853
Change from corretto to temurin
gortiz Oct 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 39 additions & 10 deletions .github/workflows/pinot_tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,12 +77,22 @@ jobs:
fail-fast: false
matrix:
testset: [ 1, 2 ]
java: [ 11, 17, 20 ]
java: [ 11, 21 ]
distribution: [ "temurin" ]
name: Pinot Unit Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
skip_bytebuffer: [false]
include:
- java: 21
testset: 1
distribution: "temurin"
skip_bytebuffer: true
- java: 21
testset: 2
distribution: "temurin"
skip_bytebuffer: true
name: Pinot Unit Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}}) with skip bytebuffers:${{matrix.skip_bytebuffer}}
steps:
- uses: actions/checkout@v3
- name: Set up JDK ${{ matrix.java }}
- name: Set up JDK ${{ matrix.java }}-${{ matrix.distribution }}
uses: actions/setup-java@v3
with:
java-version: ${{ matrix.java }}
Expand Down Expand Up @@ -116,6 +126,7 @@ jobs:
env:
RUN_INTEGRATION_TESTS: false
RUN_TEST_SET: ${{ matrix.testset }}
PINOT_OFFHEAP_SKIP_BYTEBUFFER: ${{ matrix.skip_bytebuffer }}
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
MAVEN_OPTS: >
-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
Expand All @@ -132,7 +143,7 @@ jobs:
continue-on-error: true
timeout-minutes: 5
with:
flags: unittests,unittests${{ matrix.testset }},${{matrix.distribution}},java-${{matrix.java}}
flags: unittests,unittests${{ matrix.testset }},${{matrix.distribution}},java-${{matrix.java}},skip-bytebuffers-${{matrix.skip_bytebuffer}}
name: codecov-unit-tests
fail_ci_if_error: false
verbose: true
Expand All @@ -145,12 +156,22 @@ jobs:
fail-fast: false
matrix:
testset: [ 1, 2 ]
java: [ 11, 17, 20 ]
java: [ 11, 21 ]
distribution: [ "temurin" ]
name: Pinot Integration Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
skip_bytebuffer: [false]
include:
- java: 21
testset: 1
distribution: "temurin"
skip_bytebuffer: true
- java: 21
testset: 2
distribution: "temurin"
skip_bytebuffer: true
name: Pinot Integration Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}}) with skip bytebuffers:${{matrix.skip_bytebuffer}}
steps:
- uses: actions/checkout@v3
- name: Set up JDK ${{ matrix.java }}
- name: Set up JDK ${{ matrix.java }}-${{ matrix.distribution }}
uses: actions/setup-java@v3
with:
java-version: ${{ matrix.java }}
Expand Down Expand Up @@ -183,6 +204,7 @@ jobs:
env:
RUN_INTEGRATION_TESTS: true
RUN_TEST_SET: ${{ matrix.testset }}
PINOT_OFFHEAP_SKIP_BYTEBUFFER: ${{ matrix.skip_bytebuffer }}
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
MAVEN_OPTS: >
-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
Expand All @@ -199,7 +221,7 @@ jobs:
continue-on-error: true
timeout-minutes: 5
with:
flags: integration,integration${{ matrix.testset }},${{matrix.distribution}},java-${{matrix.java}}
flags: integration,integration${{ matrix.testset }},${{matrix.distribution}},java-${{matrix.java}},skip-bytebuffers-${{matrix.skip_bytebuffer}}
name: codecov-integration-tests
fail_ci_if_error: false
verbose: true
Expand Down Expand Up @@ -292,9 +314,14 @@ jobs:
# Changed to false in order to improve coverage using unsafe buffers
fail-fast: false
matrix:
java: [ 11, 17, 20 ]
java: [ 11, 21 ]
distribution: [ "temurin" ]
name: Pinot Quickstart on JDK ${{ matrix.java }}
skip_bytebuffer: [false]
include:
- java: 21
distribution: "temurin"
skip_bytebuffer: true
name: Pinot Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }} with skip bytebuffers:${{matrix.skip_bytebuffer}}
steps:
- uses: actions/checkout@v3
- name: Set up JDK ${{ matrix.java }}
Expand All @@ -314,4 +341,6 @@ jobs:
restore-keys: |
${{ runner.os }}-maven-
- name: Quickstart on JDK ${{ matrix.java }}
env:
PINOT_OFFHEAP_SKIP_BYTEBUFFER: ${{ matrix.skip_bytebuffer }}
run: .github/workflows/scripts/.pinot_quickstart.sh
6 changes: 2 additions & 4 deletions pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,7 @@
<properties>
<pinot.root>${basedir}/../../..</pinot.root>
<phase.prop>package</phase.prop>
<scala.major.version>2.12</scala.major.version>
<spark.version>3.5.0</spark.version>
<scala.minor.version>2.12.15</scala.minor.version>
<commons-lang3.version>3.11</commons-lang3.version>
</properties>

Expand All @@ -50,7 +48,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.major.version}</artifactId>
<artifactId>spark-core_${scala.compat.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
<exclusions>
Expand Down Expand Up @@ -99,7 +97,7 @@
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.minor.version}</version>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
Expand Down
24 changes: 17 additions & 7 deletions pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotDataBuffer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,13 @@ public abstract class PinotDataBuffer implements Closeable {
// With number of bytes less than this threshold, we get/put bytes one by one
// With number of bytes more than this threshold, we create a ByteBuffer from the buffer and use bulk get/put method
public static final int BULK_BYTES_PROCESSING_THRESHOLD = 10;
private static final String SKIP_BYTEBUFFER_ENV = "PINOT_OFFHEAP_SKIP_BYTEBUFFER";
private static final boolean DEFAULT_PRIORITIZE_BYTE_BUFFER;

static {
String skipBbEnvValue = System.getenv(SKIP_BYTEBUFFER_ENV);
DEFAULT_PRIORITIZE_BYTE_BUFFER = !Boolean.parseBoolean(skipBbEnvValue);
}

private static class BufferContext {
enum Type {
Expand Down Expand Up @@ -152,17 +159,20 @@ public static PinotBufferFactory getFactory() {
}

public static PinotBufferFactory createDefaultFactory() {
return createDefaultFactory(true);
return createDefaultFactory(DEFAULT_PRIORITIZE_BYTE_BUFFER);
}

public static PinotBufferFactory createDefaultFactory(boolean prioritizeByteBuffer) {
String factoryClassName;
if (JavaVersion.VERSION < 16) {
LOGGER.info("Using LArray as buffer on JVM version {}", JavaVersion.VERSION);
factoryClassName = LArrayPinotBufferFactory.class.getCanonicalName();
} else {
LOGGER.info("Using Unsafe as buffer on JVM version {}", JavaVersion.VERSION);
factoryClassName = UnsafePinotBufferFactory.class.getCanonicalName();
factoryClassName = System.getenv("PINOT_BUFFER_LIBRARY");
if (factoryClassName == null) {
if (JavaVersion.VERSION < 16) {
LOGGER.info("Using LArray as buffer on JVM version {}", JavaVersion.VERSION);
factoryClassName = LArrayPinotBufferFactory.class.getCanonicalName();
} else {
LOGGER.info("Using Unsafe as buffer on JVM version {}", JavaVersion.VERSION);
factoryClassName = UnsafePinotBufferFactory.class.getCanonicalName();
}
}
return createFactory(factoryClassName, prioritizeByteBuffer);
}
Expand Down
9 changes: 7 additions & 2 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,10 @@
<scala.version>2.12.18</scala.version>
<scala.compat.version>2.12</scala.compat.version>

<!-- Configuration for Scala -->
<scala.version>2.12.18</scala.version>
<scala.compat.version>2.12</scala.compat.version>

<flink.version>1.12.0</flink.version>
<commons-configuration2.version>2.9.0</commons-configuration2.version>
</properties>
Expand Down Expand Up @@ -1138,13 +1142,13 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>5.3.1</version>
<version>5.5.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
<version>4.7.0</version>
<version>5.2.0</version>
<scope>test</scope>
</dependency>
<dependency>
Expand Down Expand Up @@ -1495,6 +1499,7 @@
--add-opens=java.base/java.util=ALL-UNNAMED
--add-exports=java.base/jdk.internal.util.random=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
-Dnet.bytebuddy.experimental=true
</argLine>
</configuration>
<!-- Explicitly select the test provider, instead of relying on the classpath -->
Expand Down