From 82d4947b9b0be8355047645a0177af4b9ad04207 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 11 Nov 2024 09:51:28 +0530 Subject: [PATCH 01/61] Remove jdk8 from CI layer --- .github/workflows/codeql.yml | 2 +- .github/workflows/cron-job-its.yml | 22 +++++++------- .github/workflows/revised-its.yml | 12 ++++---- .github/workflows/standard-its.yml | 30 +++++++++---------- .github/workflows/static-checks.yml | 26 ++++++++-------- .../unit-and-integration-tests-unified.yml | 8 ++--- 6 files changed, 50 insertions(+), 50 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7cfeb63cbff0..8395f2839fdd 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -40,7 +40,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'zulu' - java-version: '8' + java-version: '11' cache: 'maven' # Initializes the CodeQL tools for scanning. diff --git a/.github/workflows/cron-job-its.yml b/.github/workflows/cron-job-its.yml index aa3f28382884..095b04c345a2 100644 --- a/.github/workflows/cron-job-its.yml +++ b/.github/workflows/cron-job-its.yml @@ -28,7 +28,7 @@ on: jobs: build: if: (github.event_name == 'schedule' && github.repository == 'apache/druid') - name: build (jdk8) + name: build (jdk11) runs-on: ubuntu-latest steps: - name: Checkout branch @@ -37,7 +37,7 @@ jobs: - name: setup java uses: actions/setup-java@v4 with: - java-version: '8' + java-version: '11' distribution: 'zulu' - name: Cache Maven m2 repository @@ -60,8 +60,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml needs: build with: - build_jdk: 8 - runtime_jdk: 11 + build_jdk: 11 + runtime_jdk: 17 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: middleManager group: ${{ matrix.testing_group }} @@ -74,8 +74,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml needs: build with: - build_jdk: 8 - runtime_jdk: 11 + build_jdk: 11 + runtime_jdk: 17 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: indexer group: ${{ matrix.testing_group }} @@ -88,8 +88,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml needs: build with: - build_jdk: 8 - runtime_jdk: 11 + build_jdk: 11 + runtime_jdk: 17 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: middleManager override_config_path: ./environment-configs/test-groups/prepopulated-data @@ -103,8 +103,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml needs: build with: - build_jdk: 8 - runtime_jdk: 11 + build_jdk: 11 + runtime_jdk: 17 testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties use_indexer: ${{ matrix.indexer }} group: other @@ -122,7 +122,7 @@ jobs: - name: setup java uses: actions/setup-java@v4 with: - java-version: '8' + java-version: '11' distribution: 'zulu' cache: maven diff --git a/.github/workflows/revised-its.yml b/.github/workflows/revised-its.yml index 412a3e469737..a765011e32ee 100644 --- a/.github/workflows/revised-its.yml +++ b/.github/workflows/revised-its.yml @@ -66,8 +66,8 @@ jobs: strategy: fail-fast: false matrix: - #jdk: [8, 11, 17] - jdk: [8] + #jdk: [11, 17] + jdk: [11] it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat, Security, Query] #indexer: [indexer, middleManager] indexer: [middleManager] @@ -86,8 +86,8 @@ jobs: uses: ./.github/workflows/reusable-revised-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 8 - runtime_jdk: 11 + build_jdk: 11 + runtime_jdk: 17 use_indexer: middleManager script: ./it.sh github S3DeepStorage it: S3DeepStorage @@ -103,8 +103,8 @@ jobs: uses: ./.github/workflows/reusable-revised-its.yml if: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' && (needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true') }} with: - build_jdk: 8 - runtime_jdk: 8 + build_jdk: 11 + runtime_jdk: 11 use_indexer: middleManager script: ./it.sh github BackwardCompatibilityMain it: BackwardCompatibilityMain diff --git a/.github/workflows/standard-its.yml b/.github/workflows/standard-its.yml index 0083e952ea3d..9199b9644137 100644 --- a/.github/workflows/standard-its.yml +++ b/.github/workflows/standard-its.yml @@ -51,8 +51,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 8 - runtime_jdk: 8 + build_jdk: 11 + runtime_jdk: 11 testing_groups: -Dgroups=${{ matrix.testing_group }} override_config_path: ./environment-configs/test-groups/prepopulated-data use_indexer: middleManager @@ -67,8 +67,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 8 - runtime_jdk: 8 + build_jdk: 11 + runtime_jdk: 11 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: indexer group: ${{ matrix.testing_group }} @@ -82,8 +82,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 8 - runtime_jdk: 8 + build_jdk: 11 + runtime_jdk: 11 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: middleManager override_config_path: ./environment-configs/test-groups/prepopulated-data @@ -94,7 +94,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [8, 17, 21] + jdk: [11, 17, 21] uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: @@ -115,8 +115,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 8 - runtime_jdk: 8 + build_jdk: 11 + runtime_jdk: 11 testing_groups: -Dgroups=shuffle-deep-store use_indexer: ${{ matrix.indexer }} override_config_path: ./environment-configs/test-groups/shuffle-deep-store @@ -127,8 +127,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 8 - runtime_jdk: 8 + build_jdk: 11 + runtime_jdk: 11 testing_groups: -Dgroups=custom-coordinator-duties use_indexer: middleManager override_config_path: ./environment-configs/test-groups/custom-coordinator-duties @@ -136,7 +136,7 @@ jobs: integration-k8s-leadership-tests: needs: changes - name: (Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITNestedQueryPushDownTest integration test + name: (Compile=openjdk11, Run=openjdk11, Cluster Build On K8s) ITNestedQueryPushDownTest integration test if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} runs-on: ubuntu-22.04 env: @@ -154,7 +154,7 @@ jobs: - name: setup java uses: actions/setup-java@v4 with: - java-version: '8' + java-version: '11' distribution: 'zulu' # the build step produces SNAPSHOT artifacts into the local maven repository, @@ -195,8 +195,8 @@ jobs: indexer: [middleManager, indexer] uses: ./.github/workflows/reusable-standard-its.yml with: - build_jdk: 8 - runtime_jdk: 8 + build_jdk: 11 + runtime_jdk: 11 testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties,centralized-datasource-schema,cds-task-schema-publish-disabled,cds-coordinator-metadata-query-disabled use_indexer: ${{ matrix.indexer }} group: other diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index 6ece670dae2b..05425adb41fa 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -42,7 +42,7 @@ jobs: fail-fast: false matrix: # Use JDK 21.0.4 to work around https://github.com/apache/druid/issues/17429 - java: [ '8', '11', '17', '21.0.4' ] + java: [ '11', '17', '21.0.4' ] runs-on: ubuntu-latest steps: - name: checkout branch @@ -64,7 +64,7 @@ jobs: - name: script checks # who watches the watchers? - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '11' }} run: ./check_test_suite_test.py - name: (openjdk17) strict compilation @@ -74,43 +74,43 @@ jobs: run: ${MVN} clean -DstrictCompile compile test-compile --fail-at-end ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} - name: maven install - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '11' }} run: | echo 'Running Maven install...' && ${MVN} clean install -q -ff -pl '!distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} - name: checkstyle - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '11' }} run: ${MVN} checkstyle:checkstyle --fail-at-end - name: license checks - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '11' }} run: ./.github/scripts/license_checks_script.sh - name: analyze dependencies - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '11' }} run: | ./.github/scripts/analyze_dependencies_script.sh - name: animal sniffer checks - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '11' }} run: ${MVN} animal-sniffer:check --fail-at-end - name: enforcer checks - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '11' }} run: ${MVN} enforcer:enforce --fail-at-end - name: forbidden api checks - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '11' }} run: ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end - name: pmd checks - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '11' }} run: ${MVN} pmd:check --fail-at-end # TODO: consider adding pmd:cpd-check - name: spotbugs checks - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '11' }} run: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks' intellij-inspections: @@ -124,7 +124,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'zulu' - java-version: '8' + java-version: '11' cache: 'maven' - name: maven install @@ -154,7 +154,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'zulu' - java-version: '8' + java-version: '11' cache: 'maven' - name: maven install diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index f8aff5f56e15..714fce69f292 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -80,7 +80,7 @@ jobs: fail-fast: false matrix: # Use JDK 21.0.4 to work around https://github.com/apache/druid/issues/17429 - jdk: [ '8', '11', '17', '21.0.4' ] + jdk: [ '11', '17', '21.0.4' ] runs-on: ubuntu-latest steps: - name: Checkout branch @@ -162,7 +162,7 @@ jobs: fail-fast: false matrix: # Use JDK 21.0.4 to work around https://github.com/apache/druid/issues/17429 - jdk: [ '11', '17', '21.0.4' ] + jdk: [ '17', '21.0.4' ] name: "unit tests (jdk${{ matrix.jdk }}, sql-compat=true)" uses: ./.github/workflows/unit-tests.yml needs: unit-tests @@ -176,11 +176,11 @@ jobs: fail-fast: false matrix: sql_compatibility: [ false, true ] - name: "unit tests (jdk8, sql-compat=${{ matrix.sql_compatibility }})" + name: "unit tests (jdk11, sql-compat=${{ matrix.sql_compatibility }})" uses: ./.github/workflows/unit-tests.yml needs: build with: - jdk: 8 + jdk: 11 sql_compatibility: ${{ matrix.sql_compatibility }} standard-its: From afaab174b7e9fbf0cd1e762ffcfdfa545c0ed219 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 11 Nov 2024 10:16:45 +0530 Subject: [PATCH 02/61] Update docker images --- integration-tests-ex/image/docker/Dockerfile | 2 +- integration-tests/docker/Dockerfile | 2 +- integration-tests/script/docker_build_containers.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-tests-ex/image/docker/Dockerfile b/integration-tests-ex/image/docker/Dockerfile index 90955eae3c11..9fdce7aea6ee 100644 --- a/integration-tests-ex/image/docker/Dockerfile +++ b/integration-tests-ex/image/docker/Dockerfile @@ -28,7 +28,7 @@ # This Dockerfile prefers to use the COPY command over ADD. # See: https://phoenixnap.com/kb/docker-add-vs-copy -ARG JDK_VERSION=8-slim-buster +ARG JDK_VERSION=11-slim-buster # The FROM image provides Java on top of Debian, and # thus provides bash, apt-get, etc. diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile index 40b1cee4f9bf..98d9ecd9446f 100644 --- a/integration-tests/docker/Dockerfile +++ b/integration-tests/docker/Dockerfile @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG JDK_VERSION=8-slim-buster +ARG JDK_VERSION=11-slim-buster FROM openjdk:$JDK_VERSION as druidbase # Bundle everything into one script so cleanup can reduce image size. diff --git a/integration-tests/script/docker_build_containers.sh b/integration-tests/script/docker_build_containers.sh index ba2dd7dcd822..bed98c7db5f8 100755 --- a/integration-tests/script/docker_build_containers.sh +++ b/integration-tests/script/docker_build_containers.sh @@ -26,7 +26,7 @@ then else echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is set with value ${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" case "${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" in - 8 | 11 | 17 | 21) + 11 | 17 | 21) echo "Build druid-cluster with Java $DRUID_INTEGRATION_TEST_JVM_RUNTIME" docker build -t druid/cluster \ --build-arg JDK_VERSION=$DRUID_INTEGRATION_TEST_JVM_RUNTIME-slim-buster \ From 229f93f19141fa8c4dcf68cd3b17051071117e7a Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 11 Nov 2024 10:25:12 +0530 Subject: [PATCH 03/61] Fix spotbugs check: RCN_REDUNDANT_NULLCHECK_OF_NULL_VALUE --- .../query/rowsandcols/CursorFactoryRowsAndColumns.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/CursorFactoryRowsAndColumns.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/CursorFactoryRowsAndColumns.java index 46fda857516f..e636921baf72 100644 --- a/processing/src/main/java/org/apache/druid/query/rowsandcols/CursorFactoryRowsAndColumns.java +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/CursorFactoryRowsAndColumns.java @@ -112,12 +112,8 @@ private static RowsAndColumns materialize(CursorFactory cursorFactory) cursor.advance(); } - if (writer == null) { - return new EmptyRowsAndColumns(); - } else { - final byte[] bytes = writer.toByteArray(); - return new ColumnBasedFrameRowsAndColumns(Frame.wrap(bytes), rowSignature); - } + final byte[] bytes = writer.toByteArray(); + return new ColumnBasedFrameRowsAndColumns(Frame.wrap(bytes), rowSignature); } } } From 7675c1f5189a8620e0a5d94ee120c3270a278017 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 11 Nov 2024 10:40:50 +0530 Subject: [PATCH 04/61] Update java version in pom.xml files --- benchmarks/pom.xml | 2 +- pom.xml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 89609f573d14..d444132a95c2 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -239,7 +239,7 @@ UTF-8 1.21 - 1.8 + 11 benchmarks diff --git a/pom.xml b/pom.xml index a2d7c2aa3f04..8df88c9ae051 100644 --- a/pom.xml +++ b/pom.xml @@ -70,9 +70,9 @@ - 1.8 - 1.8 - 8 + 11 + ${java.version} + ${java.version} UTF-8 0.9.0.M2 5.5.0 From f0ccdcae1093e6160a5cfb8be7efa714fac3e823 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 11 Nov 2024 11:08:04 +0530 Subject: [PATCH 05/61] Fix compilation issues because of javax.annotation not existing in updated java version --- extensions-contrib/grpc-query/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/extensions-contrib/grpc-query/pom.xml b/extensions-contrib/grpc-query/pom.xml index 101e2f34b74c..d1e511bcd66e 100644 --- a/extensions-contrib/grpc-query/pom.xml +++ b/extensions-contrib/grpc-query/pom.xml @@ -262,6 +262,11 @@ reflections test + + javax.annotation + javax.annotation-api + 1.2 + From b60238c85c233ec14ca18e60748f5287b768ab09 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 11 Nov 2024 12:22:52 +0530 Subject: [PATCH 06/61] Fix static-checks: Unsupported class file major version 55 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8df88c9ae051..b17d1a175ca7 100644 --- a/pom.xml +++ b/pom.xml @@ -1830,7 +1830,7 @@ org.apache.maven.plugins maven-dependency-plugin - 3.1.1 + 3.1.2 org.apache.maven.plugins From bbccb60f33f119ba27cf64599cf3bd0f33839b64 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 11 Nov 2024 12:44:25 +0530 Subject: [PATCH 07/61] Fix static-checks: grpc-query extension issues --- extensions-contrib/grpc-query/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/extensions-contrib/grpc-query/pom.xml b/extensions-contrib/grpc-query/pom.xml index d1e511bcd66e..e46f0c35fd2d 100644 --- a/extensions-contrib/grpc-query/pom.xml +++ b/extensions-contrib/grpc-query/pom.xml @@ -266,6 +266,7 @@ javax.annotation javax.annotation-api 1.2 + provided From 4c40ec00be91c962e0e0ed1ca8cc5af796ed8c05 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 11 Nov 2024 14:13:29 +0530 Subject: [PATCH 08/61] Fix spotbugs check: VO_VOLATILE_INCREMENT --- .../curator/discovery/CuratorDruidLeaderSelector.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/apache/druid/curator/discovery/CuratorDruidLeaderSelector.java b/server/src/main/java/org/apache/druid/curator/discovery/CuratorDruidLeaderSelector.java index b2179f5d69d4..d11298031069 100644 --- a/server/src/main/java/org/apache/druid/curator/discovery/CuratorDruidLeaderSelector.java +++ b/server/src/main/java/org/apache/druid/curator/discovery/CuratorDruidLeaderSelector.java @@ -37,6 +37,7 @@ import javax.annotation.Nullable; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; /** @@ -58,7 +59,7 @@ public class CuratorDruidLeaderSelector implements DruidLeaderSelector private final AtomicReference leaderLatch = new AtomicReference<>(); private volatile boolean leader = false; - private volatile int term = 0; + private final AtomicInteger term = new AtomicInteger(0); public CuratorDruidLeaderSelector(CuratorFramework curator, @Self DruidNode self, String latchPath) { @@ -99,7 +100,7 @@ public void isLeader() } leader = true; - term++; + term.incrementAndGet(); listener.becomeLeader(); } catch (Exception ex) { @@ -161,7 +162,7 @@ public boolean isLeader() @Override public int localTerm() { - return term; + return term.get(); } @Override From 4ddeb2dc0cda5cf71742125fa109b93bdacf3363 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 11 Nov 2024 14:22:29 +0530 Subject: [PATCH 09/61] Fix spotbugs check: NP_NONNULL_PARAM_VIOLATION --- .../main/java/org/apache/druid/indexer/IndexGeneratorJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java index 8a124435faa5..a41ced78e7c8 100644 --- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java +++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java @@ -293,7 +293,7 @@ private static IncrementalIndex makeIncrementalIndex( AggregatorFactory[] aggs, HadoopDruidIndexerConfig config, @Nullable Iterable oldDimOrder, - Map oldCapabilities + @Nullable Map oldCapabilities ) { final HadoopTuningConfig tuningConfig = config.getSchema().getTuningConfig(); From 87ff5b6343db187a7b4235230ce818a54e89eb70 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 11 Nov 2024 16:56:40 +0530 Subject: [PATCH 10/61] Trigger Build From 1d70a4fe3cf6b652abcefb1749d37587bc89e929 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 11 Nov 2024 17:21:18 +0530 Subject: [PATCH 11/61] Handle animal-sniffer-maven-plugin issues with java.nio.* classes --- pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pom.xml b/pom.xml index b17d1a175ca7..9def00fad4e9 100644 --- a/pom.xml +++ b/pom.xml @@ -1622,6 +1622,16 @@ sun.misc.Unsafe java.lang.invoke.MethodHandle + + java.nio.ByteBuffer + java.nio.IntBuffer + java.nio.CharBuffer + java.nio.FloatBuffer + java.nio.DoubleBuffer + java.nio.MappedByteBuffer From ea17aa1c463d4cc6e0b6a6ed4b2ba92726251ca7 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 11 Nov 2024 21:40:44 +0530 Subject: [PATCH 12/61] Handle issues with blocked repository URLs by adding mirror --- extensions-contrib/thrift-extensions/pom.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index 064def57bde0..bf947d410e22 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -39,6 +39,18 @@ 19.10.0 + + + + twitter + https://maven.twttr.com + + + org.apache.druid From 6d127c1e860c0f6751efb18621eff1d938dce087 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 11 Nov 2024 21:58:31 +0530 Subject: [PATCH 13/61] Move the mirror to parent pom.xml --- extensions-contrib/thrift-extensions/pom.xml | 12 ------------ pom.xml | 10 ++++++++++ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index bf947d410e22..064def57bde0 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -39,18 +39,6 @@ 19.10.0 - - - - twitter - https://maven.twttr.com - - - org.apache.druid diff --git a/pom.xml b/pom.xml index 9def00fad4e9..61981aff1099 100644 --- a/pom.xml +++ b/pom.xml @@ -271,6 +271,16 @@ false + + + + twitter + https://maven.twttr.com + From cbdcdde6dc878e5ead54dcfae6275b7d58e30f00 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 11 Nov 2024 22:36:12 +0530 Subject: [PATCH 14/61] Update Guice to 4.2.2 to support Java 11 --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 61981aff1099..aa9969013bf4 100644 --- a/pom.xml +++ b/pom.xml @@ -96,7 +96,8 @@ 2.35.1 8.5.4 32.0.1-jre - 4.1.0 + + 4.2.2 1.3 9.4.54.v20240208 1.19.4 From 2db93971c84b890ad463179fee2a0d11bc68a995 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 11 Nov 2024 23:07:39 +0530 Subject: [PATCH 15/61] Update guice version in licenses.yaml file --- licenses.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/licenses.yaml b/licenses.yaml index 40305fc55c66..a12ab1b01e4f 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -371,7 +371,7 @@ name: Guice license_category: binary module: java-core license_name: Apache License version 2.0 -version: 4.1.0 +version: 4.2.2 libraries: - com.google.inject: guice - com.google.inject.extensions: guice-multibindings From f9c1adbf00bdadfbbaadbb5db3cf4dd8738daac8 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Tue, 12 Nov 2024 00:03:51 +0530 Subject: [PATCH 16/61] Fix mvn analyze check error: Unused declared dependencies found com.google.inject.extensions:guice-multibindings:jar:4.2.2:compile --- benchmarks/pom.xml | 1 + extensions-contrib/cloudfiles-extensions/pom.xml | 1 + indexing-service/pom.xml | 1 + integration-tests-ex/cases/pom.xml | 1 + integration-tests/pom.xml | 1 + processing/pom.xml | 1 + server/pom.xml | 1 + services/pom.xml | 1 + sql/pom.xml | 1 + 9 files changed, 9 insertions(+) diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index d444132a95c2..155b4c049a42 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -64,6 +64,7 @@ com.google.inject.extensions guice-multibindings + provided org.apache.druid diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index 01b006dd5d5b..a29a713f2e0a 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -61,6 +61,7 @@ com.google.inject.extensions guice-multibindings ${guice.version} + provided diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 2f348e3d4839..412568ea88c9 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -91,6 +91,7 @@ com.google.inject.extensions guice-multibindings + provided javax.ws.rs diff --git a/integration-tests-ex/cases/pom.xml b/integration-tests-ex/cases/pom.xml index 3e8bf724d82f..e6726db5ab8e 100644 --- a/integration-tests-ex/cases/pom.xml +++ b/integration-tests-ex/cases/pom.xml @@ -86,6 +86,7 @@ com.google.inject.extensions guice-multibindings + provided org.apache.curator diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 5320d0c2451e..983210690983 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -281,6 +281,7 @@ com.google.inject.extensions guice-multibindings + provided com.fasterxml.jackson.core diff --git a/processing/pom.xml b/processing/pom.xml index 8fb67ccaebc8..6afb38ae48d9 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -106,6 +106,7 @@ com.google.inject.extensions guice-multibindings + provided com.google.code.findbugs diff --git a/server/pom.xml b/server/pom.xml index 3300606d1067..206dd9dbeb53 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -110,6 +110,7 @@ com.google.inject.extensions guice-multibindings + provided com.google.inject.extensions diff --git a/services/pom.xml b/services/pom.xml index b6c552d451ac..7b681a1e7e17 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -160,6 +160,7 @@ com.google.inject.extensions guice-multibindings + provided org.roaringbitmap diff --git a/sql/pom.xml b/sql/pom.xml index 6c564e5cdaa7..20bf288d90db 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -131,6 +131,7 @@ com.google.inject.extensions guice-multibindings + provided javax.ws.rs From 5152c8328909b4f9e57966abeec87734a7667526 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Tue, 12 Nov 2024 00:04:45 +0530 Subject: [PATCH 17/61] Fix druid-compressed-bigdecimal tests --- extensions-contrib/compressed-bigdecimal/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-contrib/compressed-bigdecimal/pom.xml b/extensions-contrib/compressed-bigdecimal/pom.xml index 358bc3b26c5b..43caf7c3c323 100644 --- a/extensions-contrib/compressed-bigdecimal/pom.xml +++ b/extensions-contrib/compressed-bigdecimal/pom.xml @@ -145,7 +145,7 @@ com.google.inject guice - 4.1.0 + 4.2.2 provided From 3708c0bdb339b6819dfa82ff9518e512fad2ea31 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Tue, 12 Nov 2024 01:21:46 +0530 Subject: [PATCH 18/61] Change intellij xml file to use java 11 --- .idea/misc-for-inspection.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.idea/misc-for-inspection.xml b/.idea/misc-for-inspection.xml index f890d06f1fe6..7b7aac398518 100644 --- a/.idea/misc-for-inspection.xml +++ b/.idea/misc-for-inspection.xml @@ -84,7 +84,7 @@ - + From d8234dbcd4359fcaacc0a4ddaeb7b97ab26d4166 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Wed, 13 Nov 2024 17:20:53 +0530 Subject: [PATCH 19/61] Update docs --- README.md | 2 +- check_test_suite.py | 2 +- docs/configuration/index.md | 6 +++--- .../extensions-core/simple-client-sslcontext.md | 6 +++--- docs/ingestion/input-sources.md | 8 ++++---- docs/operations/java.md | 6 +++++- docs/operations/tls-support.md | 6 +++--- docs/querying/dimensionspecs.md | 2 +- docs/querying/filters.md | 2 +- docs/querying/math-expr.md | 2 +- docs/querying/sql-scalar.md | 2 +- docs/tutorials/cluster.md | 6 +++--- docs/tutorials/index.md | 2 +- examples/bin/verify-java | 6 +++--- examples/quickstart/tutorial/hadoop/docker/Dockerfile | 6 +++--- extensions-core/simple-client-sslcontext/README.md | 2 +- .../apache/druid/indexing/overlord/RemoteTaskRunner.java | 2 +- integration-tests/README.md | 3 +-- .../main/antlr4/org/apache/druid/math/expr/antlr/Expr.g4 | 2 +- .../main/java/org/apache/druid/crypto/CryptoService.java | 2 +- .../util/common/guava/ParallelMergeCombiningSequence.java | 2 +- .../src/druid-models/ingestion-spec/ingestion-spec.tsx | 2 +- .../src/druid-models/input-source/input-source.tsx | 2 +- .../src/druid-models/timestamp-spec/timestamp-spec.tsx | 2 +- 24 files changed, 43 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index df543687b5fb..e4100e8cf8a5 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ Find articles written by community members and a calendar of upcoming events on ### Building from source -Please note that JDK 8 or JDK 11 is required to build Druid. +Please note that JDK 11 or JDK 17 is required to build Druid. See the latest [build guide](https://druid.apache.org/docs/latest/development/build.html) for instructions on building Apache Druid from source. diff --git a/check_test_suite.py b/check_test_suite.py index d55c52b0b736..eb7748d879eb 100755 --- a/check_test_suite.py +++ b/check_test_suite.py @@ -22,7 +22,7 @@ # this script does some primitive examination of git diff to determine if a test suite needs to be run or not # these jobs should always be run, no matter what -always_run_jobs = ['license checks', '(openjdk8) packaging check', '(openjdk11) packaging check'] +always_run_jobs = ['license checks', '(openjdk11) packaging check'] # ignore changes to these files completely since they don't impact CI, if the changes are only to these files then all # of CI can be skipped. however, jobs which are always run will still be run even if only these files are changed diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 8f4f8065f1d9..b8d9e7b9b00b 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -195,9 +195,9 @@ and `druid.tlsPort` properties on each service. Please see `Configuration` secti Druid uses Jetty as an embedded web server. To learn more about TLS/SSL, certificates, and related concepts in Jetty, including explanations of the configuration settings below, see "Configuring SSL/TLS KeyStores" in the [Jetty Operations Guide](https://www.eclipse.org/jetty/documentation.php). -For information about TLS/SSL support in Java in general, see the [Java Secure Socket Extension (JSSE) Reference Guide](http://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html). +For information about TLS/SSL support in Java in general, see the [Java Secure Socket Extension (JSSE) Reference Guide](https://docs.oracle.com/en/java/javase/11/security/java-secure-socket-extension-jsse-reference-guide.html). The [Java Cryptography Architecture -Standard Algorithm Name Documentation for JDK 8](http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html) lists all possible +Standard Algorithm Name Documentation for JDK 11](https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html) lists all possible values for the following properties, among others provided by the Java implementation. |Property|Description|Default|Required| @@ -230,7 +230,7 @@ These properties apply to the SSLContext that will be provided to the internal H |`druid.client.https.trustStoreAlgorithm`|Algorithm to be used by TrustManager to validate certificate chains|`javax.net.ssl.TrustManagerFactory.getDefaultAlgorithm()`|no| |`druid.client.https.trustStorePassword`|The [Password Provider](../operations/password-provider.md) or String password for the Trust Store.|none|yes| -This [document](http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html) lists all the possible +This [document](https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html) lists all the possible values for the above mentioned configs among others provided by Java implementation. ### Authentication and authorization diff --git a/docs/development/extensions-core/simple-client-sslcontext.md b/docs/development/extensions-core/simple-client-sslcontext.md index 7452a7b5bb1b..db1a37afe414 100644 --- a/docs/development/extensions-core/simple-client-sslcontext.md +++ b/docs/development/extensions-core/simple-client-sslcontext.md @@ -23,9 +23,9 @@ title: "Simple SSLContext Provider Module" --> -This Apache Druid module contains a simple implementation of [SSLContext](http://docs.oracle.com/javase/8/docs/api/javax/net/ssl/SSLContext.html) +This Apache Druid module contains a simple implementation of [SSLContext](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/javax/net/ssl/SSLContext.html) that will be injected to be used with HttpClient that Druid processes use internally to communicate with each other. To learn more about -Java's SSL support, please refer to [this](http://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html) guide. +Java's SSL support, please refer to [this](https://docs.oracle.com/en/java/javase/11/security/java-secure-socket-extension-jsse-reference-guide.html) guide. |Property|Description|Default|Required| @@ -48,5 +48,5 @@ The following table contains optional parameters for supporting client certifica |`druid.client.https.keyManagerPassword`|The [Password Provider](../../operations/password-provider.md) or String password for the Key Manager.|none|no| |`druid.client.https.validateHostnames`|Validate the hostname of the server. This should not be disabled unless you are using [custom TLS certificate checks](../../operations/tls-support.md) and know that standard hostname validation is not needed.|true|no| -This [document](http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html) lists all the possible +This [document](https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html) lists all the possible values for the above mentioned configs among others provided by Java implementation. diff --git a/docs/ingestion/input-sources.md b/docs/ingestion/input-sources.md index 5ee9fdc4d258..b96050899278 100644 --- a/docs/ingestion/input-sources.md +++ b/docs/ingestion/input-sources.md @@ -184,7 +184,7 @@ Sample specs: |uris|JSON array of URIs where S3 objects to be ingested are located.|None|`uris` or `prefixes` or `objects` must be set| |prefixes|JSON array of URI prefixes for the locations of S3 objects to be ingested. Empty objects starting with one of the given prefixes will be skipped.|None|`uris` or `prefixes` or `objects` must be set| |objects|JSON array of S3 Objects to be ingested.|None|`uris` or `prefixes` or `objects` must be set| -|objectGlob|A glob for the object part of the S3 URI. In the URI `s3://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `s3://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html#getPathMatcher-java.lang.String-).|None|no| +|objectGlob|A glob for the object part of the S3 URI. In the URI `s3://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `s3://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)).|None|no| |systemFields|JSON array of system fields to return as part of input rows. Possible values: `__file_uri` (S3 URI starting with `s3://`), `__file_bucket` (S3 bucket), and `__file_path` (S3 object key).|None|no| | endpointConfig |Config for overriding the default S3 endpoint and signing region. This would allow ingesting data from a different S3 store. Please see [s3 config](../development/extensions-core/s3.md#connecting-to-s3-configuration) for more information.|None|No (defaults will be used if not given) | clientConfig |S3 client properties for the overridden s3 endpoint. This is used in conjunction with `endPointConfig`. Please see [s3 config](../development/extensions-core/s3.md#connecting-to-s3-configuration) for more information.|None|No (defaults will be used if not given) @@ -289,7 +289,7 @@ Sample specs: |uris|JSON array of URIs where Google Cloud Storage objects to be ingested are located.|None|`uris` or `prefixes` or `objects` must be set| |prefixes|JSON array of URI prefixes for the locations of Google Cloud Storage objects to be ingested. Empty objects starting with one of the given prefixes will be skipped.|None|`uris` or `prefixes` or `objects` must be set| |objects|JSON array of Google Cloud Storage objects to be ingested.|None|`uris` or `prefixes` or `objects` must be set| -|objectGlob|A glob for the object part of the S3 URI. In the URI `s3://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `s3://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html#getPathMatcher-java.lang.String-).|None|no| +|objectGlob|A glob for the object part of the S3 URI. In the URI `s3://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `s3://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)).|None|no| Note that the Google Cloud Storage input source will skip all empty objects only when `prefixes` is specified. @@ -377,7 +377,7 @@ Sample specs: |uris|JSON array of URIs where the Azure objects to be ingested are located. Use this format: `azureStorage://STORAGE_ACCOUNT/CONTAINER/PATH_TO_FILE`|None|One of the following must be set:`uris`, `prefixes`, or `objects`.| |prefixes|JSON array of URI prefixes for the locations of Azure objects to ingest. Use this format`azureStorage://STORAGE_ACCOUNT/CONTAINER/PREFIX`. Empty objects starting with any of the given prefixes are skipped.|None|One of the following must be set:`uris`, `prefixes`, or `objects`.| |objects|JSON array of Azure objects to ingest.|None|One of the following must be set:`uris`, `prefixes`, or `objects`.| -|objectGlob|A glob for the object part of the Azure URI. In the URI `azureStorage://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `azureStorage://foo/bar/file.json` because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html#getPathMatcher-java.lang.String-).|None|no| +|objectGlob|A glob for the object part of the Azure URI. In the URI `azureStorage://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `azureStorage://foo/bar/file.json` because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)).|None|no| |systemFields|JSON array of system fields to return as part of input rows. Possible values: `__file_uri` (Azure blob URI starting with `azureStorage://`), `__file_bucket` (Azure bucket), and `__file_path` (Azure object path).|None|no| |properties|Properties object for overriding the default Azure configuration. See below for more information.|None|No (defaults will be used if not given)| @@ -471,7 +471,7 @@ Sample specs: |uris|JSON array of URIs where the Azure objects to be ingested are located, in the form `azure:///`|None|`uris` or `prefixes` or `objects` must be set| |prefixes|JSON array of URI prefixes for the locations of Azure objects to ingest, in the form `azure:///`. Empty objects starting with one of the given prefixes are skipped.|None|`uris` or `prefixes` or `objects` must be set| |objects|JSON array of Azure objects to ingest.|None|`uris` or `prefixes` or `objects` must be set| -|objectGlob|A glob for the object part of the Azure URI. In the URI `azure://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `azure://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html#getPathMatcher-java.lang.String-).|None|no| +|objectGlob|A glob for the object part of the Azure URI. In the URI `azure://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `azure://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)).|None|no| |systemFields|JSON array of system fields to return as part of input rows. Possible values: `__file_uri` (Azure blob URI starting with `azure://`), `__file_bucket` (Azure bucket), and `__file_path` (Azure object path).|None|no| Note that the Azure input source skips all empty objects only when `prefixes` is specified. diff --git a/docs/operations/java.md b/docs/operations/java.md index 2af122441162..97e9d95e4634 100644 --- a/docs/operations/java.md +++ b/docs/operations/java.md @@ -27,7 +27,11 @@ a Java runtime for Druid. ## Selecting a Java runtime -Druid fully supports Java 8u92+, Java 11, and Java 17. The project team recommends Java 17. +Druid fully supports Java 11 and Java 17. The project team recommends Java 17. + +:::info +Note: Starting with Apache Druid 32.0.0, support for Java 8u92+ has been removed. +::: The project team recommends using an OpenJDK-based Java distribution. There are many free and actively-supported distributions available, including diff --git a/docs/operations/tls-support.md b/docs/operations/tls-support.md index b5db993eeeb9..543f177bfedd 100644 --- a/docs/operations/tls-support.md +++ b/docs/operations/tls-support.md @@ -37,10 +37,10 @@ Apache Druid uses Jetty as its embedded web server. To get familiar with TLS/SSL, along with related concepts like keys and certificates, read [Configuring Secure Protocols](https://www.eclipse.org/jetty/documentation/jetty-12/operations-guide/index.html#og-protocols-ssl) in the Jetty documentation. -To get more in-depth knowledge of TLS/SSL support in Java in general, refer to the [Java Secure Socket Extension (JSSE) Reference Guide](http://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html). +To get more in-depth knowledge of TLS/SSL support in Java in general, refer to the [Java Secure Socket Extension (JSSE) Reference Guide](https://docs.oracle.com/en/java/javase/11/security/java-secure-socket-extension-jsse-reference-guide.html). The [Class SslContextFactory](https://www.eclipse.org/jetty/javadoc/jetty-11/org/eclipse/jetty/util/ssl/SslContextFactory.html) reference doc can help in understanding TLS/SSL configurations listed below. Finally, [Java Cryptography Architecture -Standard Algorithm Name Documentation for JDK 8](http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html) lists all possible +Standard Algorithm Name Documentation for JDK 11](https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html) lists all possible values for the configs below, among others provided by Java implementation. |Property|Description|Default|Required| @@ -79,7 +79,7 @@ The following table contains non-mandatory advanced configuration options, use c ## Internal communication over TLS Whenever possible Druid processes will use HTTPS to talk to each other. To enable this communication Druid's HttpClient needs to -be configured with a proper [SSLContext](http://docs.oracle.com/javase/8/docs/api/javax/net/ssl/SSLContext.html) that is able +be configured with a proper [SSLContext](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/javax/net/ssl/SSLContext.html) that is able to validate the Server Certificates, otherwise communication will fail. Since, there are various ways to configure SSLContext, by default, Druid looks for an instance of SSLContext Guice binding diff --git a/docs/querying/dimensionspecs.md b/docs/querying/dimensionspecs.md index cda6d2897998..68773abdae10 100644 --- a/docs/querying/dimensionspecs.md +++ b/docs/querying/dimensionspecs.md @@ -255,7 +255,7 @@ For a regular dimension, it assumes the string is formatted in [ISO-8601 date and time format](https://en.wikipedia.org/wiki/ISO_8601). * `format` : date time format for the resulting dimension value, in [Joda Time DateTimeFormat](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html), or null to use the default ISO8601 format. -* `locale` : locale (language and country) to use, given as a [IETF BCP 47 language tag](http://www.oracle.com/technetwork/java/javase/java8locales-2095355.html#util-text), e.g. `en-US`, `en-GB`, `fr-FR`, `fr-CA`, etc. +* `locale` : locale (language and country) to use, given as a [IETF BCP 47 language tag](https://www.oracle.com/java/technologies/javase/jdk11-suported-locales.html#util-text), e.g. `en-US`, `en-GB`, `fr-FR`, `fr-CA`, etc. * `timeZone` : time zone to use in [IANA tz database format](http://en.wikipedia.org/wiki/List_of_tz_database_time_zones), e.g. `Europe/Berlin` (this can possibly be different than the aggregation time-zone) * `granularity` : [granularity](granularities.md) to apply before formatting, or omit to not apply any granularity. * `asMillis` : boolean value, set to true to treat input strings as millis rather than ISO8601 strings. Additionally, if `format` is null or not specified, output will be in millis rather than ISO8601. diff --git a/docs/querying/filters.md b/docs/querying/filters.md index ee50b370f126..20b270903151 100644 --- a/docs/querying/filters.md +++ b/docs/querying/filters.md @@ -439,7 +439,7 @@ The regular expression filter is similar to the selector filter, but using regul | -------- | ----------- | -------- | | `type` | Must be "regex".| Yes | | `dimension` | Input column or virtual column name to filter on. | Yes | -| `pattern` | String pattern to match - any standard [Java regular expression](http://docs.oracle.com/javase/6/docs/api/java/util/regex/Pattern.html). | Yes | +| `pattern` | String pattern to match - any standard [Java regular expression](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/regex/Pattern.html). | Yes | | `extractionFn` | [Extraction function](./dimensionspecs.md#extraction-functions) to apply to `dimension` prior to value matching. See [filtering with extraction functions](#filtering-with-extraction-functions) for details. | No | Note that it is often more optimal to use a like filter instead of a regex for simple matching of prefixes. diff --git a/docs/querying/math-expr.md b/docs/querying/math-expr.md index 0893fc4e2366..84494acfaabd 100644 --- a/docs/querying/math-expr.md +++ b/docs/querying/math-expr.md @@ -81,7 +81,7 @@ The following built-in functions are available. |name|description| |----|-----------| |concat|concat(expr, expr...) concatenate a list of strings| -|format|format(pattern[, args...]) returns a string formatted in the manner of Java's [String.format](https://docs.oracle.com/javase/8/docs/api/java/lang/String.html#format-java.lang.String-java.lang.Object...-).| +|format|format(pattern[, args...]) returns a string formatted in the manner of Java's [String.format](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/String.html#format(java.lang.String,java.lang.Object...)).| |like|like(expr, pattern[, escape]) is equivalent to SQL `expr LIKE pattern`| |lookup|lookup(expr, lookup-name[,replaceMissingValueWith]) looks up expr in a registered,`replaceMissingValueWith` is an optional constant string [query-time lookup](../querying/lookups.md)| |parse_long|parse_long(string[, radix]) parses a string as a long with the given radix, or 10 (decimal) if a radix is not provided.| diff --git a/docs/querying/sql-scalar.md b/docs/querying/sql-scalar.md index f4c565cda839..2c08fc636697 100644 --- a/docs/querying/sql-scalar.md +++ b/docs/querying/sql-scalar.md @@ -115,7 +115,7 @@ String functions accept strings and return a type appropriate to the function. |`REPLACE(expr, substring, replacement)`|Replaces instances of `substring` in `expr` with `replacement` and returns the result.| |`REPEAT(expr, N)`|Repeats `expr` `N` times.| |`REVERSE(expr)`|Reverses `expr`.| -|`STRING_FORMAT(pattern[, args...])`|Returns a string formatted in the manner of Java's [String.format](https://docs.oracle.com/javase/8/docs/api/java/lang/String.html#format-java.lang.String-java.lang.Object...-).| +|`STRING_FORMAT(pattern[, args...])`|Returns a string formatted in the manner of Java's [String.format](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/String.html#format(java.lang.String,java.lang.Object...)).| |`STRPOS(expr, substring)`|Returns the index of `substring` within `expr`, with indexes starting from 1. If `substring` is not found, returns 0.| |`SUBSTRING(expr, index[, length])`|Returns a substring of `expr` starting at a given one-based index. If `length` is omitted, extracts characters to the end of the string, otherwise returns a substring of `length` UTF-16 characters.| |`SUBSTR(expr, index[, length])`|Alias for `SUBSTRING`.| diff --git a/docs/tutorials/cluster.md b/docs/tutorials/cluster.md index 8c4ad3e12346..f5d73d6808f6 100644 --- a/docs/tutorials/cluster.md +++ b/docs/tutorials/cluster.md @@ -133,7 +133,7 @@ The [basic cluster tuning guide](../operations/basic-cluster-tuning.md) has info We recommend running your favorite Linux distribution. You will also need -* [Java 8u92+, 11, or 17](../operations/java.md) +* [Java 11 or 17](../operations/java.md) * Python 2 or Python 3 :::info @@ -141,8 +141,8 @@ We recommend running your favorite Linux distribution. You will also need `DRUID_JAVA_HOME` or `JAVA_HOME`. For more details run the `bin/verify-java` script. ::: -For information about installing Java, see the documentation for your OS package manager. If your Ubuntu-based OS does not have a recent enough version of Java, WebUpd8 offers [packages for those -OSes](http://www.webupd8.org/2012/09/install-oracle-java-8-in-ubuntu-via-ppa.html). +For information about installing Java, see the documentation for your OS package manager. If your Ubuntu-based OS does not have a recent enough version of Java, Linux Uprising offers [packages for those +OSes](https://launchpad.net/~linuxuprising/+archive/ubuntu/java). ## Download the distribution diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index 90e9836e28a8..813ac9859d96 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -40,7 +40,7 @@ You can follow these steps on a relatively modest machine, such as a workstation The software requirements for the installation machine are: * Linux, Mac OS X, or other Unix-like OS. (Windows is not supported) -* [Java 8u92+, 11, or 17](../operations/java.md) +* [Java 11 or 17](../operations/java.md) * Python 3 (preferred) or Python 2 * Perl 5 diff --git a/examples/bin/verify-java b/examples/bin/verify-java index 4759b38089e9..33003dc44751 100755 --- a/examples/bin/verify-java +++ b/examples/bin/verify-java @@ -28,14 +28,14 @@ sub fail_check { : "No Java runtime was detected on your system."; print STDERR <<"EOT"; -Druid requires Java 8, 11, or 17. $current_version_text +Druid requires Java 11 or 17. $current_version_text If you believe this check is in error, or you want to proceed with a potentially unsupported Java runtime, you can skip this check using an environment variable: export DRUID_SKIP_JAVA_CHECK=1 -Otherwise, install Java 8, 11, or 17 in one of the following locations. +Otherwise, install Java 11 or 17 in one of the following locations. * DRUID_JAVA_HOME * JAVA_HOME @@ -68,6 +68,6 @@ if ($?) { } # If we know it won't work, die. Otherwise hope for the best. -if ($java_version =~ /version \"((\d+)\.(\d+).*?)\"/ && !($2 == 1 && $3 == 8) && $2 != 11 && $2 != 17 ) { +if ($java_version =~ /version \"((\d+)\.(\d+).*?)\"/ && $2 != 11 && $2 != 17) { fail_check($1); } diff --git a/examples/quickstart/tutorial/hadoop/docker/Dockerfile b/examples/quickstart/tutorial/hadoop/docker/Dockerfile index 9cada259cc14..17ef14fb2c17 100644 --- a/examples/quickstart/tutorial/hadoop/docker/Dockerfile +++ b/examples/quickstart/tutorial/hadoop/docker/Dockerfile @@ -52,11 +52,11 @@ RUN rpm --import http://repos.azulsystems.com/RPM-GPG-KEY-azulsystems && \ rpm -ivh zulu-repo-${ZULU_REPO_VER}.noarch.rpm && \ yum -q -y update && \ yum -q -y upgrade && \ - yum -q -y install zulu8-jdk && \ + yum -q -y install zulu11-jdk && \ yum clean all && \ rm -rf /var/cache/yum zulu-repo_${ZULU_REPO_VER}.noarch.rpm -ENV JAVA_HOME=/usr/lib/jvm/zulu8 +ENV JAVA_HOME=/usr/lib/jvm/zulu11 ENV PATH $PATH:$JAVA_HOME/bin # hadoop @@ -73,7 +73,7 @@ ENV HADOOP_CONF_DIR /usr/local/hadoop/etc/hadoop ENV YARN_CONF_DIR $HADOOP_HOME/etc/hadoop # in hadoop 3 the example file is nearly empty so we can just append stuff -RUN sed -i '$ a export JAVA_HOME=/usr/lib/jvm/zulu8' $HADOOP_HOME/etc/hadoop/hadoop-env.sh +RUN sed -i '$ a export JAVA_HOME=/usr/lib/jvm/zulu11' $HADOOP_HOME/etc/hadoop/hadoop-env.sh RUN sed -i '$ a export HADOOP_HOME=/usr/local/hadoop' $HADOOP_HOME/etc/hadoop/hadoop-env.sh RUN sed -i '$ a export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/' $HADOOP_HOME/etc/hadoop/hadoop-env.sh RUN sed -i '$ a export HDFS_NAMENODE_USER=root' $HADOOP_HOME/etc/hadoop/hadoop-env.sh diff --git a/extensions-core/simple-client-sslcontext/README.md b/extensions-core/simple-client-sslcontext/README.md index 4a5a97d19039..29276f4cf249 100644 --- a/extensions-core/simple-client-sslcontext/README.md +++ b/extensions-core/simple-client-sslcontext/README.md @@ -17,6 +17,6 @@ ~ under the License. --> -This module contains a simple implementation of [SslContext](http://docs.oracle.com/javase/8/docs/api/javax/net/ssl/SSLContext.html) +This module contains a simple implementation of [SslContext](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/javax/net/ssl/SSLContext.html) that will be injected to be used with HttpClient that Druid nodes use internally to communicate with each other. More details [here](https://druid.apache.org/docs/latest/development/extensions-core/simple-client-sslcontext.html). diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/RemoteTaskRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/RemoteTaskRunner.java index 1d06321ddc49..4fd7b57f38b5 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/RemoteTaskRunner.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/RemoteTaskRunner.java @@ -1515,7 +1515,7 @@ void checkBlackListedNodes() boolean shouldRunPendingTasks = false; // must be synchronized while iterating: - // https://docs.oracle.com/javase/8/docs/api/java/util/Collections.html#synchronizedSet-java.util.Set- + // https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/Collections.html#synchronizedSet(java.util.Set) synchronized (blackListedWorkers) { for (Iterator iterator = blackListedWorkers.iterator(); iterator.hasNext(); ) { ZkWorker zkWorker = iterator.next(); diff --git a/integration-tests/README.md b/integration-tests/README.md index 25597df5c923..70c5e6c5edba 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -88,8 +88,7 @@ if your test has the annotation: `@Test(groups = TestNGGroup.BATCH_INDEX)` then * Add `-pl :druid-integration-tests` when running integration tests for the second time or later without changing the code of core modules in between to skip up-to-date checks for the whole module dependency tree. -* Integration tests can also be run with either Java 8 or Java 11 by adding `-Djvm.runtime=#` to the `mvn` command, where `#` -can either be 8 or 11. +* Integration tests can also be run with a specific Java version by adding `-Djvm.runtime=#` to the `mvn` command (where `#` can be 11, for example). * Druid's configuration (using Docker) can be overridden by providing `-Doverride.config.path=`. The file must contain one property per line, the key must start with `druid_` and the format should be snake case. diff --git a/processing/src/main/antlr4/org/apache/druid/math/expr/antlr/Expr.g4 b/processing/src/main/antlr4/org/apache/druid/math/expr/antlr/Expr.g4 index 1d52a13ecfc7..fd8288b2b086 100644 --- a/processing/src/main/antlr4/org/apache/druid/math/expr/antlr/Expr.g4 +++ b/processing/src/main/antlr4/org/apache/druid/math/expr/antlr/Expr.g4 @@ -59,7 +59,7 @@ NULL : 'null'; LONG : [0-9]+; EXP: [eE] [-]? LONG; // DOUBLE provides partial support for java double format -// see: https://docs.oracle.com/javase/8/docs/api/java/lang/Double.html#valueOf-java.lang.String- +// see: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/Double.html#valueOf(java.lang.String) DOUBLE : 'NaN' | 'Infinity' | (LONG '.' LONG?) | (LONG EXP) | (LONG '.' LONG? EXP); IDENTIFIER : [_$a-zA-Z][_$a-zA-Z0-9]* | '"' (ESC | ~ [\"\\])* '"'; WS : [ \t\r\n]+ -> skip ; diff --git a/processing/src/main/java/org/apache/druid/crypto/CryptoService.java b/processing/src/main/java/org/apache/druid/crypto/CryptoService.java index 92d4f6602c45..9e63c0adfa94 100644 --- a/processing/src/main/java/org/apache/druid/crypto/CryptoService.java +++ b/processing/src/main/java/org/apache/druid/crypto/CryptoService.java @@ -42,7 +42,7 @@ * using javax.crypto package. * * To learn about possible algorithms supported and their names, - * See https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html + * See https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html */ public class CryptoService { diff --git a/processing/src/main/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequence.java b/processing/src/main/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequence.java index ca2708700f08..53b7d68636dc 100644 --- a/processing/src/main/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequence.java +++ b/processing/src/main/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequence.java @@ -808,7 +808,7 @@ protected void compute() /** * {@link ForkJoinPool} friendly {@link BlockingQueue} feeder, adapted from 'QueueTaker' of Java documentation on * {@link ForkJoinPool.ManagedBlocker}, - * https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ForkJoinPool.ManagedBlocker.html + * https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/concurrent/ForkJoinPool.ManagedBlocker.html */ static class QueuePusher implements ForkJoinPool.ManagedBlocker { diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx index 69b2f97b0360..f0f0e7e281f9 100644 --- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx +++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx @@ -647,7 +647,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F

For more information, refer to the documentation for{' '} - + FileSystem#getPathMatcher . diff --git a/web-console/src/druid-models/input-source/input-source.tsx b/web-console/src/druid-models/input-source/input-source.tsx index d3f24ea4d948..f2428d8a3835 100644 --- a/web-console/src/druid-models/input-source/input-source.tsx +++ b/web-console/src/druid-models/input-source/input-source.tsx @@ -507,7 +507,7 @@ export const INPUT_SOURCE_FIELDS: Field[] = [

For more information, refer to the documentation for{' '} - + FileSystem#getPathMatcher . diff --git a/web-console/src/druid-models/timestamp-spec/timestamp-spec.tsx b/web-console/src/druid-models/timestamp-spec/timestamp-spec.tsx index 0ec00fd944a7..a80224f12860 100644 --- a/web-console/src/druid-models/timestamp-spec/timestamp-spec.tsx +++ b/web-console/src/druid-models/timestamp-spec/timestamp-spec.tsx @@ -136,7 +136,7 @@ export const TIMESTAMP_SPEC_FIELDS: Field[] = [ info: (

Specify your timestamp format by using the suggestions menu or typing in a{' '} - + format string . From 2c7ddad3601c4ca46d9d0b40f6cc2ce1fb9011d0 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Wed, 13 Nov 2024 18:03:57 +0530 Subject: [PATCH 20/61] Improve doc line --- docs/operations/java.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/java.md b/docs/operations/java.md index 97e9d95e4634..7021ae3e0eff 100644 --- a/docs/operations/java.md +++ b/docs/operations/java.md @@ -30,7 +30,7 @@ a Java runtime for Druid. Druid fully supports Java 11 and Java 17. The project team recommends Java 17. :::info -Note: Starting with Apache Druid 32.0.0, support for Java 8u92+ has been removed. +Note: Starting with Apache Druid 32.0.0, support for Java 8 has been removed. ::: The project team recommends using an OpenJDK-based Java distribution. There are many free and actively-supported From a9a194fae96379bcf1a3d2656191f370074c5ce5 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Thu, 14 Nov 2024 15:21:20 +0530 Subject: [PATCH 21/61] Add check for Java 11 that was removed by mistake --- .github/workflows/unit-and-integration-tests-unified.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index 714fce69f292..b029cf77c463 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -162,7 +162,7 @@ jobs: fail-fast: false matrix: # Use JDK 21.0.4 to work around https://github.com/apache/druid/issues/17429 - jdk: [ '17', '21.0.4' ] + jdk: [ '11', '17', '21.0.4' ] name: "unit tests (jdk${{ matrix.jdk }}, sql-compat=true)" uses: ./.github/workflows/unit-tests.yml needs: unit-tests From b6e2e4f6a89aff855b7783c9bc5430a893b5df11 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Fri, 15 Nov 2024 10:14:53 +0530 Subject: [PATCH 22/61] Change default to Java 17: draft commit v1 --- .github/workflows/codeql.yml | 2 +- .github/workflows/cron-job-its.yml | 21 +++++++------- .github/workflows/revised-its.yml | 11 ++++---- .github/workflows/standard-its.yml | 28 +++++++++---------- .github/workflows/static-checks.yml | 22 +++++++-------- .../unit-and-integration-tests-unified.yml | 4 +-- .idea/misc-for-inspection.xml | 2 +- check_test_suite.py | 2 +- .../tutorial/hadoop/docker/Dockerfile | 6 ++-- .../cloudfiles-extensions/pom.xml | 2 +- integration-tests-ex/image/docker/Dockerfile | 2 +- integration-tests/docker/Dockerfile | 2 +- pom.xml | 13 +++++---- 13 files changed, 60 insertions(+), 57 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 8395f2839fdd..c47e3fa4557f 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -40,7 +40,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'zulu' - java-version: '11' + java-version: '17' cache: 'maven' # Initializes the CodeQL tools for scanning. diff --git a/.github/workflows/cron-job-its.yml b/.github/workflows/cron-job-its.yml index 095b04c345a2..877c3389ebb4 100644 --- a/.github/workflows/cron-job-its.yml +++ b/.github/workflows/cron-job-its.yml @@ -28,7 +28,7 @@ on: jobs: build: if: (github.event_name == 'schedule' && github.repository == 'apache/druid') - name: build (jdk11) + name: build (jdk17) runs-on: ubuntu-latest steps: - name: Checkout branch @@ -37,7 +37,7 @@ jobs: - name: setup java uses: actions/setup-java@v4 with: - java-version: '11' + java-version: '17' distribution: 'zulu' - name: Cache Maven m2 repository @@ -52,6 +52,7 @@ jobs: run: | ./it.sh ci + # This test also validates building Druid on Java 21, and running it on Java 17. integration-index-tests-middleManager: strategy: fail-fast: false @@ -60,7 +61,7 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml needs: build with: - build_jdk: 11 + build_jdk: 21.0.4 runtime_jdk: 17 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: middleManager @@ -74,8 +75,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml needs: build with: - build_jdk: 11 - runtime_jdk: 17 + build_jdk: 17 + runtime_jdk: 21.0.4 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: indexer group: ${{ matrix.testing_group }} @@ -88,8 +89,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml needs: build with: - build_jdk: 11 - runtime_jdk: 17 + build_jdk: 17 + runtime_jdk: 21.0.4 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: middleManager override_config_path: ./environment-configs/test-groups/prepopulated-data @@ -103,8 +104,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml needs: build with: - build_jdk: 11 - runtime_jdk: 17 + build_jdk: 17 + runtime_jdk: 21.0.4 testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties use_indexer: ${{ matrix.indexer }} group: other @@ -122,7 +123,7 @@ jobs: - name: setup java uses: actions/setup-java@v4 with: - java-version: '11' + java-version: '17' distribution: 'zulu' cache: maven diff --git a/.github/workflows/revised-its.yml b/.github/workflows/revised-its.yml index a765011e32ee..9585fe996e11 100644 --- a/.github/workflows/revised-its.yml +++ b/.github/workflows/revised-its.yml @@ -66,8 +66,7 @@ jobs: strategy: fail-fast: false matrix: - #jdk: [11, 17] - jdk: [11] + jdk: [11, 17] it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat, Security, Query] #indexer: [indexer, middleManager] indexer: [middleManager] @@ -86,8 +85,8 @@ jobs: uses: ./.github/workflows/reusable-revised-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 11 - runtime_jdk: 17 + build_jdk: 17 + runtime_jdk: 21.0.4 use_indexer: middleManager script: ./it.sh github S3DeepStorage it: S3DeepStorage @@ -103,8 +102,8 @@ jobs: uses: ./.github/workflows/reusable-revised-its.yml if: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' && (needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true') }} with: - build_jdk: 11 - runtime_jdk: 11 + build_jdk: 17 + runtime_jdk: 17 use_indexer: middleManager script: ./it.sh github BackwardCompatibilityMain it: BackwardCompatibilityMain diff --git a/.github/workflows/standard-its.yml b/.github/workflows/standard-its.yml index 9199b9644137..3ea728dd7c7f 100644 --- a/.github/workflows/standard-its.yml +++ b/.github/workflows/standard-its.yml @@ -51,8 +51,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 11 - runtime_jdk: 11 + build_jdk: 17 + runtime_jdk: 17 testing_groups: -Dgroups=${{ matrix.testing_group }} override_config_path: ./environment-configs/test-groups/prepopulated-data use_indexer: middleManager @@ -67,8 +67,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 11 - runtime_jdk: 11 + build_jdk: 17 + runtime_jdk: 17 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: indexer group: ${{ matrix.testing_group }} @@ -82,8 +82,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 11 - runtime_jdk: 11 + build_jdk: 17 + runtime_jdk: 17 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: middleManager override_config_path: ./environment-configs/test-groups/prepopulated-data @@ -115,8 +115,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 11 - runtime_jdk: 11 + build_jdk: 17 + runtime_jdk: 17 testing_groups: -Dgroups=shuffle-deep-store use_indexer: ${{ matrix.indexer }} override_config_path: ./environment-configs/test-groups/shuffle-deep-store @@ -127,8 +127,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 11 - runtime_jdk: 11 + build_jdk: 17 + runtime_jdk: 17 testing_groups: -Dgroups=custom-coordinator-duties use_indexer: middleManager override_config_path: ./environment-configs/test-groups/custom-coordinator-duties @@ -136,7 +136,7 @@ jobs: integration-k8s-leadership-tests: needs: changes - name: (Compile=openjdk11, Run=openjdk11, Cluster Build On K8s) ITNestedQueryPushDownTest integration test + name: (Compile=openjdk17, Run=openjdk17, Cluster Build On K8s) ITNestedQueryPushDownTest integration test if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} runs-on: ubuntu-22.04 env: @@ -154,7 +154,7 @@ jobs: - name: setup java uses: actions/setup-java@v4 with: - java-version: '11' + java-version: '17' distribution: 'zulu' # the build step produces SNAPSHOT artifacts into the local maven repository, @@ -195,8 +195,8 @@ jobs: indexer: [middleManager, indexer] uses: ./.github/workflows/reusable-standard-its.yml with: - build_jdk: 11 - runtime_jdk: 11 + build_jdk: 17 + runtime_jdk: 17 testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties,centralized-datasource-schema,cds-task-schema-publish-disabled,cds-coordinator-metadata-query-disabled use_indexer: ${{ matrix.indexer }} group: other diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index 949d0df412f9..778a79db7618 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -64,7 +64,7 @@ jobs: - name: script checks # who watches the watchers? - if: ${{ matrix.java == '11' }} + if: ${{ matrix.java == '17' }} run: ./check_test_suite_test.py - name: (openjdk17) strict compilation @@ -74,43 +74,43 @@ jobs: run: ${MVN} clean -DstrictCompile compile test-compile --fail-at-end ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} - name: maven install - if: ${{ matrix.java == '11' }} + if: ${{ matrix.java == '17' }} run: | echo 'Running Maven install...' && ${MVN} clean install -q -ff -pl '!distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} - name: checkstyle - if: ${{ matrix.java == '11' }} + if: ${{ matrix.java == '17' }} run: ${MVN} checkstyle:checkstyle --fail-at-end - name: license checks - if: ${{ matrix.java == '11' }} + if: ${{ matrix.java == '17' }} run: ./.github/scripts/license_checks_script.sh - name: analyze dependencies - if: ${{ matrix.java == '11' }} + if: ${{ matrix.java == '17' }} run: | ./.github/scripts/analyze_dependencies_script.sh - name: animal sniffer checks - if: ${{ matrix.java == '11' }} + if: ${{ matrix.java == '17' }} run: ${MVN} animal-sniffer:check --fail-at-end - name: enforcer checks - if: ${{ matrix.java == '11' }} + if: ${{ matrix.java == '17' }} run: ${MVN} enforcer:enforce --fail-at-end - name: forbidden api checks - if: ${{ matrix.java == '11' }} + if: ${{ matrix.java == '17' }} run: ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end - name: pmd checks - if: ${{ matrix.java == '11' }} + if: ${{ matrix.java == '17' }} run: ${MVN} pmd:check --fail-at-end # TODO: consider adding pmd:cpd-check - name: spotbugs checks - if: ${{ matrix.java == '11' }} + if: ${{ matrix.java == '17' }} run: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks' openrewrite: @@ -122,7 +122,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'zulu' - java-version: '11' + java-version: '17' cache: 'maven' - name: maven install diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index b029cf77c463..9f1294e18409 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -176,11 +176,11 @@ jobs: fail-fast: false matrix: sql_compatibility: [ false, true ] - name: "unit tests (jdk11, sql-compat=${{ matrix.sql_compatibility }})" + name: "unit tests (jdk17, sql-compat=${{ matrix.sql_compatibility }})" uses: ./.github/workflows/unit-tests.yml needs: build with: - jdk: 11 + jdk: 17 sql_compatibility: ${{ matrix.sql_compatibility }} standard-its: diff --git a/.idea/misc-for-inspection.xml b/.idea/misc-for-inspection.xml index 7b7aac398518..66feb815ad1d 100644 --- a/.idea/misc-for-inspection.xml +++ b/.idea/misc-for-inspection.xml @@ -84,7 +84,7 @@ - + diff --git a/check_test_suite.py b/check_test_suite.py index eb7748d879eb..62a5902421aa 100755 --- a/check_test_suite.py +++ b/check_test_suite.py @@ -22,7 +22,7 @@ # this script does some primitive examination of git diff to determine if a test suite needs to be run or not # these jobs should always be run, no matter what -always_run_jobs = ['license checks', '(openjdk11) packaging check'] +always_run_jobs = ['license checks', '(openjdk17) packaging check'] # ignore changes to these files completely since they don't impact CI, if the changes are only to these files then all # of CI can be skipped. however, jobs which are always run will still be run even if only these files are changed diff --git a/examples/quickstart/tutorial/hadoop/docker/Dockerfile b/examples/quickstart/tutorial/hadoop/docker/Dockerfile index 17ef14fb2c17..d6ae8aedcb15 100644 --- a/examples/quickstart/tutorial/hadoop/docker/Dockerfile +++ b/examples/quickstart/tutorial/hadoop/docker/Dockerfile @@ -52,11 +52,11 @@ RUN rpm --import http://repos.azulsystems.com/RPM-GPG-KEY-azulsystems && \ rpm -ivh zulu-repo-${ZULU_REPO_VER}.noarch.rpm && \ yum -q -y update && \ yum -q -y upgrade && \ - yum -q -y install zulu11-jdk && \ + yum -q -y install zulu17-jdk && \ yum clean all && \ rm -rf /var/cache/yum zulu-repo_${ZULU_REPO_VER}.noarch.rpm -ENV JAVA_HOME=/usr/lib/jvm/zulu11 +ENV JAVA_HOME=/usr/lib/jvm/zulu17 ENV PATH $PATH:$JAVA_HOME/bin # hadoop @@ -73,7 +73,7 @@ ENV HADOOP_CONF_DIR /usr/local/hadoop/etc/hadoop ENV YARN_CONF_DIR $HADOOP_HOME/etc/hadoop # in hadoop 3 the example file is nearly empty so we can just append stuff -RUN sed -i '$ a export JAVA_HOME=/usr/lib/jvm/zulu11' $HADOOP_HOME/etc/hadoop/hadoop-env.sh +RUN sed -i '$ a export JAVA_HOME=/usr/lib/jvm/zulu17' $HADOOP_HOME/etc/hadoop/hadoop-env.sh RUN sed -i '$ a export HADOOP_HOME=/usr/local/hadoop' $HADOOP_HOME/etc/hadoop/hadoop-env.sh RUN sed -i '$ a export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/' $HADOOP_HOME/etc/hadoop/hadoop-env.sh RUN sed -i '$ a export HDFS_NAMENODE_USER=root' $HADOOP_HOME/etc/hadoop/hadoop-env.sh diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index a29a713f2e0a..3264045028f3 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -60,7 +60,7 @@ com.google.inject.extensions guice-multibindings - ${guice.version} + ${guice-multibindings.version} provided diff --git a/integration-tests-ex/image/docker/Dockerfile b/integration-tests-ex/image/docker/Dockerfile index 9fdce7aea6ee..deb5bb8d9b81 100644 --- a/integration-tests-ex/image/docker/Dockerfile +++ b/integration-tests-ex/image/docker/Dockerfile @@ -28,7 +28,7 @@ # This Dockerfile prefers to use the COPY command over ADD. # See: https://phoenixnap.com/kb/docker-add-vs-copy -ARG JDK_VERSION=11-slim-buster +ARG JDK_VERSION=17-slim-buster # The FROM image provides Java on top of Debian, and # thus provides bash, apt-get, etc. diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile index 98d9ecd9446f..df8c6f2598be 100644 --- a/integration-tests/docker/Dockerfile +++ b/integration-tests/docker/Dockerfile @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG JDK_VERSION=11-slim-buster +ARG JDK_VERSION=17-slim-buster FROM openjdk:$JDK_VERSION as druidbase # Bundle everything into one script so cleanup can reduce image size. diff --git a/pom.xml b/pom.xml index aa9969013bf4..869b8022164b 100644 --- a/pom.xml +++ b/pom.xml @@ -70,7 +70,7 @@ - 11 + 17 ${java.version} ${java.version} UTF-8 @@ -96,8 +96,9 @@ 2.35.1 8.5.4 32.0.1-jre - - 4.2.2 + + 5.1.0 + 4.2.2 1.3 9.4.54.v20240208 1.19.4 @@ -579,7 +580,7 @@ com.google.inject.extensions guice-multibindings - ${guice.version} + ${guice-multibindings.version} com.google.inject.extensions @@ -2055,9 +2056,11 @@ -J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + -J--add-exports=java.base/sun.nio.ch=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED - + -J--add-opens=java.base/java.lang=ALL-UNNAMED + com.google.errorprone From 49af8892b48836fb1c7bcb9cf9cf97927976182e Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Fri, 15 Nov 2024 10:36:50 +0530 Subject: [PATCH 23/61] Exclude spotbugs failure patterns --- codestyle/spotbugs-exclude.xml | 11 +++++++++++ pom.xml | 5 +++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/codestyle/spotbugs-exclude.xml b/codestyle/spotbugs-exclude.xml index 764ae7c73513..9a2ba6ecf1c9 100644 --- a/codestyle/spotbugs-exclude.xml +++ b/codestyle/spotbugs-exclude.xml @@ -137,4 +137,15 @@ + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 869b8022164b..be7074b916fb 100644 --- a/pom.xml +++ b/pom.xml @@ -73,6 +73,7 @@ 17 ${java.version} ${java.version} + ${java.version} UTF-8 0.9.0.M2 5.5.0 @@ -1519,13 +1520,13 @@ com.github.spotbugs spotbugs-maven-plugin - 4.2.0 + 4.8.6.6 com.github.spotbugs spotbugs - 4.2.2 + 4.8.6 From 68783e1b894725ff42a96b9ecfa5eb0de355fa59 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Fri, 15 Nov 2024 10:39:04 +0530 Subject: [PATCH 24/61] Update licenses.yaml --- licenses.yaml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/licenses.yaml b/licenses.yaml index a12ab1b01e4f..2fa359adbdd2 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -367,23 +367,19 @@ libraries: --- -name: Guice +name: Guice Core and Extensions license_category: binary module: java-core license_name: Apache License version 2.0 -version: 4.2.2 +version: 5.1.0 libraries: - com.google.inject: guice - - com.google.inject.extensions: guice-multibindings - com.google.inject.extensions: guice-servlet - com.google.inject.extensions: guice-assistedinject notices: - guice: | Google Guice - Core Library Copyright 2006-2016 Google, Inc. - - guice-multibindings: | - Google Guice - Extensions - MultiBindings - Copyright 2006-2016 Google, Inc. - guice-servlet: | Google Guice - Extensions - Servlet Copyright 2006-2016 Google, Inc. @@ -393,6 +389,20 @@ notices: --- +name: Guice MultiBindings Extension +license_category: binary +module: java-core +license_name: Apache License version 2.0 +version: 4.2.2 +libraries: + - com.google.inject.extensions: guice-multibindings +notices: + - guice-multibindings: | + Google Guice - Extensions - MultiBindings + Copyright 2006-2016 Google, Inc. + +--- + name: JsonPath license_category: binary module: java-core From 81c3ef5e1fc6c60b6e9bd3746d36d9979c604879 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Fri, 15 Nov 2024 10:44:41 +0530 Subject: [PATCH 25/61] Set release version as Java 11 --- pom.xml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index be7074b916fb..07d23b19bee5 100644 --- a/pom.xml +++ b/pom.xml @@ -71,9 +71,7 @@ 17 - ${java.version} - ${java.version} - ${java.version} + 11 UTF-8 0.9.0.M2 5.5.0 From 250525746aeb5121cae9ec2f45113ebb7ae13c4d Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Fri, 15 Nov 2024 11:13:16 +0530 Subject: [PATCH 26/61] Change docker image in distribution Dockerfile --- distribution/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/docker/Dockerfile b/distribution/docker/Dockerfile index f9764c2df967..a016b3e12605 100644 --- a/distribution/docker/Dockerfile +++ b/distribution/docker/Dockerfile @@ -23,7 +23,7 @@ ARG JDK_VERSION=17 # This is because it's not able to build the distribution on arm64 due to dependency problem of web-console. See: https://github.com/apache/druid/issues/13012 # Since only java jars are shipped in the final image, it's OK to build the distribution on x64. # Once the web-console dependency problem is resolved, we can remove the --platform directive. -FROM --platform=linux/amd64 maven:3.8.6-jdk-11-slim as builder +FROM --platform=linux/amd64 maven:3.8.5-openjdk-17-slim as builder # Rebuild from source in this stage # This can be unset if the tarball was already built outside of Docker From 6f637f8dbb80977029b1daf5955fe0bd4b44972f Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Fri, 15 Nov 2024 11:19:09 +0530 Subject: [PATCH 27/61] Change java.version to 11 in pom.xml --- pom.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 07d23b19bee5..74482911815b 100644 --- a/pom.xml +++ b/pom.xml @@ -70,8 +70,9 @@ - 17 - 11 + 11 + ${java.version} + ${java.version} UTF-8 0.9.0.M2 5.5.0 From 259f2d4af019e4bf941a245d5ef7e273bac994e7 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Fri, 15 Nov 2024 12:16:26 +0530 Subject: [PATCH 28/61] This reverts commit 68783e1b894725ff42a96b9ecfa5eb0de355fa59. Downgrade guice back to 4.2.2 --- .../cloudfiles-extensions/pom.xml | 2 +- licenses.yaml | 22 +++++-------------- pom.xml | 7 +++--- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index 3264045028f3..a29a713f2e0a 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -60,7 +60,7 @@ com.google.inject.extensions guice-multibindings - ${guice-multibindings.version} + ${guice.version} provided diff --git a/licenses.yaml b/licenses.yaml index 2fa359adbdd2..a12ab1b01e4f 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -367,19 +367,23 @@ libraries: --- -name: Guice Core and Extensions +name: Guice license_category: binary module: java-core license_name: Apache License version 2.0 -version: 5.1.0 +version: 4.2.2 libraries: - com.google.inject: guice + - com.google.inject.extensions: guice-multibindings - com.google.inject.extensions: guice-servlet - com.google.inject.extensions: guice-assistedinject notices: - guice: | Google Guice - Core Library Copyright 2006-2016 Google, Inc. + - guice-multibindings: | + Google Guice - Extensions - MultiBindings + Copyright 2006-2016 Google, Inc. - guice-servlet: | Google Guice - Extensions - Servlet Copyright 2006-2016 Google, Inc. @@ -389,20 +393,6 @@ notices: --- -name: Guice MultiBindings Extension -license_category: binary -module: java-core -license_name: Apache License version 2.0 -version: 4.2.2 -libraries: - - com.google.inject.extensions: guice-multibindings -notices: - - guice-multibindings: | - Google Guice - Extensions - MultiBindings - Copyright 2006-2016 Google, Inc. - ---- - name: JsonPath license_category: binary module: java-core diff --git a/pom.xml b/pom.xml index 74482911815b..e70d6f19e0cf 100644 --- a/pom.xml +++ b/pom.xml @@ -96,9 +96,8 @@ 2.35.1 8.5.4 32.0.1-jre - - 5.1.0 - 4.2.2 + + 4.2.2 1.3 9.4.54.v20240208 1.19.4 @@ -580,7 +579,7 @@ com.google.inject.extensions guice-multibindings - ${guice-multibindings.version} + ${guice.version} com.google.inject.extensions From 7241a876986ca8e26e137b2a6ef44fdda6009709 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Fri, 15 Nov 2024 12:41:05 +0530 Subject: [PATCH 29/61] Downgrade spotbugs back to 4.2.2 --- codestyle/spotbugs-exclude.xml | 11 ----------- pom.xml | 4 ++-- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/codestyle/spotbugs-exclude.xml b/codestyle/spotbugs-exclude.xml index 9a2ba6ecf1c9..764ae7c73513 100644 --- a/codestyle/spotbugs-exclude.xml +++ b/codestyle/spotbugs-exclude.xml @@ -137,15 +137,4 @@ - - - - - - - - - - diff --git a/pom.xml b/pom.xml index e70d6f19e0cf..f36f6de80d08 100644 --- a/pom.xml +++ b/pom.xml @@ -1518,13 +1518,13 @@ com.github.spotbugs spotbugs-maven-plugin - 4.8.6.6 + 4.2.0 com.github.spotbugs spotbugs - 4.8.6 + 4.2.2 From 79581329aa7ada3e22e65c01f8db22c55fc81024 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Fri, 15 Nov 2024 13:19:08 +0530 Subject: [PATCH 30/61] Remove redundant add-opens --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index f36f6de80d08..7fdbe3319431 100644 --- a/pom.xml +++ b/pom.xml @@ -2058,7 +2058,6 @@ -J--add-exports=java.base/sun.nio.ch=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED - -J--add-opens=java.base/java.lang=ALL-UNNAMED From b53b1f8f70c6da26cad30ba94c3873117446baa1 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Fri, 15 Nov 2024 13:19:14 +0530 Subject: [PATCH 31/61] Revert "Downgrade spotbugs back to 4.2.2" This reverts commit 7241a876986ca8e26e137b2a6ef44fdda6009709. --- codestyle/spotbugs-exclude.xml | 11 +++++++++++ pom.xml | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/codestyle/spotbugs-exclude.xml b/codestyle/spotbugs-exclude.xml index 764ae7c73513..9a2ba6ecf1c9 100644 --- a/codestyle/spotbugs-exclude.xml +++ b/codestyle/spotbugs-exclude.xml @@ -137,4 +137,15 @@ + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 7fdbe3319431..dbeccf71d24e 100644 --- a/pom.xml +++ b/pom.xml @@ -1518,13 +1518,13 @@ com.github.spotbugs spotbugs-maven-plugin - 4.2.0 + 4.8.6.6 com.github.spotbugs spotbugs - 4.2.2 + 4.8.6 From 534e185f3df07e6e7203e89c0eb5cb469efd2671 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 15:40:36 +0530 Subject: [PATCH 32/61] Address Guice related review comments --- extensions-contrib/cloudfiles-extensions/pom.xml | 2 -- extensions-contrib/compressed-bigdecimal/pom.xml | 1 - integration-tests/pom.xml | 1 - pom.xml | 1 - 4 files changed, 5 deletions(-) diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index a29a713f2e0a..0fd5018d1311 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -48,7 +48,6 @@ com.google.inject guice - ${guice.version} aopalliance @@ -60,7 +59,6 @@ com.google.inject.extensions guice-multibindings - ${guice.version} provided diff --git a/extensions-contrib/compressed-bigdecimal/pom.xml b/extensions-contrib/compressed-bigdecimal/pom.xml index 43caf7c3c323..2ef1a7740cc9 100644 --- a/extensions-contrib/compressed-bigdecimal/pom.xml +++ b/extensions-contrib/compressed-bigdecimal/pom.xml @@ -145,7 +145,6 @@ com.google.inject guice - 4.2.2 provided diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 983210690983..40110663009d 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -372,7 +372,6 @@ com.google.inject.extensions guice-servlet - ${guice.version} io.confluent diff --git a/pom.xml b/pom.xml index dbeccf71d24e..487c9044c66a 100644 --- a/pom.xml +++ b/pom.xml @@ -96,7 +96,6 @@ 2.35.1 8.5.4 32.0.1-jre - 4.2.2 1.3 9.4.54.v20240208 From b1022aa0c253d8ecc4bbf2b2a9cf2cbbc6c02c96 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 16:45:03 +0530 Subject: [PATCH 33/61] Remove javac.target from benchmarks/pom.xml --- benchmarks/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 155b4c049a42..bd8c89b4d7f6 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -240,7 +240,6 @@ UTF-8 1.21 - 11 benchmarks From 46f495bd60896d45b1aed4db6dc2a3676cb7ac00 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 16:47:18 +0530 Subject: [PATCH 34/61] Remove source and target in favor of maven.compiler.release --- pom.xml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 487c9044c66a..66b16574b9d8 100644 --- a/pom.xml +++ b/pom.xml @@ -71,8 +71,7 @@ 11 - ${java.version} - ${java.version} + ${java.version} UTF-8 0.9.0.M2 5.5.0 @@ -1564,7 +1563,7 @@ jdk-unsafe @@ -1937,8 +1936,7 @@ maven-compiler-plugin 3.11.0 - ${maven.compiler.source} - ${maven.compiler.target} + ${maven.compiler.release} From 42a3da3b410bd79562908edc54ba2a4190bf1739 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 17:16:39 +0530 Subject: [PATCH 35/61] Fix revised ITs by adding add-opens JVM args --- integration-tests-ex/cases/pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration-tests-ex/cases/pom.xml b/integration-tests-ex/cases/pom.xml index e6726db5ab8e..94c42a0555b4 100644 --- a/integration-tests-ex/cases/pom.xml +++ b/integration-tests-ex/cases/pom.xml @@ -479,6 +479,9 @@ maven-failsafe-plugin 3.0.0-M7 + + --add-opens java.base/java.lang=ALL-UNNAMED + From a47f575bca489f38d7205b22bbdd80ecaec84af1 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 17:28:54 +0530 Subject: [PATCH 36/61] Swap build_jdk and runtime_jdk for the experimental test --- .github/workflows/cron-job-its.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cron-job-its.yml b/.github/workflows/cron-job-its.yml index 877c3389ebb4..31d15aaec715 100644 --- a/.github/workflows/cron-job-its.yml +++ b/.github/workflows/cron-job-its.yml @@ -52,7 +52,6 @@ jobs: run: | ./it.sh ci - # This test also validates building Druid on Java 21, and running it on Java 17. integration-index-tests-middleManager: strategy: fail-fast: false @@ -61,8 +60,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml needs: build with: - build_jdk: 21.0.4 - runtime_jdk: 17 + build_jdk: 17 + runtime_jdk: 21.0.4 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: middleManager group: ${{ matrix.testing_group }} From f3bd3a680a1d3e3557c0602d173554df77bc8872 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 18:42:46 +0530 Subject: [PATCH 37/61] Specify target JDK for maven-pmd-plugin --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 66b16574b9d8..97bc003184e3 100644 --- a/pom.xml +++ b/pom.xml @@ -1543,6 +1543,7 @@ target/generated-sources/ + ${maven.compiler.release} From 5a5d859d0bb0bae5518068fbf4fd787157f2faaf Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 20:44:14 +0530 Subject: [PATCH 38/61] Revert "Address Guice related review comments" This reverts commit 534e185f3df07e6e7203e89c0eb5cb469efd2671. --- extensions-contrib/cloudfiles-extensions/pom.xml | 2 ++ extensions-contrib/compressed-bigdecimal/pom.xml | 1 + integration-tests/pom.xml | 1 + pom.xml | 1 + 4 files changed, 5 insertions(+) diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index 0fd5018d1311..a29a713f2e0a 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -48,6 +48,7 @@ com.google.inject guice + ${guice.version} aopalliance @@ -59,6 +60,7 @@ com.google.inject.extensions guice-multibindings + ${guice.version} provided diff --git a/extensions-contrib/compressed-bigdecimal/pom.xml b/extensions-contrib/compressed-bigdecimal/pom.xml index 2ef1a7740cc9..43caf7c3c323 100644 --- a/extensions-contrib/compressed-bigdecimal/pom.xml +++ b/extensions-contrib/compressed-bigdecimal/pom.xml @@ -145,6 +145,7 @@ com.google.inject guice + 4.2.2 provided diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 40110663009d..983210690983 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -372,6 +372,7 @@ com.google.inject.extensions guice-servlet + ${guice.version} io.confluent diff --git a/pom.xml b/pom.xml index 97bc003184e3..a37b47dcbf2f 100644 --- a/pom.xml +++ b/pom.xml @@ -95,6 +95,7 @@ 2.35.1 8.5.4 32.0.1-jre + 4.2.2 1.3 9.4.54.v20240208 From 6abf4d78f576bffbea93bea41e2223294b49b0c3 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 20:45:01 +0530 Subject: [PATCH 39/61] Revert "Revert "Address Guice related review comments"" This reverts commit 5a5d859d0bb0bae5518068fbf4fd787157f2faaf. --- extensions-contrib/cloudfiles-extensions/pom.xml | 2 -- extensions-contrib/compressed-bigdecimal/pom.xml | 1 - integration-tests/pom.xml | 1 - pom.xml | 1 - 4 files changed, 5 deletions(-) diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index a29a713f2e0a..0fd5018d1311 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -48,7 +48,6 @@ com.google.inject guice - ${guice.version} aopalliance @@ -60,7 +59,6 @@ com.google.inject.extensions guice-multibindings - ${guice.version} provided diff --git a/extensions-contrib/compressed-bigdecimal/pom.xml b/extensions-contrib/compressed-bigdecimal/pom.xml index 43caf7c3c323..2ef1a7740cc9 100644 --- a/extensions-contrib/compressed-bigdecimal/pom.xml +++ b/extensions-contrib/compressed-bigdecimal/pom.xml @@ -145,7 +145,6 @@ com.google.inject guice - 4.2.2 provided diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 983210690983..40110663009d 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -372,7 +372,6 @@ com.google.inject.extensions guice-servlet - ${guice.version} io.confluent diff --git a/pom.xml b/pom.xml index a37b47dcbf2f..97bc003184e3 100644 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,6 @@ 2.35.1 8.5.4 32.0.1-jre - 4.2.2 1.3 9.4.54.v20240208 From f57c64d99abd37ca87387ff362568f3c7a59266a Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 20:49:09 +0530 Subject: [PATCH 40/61] Try adding scope provided to guice dependencies --- extensions-contrib/cloudfiles-extensions/pom.xml | 1 + integration-tests/pom.xml | 1 + 2 files changed, 2 insertions(+) diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index 0fd5018d1311..8fc13f3b186f 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -54,6 +54,7 @@ aopalliance + provided diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 40110663009d..29314e018c78 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -372,6 +372,7 @@ com.google.inject.extensions guice-servlet + provided io.confluent From 41bd77e56eb8f363c91cdb2e5bd14d0fc416b70d Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 21:04:34 +0530 Subject: [PATCH 41/61] Try reverting Guice changes: Attempt 1 --- extensions-contrib/cloudfiles-extensions/pom.xml | 3 ++- extensions-contrib/compressed-bigdecimal/pom.xml | 1 + integration-tests/pom.xml | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index 8fc13f3b186f..a29a713f2e0a 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -48,18 +48,19 @@ com.google.inject guice + ${guice.version} aopalliance aopalliance - provided com.google.inject.extensions guice-multibindings + ${guice.version} provided diff --git a/extensions-contrib/compressed-bigdecimal/pom.xml b/extensions-contrib/compressed-bigdecimal/pom.xml index 2ef1a7740cc9..9c8878ca243f 100644 --- a/extensions-contrib/compressed-bigdecimal/pom.xml +++ b/extensions-contrib/compressed-bigdecimal/pom.xml @@ -145,6 +145,7 @@ com.google.inject guice + ${guice.version} provided diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 29314e018c78..983210690983 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -372,7 +372,7 @@ com.google.inject.extensions guice-servlet - provided + ${guice.version} io.confluent From 2dd49d9aeee84473a44c7be33702ab2b52db16fe Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 21:19:19 +0530 Subject: [PATCH 42/61] Try reverting Guice changes: Attempt 2 --- extensions-contrib/compressed-bigdecimal/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-contrib/compressed-bigdecimal/pom.xml b/extensions-contrib/compressed-bigdecimal/pom.xml index 9c8878ca243f..43caf7c3c323 100644 --- a/extensions-contrib/compressed-bigdecimal/pom.xml +++ b/extensions-contrib/compressed-bigdecimal/pom.xml @@ -145,7 +145,7 @@ com.google.inject guice - ${guice.version} + 4.2.2 provided From 0a596021065acb918794c5105bb0c01640ad7f38 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 21:27:43 +0530 Subject: [PATCH 43/61] Revert back to working version of Guice --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 97bc003184e3..a37b47dcbf2f 100644 --- a/pom.xml +++ b/pom.xml @@ -95,6 +95,7 @@ 2.35.1 8.5.4 32.0.1-jre + 4.2.2 1.3 9.4.54.v20240208 From 3ab91341fde09d393347203c9118d5987a04704d Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 21:33:00 +0530 Subject: [PATCH 44/61] Trigger Build From 88183be9cf2af7303873e33a6802e1f388f729f1 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 21:45:33 +0530 Subject: [PATCH 45/61] Trigger Build: Attempt 2 From 12f63cfd4378eec9b45a30242f192557fb227047 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 21:51:52 +0530 Subject: [PATCH 46/61] Try maven 3.8.4 image to see if it helps with multithreading flakes --- distribution/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/docker/Dockerfile b/distribution/docker/Dockerfile index a016b3e12605..0ac517a29ba1 100644 --- a/distribution/docker/Dockerfile +++ b/distribution/docker/Dockerfile @@ -23,7 +23,7 @@ ARG JDK_VERSION=17 # This is because it's not able to build the distribution on arm64 due to dependency problem of web-console. See: https://github.com/apache/druid/issues/13012 # Since only java jars are shipped in the final image, it's OK to build the distribution on x64. # Once the web-console dependency problem is resolved, we can remove the --platform directive. -FROM --platform=linux/amd64 maven:3.8.5-openjdk-17-slim as builder +FROM --platform=linux/amd64 maven:3.8.4-openjdk-17-slim as builder # Rebuild from source in this stage # This can be unset if the tarball was already built outside of Docker From 65d40b58977053e46af1755c02d94facfc4fd00d Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 21:56:18 +0530 Subject: [PATCH 47/61] Trigger Build From 11a73239c2a16f1c3a3c3b054bee210458f20ea2 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 22:02:31 +0530 Subject: [PATCH 48/61] Trigger Build: Attempt 2 From 9ce24090b54626024da151102e215faad73b3d00 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 22:07:20 +0530 Subject: [PATCH 49/61] Address Guice related review comments --- extensions-contrib/cloudfiles-extensions/pom.xml | 2 -- extensions-contrib/compressed-bigdecimal/pom.xml | 1 - integration-tests/pom.xml | 1 - pom.xml | 1 - 4 files changed, 5 deletions(-) diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index a29a713f2e0a..0fd5018d1311 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -48,7 +48,6 @@ com.google.inject guice - ${guice.version} aopalliance @@ -60,7 +59,6 @@ com.google.inject.extensions guice-multibindings - ${guice.version} provided diff --git a/extensions-contrib/compressed-bigdecimal/pom.xml b/extensions-contrib/compressed-bigdecimal/pom.xml index 43caf7c3c323..2ef1a7740cc9 100644 --- a/extensions-contrib/compressed-bigdecimal/pom.xml +++ b/extensions-contrib/compressed-bigdecimal/pom.xml @@ -145,7 +145,6 @@ com.google.inject guice - 4.2.2 provided diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 983210690983..40110663009d 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -372,7 +372,6 @@ com.google.inject.extensions guice-servlet - ${guice.version} io.confluent diff --git a/pom.xml b/pom.xml index a37b47dcbf2f..97bc003184e3 100644 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,6 @@ 2.35.1 8.5.4 32.0.1-jre - 4.2.2 1.3 9.4.54.v20240208 From 71825a0eb4a7d24f63e9b5628e016e3743c52c4c Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 22:18:02 +0530 Subject: [PATCH 50/61] Trigger Build From 2d211b7b25dd5398570d836488c6448f518b84b1 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Mon, 18 Nov 2024 22:28:58 +0530 Subject: [PATCH 51/61] Trigger Build: Attempt 2 From d1bf1b74c58e83b563df0c35274ed0f47bcfa899 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Tue, 19 Nov 2024 12:23:02 +0530 Subject: [PATCH 52/61] Remove java-9+ profile --- pom.xml | 77 +++++++++++++++++++++------------------------------------ 1 file changed, 28 insertions(+), 49 deletions(-) diff --git a/pom.xml b/pom.xml index 97bc003184e3..3db693fc70ec 100644 --- a/pom.xml +++ b/pom.xml @@ -130,7 +130,33 @@ 1.42.3 v1-rev20230606-2.0.0 2.29.1 - + + + + + + + + + + --add-exports=java.base/jdk.internal.ref=ALL-UNNAMED + --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED + + + --add-opens=java.base/java.io=ALL-UNNAMED + + + --add-opens=java.base/java.lang=ALL-UNNAMED + + + --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED + + + --add-opens=java.base/java.util=ALL-UNNAMED + maven.org Maven Central Repository @@ -1936,6 +1962,7 @@ org.apache.maven.plugins maven-compiler-plugin 3.11.0 + true ${maven.compiler.release} @@ -1975,54 +2002,6 @@ - - java-9+ - - [9,) - - - - - - - - - - - - --add-exports=java.base/jdk.internal.ref=ALL-UNNAMED - --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED - --add-opens=java.base/java.nio=ALL-UNNAMED - --add-opens=java.base/sun.nio.ch=ALL-UNNAMED - --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED - - - --add-opens=java.base/java.io=ALL-UNNAMED - - - --add-opens=java.base/java.lang=ALL-UNNAMED - - - --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED - - - --add-opens=java.base/java.util=ALL-UNNAMED - - - - - - org.apache.maven.plugins - maven-compiler-plugin - true - - - ${java.version} - - - - - strict From a40b0d338387105bb246ebdef9abb67c79b8f64c Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Tue, 19 Nov 2024 14:38:43 +0530 Subject: [PATCH 53/61] Run revised-it test suite only for JDK 17 --- .github/workflows/revised-its.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/revised-its.yml b/.github/workflows/revised-its.yml index 9585fe996e11..3e7394268020 100644 --- a/.github/workflows/revised-its.yml +++ b/.github/workflows/revised-its.yml @@ -66,7 +66,8 @@ jobs: strategy: fail-fast: false matrix: - jdk: [11, 17] + # jdk: [11, 17] + jdk: [17] it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat, Security, Query] #indexer: [indexer, middleManager] indexer: [middleManager] From c6f2364cded7a3c603cd8612379ab6eb665cee7f Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Tue, 19 Nov 2024 14:43:59 +0530 Subject: [PATCH 54/61] Add strong encapsulation args to int-tests-config-file profile --- integration-tests/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 40110663009d..7bea258c6472 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -772,6 +772,7 @@ + ${jdk.strong.encapsulation.argLine} -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Ddruid.test.config.type=configFile From e684f3c70a302e413369b08147b3965ad96cad8d Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Tue, 19 Nov 2024 19:45:18 +0530 Subject: [PATCH 55/61] Trigger Build From 2098c6474898ea7e02e03a3732b285dff4b32cca Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Wed, 20 Nov 2024 09:26:15 +0530 Subject: [PATCH 56/61] Update maven-dependency-plugin and maven-pmd-plugin --- codestyle/pmd-ruleset.xml | 1 - pom.xml | 9 +++++++-- server/pom.xml | 5 +++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/codestyle/pmd-ruleset.xml b/codestyle/pmd-ruleset.xml index f10d4f3da197..7d7285d0a516 100644 --- a/codestyle/pmd-ruleset.xml +++ b/codestyle/pmd-ruleset.xml @@ -29,5 +29,4 @@ This ruleset defines the PMD rules for the Apache Druid project. - diff --git a/pom.xml b/pom.xml index 3db693fc70ec..f6e9728ae03a 100644 --- a/pom.xml +++ b/pom.xml @@ -1559,7 +1559,7 @@ org.apache.maven.plugins maven-pmd-plugin - 3.16.0 + 3.26.0 false true @@ -1876,7 +1876,12 @@ org.apache.maven.plugins maven-dependency-plugin - 3.1.2 + 3.3.0 + + + * + + org.apache.maven.plugins diff --git a/server/pom.xml b/server/pom.xml index 206dd9dbeb53..45ca282f836f 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -217,6 +217,11 @@ org.apache.logging.log4j log4j-core + + org.checkerframework + checker-qual + ${checkerframework.version} + com.fasterxml.jackson.datatype jackson-datatype-joda From 0c17ecee355f8bd88ef517eb71b7274d68369fe9 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Wed, 20 Nov 2024 15:00:18 +0530 Subject: [PATCH 57/61] Fix ITMultiStageQueryWorkerFaultTolerance --- ...ITMultiStageQueryWorkerFaultTolerance.java | 3 +- .../wikipedia_msq_select_query_ha.json | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQueryWorkerFaultTolerance.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQueryWorkerFaultTolerance.java index f45ce3ad0e96..462cf5fb992f 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQueryWorkerFaultTolerance.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQueryWorkerFaultTolerance.java @@ -99,7 +99,7 @@ public void testMsqIngestionAndQuerying() throws Exception + " regionIsoCode\n" + "FROM TABLE(\n" + " EXTERN(\n" - + " '{\"type\":\"local\",\"files\":[\"/resources/data/batch_index/json/wikipedia_index_data1.json\",\"/resources/data/batch_index/json/wikipedia_index_data1.json\"]}',\n" + + " '{\"type\":\"local\",\"files\":[\"/resources/data/batch_index/json/wikipedia_index_data1.json\",\"/resources/data/batch_index/json/wikipedia_index_data1.json\",\"/resources/data/batch_index/json/wikipedia_index_data1.json\",\"/resources/data/batch_index/json/wikipedia_index_data1.json\"]}',\n" + " '{\"type\":\"json\"}',\n" + " '[{\"type\":\"string\",\"name\":\"timestamp\"},{\"type\":\"string\",\"name\":\"isRobot\"},{\"type\":\"string\",\"name\":\"diffUrl\"},{\"type\":\"long\",\"name\":\"added\"},{\"type\":\"string\",\"name\":\"countryIsoCode\"},{\"type\":\"string\",\"name\":\"regionName\"},{\"type\":\"string\",\"name\":\"channel\"},{\"type\":\"string\",\"name\":\"flags\"},{\"type\":\"long\",\"name\":\"delta\"},{\"type\":\"string\",\"name\":\"isUnpatrolled\"},{\"type\":\"string\",\"name\":\"isNew\"},{\"type\":\"double\",\"name\":\"deltaBucket\"},{\"type\":\"string\",\"name\":\"isMinor\"},{\"type\":\"string\",\"name\":\"isAnonymous\"},{\"type\":\"long\",\"name\":\"deleted\"},{\"type\":\"string\",\"name\":\"cityName\"},{\"type\":\"long\",\"name\":\"metroCode\"},{\"type\":\"string\",\"name\":\"namespace\"},{\"type\":\"string\",\"name\":\"comment\"},{\"type\":\"string\",\"name\":\"page\"},{\"type\":\"long\",\"name\":\"commentLength\"},{\"type\":\"string\",\"name\":\"countryName\"},{\"type\":\"string\",\"name\":\"user\"},{\"type\":\"string\",\"name\":\"regionIsoCode\"}]'\n" + " )\n" @@ -139,7 +139,6 @@ public void testMsqIngestionAndQuerying() throws Exception private void killTaskAbruptly(String taskIdToKill) { - String command = "jps -mlv | grep -i peon | grep -i " + taskIdToKill + " |awk '{print $1}'"; ITRetryUtil.retryUntil(() -> { diff --git a/integration-tests-ex/cases/src/test/resources/multi-stage-query/wikipedia_msq_select_query_ha.json b/integration-tests-ex/cases/src/test/resources/multi-stage-query/wikipedia_msq_select_query_ha.json index 992eda01a26a..5106257eb6a8 100644 --- a/integration-tests-ex/cases/src/test/resources/multi-stage-query/wikipedia_msq_select_query_ha.json +++ b/integration-tests-ex/cases/src/test/resources/multi-stage-query/wikipedia_msq_select_query_ha.json @@ -18,6 +18,30 @@ "deleted": 200, "namespace": "article" }, + { + "__time": 1377910953000, + "isRobot": null, + "added": 57, + "delta": -143, + "deleted": 200, + "namespace": "article" + }, + { + "__time": 1377910953000, + "isRobot": null, + "added": 57, + "delta": -143, + "deleted": 200, + "namespace": "article" + }, + { + "__time": 1377919965000, + "isRobot": null, + "added": 459, + "delta": 330, + "deleted": 129, + "namespace": "wikipedia" + }, { "__time": 1377919965000, "isRobot": null, @@ -34,6 +58,30 @@ "deleted": 129, "namespace": "wikipedia" }, + { + "__time": 1377919965000, + "isRobot": null, + "added": 459, + "delta": 330, + "deleted": 129, + "namespace": "wikipedia" + }, + { + "__time": 1377933081000, + "isRobot": null, + "added": 123, + "delta": 111, + "deleted": 12, + "namespace": "article" + }, + { + "__time": 1377933081000, + "isRobot": null, + "added": 123, + "delta": 111, + "deleted": 12, + "namespace": "article" + }, { "__time": 1377933081000, "isRobot": null, From df876700871c4dafdeb46ef0cde160b364eb563d Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Wed, 20 Nov 2024 18:11:05 +0530 Subject: [PATCH 58/61] Fix ITMultiStageQueryWorkerFaultTolerance: Attempt 2 --- .../msq/ITMultiStageQueryWorkerFaultTolerance.java | 10 +++------- .../apache/druid/testing/utils/MsqTestQueryHelper.java | 1 - 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQueryWorkerFaultTolerance.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQueryWorkerFaultTolerance.java index 462cf5fb992f..13c3fdc38463 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQueryWorkerFaultTolerance.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQueryWorkerFaultTolerance.java @@ -157,21 +157,17 @@ private void killTaskAbruptly(String taskIdToKill) } String pidToKill = stdOut.lhs.trim(); if (pidToKill.length() != 0) { - LOG.info("Found PID to kill %s", pidToKill); - // kill worker after 5 seconds - Thread.sleep(5000); LOG.info("Killing pid %s", pidToKill); - druidClusterAdminClient.runCommandInMiddleManagerContainer( + final Pair killResult = druidClusterAdminClient.runCommandInMiddleManagerContainer( "/bin/bash", "-c", "kill -9 " + pidToKill ); + LOG.info(StringUtils.format("Kill command stdout: %s, stderr: %s", killResult.lhs, killResult.rhs)); return true; } else { return false; } - }, true, 6000, 50, StringUtils.format("Figuring out PID for task[%s] to kill abruptly", taskIdToKill)); - - + }, true, 2000, 100, StringUtils.format("Figuring out PID for task[%s] to kill abruptly", taskIdToKill)); } } diff --git a/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java b/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java index 2a4d04f42dbd..c6c360512c8a 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java @@ -190,7 +190,6 @@ public TaskState pollTaskIdForCompletion(String taskId) throws Exception throw new TaskStillRunningException(); }, (Throwable t) -> t instanceof TaskStillRunningException, - 99, 100 ); } From 4daeb038fc7c0260413450cd5fd64575ffe04b61 Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Wed, 20 Nov 2024 19:36:34 +0530 Subject: [PATCH 59/61] Trigger Build From 028689895bc55fa0c469b69191c7e10a424d7f8a Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Wed, 20 Nov 2024 22:20:52 +0530 Subject: [PATCH 60/61] Revert change of MsqTestQueryHelper --- .../java/org/apache/druid/testing/utils/MsqTestQueryHelper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java b/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java index c6c360512c8a..2a4d04f42dbd 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java @@ -190,6 +190,7 @@ public TaskState pollTaskIdForCompletion(String taskId) throws Exception throw new TaskStillRunningException(); }, (Throwable t) -> t instanceof TaskStillRunningException, + 99, 100 ); } From 3cd2e6d7704482f97e380f889a1cfaf78960fbbf Mon Sep 17 00:00:00 2001 From: Akshat Jain Date: Thu, 21 Nov 2024 08:42:33 +0530 Subject: [PATCH 61/61] Remove java17 from the matrix of 2nd phase of unit tests --- .github/workflows/unit-and-integration-tests-unified.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index 9f1294e18409..81375f33c090 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -162,7 +162,7 @@ jobs: fail-fast: false matrix: # Use JDK 21.0.4 to work around https://github.com/apache/druid/issues/17429 - jdk: [ '11', '17', '21.0.4' ] + jdk: [ '11', '21.0.4' ] name: "unit tests (jdk${{ matrix.jdk }}, sql-compat=true)" uses: ./.github/workflows/unit-tests.yml needs: unit-tests