diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7cfeb63cbff0..c47e3fa4557f 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -40,7 +40,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'zulu' - java-version: '8' + java-version: '17' cache: 'maven' # Initializes the CodeQL tools for scanning. diff --git a/.github/workflows/cron-job-its.yml b/.github/workflows/cron-job-its.yml index aa3f28382884..31d15aaec715 100644 --- a/.github/workflows/cron-job-its.yml +++ b/.github/workflows/cron-job-its.yml @@ -28,7 +28,7 @@ on: jobs: build: if: (github.event_name == 'schedule' && github.repository == 'apache/druid') - name: build (jdk8) + name: build (jdk17) runs-on: ubuntu-latest steps: - name: Checkout branch @@ -37,7 +37,7 @@ jobs: - name: setup java uses: actions/setup-java@v4 with: - java-version: '8' + java-version: '17' distribution: 'zulu' - name: Cache Maven m2 repository @@ -60,8 +60,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml needs: build with: - build_jdk: 8 - runtime_jdk: 11 + build_jdk: 17 + runtime_jdk: 21.0.4 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: middleManager group: ${{ matrix.testing_group }} @@ -74,8 +74,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml needs: build with: - build_jdk: 8 - runtime_jdk: 11 + build_jdk: 17 + runtime_jdk: 21.0.4 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: indexer group: ${{ matrix.testing_group }} @@ -88,8 +88,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml needs: build with: - build_jdk: 8 - runtime_jdk: 11 + build_jdk: 17 + runtime_jdk: 21.0.4 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: middleManager override_config_path: ./environment-configs/test-groups/prepopulated-data @@ -103,8 +103,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml needs: build with: - build_jdk: 8 - runtime_jdk: 11 + build_jdk: 17 + runtime_jdk: 21.0.4 testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties use_indexer: ${{ matrix.indexer }} group: other @@ -122,7 +122,7 @@ jobs: - name: setup java uses: actions/setup-java@v4 with: - java-version: '8' + java-version: '17' distribution: 'zulu' cache: maven diff --git a/.github/workflows/revised-its.yml b/.github/workflows/revised-its.yml index 412a3e469737..3e7394268020 100644 --- a/.github/workflows/revised-its.yml +++ b/.github/workflows/revised-its.yml @@ -66,8 +66,8 @@ jobs: strategy: fail-fast: false matrix: - #jdk: [8, 11, 17] - jdk: [8] + # jdk: [11, 17] + jdk: [17] it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat, Security, Query] #indexer: [indexer, middleManager] indexer: [middleManager] @@ -86,8 +86,8 @@ jobs: uses: ./.github/workflows/reusable-revised-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 8 - runtime_jdk: 11 + build_jdk: 17 + runtime_jdk: 21.0.4 use_indexer: middleManager script: ./it.sh github S3DeepStorage it: S3DeepStorage @@ -103,8 +103,8 @@ jobs: uses: ./.github/workflows/reusable-revised-its.yml if: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' && (needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true') }} with: - build_jdk: 8 - runtime_jdk: 8 + build_jdk: 17 + runtime_jdk: 17 use_indexer: middleManager script: ./it.sh github BackwardCompatibilityMain it: BackwardCompatibilityMain diff --git a/.github/workflows/standard-its.yml b/.github/workflows/standard-its.yml index 0083e952ea3d..3ea728dd7c7f 100644 --- a/.github/workflows/standard-its.yml +++ b/.github/workflows/standard-its.yml @@ -51,8 +51,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 8 - runtime_jdk: 8 + build_jdk: 17 + runtime_jdk: 17 testing_groups: -Dgroups=${{ matrix.testing_group }} override_config_path: ./environment-configs/test-groups/prepopulated-data use_indexer: middleManager @@ -67,8 +67,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 8 - runtime_jdk: 8 + build_jdk: 17 + runtime_jdk: 17 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: indexer group: ${{ matrix.testing_group }} @@ -82,8 +82,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 8 - runtime_jdk: 8 + build_jdk: 17 + runtime_jdk: 17 testing_groups: -Dgroups=${{ matrix.testing_group }} use_indexer: middleManager override_config_path: ./environment-configs/test-groups/prepopulated-data @@ -94,7 +94,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [8, 17, 21] + jdk: [11, 17, 21] uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: @@ -115,8 +115,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 8 - runtime_jdk: 8 + build_jdk: 17 + runtime_jdk: 17 testing_groups: -Dgroups=shuffle-deep-store use_indexer: ${{ matrix.indexer }} override_config_path: ./environment-configs/test-groups/shuffle-deep-store @@ -127,8 +127,8 @@ jobs: uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: - build_jdk: 8 - runtime_jdk: 8 + build_jdk: 17 + runtime_jdk: 17 testing_groups: -Dgroups=custom-coordinator-duties use_indexer: middleManager override_config_path: ./environment-configs/test-groups/custom-coordinator-duties @@ -136,7 +136,7 @@ jobs: integration-k8s-leadership-tests: needs: changes - name: (Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITNestedQueryPushDownTest integration test + name: (Compile=openjdk17, Run=openjdk17, Cluster Build On K8s) ITNestedQueryPushDownTest integration test if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} runs-on: ubuntu-22.04 env: @@ -154,7 +154,7 @@ jobs: - name: setup java uses: actions/setup-java@v4 with: - java-version: '8' + java-version: '17' distribution: 'zulu' # the build step produces SNAPSHOT artifacts into the local maven repository, @@ -195,8 +195,8 @@ jobs: indexer: [middleManager, indexer] uses: ./.github/workflows/reusable-standard-its.yml with: - build_jdk: 8 - runtime_jdk: 8 + build_jdk: 17 + runtime_jdk: 17 testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties,centralized-datasource-schema,cds-task-schema-publish-disabled,cds-coordinator-metadata-query-disabled use_indexer: ${{ matrix.indexer }} group: other diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index f32b362f84d9..778a79db7618 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -42,7 +42,7 @@ jobs: fail-fast: false matrix: # Use JDK 21.0.4 to work around https://github.com/apache/druid/issues/17429 - java: [ '8', '11', '17', '21.0.4' ] + java: [ '11', '17', '21.0.4' ] runs-on: ubuntu-latest steps: - name: checkout branch @@ -64,7 +64,7 @@ jobs: - name: script checks # who watches the watchers? - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '17' }} run: ./check_test_suite_test.py - name: (openjdk17) strict compilation @@ -74,43 +74,43 @@ jobs: run: ${MVN} clean -DstrictCompile compile test-compile --fail-at-end ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} - name: maven install - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '17' }} run: | echo 'Running Maven install...' && ${MVN} clean install -q -ff -pl '!distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} - name: checkstyle - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '17' }} run: ${MVN} checkstyle:checkstyle --fail-at-end - name: license checks - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '17' }} run: ./.github/scripts/license_checks_script.sh - name: analyze dependencies - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '17' }} run: | ./.github/scripts/analyze_dependencies_script.sh - name: animal sniffer checks - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '17' }} run: ${MVN} animal-sniffer:check --fail-at-end - name: enforcer checks - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '17' }} run: ${MVN} enforcer:enforce --fail-at-end - name: forbidden api checks - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '17' }} run: ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end - name: pmd checks - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '17' }} run: ${MVN} pmd:check --fail-at-end # TODO: consider adding pmd:cpd-check - name: spotbugs checks - if: ${{ matrix.java == '8' }} + if: ${{ matrix.java == '17' }} run: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks' openrewrite: @@ -122,7 +122,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: 'zulu' - java-version: '8' + java-version: '17' cache: 'maven' - name: maven install diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index f8aff5f56e15..81375f33c090 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -80,7 +80,7 @@ jobs: fail-fast: false matrix: # Use JDK 21.0.4 to work around https://github.com/apache/druid/issues/17429 - jdk: [ '8', '11', '17', '21.0.4' ] + jdk: [ '11', '17', '21.0.4' ] runs-on: ubuntu-latest steps: - name: Checkout branch @@ -162,7 +162,7 @@ jobs: fail-fast: false matrix: # Use JDK 21.0.4 to work around https://github.com/apache/druid/issues/17429 - jdk: [ '11', '17', '21.0.4' ] + jdk: [ '11', '21.0.4' ] name: "unit tests (jdk${{ matrix.jdk }}, sql-compat=true)" uses: ./.github/workflows/unit-tests.yml needs: unit-tests @@ -176,11 +176,11 @@ jobs: fail-fast: false matrix: sql_compatibility: [ false, true ] - name: "unit tests (jdk8, sql-compat=${{ matrix.sql_compatibility }})" + name: "unit tests (jdk17, sql-compat=${{ matrix.sql_compatibility }})" uses: ./.github/workflows/unit-tests.yml needs: build with: - jdk: 8 + jdk: 17 sql_compatibility: ${{ matrix.sql_compatibility }} standard-its: diff --git a/.idea/misc-for-inspection.xml b/.idea/misc-for-inspection.xml index f890d06f1fe6..66feb815ad1d 100644 --- a/.idea/misc-for-inspection.xml +++ b/.idea/misc-for-inspection.xml @@ -84,7 +84,7 @@ - + diff --git a/README.md b/README.md index df543687b5fb..e4100e8cf8a5 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ Find articles written by community members and a calendar of upcoming events on ### Building from source -Please note that JDK 8 or JDK 11 is required to build Druid. +Please note that JDK 11 or JDK 17 is required to build Druid. See the latest [build guide](https://druid.apache.org/docs/latest/development/build.html) for instructions on building Apache Druid from source. diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 89609f573d14..bd8c89b4d7f6 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -64,6 +64,7 @@ com.google.inject.extensions guice-multibindings + provided org.apache.druid @@ -239,7 +240,6 @@ UTF-8 1.21 - 1.8 benchmarks diff --git a/check_test_suite.py b/check_test_suite.py index d55c52b0b736..62a5902421aa 100755 --- a/check_test_suite.py +++ b/check_test_suite.py @@ -22,7 +22,7 @@ # this script does some primitive examination of git diff to determine if a test suite needs to be run or not # these jobs should always be run, no matter what -always_run_jobs = ['license checks', '(openjdk8) packaging check', '(openjdk11) packaging check'] +always_run_jobs = ['license checks', '(openjdk17) packaging check'] # ignore changes to these files completely since they don't impact CI, if the changes are only to these files then all # of CI can be skipped. however, jobs which are always run will still be run even if only these files are changed diff --git a/codestyle/pmd-ruleset.xml b/codestyle/pmd-ruleset.xml index f10d4f3da197..7d7285d0a516 100644 --- a/codestyle/pmd-ruleset.xml +++ b/codestyle/pmd-ruleset.xml @@ -29,5 +29,4 @@ This ruleset defines the PMD rules for the Apache Druid project. - diff --git a/codestyle/spotbugs-exclude.xml b/codestyle/spotbugs-exclude.xml index 764ae7c73513..9a2ba6ecf1c9 100644 --- a/codestyle/spotbugs-exclude.xml +++ b/codestyle/spotbugs-exclude.xml @@ -137,4 +137,15 @@ + + + + + + + + + + diff --git a/distribution/docker/Dockerfile b/distribution/docker/Dockerfile index f9764c2df967..0ac517a29ba1 100644 --- a/distribution/docker/Dockerfile +++ b/distribution/docker/Dockerfile @@ -23,7 +23,7 @@ ARG JDK_VERSION=17 # This is because it's not able to build the distribution on arm64 due to dependency problem of web-console. See: https://github.com/apache/druid/issues/13012 # Since only java jars are shipped in the final image, it's OK to build the distribution on x64. # Once the web-console dependency problem is resolved, we can remove the --platform directive. -FROM --platform=linux/amd64 maven:3.8.6-jdk-11-slim as builder +FROM --platform=linux/amd64 maven:3.8.4-openjdk-17-slim as builder # Rebuild from source in this stage # This can be unset if the tarball was already built outside of Docker diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 8f4f8065f1d9..b8d9e7b9b00b 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -195,9 +195,9 @@ and `druid.tlsPort` properties on each service. Please see `Configuration` secti Druid uses Jetty as an embedded web server. To learn more about TLS/SSL, certificates, and related concepts in Jetty, including explanations of the configuration settings below, see "Configuring SSL/TLS KeyStores" in the [Jetty Operations Guide](https://www.eclipse.org/jetty/documentation.php). -For information about TLS/SSL support in Java in general, see the [Java Secure Socket Extension (JSSE) Reference Guide](http://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html). +For information about TLS/SSL support in Java in general, see the [Java Secure Socket Extension (JSSE) Reference Guide](https://docs.oracle.com/en/java/javase/11/security/java-secure-socket-extension-jsse-reference-guide.html). The [Java Cryptography Architecture -Standard Algorithm Name Documentation for JDK 8](http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html) lists all possible +Standard Algorithm Name Documentation for JDK 11](https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html) lists all possible values for the following properties, among others provided by the Java implementation. |Property|Description|Default|Required| @@ -230,7 +230,7 @@ These properties apply to the SSLContext that will be provided to the internal H |`druid.client.https.trustStoreAlgorithm`|Algorithm to be used by TrustManager to validate certificate chains|`javax.net.ssl.TrustManagerFactory.getDefaultAlgorithm()`|no| |`druid.client.https.trustStorePassword`|The [Password Provider](../operations/password-provider.md) or String password for the Trust Store.|none|yes| -This [document](http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html) lists all the possible +This [document](https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html) lists all the possible values for the above mentioned configs among others provided by Java implementation. ### Authentication and authorization diff --git a/docs/development/extensions-core/simple-client-sslcontext.md b/docs/development/extensions-core/simple-client-sslcontext.md index 7452a7b5bb1b..db1a37afe414 100644 --- a/docs/development/extensions-core/simple-client-sslcontext.md +++ b/docs/development/extensions-core/simple-client-sslcontext.md @@ -23,9 +23,9 @@ title: "Simple SSLContext Provider Module" --> -This Apache Druid module contains a simple implementation of [SSLContext](http://docs.oracle.com/javase/8/docs/api/javax/net/ssl/SSLContext.html) +This Apache Druid module contains a simple implementation of [SSLContext](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/javax/net/ssl/SSLContext.html) that will be injected to be used with HttpClient that Druid processes use internally to communicate with each other. To learn more about -Java's SSL support, please refer to [this](http://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html) guide. +Java's SSL support, please refer to [this](https://docs.oracle.com/en/java/javase/11/security/java-secure-socket-extension-jsse-reference-guide.html) guide. |Property|Description|Default|Required| @@ -48,5 +48,5 @@ The following table contains optional parameters for supporting client certifica |`druid.client.https.keyManagerPassword`|The [Password Provider](../../operations/password-provider.md) or String password for the Key Manager.|none|no| |`druid.client.https.validateHostnames`|Validate the hostname of the server. This should not be disabled unless you are using [custom TLS certificate checks](../../operations/tls-support.md) and know that standard hostname validation is not needed.|true|no| -This [document](http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html) lists all the possible +This [document](https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html) lists all the possible values for the above mentioned configs among others provided by Java implementation. diff --git a/docs/ingestion/input-sources.md b/docs/ingestion/input-sources.md index 5ee9fdc4d258..b96050899278 100644 --- a/docs/ingestion/input-sources.md +++ b/docs/ingestion/input-sources.md @@ -184,7 +184,7 @@ Sample specs: |uris|JSON array of URIs where S3 objects to be ingested are located.|None|`uris` or `prefixes` or `objects` must be set| |prefixes|JSON array of URI prefixes for the locations of S3 objects to be ingested. Empty objects starting with one of the given prefixes will be skipped.|None|`uris` or `prefixes` or `objects` must be set| |objects|JSON array of S3 Objects to be ingested.|None|`uris` or `prefixes` or `objects` must be set| -|objectGlob|A glob for the object part of the S3 URI. In the URI `s3://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `s3://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html#getPathMatcher-java.lang.String-).|None|no| +|objectGlob|A glob for the object part of the S3 URI. In the URI `s3://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `s3://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)).|None|no| |systemFields|JSON array of system fields to return as part of input rows. Possible values: `__file_uri` (S3 URI starting with `s3://`), `__file_bucket` (S3 bucket), and `__file_path` (S3 object key).|None|no| | endpointConfig |Config for overriding the default S3 endpoint and signing region. This would allow ingesting data from a different S3 store. Please see [s3 config](../development/extensions-core/s3.md#connecting-to-s3-configuration) for more information.|None|No (defaults will be used if not given) | clientConfig |S3 client properties for the overridden s3 endpoint. This is used in conjunction with `endPointConfig`. Please see [s3 config](../development/extensions-core/s3.md#connecting-to-s3-configuration) for more information.|None|No (defaults will be used if not given) @@ -289,7 +289,7 @@ Sample specs: |uris|JSON array of URIs where Google Cloud Storage objects to be ingested are located.|None|`uris` or `prefixes` or `objects` must be set| |prefixes|JSON array of URI prefixes for the locations of Google Cloud Storage objects to be ingested. Empty objects starting with one of the given prefixes will be skipped.|None|`uris` or `prefixes` or `objects` must be set| |objects|JSON array of Google Cloud Storage objects to be ingested.|None|`uris` or `prefixes` or `objects` must be set| -|objectGlob|A glob for the object part of the S3 URI. In the URI `s3://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `s3://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html#getPathMatcher-java.lang.String-).|None|no| +|objectGlob|A glob for the object part of the S3 URI. In the URI `s3://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `s3://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)).|None|no| Note that the Google Cloud Storage input source will skip all empty objects only when `prefixes` is specified. @@ -377,7 +377,7 @@ Sample specs: |uris|JSON array of URIs where the Azure objects to be ingested are located. Use this format: `azureStorage://STORAGE_ACCOUNT/CONTAINER/PATH_TO_FILE`|None|One of the following must be set:`uris`, `prefixes`, or `objects`.| |prefixes|JSON array of URI prefixes for the locations of Azure objects to ingest. Use this format`azureStorage://STORAGE_ACCOUNT/CONTAINER/PREFIX`. Empty objects starting with any of the given prefixes are skipped.|None|One of the following must be set:`uris`, `prefixes`, or `objects`.| |objects|JSON array of Azure objects to ingest.|None|One of the following must be set:`uris`, `prefixes`, or `objects`.| -|objectGlob|A glob for the object part of the Azure URI. In the URI `azureStorage://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `azureStorage://foo/bar/file.json` because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html#getPathMatcher-java.lang.String-).|None|no| +|objectGlob|A glob for the object part of the Azure URI. In the URI `azureStorage://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `azureStorage://foo/bar/file.json` because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)).|None|no| |systemFields|JSON array of system fields to return as part of input rows. Possible values: `__file_uri` (Azure blob URI starting with `azureStorage://`), `__file_bucket` (Azure bucket), and `__file_path` (Azure object path).|None|no| |properties|Properties object for overriding the default Azure configuration. See below for more information.|None|No (defaults will be used if not given)| @@ -471,7 +471,7 @@ Sample specs: |uris|JSON array of URIs where the Azure objects to be ingested are located, in the form `azure:///`|None|`uris` or `prefixes` or `objects` must be set| |prefixes|JSON array of URI prefixes for the locations of Azure objects to ingest, in the form `azure:///`. Empty objects starting with one of the given prefixes are skipped.|None|`uris` or `prefixes` or `objects` must be set| |objects|JSON array of Azure objects to ingest.|None|`uris` or `prefixes` or `objects` must be set| -|objectGlob|A glob for the object part of the Azure URI. In the URI `azure://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `azure://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html#getPathMatcher-java.lang.String-).|None|no| +|objectGlob|A glob for the object part of the Azure URI. In the URI `azure://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `azure://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)).|None|no| |systemFields|JSON array of system fields to return as part of input rows. Possible values: `__file_uri` (Azure blob URI starting with `azure://`), `__file_bucket` (Azure bucket), and `__file_path` (Azure object path).|None|no| Note that the Azure input source skips all empty objects only when `prefixes` is specified. diff --git a/docs/operations/java.md b/docs/operations/java.md index 2af122441162..7021ae3e0eff 100644 --- a/docs/operations/java.md +++ b/docs/operations/java.md @@ -27,7 +27,11 @@ a Java runtime for Druid. ## Selecting a Java runtime -Druid fully supports Java 8u92+, Java 11, and Java 17. The project team recommends Java 17. +Druid fully supports Java 11 and Java 17. The project team recommends Java 17. + +:::info +Note: Starting with Apache Druid 32.0.0, support for Java 8 has been removed. +::: The project team recommends using an OpenJDK-based Java distribution. There are many free and actively-supported distributions available, including diff --git a/docs/operations/tls-support.md b/docs/operations/tls-support.md index b5db993eeeb9..543f177bfedd 100644 --- a/docs/operations/tls-support.md +++ b/docs/operations/tls-support.md @@ -37,10 +37,10 @@ Apache Druid uses Jetty as its embedded web server. To get familiar with TLS/SSL, along with related concepts like keys and certificates, read [Configuring Secure Protocols](https://www.eclipse.org/jetty/documentation/jetty-12/operations-guide/index.html#og-protocols-ssl) in the Jetty documentation. -To get more in-depth knowledge of TLS/SSL support in Java in general, refer to the [Java Secure Socket Extension (JSSE) Reference Guide](http://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html). +To get more in-depth knowledge of TLS/SSL support in Java in general, refer to the [Java Secure Socket Extension (JSSE) Reference Guide](https://docs.oracle.com/en/java/javase/11/security/java-secure-socket-extension-jsse-reference-guide.html). The [Class SslContextFactory](https://www.eclipse.org/jetty/javadoc/jetty-11/org/eclipse/jetty/util/ssl/SslContextFactory.html) reference doc can help in understanding TLS/SSL configurations listed below. Finally, [Java Cryptography Architecture -Standard Algorithm Name Documentation for JDK 8](http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html) lists all possible +Standard Algorithm Name Documentation for JDK 11](https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html) lists all possible values for the configs below, among others provided by Java implementation. |Property|Description|Default|Required| @@ -79,7 +79,7 @@ The following table contains non-mandatory advanced configuration options, use c ## Internal communication over TLS Whenever possible Druid processes will use HTTPS to talk to each other. To enable this communication Druid's HttpClient needs to -be configured with a proper [SSLContext](http://docs.oracle.com/javase/8/docs/api/javax/net/ssl/SSLContext.html) that is able +be configured with a proper [SSLContext](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/javax/net/ssl/SSLContext.html) that is able to validate the Server Certificates, otherwise communication will fail. Since, there are various ways to configure SSLContext, by default, Druid looks for an instance of SSLContext Guice binding diff --git a/docs/querying/dimensionspecs.md b/docs/querying/dimensionspecs.md index cda6d2897998..68773abdae10 100644 --- a/docs/querying/dimensionspecs.md +++ b/docs/querying/dimensionspecs.md @@ -255,7 +255,7 @@ For a regular dimension, it assumes the string is formatted in [ISO-8601 date and time format](https://en.wikipedia.org/wiki/ISO_8601). * `format` : date time format for the resulting dimension value, in [Joda Time DateTimeFormat](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html), or null to use the default ISO8601 format. -* `locale` : locale (language and country) to use, given as a [IETF BCP 47 language tag](http://www.oracle.com/technetwork/java/javase/java8locales-2095355.html#util-text), e.g. `en-US`, `en-GB`, `fr-FR`, `fr-CA`, etc. +* `locale` : locale (language and country) to use, given as a [IETF BCP 47 language tag](https://www.oracle.com/java/technologies/javase/jdk11-suported-locales.html#util-text), e.g. `en-US`, `en-GB`, `fr-FR`, `fr-CA`, etc. * `timeZone` : time zone to use in [IANA tz database format](http://en.wikipedia.org/wiki/List_of_tz_database_time_zones), e.g. `Europe/Berlin` (this can possibly be different than the aggregation time-zone) * `granularity` : [granularity](granularities.md) to apply before formatting, or omit to not apply any granularity. * `asMillis` : boolean value, set to true to treat input strings as millis rather than ISO8601 strings. Additionally, if `format` is null or not specified, output will be in millis rather than ISO8601. diff --git a/docs/querying/filters.md b/docs/querying/filters.md index ee50b370f126..20b270903151 100644 --- a/docs/querying/filters.md +++ b/docs/querying/filters.md @@ -439,7 +439,7 @@ The regular expression filter is similar to the selector filter, but using regul | -------- | ----------- | -------- | | `type` | Must be "regex".| Yes | | `dimension` | Input column or virtual column name to filter on. | Yes | -| `pattern` | String pattern to match - any standard [Java regular expression](http://docs.oracle.com/javase/6/docs/api/java/util/regex/Pattern.html). | Yes | +| `pattern` | String pattern to match - any standard [Java regular expression](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/regex/Pattern.html). | Yes | | `extractionFn` | [Extraction function](./dimensionspecs.md#extraction-functions) to apply to `dimension` prior to value matching. See [filtering with extraction functions](#filtering-with-extraction-functions) for details. | No | Note that it is often more optimal to use a like filter instead of a regex for simple matching of prefixes. diff --git a/docs/querying/math-expr.md b/docs/querying/math-expr.md index 0893fc4e2366..84494acfaabd 100644 --- a/docs/querying/math-expr.md +++ b/docs/querying/math-expr.md @@ -81,7 +81,7 @@ The following built-in functions are available. |name|description| |----|-----------| |concat|concat(expr, expr...) concatenate a list of strings| -|format|format(pattern[, args...]) returns a string formatted in the manner of Java's [String.format](https://docs.oracle.com/javase/8/docs/api/java/lang/String.html#format-java.lang.String-java.lang.Object...-).| +|format|format(pattern[, args...]) returns a string formatted in the manner of Java's [String.format](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/String.html#format(java.lang.String,java.lang.Object...)).| |like|like(expr, pattern[, escape]) is equivalent to SQL `expr LIKE pattern`| |lookup|lookup(expr, lookup-name[,replaceMissingValueWith]) looks up expr in a registered,`replaceMissingValueWith` is an optional constant string [query-time lookup](../querying/lookups.md)| |parse_long|parse_long(string[, radix]) parses a string as a long with the given radix, or 10 (decimal) if a radix is not provided.| diff --git a/docs/querying/sql-scalar.md b/docs/querying/sql-scalar.md index f4c565cda839..2c08fc636697 100644 --- a/docs/querying/sql-scalar.md +++ b/docs/querying/sql-scalar.md @@ -115,7 +115,7 @@ String functions accept strings and return a type appropriate to the function. |`REPLACE(expr, substring, replacement)`|Replaces instances of `substring` in `expr` with `replacement` and returns the result.| |`REPEAT(expr, N)`|Repeats `expr` `N` times.| |`REVERSE(expr)`|Reverses `expr`.| -|`STRING_FORMAT(pattern[, args...])`|Returns a string formatted in the manner of Java's [String.format](https://docs.oracle.com/javase/8/docs/api/java/lang/String.html#format-java.lang.String-java.lang.Object...-).| +|`STRING_FORMAT(pattern[, args...])`|Returns a string formatted in the manner of Java's [String.format](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/String.html#format(java.lang.String,java.lang.Object...)).| |`STRPOS(expr, substring)`|Returns the index of `substring` within `expr`, with indexes starting from 1. If `substring` is not found, returns 0.| |`SUBSTRING(expr, index[, length])`|Returns a substring of `expr` starting at a given one-based index. If `length` is omitted, extracts characters to the end of the string, otherwise returns a substring of `length` UTF-16 characters.| |`SUBSTR(expr, index[, length])`|Alias for `SUBSTRING`.| diff --git a/docs/tutorials/cluster.md b/docs/tutorials/cluster.md index 8c4ad3e12346..f5d73d6808f6 100644 --- a/docs/tutorials/cluster.md +++ b/docs/tutorials/cluster.md @@ -133,7 +133,7 @@ The [basic cluster tuning guide](../operations/basic-cluster-tuning.md) has info We recommend running your favorite Linux distribution. You will also need -* [Java 8u92+, 11, or 17](../operations/java.md) +* [Java 11 or 17](../operations/java.md) * Python 2 or Python 3 :::info @@ -141,8 +141,8 @@ We recommend running your favorite Linux distribution. You will also need `DRUID_JAVA_HOME` or `JAVA_HOME`. For more details run the `bin/verify-java` script. ::: -For information about installing Java, see the documentation for your OS package manager. If your Ubuntu-based OS does not have a recent enough version of Java, WebUpd8 offers [packages for those -OSes](http://www.webupd8.org/2012/09/install-oracle-java-8-in-ubuntu-via-ppa.html). +For information about installing Java, see the documentation for your OS package manager. If your Ubuntu-based OS does not have a recent enough version of Java, Linux Uprising offers [packages for those +OSes](https://launchpad.net/~linuxuprising/+archive/ubuntu/java). ## Download the distribution diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index 90e9836e28a8..813ac9859d96 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -40,7 +40,7 @@ You can follow these steps on a relatively modest machine, such as a workstation The software requirements for the installation machine are: * Linux, Mac OS X, or other Unix-like OS. (Windows is not supported) -* [Java 8u92+, 11, or 17](../operations/java.md) +* [Java 11 or 17](../operations/java.md) * Python 3 (preferred) or Python 2 * Perl 5 diff --git a/examples/bin/verify-java b/examples/bin/verify-java index 4759b38089e9..33003dc44751 100755 --- a/examples/bin/verify-java +++ b/examples/bin/verify-java @@ -28,14 +28,14 @@ sub fail_check { : "No Java runtime was detected on your system."; print STDERR <<"EOT"; -Druid requires Java 8, 11, or 17. $current_version_text +Druid requires Java 11 or 17. $current_version_text If you believe this check is in error, or you want to proceed with a potentially unsupported Java runtime, you can skip this check using an environment variable: export DRUID_SKIP_JAVA_CHECK=1 -Otherwise, install Java 8, 11, or 17 in one of the following locations. +Otherwise, install Java 11 or 17 in one of the following locations. * DRUID_JAVA_HOME * JAVA_HOME @@ -68,6 +68,6 @@ if ($?) { } # If we know it won't work, die. Otherwise hope for the best. -if ($java_version =~ /version \"((\d+)\.(\d+).*?)\"/ && !($2 == 1 && $3 == 8) && $2 != 11 && $2 != 17 ) { +if ($java_version =~ /version \"((\d+)\.(\d+).*?)\"/ && $2 != 11 && $2 != 17) { fail_check($1); } diff --git a/examples/quickstart/tutorial/hadoop/docker/Dockerfile b/examples/quickstart/tutorial/hadoop/docker/Dockerfile index 9cada259cc14..d6ae8aedcb15 100644 --- a/examples/quickstart/tutorial/hadoop/docker/Dockerfile +++ b/examples/quickstart/tutorial/hadoop/docker/Dockerfile @@ -52,11 +52,11 @@ RUN rpm --import http://repos.azulsystems.com/RPM-GPG-KEY-azulsystems && \ rpm -ivh zulu-repo-${ZULU_REPO_VER}.noarch.rpm && \ yum -q -y update && \ yum -q -y upgrade && \ - yum -q -y install zulu8-jdk && \ + yum -q -y install zulu17-jdk && \ yum clean all && \ rm -rf /var/cache/yum zulu-repo_${ZULU_REPO_VER}.noarch.rpm -ENV JAVA_HOME=/usr/lib/jvm/zulu8 +ENV JAVA_HOME=/usr/lib/jvm/zulu17 ENV PATH $PATH:$JAVA_HOME/bin # hadoop @@ -73,7 +73,7 @@ ENV HADOOP_CONF_DIR /usr/local/hadoop/etc/hadoop ENV YARN_CONF_DIR $HADOOP_HOME/etc/hadoop # in hadoop 3 the example file is nearly empty so we can just append stuff -RUN sed -i '$ a export JAVA_HOME=/usr/lib/jvm/zulu8' $HADOOP_HOME/etc/hadoop/hadoop-env.sh +RUN sed -i '$ a export JAVA_HOME=/usr/lib/jvm/zulu17' $HADOOP_HOME/etc/hadoop/hadoop-env.sh RUN sed -i '$ a export HADOOP_HOME=/usr/local/hadoop' $HADOOP_HOME/etc/hadoop/hadoop-env.sh RUN sed -i '$ a export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/' $HADOOP_HOME/etc/hadoop/hadoop-env.sh RUN sed -i '$ a export HDFS_NAMENODE_USER=root' $HADOOP_HOME/etc/hadoop/hadoop-env.sh diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index 01b006dd5d5b..0fd5018d1311 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -48,7 +48,6 @@ com.google.inject guice - ${guice.version} aopalliance @@ -60,7 +59,7 @@ com.google.inject.extensions guice-multibindings - ${guice.version} + provided diff --git a/extensions-contrib/compressed-bigdecimal/pom.xml b/extensions-contrib/compressed-bigdecimal/pom.xml index 358bc3b26c5b..2ef1a7740cc9 100644 --- a/extensions-contrib/compressed-bigdecimal/pom.xml +++ b/extensions-contrib/compressed-bigdecimal/pom.xml @@ -145,7 +145,6 @@ com.google.inject guice - 4.1.0 provided diff --git a/extensions-contrib/grpc-query/pom.xml b/extensions-contrib/grpc-query/pom.xml index 101e2f34b74c..e46f0c35fd2d 100644 --- a/extensions-contrib/grpc-query/pom.xml +++ b/extensions-contrib/grpc-query/pom.xml @@ -262,6 +262,12 @@ reflections test + + javax.annotation + javax.annotation-api + 1.2 + provided + diff --git a/extensions-core/simple-client-sslcontext/README.md b/extensions-core/simple-client-sslcontext/README.md index 4a5a97d19039..29276f4cf249 100644 --- a/extensions-core/simple-client-sslcontext/README.md +++ b/extensions-core/simple-client-sslcontext/README.md @@ -17,6 +17,6 @@ ~ under the License. --> -This module contains a simple implementation of [SslContext](http://docs.oracle.com/javase/8/docs/api/javax/net/ssl/SSLContext.html) +This module contains a simple implementation of [SslContext](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/javax/net/ssl/SSLContext.html) that will be injected to be used with HttpClient that Druid nodes use internally to communicate with each other. More details [here](https://druid.apache.org/docs/latest/development/extensions-core/simple-client-sslcontext.html). diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java index 8a124435faa5..a41ced78e7c8 100644 --- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java +++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java @@ -293,7 +293,7 @@ private static IncrementalIndex makeIncrementalIndex( AggregatorFactory[] aggs, HadoopDruidIndexerConfig config, @Nullable Iterable oldDimOrder, - Map oldCapabilities + @Nullable Map oldCapabilities ) { final HadoopTuningConfig tuningConfig = config.getSchema().getTuningConfig(); diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 2f348e3d4839..412568ea88c9 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -91,6 +91,7 @@ com.google.inject.extensions guice-multibindings + provided javax.ws.rs diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/RemoteTaskRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/RemoteTaskRunner.java index 1d06321ddc49..4fd7b57f38b5 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/RemoteTaskRunner.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/RemoteTaskRunner.java @@ -1515,7 +1515,7 @@ void checkBlackListedNodes() boolean shouldRunPendingTasks = false; // must be synchronized while iterating: - // https://docs.oracle.com/javase/8/docs/api/java/util/Collections.html#synchronizedSet-java.util.Set- + // https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/Collections.html#synchronizedSet(java.util.Set) synchronized (blackListedWorkers) { for (Iterator iterator = blackListedWorkers.iterator(); iterator.hasNext(); ) { ZkWorker zkWorker = iterator.next(); diff --git a/integration-tests-ex/cases/pom.xml b/integration-tests-ex/cases/pom.xml index 3e8bf724d82f..94c42a0555b4 100644 --- a/integration-tests-ex/cases/pom.xml +++ b/integration-tests-ex/cases/pom.xml @@ -86,6 +86,7 @@ com.google.inject.extensions guice-multibindings + provided org.apache.curator @@ -478,6 +479,9 @@ maven-failsafe-plugin 3.0.0-M7 + + --add-opens java.base/java.lang=ALL-UNNAMED + diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQueryWorkerFaultTolerance.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQueryWorkerFaultTolerance.java index f45ce3ad0e96..13c3fdc38463 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQueryWorkerFaultTolerance.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQueryWorkerFaultTolerance.java @@ -99,7 +99,7 @@ public void testMsqIngestionAndQuerying() throws Exception + " regionIsoCode\n" + "FROM TABLE(\n" + " EXTERN(\n" - + " '{\"type\":\"local\",\"files\":[\"/resources/data/batch_index/json/wikipedia_index_data1.json\",\"/resources/data/batch_index/json/wikipedia_index_data1.json\"]}',\n" + + " '{\"type\":\"local\",\"files\":[\"/resources/data/batch_index/json/wikipedia_index_data1.json\",\"/resources/data/batch_index/json/wikipedia_index_data1.json\",\"/resources/data/batch_index/json/wikipedia_index_data1.json\",\"/resources/data/batch_index/json/wikipedia_index_data1.json\"]}',\n" + " '{\"type\":\"json\"}',\n" + " '[{\"type\":\"string\",\"name\":\"timestamp\"},{\"type\":\"string\",\"name\":\"isRobot\"},{\"type\":\"string\",\"name\":\"diffUrl\"},{\"type\":\"long\",\"name\":\"added\"},{\"type\":\"string\",\"name\":\"countryIsoCode\"},{\"type\":\"string\",\"name\":\"regionName\"},{\"type\":\"string\",\"name\":\"channel\"},{\"type\":\"string\",\"name\":\"flags\"},{\"type\":\"long\",\"name\":\"delta\"},{\"type\":\"string\",\"name\":\"isUnpatrolled\"},{\"type\":\"string\",\"name\":\"isNew\"},{\"type\":\"double\",\"name\":\"deltaBucket\"},{\"type\":\"string\",\"name\":\"isMinor\"},{\"type\":\"string\",\"name\":\"isAnonymous\"},{\"type\":\"long\",\"name\":\"deleted\"},{\"type\":\"string\",\"name\":\"cityName\"},{\"type\":\"long\",\"name\":\"metroCode\"},{\"type\":\"string\",\"name\":\"namespace\"},{\"type\":\"string\",\"name\":\"comment\"},{\"type\":\"string\",\"name\":\"page\"},{\"type\":\"long\",\"name\":\"commentLength\"},{\"type\":\"string\",\"name\":\"countryName\"},{\"type\":\"string\",\"name\":\"user\"},{\"type\":\"string\",\"name\":\"regionIsoCode\"}]'\n" + " )\n" @@ -139,7 +139,6 @@ public void testMsqIngestionAndQuerying() throws Exception private void killTaskAbruptly(String taskIdToKill) { - String command = "jps -mlv | grep -i peon | grep -i " + taskIdToKill + " |awk '{print $1}'"; ITRetryUtil.retryUntil(() -> { @@ -158,21 +157,17 @@ private void killTaskAbruptly(String taskIdToKill) } String pidToKill = stdOut.lhs.trim(); if (pidToKill.length() != 0) { - LOG.info("Found PID to kill %s", pidToKill); - // kill worker after 5 seconds - Thread.sleep(5000); LOG.info("Killing pid %s", pidToKill); - druidClusterAdminClient.runCommandInMiddleManagerContainer( + final Pair killResult = druidClusterAdminClient.runCommandInMiddleManagerContainer( "/bin/bash", "-c", "kill -9 " + pidToKill ); + LOG.info(StringUtils.format("Kill command stdout: %s, stderr: %s", killResult.lhs, killResult.rhs)); return true; } else { return false; } - }, true, 6000, 50, StringUtils.format("Figuring out PID for task[%s] to kill abruptly", taskIdToKill)); - - + }, true, 2000, 100, StringUtils.format("Figuring out PID for task[%s] to kill abruptly", taskIdToKill)); } } diff --git a/integration-tests-ex/cases/src/test/resources/multi-stage-query/wikipedia_msq_select_query_ha.json b/integration-tests-ex/cases/src/test/resources/multi-stage-query/wikipedia_msq_select_query_ha.json index 992eda01a26a..5106257eb6a8 100644 --- a/integration-tests-ex/cases/src/test/resources/multi-stage-query/wikipedia_msq_select_query_ha.json +++ b/integration-tests-ex/cases/src/test/resources/multi-stage-query/wikipedia_msq_select_query_ha.json @@ -18,6 +18,30 @@ "deleted": 200, "namespace": "article" }, + { + "__time": 1377910953000, + "isRobot": null, + "added": 57, + "delta": -143, + "deleted": 200, + "namespace": "article" + }, + { + "__time": 1377910953000, + "isRobot": null, + "added": 57, + "delta": -143, + "deleted": 200, + "namespace": "article" + }, + { + "__time": 1377919965000, + "isRobot": null, + "added": 459, + "delta": 330, + "deleted": 129, + "namespace": "wikipedia" + }, { "__time": 1377919965000, "isRobot": null, @@ -34,6 +58,30 @@ "deleted": 129, "namespace": "wikipedia" }, + { + "__time": 1377919965000, + "isRobot": null, + "added": 459, + "delta": 330, + "deleted": 129, + "namespace": "wikipedia" + }, + { + "__time": 1377933081000, + "isRobot": null, + "added": 123, + "delta": 111, + "deleted": 12, + "namespace": "article" + }, + { + "__time": 1377933081000, + "isRobot": null, + "added": 123, + "delta": 111, + "deleted": 12, + "namespace": "article" + }, { "__time": 1377933081000, "isRobot": null, diff --git a/integration-tests-ex/image/docker/Dockerfile b/integration-tests-ex/image/docker/Dockerfile index 90955eae3c11..deb5bb8d9b81 100644 --- a/integration-tests-ex/image/docker/Dockerfile +++ b/integration-tests-ex/image/docker/Dockerfile @@ -28,7 +28,7 @@ # This Dockerfile prefers to use the COPY command over ADD. # See: https://phoenixnap.com/kb/docker-add-vs-copy -ARG JDK_VERSION=8-slim-buster +ARG JDK_VERSION=17-slim-buster # The FROM image provides Java on top of Debian, and # thus provides bash, apt-get, etc. diff --git a/integration-tests/README.md b/integration-tests/README.md index 25597df5c923..70c5e6c5edba 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -88,8 +88,7 @@ if your test has the annotation: `@Test(groups = TestNGGroup.BATCH_INDEX)` then * Add `-pl :druid-integration-tests` when running integration tests for the second time or later without changing the code of core modules in between to skip up-to-date checks for the whole module dependency tree. -* Integration tests can also be run with either Java 8 or Java 11 by adding `-Djvm.runtime=#` to the `mvn` command, where `#` -can either be 8 or 11. +* Integration tests can also be run with a specific Java version by adding `-Djvm.runtime=#` to the `mvn` command (where `#` can be 11, for example). * Druid's configuration (using Docker) can be overridden by providing `-Doverride.config.path=`. The file must contain one property per line, the key must start with `druid_` and the format should be snake case. diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile index 40b1cee4f9bf..df8c6f2598be 100644 --- a/integration-tests/docker/Dockerfile +++ b/integration-tests/docker/Dockerfile @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG JDK_VERSION=8-slim-buster +ARG JDK_VERSION=17-slim-buster FROM openjdk:$JDK_VERSION as druidbase # Bundle everything into one script so cleanup can reduce image size. diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 5320d0c2451e..7bea258c6472 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -281,6 +281,7 @@ com.google.inject.extensions guice-multibindings + provided com.fasterxml.jackson.core @@ -371,7 +372,6 @@ com.google.inject.extensions guice-servlet - ${guice.version} io.confluent @@ -772,6 +772,7 @@ + ${jdk.strong.encapsulation.argLine} -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Ddruid.test.config.type=configFile diff --git a/integration-tests/script/docker_build_containers.sh b/integration-tests/script/docker_build_containers.sh index ba2dd7dcd822..bed98c7db5f8 100755 --- a/integration-tests/script/docker_build_containers.sh +++ b/integration-tests/script/docker_build_containers.sh @@ -26,7 +26,7 @@ then else echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is set with value ${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" case "${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" in - 8 | 11 | 17 | 21) + 11 | 17 | 21) echo "Build druid-cluster with Java $DRUID_INTEGRATION_TEST_JVM_RUNTIME" docker build -t druid/cluster \ --build-arg JDK_VERSION=$DRUID_INTEGRATION_TEST_JVM_RUNTIME-slim-buster \ diff --git a/licenses.yaml b/licenses.yaml index 40305fc55c66..a12ab1b01e4f 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -371,7 +371,7 @@ name: Guice license_category: binary module: java-core license_name: Apache License version 2.0 -version: 4.1.0 +version: 4.2.2 libraries: - com.google.inject: guice - com.google.inject.extensions: guice-multibindings diff --git a/pom.xml b/pom.xml index a2d7c2aa3f04..f6e9728ae03a 100644 --- a/pom.xml +++ b/pom.xml @@ -70,9 +70,8 @@ - 1.8 - 1.8 - 8 + 11 + ${java.version} UTF-8 0.9.0.M2 5.5.0 @@ -96,7 +95,7 @@ 2.35.1 8.5.4 32.0.1-jre - 4.1.0 + 4.2.2 1.3 9.4.54.v20240208 1.19.4 @@ -131,7 +130,33 @@ 1.42.3 v1-rev20230606-2.0.0 2.29.1 - + + + + + + + + + + --add-exports=java.base/jdk.internal.ref=ALL-UNNAMED + --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED + + + --add-opens=java.base/java.io=ALL-UNNAMED + + + --add-opens=java.base/java.lang=ALL-UNNAMED + + + --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED + + + --add-opens=java.base/java.util=ALL-UNNAMED + maven.org Maven Central Repository @@ -271,6 +296,16 @@ false + + + + twitter + https://maven.twttr.com + @@ -1507,13 +1542,13 @@ com.github.spotbugs spotbugs-maven-plugin - 4.2.0 + 4.8.6.6 com.github.spotbugs spotbugs - 4.2.2 + 4.8.6 @@ -1524,7 +1559,7 @@ org.apache.maven.plugins maven-pmd-plugin - 3.16.0 + 3.26.0 false true @@ -1534,6 +1569,7 @@ target/generated-sources/ + ${maven.compiler.release} @@ -1554,7 +1590,7 @@ jdk-unsafe @@ -1622,6 +1658,16 @@ sun.misc.Unsafe java.lang.invoke.MethodHandle + + java.nio.ByteBuffer + java.nio.IntBuffer + java.nio.CharBuffer + java.nio.FloatBuffer + java.nio.DoubleBuffer + java.nio.MappedByteBuffer @@ -1830,7 +1876,12 @@ org.apache.maven.plugins maven-dependency-plugin - 3.1.1 + 3.3.0 + + + * + + org.apache.maven.plugins @@ -1916,9 +1967,9 @@ org.apache.maven.plugins maven-compiler-plugin 3.11.0 + true - ${maven.compiler.source} - ${maven.compiler.target} + ${maven.compiler.release} @@ -1956,54 +2007,6 @@ - - java-9+ - - [9,) - - - - - - - - - - - - --add-exports=java.base/jdk.internal.ref=ALL-UNNAMED - --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED - --add-opens=java.base/java.nio=ALL-UNNAMED - --add-opens=java.base/sun.nio.ch=ALL-UNNAMED - --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED - - - --add-opens=java.base/java.io=ALL-UNNAMED - - - --add-opens=java.base/java.lang=ALL-UNNAMED - - - --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED - - - --add-opens=java.base/java.util=ALL-UNNAMED - - - - - - org.apache.maven.plugins - maven-compiler-plugin - true - - - ${java.version} - - - - - strict @@ -2034,9 +2037,10 @@ -J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + -J--add-exports=java.base/sun.nio.ch=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED - + com.google.errorprone diff --git a/processing/pom.xml b/processing/pom.xml index 8fb67ccaebc8..6afb38ae48d9 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -106,6 +106,7 @@ com.google.inject.extensions guice-multibindings + provided com.google.code.findbugs diff --git a/processing/src/main/antlr4/org/apache/druid/math/expr/antlr/Expr.g4 b/processing/src/main/antlr4/org/apache/druid/math/expr/antlr/Expr.g4 index 1d52a13ecfc7..fd8288b2b086 100644 --- a/processing/src/main/antlr4/org/apache/druid/math/expr/antlr/Expr.g4 +++ b/processing/src/main/antlr4/org/apache/druid/math/expr/antlr/Expr.g4 @@ -59,7 +59,7 @@ NULL : 'null'; LONG : [0-9]+; EXP: [eE] [-]? LONG; // DOUBLE provides partial support for java double format -// see: https://docs.oracle.com/javase/8/docs/api/java/lang/Double.html#valueOf-java.lang.String- +// see: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/Double.html#valueOf(java.lang.String) DOUBLE : 'NaN' | 'Infinity' | (LONG '.' LONG?) | (LONG EXP) | (LONG '.' LONG? EXP); IDENTIFIER : [_$a-zA-Z][_$a-zA-Z0-9]* | '"' (ESC | ~ [\"\\])* '"'; WS : [ \t\r\n]+ -> skip ; diff --git a/processing/src/main/java/org/apache/druid/crypto/CryptoService.java b/processing/src/main/java/org/apache/druid/crypto/CryptoService.java index 92d4f6602c45..9e63c0adfa94 100644 --- a/processing/src/main/java/org/apache/druid/crypto/CryptoService.java +++ b/processing/src/main/java/org/apache/druid/crypto/CryptoService.java @@ -42,7 +42,7 @@ * using javax.crypto package. * * To learn about possible algorithms supported and their names, - * See https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html + * See https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html */ public class CryptoService { diff --git a/processing/src/main/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequence.java b/processing/src/main/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequence.java index ca2708700f08..53b7d68636dc 100644 --- a/processing/src/main/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequence.java +++ b/processing/src/main/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequence.java @@ -808,7 +808,7 @@ protected void compute() /** * {@link ForkJoinPool} friendly {@link BlockingQueue} feeder, adapted from 'QueueTaker' of Java documentation on * {@link ForkJoinPool.ManagedBlocker}, - * https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ForkJoinPool.ManagedBlocker.html + * https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/concurrent/ForkJoinPool.ManagedBlocker.html */ static class QueuePusher implements ForkJoinPool.ManagedBlocker { diff --git a/processing/src/main/java/org/apache/druid/query/rowsandcols/CursorFactoryRowsAndColumns.java b/processing/src/main/java/org/apache/druid/query/rowsandcols/CursorFactoryRowsAndColumns.java index 46fda857516f..e636921baf72 100644 --- a/processing/src/main/java/org/apache/druid/query/rowsandcols/CursorFactoryRowsAndColumns.java +++ b/processing/src/main/java/org/apache/druid/query/rowsandcols/CursorFactoryRowsAndColumns.java @@ -112,12 +112,8 @@ private static RowsAndColumns materialize(CursorFactory cursorFactory) cursor.advance(); } - if (writer == null) { - return new EmptyRowsAndColumns(); - } else { - final byte[] bytes = writer.toByteArray(); - return new ColumnBasedFrameRowsAndColumns(Frame.wrap(bytes), rowSignature); - } + final byte[] bytes = writer.toByteArray(); + return new ColumnBasedFrameRowsAndColumns(Frame.wrap(bytes), rowSignature); } } } diff --git a/server/pom.xml b/server/pom.xml index 3300606d1067..45ca282f836f 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -110,6 +110,7 @@ com.google.inject.extensions guice-multibindings + provided com.google.inject.extensions @@ -216,6 +217,11 @@ org.apache.logging.log4j log4j-core + + org.checkerframework + checker-qual + ${checkerframework.version} + com.fasterxml.jackson.datatype jackson-datatype-joda diff --git a/server/src/main/java/org/apache/druid/curator/discovery/CuratorDruidLeaderSelector.java b/server/src/main/java/org/apache/druid/curator/discovery/CuratorDruidLeaderSelector.java index b2179f5d69d4..d11298031069 100644 --- a/server/src/main/java/org/apache/druid/curator/discovery/CuratorDruidLeaderSelector.java +++ b/server/src/main/java/org/apache/druid/curator/discovery/CuratorDruidLeaderSelector.java @@ -37,6 +37,7 @@ import javax.annotation.Nullable; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; /** @@ -58,7 +59,7 @@ public class CuratorDruidLeaderSelector implements DruidLeaderSelector private final AtomicReference leaderLatch = new AtomicReference<>(); private volatile boolean leader = false; - private volatile int term = 0; + private final AtomicInteger term = new AtomicInteger(0); public CuratorDruidLeaderSelector(CuratorFramework curator, @Self DruidNode self, String latchPath) { @@ -99,7 +100,7 @@ public void isLeader() } leader = true; - term++; + term.incrementAndGet(); listener.becomeLeader(); } catch (Exception ex) { @@ -161,7 +162,7 @@ public boolean isLeader() @Override public int localTerm() { - return term; + return term.get(); } @Override diff --git a/services/pom.xml b/services/pom.xml index b6c552d451ac..7b681a1e7e17 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -160,6 +160,7 @@ com.google.inject.extensions guice-multibindings + provided org.roaringbitmap diff --git a/sql/pom.xml b/sql/pom.xml index 6c564e5cdaa7..20bf288d90db 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -131,6 +131,7 @@ com.google.inject.extensions guice-multibindings + provided javax.ws.rs diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx index 69b2f97b0360..f0f0e7e281f9 100644 --- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx +++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx @@ -647,7 +647,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F

For more information, refer to the documentation for{' '} - + FileSystem#getPathMatcher . diff --git a/web-console/src/druid-models/input-source/input-source.tsx b/web-console/src/druid-models/input-source/input-source.tsx index d3f24ea4d948..f2428d8a3835 100644 --- a/web-console/src/druid-models/input-source/input-source.tsx +++ b/web-console/src/druid-models/input-source/input-source.tsx @@ -507,7 +507,7 @@ export const INPUT_SOURCE_FIELDS: Field[] = [

For more information, refer to the documentation for{' '} - + FileSystem#getPathMatcher . diff --git a/web-console/src/druid-models/timestamp-spec/timestamp-spec.tsx b/web-console/src/druid-models/timestamp-spec/timestamp-spec.tsx index 0ec00fd944a7..a80224f12860 100644 --- a/web-console/src/druid-models/timestamp-spec/timestamp-spec.tsx +++ b/web-console/src/druid-models/timestamp-spec/timestamp-spec.tsx @@ -136,7 +136,7 @@ export const TIMESTAMP_SPEC_FIELDS: Field[] = [ info: (

Specify your timestamp format by using the suggestions menu or typing in a{' '} - + format string .