From 69961909a420a6c436a7559d194f93e49b2e5da1 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Sat, 30 Jul 2022 08:39:59 -0700 Subject: [PATCH 01/12] Improved Java 17 support and Java runtime docs. 1) Add a "Java runtime" doc page with information about supported Java versions, garbage collection, and strong encapsulation.. 2) Update asm and equalsverifier to versions that support Java 17. 3) Add additional "--add-opens" lines to surefire configuration, so tests can pass successfully under Java 17. 4) Switch openjdk15 tests to openjdk17. 5) Update FrameFile to specifically mention Java runtime incompatibility as the cause of not being able to use Memory.map. 6) Update SegmentLoadDropHandler to log an error for Errors too, not just Exceptions. This is important because an IllegalAccessError is encountered when the correct "--add-opens" line is not provided, which would otherwise be silently ignored. 7) Update example configs to use druid.indexer.runner.javaOptsArray instead of druid.indexer.runner.javaOpts. (The latter is deprecated.) --- .travis.yml | 36 +++---- .../java/org/apache/druid/utils/JvmUtils.java | 31 +++++-- docs/configuration/index.md | 2 +- docs/configuration/logging.md | 12 +-- docs/operations/java.md | 93 +++++++++++++++++++ docs/tutorials/cluster.md | 2 +- docs/tutorials/index.md | 5 +- .../data/middleManager/runtime.properties | 2 +- .../large/middleManager/runtime.properties | 2 +- .../medium/middleManager/runtime.properties | 2 +- .../middleManager/runtime.properties | 2 +- .../middleManager/runtime.properties | 2 +- .../small/middleManager/runtime.properties | 2 +- .../xlarge/middleManager/runtime.properties | 2 +- licenses.yaml | 2 +- pom.xml | 18 +++- .../apache/druid/frame/file/FrameFile.java | 21 ++++- .../druid/frame/testutil/FrameTestUtil.java | 5 +- .../coordination/SegmentLoadDropHandler.java | 4 +- website/sidebars.json | 1 + 20 files changed, 190 insertions(+), 56 deletions(-) create mode 100644 docs/operations/java.md diff --git a/.travis.yml b/.travis.yml index d92b66c33367..aed248afdff8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -188,9 +188,9 @@ jobs: jdk: openjdk11 - <<: *package - name: "(openjdk15) packaging check" + name: "(openjdk17) packaging check" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - &test_processing_module name: "(openjdk8) processing module test" @@ -259,9 +259,9 @@ jobs: jdk: openjdk11 - <<: *test_processing_module - name: "(openjdk15) processing module test" + name: "(openjdk17) processing module test" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - &test_processing_module_sqlcompat <<: *test_processing_module @@ -276,9 +276,9 @@ jobs: jdk: openjdk11 - <<: *test_processing_module_sqlcompat - name: "(openjdk15) processing module test (SQL Compatibility)" + name: "(openjdk17) processing module test (SQL Compatibility)" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - &test_indexing_module <<: *test_processing_module @@ -292,9 +292,9 @@ jobs: jdk: openjdk11 - <<: *test_indexing_module - name: "(openjdk15) indexing modules test" + name: "(openjdk17) indexing modules test" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - &test_indexing_module_sqlcompat <<: *test_indexing_module @@ -308,9 +308,9 @@ jobs: jdk: openjdk11 - <<: *test_indexing_module_sqlcompat - name: "(openjdk15) indexing modules test (SQL Compatibility)" + name: "(openjdk17) indexing modules test (SQL Compatibility)" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - &test_server_module <<: *test_processing_module @@ -324,9 +324,9 @@ jobs: jdk: openjdk11 - <<: *test_server_module - name: "(openjdk15) server module test" + name: "(openjdk17) server module test" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - &test_server_module_sqlcompat <<: *test_server_module @@ -339,9 +339,9 @@ jobs: jdk: openjdk11 - <<: *test_server_module_sqlcompat - name: "(openjdk15) server module test (SQL Compatibility)" + name: "(openjdk17) server module test (SQL Compatibility)" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - &test_other_modules <<: *test_processing_module @@ -355,9 +355,9 @@ jobs: jdk: openjdk11 - <<: *test_other_modules - name: "(openjdk15) other modules test" + name: "(openjdk17) other modules test" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - &test_other_modules_sqlcompat <<: *test_other_modules @@ -370,9 +370,9 @@ jobs: jdk: openjdk11 - <<: *test_other_modules_sqlcompat - name: "(openjdk15) other modules test (SQL Compatibility)" + name: "(openjdk17) other modules test (SQL Compatibility)" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - name: "web console" install: skip diff --git a/core/src/main/java/org/apache/druid/utils/JvmUtils.java b/core/src/main/java/org/apache/druid/utils/JvmUtils.java index 5c4869d4118f..64bb3778f3d8 100644 --- a/core/src/main/java/org/apache/druid/utils/JvmUtils.java +++ b/core/src/main/java/org/apache/druid/utils/JvmUtils.java @@ -19,6 +19,7 @@ package org.apache.druid.utils; +import com.google.common.primitives.Ints; import com.google.inject.Inject; import java.io.File; @@ -36,19 +37,37 @@ public class JvmUtils { - private static final boolean IS_JAVA9_COMPATIBLE = isJava9Compatible(System.getProperty("java.specification.version")); + public static final int UNKNOWN_VERSION = -1; + private static final int MAJOR_VERSION = computeMajorVersion(); - private static boolean isJava9Compatible(String versionString) + private static int computeMajorVersion() { - final StringTokenizer st = new StringTokenizer(versionString, "."); - int majorVersion = Integer.parseInt(st.nextToken()); + final StringTokenizer st = new StringTokenizer(System.getProperty("java.specification.version"), "."); + if (!st.hasMoreTokens()) { + return UNKNOWN_VERSION; + } + + final String majorVersionString = st.nextToken(); + final Integer majorVersion = Ints.tryParse(majorVersionString); + return majorVersion == null ? UNKNOWN_VERSION : majorVersion; + } - return majorVersion >= 9; + /** + * Returns the major version of the current Java runtime for Java 9 and above. For example: 9, 11, 17, etc. + * + * Returns 1 for Java 8 and earlier. + * + * Returns {@link #UNKNOWN_VERSION} if the major version cannot be determined. This is a negative number and is + * therefore lower than all valid versions. + */ + public static int majorVersion() + { + return MAJOR_VERSION; } public static boolean isIsJava9Compatible() { - return IS_JAVA9_COMPATIBLE; + return MAJOR_VERSION >= 9; } @Inject diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 2246727ad288..da54aa7dd535 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -1371,7 +1371,7 @@ Middle managers pass their configurations down to their child peons. The MiddleM |`druid.indexer.runner.classpath`|Java classpath for the peon.|System.getProperty("java.class.path")| |`druid.indexer.runner.javaCommand`|Command required to execute java.|java| |`druid.indexer.runner.javaOpts`|*DEPRECATED* A string of -X Java options to pass to the peon's JVM. Quotable parameters or parameters with spaces are encouraged to use javaOptsArray|""| -|`druid.indexer.runner.javaOptsArray`|A JSON array of strings to be passed in as options to the peon's JVM. This is additive to javaOpts and is recommended for properly handling arguments which contain quotes or spaces like `["-XX:OnOutOfMemoryError=kill -9 %p"]`|`[]`| +|`druid.indexer.runner.javaOptsArray`|A JSON array of strings to be passed in as options to the peon's JVM. This is additive to `druid.indexer.runner.javaOpts` and is recommended for properly handling arguments which contain quotes or spaces like `["-XX:OnOutOfMemoryError=kill -9 %p"]`|`[]`| |`druid.indexer.runner.maxZnodeBytes`|The maximum size Znode in bytes that can be created in ZooKeeper, should be in the range of [10KiB, 2GiB). [Human-readable format](human-readable-byte.md) is supported.|512KiB| |`druid.indexer.runner.startPort`|Starting port used for peon processes, should be greater than 1023 and less than 65536.|8100| |`druid.indexer.runner.endPort`|Ending port used for peon processes, should be greater than or equal to `druid.indexer.runner.startPort` and less than 65536.|65535| diff --git a/docs/configuration/logging.md b/docs/configuration/logging.md index 9641b9d64a9a..1be881a0cbfd 100644 --- a/docs/configuration/logging.md +++ b/docs/configuration/logging.md @@ -104,16 +104,8 @@ WARNING: Use --illegal-access=warn to enable warnings of further illegal reflect WARNING: All illegal access operations will be denied in a future release ``` -These messages do not cause harm, but you can avoid them by adding the following lines to your `jvm.config` files. These -lines are not part of the default JVM configs that ship with Druid, because Java 8 will not recognize these options and -will fail to start up. - -``` ---add-exports=java.base/jdk.internal.ref=ALL-UNNAMED ---add-exports=java.base/jdk.internal.perf=ALL-UNNAMED ---add-opens=java.base/java.lang=ALL-UNNAMED ---add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED -``` +To avoid these, add the `--add-exports` and `--add-opens` command line parameters described in the documentation section +about [Java strong encapsulation](../operations/java.md#strong-encapsulation). ## My logs are really chatty, can I set them to asynchronously write? diff --git a/docs/operations/java.md b/docs/operations/java.md new file mode 100644 index 000000000000..9808561ace4b --- /dev/null +++ b/docs/operations/java.md @@ -0,0 +1,93 @@ +--- +id: java +title: "Java runtime" +--- + + + +Apache Druid is written in Java and requires a Java runtime. This page provides details about obtaining and configuring +a Java runtime for Druid. + +## Selecting a Java runtime + +Druid fully supports Java 8 and 11. The project team recommends Java 11. + +The project team does not recommend running with Java 17, because certain Druid functionality is not currently +compatible with Java 17. + +There are many free and actively-supported Java runtimes available for these versions, such as +[Corretto](https://aws.amazon.com/corretto/) and [Zulu](https://www.azul.com/downloads/?package=jdk#download-openjdk). +The project team does not recommend any specific runtime over any other. + +Druid relies on the environment variables `JAVA_HOME` or `DRUID_JAVA_HOME` to find Java on the machine. You can set +`DRUID_JAVA_HOME` if there is more than one instance of Java. To verify Java requirements for your environment, run the +`bin/verify-java` script. + +## Garbage collection + +In general, we recommend using the G1 collector with default settings. This is the default collector in Java 11. +To enable G1 on Java 8, use `+XX:UseG1GC`. There is no harm in explicitly specifying this on Java 11 as well. + +Garbage collector selection and tuning is a form of sport in the Java community. There may be situations where adjusting +garbage collection configuration improves or worsens performance. The project team's guidance is that most people do +not need to stray away from G1 with default settings. + +## Strong encapsulation + +Java 9 and beyond (including Java 11) include the capability for +[strong encapsulation](https://dev.java/learn/strong-encapsulation-\(of-jdk-internals\)/) of internal JDK APIs. Druid +uses certain internal JDK APIs for functionality- and performance-related reasons. In Java 11, this leads to log +messages like the following: + +``` +WARNING: An illegal reflective access operation has occurred +WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations +WARNING: All illegal access operations will be denied in a future release +``` + +These warning messages are harmless, and can be ignored. However, you can avoid them entirely if you wish by adding the +following Java command line parameters. These paramaters are not part of the default configurations that ship with +Druid, because Java 8 does not recognize these parameters and fails to start up if they are provided. + +To do this, add the following lines to your `jvm.config` files: + +``` +--add-exports=java.base/jdk.internal.perf=ALL-UNNAMED +--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED +--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED +--add-opens=java.base/java.lang=ALL-UNNAMED +--add-opens=java.base/java.io=ALL-UNNAMED +--add-opens=java.base/java.nio=ALL-UNNAMED +--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED +--add-opens=java.base/sun.nio.ch=ALL-UNNAMED +--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED +``` + +Additionally, tasks run by [MiddleManagers](../design/architecture.md) execute in separate JVMs. The command line for +these JVMs is given by `druid.indexer.runner.javaOptsArray` or `druid.indexer.runner.javaOpts` in +`middleManager/runtime.properties`. Java command line parameters for tasks must be specified here. For example, use +a line like the following: + +``` +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager","--add-exports=java.base/jdk.internal.perf=ALL-UNNAMED","--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED","--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED","--add-opens=java.base/java.lang=ALL-UNNAMED","--add-opens=java.base/java.io=ALL-UNNAMED","--add-opens=java.base/java.nio=ALL-UNNAMED","--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED","--add-opens=java.base/sun.nio.ch=ALL-UNNAMED","--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED"] +``` + +The `Xms`, `Xmx`, and `MaxDirectMemorySize` parameters in the line above are merely an example. You may use different +values in your specific environment. diff --git a/docs/tutorials/cluster.md b/docs/tutorials/cluster.md index eac469d1e300..b61953c2f427 100644 --- a/docs/tutorials/cluster.md +++ b/docs/tutorials/cluster.md @@ -130,7 +130,7 @@ The [basic cluster tuning guide](../operations/basic-cluster-tuning.md) has info ## Select OS -We recommend running your favorite Linux distribution. You will also need Java 8 or 11. +We recommend running your favorite Linux distribution. You will also need [Java 8 or 11](../operations/java.md). > If needed, you can specify where to find Java using the environment variables > `DRUID_JAVA_HOME` or `JAVA_HOME`. For more details run the `bin/verify-java` script. diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index 5adcf5adfb6e..1fab7f8fe342 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -45,8 +45,9 @@ information on deploying Druid services across clustered machines. The software requirements for the installation machine are: -* Linux, Mac OS X, or other Unix-like OS (Windows is not supported) -* Java 8, Update 92 or later (8u92+) or Java 11 +* Linux, Mac OS X, or other Unix-like OS (Windows is not supported). +* Java 8, Update 92 or later (8u92+) or Java 11. See the [Java runtime](../operations/java.md) page for additional + information about selecting and configuring a Java runtime. > Druid relies on the environment variables `JAVA_HOME` or `DRUID_JAVA_HOME` to find Java on the machine. You can set `DRUID_JAVA_HOME` if there is more than one instance of Java. To verify Java requirements for your environment, run the diff --git a/examples/conf/druid/cluster/data/middleManager/runtime.properties b/examples/conf/druid/cluster/data/middleManager/runtime.properties index 564b98e19957..6be77c3635e2 100644 --- a/examples/conf/druid/cluster/data/middleManager/runtime.properties +++ b/examples/conf/druid/cluster/data/middleManager/runtime.properties @@ -24,7 +24,7 @@ druid.plaintextPort=8091 druid.worker.capacity=4 # Task launch parameters -druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task # HTTP server threads diff --git a/examples/conf/druid/single-server/large/middleManager/runtime.properties b/examples/conf/druid/single-server/large/middleManager/runtime.properties index 2c1cf0727904..e11aa40614be 100644 --- a/examples/conf/druid/single-server/large/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/large/middleManager/runtime.properties @@ -24,7 +24,7 @@ druid.plaintextPort=8091 druid.worker.capacity=8 # Task launch parameters -druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task # HTTP server threads diff --git a/examples/conf/druid/single-server/medium/middleManager/runtime.properties b/examples/conf/druid/single-server/medium/middleManager/runtime.properties index 564b98e19957..6be77c3635e2 100644 --- a/examples/conf/druid/single-server/medium/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/medium/middleManager/runtime.properties @@ -24,7 +24,7 @@ druid.plaintextPort=8091 druid.worker.capacity=4 # Task launch parameters -druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task # HTTP server threads diff --git a/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties b/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties index 1f8ae4696532..03028877f2f3 100644 --- a/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties @@ -24,7 +24,7 @@ druid.plaintextPort=8091 druid.worker.capacity=2 # Task launch parameters -druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task # HTTP server threads diff --git a/examples/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties b/examples/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties index 85e8c4c40205..cfb02d7e8df3 100644 --- a/examples/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties @@ -24,7 +24,7 @@ druid.plaintextPort=8091 druid.worker.capacity=2 # Task launch parameters -druid.indexer.runner.javaOpts=-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.runner.javaOptsArray=["-server","-Xms256m","-Xmx256m","-XX:MaxDirectMemorySize=300m","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task # HTTP server threads diff --git a/examples/conf/druid/single-server/small/middleManager/runtime.properties b/examples/conf/druid/single-server/small/middleManager/runtime.properties index dd902d349b1b..c4d95ca66cc2 100644 --- a/examples/conf/druid/single-server/small/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/small/middleManager/runtime.properties @@ -24,7 +24,7 @@ druid.plaintextPort=8091 druid.worker.capacity=3 # Task launch parameters -druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task # HTTP server threads diff --git a/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties b/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties index 8a5924be709d..11f048d1c0b9 100644 --- a/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties @@ -24,7 +24,7 @@ druid.plaintextPort=8091 druid.worker.capacity=16 # Task launch parameters -druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task # HTTP server threads diff --git a/licenses.yaml b/licenses.yaml index c64e372cc74c..a2d4d580b2d3 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -4052,7 +4052,7 @@ name: ASM license_category: binary module: java-core license_name: BSD-3-Clause License -version: 7.1 +version: 9.3 copyright: INRIA, France Telecom license_file_path: licenses/bin/asm.BSD3 libraries: diff --git a/pom.xml b/pom.xml index ad6297b7e00a..df4657bda857 100644 --- a/pom.xml +++ b/pom.xml @@ -815,12 +815,12 @@ org.ow2.asm asm - 7.1 + 9.3 org.ow2.asm asm-commons - 7.1 + 9.3 org.asynchttpclient @@ -1122,7 +1122,7 @@ nl.jqno.equalsverifier equalsverifier - 3.5.5 + 3.10.1 test @@ -1669,7 +1669,7 @@ - + --add-exports=java.base/jdk.internal.perf=ALL-UNNAMED --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED @@ -1679,6 +1679,16 @@ --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + + + --add-opens=java.base/java.io=ALL-UNNAMED + + + --add-opens=java.base/java.lang=ALL-UNNAMED + + + + --add-opens=java.base/java.util=ALL-UNNAMED diff --git a/processing/src/main/java/org/apache/druid/frame/file/FrameFile.java b/processing/src/main/java/org/apache/druid/frame/file/FrameFile.java index fce9612b3636..73bdd45b12ff 100644 --- a/processing/src/main/java/org/apache/druid/frame/file/FrameFile.java +++ b/processing/src/main/java/org/apache/druid/frame/file/FrameFile.java @@ -34,6 +34,7 @@ import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.segment.ReferenceCountingCloseableObject; import org.apache.druid.utils.CloseableUtils; +import org.apache.druid.utils.JvmUtils; import java.io.Closeable; import java.io.File; @@ -330,7 +331,25 @@ private static Pair mapFileBB(final File file) throws IOExcep */ private static Pair mapFileDS(final File file) { - final MapHandle mapHandle = Memory.map(file, 0, file.length(), ByteOrder.LITTLE_ENDIAN); + final MapHandle mapHandle; + + try { + mapHandle = Memory.map(file, 0, file.length(), ByteOrder.LITTLE_ENDIAN); + } + catch (NoClassDefFoundError | ExceptionInInitializerError e) { + // Memory.map does not work on JDK 14+ due to issues with AllocateDirectMap. + if (JvmUtils.majorVersion() >= 14) { + throw new ISE( + "Cannot read frame files larger than %,d bytes with Java %d. Try using Java 11.", + Integer.MAX_VALUE, + JvmUtils.majorVersion() + ); + } else { + // We don't have a good reason why this happened. Throw the original error. + throw e; + } + } + return Pair.of( mapHandle.get(), () -> { diff --git a/processing/src/test/java/org/apache/druid/frame/testutil/FrameTestUtil.java b/processing/src/test/java/org/apache/druid/frame/testutil/FrameTestUtil.java index 0884b60437da..14b4be36293e 100644 --- a/processing/src/test/java/org/apache/druid/frame/testutil/FrameTestUtil.java +++ b/processing/src/test/java/org/apache/druid/frame/testutil/FrameTestUtil.java @@ -47,6 +47,7 @@ import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorCursor; import org.apache.druid.timeline.SegmentId; +import org.apache.druid.utils.JvmUtils; import org.junit.Assert; import javax.annotation.Nullable; @@ -58,7 +59,6 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.StringTokenizer; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -295,8 +295,7 @@ public static Sequence> readRowsFromVectorCursor(final VectorCursor */ public static boolean jdkCanDataSketchesMemoryMap() { - final StringTokenizer st = new StringTokenizer(System.getProperty("java.specification.version"), "."); - return Integer.parseInt(st.nextToken()) < 14; + return JvmUtils.majorVersion() < 14; } private static Supplier dimensionSelectorReader(final DimensionSelector selector) diff --git a/server/src/main/java/org/apache/druid/server/coordination/SegmentLoadDropHandler.java b/server/src/main/java/org/apache/druid/server/coordination/SegmentLoadDropHandler.java index ba918d8fbd1e..2ba5dc933042 100644 --- a/server/src/main/java/org/apache/druid/server/coordination/SegmentLoadDropHandler.java +++ b/server/src/main/java/org/apache/druid/server/coordination/SegmentLoadDropHandler.java @@ -351,11 +351,11 @@ each time when addSegment() is called, it has to wait for the lock in order to m result = Status.SUCCESS; } - catch (Exception e) { + catch (Throwable e) { log.makeAlert(e, "Failed to load segment for dataSource") .addData("segment", segment) .emit(); - result = Status.failed(e.getMessage()); + result = Status.failed(e.toString()); } finally { updateRequestStatus(new SegmentChangeRequestLoad(segment), result); diff --git a/website/sidebars.json b/website/sidebars.json index 7f703ef15264..1a1b27208cb7 100644 --- a/website/sidebars.json +++ b/website/sidebars.json @@ -143,6 +143,7 @@ "Operations": [ "operations/druid-console", "operations/getting-started", + "operations/java", { "type": "subcategory", "label": "Security", From c7dc1c92f8615331fff9cde48c68d017f100ce1a Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Sat, 30 Jul 2022 10:12:51 -0700 Subject: [PATCH 02/12] Adjustments. --- distribution/bin/check-licenses.py | 1 + docs/operations/java.md | 4 +- .../apache/druid/frame/file/FrameFile.java | 30 +++++++----- .../druid/frame/file/FrameFileTest.java | 49 +++++++++++++++++++ website/.spelling | 2 + 5 files changed, 73 insertions(+), 13 deletions(-) diff --git a/distribution/bin/check-licenses.py b/distribution/bin/check-licenses.py index 1de9b1e22e7a..b5a2c2e933dd 100755 --- a/distribution/bin/check-licenses.py +++ b/distribution/bin/check-licenses.py @@ -249,6 +249,7 @@ def build_compatible_license_names(): compatible_licenses['New BSD License'] = 'BSD-3-Clause License' compatible_licenses['3-Clause BSD License'] = 'BSD-3-Clause License' compatible_licenses['BSD 3-Clause'] = 'BSD-3-Clause License' + compatible_licenses['BSD-3-Clause'] = 'BSD-3-Clause License' compatible_licenses['ICU License'] = 'ICU License' diff --git a/docs/operations/java.md b/docs/operations/java.md index 9808561ace4b..c64264d46ec5 100644 --- a/docs/operations/java.md +++ b/docs/operations/java.md @@ -32,7 +32,7 @@ Druid fully supports Java 8 and 11. The project team recommends Java 11. The project team does not recommend running with Java 17, because certain Druid functionality is not currently compatible with Java 17. -There are many free and actively-supported Java runtimes available for these versions, such as +There are many free and actively-supported Java runtime environments available, such as [Corretto](https://aws.amazon.com/corretto/) and [Zulu](https://www.azul.com/downloads/?package=jdk#download-openjdk). The project team does not recommend any specific runtime over any other. @@ -63,7 +63,7 @@ WARNING: All illegal access operations will be denied in a future release ``` These warning messages are harmless, and can be ignored. However, you can avoid them entirely if you wish by adding the -following Java command line parameters. These paramaters are not part of the default configurations that ship with +following Java command line parameters. These parameters are not part of the default configurations that ship with Druid, because Java 8 does not recognize these parameters and fails to start up if they are provided. To do this, add the following lines to your `jvm.config` files: diff --git a/processing/src/main/java/org/apache/druid/frame/file/FrameFile.java b/processing/src/main/java/org/apache/druid/frame/file/FrameFile.java index 73bdd45b12ff..886d6ec1316c 100644 --- a/processing/src/main/java/org/apache/druid/frame/file/FrameFile.java +++ b/processing/src/main/java/org/apache/druid/frame/file/FrameFile.java @@ -19,6 +19,7 @@ package org.apache.druid.frame.file; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.io.Files; import org.apache.datasketches.memory.MapHandle; @@ -29,6 +30,7 @@ import org.apache.druid.java.util.common.IOE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.RE; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.logger.Logger; @@ -337,17 +339,7 @@ private static Pair mapFileDS(final File file) mapHandle = Memory.map(file, 0, file.length(), ByteOrder.LITTLE_ENDIAN); } catch (NoClassDefFoundError | ExceptionInInitializerError e) { - // Memory.map does not work on JDK 14+ due to issues with AllocateDirectMap. - if (JvmUtils.majorVersion() >= 14) { - throw new ISE( - "Cannot read frame files larger than %,d bytes with Java %d. Try using Java 11.", - Integer.MAX_VALUE, - JvmUtils.majorVersion() - ); - } else { - // We don't have a good reason why this happened. Throw the original error. - throw e; - } + throw handleMemoryMapError(e, JvmUtils.majorVersion()); } return Pair.of( @@ -363,6 +355,22 @@ private static Pair mapFileDS(final File file) ); } + @VisibleForTesting + static RuntimeException handleMemoryMapError(Throwable e, int javaMajorVersion) + { + // Memory.map does not work on JDK 14+ due to issues with AllocateDirectMap. + if (javaMajorVersion >= 14) { + throw new ISE( + "Cannot read frame files larger than %,d bytes with Java %d. Try using Java 11.", + Integer.MAX_VALUE, + javaMajorVersion + ); + } else { + // We don't have a good reason why this happened. Throw the original error. + throw new RE(e, "Could not map frame file"); + } + } + private static long getFrameEndPosition(final Memory memory, final int frameNumber, final int numFrames) { final long frameEndPointerPosition = diff --git a/processing/src/test/java/org/apache/druid/frame/file/FrameFileTest.java b/processing/src/test/java/org/apache/druid/frame/file/FrameFileTest.java index 6eec5e5ce11c..034f732820a4 100644 --- a/processing/src/test/java/org/apache/druid/frame/file/FrameFileTest.java +++ b/processing/src/test/java/org/apache/druid/frame/file/FrameFileTest.java @@ -41,12 +41,15 @@ import org.apache.druid.segment.incremental.IncrementalIndexStorageAdapter; import org.apache.druid.testing.InitializedNullHandlingTest; import org.apache.druid.timeline.SegmentId; +import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; import org.junit.Assert; import org.junit.Assume; import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.internal.matchers.ThrowableMessageMatcher; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; @@ -372,6 +375,52 @@ public void test_newReference() throws IOException frameFile1.newReference(); } + @Test + public void test_handleMemoryMapError_java11() + { + @SuppressWarnings("ThrowableNotThrown") + final RuntimeException e = Assert.assertThrows( + RuntimeException.class, + () -> FrameFile.handleMemoryMapError(new IllegalAccessError("foo"), 11) + ); + + MatcherAssert.assertThat( + e, + ThrowableMessageMatcher.hasMessage(CoreMatchers.equalTo("Could not map frame file")) + ); + + // Include the original error, since we don't have a better explanation. + MatcherAssert.assertThat( + e.getCause(), + CoreMatchers.instanceOf(IllegalAccessError.class) + ); + } + + @Test + public void test_handleMemoryMapError_java17() + { + @SuppressWarnings("ThrowableNotThrown") + final IllegalStateException e = Assert.assertThrows( + IllegalStateException.class, + () -> FrameFile.handleMemoryMapError(new IllegalAccessError("foo"), 17) + ); + + MatcherAssert.assertThat( + e, + ThrowableMessageMatcher.hasMessage( + CoreMatchers.containsString( + StringUtils.format( + "Cannot read frame files larger than %,d bytes with Java 17.", + Integer.MAX_VALUE + ) + ) + ) + ); + + // Cause not included; we want to keep logs relatively cleaner and highlight the actual issue. + Assert.assertNull(e.getCause()); + } + private int computeExpectedNumFrames() { return IntMath.divide(countRows(adapter), maxRowsPerFrame, RoundingMode.CEILING); diff --git a/website/.spelling b/website/.spelling index 3db7cb020ecf..5e7661950226 100644 --- a/website/.spelling +++ b/website/.spelling @@ -1429,6 +1429,8 @@ time-iso8601 hadoopStorageDirectory - ../docs/operations/insert-segment-to-db.md 0.14.x + - ../docs/operations/java.md +G1 - ../docs/operations/metrics.md 0.14.x 1s From 532dada019e51cc0512348e3a406b77df635ae0a Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Sun, 31 Jul 2022 10:58:53 -0700 Subject: [PATCH 03/12] Use run-java in more places. --- distribution/pom.xml | 8 ++++---- examples/bin/run-druid | 7 +------ examples/bin/run-zk | 7 +------ .../druid/cluster/data/middleManager/runtime.properties | 1 + .../single-server/large/middleManager/runtime.properties | 1 + .../single-server/medium/middleManager/runtime.properties | 1 + .../micro-quickstart/middleManager/runtime.properties | 1 + .../nano-quickstart/middleManager/runtime.properties | 1 + .../single-server/small/middleManager/runtime.properties | 1 + .../single-server/xlarge/middleManager/runtime.properties | 1 + 10 files changed, 13 insertions(+), 16 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index 6654bdd072de..84d3c218fb83 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -173,7 +173,7 @@ exec - java + ${project.parent.basedir}/examples/bin/run-java -classpath @@ -350,7 +350,7 @@ exec - java + ${project.parent.basedir}/examples/bin/run-java -classpath @@ -548,7 +548,7 @@ exec - java + ${project.parent.basedir}/examples/bin/run-java -classpath @@ -642,7 +642,7 @@ exec - java + ${project.parent.basedir}/examples/bin/run-java -classpath diff --git a/examples/bin/run-druid b/examples/bin/run-druid index 93765f6f97ea..bc484e9d6420 100755 --- a/examples/bin/run-druid +++ b/examples/bin/run-druid @@ -36,11 +36,6 @@ fi CONFDIR="$(cd "$CONFDIR" && pwd)" WHEREAMI="$(cd "$WHEREAMI" && pwd)" -JAVA_BIN="$(source "$WHEREAMI"/java-util && get_java_bin_dir)" -if [ -z "$JAVA_BIN" ]; then - >&2 echo "Could not find java - please run $WHEREAMI/verify-java to confirm it is installed." - exit 1 -fi LOG_DIR="${DRUID_LOG_DIR:=${WHEREAMI}/../log}" # Remove possible ending slash @@ -53,6 +48,6 @@ if [ ! -d "$LOG_DIR" ]; then mkdir -p $LOG_DIR; fi echo "Running [$1], logging to [$LOG_DIR/$1.log] if no changes made to log4j2.xml" cd "$WHEREAMI/.." -exec "$JAVA_BIN"/java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ +exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" \ `cat "$CONFDIR"/$WHATAMI/main.config | xargs` diff --git a/examples/bin/run-zk b/examples/bin/run-zk index 8dc5b648584b..81a516315224 100755 --- a/examples/bin/run-zk +++ b/examples/bin/run-zk @@ -35,11 +35,6 @@ fi CONFDIR="$(cd "$CONFDIR" && pwd)/zk" WHEREAMI="$(cd "$WHEREAMI" && pwd)" -JAVA_BIN="$(source "$WHEREAMI"/java-util && get_java_bin_dir)" -if [ -z "$JAVA_BIN" ]; then - >&2 echo "Could not find java - please run $WHEREAMI/verify-java to confirm it is installed." - exit 1 -fi LOG_DIR="${DRUID_LOG_DIR:=${WHEREAMI}/../log}" # Remove possible ending slash @@ -52,7 +47,7 @@ if [ ! -d "$LOG_DIR" ]; then mkdir -p $LOG_DIR; fi echo "Running [ZooKeeper], logging to [$LOG_DIR/zookeeper.log] if no changes made to log4j2.xml" cd "$WHEREAMI/.." -exec "$JAVA_BIN"/java "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/jvm.config | xargs` \ +exec "$WHEREAMI"/run-java "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/jvm.config | xargs` \ -cp "$WHEREAMI/../lib/*:$CONFDIR" \ -Dzookeeper.jmx.log4j.disable=true \ org.apache.zookeeper.server.quorum.QuorumPeerMain \ diff --git a/examples/conf/druid/cluster/data/middleManager/runtime.properties b/examples/conf/druid/cluster/data/middleManager/runtime.properties index 6be77c3635e2..c44add1b8aca 100644 --- a/examples/conf/druid/cluster/data/middleManager/runtime.properties +++ b/examples/conf/druid/cluster/data/middleManager/runtime.properties @@ -24,6 +24,7 @@ druid.plaintextPort=8091 druid.worker.capacity=4 # Task launch parameters +druid.indexer.runner.javaCommand=bin/run-java druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task diff --git a/examples/conf/druid/single-server/large/middleManager/runtime.properties b/examples/conf/druid/single-server/large/middleManager/runtime.properties index e11aa40614be..bdfaa6b22224 100644 --- a/examples/conf/druid/single-server/large/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/large/middleManager/runtime.properties @@ -24,6 +24,7 @@ druid.plaintextPort=8091 druid.worker.capacity=8 # Task launch parameters +druid.indexer.runner.javaCommand=bin/run-java druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task diff --git a/examples/conf/druid/single-server/medium/middleManager/runtime.properties b/examples/conf/druid/single-server/medium/middleManager/runtime.properties index 6be77c3635e2..c44add1b8aca 100644 --- a/examples/conf/druid/single-server/medium/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/medium/middleManager/runtime.properties @@ -24,6 +24,7 @@ druid.plaintextPort=8091 druid.worker.capacity=4 # Task launch parameters +druid.indexer.runner.javaCommand=bin/run-java druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task diff --git a/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties b/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties index 03028877f2f3..e7262930f498 100644 --- a/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties @@ -24,6 +24,7 @@ druid.plaintextPort=8091 druid.worker.capacity=2 # Task launch parameters +druid.indexer.runner.javaCommand=bin/run-java druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task diff --git a/examples/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties b/examples/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties index cfb02d7e8df3..4eadb41bf661 100644 --- a/examples/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties @@ -24,6 +24,7 @@ druid.plaintextPort=8091 druid.worker.capacity=2 # Task launch parameters +druid.indexer.runner.javaCommand=bin/run-java druid.indexer.runner.javaOptsArray=["-server","-Xms256m","-Xmx256m","-XX:MaxDirectMemorySize=300m","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task diff --git a/examples/conf/druid/single-server/small/middleManager/runtime.properties b/examples/conf/druid/single-server/small/middleManager/runtime.properties index c4d95ca66cc2..603ae60776ca 100644 --- a/examples/conf/druid/single-server/small/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/small/middleManager/runtime.properties @@ -24,6 +24,7 @@ druid.plaintextPort=8091 druid.worker.capacity=3 # Task launch parameters +druid.indexer.runner.javaCommand=bin/run-java druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task diff --git a/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties b/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties index 11f048d1c0b9..6dd805f5d60c 100644 --- a/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties @@ -24,6 +24,7 @@ druid.plaintextPort=8091 druid.worker.capacity=16 # Task launch parameters +druid.indexer.runner.javaCommand=bin/run-java druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task From 52b836af0deaf91c1e54216796dfcf27c9cb6fbe Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Sun, 31 Jul 2022 12:26:40 -0700 Subject: [PATCH 04/12] Add run-java. --- examples/bin/run-java | 90 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100755 examples/bin/run-java diff --git a/examples/bin/run-java b/examples/bin/run-java new file mode 100755 index 000000000000..ca3122f1ca92 --- /dev/null +++ b/examples/bin/run-java @@ -0,0 +1,90 @@ +#!/bin/bash -eu + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +WHEREAMI="$(dirname "$0")" +JAVA_BIN="$(source "$WHEREAMI"/java-util && get_java_bin_dir)/java" +if [ -z "$JAVA_BIN" ]; then + >&2 echo "Could not find java - please run $WHEREAMI/verify-java to confirm it is installed." + exit 1 +fi + +JAVA_MAJOR="$("$JAVA_BIN" -version 2>&1 | grep ' version ' | head -n1 | sed -E 's/.*version "([^"]+)".*/\1/' | cut -d. -f1)" + +if [ "$JAVA_MAJOR" != "" ] && [ "$JAVA_MAJOR" -ge "17" ] +then + # Must disable strong encapsulation for certain packages on Java 17. + exec "$JAVA_BIN" \ + --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED \ + --add-exports=java.base/jdk.internal.perf=ALL-UNNAMED \ + --add-exports=java.base/jdk.internal.ref=ALL-UNNAMED \ + --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \ + --add-opens=java.base/java.nio=ALL-UNNAMED \ + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \ + --add-opens=java.base/java.io=ALL-UNNAMED \ + --add-opens=java.base/java.lang=ALL-UNNAMED \ + --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED \ + $* +else + exec "$JAVA_BIN" $* +fi +#!/bin/bash -eu + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +WHEREAMI="$(dirname "$0")" +JAVA_BIN="$(source "$WHEREAMI"/java-util && get_java_bin_dir)/java" +if [ -z "$JAVA_BIN" ]; then + >&2 echo "Could not find java - please run $WHEREAMI/verify-java to confirm it is installed." + exit 1 +fi + +JAVA_MAJOR="$("$JAVA_BIN" -version 2>&1 | grep ' version ' | head -n1 | sed -E 's/.*version "([^"]+)".*/\1/' | cut -d. -f1)" + +if [ "$JAVA_MAJOR" != "" ] && [ "$JAVA_MAJOR" -ge "17" ] +then + # Must disable strong encapsulation for certain packages on Java 17. + exec "$JAVA_BIN" \ + --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED \ + --add-exports=java.base/jdk.internal.perf=ALL-UNNAMED \ + --add-exports=java.base/jdk.internal.ref=ALL-UNNAMED \ + --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \ + --add-opens=java.base/java.nio=ALL-UNNAMED \ + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \ + --add-opens=java.base/java.io=ALL-UNNAMED \ + --add-opens=java.base/java.lang=ALL-UNNAMED \ + --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED \ + $* +else + exec "$JAVA_BIN" $* +fi From e65b3496addc3064376c859492c7f8c348dcf116 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Sun, 31 Jul 2022 12:28:57 -0700 Subject: [PATCH 05/12] Update .gitignore. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 53d3303bf9b2..dfd02101ec62 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,4 @@ README .pmdruleset.xml .java-version integration-tests/gen-scripts/ -bin/ +/bin/ From 411e246f207b50db1ecc012fe59cbcb00779e067 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Sun, 31 Jul 2022 13:27:04 -0700 Subject: [PATCH 06/12] Exclude hadoop-client-api. Brought in when building on Java 17. --- extensions-core/orc-extensions/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/extensions-core/orc-extensions/pom.xml b/extensions-core/orc-extensions/pom.xml index 2035974568be..a620d15abeaa 100644 --- a/extensions-core/orc-extensions/pom.xml +++ b/extensions-core/orc-extensions/pom.xml @@ -138,6 +138,10 @@ org.apache.hadoop hadoop-hdfs + + org.apache.hadoop + hadoop-client-api + org.slf4j slf4j-api From 246cd9b753ed9c12c16654ba84e8659135fcef4a Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Sun, 31 Jul 2022 14:35:11 -0700 Subject: [PATCH 07/12] Swap one more usage of java. --- web-console/script/druid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-console/script/druid b/web-console/script/druid index 192d43281d69..9f54949b96cd 100755 --- a/web-console/script/druid +++ b/web-console/script/druid @@ -60,7 +60,7 @@ function _build_distribution() { && cd distribution/target \ && tar xzf "apache-druid-$(_get_druid_version)-bin.tar.gz" \ && cd apache-druid-$(_get_druid_version) \ - && java -classpath "lib/*" org.apache.druid.cli.Main tools pull-deps -c org.apache.druid.extensions:druid-testing-tools \ + && bin/run-java -classpath "lib/*" org.apache.druid.cli.Main tools pull-deps -c org.apache.druid.extensions:druid-testing-tools \ && echo -e "\n\ndruid.extensions.loadList=[\"druid-hdfs-storage\", \"druid-kafka-indexing-service\", \"druid-datasketches\", \"druid-testing-tools\"]" >> conf/druid/single-server/micro-quickstart/_common/common.runtime.properties \ && echo -e "\n\ndruid.server.http.allowedHttpMethods=[\"HEAD\"]" >> conf/druid/single-server/micro-quickstart/_common/common.runtime.properties \ ) From 58d98119a9a229b18abbbe06a57fbbd60c89da67 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Sun, 31 Jul 2022 15:10:19 -0700 Subject: [PATCH 08/12] Fix the run-java script. --- examples/bin/run-java | 49 ++----------------------------------------- 1 file changed, 2 insertions(+), 47 deletions(-) diff --git a/examples/bin/run-java b/examples/bin/run-java index ca3122f1ca92..922503ad1741 100755 --- a/examples/bin/run-java +++ b/examples/bin/run-java @@ -39,52 +39,7 @@ then --add-opens=java.base/java.io=ALL-UNNAMED \ --add-opens=java.base/java.lang=ALL-UNNAMED \ --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED \ - $* + "$@" else - exec "$JAVA_BIN" $* -fi -#!/bin/bash -eu - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -WHEREAMI="$(dirname "$0")" -JAVA_BIN="$(source "$WHEREAMI"/java-util && get_java_bin_dir)/java" -if [ -z "$JAVA_BIN" ]; then - >&2 echo "Could not find java - please run $WHEREAMI/verify-java to confirm it is installed." - exit 1 -fi - -JAVA_MAJOR="$("$JAVA_BIN" -version 2>&1 | grep ' version ' | head -n1 | sed -E 's/.*version "([^"]+)".*/\1/' | cut -d. -f1)" - -if [ "$JAVA_MAJOR" != "" ] && [ "$JAVA_MAJOR" -ge "17" ] -then - # Must disable strong encapsulation for certain packages on Java 17. - exec "$JAVA_BIN" \ - --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED \ - --add-exports=java.base/jdk.internal.perf=ALL-UNNAMED \ - --add-exports=java.base/jdk.internal.ref=ALL-UNNAMED \ - --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \ - --add-opens=java.base/java.nio=ALL-UNNAMED \ - --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \ - --add-opens=java.base/java.io=ALL-UNNAMED \ - --add-opens=java.base/java.lang=ALL-UNNAMED \ - --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED \ - $* -else - exec "$JAVA_BIN" $* + exec "$JAVA_BIN" "$@" fi From e01dc7d60704051300b0e2a538ba223ecfc032bc Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Mon, 1 Aug 2022 07:09:57 -0700 Subject: [PATCH 09/12] Fix flag. --- docs/operations/java.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/java.md b/docs/operations/java.md index c64264d46ec5..79c735043c77 100644 --- a/docs/operations/java.md +++ b/docs/operations/java.md @@ -43,7 +43,7 @@ Druid relies on the environment variables `JAVA_HOME` or `DRUID_JAVA_HOME` to fi ## Garbage collection In general, we recommend using the G1 collector with default settings. This is the default collector in Java 11. -To enable G1 on Java 8, use `+XX:UseG1GC`. There is no harm in explicitly specifying this on Java 11 as well. +To enable G1 on Java 8, use `-XX:+UseG1GC`. There is no harm in explicitly specifying this on Java 11 as well. Garbage collector selection and tuning is a form of sport in the Java community. There may be situations where adjusting garbage collection configuration improves or worsens performance. The project team's guidance is that most people do From 4706f4395e862f851bb78b7a224d312e3b8dfa00 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 2 Aug 2022 07:23:51 -0700 Subject: [PATCH 10/12] Include link to Temurin. --- docs/development/build.md | 5 +---- docs/operations/java.md | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/docs/development/build.md b/docs/development/build.md index 8487161d35d9..9036c8fbc45b 100644 --- a/docs/development/build.md +++ b/docs/development/build.md @@ -33,10 +33,7 @@ make sure it has `/master/` in the URL. ##### Installing Java and Maven -- JDK 8, 8u92+ or JDK 11 - - We recommend using an OpenJDK distribution that provides long-term support and open-source licensing, - like [Amazon Corretto](https://aws.amazon.com/corretto/) or [Azul Zulu](https://www.azul.com/downloads/zulu/). +- JDK 8, 8u92+ or JDK 11. See our [Java documentation](../operations/java.md) for information about obtaining a JDK. - [Maven version 3.x](http://maven.apache.org/download.cgi) ##### Other dependencies diff --git a/docs/operations/java.md b/docs/operations/java.md index 79c735043c77..e5d852bc91a7 100644 --- a/docs/operations/java.md +++ b/docs/operations/java.md @@ -27,14 +27,15 @@ a Java runtime for Druid. ## Selecting a Java runtime -Druid fully supports Java 8 and 11. The project team recommends Java 11. +Druid fully supports Java 8 and 11. The project team recommends Java 11. The project team does not recommend running +with Java 17, because certain Druid functionality is not currently compatible with Java 17. -The project team does not recommend running with Java 17, because certain Druid functionality is not currently -compatible with Java 17. - -There are many free and actively-supported Java runtime environments available, such as -[Corretto](https://aws.amazon.com/corretto/) and [Zulu](https://www.azul.com/downloads/?package=jdk#download-openjdk). -The project team does not recommend any specific runtime over any other. +The project team recommends using an OpenJDK-based Java distribution. There are many free and actively-supported +distributions available, including +[Amazon Corretto](https://docs.aws.amazon.com/corretto/latest/corretto-11-ug/what-is-corretto-11.html), +[Azul Zulu](https://www.azul.com/downloads/?version=java-11-lts&package=jdk), and +[Eclipse Temurin](https://adoptium.net/temurin/releases?version=11). +The project team does not recommend any specific distribution over any other. Druid relies on the environment variables `JAVA_HOME` or `DRUID_JAVA_HOME` to find Java on the machine. You can set `DRUID_JAVA_HOME` if there is more than one instance of Java. To verify Java requirements for your environment, run the @@ -42,8 +43,8 @@ Druid relies on the environment variables `JAVA_HOME` or `DRUID_JAVA_HOME` to fi ## Garbage collection -In general, we recommend using the G1 collector with default settings. This is the default collector in Java 11. -To enable G1 on Java 8, use `-XX:+UseG1GC`. There is no harm in explicitly specifying this on Java 11 as well. +In general, the project team recommends using the G1 collector with default settings. This is the default collector in +Java 11. To enable G1 on Java 8, use `-XX:+UseG1GC`. There is no harm in explicitly specifying this on Java 11 as well. Garbage collector selection and tuning is a form of sport in the Java community. There may be situations where adjusting garbage collection configuration improves or worsens performance. The project team's guidance is that most people do From 0287d9bb7e6bf02cbfb45e7a323448613cd57429 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 2 Aug 2022 15:45:52 -0700 Subject: [PATCH 11/12] Spelling. --- docs/operations/metrics.md | 2 +- website/.spelling | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/operations/metrics.md b/docs/operations/metrics.md index 787fade39434..5b58aceab4bd 100644 --- a/docs/operations/metrics.md +++ b/docs/operations/metrics.md @@ -372,7 +372,7 @@ These metrics are only available if the SysMonitor module is included. |`sys/net/write/size`|Bytes written to the network.|netName, netAddress, netHwaddr|Varies.| |`sys/net/read/size`|Bytes read from the network.|netName, netAddress, netHwaddr|Varies.| |`sys/fs/used`|Filesystem bytes used.|fsDevName, fsDirName, fsTypeName, fsSysTypeName, fsOptions.|< max| -|`sys/fs/max`|Filesystesm bytes max.|fsDevName, fsDirName, fsTypeName, fsSysTypeName, fsOptions.|Varies.| +|`sys/fs/max`|Filesystem bytes max.|fsDevName, fsDirName, fsTypeName, fsSysTypeName, fsOptions.|Varies.| |`sys/mem/used`|Memory used.||< max| |`sys/mem/max`|Memory max.||Varies.| |`sys/storage/used`|Disk space used.|fsDirName.|Varies.| diff --git a/website/.spelling b/website/.spelling index 5e7661950226..088d27f4d0fa 100644 --- a/website/.spelling +++ b/website/.spelling @@ -1431,13 +1431,14 @@ hadoopStorageDirectory 0.14.x - ../docs/operations/java.md G1 +Temurin - ../docs/operations/metrics.md 0.14.x 1s Bufferpool EventReceiverFirehose EventReceiverFirehoseMonitor -Filesystesm +Filesystem JVMMonitor QueryCountStatsMonitor RealtimeMetricsMonitor From a66ce4add0b1d0ffa7f0572c099bf401343a8a46 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 3 Aug 2022 08:01:05 -0700 Subject: [PATCH 12/12] Update examples/bin/run-java MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Xavier Léauté --- examples/bin/run-java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/bin/run-java b/examples/bin/run-java index 922503ad1741..7438b1d84671 100755 --- a/examples/bin/run-java +++ b/examples/bin/run-java @@ -24,7 +24,7 @@ if [ -z "$JAVA_BIN" ]; then exit 1 fi -JAVA_MAJOR="$("$JAVA_BIN" -version 2>&1 | grep ' version ' | head -n1 | sed -E 's/.*version "([^"]+)".*/\1/' | cut -d. -f1)" +JAVA_MAJOR="$("$JAVA_BIN" -version 2>&1 | sed -n -E 's/.* version "([^."]*).*/\1/p')" if [ "$JAVA_MAJOR" != "" ] && [ "$JAVA_MAJOR" -ge "17" ] then