diff --git a/.gitignore b/.gitignore index 53d3303bf9b2..dfd02101ec62 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,4 @@ README .pmdruleset.xml .java-version integration-tests/gen-scripts/ -bin/ +/bin/ diff --git a/.travis.yml b/.travis.yml index d92b66c33367..aed248afdff8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -188,9 +188,9 @@ jobs: jdk: openjdk11 - <<: *package - name: "(openjdk15) packaging check" + name: "(openjdk17) packaging check" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - &test_processing_module name: "(openjdk8) processing module test" @@ -259,9 +259,9 @@ jobs: jdk: openjdk11 - <<: *test_processing_module - name: "(openjdk15) processing module test" + name: "(openjdk17) processing module test" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - &test_processing_module_sqlcompat <<: *test_processing_module @@ -276,9 +276,9 @@ jobs: jdk: openjdk11 - <<: *test_processing_module_sqlcompat - name: "(openjdk15) processing module test (SQL Compatibility)" + name: "(openjdk17) processing module test (SQL Compatibility)" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - &test_indexing_module <<: *test_processing_module @@ -292,9 +292,9 @@ jobs: jdk: openjdk11 - <<: *test_indexing_module - name: "(openjdk15) indexing modules test" + name: "(openjdk17) indexing modules test" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - &test_indexing_module_sqlcompat <<: *test_indexing_module @@ -308,9 +308,9 @@ jobs: jdk: openjdk11 - <<: *test_indexing_module_sqlcompat - name: "(openjdk15) indexing modules test (SQL Compatibility)" + name: "(openjdk17) indexing modules test (SQL Compatibility)" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - &test_server_module <<: *test_processing_module @@ -324,9 +324,9 @@ jobs: jdk: openjdk11 - <<: *test_server_module - name: "(openjdk15) server module test" + name: "(openjdk17) server module test" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - &test_server_module_sqlcompat <<: *test_server_module @@ -339,9 +339,9 @@ jobs: jdk: openjdk11 - <<: *test_server_module_sqlcompat - name: "(openjdk15) server module test (SQL Compatibility)" + name: "(openjdk17) server module test (SQL Compatibility)" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - &test_other_modules <<: *test_processing_module @@ -355,9 +355,9 @@ jobs: jdk: openjdk11 - <<: *test_other_modules - name: "(openjdk15) other modules test" + name: "(openjdk17) other modules test" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - &test_other_modules_sqlcompat <<: *test_other_modules @@ -370,9 +370,9 @@ jobs: jdk: openjdk11 - <<: *test_other_modules_sqlcompat - name: "(openjdk15) other modules test (SQL Compatibility)" + name: "(openjdk17) other modules test (SQL Compatibility)" stage: Tests - phase 2 - jdk: openjdk15 + jdk: openjdk17 - name: "web console" install: skip diff --git a/core/src/main/java/org/apache/druid/utils/JvmUtils.java b/core/src/main/java/org/apache/druid/utils/JvmUtils.java index 5c4869d4118f..64bb3778f3d8 100644 --- a/core/src/main/java/org/apache/druid/utils/JvmUtils.java +++ b/core/src/main/java/org/apache/druid/utils/JvmUtils.java @@ -19,6 +19,7 @@ package org.apache.druid.utils; +import com.google.common.primitives.Ints; import com.google.inject.Inject; import java.io.File; @@ -36,19 +37,37 @@ public class JvmUtils { - private static final boolean IS_JAVA9_COMPATIBLE = isJava9Compatible(System.getProperty("java.specification.version")); + public static final int UNKNOWN_VERSION = -1; + private static final int MAJOR_VERSION = computeMajorVersion(); - private static boolean isJava9Compatible(String versionString) + private static int computeMajorVersion() { - final StringTokenizer st = new StringTokenizer(versionString, "."); - int majorVersion = Integer.parseInt(st.nextToken()); + final StringTokenizer st = new StringTokenizer(System.getProperty("java.specification.version"), "."); + if (!st.hasMoreTokens()) { + return UNKNOWN_VERSION; + } + + final String majorVersionString = st.nextToken(); + final Integer majorVersion = Ints.tryParse(majorVersionString); + return majorVersion == null ? UNKNOWN_VERSION : majorVersion; + } - return majorVersion >= 9; + /** + * Returns the major version of the current Java runtime for Java 9 and above. For example: 9, 11, 17, etc. + * + * Returns 1 for Java 8 and earlier. + * + * Returns {@link #UNKNOWN_VERSION} if the major version cannot be determined. This is a negative number and is + * therefore lower than all valid versions. + */ + public static int majorVersion() + { + return MAJOR_VERSION; } public static boolean isIsJava9Compatible() { - return IS_JAVA9_COMPATIBLE; + return MAJOR_VERSION >= 9; } @Inject diff --git a/distribution/bin/check-licenses.py b/distribution/bin/check-licenses.py index 1de9b1e22e7a..b5a2c2e933dd 100755 --- a/distribution/bin/check-licenses.py +++ b/distribution/bin/check-licenses.py @@ -249,6 +249,7 @@ def build_compatible_license_names(): compatible_licenses['New BSD License'] = 'BSD-3-Clause License' compatible_licenses['3-Clause BSD License'] = 'BSD-3-Clause License' compatible_licenses['BSD 3-Clause'] = 'BSD-3-Clause License' + compatible_licenses['BSD-3-Clause'] = 'BSD-3-Clause License' compatible_licenses['ICU License'] = 'ICU License' diff --git a/distribution/pom.xml b/distribution/pom.xml index 6654bdd072de..84d3c218fb83 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -173,7 +173,7 @@ exec - java + ${project.parent.basedir}/examples/bin/run-java -classpath @@ -350,7 +350,7 @@ exec - java + ${project.parent.basedir}/examples/bin/run-java -classpath @@ -548,7 +548,7 @@ exec - java + ${project.parent.basedir}/examples/bin/run-java -classpath @@ -642,7 +642,7 @@ exec - java + ${project.parent.basedir}/examples/bin/run-java -classpath diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 2246727ad288..da54aa7dd535 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -1371,7 +1371,7 @@ Middle managers pass their configurations down to their child peons. The MiddleM |`druid.indexer.runner.classpath`|Java classpath for the peon.|System.getProperty("java.class.path")| |`druid.indexer.runner.javaCommand`|Command required to execute java.|java| |`druid.indexer.runner.javaOpts`|*DEPRECATED* A string of -X Java options to pass to the peon's JVM. Quotable parameters or parameters with spaces are encouraged to use javaOptsArray|""| -|`druid.indexer.runner.javaOptsArray`|A JSON array of strings to be passed in as options to the peon's JVM. This is additive to javaOpts and is recommended for properly handling arguments which contain quotes or spaces like `["-XX:OnOutOfMemoryError=kill -9 %p"]`|`[]`| +|`druid.indexer.runner.javaOptsArray`|A JSON array of strings to be passed in as options to the peon's JVM. This is additive to `druid.indexer.runner.javaOpts` and is recommended for properly handling arguments which contain quotes or spaces like `["-XX:OnOutOfMemoryError=kill -9 %p"]`|`[]`| |`druid.indexer.runner.maxZnodeBytes`|The maximum size Znode in bytes that can be created in ZooKeeper, should be in the range of [10KiB, 2GiB). [Human-readable format](human-readable-byte.md) is supported.|512KiB| |`druid.indexer.runner.startPort`|Starting port used for peon processes, should be greater than 1023 and less than 65536.|8100| |`druid.indexer.runner.endPort`|Ending port used for peon processes, should be greater than or equal to `druid.indexer.runner.startPort` and less than 65536.|65535| diff --git a/docs/configuration/logging.md b/docs/configuration/logging.md index 9641b9d64a9a..1be881a0cbfd 100644 --- a/docs/configuration/logging.md +++ b/docs/configuration/logging.md @@ -104,16 +104,8 @@ WARNING: Use --illegal-access=warn to enable warnings of further illegal reflect WARNING: All illegal access operations will be denied in a future release ``` -These messages do not cause harm, but you can avoid them by adding the following lines to your `jvm.config` files. These -lines are not part of the default JVM configs that ship with Druid, because Java 8 will not recognize these options and -will fail to start up. - -``` ---add-exports=java.base/jdk.internal.ref=ALL-UNNAMED ---add-exports=java.base/jdk.internal.perf=ALL-UNNAMED ---add-opens=java.base/java.lang=ALL-UNNAMED ---add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED -``` +To avoid these, add the `--add-exports` and `--add-opens` command line parameters described in the documentation section +about [Java strong encapsulation](../operations/java.md#strong-encapsulation). ## My logs are really chatty, can I set them to asynchronously write? diff --git a/docs/development/build.md b/docs/development/build.md index 8487161d35d9..9036c8fbc45b 100644 --- a/docs/development/build.md +++ b/docs/development/build.md @@ -33,10 +33,7 @@ make sure it has `/master/` in the URL. ##### Installing Java and Maven -- JDK 8, 8u92+ or JDK 11 - - We recommend using an OpenJDK distribution that provides long-term support and open-source licensing, - like [Amazon Corretto](https://aws.amazon.com/corretto/) or [Azul Zulu](https://www.azul.com/downloads/zulu/). +- JDK 8, 8u92+ or JDK 11. See our [Java documentation](../operations/java.md) for information about obtaining a JDK. - [Maven version 3.x](http://maven.apache.org/download.cgi) ##### Other dependencies diff --git a/docs/operations/java.md b/docs/operations/java.md new file mode 100644 index 000000000000..e5d852bc91a7 --- /dev/null +++ b/docs/operations/java.md @@ -0,0 +1,94 @@ +--- +id: java +title: "Java runtime" +--- + + + +Apache Druid is written in Java and requires a Java runtime. This page provides details about obtaining and configuring +a Java runtime for Druid. + +## Selecting a Java runtime + +Druid fully supports Java 8 and 11. The project team recommends Java 11. The project team does not recommend running +with Java 17, because certain Druid functionality is not currently compatible with Java 17. + +The project team recommends using an OpenJDK-based Java distribution. There are many free and actively-supported +distributions available, including +[Amazon Corretto](https://docs.aws.amazon.com/corretto/latest/corretto-11-ug/what-is-corretto-11.html), +[Azul Zulu](https://www.azul.com/downloads/?version=java-11-lts&package=jdk), and +[Eclipse Temurin](https://adoptium.net/temurin/releases?version=11). +The project team does not recommend any specific distribution over any other. + +Druid relies on the environment variables `JAVA_HOME` or `DRUID_JAVA_HOME` to find Java on the machine. You can set +`DRUID_JAVA_HOME` if there is more than one instance of Java. To verify Java requirements for your environment, run the +`bin/verify-java` script. + +## Garbage collection + +In general, the project team recommends using the G1 collector with default settings. This is the default collector in +Java 11. To enable G1 on Java 8, use `-XX:+UseG1GC`. There is no harm in explicitly specifying this on Java 11 as well. + +Garbage collector selection and tuning is a form of sport in the Java community. There may be situations where adjusting +garbage collection configuration improves or worsens performance. The project team's guidance is that most people do +not need to stray away from G1 with default settings. + +## Strong encapsulation + +Java 9 and beyond (including Java 11) include the capability for +[strong encapsulation](https://dev.java/learn/strong-encapsulation-\(of-jdk-internals\)/) of internal JDK APIs. Druid +uses certain internal JDK APIs for functionality- and performance-related reasons. In Java 11, this leads to log +messages like the following: + +``` +WARNING: An illegal reflective access operation has occurred +WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations +WARNING: All illegal access operations will be denied in a future release +``` + +These warning messages are harmless, and can be ignored. However, you can avoid them entirely if you wish by adding the +following Java command line parameters. These parameters are not part of the default configurations that ship with +Druid, because Java 8 does not recognize these parameters and fails to start up if they are provided. + +To do this, add the following lines to your `jvm.config` files: + +``` +--add-exports=java.base/jdk.internal.perf=ALL-UNNAMED +--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED +--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED +--add-opens=java.base/java.lang=ALL-UNNAMED +--add-opens=java.base/java.io=ALL-UNNAMED +--add-opens=java.base/java.nio=ALL-UNNAMED +--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED +--add-opens=java.base/sun.nio.ch=ALL-UNNAMED +--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED +``` + +Additionally, tasks run by [MiddleManagers](../design/architecture.md) execute in separate JVMs. The command line for +these JVMs is given by `druid.indexer.runner.javaOptsArray` or `druid.indexer.runner.javaOpts` in +`middleManager/runtime.properties`. Java command line parameters for tasks must be specified here. For example, use +a line like the following: + +``` +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager","--add-exports=java.base/jdk.internal.perf=ALL-UNNAMED","--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED","--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED","--add-opens=java.base/java.lang=ALL-UNNAMED","--add-opens=java.base/java.io=ALL-UNNAMED","--add-opens=java.base/java.nio=ALL-UNNAMED","--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED","--add-opens=java.base/sun.nio.ch=ALL-UNNAMED","--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED"] +``` + +The `Xms`, `Xmx`, and `MaxDirectMemorySize` parameters in the line above are merely an example. You may use different +values in your specific environment. diff --git a/docs/operations/metrics.md b/docs/operations/metrics.md index 787fade39434..5b58aceab4bd 100644 --- a/docs/operations/metrics.md +++ b/docs/operations/metrics.md @@ -372,7 +372,7 @@ These metrics are only available if the SysMonitor module is included. |`sys/net/write/size`|Bytes written to the network.|netName, netAddress, netHwaddr|Varies.| |`sys/net/read/size`|Bytes read from the network.|netName, netAddress, netHwaddr|Varies.| |`sys/fs/used`|Filesystem bytes used.|fsDevName, fsDirName, fsTypeName, fsSysTypeName, fsOptions.|< max| -|`sys/fs/max`|Filesystesm bytes max.|fsDevName, fsDirName, fsTypeName, fsSysTypeName, fsOptions.|Varies.| +|`sys/fs/max`|Filesystem bytes max.|fsDevName, fsDirName, fsTypeName, fsSysTypeName, fsOptions.|Varies.| |`sys/mem/used`|Memory used.||< max| |`sys/mem/max`|Memory max.||Varies.| |`sys/storage/used`|Disk space used.|fsDirName.|Varies.| diff --git a/docs/tutorials/cluster.md b/docs/tutorials/cluster.md index eac469d1e300..b61953c2f427 100644 --- a/docs/tutorials/cluster.md +++ b/docs/tutorials/cluster.md @@ -130,7 +130,7 @@ The [basic cluster tuning guide](../operations/basic-cluster-tuning.md) has info ## Select OS -We recommend running your favorite Linux distribution. You will also need Java 8 or 11. +We recommend running your favorite Linux distribution. You will also need [Java 8 or 11](../operations/java.md). > If needed, you can specify where to find Java using the environment variables > `DRUID_JAVA_HOME` or `JAVA_HOME`. For more details run the `bin/verify-java` script. diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index 5adcf5adfb6e..1fab7f8fe342 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -45,8 +45,9 @@ information on deploying Druid services across clustered machines. The software requirements for the installation machine are: -* Linux, Mac OS X, or other Unix-like OS (Windows is not supported) -* Java 8, Update 92 or later (8u92+) or Java 11 +* Linux, Mac OS X, or other Unix-like OS (Windows is not supported). +* Java 8, Update 92 or later (8u92+) or Java 11. See the [Java runtime](../operations/java.md) page for additional + information about selecting and configuring a Java runtime. > Druid relies on the environment variables `JAVA_HOME` or `DRUID_JAVA_HOME` to find Java on the machine. You can set `DRUID_JAVA_HOME` if there is more than one instance of Java. To verify Java requirements for your environment, run the diff --git a/examples/bin/run-druid b/examples/bin/run-druid index 93765f6f97ea..bc484e9d6420 100755 --- a/examples/bin/run-druid +++ b/examples/bin/run-druid @@ -36,11 +36,6 @@ fi CONFDIR="$(cd "$CONFDIR" && pwd)" WHEREAMI="$(cd "$WHEREAMI" && pwd)" -JAVA_BIN="$(source "$WHEREAMI"/java-util && get_java_bin_dir)" -if [ -z "$JAVA_BIN" ]; then - >&2 echo "Could not find java - please run $WHEREAMI/verify-java to confirm it is installed." - exit 1 -fi LOG_DIR="${DRUID_LOG_DIR:=${WHEREAMI}/../log}" # Remove possible ending slash @@ -53,6 +48,6 @@ if [ ! -d "$LOG_DIR" ]; then mkdir -p $LOG_DIR; fi echo "Running [$1], logging to [$LOG_DIR/$1.log] if no changes made to log4j2.xml" cd "$WHEREAMI/.." -exec "$JAVA_BIN"/java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ +exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" \ `cat "$CONFDIR"/$WHATAMI/main.config | xargs` diff --git a/examples/bin/run-java b/examples/bin/run-java new file mode 100755 index 000000000000..7438b1d84671 --- /dev/null +++ b/examples/bin/run-java @@ -0,0 +1,45 @@ +#!/bin/bash -eu + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +WHEREAMI="$(dirname "$0")" +JAVA_BIN="$(source "$WHEREAMI"/java-util && get_java_bin_dir)/java" +if [ -z "$JAVA_BIN" ]; then + >&2 echo "Could not find java - please run $WHEREAMI/verify-java to confirm it is installed." + exit 1 +fi + +JAVA_MAJOR="$("$JAVA_BIN" -version 2>&1 | sed -n -E 's/.* version "([^."]*).*/\1/p')" + +if [ "$JAVA_MAJOR" != "" ] && [ "$JAVA_MAJOR" -ge "17" ] +then + # Must disable strong encapsulation for certain packages on Java 17. + exec "$JAVA_BIN" \ + --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED \ + --add-exports=java.base/jdk.internal.perf=ALL-UNNAMED \ + --add-exports=java.base/jdk.internal.ref=ALL-UNNAMED \ + --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \ + --add-opens=java.base/java.nio=ALL-UNNAMED \ + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \ + --add-opens=java.base/java.io=ALL-UNNAMED \ + --add-opens=java.base/java.lang=ALL-UNNAMED \ + --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED \ + "$@" +else + exec "$JAVA_BIN" "$@" +fi diff --git a/examples/bin/run-zk b/examples/bin/run-zk index 8dc5b648584b..81a516315224 100755 --- a/examples/bin/run-zk +++ b/examples/bin/run-zk @@ -35,11 +35,6 @@ fi CONFDIR="$(cd "$CONFDIR" && pwd)/zk" WHEREAMI="$(cd "$WHEREAMI" && pwd)" -JAVA_BIN="$(source "$WHEREAMI"/java-util && get_java_bin_dir)" -if [ -z "$JAVA_BIN" ]; then - >&2 echo "Could not find java - please run $WHEREAMI/verify-java to confirm it is installed." - exit 1 -fi LOG_DIR="${DRUID_LOG_DIR:=${WHEREAMI}/../log}" # Remove possible ending slash @@ -52,7 +47,7 @@ if [ ! -d "$LOG_DIR" ]; then mkdir -p $LOG_DIR; fi echo "Running [ZooKeeper], logging to [$LOG_DIR/zookeeper.log] if no changes made to log4j2.xml" cd "$WHEREAMI/.." -exec "$JAVA_BIN"/java "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/jvm.config | xargs` \ +exec "$WHEREAMI"/run-java "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/jvm.config | xargs` \ -cp "$WHEREAMI/../lib/*:$CONFDIR" \ -Dzookeeper.jmx.log4j.disable=true \ org.apache.zookeeper.server.quorum.QuorumPeerMain \ diff --git a/examples/conf/druid/cluster/data/middleManager/runtime.properties b/examples/conf/druid/cluster/data/middleManager/runtime.properties index 564b98e19957..c44add1b8aca 100644 --- a/examples/conf/druid/cluster/data/middleManager/runtime.properties +++ b/examples/conf/druid/cluster/data/middleManager/runtime.properties @@ -24,7 +24,8 @@ druid.plaintextPort=8091 druid.worker.capacity=4 # Task launch parameters -druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.runner.javaCommand=bin/run-java +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task # HTTP server threads diff --git a/examples/conf/druid/single-server/large/middleManager/runtime.properties b/examples/conf/druid/single-server/large/middleManager/runtime.properties index 2c1cf0727904..bdfaa6b22224 100644 --- a/examples/conf/druid/single-server/large/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/large/middleManager/runtime.properties @@ -24,7 +24,8 @@ druid.plaintextPort=8091 druid.worker.capacity=8 # Task launch parameters -druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.runner.javaCommand=bin/run-java +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task # HTTP server threads diff --git a/examples/conf/druid/single-server/medium/middleManager/runtime.properties b/examples/conf/druid/single-server/medium/middleManager/runtime.properties index 564b98e19957..c44add1b8aca 100644 --- a/examples/conf/druid/single-server/medium/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/medium/middleManager/runtime.properties @@ -24,7 +24,8 @@ druid.plaintextPort=8091 druid.worker.capacity=4 # Task launch parameters -druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.runner.javaCommand=bin/run-java +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task # HTTP server threads diff --git a/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties b/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties index 1f8ae4696532..e7262930f498 100644 --- a/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties @@ -24,7 +24,8 @@ druid.plaintextPort=8091 druid.worker.capacity=2 # Task launch parameters -druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.runner.javaCommand=bin/run-java +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task # HTTP server threads diff --git a/examples/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties b/examples/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties index 85e8c4c40205..4eadb41bf661 100644 --- a/examples/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties @@ -24,7 +24,8 @@ druid.plaintextPort=8091 druid.worker.capacity=2 # Task launch parameters -druid.indexer.runner.javaOpts=-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.runner.javaCommand=bin/run-java +druid.indexer.runner.javaOptsArray=["-server","-Xms256m","-Xmx256m","-XX:MaxDirectMemorySize=300m","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task # HTTP server threads diff --git a/examples/conf/druid/single-server/small/middleManager/runtime.properties b/examples/conf/druid/single-server/small/middleManager/runtime.properties index dd902d349b1b..603ae60776ca 100644 --- a/examples/conf/druid/single-server/small/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/small/middleManager/runtime.properties @@ -24,7 +24,8 @@ druid.plaintextPort=8091 druid.worker.capacity=3 # Task launch parameters -druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.runner.javaCommand=bin/run-java +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task # HTTP server threads diff --git a/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties b/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties index 8a5924be709d..6dd805f5d60c 100644 --- a/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties @@ -24,7 +24,8 @@ druid.plaintextPort=8091 druid.worker.capacity=16 # Task launch parameters -druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +druid.indexer.runner.javaCommand=bin/run-java +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task # HTTP server threads diff --git a/extensions-core/orc-extensions/pom.xml b/extensions-core/orc-extensions/pom.xml index 2035974568be..a620d15abeaa 100644 --- a/extensions-core/orc-extensions/pom.xml +++ b/extensions-core/orc-extensions/pom.xml @@ -138,6 +138,10 @@ org.apache.hadoop hadoop-hdfs + + org.apache.hadoop + hadoop-client-api + org.slf4j slf4j-api diff --git a/licenses.yaml b/licenses.yaml index c64e372cc74c..a2d4d580b2d3 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -4052,7 +4052,7 @@ name: ASM license_category: binary module: java-core license_name: BSD-3-Clause License -version: 7.1 +version: 9.3 copyright: INRIA, France Telecom license_file_path: licenses/bin/asm.BSD3 libraries: diff --git a/pom.xml b/pom.xml index ad6297b7e00a..df4657bda857 100644 --- a/pom.xml +++ b/pom.xml @@ -815,12 +815,12 @@ org.ow2.asm asm - 7.1 + 9.3 org.ow2.asm asm-commons - 7.1 + 9.3 org.asynchttpclient @@ -1122,7 +1122,7 @@ nl.jqno.equalsverifier equalsverifier - 3.5.5 + 3.10.1 test @@ -1669,7 +1669,7 @@ - + --add-exports=java.base/jdk.internal.perf=ALL-UNNAMED --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED @@ -1679,6 +1679,16 @@ --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + + + --add-opens=java.base/java.io=ALL-UNNAMED + + + --add-opens=java.base/java.lang=ALL-UNNAMED + + + + --add-opens=java.base/java.util=ALL-UNNAMED diff --git a/processing/src/main/java/org/apache/druid/frame/file/FrameFile.java b/processing/src/main/java/org/apache/druid/frame/file/FrameFile.java index fce9612b3636..886d6ec1316c 100644 --- a/processing/src/main/java/org/apache/druid/frame/file/FrameFile.java +++ b/processing/src/main/java/org/apache/druid/frame/file/FrameFile.java @@ -19,6 +19,7 @@ package org.apache.druid.frame.file; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.io.Files; import org.apache.datasketches.memory.MapHandle; @@ -29,11 +30,13 @@ import org.apache.druid.java.util.common.IOE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.RE; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.segment.ReferenceCountingCloseableObject; import org.apache.druid.utils.CloseableUtils; +import org.apache.druid.utils.JvmUtils; import java.io.Closeable; import java.io.File; @@ -330,7 +333,15 @@ private static Pair mapFileBB(final File file) throws IOExcep */ private static Pair mapFileDS(final File file) { - final MapHandle mapHandle = Memory.map(file, 0, file.length(), ByteOrder.LITTLE_ENDIAN); + final MapHandle mapHandle; + + try { + mapHandle = Memory.map(file, 0, file.length(), ByteOrder.LITTLE_ENDIAN); + } + catch (NoClassDefFoundError | ExceptionInInitializerError e) { + throw handleMemoryMapError(e, JvmUtils.majorVersion()); + } + return Pair.of( mapHandle.get(), () -> { @@ -344,6 +355,22 @@ private static Pair mapFileDS(final File file) ); } + @VisibleForTesting + static RuntimeException handleMemoryMapError(Throwable e, int javaMajorVersion) + { + // Memory.map does not work on JDK 14+ due to issues with AllocateDirectMap. + if (javaMajorVersion >= 14) { + throw new ISE( + "Cannot read frame files larger than %,d bytes with Java %d. Try using Java 11.", + Integer.MAX_VALUE, + javaMajorVersion + ); + } else { + // We don't have a good reason why this happened. Throw the original error. + throw new RE(e, "Could not map frame file"); + } + } + private static long getFrameEndPosition(final Memory memory, final int frameNumber, final int numFrames) { final long frameEndPointerPosition = diff --git a/processing/src/test/java/org/apache/druid/frame/file/FrameFileTest.java b/processing/src/test/java/org/apache/druid/frame/file/FrameFileTest.java index 6eec5e5ce11c..034f732820a4 100644 --- a/processing/src/test/java/org/apache/druid/frame/file/FrameFileTest.java +++ b/processing/src/test/java/org/apache/druid/frame/file/FrameFileTest.java @@ -41,12 +41,15 @@ import org.apache.druid.segment.incremental.IncrementalIndexStorageAdapter; import org.apache.druid.testing.InitializedNullHandlingTest; import org.apache.druid.timeline.SegmentId; +import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; import org.junit.Assert; import org.junit.Assume; import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.internal.matchers.ThrowableMessageMatcher; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; @@ -372,6 +375,52 @@ public void test_newReference() throws IOException frameFile1.newReference(); } + @Test + public void test_handleMemoryMapError_java11() + { + @SuppressWarnings("ThrowableNotThrown") + final RuntimeException e = Assert.assertThrows( + RuntimeException.class, + () -> FrameFile.handleMemoryMapError(new IllegalAccessError("foo"), 11) + ); + + MatcherAssert.assertThat( + e, + ThrowableMessageMatcher.hasMessage(CoreMatchers.equalTo("Could not map frame file")) + ); + + // Include the original error, since we don't have a better explanation. + MatcherAssert.assertThat( + e.getCause(), + CoreMatchers.instanceOf(IllegalAccessError.class) + ); + } + + @Test + public void test_handleMemoryMapError_java17() + { + @SuppressWarnings("ThrowableNotThrown") + final IllegalStateException e = Assert.assertThrows( + IllegalStateException.class, + () -> FrameFile.handleMemoryMapError(new IllegalAccessError("foo"), 17) + ); + + MatcherAssert.assertThat( + e, + ThrowableMessageMatcher.hasMessage( + CoreMatchers.containsString( + StringUtils.format( + "Cannot read frame files larger than %,d bytes with Java 17.", + Integer.MAX_VALUE + ) + ) + ) + ); + + // Cause not included; we want to keep logs relatively cleaner and highlight the actual issue. + Assert.assertNull(e.getCause()); + } + private int computeExpectedNumFrames() { return IntMath.divide(countRows(adapter), maxRowsPerFrame, RoundingMode.CEILING); diff --git a/processing/src/test/java/org/apache/druid/frame/testutil/FrameTestUtil.java b/processing/src/test/java/org/apache/druid/frame/testutil/FrameTestUtil.java index 0884b60437da..14b4be36293e 100644 --- a/processing/src/test/java/org/apache/druid/frame/testutil/FrameTestUtil.java +++ b/processing/src/test/java/org/apache/druid/frame/testutil/FrameTestUtil.java @@ -47,6 +47,7 @@ import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorCursor; import org.apache.druid.timeline.SegmentId; +import org.apache.druid.utils.JvmUtils; import org.junit.Assert; import javax.annotation.Nullable; @@ -58,7 +59,6 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.StringTokenizer; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -295,8 +295,7 @@ public static Sequence> readRowsFromVectorCursor(final VectorCursor */ public static boolean jdkCanDataSketchesMemoryMap() { - final StringTokenizer st = new StringTokenizer(System.getProperty("java.specification.version"), "."); - return Integer.parseInt(st.nextToken()) < 14; + return JvmUtils.majorVersion() < 14; } private static Supplier dimensionSelectorReader(final DimensionSelector selector) diff --git a/server/src/main/java/org/apache/druid/server/coordination/SegmentLoadDropHandler.java b/server/src/main/java/org/apache/druid/server/coordination/SegmentLoadDropHandler.java index ba918d8fbd1e..2ba5dc933042 100644 --- a/server/src/main/java/org/apache/druid/server/coordination/SegmentLoadDropHandler.java +++ b/server/src/main/java/org/apache/druid/server/coordination/SegmentLoadDropHandler.java @@ -351,11 +351,11 @@ each time when addSegment() is called, it has to wait for the lock in order to m result = Status.SUCCESS; } - catch (Exception e) { + catch (Throwable e) { log.makeAlert(e, "Failed to load segment for dataSource") .addData("segment", segment) .emit(); - result = Status.failed(e.getMessage()); + result = Status.failed(e.toString()); } finally { updateRequestStatus(new SegmentChangeRequestLoad(segment), result); diff --git a/web-console/script/druid b/web-console/script/druid index 192d43281d69..9f54949b96cd 100755 --- a/web-console/script/druid +++ b/web-console/script/druid @@ -60,7 +60,7 @@ function _build_distribution() { && cd distribution/target \ && tar xzf "apache-druid-$(_get_druid_version)-bin.tar.gz" \ && cd apache-druid-$(_get_druid_version) \ - && java -classpath "lib/*" org.apache.druid.cli.Main tools pull-deps -c org.apache.druid.extensions:druid-testing-tools \ + && bin/run-java -classpath "lib/*" org.apache.druid.cli.Main tools pull-deps -c org.apache.druid.extensions:druid-testing-tools \ && echo -e "\n\ndruid.extensions.loadList=[\"druid-hdfs-storage\", \"druid-kafka-indexing-service\", \"druid-datasketches\", \"druid-testing-tools\"]" >> conf/druid/single-server/micro-quickstart/_common/common.runtime.properties \ && echo -e "\n\ndruid.server.http.allowedHttpMethods=[\"HEAD\"]" >> conf/druid/single-server/micro-quickstart/_common/common.runtime.properties \ ) diff --git a/website/.spelling b/website/.spelling index 3db7cb020ecf..088d27f4d0fa 100644 --- a/website/.spelling +++ b/website/.spelling @@ -1429,13 +1429,16 @@ time-iso8601 hadoopStorageDirectory - ../docs/operations/insert-segment-to-db.md 0.14.x + - ../docs/operations/java.md +G1 +Temurin - ../docs/operations/metrics.md 0.14.x 1s Bufferpool EventReceiverFirehose EventReceiverFirehoseMonitor -Filesystesm +Filesystem JVMMonitor QueryCountStatsMonitor RealtimeMetricsMonitor diff --git a/website/sidebars.json b/website/sidebars.json index 7f703ef15264..1a1b27208cb7 100644 --- a/website/sidebars.json +++ b/website/sidebars.json @@ -143,6 +143,7 @@ "Operations": [ "operations/druid-console", "operations/getting-started", + "operations/java", { "type": "subcategory", "label": "Security",