diff --git a/docs/operations/python.md b/docs/operations/python.md new file mode 100644 index 000000000000..9f5b9c34909e --- /dev/null +++ b/docs/operations/python.md @@ -0,0 +1,49 @@ +--- +id: python +title: "Python Installation" +--- + + + +Apache Druid startup script requires Python2 or Python3 interpreter. +Since Python2 is deprecated, this document has instructions to install Python3 interpreter. + +## Python3 interpreter installation instructions + +### Linux + +#### Debian or Ubuntu + - `sudo apt update` + - `sudo apt install -y python3-pip` +#### RHEL + - `sudo yum install -y epel-release` + - `sudo yum install -y python3-pip` + +### MacOS + +#### Install with Homebrew +Refer [Installing Python 3 on Mac OS X](https://docs.python-guide.org/starting/install3/osx/) + +#### Install the official Python release +* Browse to the [Python Downloads Page](https://www.python.org/downloads/) and download the latest version (3.x.x) + +Verify if Python3 is installed by issuing `python3 --version` command. + + diff --git a/docs/operations/single-server.md b/docs/operations/single-server.md index 35413952e222..48459a286042 100644 --- a/docs/operations/single-server.md +++ b/docs/operations/single-server.md @@ -23,14 +23,17 @@ title: "Single server deployment" --> -Druid includes a set of reference configurations and launch scripts for single-machine deployments: - -- `nano-quickstart` -- `micro-quickstart` -- `small` -- `medium` -- `large` -- `xlarge` +Druid includes a set of reference configurations and launch scripts for single-machine deployments. +These configuration bundles are located in `conf/druid/single-server/`. + +The `auto` configuration sizes runtime parameters based on available processors and memory. Other configurations include hard-coded runtime parameters for various server sizes. Most users should stick with `auto`. Refer below [Druid auto start](#druid-auto-start) +- `auto` (run script: `bin/start-druid`) +- `nano-quickstart` (run script: `bin/start-nano-quickstart`) +- `micro-quickstart` (run script: `bin/start-micro-quickstart`) +- `small` (run script: `bin/start-single-server-small`) +- `medium` (run script: `bin/start-single-server-medium`) +- `large` (run script: `bin/start-single-server-large`) +- `xlarge` (run script: `bin/start-single-server-xlarge`) The `micro-quickstart` is sized for small machines like laptops and is intended for quick evaluation use-cases. @@ -44,6 +47,18 @@ The example configurations run the Druid Coordinator and Overlord together in a While example configurations are provided for very large single machines, at higher scales we recommend running Druid in a [clustered deployment](../tutorials/cluster.md), for fault-tolerance and reduced resource contention. +## Druid auto start + +Druid includes a launch script, `bin/start-druid` that automatically sets various memory-related parameters based on available processors and memory. It accepts optional arguments such as list of services, total memory and a config directory to override default JVM arguments and service-specific runtime properties. + +`start-druid` is a generic launch script capable of starting any set of Druid services on a server. +It accepts optional arguments such as list of services, total memory and a config directory to override default JVM arguments and service-specific runtime properties. +Druid services will use all processors and up to 80% memory on the system. +For details about possible arguments, run `bin/start-druid --help`. + +The corresponding launch scripts (e.g. `start-micro-quickstart`) are now deprecated. + + ## Single server reference configurations ### Nano-Quickstart: 1 CPU, 4GiB RAM @@ -74,5 +89,4 @@ While example configurations are provided for very large single machines, at hig ### X-Large: 64 CPU, 512GiB RAM (~i3.16xlarge) - Launch command: `bin/start-xlarge` -- Configuration directory: `conf/druid/single-server/xlarge` - +- Configuration directory: `conf/druid/single-server/xlarge` \ No newline at end of file diff --git a/docs/tutorials/cluster.md b/docs/tutorials/cluster.md index b61953c2f427..5ee68e2cd13e 100644 --- a/docs/tutorials/cluster.md +++ b/docs/tutorials/cluster.md @@ -130,7 +130,10 @@ The [basic cluster tuning guide](../operations/basic-cluster-tuning.md) has info ## Select OS -We recommend running your favorite Linux distribution. You will also need [Java 8 or 11](../operations/java.md). +We recommend running your favorite Linux distribution. You will also need + +* [Java 8 or 11](../operations/java.md). +* [Python2 or Python3](../operations/python.md) > If needed, you can specify where to find Java using the environment variables > `DRUID_JAVA_HOME` or `JAVA_HOME`. For more details run the `bin/verify-java` script. diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index 7d3d2ab68ec1..d854a691e6b9 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -22,8 +22,7 @@ title: "Quickstart (local)" ~ under the License. --> - -This quickstart gets you started with Apache Druid using the [`micro-quickstart`](../operations/single-server.md#micro-quickstart-4-cpu-16gib-ram) configuration, and introduces you to Druid ingestion and query features. +This quickstart gets you started with Apache Druid and introduces you to Druid ingestion and query features. For this tutorial, we recommend a machine with at least 6 GB of RAM. In this quickstart, you'll do the following: - install Druid @@ -37,15 +36,16 @@ Druid supports a variety of ingestion options. Once you're done with this tutori You can follow these steps on a relatively modest machine, such as a workstation or virtual server with 16 GiB of RAM. -Druid comes equipped with several [startup configuration profiles](../operations/single-server.md) for a -range of machine sizes. These range from `nano` (1 CPU, 4GiB RAM) to `x-large` (64 CPU, 512GiB RAM). For more -information, see [Single server deployment](../operations/single-server.md). For information on deploying Druid services -across clustered machines, see [Clustered deployment](./cluster.md). +Druid comes equipped with launch scripts that can be used to start all processes on a single server. Here, we will use [`auto`](../operations/single-server.md#druid-auto-start), which automatically sets various runtime properties based on available processors and memory. + +In addition, Druid includes several [bundled non-automatic profiles](../operations/single-server.md) for a range of machine sizes. These range from nano (1 CPU, 4GiB RAM) to x-large (64 CPU, 512GiB RAM). +We won't use those here, but for more information, see [Single server deployment](../operations/single-server.md). For additional information on deploying Druid services across clustered machines, see [Clustered deployment](./cluster.md). The software requirements for the installation machine are: * Linux, Mac OS X, or other Unix-like OS. (Windows is not supported.) * Java 8u92+ or Java 11. +* [Python2 or Python3](../operations/python.md) > Druid relies on the environment variables `JAVA_HOME` or `DRUID_JAVA_HOME` to find Java on the machine. You can set `DRUID_JAVA_HOME` if there is more than one instance of Java. To verify Java requirements for your environment, run the @@ -72,30 +72,31 @@ The distribution directory contains `LICENSE` and `NOTICE` files and subdirector ## Start up Druid services -Start up Druid services using the `micro-quickstart` single-machine configuration. +Start up Druid services using the `auto` single-machine configuration. This configuration includes default settings that are appropriate for this tutorial, such as loading the `druid-multi-stage-query` extension by default so that you can use the MSQ task engine. -You can view that setting and others in the configuration files in the `conf/druid/single-server/micro-quickstart/`. +You can view that setting and others in the configuration files in the `conf/druid/auto`. From the apache-druid-{{DRUIDVERSION}} package root, run the following command: ```bash -./bin/start-micro-quickstart +./bin/start-druid ``` This brings up instances of ZooKeeper and the Druid services: ```bash -$ ./bin/start-micro-quickstart -[Thu Sep 8 18:30:00 2022] Starting Apache Druid. -[Thu Sep 8 18:30:00 2022] Open http://localhost:8888/ in your browser to access the web console. -[Thu Sep 8 18:30:00 2022] Or, if you have enabled TLS, use https on port 9088. -[Thu Sep 8 18:30:00 2022] Running command[zk], logging to[/apache-druid-{{DRUIDVERSION}}/var/sv/zk.log]: bin/run-zk conf -[Thu Sep 8 18:30:00 2022] Running command[coordinator-overlord], logging to[/apache-druid-{{DRUIDVERSION}}/var/sv/coordinator-overlord.log]: bin/run-druid coordinator-overlord conf/druid/single-server/micro-quickstart -[Thu Sep 8 18:30:00 2022] Running command[broker], logging to[/apache-druid-{{DRUIDVERSION}}/var/sv/broker.log]: bin/run-druid broker conf/druid/single-server/micro-quickstart -[Thu Sep 8 18:30:00 2022] Running command[router], logging to[/apache-druid-{{DRUIDVERSION}}/var/sv/router.log]: bin/run-druid router conf/druid/single-server/micro-quickstart -[Thu Sep 8 18:30:00 2022] Running command[historical], logging to[/apache-druid-{{DRUIDVERSION}}/var/sv/historical.log]: bin/run-druid historical conf/druid/single-server/micro-quickstart -[Thu Sep 8 18:30:00 2022] Running command[middleManager], logging to[/apache-druid-{{DRUIDVERSION}}/var/sv/middleManager.log]: bin/run-druid middleManager conf/druid/single-server/micro-quickstart +$ ./bin/start-druid +[Tue Nov 29 16:31:06 2022] Starting Apache Druid. +[Tue Nov 29 16:31:06 2022] Open http://localhost:8888/ in your browser to access the web console. +[Tue Nov 29 16:31:06 2022] Or, if you have enabled TLS, use https on port 9088. +[Tue Nov 29 16:31:06 2022] Starting services with log directory [/apache-druid-{{DRUIDVERSION}}/log]. +[Tue Nov 29 16:31:06 2022] Running command[zk]: bin/run-zk conf +[Tue Nov 29 16:31:06 2022] Running command[broker]: bin/run-druid broker /apache-druid-{{DRUIDVERSION}}/conf/druid/single-server/quickstart '-Xms1187m -Xmx1187m -XX:MaxDirectMemorySize=791m' +[Tue Nov 29 16:31:06 2022] Running command[router]: bin/run-druid router /apache-druid-{{DRUIDVERSION}}/conf/druid/single-server/quickstart '-Xms128m -Xmx128m' +[Tue Nov 29 16:31:06 2022] Running command[coordinator-overlord]: bin/run-druid coordinator-overlord /apache-druid-{{DRUIDVERSION}}/conf/druid/single-server/quickstart '-Xms1290m -Xmx1290m' +[Tue Nov 29 16:31:06 2022] Running command[historical]: bin/run-druid historical /apache-druid-{{DRUIDVERSION}}/conf/druid/single-server/quickstart '-Xms1376m -Xmx1376m -XX:MaxDirectMemorySize=2064m' +[Tue Nov 29 16:31:06 2022] Running command[middleManager]: bin/run-druid middleManager /apache-druid-{{DRUIDVERSION}}/conf/druid/single-server/quickstart '-Xms64m -Xmx64m' '-Ddruid.worker.capacity=2 -Ddruid.indexer.runner.javaOptsArray=["-server","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager","-Xms256m","-Xmx256m","-XX:MaxDirectMemorySize=256m"]' ``` All persistent state, such as the cluster metadata store and segments for the services, are kept in the `var` directory under @@ -103,7 +104,7 @@ the Druid root directory, apache-druid-{{DRUIDVERSION}}. Each service writes to At any time, you can revert Druid to its original, post-installation state by deleting the entire `var` directory. You may want to do this, for example, between Druid tutorials or after experimentation, to start with a fresh instance. -To stop Druid at any time, use CTRL+C in the terminal. This exits the `bin/start-micro-quickstart` script and terminates all Druid processes. +To stop Druid at any time, use CTRL+C in the terminal. This exits the `bin/start-druid` script and terminates all Druid processes. ## Open the web console @@ -222,4 +223,4 @@ See the following topics for more information: * [Tutorial: Load stream data from Apache Kafka](./tutorial-kafka.md) to load streaming data from a Kafka topic. * [Extensions](../development/extensions.md) for details on Druid extensions. -Remember that after stopping Druid services, you can start clean next time by deleting the `var` directory from the Druid root directory and running the `bin/start-micro-quickstart` script again. You may want to do this before using other data ingestion tutorials, since they use the same Wikipedia datasource. +Remember that after stopping Druid services, you can start clean next time by deleting the `var` directory from the Druid root directory and running the `bin/start-druid` script again. You may want to do this before using other data ingestion tutorials, since they use the same Wikipedia datasource. diff --git a/docs/tutorials/tutorial-batch-hadoop.md b/docs/tutorials/tutorial-batch-hadoop.md index 47cd2d6bcbe5..234e8426b064 100644 --- a/docs/tutorials/tutorial-batch-hadoop.md +++ b/docs/tutorials/tutorial-batch-hadoop.md @@ -28,7 +28,7 @@ This tutorial shows you how to load data files into Apache Druid using a remote For this tutorial, we'll assume that you've already completed the previous [batch ingestion tutorial](tutorial-batch.md) using Druid's native batch ingestion system and are using the -`micro-quickstart` single-machine configuration as described in the [quickstart](index.md). +`auto` single-machine configuration as described in the [quickstart](../operations/single-server.md#druid-auto-start). ## Install Docker @@ -156,7 +156,7 @@ cp /tmp/shared/hadoop_xml/*.xml {PATH_TO_DRUID}/conf/druid/single-server/micro-q ### Update Druid segment and log storage -In your favorite text editor, open `conf/druid/single-server/micro-quickstart/_common/common.runtime.properties`, and make the following edits: +In your favorite text editor, open `conf/druid/auto/_common/common.runtime.properties`, and make the following edits: #### Disable local deep storage and enable HDFS deep storage @@ -196,7 +196,7 @@ druid.indexer.logs.directory=/druid/indexing-logs Once the Hadoop .xml files have been copied to the Druid cluster and the segment/log storage configuration has been updated to use HDFS, the Druid cluster needs to be restarted for the new configurations to take effect. -If the cluster is still running, CTRL-C to terminate the `bin/start-micro-quickstart` script, and re-run it to bring the Druid services back up. +If the cluster is still running, CTRL-C to terminate the `bin/start-druid` script, and re-run it to bring the Druid services back up. ## Load batch data @@ -221,7 +221,7 @@ This tutorial is only meant to be used together with the [query tutorial](../tut If you wish to go through any of the other tutorials, you will need to: * Shut down the cluster and reset the cluster state by removing the contents of the `var` directory under the druid package. -* Revert the deep storage and task storage config back to local types in `conf/druid/single-server/micro-quickstart/_common/common.runtime.properties` +* Revert the deep storage and task storage config back to local types in `conf/druid/auto/_common/common.runtime.properties` * Restart the cluster This is necessary because the other ingestion tutorials will write to the same "wikipedia" datasource, and later tutorials expect the cluster to use local deep storage. diff --git a/docs/tutorials/tutorial-kafka.md b/docs/tutorials/tutorial-kafka.md index eb06f4239f95..0a47d3237fa6 100644 --- a/docs/tutorials/tutorial-kafka.md +++ b/docs/tutorials/tutorial-kafka.md @@ -30,7 +30,7 @@ The tutorial guides you through the steps to load sample nested clickstream data ## Prerequisites -Before you follow the steps in this tutorial, download Druid as described in the [quickstart](index.md) using the [micro-quickstart](../operations/single-server.md#micro-quickstart-4-cpu-16gib-ram) single-machine configuration and have it running on your local machine. You don't need to have loaded any data. +Before you follow the steps in this tutorial, download Druid as described in the [quickstart](index.md) using the [auto](../operations/single-server.md#druid-auto-start) single-machine configuration and have it running on your local machine. You don't need to have loaded any data. ## Download and start Kafka diff --git a/examples/bin/run-druid b/examples/bin/run-druid index c302672209a0..4be0afc5854b 100755 --- a/examples/bin/run-druid +++ b/examples/bin/run-druid @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -if [ "$#" -gt 2 ] || [ "$#" -eq 0 ] +if [ "$#" -gt 4 ] || [ "$#" -eq 0 ] then >&2 echo "usage: $0 [conf-dir]" exit 1 @@ -47,7 +47,45 @@ if [ ! -d "$LOG_DIR" ]; then mkdir -p $LOG_DIR; fi echo "Running [$1], logging to [$LOG_DIR/$1.log] if no changes made to log4j2.xml" +if [ "$WHATAMI" = 'coordinator-overlord' ] +then + SERVER_NAME=coordinator +else + SERVER_NAME="$WHATAMI" +fi + + +if [ ! -f "$CONFDIR"/$WHATAMI/main.config ]; + then + MAIN_CLASS="org.apache.druid.cli.Main server $SERVER_NAME" + else + MAIN_CLASS=`cat "$CONFDIR"/$WHATAMI/main.config | xargs` +fi + cd "$WHEREAMI/.." -exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ - -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" \ - `cat "$CONFDIR"/$WHATAMI/main.config | xargs` + +CLASS_PATH="$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" + +if [ "$#" -eq 3 ] || [ "$#" -eq 4 ] +then + # args: or + JVMARGS=`cat "$CONFDIR/_common/common.jvm.config" | xargs` + JVMARGS+=' ' + JVMARGS+=$3 + + if [ "$#" -eq 3 ] + then + # args: + exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" $JVMARGS \ + -cp $CLASS_PATH $MAIN_CLASS + else + # args: + exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 $4 "-Ddruid.log.path=$LOG_DIR" $JVMARGS \ + -cp $CLASS_PATH $MAIN_CLASS + fi +else + # args: + exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" \ + `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ + -cp $CLASS_PATH $MAIN_CLASS +fi diff --git a/examples/bin/start-druid b/examples/bin/start-druid new file mode 100755 index 000000000000..81d8938adbb2 --- /dev/null +++ b/examples/bin/start-druid @@ -0,0 +1,35 @@ +#!/bin/bash -eu + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +PWD="$(pwd)" +WHEREAMI="$(dirname "$0")" +WHEREAMI="$(cd "$WHEREAMI" && pwd)" + +if [ -x "$(command -v python3)" ] +then + exec python3 "$WHEREAMI/start-druid-main.py" "$@" +elif [ -x "$(command -v python2)" ] +then + exec python2 "$WHEREAMI/start-druid-main.py" "$@" +elif [ -x "$(command -v python)" ] +then + exec python "$WHEREAMI/start-druid-main.py" "$@" +else + echo "python interepreter not found" +fi diff --git a/examples/bin/start-druid-main.py b/examples/bin/start-druid-main.py new file mode 100644 index 000000000000..d1f4e6114918 --- /dev/null +++ b/examples/bin/start-druid-main.py @@ -0,0 +1,641 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import sys +import os +import multiprocessing +import argparse +import subprocess +import platform + +BASE_CONFIG_PATH = "conf/druid/auto" + +MEM_GB_SUFFIX = "g" +MEM_MB_SUFFIX = "m" +XMX_PARAMETER = "-Xmx" +XMS_PARAMETER = "-Xms" +DIRECT_MEM_PARAMETER = "-XX:MaxDirectMemorySize" +SERVICE_SEPARATOR = "," + +TASK_JAVA_OPTS_ARRAY = ["-server", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-XX:+ExitOnOutOfMemoryError", + "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] +TASK_JAVA_OPTS_PROPERTY = "druid.indexer.runner.javaOptsArray" +TASK_WORKER_CAPACITY_PROPERTY = "druid.worker.capacity" +TASK_COUNT = "task-count" +TASK_MEM_TYPE_LOW = "low" +TASK_MEM_TYPE_HIGH = "high" +TASK_MEM_MAP = { + TASK_MEM_TYPE_LOW: ["-Xms256m", "-Xmx256m", "-XX:MaxDirectMemorySize=256m"], + TASK_MEM_TYPE_HIGH: ["-Xms1g", "-Xmx1g", "-XX:MaxDirectMemorySize=1g"] +} + +BROKER = "broker" +ROUTER = "router" +COORDINATOR = "coordinator-overlord" +HISTORICAL = "historical" +MIDDLE_MANAGER = "middleManager" +TASKS = "tasks" +INDEXER = "indexer" + +DEFAULT_SERVICES = [ + BROKER, + ROUTER, + COORDINATOR, + HISTORICAL, + MIDDLE_MANAGER +] + +SUPPORTED_SERVICES = [ + BROKER, + ROUTER, + COORDINATOR, + HISTORICAL, + MIDDLE_MANAGER, + INDEXER +] + +SERVICE_MEMORY_RATIO = { + MIDDLE_MANAGER: 1, + ROUTER: 2, + COORDINATOR: 30, + BROKER: 46, + HISTORICAL: 80, + TASKS: 30, + INDEXER: 32 +} + +MINIMUM_MEMORY_MB = { + MIDDLE_MANAGER: 64, + ROUTER: 128, + TASKS: 1024, + BROKER: 900, + COORDINATOR: 256, + HISTORICAL: 900, + INDEXER: 1124 +} + +HEAP_TO_TOTAL_MEM_RATIO = { + MIDDLE_MANAGER: 1, + ROUTER: 1, + COORDINATOR: 1, + BROKER: 0.60, + HISTORICAL: 0.40, + TASKS: 0.50, + INDEXER: 0.50 +} + +LOGGING_ENABLED = False + + +def print_if_verbose(message): + if LOGGING_ENABLED: + print(message) + + +def configure_parser(): + parser = argparse.ArgumentParser( + prog='start-druid', + formatter_class=argparse.RawTextHelpFormatter, + epilog= + """ +sample usage: + start-druid + Start up all the services (including zk). + start-druid -m=100g + Start up all the services (including zk) + using a total memory of 100GB. + start-druid -m=100g --compute + Compute memory distribution and validate arguments. + start-druid -m=100g -s=broker,router + Starts a broker and a router, using a total memory of 100GB. + start-druid -m=100g --s=broker,router \\ + -c=conf/druid/single-server/custom + Starts a broker and a router, using a total memory of 100GB. + Reads configs for each service (jvm.config, runtime.properties) + from respective folders inside the given root config path. + start-druid -s=broker,router \\ + -c=conf/druid/single-server/custom + Starts a broker and a router service, reading service configs + from the given root directory. Calculates memory requirements for + each service, if required, using upto 80% of the total system memory. + start-druid -m=100g \\ + -s=broker,router \\ + -c=conf/druid/single-server/custom \\ + --zk + Starts broker, router and zookeeper. + zookeeper config is read from conf/zk. +""" + ) + parser.add_argument('--memory', '-m', type=str, required=False, + help='Total memory for all processes (services and tasks, if any). \n' + 'This parameter is ignored if each service already has a jvm.config \n' + 'in the given conf directory. e.g. 500m, 4g, 6g\n') + parser.add_argument('--services', '-s', type=str, required=False, + help='List of services to be started, subset of \n' + '{broker, router, middleManager, historical, coordinator-overlord, indexer}. \n' + 'If the argument is not given, broker, router, middleManager, historical, coordinator-overlord \n' + 'and zookeeper is started. e.g. -s=broker,historical') + parser.add_argument('--config', '-c', type=str, required=False, + help='Relative path to the directory containing common and service \n' + 'specific properties to be overridden. \n' + 'This directory must contain \'_common\' directory with \n' + '\'common.jvm.config\' & \'common.runtime.properties\' files. \n' + 'If this argument is not given, config from \n' + 'conf/druid/auto directory is used.\n') + parser.add_argument('--compute', action='store_true', + help='Does not start Druid, only displays the memory allocated \n' + 'to each service if started with the given total memory.\n') + parser.add_argument('--zk', '-zk', action='store_true', + help='Specification to run zookeeper, \n' + 'zk config is picked up from conf/zk.') + parser.add_argument('--verbose', action='store_true', help='Log details') + + parser.set_defaults(zk=False) + parser.set_defaults(compute=False) + parser.set_defaults(verbose=False) + + return parser + + +def is_file(path): + return os.path.isfile(path) + + +def is_dir(path): + return os.path.isdir(path) + + +def resolve_path(path): + return os.path.abspath(path) + + +def validate_common_jvm_args(config): + if is_file('{0}/_common/common.jvm.config'.format(config)) is False: + raise ValueError('_common/common.jvm.config file is missing in the root config, ' + 'check {0}/_common directory'.format(BASE_CONFIG_PATH)) + + +def validate_common_directory(config): + if is_dir('{0}/_common'.format(config)) is False: + raise ValueError( + '_common directory is missing in the root config, check {0}/_common directory'.format(BASE_CONFIG_PATH)) + + if is_file('{0}/_common/common.runtime.properties'.format(config)) is False: + raise ValueError('_common/common.runtime.properties file is missing in the root config, ' + 'check {0}/_common directory'.format(BASE_CONFIG_PATH)) + + +def parse_arguments(args): + service_list = [] + config = "" + total_memory = "" + compute = False + zk = False + + if args.compute: + compute = True + if args.zk: + zk = True + if args.config is not None: + config = resolve_path(os.path.join(os.getcwd(), args.config)) + if is_dir(config) is False: + raise ValueError('config {0} not found'.format(config)) + if args.memory is not None: + total_memory = args.memory + if args.services is not None: + services = args.services.split(SERVICE_SEPARATOR) + + for service in services: + if service not in SUPPORTED_SERVICES: + raise ValueError('Invalid service name {0}, should be one of {1}'.format(service, DEFAULT_SERVICES)) + + if service in service_list: + raise ValueError('{0} is specified multiple times'.format(service)) + + service_list.append(service) + + if INDEXER in services and MIDDLE_MANAGER in services: + raise ValueError('one of indexer and middleManager can run') + + if len(service_list) == 0: + # start all services + service_list = DEFAULT_SERVICES + zk = True + + return config, total_memory, service_list, zk, compute + + +def print_startup_config(service_list, config, zk): + print_if_verbose('Starting {0}'.format(service_list)) + print_if_verbose('Reading config from {0}'.format(config)) + if zk: + zk_config = resolve_path('{0}/../conf/zk'.format(os.getcwd())) + print_if_verbose('Starting zk, reading default config from {0}'.format(zk_config)) + print_if_verbose('\n') + + +def task_memory_params_present(config, service): + java_opts_property_present = False + worker_capacity_property_present = False + + if is_file('{0}/{1}/runtime.properties'.format(config, service)): + with open('{0}/{1}/runtime.properties'.format(config, service)) as file: + for line in file: + if line.startswith(TASK_JAVA_OPTS_PROPERTY): + java_opts_property_present = True + elif line.startswith(TASK_WORKER_CAPACITY_PROPERTY): + worker_capacity_property_present = True + + return java_opts_property_present, worker_capacity_property_present + + +def verify_service_config(service, config): + path = '{0}/{1}/jvm.config'.format(config, service) + + required_parameters = [XMX_PARAMETER, XMS_PARAMETER] + + if HEAP_TO_TOTAL_MEM_RATIO.get(service) != 1: + required_parameters.append(DIRECT_MEM_PARAMETER) + + with open(path) as file: + for line in file: + if line.startswith(XMX_PARAMETER) and XMX_PARAMETER in required_parameters: + required_parameters.remove(XMX_PARAMETER) + if line.startswith(XMS_PARAMETER) and XMS_PARAMETER in required_parameters: + required_parameters.remove(XMS_PARAMETER) + if line.startswith(DIRECT_MEM_PARAMETER) and DIRECT_MEM_PARAMETER in required_parameters: + required_parameters.remove(DIRECT_MEM_PARAMETER) + + if len(required_parameters) > 0: + params = ",".join(required_parameters) + raise ValueError('{0} missing in {1}/jvm.config'.format(params, service)) + + if service == MIDDLE_MANAGER: + if is_file('{0}/{1}/runtime.properties'.format(config, service)) is False: + raise ValueError('{0}/runtime.properties file is missing in the root config'.format(service)) + + mm_task_java_opts_prop, mm_task_worker_capacity_prop = task_memory_params_present(config, MIDDLE_MANAGER) + + if mm_task_java_opts_property is False: + raise ValueError('{0} property missing in {1}/runtime.properties'.format(TASK_JAVA_OPTS_PROPERTY, service)) + + +def should_compute_memory(config, total_memory, service_list): + """ + if memory argument is given, memory for services and tasks is computed, jvm.config file + or runtime.properties with task memory specification shouldn't be present + Alternatively, all memory related parameters are specified + which implies following should be present: + jvm.config file for all services with -Xmx=***, Xms=*** parameters + -XX:MaxDirectMemorySize=** in jvm.config for broker and historical + druid.indexer.runner.javaOptsArray (optionally druid.worker.capacity) in + rootDirectory/middleManager/runtime.properties + """ + + jvm_config_count = 0 + for service in service_list: + if is_file('{0}/{1}/jvm.config'.format(config, service)): + jvm_config_count += 1 + + mm_task_property_present = False + if MIDDLE_MANAGER in service_list: + mm_task_java_opts_prop, mm_task_worker_capacity_prop = task_memory_params_present(config, MIDDLE_MANAGER) + mm_task_property_present = mm_task_java_opts_prop or mm_task_worker_capacity_prop + + indexer_task_worker_capacity_prop = False + if INDEXER in service_list: + indexer_task_java_opts_prop, indexer_task_worker_capacity_prop = task_memory_params_present(config, INDEXER) + + # possible error states + # 1. memory argument is specified, also jvm.config or middleManger/runtime.properties having + # druid.indexer.runner.javaOptsArray or druid.worker.capacity parameters is present + # 2. jvm.config is not present for any service, but middleManger/runtime.properties has + # druid.indexer.runner.javaOptsArray or druid.worker.capacity parameters + # or indexer/runtime.properties has druid.worker.capacity + # 3. jvm.config present for some but not all services + # 4. jvm.config file is present for all services, but it doesn't contain required parameters + # 5. lastly, if middleManager is to be started, and it is missing task memory properties + if jvm_config_count > 0 or mm_task_property_present or indexer_task_worker_capacity_prop: + if total_memory != "": + raise ValueError( + "If jvm.config for services and/or middleManager/indexer configs " + "(druid.worker.capacity, druid.indexer.runner.javaOptsArray) is present, " + "memory argument shouldn't be specified") + if jvm_config_count == 0 and mm_task_property_present: + raise ValueError("middleManger configs (druid.indexer.runner.javaOptsArray or druid.worker.capacity) " + "is present in middleManager/runtime.properties, " + "add jvm.config for all other services") + if jvm_config_count == 0 and indexer_task_worker_capacity_prop: + raise ValueError("indexer configs (druid.worker.capacity) " + "is present in indexer/runtime.properties, " + "add jvm.config for all other services") + if jvm_config_count != len(service_list): + raise ValueError("jvm.config file should be present for all services or none") + for service in service_list: + verify_service_config(service, config) + + return False + + # compute memory only when none of the specified services contains jvm.config, + # if middleManager is to be started it shouldn't contain task memory properties + # if indexer is present it shouldn't contain task memory properties + return True + + +def get_physical_memory_linux(): + mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') + mem_mbs = int(mem_bytes / (1024 * 1024)) + return mem_mbs + + +def get_physical_memory_osx(): + p1 = subprocess.Popen(['sysctl', '-a'], stdout=subprocess.PIPE) + p2 = subprocess.check_output(['grep', 'hw.memsize'], stdin=p1.stdout) + p2 = p2.decode('utf-8') + fields = p2.split(':') + + mem_mbs = int(int(fields[1]) / (1024 * 1024)) + + return mem_mbs + + +def get_physical_memory(): + operating_system = platform.system() + print_if_verbose('operating system is {0}'.format(operating_system)) + + system_memory = None + + try: + if operating_system == 'Darwin': + system_memory = get_physical_memory_osx() + elif operating_system == 'Linux': + system_memory = get_physical_memory_linux() + except Exception: + pass + + return system_memory + + +def convert_total_memory_string(memory): + try: + if memory == '': + physical_memory = get_physical_memory() + + if physical_memory is None: + raise ValueError('Please specify memory argument') + + return physical_memory + elif memory.endswith(MEM_MB_SUFFIX): + return int(memory[:-1]) + elif memory.endswith(MEM_GB_SUFFIX): + return 1024 * int(memory[:-1]) + else: + raise ValueError('Incorrect format for memory argument, expected format is ') + except ValueError as e: + raise e + except Exception: + raise ValueError('Incorrect format for memory argument, expected format is ') + + +def check_memory_constraint(total_memory, services): + # 80% of total memory >= sum of lower bound service memory should be + lower_bound_memory = 0 + + service_list = list(services) + if MIDDLE_MANAGER in services: + service_list.append(TASKS) + + for service in service_list: + lower_bound_memory += MINIMUM_MEMORY_MB.get(service) + + required_memory = int(lower_bound_memory / 0.8) + + if total_memory < required_memory: + raise ValueError('Minimum memory required for starting services is {0}m'.format(required_memory)) + + if total_memory >= 2 * lower_bound_memory: + return int(total_memory / 2) + else: + return lower_bound_memory + + +def build_mm_task_java_opts_array(memory_type): + task_memory = '-D{0}=['.format(TASK_JAVA_OPTS_PROPERTY) + + mem_array = TASK_MEM_MAP.get(memory_type) + + java_opts_list = TASK_JAVA_OPTS_ARRAY + mem_array + + for item in java_opts_list: + task_memory += '\"{0}\",'.format(item) + + task_memory = task_memory[:-1] + task_memory += ']' + return task_memory + + +def compute_tasks_memory(allocated_memory): + if allocated_memory >= 4096: + task_count = int(allocated_memory / 2048) + memory_type = TASK_MEM_TYPE_HIGH + task_memory_mb = 2048 + else: + task_count = 2 + memory_type = TASK_MEM_TYPE_LOW + task_memory_mb = 512 + task_count = min(task_count, multiprocessing.cpu_count()) + + return memory_type, task_count, task_memory_mb + + +def build_memory_config(service, allocated_memory): + if service == TASKS: + memory_type, task_count, task_memory = compute_tasks_memory(allocated_memory) + java_opts_array = build_mm_task_java_opts_array(memory_type) + return ['-D{0}={1}'.format(TASK_WORKER_CAPACITY_PROPERTY, task_count), + java_opts_array], task_memory * task_count + elif service == INDEXER: + heap_memory = HEAP_TO_TOTAL_MEM_RATIO.get(service) * allocated_memory + direct_memory = int(allocated_memory - heap_memory) + heap_memory = int(heap_memory) + memory_type, task_count, task_memory = compute_tasks_memory(allocated_memory) + return ['-D{0}={1}'.format(TASK_WORKER_CAPACITY_PROPERTY, task_count), + '-Xms{0}m -Xmx{0}m -XX:MaxDirectMemorySize={1}m'.format(heap_memory, direct_memory)], \ + task_memory * task_count + else: + heap_memory = HEAP_TO_TOTAL_MEM_RATIO.get(service) * allocated_memory + direct_memory = int(allocated_memory - heap_memory) + heap_memory = int(heap_memory) + + if direct_memory == 0: + return '-Xms{0}m -Xmx{0}m'.format(heap_memory), allocated_memory + + return '-Xms{0}m -Xmx{0}m -XX:MaxDirectMemorySize={1}m'.format(heap_memory, direct_memory), allocated_memory + + +def distribute_memory(services, total_memory): + service_memory_config = {} + + memory_weight_sum = 0 + + service_list = list(services) + if MIDDLE_MANAGER in services: + service_list.append(TASKS) + + for service in service_list: + memory_weight_sum += SERVICE_MEMORY_RATIO.get(service) + + multiplier = total_memory / memory_weight_sum + + lower_bound_memory_allocation = 0 + allocated_services = set() + + for service in service_list: + allocated_memory = SERVICE_MEMORY_RATIO.get(service) * multiplier + if service in MINIMUM_MEMORY_MB and allocated_memory < MINIMUM_MEMORY_MB.get(service): + allocated_memory = MINIMUM_MEMORY_MB.get(service) + service_memory_config[service], allocated_memory = build_memory_config(service, allocated_memory) + lower_bound_memory_allocation += allocated_memory + allocated_services.add(service) + + if lower_bound_memory_allocation > 0: + # compute the multiplier again for remaining services + memory_weight_sum = 0 + for service in service_list: + if service in allocated_services: + continue + memory_weight_sum += SERVICE_MEMORY_RATIO.get(service) + multiplier = (total_memory - lower_bound_memory_allocation) / memory_weight_sum + + for service in service_list: + if service in allocated_services: + continue + allocated_memory = SERVICE_MEMORY_RATIO.get(service) * multiplier + if service in MINIMUM_MEMORY_MB and allocated_memory < MINIMUM_MEMORY_MB.get(service): + allocated_memory = MINIMUM_MEMORY_MB.get(service) + + service_memory_config[service], allocated_memory = build_memory_config(service, allocated_memory) + + print_if_verbose('\nMemory distribution for services:') + for key, value in service_memory_config.items(): + print_if_verbose('{0}, {1}'.format(key, value)) + print_if_verbose('\n') + + return service_memory_config + + +def append_command(commands, command): + commands.append('--command') + commands.append(command) + + +def build_supervise_script_arguments(service_list, service_memory_config, config, zk): + commands = [] + commands.append('supervise') + + append_command(commands, ":verify bin/verify-java") + append_command(commands, ":verify bin/verify-default-ports") + append_command(commands, ":notify bin/greet") + append_command(commands, ":kill-timeout 10") + + if zk: + append_command(commands, "!p10 zk bin/run-zk conf") + + for service in service_list: + memory_config = service_memory_config.get(service) + + prefix = '' + if service == MIDDLE_MANAGER: + prefix = '!p90 ' + + if memory_config is None: + append_command(commands, '{0}{1} bin/run-druid {1} {2}'.format(prefix, service, config)) + else: + if service == MIDDLE_MANAGER: + task_config = service_memory_config.get(TASKS) + task_count = task_config[0] + task_memory = task_config[1] + append_command( + commands, + '{0}{1} bin/run-druid {1} {2} \'{3}\' \'{4} {5}\'' + .format(prefix, service, config, memory_config, task_count, task_memory)) + elif service == INDEXER: + task_count = memory_config[0] + jvm_args = memory_config[1] + append_command( + commands, + '{0}{1} bin/run-druid {1} {2} \'{3}\' \'{4}\'' + .format(prefix, service, config, jvm_args, task_count)) + else: + append_command(commands, + '{0}{1} bin/run-druid {1} {2} \'{3}\''.format(prefix, service, config, memory_config)) + + print_if_verbose('Supervise script args:') + for item in commands: + print_if_verbose(item) + + print_if_verbose('\n') + + return commands + + +def main(): + parser = configure_parser() + args = parser.parse_args() + + global LOGGING_ENABLED + LOGGING_ENABLED = args.verbose or args.compute + + config, total_memory, service_list, zk, compute = parse_arguments(args) + + # change directory to bin + os.chdir(os.path.dirname(sys.argv[0])) + + if config == "": + config = resolve_path('{0}/../{1}'.format(os.getcwd(), BASE_CONFIG_PATH)) + + validate_common_directory(config) + + print_startup_config(service_list, config, zk) + + service_memory_config = {} + + if should_compute_memory(config, total_memory, service_list): + # if memory is to be computed, _common directory should contain common.jvm.config + validate_common_jvm_args(config) + memory_in_mega_bytes = convert_total_memory_string(total_memory) + print_if_verbose('Total memory is {0}m\n'.format(memory_in_mega_bytes)) + memory_to_be_used = check_memory_constraint(memory_in_mega_bytes, service_list) + print_if_verbose('Memory used for services & tasks {0}m\n'.format(memory_to_be_used)) + service_memory_config = distribute_memory(service_list, memory_to_be_used) + else: + print_if_verbose('Not computing memory distribution, reading memory specification from service jvm.config & ' + 'middleManager/runtime.properties\n') + + script_arguments = build_supervise_script_arguments(service_list, service_memory_config, config, zk) + + if compute: + return + + os.execv('./supervise', script_arguments) + + +try: + main() +except (KeyboardInterrupt, ValueError) as error: + print(error) + sys.exit(1) diff --git a/examples/bin/supervise b/examples/bin/supervise index 81b7e57b049e..ba336c31aa66 100755 --- a/examples/bin/supervise +++ b/examples/bin/supervise @@ -46,6 +46,51 @@ sub usage die "usage: $0 -c [-d ] [-t ] [--svlogd ]\n"; } +sub process_config +{ + my @lines = @_; + my @commands; + my @verify; + my @notify; + my $kill_timeout; + for my $line (@lines) + { + if ($line =~ /^(:verify|:notify|:kill-timeout|(?:\!p[0-9]+\s+)?[^:]\S+)\s+(.+)$/) { + my $name = $1; + my $order = 50; + my $command = $2; + + if ($name =~ /^(?:\!p([0-9]+)\s+)(.*)$/) { + $order = $1; + $name = $2; + } + + if ($name eq ':verify') { + push @verify, $command; + } elsif ($name eq ':notify') { + push @notify, $command; + } elsif ($name eq ':kill-timeout') { + $kill_timeout = int($command); + } else { + die "Duplicate command: $line\n" if grep { $_->{name} eq $name } @commands; + push @commands, { + name => $name, + command => $command, + order => $order, # Stop order for this command + pid => 0, # Current pid, or 0 if not running + down => 0, # Time the proc should be down until + killed => 0, # Signal we sent to this process + restarting => 0, # True if this command is currently restarting + }; + } + } else { + die "Syntax error: $line\n"; + } + } + + return { commands => \@commands, verify => \@verify, notify => \@notify, 'kill-timeout' => $kill_timeout }; +} + sub read_config_file { my ($config_file) = @_; @@ -53,49 +98,20 @@ sub read_config_file open my $config_fh, "<", $config_file or die "open $config_file: $!"; - my @commands; - my @verify; - my @notify; - my $kill_timeout; + my @lines; while (my $line = <$config_fh>) { chomp $line; next if $line =~ /^(\s*\#.*|\s*)$/; if ($line =~ /^(:verify|:notify|:kill-timeout|(?:\!p[0-9]+\s+)?[^:]\S+)\s+(.+)$/) { - my $name = $1; - my $order = 50; - my $command = $2; - - if ($name =~ /^(?:\!p([0-9]+)\s+)(.*)$/) { - $order = $1; - $name = $2; - } - - if ($name eq ':verify') { - push @verify, $command; - } elsif ($name eq ':notify') { - push @notify, $command; - } elsif ($name eq ':kill-timeout') { - $kill_timeout = int($command); - } else { - die "Duplicate command: $line\n" if grep { $_->{name} eq $name } @commands; - push @commands, { - name => $name, - command => $command, - order => $order, # Stop order for this command - pid => 0, # Current pid, or 0 if not running - down => 0, # Time the proc should be down until - killed => 0, # Signal we sent to this process - restarting => 0, # True if this command is currently restarting - }; - } + push @lines, $line } else { die "Syntax error: $line\n"; } } close $config_fh; - return { commands => \@commands, verify => \@verify, notify => \@notify, 'kill-timeout' => $kill_timeout }; + return @lines; } sub stringify_exit_status @@ -179,13 +195,23 @@ usage() unless GetOptions( 'vardir|d=s', 'kill-timeout|t=i', 'chdir=s', - 'svlogd:s' + 'svlogd:s', + 'command=s@' ); -usage() unless $opt{'conf'} && $opt{'vardir'}; +usage() unless (($opt{'command'} && @{$opt{'command'}}) || $opt{'conf'}) && $opt{'vardir'}; + +my @config_lines; + +# get commands to execute either from reading the config file or command line +if (not defined $opt{'conf'}) { + @config_lines = @{$opt{'command'}} +} else { + @config_lines = read_config_file($opt{'conf'}); +} + +my $config = process_config(@config_lines); -# Read config file -my $config = read_config_file($opt{'conf'}); @commands = @{$config->{commands}}; if (!@commands) { diff --git a/examples/conf/druid/auto/_common/common.jvm.config b/examples/conf/druid/auto/_common/common.jvm.config new file mode 100644 index 000000000000..fd74cf358979 --- /dev/null +++ b/examples/conf/druid/auto/_common/common.jvm.config @@ -0,0 +1,7 @@ +-server +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/auto/_common/common.runtime.properties b/examples/conf/druid/auto/_common/common.runtime.properties new file mode 100644 index 000000000000..b0adb0695cd7 --- /dev/null +++ b/examples/conf/druid/auto/_common/common.runtime.properties @@ -0,0 +1,158 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Extensions specified in the load list will be loaded by Druid +# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead +# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead + +# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. +# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. +# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query"] + +# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory +# and uncomment the line below to point to your directory. +#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies + + +# +# Hostname +# +druid.host=localhost + +# +# Logging +# + +# Log all runtime properties on startup. Disable to avoid logging properties on startup: +druid.startup.logging.logProperties=true + +# +# Zookeeper +# + +druid.zk.service.host=localhost +druid.zk.paths.base=/druid + +# +# Metadata storage +# + +# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): +druid.metadata.storage.type=derby +druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true +druid.metadata.storage.connector.host=localhost +druid.metadata.storage.connector.port=1527 + +# For MySQL (make sure to include the MySQL JDBC driver on the classpath): +#druid.metadata.storage.type=mysql +#druid.metadata.storage.connector.connectURI=jdbc:mysql://db.example.com:3306/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# For PostgreSQL: +#druid.metadata.storage.type=postgresql +#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# +# Deep storage +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.storage.type=local +druid.storage.storageDirectory=var/druid/segments + +# For HDFS: +#druid.storage.type=hdfs +#druid.storage.storageDirectory=/druid/segments + +# For S3: +#druid.storage.type=s3 +#druid.storage.bucket=your-bucket +#druid.storage.baseKey=druid/segments +#druid.s3.accessKey=... +#druid.s3.secretKey=... + +# +# Indexing service logs +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.indexer.logs.type=file +druid.indexer.logs.directory=var/druid/indexing-logs + +# For HDFS: +#druid.indexer.logs.type=hdfs +#druid.indexer.logs.directory=/druid/indexing-logs + +# For S3: +#druid.indexer.logs.type=s3 +#druid.indexer.logs.s3Bucket=your-bucket +#druid.indexer.logs.s3Prefix=druid/indexing-logs + +# +# Service discovery +# + +druid.selectors.indexing.serviceName=druid/overlord +druid.selectors.coordinator.serviceName=druid/coordinator + +# +# Monitoring +# + +druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] +druid.emitter=noop +druid.emitter.logging.logLevel=info + +# Storage type of double columns +# ommiting this will lead to index double as float at the storage layer + +druid.indexing.doubleStorage=double + +# +# Security +# +druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password", "password", "key", "token", "pwd"] + + +# +# SQL +# +druid.sql.enable=true + +# Planning SQL query when there is aggregate distinct in the statement +druid.sql.planner.useGroupingSetForExactDistinct=true + +# +# Lookups +# +druid.lookup.enableLookupSyncOnStartup=false + +# +# Expression processing config +# +druid.expressions.useStrictBooleans=true + +# +# Http client +# +druid.global.http.eagerInitialization=false diff --git a/examples/conf/druid/auto/_common/log4j2.xml b/examples/conf/druid/auto/_common/log4j2.xml new file mode 100644 index 000000000000..66dc13da4c5e --- /dev/null +++ b/examples/conf/druid/auto/_common/log4j2.xml @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/conf/druid/auto/broker/runtime.properties b/examples/conf/druid/auto/broker/runtime.properties new file mode 100644 index 000000000000..f4c494019933 --- /dev/null +++ b/examples/conf/druid/auto/broker/runtime.properties @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/broker +druid.plaintextPort=8082 + +# HTTP server settings +# HTTP server thread pool size. Higher values increase peak load on the Broker, but +# may be useful for high-concurrency workloads. +# Default is max(10, (Number of processors * 17) / 16 + 2) + 30. +# druid.server.http.numThreads=N + +# HTTP client settings +# Connection pool size from the Broker to each data server. May be useful to +# raise this for high-concurrency workloads. +# druid.broker.http.numConnections=20 + +# Processing threads and buffers +# Determined automatically based on available memory. For details on how to manually set parameters: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers +druid.processing.tmpDir=var/druid/processing + +# Query cache disabled -- push down caching and merging instead +druid.broker.cache.useCache=false +druid.broker.cache.populateCache=false diff --git a/examples/conf/druid/auto/coordinator-overlord/runtime.properties b/examples/conf/druid/auto/coordinator-overlord/runtime.properties new file mode 100644 index 000000000000..c053823f8ed8 --- /dev/null +++ b/examples/conf/druid/auto/coordinator-overlord/runtime.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/coordinator +druid.plaintextPort=8081 + +druid.coordinator.startDelay=PT10S +druid.coordinator.period=PT5S +druid.manager.segments.pollDuration=PT5S + +# Run the overlord service in the coordinator process +druid.coordinator.asOverlord.enabled=true +druid.coordinator.asOverlord.overlordService=druid/overlord + +druid.indexer.queue.startDelay=PT5S + +druid.indexer.storage.type=metadata diff --git a/examples/conf/druid/auto/historical/runtime.properties b/examples/conf/druid/auto/historical/runtime.properties new file mode 100644 index 000000000000..6c241aa7cf06 --- /dev/null +++ b/examples/conf/druid/auto/historical/runtime.properties @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# HTTP server thread pool size. Higher values increase peak load on the Broker, but +# may be useful for high-concurrency workloads. +# Default is max(10, (Number of processors * 17) / 16 + 2) + 30. +# druid.server.http.numThreads=N + +# Processing threads and buffers +# Determined automatically based on available memory. For details on how to manually set parameters: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers +druid.processing.tmpDir=var/druid/processing + +# Segment storage +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g"}] +# Query cache +druid.historical.cache.useCache=true +druid.historical.cache.populateCache=true +druid.cache.type=caffeine diff --git a/examples/conf/druid/auto/indexer/runtime.properties b/examples/conf/druid/auto/indexer/runtime.properties new file mode 100644 index 000000000000..5aef64535b67 --- /dev/null +++ b/examples/conf/druid/auto/indexer/runtime.properties @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/indexer +druid.plaintextPort=8091 + +# Number of tasks (druid.worker.capacity) is automatically +# determined based on available processor. + +# Task launch parameters +druid.indexer.task.baseTaskDir=var/druid/task + +# Processing threads and buffers on Indexer +# Determined automatically based on available memory. For details on how to manually set parameters: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/examples/conf/druid/auto/middleManager/runtime.properties b/examples/conf/druid/auto/middleManager/runtime.properties new file mode 100644 index 000000000000..08c58bae6de0 --- /dev/null +++ b/examples/conf/druid/auto/middleManager/runtime.properties @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/middleManager +druid.plaintextPort=8091 + +# Number of tasks (druid.worker.capacity) and memory usage per task (druid.indexer.runner.javaOptsArray) is automatically +# determined based on available memory. For details on how to manually set parameters, see: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#middlemanager + +# Task launch parameters +druid.indexer.runner.javaCommand=bin/run-java +druid.indexer.task.baseTaskDir=var/druid/task + +# Processing threads and buffers on Peons +# Determined automatically based on available memory. For details on how to manually set parameters: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/examples/conf/druid/auto/router/runtime.properties b/examples/conf/druid/auto/router/runtime.properties new file mode 100644 index 000000000000..3858dec044bd --- /dev/null +++ b/examples/conf/druid/auto/router/runtime.properties @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/router +druid.plaintextPort=8888 + +# Service discovery +druid.router.defaultBrokerServiceName=druid/broker +druid.router.coordinatorServiceName=druid/coordinator + +# Management proxy to coordinator / overlord: required for unified web console. +druid.router.managementProxy.enabled=true diff --git a/server/src/main/java/org/apache/druid/client/cache/CaffeineCache.java b/server/src/main/java/org/apache/druid/client/cache/CaffeineCache.java index b7a1ea826c58..362a387231a6 100644 --- a/server/src/main/java/org/apache/druid/client/cache/CaffeineCache.java +++ b/server/src/main/java/org/apache/druid/client/cache/CaffeineCache.java @@ -70,7 +70,7 @@ public static CaffeineCache create(final CaffeineCacheConfig config, final Execu if (config.getSizeInBytes() >= 0) { builder.maximumWeight(config.getSizeInBytes()); } else { - builder.maximumWeight(Math.min(MAX_DEFAULT_BYTES, JvmUtils.getRuntimeInfo().getMaxHeapSizeBytes() / 10)); + builder.maximumWeight(Math.min(MAX_DEFAULT_BYTES, JvmUtils.getRuntimeInfo().getMaxHeapSizeBytes() / 20)); } builder .weigher((NamedKey key, byte[] value) -> value.length diff --git a/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java b/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java index a232f6d01aac..842abd0e5347 100644 --- a/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java +++ b/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java @@ -101,7 +101,7 @@ public DataNodeService getDataNodeService( serverTypeConfig.getServerType() ); if (ServerType.HISTORICAL.equals(serverTypeConfig.getServerType())) { - throw new ProvisionException("Segment cache locations must be set on historicals."); + throw new ProvisionException("druid.segmentCache.locations must be set on historicals."); } } diff --git a/server/src/test/java/org/apache/druid/guice/StorageNodeModuleTest.java b/server/src/test/java/org/apache/druid/guice/StorageNodeModuleTest.java index e849bb6910e8..082361a1c885 100644 --- a/server/src/test/java/org/apache/druid/guice/StorageNodeModuleTest.java +++ b/server/src/test/java/org/apache/druid/guice/StorageNodeModuleTest.java @@ -112,7 +112,7 @@ public void getDataNodeServiceWithNoServerTypeConfigShouldThrowProvisionExceptio public void getDataNodeServiceWithNoSegmentCacheConfiguredThrowProvisionException() { exceptionRule.expect(ProvisionException.class); - exceptionRule.expectMessage("Segment cache locations must be set on historicals."); + exceptionRule.expectMessage("druid.segmentCache.locations must be set on historicals."); Mockito.doReturn(ServerType.HISTORICAL).when(serverTypeConfig).getServerType(); mockSegmentCacheNotConfigured(); injector.getInstance(DataNodeService.class); diff --git a/web-console/src/variables.scss b/web-console/src/variables.scss index 1dc4d0124cf3..fec32add0008 100644 --- a/web-console/src/variables.scss +++ b/web-console/src/variables.scss @@ -25,7 +25,7 @@ $view-control-bar-height: 30px; $standard-padding: 15px; $thin-padding: 10px; -// various style variables +// various style Variables $druid-brand: #2ceefb; $druid-brand2: #00b6bf; diff --git a/website/.spelling b/website/.spelling index d02f4f846799..a3212dc0ff38 100644 --- a/website/.spelling +++ b/website/.spelling @@ -424,6 +424,10 @@ programmatically proto proxied proxyConfig +python2 +python3 +Python2 +Python3 QPS quantile quantiles @@ -2297,3 +2301,8 @@ Czechia Zeelund - ../docs/tutorials/docker.md nano + - ../docs/operations/python.md +MacOS +RHEL +psutil +pathlib