From 8c9dcc251221deb99eda1a63fc0689cbb2887d9a Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 15 Nov 2022 11:08:35 +0530 Subject: [PATCH 01/82] Druid automated quickstart --- examples/bin/druid-automated-quickstart.py | 299 ++++++++++++++++++ examples/bin/run-druid | 33 +- examples/bin/start-druid | 24 ++ .../quickstart/_common/common.jvm.config | 7 + .../_common/common.runtime.properties | 158 +++++++++ .../quickstart/_common/log4j2.xml | 86 +++++ .../quickstart/broker/runtime.properties | 21 ++ .../coordinator-overlord/runtime.properties | 21 ++ .../quickstart/historical/jvm.config | 0 .../quickstart/historical/runtime.properties | 24 ++ .../middleManager/runtime.properties | 21 ++ .../quickstart/router/runtime.properties | 21 ++ 12 files changed, 710 insertions(+), 5 deletions(-) create mode 100644 examples/bin/druid-automated-quickstart.py create mode 100644 examples/bin/start-druid create mode 100644 examples/conf/druid/single-server/quickstart/_common/common.jvm.config create mode 100644 examples/conf/druid/single-server/quickstart/_common/common.runtime.properties create mode 100644 examples/conf/druid/single-server/quickstart/_common/log4j2.xml create mode 100644 examples/conf/druid/single-server/quickstart/broker/runtime.properties create mode 100644 examples/conf/druid/single-server/quickstart/coordinator-overlord/runtime.properties create mode 100644 examples/conf/druid/single-server/quickstart/historical/jvm.config create mode 100644 examples/conf/druid/single-server/quickstart/historical/runtime.properties create mode 100644 examples/conf/druid/single-server/quickstart/middleManager/runtime.properties create mode 100644 examples/conf/druid/single-server/quickstart/router/runtime.properties diff --git a/examples/bin/druid-automated-quickstart.py b/examples/bin/druid-automated-quickstart.py new file mode 100644 index 000000000000..24bee48fbb96 --- /dev/null +++ b/examples/bin/druid-automated-quickstart.py @@ -0,0 +1,299 @@ +import sys +import os +import psutil +from pathlib import Path + +SUPERVISE_CONFIG_FILE_PATH = "../conf/supervise/single-server/quickstart.conf" +QUICKSTART_BASE_CONFIG_PATH = "conf/druid/single-server/quickstart" +HELP_ARG_IDENTIFIER = "help" +COMPUTE_ONLY_ARG_IDENTIFIER = "computeOnly" +RUN_ZK_IDENTIFIER = "runZk" +BASE_CONFIG_PATH_IDENTIFIER = "baseConfigPath" +MEMORY_ARG_IDENTIFIER = "totalMemory" +MEMORY_GIGABYTES_IDENTIFIER = "g" +MEMORY_MEGABYTES_IDENTIFIER = "m" +ARG_SEPARATOR = "=" + +BROKER_SERVICE_NAME = "broker" +ROUTER_SERVICE_NAME = "router" +COORDINATOR_SERVICE_NAME = "coordinator-overlord" +HISTORICAL_SERVICE_NAME = "historical" +MIDDLE_MANAGER_SERVICE_NAME = "middleManager" + +DEFAULT_SERVICES = [ + BROKER_SERVICE_NAME, + ROUTER_SERVICE_NAME, + COORDINATOR_SERVICE_NAME, + HISTORICAL_SERVICE_NAME, + MIDDLE_MANAGER_SERVICE_NAME +] + +SERVICE_MEMORY_DISTRIBUTION_WEIGHT = { + MIDDLE_MANAGER_SERVICE_NAME: 0.25, + ROUTER_SERVICE_NAME: 1, + COORDINATOR_SERVICE_NAME: 18, + BROKER_SERVICE_NAME: 28, + HISTORICAL_SERVICE_NAME: 50 +} + +SERVICE_MEMORY_HEAP_RATIO = { + MIDDLE_MANAGER_SERVICE_NAME: 1, + ROUTER_SERVICE_NAME: 1, + COORDINATOR_SERVICE_NAME: 1, + BROKER_SERVICE_NAME: 0.60, + HISTORICAL_SERVICE_NAME: 0.40 +} + +def check_argument_type(argument, type): + split_args = argument.split(ARG_SEPARATOR) + return split_args[0] == type + +def get_argument_value(argument): + split_args = argument.split(ARG_SEPARATOR) + return split_args[1] + +def parse_arguments(): + num_args = len(sys.argv) + service_list = [] + service_path_list = [] + base_config_path = "" + total_memory = "" + compute_only = False + run_zk = False + + for argument in sys.argv[1:]: + if check_argument_type(argument, COMPUTE_ONLY_ARG_IDENTIFIER): + compute_only = True + elif check_argument_type(argument, RUN_ZK_IDENTIFIER): + run_zk = True + elif check_argument_type(argument, BASE_CONFIG_PATH_IDENTIFIER): + base_config_path = os.path.join(os.getcwd(), get_argument_value(argument)) + elif (check_argument_type(argument, MEMORY_ARG_IDENTIFIER)): + total_memory = get_argument_value(argument) + else: + split_args = argument.split(ARG_SEPARATOR) + service = split_args[0] + + if service not in DEFAULT_SERVICES: + raise Exception(f'{service} is not a valid service name, should be one of {DEFAULT_SERVICES}') + + path = "" + + if len(split_args) == 2: + path = split_args[1] + complete_path = os.path.join(base_config_path, path) + if os.path.exists(os.path.join(complete_path)) is False: + raise Exception(f'Path `{complete_path}` specified for service `{service}` doesn\'t exist ') + service_list.append(service) + service_path_list.append(path) + + if len(service_list) == 0: + # start all services + service_list = DEFAULT_SERVICES + service_path_list = [""] * len(DEFAULT_SERVICES) + run_zk = True + + return base_config_path, total_memory, list(zip(service_list, service_path_list)), run_zk, compute_only + +def should_compute_memory(base_config_path, total_memory, service_config): + # if jvm file is present for any of the services + # it should be present for all services and totalMemory should not be specified + # if totalMemory is given, jvm file shouldn't be present for any service + + jvm_config_count = 0 + for item in service_config: + if item[1] != "": + if Path(f'{base_config_path}/{item[1]}/jvm.config').is_file(): + jvm_config_count += 1 + elif jvm_config_count > 0: + raise Exception('jvm.config file is missing for service {item[0]}, jvm.config should be specified for all the services or none') + + if jvm_config_count > 0 and (jvm_config_count != len(service_config) or total_memory != ""): + if jvm_config_count != len(service_config): + raise Exception("jvm.config file should be present for all services or none") + if total_memory != "": + raise Exception("If jvm.config is given for services, `totalMemory` argument shouldn't be specified") + + return jvm_config_count == 0 + +def compute_system_memory(): + system_memory = psutil.virtual_memory().total # mem in bytes + memory_for_druid = int((system_memory * 0.8) / (1024 * 1024)) + return memory_for_druid + +def convert_total_memory_string(memory): + if memory == "": + computed_memory = compute_system_memory() + print(f'`{MEMORY_ARG_IDENTIFIER}` argument is not specified Druid will use 80% of system memory: {computed_memory}m') + return computed_memory + elif memory.endswith(MEMORY_MEGABYTES_IDENTIFIER): + return int(memory[:-1]) + elif memory.endswith(MEMORY_GIGABYTES_IDENTIFIER): + return 1024 * int(memory[:-1]) + else: + raise Exception('Incorrect format for totalMemory argument, expected format is m or g') + +def build_memory_config_string(heap_memory, direct_memory): + if direct_memory == 0: + return f'-Xms{heap_memory}m -Xmx{heap_memory}m' + return f'-Xms{heap_memory}m -Xmx{heap_memory}m -XX:MaxDirectMemorySize={direct_memory}m' + +def distribute_memory_over_services(service_config, total_memory): + service_memory_config = {} + service_instance_map = {} + + for item in service_config: + service_instance_map[item[0]] = service_instance_map.get(item[0], 0) + 1 + + memory_weight_sum = 0 + for key, value in service_instance_map.items(): + memory_weight_sum += SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(key) * value + + multiplier = total_memory / memory_weight_sum + + for key, value in service_instance_map.items(): + allocated_memory = SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(key) * multiplier + heap_memory = SERVICE_MEMORY_HEAP_RATIO.get(key) * allocated_memory + direct_memory = allocated_memory - heap_memory + service_memory_config[key] = build_memory_config_string(int(heap_memory), int(direct_memory)) + + print(f'\nMemory distribution for services') + for key, value in service_memory_config.items(): + print(f'{key}, memory_config: {value}, instance_count: {service_instance_map[key]}') + print('\n') + + return service_memory_config + +def create_supervise_config_file(service_config, service_memory_config, base_config_path, run_zk): + + with open(SUPERVISE_CONFIG_FILE_PATH, 'w+') as the_file: + the_file.write(":verify bin/verify-java\n") + the_file.write(":verify bin/verify-default-ports\n") + the_file.write(":notify bin/greet\n") + the_file.write(":kill-timeout 10\n") + the_file.write("\n") + + if run_zk: + the_file.write("!p10 zk bin/run-zk conf\n") + + for item in service_config: + service = item[0] + prefix = '' + if service == MIDDLE_MANAGER_SERVICE_NAME: + prefix = '!p90 ' + if item[1] == "": + service_path = item[0] + else: + service_path = item[1] + jvm_args = service_memory_config.get(item[0]) + + if jvm_args == "": + the_file.write(f'{prefix}{service} bin/run-druid {service} {base_config_path} {service_path}\n') + else: + the_file.write(f'{prefix}{service} bin/run-druid {service} {base_config_path} {service_path} \'{jvm_args}\'\n') + +def print_service_config(service_config, base_config_path): + print('Services to start:') + for item in service_config: + if item[1] == "": + print(f'{item[0]}, using default config from {os.getcwd()}/../{QUICKSTART_BASE_CONFIG_PATH}') + else: + print(f'{item[0]}, using config from {base_config_path}/{item[1]}') + print('\n') + +def display_help(): + text = """ + Usage: start-druid [options] + + where options include: + totalMemory= + memory for druid cluster, if totalMemory is not specified + 80 percent of system memory is used. Note, if service + specific jvm config is present, + totalMemory shouldn't be specified + baseConfigPath= + relative path to base directory containing common and service specific + properties to be overridden, this directory must contain `_common` + directory with `common.jvm.config` & `common.runtime.properties` + if `baseConfigPath` is not specified, config from + conf/druid/single-server/quickstart direcotry is used + computeOnly + command dry-run, validates the arguments and + display the memory distribution for services + runZk + specification to run zookeeper, zk config is picked up from conf/zk + =[subdirectory] + service_identifier is the service to be started, multiple services + can be specified, `service_identifier` should be one of + [broker, router, middleManager, historical, coordinator-overlord] + `subdirectory` is optional directory within `baseConfigPath` + containing runtime properties or/and jvm properties + Note, if jvm.config file is present for one service, it must be + present for all other services + If no service is specified, all services and zookeeper are started + + sample usage: + start-druid + start up all the services using the default system memory + start-druid totalMemory=100g + start up all the services using the given memory + start-druid totalMemory=100g computeOnly + compute memory distribution for all the services + start-druid totalMemory=100g broker router historical + starts `broker`, `router` and `historical` services, using `100g` of memory + start-druid totalMemory=100g baseConfigPath=/conf/druid/single-server/large broker router historical + starts `broker`, `router` and `historical` service, using 100g of memory, + use common configs from specified `baseConfigPath` + start-druid totalMemory=100g baseConfigPath=/conf/druid/single-server/large broker=broker router=router historical=historical + starts `broker`, `router` and `historical` services, using 100g of memory, use common configs + from specified `baseConfigPath`, use service specific config from specified directories + if jvm.config is specified for all the services, memory distribution is not computed + start-druid totalMemory=100g baseConfigPath=/conf/druid/single-server/large broker=broker1 broker=broker2 + starts 2 instances of `broker` + config is read from respective directories, depending on whether jvm.config is specified, + memory distribution is computed + start-druid totalMemory=100g baseConfigPath=conf/druid/profile broker=broker1 historical=historical1 + if either of `broker1`, `historical1` subdirectory contains jvm.config, + exception is thrown since `totalMemory` argument is specified + start-druid baseConfigPath=conf/druid/profile broker=broker1 historical=historical1 + exception is thrown if either of `broker1`, `historical1` + subdirectory contains jvm.config but not both + If none of the subdirectory contains jvm.config, memory distribution is computed + """ + + print(text) + + +def main(): + for argument in sys.argv[1:]: + if check_argument_type(argument, HELP_ARG_IDENTIFIER): + display_help() + return + + print("Druid auto tuning quickstart\n") + + base_config_path, total_memory, service_config, run_zk, compute_only = parse_arguments() + + # change directory to bin + os.chdir(os.path.dirname(sys.argv[0])) + + print(f'Arguments passed, baseConfigPath: "{base_config_path}", totalMemory: "{total_memory}"\n') + print_service_config(service_config, base_config_path) + + service_memory_config = {} + if (should_compute_memory(base_config_path, total_memory, service_config)): + memory_in_mega_bytes = convert_total_memory_string(total_memory) + service_memory_config = distribute_memory_over_services(service_config, memory_in_mega_bytes) + + if compute_only: + return + + if base_config_path == "": + base_config_path = QUICKSTART_BASE_CONFIG_PATH + + create_supervise_config_file(service_config, service_memory_config, base_config_path, run_zk) + + os.system(f'exec ./supervise -c {SUPERVISE_CONFIG_FILE_PATH}') + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/examples/bin/run-druid b/examples/bin/run-druid index c302672209a0..c98c2e53bc3c 100755 --- a/examples/bin/run-druid +++ b/examples/bin/run-druid @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -if [ "$#" -gt 2 ] || [ "$#" -eq 0 ] +if [ "$#" -gt 4 ] || [ "$#" -eq 0 ] then >&2 echo "usage: $0 [conf-dir]" exit 1 @@ -47,7 +47,30 @@ if [ ! -d "$LOG_DIR" ]; then mkdir -p $LOG_DIR; fi echo "Running [$1], logging to [$LOG_DIR/$1.log] if no changes made to log4j2.xml" -cd "$WHEREAMI/.." -exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ - -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" \ - `cat "$CONFDIR"/$WHATAMI/main.config | xargs` +if [ "$WHATAMI" = 'coordinator-overlord' ] +then + SERVER_NAME=coordinator +else + SERVER_NAME="$WHATAMI" +fi + +if [ "$#" -eq 3 ] +then + cd "$WHEREAMI/.." + exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$3"/jvm.config | xargs` \ + -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" org.apache.druid.cli.Main server $SERVER_NAME +elif [ "$#" -eq 4 ] +then + JVMARGS=`cat "$CONFDIR/_common/common.jvm.config" | xargs` + JVMARGS+=' ' + JVMARGS+=$4 + + cd "$WHEREAMI/.." + exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" $JVMARGS \ + -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" org.apache.druid.cli.Main server $SERVER_NAME +else + cd "$WHEREAMI/.." + exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ + -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" \ + `cat "$CONFDIR"/$WHATAMI/main.config | xargs` +fi \ No newline at end of file diff --git a/examples/bin/start-druid b/examples/bin/start-druid new file mode 100644 index 000000000000..13781fca8e9b --- /dev/null +++ b/examples/bin/start-druid @@ -0,0 +1,24 @@ +#!/bin/bash -eu + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +PWD="$(pwd)" +WHEREAMI="$(dirname "$0")" +WHEREAMI="$(cd "$WHEREAMI" && pwd)" + +python3 "$WHEREAMI"/druid-automated-quickstart.py $@ \ No newline at end of file diff --git a/examples/conf/druid/single-server/quickstart/_common/common.jvm.config b/examples/conf/druid/single-server/quickstart/_common/common.jvm.config new file mode 100644 index 000000000000..4db1d94ec38e --- /dev/null +++ b/examples/conf/druid/single-server/quickstart/_common/common.jvm.config @@ -0,0 +1,7 @@ +-server +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \ No newline at end of file diff --git a/examples/conf/druid/single-server/quickstart/_common/common.runtime.properties b/examples/conf/druid/single-server/quickstart/_common/common.runtime.properties new file mode 100644 index 000000000000..b0adb0695cd7 --- /dev/null +++ b/examples/conf/druid/single-server/quickstart/_common/common.runtime.properties @@ -0,0 +1,158 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Extensions specified in the load list will be loaded by Druid +# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead +# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead + +# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. +# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. +# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query"] + +# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory +# and uncomment the line below to point to your directory. +#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies + + +# +# Hostname +# +druid.host=localhost + +# +# Logging +# + +# Log all runtime properties on startup. Disable to avoid logging properties on startup: +druid.startup.logging.logProperties=true + +# +# Zookeeper +# + +druid.zk.service.host=localhost +druid.zk.paths.base=/druid + +# +# Metadata storage +# + +# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): +druid.metadata.storage.type=derby +druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true +druid.metadata.storage.connector.host=localhost +druid.metadata.storage.connector.port=1527 + +# For MySQL (make sure to include the MySQL JDBC driver on the classpath): +#druid.metadata.storage.type=mysql +#druid.metadata.storage.connector.connectURI=jdbc:mysql://db.example.com:3306/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# For PostgreSQL: +#druid.metadata.storage.type=postgresql +#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# +# Deep storage +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.storage.type=local +druid.storage.storageDirectory=var/druid/segments + +# For HDFS: +#druid.storage.type=hdfs +#druid.storage.storageDirectory=/druid/segments + +# For S3: +#druid.storage.type=s3 +#druid.storage.bucket=your-bucket +#druid.storage.baseKey=druid/segments +#druid.s3.accessKey=... +#druid.s3.secretKey=... + +# +# Indexing service logs +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.indexer.logs.type=file +druid.indexer.logs.directory=var/druid/indexing-logs + +# For HDFS: +#druid.indexer.logs.type=hdfs +#druid.indexer.logs.directory=/druid/indexing-logs + +# For S3: +#druid.indexer.logs.type=s3 +#druid.indexer.logs.s3Bucket=your-bucket +#druid.indexer.logs.s3Prefix=druid/indexing-logs + +# +# Service discovery +# + +druid.selectors.indexing.serviceName=druid/overlord +druid.selectors.coordinator.serviceName=druid/coordinator + +# +# Monitoring +# + +druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] +druid.emitter=noop +druid.emitter.logging.logLevel=info + +# Storage type of double columns +# ommiting this will lead to index double as float at the storage layer + +druid.indexing.doubleStorage=double + +# +# Security +# +druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password", "password", "key", "token", "pwd"] + + +# +# SQL +# +druid.sql.enable=true + +# Planning SQL query when there is aggregate distinct in the statement +druid.sql.planner.useGroupingSetForExactDistinct=true + +# +# Lookups +# +druid.lookup.enableLookupSyncOnStartup=false + +# +# Expression processing config +# +druid.expressions.useStrictBooleans=true + +# +# Http client +# +druid.global.http.eagerInitialization=false diff --git a/examples/conf/druid/single-server/quickstart/_common/log4j2.xml b/examples/conf/druid/single-server/quickstart/_common/log4j2.xml new file mode 100644 index 000000000000..66dc13da4c5e --- /dev/null +++ b/examples/conf/druid/single-server/quickstart/_common/log4j2.xml @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/conf/druid/single-server/quickstart/broker/runtime.properties b/examples/conf/druid/single-server/quickstart/broker/runtime.properties new file mode 100644 index 000000000000..4d67149d29e9 --- /dev/null +++ b/examples/conf/druid/single-server/quickstart/broker/runtime.properties @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/broker +druid.plaintextPort=8082 \ No newline at end of file diff --git a/examples/conf/druid/single-server/quickstart/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/quickstart/coordinator-overlord/runtime.properties new file mode 100644 index 000000000000..855eeeafa7dd --- /dev/null +++ b/examples/conf/druid/single-server/quickstart/coordinator-overlord/runtime.properties @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/coordinator +druid.plaintextPort=8081 \ No newline at end of file diff --git a/examples/conf/druid/single-server/quickstart/historical/jvm.config b/examples/conf/druid/single-server/quickstart/historical/jvm.config new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/examples/conf/druid/single-server/quickstart/historical/runtime.properties b/examples/conf/druid/single-server/quickstart/historical/runtime.properties new file mode 100644 index 000000000000..4ce14669dbaf --- /dev/null +++ b/examples/conf/druid/single-server/quickstart/historical/runtime.properties @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/historical +druid.plaintextPort=8083 + +# Segment storage +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g"}] \ No newline at end of file diff --git a/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties b/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties new file mode 100644 index 000000000000..fdec5cfca425 --- /dev/null +++ b/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/middleManager +druid.plaintextPort=8091 diff --git a/examples/conf/druid/single-server/quickstart/router/runtime.properties b/examples/conf/druid/single-server/quickstart/router/runtime.properties new file mode 100644 index 000000000000..3754ec172210 --- /dev/null +++ b/examples/conf/druid/single-server/quickstart/router/runtime.properties @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/router +druid.plaintextPort=8888 \ No newline at end of file From 342e5d72b1ce5d012622f501649b36320ecc5598 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 15 Nov 2022 11:35:00 +0530 Subject: [PATCH 02/82] remove conf/druid/single-server/quickstart/_common/historical/jvm.config --- .../conf/druid/single-server/quickstart/historical/jvm.config | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 examples/conf/druid/single-server/quickstart/historical/jvm.config diff --git a/examples/conf/druid/single-server/quickstart/historical/jvm.config b/examples/conf/druid/single-server/quickstart/historical/jvm.config deleted file mode 100644 index e69de29bb2d1..000000000000 From 6a4a48edd6ceed77ee26c40c55625a76f200b131 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 15 Nov 2022 11:53:25 +0530 Subject: [PATCH 03/82] Minor changes in python script --- examples/bin/druid-automated-quickstart.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/examples/bin/druid-automated-quickstart.py b/examples/bin/druid-automated-quickstart.py index 24bee48fbb96..de6827f61f67 100644 --- a/examples/bin/druid-automated-quickstart.py +++ b/examples/bin/druid-automated-quickstart.py @@ -53,7 +53,6 @@ def get_argument_value(argument): return split_args[1] def parse_arguments(): - num_args = len(sys.argv) service_list = [] service_path_list = [] base_config_path = "" @@ -157,7 +156,7 @@ def distribute_memory_over_services(service_config, total_memory): direct_memory = allocated_memory - heap_memory service_memory_config[key] = build_memory_config_string(int(heap_memory), int(direct_memory)) - print(f'\nMemory distribution for services') + print(f'\nMemory distribution for services:') for key, value in service_memory_config.items(): print(f'{key}, memory_config: {value}, instance_count: {service_instance_map[key]}') print('\n') @@ -199,7 +198,6 @@ def print_service_config(service_config, base_config_path): print(f'{item[0]}, using default config from {os.getcwd()}/../{QUICKSTART_BASE_CONFIG_PATH}') else: print(f'{item[0]}, using config from {base_config_path}/{item[1]}') - print('\n') def display_help(): text = """ @@ -270,14 +268,14 @@ def main(): display_help() return - print("Druid auto tuning quickstart\n") + print("Druid automated quickstart\n") base_config_path, total_memory, service_config, run_zk, compute_only = parse_arguments() # change directory to bin os.chdir(os.path.dirname(sys.argv[0])) - print(f'Arguments passed, baseConfigPath: "{base_config_path}", totalMemory: "{total_memory}"\n') + print(f'Arguments passed: baseConfigPath: "{base_config_path}", totalMemory: "{total_memory}"\n') print_service_config(service_config, base_config_path) service_memory_config = {} From 36b86b6336b366fa0ac0135177e6e556c2621974 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 15 Nov 2022 14:38:12 +0530 Subject: [PATCH 04/82] Add lower bound memory for some services --- examples/bin/druid-automated-quickstart.py | 48 +++++++++++++++++++--- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/examples/bin/druid-automated-quickstart.py b/examples/bin/druid-automated-quickstart.py index de6827f61f67..ae4bc6fddfc6 100644 --- a/examples/bin/druid-automated-quickstart.py +++ b/examples/bin/druid-automated-quickstart.py @@ -29,11 +29,16 @@ ] SERVICE_MEMORY_DISTRIBUTION_WEIGHT = { - MIDDLE_MANAGER_SERVICE_NAME: 0.25, + MIDDLE_MANAGER_SERVICE_NAME: 0.5, ROUTER_SERVICE_NAME: 1, COORDINATOR_SERVICE_NAME: 18, BROKER_SERVICE_NAME: 28, - HISTORICAL_SERVICE_NAME: 50 + HISTORICAL_SERVICE_NAME: 45 +} + +SERVICE_MEMORY_LOWER_BOUND = { + MIDDLE_MANAGER_SERVICE_NAME: 64, + ROUTER_SERVICE_NAME: 128 } SERVICE_MEMORY_HEAP_RATIO = { @@ -150,8 +155,34 @@ def distribute_memory_over_services(service_config, total_memory): multiplier = total_memory / memory_weight_sum + lower_bound_memory_allocation = 0 + allocated_services = set() + for key, value in service_instance_map.items(): + allocated_memory = SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(key) * multiplier + if key in SERVICE_MEMORY_LOWER_BOUND and allocated_memory < SERVICE_MEMORY_LOWER_BOUND.get(key): + allocated_memory = SERVICE_MEMORY_LOWER_BOUND.get(key) + heap_memory = SERVICE_MEMORY_HEAP_RATIO.get(key) * allocated_memory + direct_memory = allocated_memory - heap_memory + service_memory_config[key] = build_memory_config_string(int(heap_memory), int(direct_memory)) + lower_bound_memory_allocation += allocated_memory + allocated_services.add(key) + + if lower_bound_memory_allocation > 0: + # compute the multiplier again for remaing services + memory_weight_sum = 0 + for key, value in service_instance_map.items(): + if key in allocated_services: + continue + memory_weight_sum += SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(key) * value + multiplier = (total_memory - lower_bound_memory_allocation) / memory_weight_sum + for key, value in service_instance_map.items(): + if key in allocated_services: + continue allocated_memory = SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(key) * multiplier + if key in SERVICE_MEMORY_LOWER_BOUND and allocated_memory < SERVICE_MEMORY_LOWER_BOUND.get(key): + allocated_memory = SERVICE_MEMORY_LOWER_BOUND.get(key) + heap_memory = SERVICE_MEMORY_HEAP_RATIO.get(key) * allocated_memory direct_memory = allocated_memory - heap_memory service_memory_config[key] = build_memory_config_string(int(heap_memory), int(direct_memory)) @@ -191,13 +222,15 @@ def create_supervise_config_file(service_config, service_memory_config, base_con else: the_file.write(f'{prefix}{service} bin/run-druid {service} {base_config_path} {service_path} \'{jvm_args}\'\n') -def print_service_config(service_config, base_config_path): +def print_service_config(service_config, base_config_path, run_zk): print('Services to start:') for item in service_config: if item[1] == "": print(f'{item[0]}, using default config from {os.getcwd()}/../{QUICKSTART_BASE_CONFIG_PATH}') else: print(f'{item[0]}, using config from {base_config_path}/{item[1]}') + if run_zk: + print(f'zk, using default config from {os.getcwd()}/../conf/zk') def display_help(): text = """ @@ -206,9 +239,12 @@ def display_help(): where options include: totalMemory= memory for druid cluster, if totalMemory is not specified - 80 percent of system memory is used. Note, if service - specific jvm config is present, + 80 percent of system memory is used. + Note, if service specific jvm config is present, totalMemory shouldn't be specified + Integer value is supported with `m` or `g` suffix, + denoting memory in mb or gb + Memory should be greater than equals 2g baseConfigPath= relative path to base directory containing common and service specific properties to be overridden, this directory must contain `_common` @@ -276,7 +312,7 @@ def main(): os.chdir(os.path.dirname(sys.argv[0])) print(f'Arguments passed: baseConfigPath: "{base_config_path}", totalMemory: "{total_memory}"\n') - print_service_config(service_config, base_config_path) + print_service_config(service_config, base_config_path, run_zk) service_memory_config = {} if (should_compute_memory(base_config_path, total_memory, service_config)): From dbfb465610897af715307135f80cd7065526d3ae Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 15 Nov 2022 16:41:41 +0530 Subject: [PATCH 05/82] Additional runtime properties for services --- .../quickstart/broker/runtime.properties | 18 +++++++++++++++++- .../coordinator-overlord/runtime.properties | 14 +++++++++++++- .../quickstart/historical/runtime.properties | 17 ++++++++++++++++- .../middleManager/runtime.properties | 19 +++++++++++++++++++ .../quickstart/router/runtime.properties | 15 ++++++++++++++- 5 files changed, 79 insertions(+), 4 deletions(-) diff --git a/examples/conf/druid/single-server/quickstart/broker/runtime.properties b/examples/conf/druid/single-server/quickstart/broker/runtime.properties index 4d67149d29e9..ac42accd8576 100644 --- a/examples/conf/druid/single-server/quickstart/broker/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/broker/runtime.properties @@ -18,4 +18,20 @@ # druid.service=druid/broker -druid.plaintextPort=8082 \ No newline at end of file +druid.plaintextPort=8082 + +# HTTP server settings +druid.server.http.numThreads=12 + +# HTTP client settings +druid.broker.http.numConnections=10 +druid.broker.http.maxQueuedBytes=5MiB + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=100MiB +druid.processing.numMergeBuffers=2 +druid.processing.tmpDir=var/druid/processing + +# Query cache disabled -- push down caching and merging instead +druid.broker.cache.useCache=false +druid.broker.cache.populateCache=false \ No newline at end of file diff --git a/examples/conf/druid/single-server/quickstart/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/quickstart/coordinator-overlord/runtime.properties index 855eeeafa7dd..c808bb6d2e4b 100644 --- a/examples/conf/druid/single-server/quickstart/coordinator-overlord/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/coordinator-overlord/runtime.properties @@ -18,4 +18,16 @@ # druid.service=druid/coordinator -druid.plaintextPort=8081 \ No newline at end of file +druid.plaintextPort=8081 + +druid.coordinator.startDelay=PT10S +druid.coordinator.period=PT5S +druid.manager.segments.pollDuration=PT5S + +# Run the overlord service in the coordinator process +druid.coordinator.asOverlord.enabled=true +druid.coordinator.asOverlord.overlordService=druid/overlord + +druid.indexer.queue.startDelay=PT5S + +druid.indexer.storage.type=metadata \ No newline at end of file diff --git a/examples/conf/druid/single-server/quickstart/historical/runtime.properties b/examples/conf/druid/single-server/quickstart/historical/runtime.properties index 4ce14669dbaf..76eaa5d4b0cd 100644 --- a/examples/conf/druid/single-server/quickstart/historical/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/historical/runtime.properties @@ -20,5 +20,20 @@ druid.service=druid/historical druid.plaintextPort=8083 +# HTTP server threads +druid.server.http.numThreads=12 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=200MiB +druid.processing.numMergeBuffers=2 +druid.processing.numThreads=2 +druid.processing.tmpDir=var/druid/processing + # Segment storage -druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g"}] \ No newline at end of file +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g"}] + +# Query cache +druid.historical.cache.useCache=true +druid.historical.cache.populateCache=true +druid.cache.type=caffeine +druid.cache.sizeInBytes=10MiB diff --git a/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties b/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties index fdec5cfca425..e7262930f498 100644 --- a/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties @@ -19,3 +19,22 @@ druid.service=druid/middleManager druid.plaintextPort=8091 + +# Number of tasks per middleManager +druid.worker.capacity=2 + +# Task launch parameters +druid.indexer.runner.javaCommand=bin/run-java +druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] +druid.indexer.task.baseTaskDir=var/druid/task + +# HTTP server threads +druid.server.http.numThreads=12 + +# Processing threads and buffers on Peons +druid.indexer.fork.property.druid.processing.numMergeBuffers=2 +druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB +druid.indexer.fork.property.druid.processing.numThreads=1 + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/examples/conf/druid/single-server/quickstart/router/runtime.properties b/examples/conf/druid/single-server/quickstart/router/runtime.properties index 3754ec172210..e5448b38d0cc 100644 --- a/examples/conf/druid/single-server/quickstart/router/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/router/runtime.properties @@ -18,4 +18,17 @@ # druid.service=druid/router -druid.plaintextPort=8888 \ No newline at end of file +druid.plaintextPort=8888 + +# HTTP proxy +druid.router.http.numConnections=50 +druid.router.http.readTimeout=PT5M +druid.router.http.numMaxThreads=100 +druid.server.http.numThreads=100 + +# Service discovery +druid.router.defaultBrokerServiceName=druid/broker +druid.router.coordinatorServiceName=druid/coordinator + +# Management proxy to coordinator / overlord: required for unified web console. +druid.router.managementProxy.enabled=true \ No newline at end of file From 7324252be7ab4fd5c13adcdb494d9b015cf73186 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 18 Nov 2022 13:26:17 +0530 Subject: [PATCH 06/82] Update supervise script to accept command arguments, corresponding changes in druid-quickstart.py --- ...ated-quickstart.py => druid-quickstart.py} | 114 ++++++++++-------- 1 file changed, 62 insertions(+), 52 deletions(-) rename examples/bin/{druid-automated-quickstart.py => druid-quickstart.py} (79%) diff --git a/examples/bin/druid-automated-quickstart.py b/examples/bin/druid-quickstart.py similarity index 79% rename from examples/bin/druid-automated-quickstart.py rename to examples/bin/druid-quickstart.py index ae4bc6fddfc6..8058315b3783 100644 --- a/examples/bin/druid-automated-quickstart.py +++ b/examples/bin/druid-quickstart.py @@ -3,7 +3,6 @@ import psutil from pathlib import Path -SUPERVISE_CONFIG_FILE_PATH = "../conf/supervise/single-server/quickstart.conf" QUICKSTART_BASE_CONFIG_PATH = "conf/druid/single-server/quickstart" HELP_ARG_IDENTIFIER = "help" COMPUTE_ONLY_ARG_IDENTIFIER = "computeOnly" @@ -29,11 +28,11 @@ ] SERVICE_MEMORY_DISTRIBUTION_WEIGHT = { - MIDDLE_MANAGER_SERVICE_NAME: 0.5, - ROUTER_SERVICE_NAME: 1, - COORDINATOR_SERVICE_NAME: 18, - BROKER_SERVICE_NAME: 28, - HISTORICAL_SERVICE_NAME: 45 + MIDDLE_MANAGER_SERVICE_NAME: 1, + ROUTER_SERVICE_NAME: 2, + COORDINATOR_SERVICE_NAME: 36, + BROKER_SERVICE_NAME: 56, + HISTORICAL_SERVICE_NAME: 90 } SERVICE_MEMORY_LOWER_BOUND = { @@ -41,7 +40,7 @@ ROUTER_SERVICE_NAME: 128 } -SERVICE_MEMORY_HEAP_RATIO = { +SERVICE_MEMORY_HEAP_PERCENTAGE = { MIDDLE_MANAGER_SERVICE_NAME: 1, ROUTER_SERVICE_NAME: 1, COORDINATOR_SERVICE_NAME: 1, @@ -81,15 +80,18 @@ def parse_arguments(): if service not in DEFAULT_SERVICES: raise Exception(f'{service} is not a valid service name, should be one of {DEFAULT_SERVICES}') - path = "" + subdirectory = "" if len(split_args) == 2: - path = split_args[1] - complete_path = os.path.join(base_config_path, path) - if os.path.exists(os.path.join(complete_path)) is False: - raise Exception(f'Path `{complete_path}` specified for service `{service}` doesn\'t exist ') + subdirectory = split_args[1] + + if subdirectory != "": + complete_path = os.path.join(base_config_path, subdirectory) + if os.path.exists(os.path.join(complete_path)) is False: + raise Exception(f'Path `{complete_path}` specified for service `{service}` doesn\'t exist') + service_list.append(service) - service_path_list.append(path) + service_path_list.append(subdirectory) if len(service_list) == 0: # start all services @@ -161,7 +163,7 @@ def distribute_memory_over_services(service_config, total_memory): allocated_memory = SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(key) * multiplier if key in SERVICE_MEMORY_LOWER_BOUND and allocated_memory < SERVICE_MEMORY_LOWER_BOUND.get(key): allocated_memory = SERVICE_MEMORY_LOWER_BOUND.get(key) - heap_memory = SERVICE_MEMORY_HEAP_RATIO.get(key) * allocated_memory + heap_memory = SERVICE_MEMORY_HEAP_PERCENTAGE.get(key) * allocated_memory direct_memory = allocated_memory - heap_memory service_memory_config[key] = build_memory_config_string(int(heap_memory), int(direct_memory)) lower_bound_memory_allocation += allocated_memory @@ -183,7 +185,7 @@ def distribute_memory_over_services(service_config, total_memory): if key in SERVICE_MEMORY_LOWER_BOUND and allocated_memory < SERVICE_MEMORY_LOWER_BOUND.get(key): allocated_memory = SERVICE_MEMORY_LOWER_BOUND.get(key) - heap_memory = SERVICE_MEMORY_HEAP_RATIO.get(key) * allocated_memory + heap_memory = SERVICE_MEMORY_HEAP_PERCENTAGE.get(key) * allocated_memory direct_memory = allocated_memory - heap_memory service_memory_config[key] = build_memory_config_string(int(heap_memory), int(direct_memory)) @@ -194,33 +196,40 @@ def distribute_memory_over_services(service_config, total_memory): return service_memory_config -def create_supervise_config_file(service_config, service_memory_config, base_config_path, run_zk): - - with open(SUPERVISE_CONFIG_FILE_PATH, 'w+') as the_file: - the_file.write(":verify bin/verify-java\n") - the_file.write(":verify bin/verify-default-ports\n") - the_file.write(":notify bin/greet\n") - the_file.write(":kill-timeout 10\n") - the_file.write("\n") - - if run_zk: - the_file.write("!p10 zk bin/run-zk conf\n") - - for item in service_config: - service = item[0] - prefix = '' - if service == MIDDLE_MANAGER_SERVICE_NAME: - prefix = '!p90 ' - if item[1] == "": - service_path = item[0] - else: - service_path = item[1] - jvm_args = service_memory_config.get(item[0]) - - if jvm_args == "": - the_file.write(f'{prefix}{service} bin/run-druid {service} {base_config_path} {service_path}\n') - else: - the_file.write(f'{prefix}{service} bin/run-druid {service} {base_config_path} {service_path} \'{jvm_args}\'\n') +def build_supervise_script_arguments(service_config, service_memory_config, base_config_path, run_zk): + argument_list = [] + + argument_list.append("\":verify bin/verify-java\"") + argument_list.append("\":verify bin/verify-default-ports\"") + argument_list.append("\":notify bin/greet\"") + argument_list.append("\":kill-timeout 10\"") + + if run_zk: + argument_list.append("\"!p10 zk bin/run-zk conf\"") + + for item in service_config: + service = item[0] + prefix = '' + if service == MIDDLE_MANAGER_SERVICE_NAME: + prefix = '!p90 ' + if item[1] == "": + service_path = item[0] + else: + service_path = item[1] + jvm_args = service_memory_config.get(item[0]) + + if jvm_args is None: + argument_list.append(f'\"{prefix}{service} bin/run-druid {service} {base_config_path} {service_path}\"') + else: + argument_list.append(f'\"{prefix}{service} bin/run-druid {service} {base_config_path} {service_path} \'{jvm_args}\'\"') + + print('Commands for supervise script:') + for item in argument_list: + print(item) + + print('\n') + + return ",".join(argument_list) def print_service_config(service_config, base_config_path, run_zk): print('Services to start:') @@ -231,6 +240,7 @@ def print_service_config(service_config, base_config_path, run_zk): print(f'{item[0]}, using config from {base_config_path}/{item[1]}') if run_zk: print(f'zk, using default config from {os.getcwd()}/../conf/zk') + print('\n') def display_help(): text = """ @@ -246,11 +256,11 @@ def display_help(): denoting memory in mb or gb Memory should be greater than equals 2g baseConfigPath= - relative path to base directory containing common and service specific + relative path to base directory, containing common and service specific properties to be overridden, this directory must contain `_common` directory with `common.jvm.config` & `common.runtime.properties` if `baseConfigPath` is not specified, config from - conf/druid/single-server/quickstart direcotry is used + conf/druid/single-server/quickstart directory is used computeOnly command dry-run, validates the arguments and display the memory distribution for services @@ -275,21 +285,21 @@ def display_help(): compute memory distribution for all the services start-druid totalMemory=100g broker router historical starts `broker`, `router` and `historical` services, using `100g` of memory - start-druid totalMemory=100g baseConfigPath=/conf/druid/single-server/large broker router historical + start-druid totalMemory=100g baseConfigPath=../conf/druid/single-server/large broker router historical starts `broker`, `router` and `historical` service, using 100g of memory, use common configs from specified `baseConfigPath` - start-druid totalMemory=100g baseConfigPath=/conf/druid/single-server/large broker=broker router=router historical=historical + start-druid totalMemory=100g baseConfigPath=../conf/druid/single-server/large broker=broker router=router historical=historical starts `broker`, `router` and `historical` services, using 100g of memory, use common configs from specified `baseConfigPath`, use service specific config from specified directories if jvm.config is specified for all the services, memory distribution is not computed - start-druid totalMemory=100g baseConfigPath=/conf/druid/single-server/large broker=broker1 broker=broker2 + start-druid totalMemory=100g baseConfigPath=../conf/druid/single-server/large broker=broker1 broker=broker2 starts 2 instances of `broker` config is read from respective directories, depending on whether jvm.config is specified, memory distribution is computed - start-druid totalMemory=100g baseConfigPath=conf/druid/profile broker=broker1 historical=historical1 + start-druid totalMemory=100g baseConfigPath=../conf/druid/profile broker=broker1 historical=historical1 if either of `broker1`, `historical1` subdirectory contains jvm.config, exception is thrown since `totalMemory` argument is specified - start-druid baseConfigPath=conf/druid/profile broker=broker1 historical=historical1 + start-druid baseConfigPath=../conf/druid/profile broker=broker1 historical=historical1 exception is thrown if either of `broker1`, `historical1` subdirectory contains jvm.config but not both If none of the subdirectory contains jvm.config, memory distribution is computed @@ -304,7 +314,7 @@ def main(): display_help() return - print("Druid automated quickstart\n") + print("Druid quickstart\n") base_config_path, total_memory, service_config, run_zk, compute_only = parse_arguments() @@ -325,9 +335,9 @@ def main(): if base_config_path == "": base_config_path = QUICKSTART_BASE_CONFIG_PATH - create_supervise_config_file(service_config, service_memory_config, base_config_path, run_zk) + script_arguments = build_supervise_script_arguments(service_config, service_memory_config, base_config_path, run_zk) - os.system(f'exec ./supervise -c {SUPERVISE_CONFIG_FILE_PATH}') + os.system(f'exec ./supervise -a {script_arguments}') if __name__ == '__main__': main() \ No newline at end of file From 72494e939341110b3db5558d663402503bcffd68 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 18 Nov 2022 13:29:42 +0530 Subject: [PATCH 07/82] File end newline --- examples/bin/druid-quickstart.py | 2 +- examples/bin/run-druid | 2 +- examples/bin/start-druid | 2 +- examples/bin/supervise | 99 ++++++++++++------- .../quickstart/_common/common.jvm.config | 2 +- .../quickstart/broker/runtime.properties | 2 +- .../coordinator-overlord/runtime.properties | 2 +- .../quickstart/router/runtime.properties | 2 +- 8 files changed, 70 insertions(+), 43 deletions(-) diff --git a/examples/bin/druid-quickstart.py b/examples/bin/druid-quickstart.py index 8058315b3783..136dcf9ccd15 100644 --- a/examples/bin/druid-quickstart.py +++ b/examples/bin/druid-quickstart.py @@ -340,4 +340,4 @@ def main(): os.system(f'exec ./supervise -a {script_arguments}') if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/examples/bin/run-druid b/examples/bin/run-druid index c98c2e53bc3c..005c116e6100 100755 --- a/examples/bin/run-druid +++ b/examples/bin/run-druid @@ -73,4 +73,4 @@ else exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" \ `cat "$CONFDIR"/$WHATAMI/main.config | xargs` -fi \ No newline at end of file +fi diff --git a/examples/bin/start-druid b/examples/bin/start-druid index 13781fca8e9b..41639d8cdce6 100644 --- a/examples/bin/start-druid +++ b/examples/bin/start-druid @@ -21,4 +21,4 @@ PWD="$(pwd)" WHEREAMI="$(dirname "$0")" WHEREAMI="$(cd "$WHEREAMI" && pwd)" -python3 "$WHEREAMI"/druid-automated-quickstart.py $@ \ No newline at end of file +python3 "$WHEREAMI"/druid-quickstart.py $@ diff --git a/examples/bin/supervise b/examples/bin/supervise index 81b7e57b049e..507795733c34 100755 --- a/examples/bin/supervise +++ b/examples/bin/supervise @@ -46,6 +46,52 @@ sub usage die "usage: $0 -c [-d ] [-t ] [--svlogd ]\n"; } +sub process_config +{ + my @lines = @_; + my @commands; + my @verify; + my @notify; + my $kill_timeout; + for my $line (@lines) + { + if ($line =~ /^(:verify|:notify|:kill-timeout|(?:\!p[0-9]+\s+)?[^:]\S+)\s+(.+)$/) { + + my $name = $1; + my $order = 50; + my $command = $2; + + if ($name =~ /^(?:\!p([0-9]+)\s+)(.*)$/) { + $order = $1; + $name = $2; + } + + if ($name eq ':verify') { + push @verify, $command; + } elsif ($name eq ':notify') { + push @notify, $command; + } elsif ($name eq ':kill-timeout') { + $kill_timeout = int($command); + } else { + die "Duplicate command: $line\n" if grep { $_->{name} eq $name } @commands; + push @commands, { + name => $name, + command => $command, + order => $order, # Stop order for this command + pid => 0, # Current pid, or 0 if not running + down => 0, # Time the proc should be down until + killed => 0, # Signal we sent to this process + restarting => 0, # True if this command is currently restarting + }; + } + } else { + die "Syntax error: $line\n"; + } + } + + return { commands => \@commands, verify => \@verify, notify => \@notify, 'kill-timeout' => $kill_timeout }; +} + sub read_config_file { my ($config_file) = @_; @@ -53,49 +99,20 @@ sub read_config_file open my $config_fh, "<", $config_file or die "open $config_file: $!"; - my @commands; - my @verify; - my @notify; - my $kill_timeout; + my @lines; while (my $line = <$config_fh>) { chomp $line; next if $line =~ /^(\s*\#.*|\s*)$/; if ($line =~ /^(:verify|:notify|:kill-timeout|(?:\!p[0-9]+\s+)?[^:]\S+)\s+(.+)$/) { - my $name = $1; - my $order = 50; - my $command = $2; - - if ($name =~ /^(?:\!p([0-9]+)\s+)(.*)$/) { - $order = $1; - $name = $2; - } - - if ($name eq ':verify') { - push @verify, $command; - } elsif ($name eq ':notify') { - push @notify, $command; - } elsif ($name eq ':kill-timeout') { - $kill_timeout = int($command); - } else { - die "Duplicate command: $line\n" if grep { $_->{name} eq $name } @commands; - push @commands, { - name => $name, - command => $command, - order => $order, # Stop order for this command - pid => 0, # Current pid, or 0 if not running - down => 0, # Time the proc should be down until - killed => 0, # Signal we sent to this process - restarting => 0, # True if this command is currently restarting - }; - } + push @lines, $line } else { die "Syntax error: $line\n"; } } close $config_fh; - return { commands => \@commands, verify => \@verify, notify => \@notify, 'kill-timeout' => $kill_timeout }; + return @lines; } sub stringify_exit_status @@ -179,13 +196,23 @@ usage() unless GetOptions( 'vardir|d=s', 'kill-timeout|t=i', 'chdir=s', - 'svlogd:s' + 'svlogd:s', + 'array|a=s{,}' ); -usage() unless $opt{'conf'} && $opt{'vardir'}; +usage() unless (($opt{'array'} || $opt{'conf'}) && $opt{'vardir'}); + +my @config_lines; + +# get commands to execute either from reading the config file or command line +if (not defined $opt{'conf'}) { + @config_lines = split(',', $opt{'array'}); +} else { + @config_lines = read_config_file($opt{'conf'}); +} + +my $config = process_config(@config_lines); -# Read config file -my $config = read_config_file($opt{'conf'}); @commands = @{$config->{commands}}; if (!@commands) { diff --git a/examples/conf/druid/single-server/quickstart/_common/common.jvm.config b/examples/conf/druid/single-server/quickstart/_common/common.jvm.config index 4db1d94ec38e..fd74cf358979 100644 --- a/examples/conf/druid/single-server/quickstart/_common/common.jvm.config +++ b/examples/conf/druid/single-server/quickstart/_common/common.jvm.config @@ -4,4 +4,4 @@ -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.io.tmpdir=var/tmp --Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \ No newline at end of file +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/single-server/quickstart/broker/runtime.properties b/examples/conf/druid/single-server/quickstart/broker/runtime.properties index ac42accd8576..92b85f8bc754 100644 --- a/examples/conf/druid/single-server/quickstart/broker/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/broker/runtime.properties @@ -34,4 +34,4 @@ druid.processing.tmpDir=var/druid/processing # Query cache disabled -- push down caching and merging instead druid.broker.cache.useCache=false -druid.broker.cache.populateCache=false \ No newline at end of file +druid.broker.cache.populateCache=false diff --git a/examples/conf/druid/single-server/quickstart/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/quickstart/coordinator-overlord/runtime.properties index c808bb6d2e4b..c053823f8ed8 100644 --- a/examples/conf/druid/single-server/quickstart/coordinator-overlord/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/coordinator-overlord/runtime.properties @@ -30,4 +30,4 @@ druid.coordinator.asOverlord.overlordService=druid/overlord druid.indexer.queue.startDelay=PT5S -druid.indexer.storage.type=metadata \ No newline at end of file +druid.indexer.storage.type=metadata diff --git a/examples/conf/druid/single-server/quickstart/router/runtime.properties b/examples/conf/druid/single-server/quickstart/router/runtime.properties index e5448b38d0cc..497d3b4d3aa1 100644 --- a/examples/conf/druid/single-server/quickstart/router/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/router/runtime.properties @@ -31,4 +31,4 @@ druid.router.defaultBrokerServiceName=druid/broker druid.router.coordinatorServiceName=druid/coordinator # Management proxy to coordinator / overlord: required for unified web console. -druid.router.managementProxy.enabled=true \ No newline at end of file +druid.router.managementProxy.enabled=true From e3505879770b227a087828e28a67654c3699037e Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 18 Nov 2022 18:49:11 +0530 Subject: [PATCH 08/82] Limit the ability to start multiple instances of a service, documentation changes --- examples/bin/druid-quickstart.py | 150 +++++++++--------- examples/bin/run-druid | 2 + .../medium/broker/runtime.properties | 10 -- .../medium/historical/runtime.properties | 8 - .../medium/router/runtime.properties | 6 - 5 files changed, 76 insertions(+), 100 deletions(-) diff --git a/examples/bin/druid-quickstart.py b/examples/bin/druid-quickstart.py index 136dcf9ccd15..30ca0f60aad0 100644 --- a/examples/bin/druid-quickstart.py +++ b/examples/bin/druid-quickstart.py @@ -7,7 +7,7 @@ HELP_ARG_IDENTIFIER = "help" COMPUTE_ONLY_ARG_IDENTIFIER = "computeOnly" RUN_ZK_IDENTIFIER = "runZk" -BASE_CONFIG_PATH_IDENTIFIER = "baseConfigPath" +ROOT_CONFIG_PATH_IDENTIFIER = "rootConfigPath" MEMORY_ARG_IDENTIFIER = "totalMemory" MEMORY_GIGABYTES_IDENTIFIER = "g" MEMORY_MEGABYTES_IDENTIFIER = "m" @@ -59,7 +59,7 @@ def get_argument_value(argument): def parse_arguments(): service_list = [] service_path_list = [] - base_config_path = "" + root_config_path = "" total_memory = "" compute_only = False run_zk = False @@ -69,8 +69,8 @@ def parse_arguments(): compute_only = True elif check_argument_type(argument, RUN_ZK_IDENTIFIER): run_zk = True - elif check_argument_type(argument, BASE_CONFIG_PATH_IDENTIFIER): - base_config_path = os.path.join(os.getcwd(), get_argument_value(argument)) + elif check_argument_type(argument, ROOT_CONFIG_PATH_IDENTIFIER): + root_config_path = os.path.join(os.getcwd(), get_argument_value(argument)) elif (check_argument_type(argument, MEMORY_ARG_IDENTIFIER)): total_memory = get_argument_value(argument) else: @@ -80,13 +80,16 @@ def parse_arguments(): if service not in DEFAULT_SERVICES: raise Exception(f'{service} is not a valid service name, should be one of {DEFAULT_SERVICES}') + if service in service_list: + raise Exception(f'{service} is specified multiple times') + subdirectory = "" if len(split_args) == 2: subdirectory = split_args[1] if subdirectory != "": - complete_path = os.path.join(base_config_path, subdirectory) + complete_path = os.path.join(root_config_path, subdirectory) if os.path.exists(os.path.join(complete_path)) is False: raise Exception(f'Path `{complete_path}` specified for service `{service}` doesn\'t exist') @@ -99,9 +102,9 @@ def parse_arguments(): service_path_list = [""] * len(DEFAULT_SERVICES) run_zk = True - return base_config_path, total_memory, list(zip(service_list, service_path_list)), run_zk, compute_only + return root_config_path, total_memory, list(zip(service_list, service_path_list)), run_zk, compute_only -def should_compute_memory(base_config_path, total_memory, service_config): +def should_compute_memory(root_config_path, total_memory, service_config): # if jvm file is present for any of the services # it should be present for all services and totalMemory should not be specified # if totalMemory is given, jvm file shouldn't be present for any service @@ -109,7 +112,7 @@ def should_compute_memory(base_config_path, total_memory, service_config): jvm_config_count = 0 for item in service_config: if item[1] != "": - if Path(f'{base_config_path}/{item[1]}/jvm.config').is_file(): + if Path(f'{root_config_path}/{item[1]}/jvm.config').is_file(): jvm_config_count += 1 elif jvm_config_count > 0: raise Exception('jvm.config file is missing for service {item[0]}, jvm.config should be specified for all the services or none') @@ -130,7 +133,7 @@ def compute_system_memory(): def convert_total_memory_string(memory): if memory == "": computed_memory = compute_system_memory() - print(f'`{MEMORY_ARG_IDENTIFIER}` argument is not specified Druid will use 80% of system memory: {computed_memory}m') + print(f'`{MEMORY_ARG_IDENTIFIER}` argument is not specified, Druid will use 80% of system memory: {computed_memory}m') return computed_memory elif memory.endswith(MEMORY_MEGABYTES_IDENTIFIER): return int(memory[:-1]) @@ -146,57 +149,57 @@ def build_memory_config_string(heap_memory, direct_memory): def distribute_memory_over_services(service_config, total_memory): service_memory_config = {} - service_instance_map = {} - - for item in service_config: - service_instance_map[item[0]] = service_instance_map.get(item[0], 0) + 1 memory_weight_sum = 0 - for key, value in service_instance_map.items(): - memory_weight_sum += SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(key) * value + for item in service_config: + memory_weight_sum += SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(item[0]) multiplier = total_memory / memory_weight_sum lower_bound_memory_allocation = 0 allocated_services = set() - for key, value in service_instance_map.items(): - allocated_memory = SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(key) * multiplier - if key in SERVICE_MEMORY_LOWER_BOUND and allocated_memory < SERVICE_MEMORY_LOWER_BOUND.get(key): - allocated_memory = SERVICE_MEMORY_LOWER_BOUND.get(key) - heap_memory = SERVICE_MEMORY_HEAP_PERCENTAGE.get(key) * allocated_memory + + for item in service_config: + service = item[0] + allocated_memory = SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) * multiplier + if service in SERVICE_MEMORY_LOWER_BOUND and allocated_memory < SERVICE_MEMORY_LOWER_BOUND.get(service): + allocated_memory = SERVICE_MEMORY_LOWER_BOUND.get(service) + heap_memory = SERVICE_MEMORY_HEAP_PERCENTAGE.get(service) * allocated_memory direct_memory = allocated_memory - heap_memory - service_memory_config[key] = build_memory_config_string(int(heap_memory), int(direct_memory)) + service_memory_config[service] = build_memory_config_string(int(heap_memory), int(direct_memory)) lower_bound_memory_allocation += allocated_memory - allocated_services.add(key) + allocated_services.add(service) if lower_bound_memory_allocation > 0: # compute the multiplier again for remaing services memory_weight_sum = 0 - for key, value in service_instance_map.items(): - if key in allocated_services: + for item in service_config: + service = item[0] + if service in allocated_services: continue - memory_weight_sum += SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(key) * value + memory_weight_sum += SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) * value multiplier = (total_memory - lower_bound_memory_allocation) / memory_weight_sum - for key, value in service_instance_map.items(): - if key in allocated_services: + for item in service_config: + service = item[0] + if service in allocated_services: continue - allocated_memory = SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(key) * multiplier - if key in SERVICE_MEMORY_LOWER_BOUND and allocated_memory < SERVICE_MEMORY_LOWER_BOUND.get(key): - allocated_memory = SERVICE_MEMORY_LOWER_BOUND.get(key) + allocated_memory = SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) * multiplier + if service in SERVICE_MEMORY_LOWER_BOUND and allocated_memory < SERVICE_MEMORY_LOWER_BOUND.get(service): + allocated_memory = SERVICE_MEMORY_LOWER_BOUND.get(service) - heap_memory = SERVICE_MEMORY_HEAP_PERCENTAGE.get(key) * allocated_memory + heap_memory = SERVICE_MEMORY_HEAP_PERCENTAGE.get(service) * allocated_memory direct_memory = allocated_memory - heap_memory - service_memory_config[key] = build_memory_config_string(int(heap_memory), int(direct_memory)) + service_memory_config[service] = build_memory_config_string(int(heap_memory), int(direct_memory)) print(f'\nMemory distribution for services:') for key, value in service_memory_config.items(): - print(f'{key}, memory_config: {value}, instance_count: {service_instance_map[key]}') + print(f'{key}, memory_config: {value}') print('\n') return service_memory_config -def build_supervise_script_arguments(service_config, service_memory_config, base_config_path, run_zk): +def build_supervise_script_arguments(service_config, service_memory_config, root_config_path, run_zk): argument_list = [] argument_list.append("\":verify bin/verify-java\"") @@ -219,11 +222,11 @@ def build_supervise_script_arguments(service_config, service_memory_config, base jvm_args = service_memory_config.get(item[0]) if jvm_args is None: - argument_list.append(f'\"{prefix}{service} bin/run-druid {service} {base_config_path} {service_path}\"') + argument_list.append(f'\"{prefix}{service} bin/run-druid {service} {root_config_path} {service_path}\"') else: - argument_list.append(f'\"{prefix}{service} bin/run-druid {service} {base_config_path} {service_path} \'{jvm_args}\'\"') + argument_list.append(f'\"{prefix}{service} bin/run-druid {service} {root_config_path} {service_path} \'{jvm_args}\'\"') - print('Commands for supervise script:') + print('Command for supervise script:') for item in argument_list: print(item) @@ -231,13 +234,13 @@ def build_supervise_script_arguments(service_config, service_memory_config, base return ",".join(argument_list) -def print_service_config(service_config, base_config_path, run_zk): +def print_service_config(service_config, root_config_path, run_zk): print('Services to start:') for item in service_config: if item[1] == "": print(f'{item[0]}, using default config from {os.getcwd()}/../{QUICKSTART_BASE_CONFIG_PATH}') else: - print(f'{item[0]}, using config from {base_config_path}/{item[1]}') + print(f'{item[0]}, using config from {root_config_path}/{item[1]}') if run_zk: print(f'zk, using default config from {os.getcwd()}/../conf/zk') print('\n') @@ -251,58 +254,53 @@ def display_help(): memory for druid cluster, if totalMemory is not specified 80 percent of system memory is used. Note, if service specific jvm config is present, - totalMemory shouldn't be specified - Integer value is supported with `m` or `g` suffix, - denoting memory in mb or gb + totalMemory shouldn't be specified. + Integer value is supported with `m` or `g` suffix. Memory should be greater than equals 2g - baseConfigPath= - relative path to base directory, containing common and service specific + rootConfigPath= + directory containing common and service specific properties to be overridden, this directory must contain `_common` directory with `common.jvm.config` & `common.runtime.properties` - if `baseConfigPath` is not specified, config from + if `rootConfigPath` is not specified, config from conf/druid/single-server/quickstart directory is used computeOnly command dry-run, validates the arguments and display the memory distribution for services runZk specification to run zookeeper, zk config is picked up from conf/zk - =[subdirectory] + [=subdirectory] service_identifier is the service to be started, multiple services can be specified, `service_identifier` should be one of [broker, router, middleManager, historical, coordinator-overlord] - `subdirectory` is optional directory within `baseConfigPath` + `subdirectory` is optional directory within `rootConfigPath` containing runtime properties or/and jvm properties - Note, if jvm.config file is present for one service, it must be - present for all other services - If no service is specified, all services and zookeeper are started + Note, if jvm.config file is present for one of the service, + it must be present for all other services. + If service is not explicitly specified, all services + alongwith zookeeper is started + Note each service should be specified at most once sample usage: start-druid - start up all the services using the default system memory + start up all the services (including zk) using the default system memory start-druid totalMemory=100g start up all the services using the given memory start-druid totalMemory=100g computeOnly compute memory distribution for all the services start-druid totalMemory=100g broker router historical starts `broker`, `router` and `historical` services, using `100g` of memory - start-druid totalMemory=100g baseConfigPath=../conf/druid/single-server/large broker router historical - starts `broker`, `router` and `historical` service, using 100g of memory, - use common configs from specified `baseConfigPath` - start-druid totalMemory=100g baseConfigPath=../conf/druid/single-server/large broker=broker router=router historical=historical + start-druid totalMemory=100g rootConfigPath=../conf/druid/single-server/large broker router historical + start `broker`, `router` and `historical` service, using 100g of memory, + use common configs from specified `rootConfigPath` + start-druid totalMemory=100g rootConfigPath=../conf/druid/single-server/large broker=broker router=router historical=historical starts `broker`, `router` and `historical` services, using 100g of memory, use common configs - from specified `baseConfigPath`, use service specific config from specified directories - if jvm.config is specified for all the services, memory distribution is not computed - start-druid totalMemory=100g baseConfigPath=../conf/druid/single-server/large broker=broker1 broker=broker2 - starts 2 instances of `broker` - config is read from respective directories, depending on whether jvm.config is specified, + from specified `rootConfigPath`, use service specific config from specified sub directories + Since totalMemory is specific, the service specific folder shouldn't contain jvm.config + start-druid rootConfigPath=../conf/druid/profile broker=broker historical=historical + exception is thrown if either of `broker`, `historical` + subdirectory contains jvm.config but not both. + If none of the service specific subdirectory contains jvm.config, memory distribution is computed - start-druid totalMemory=100g baseConfigPath=../conf/druid/profile broker=broker1 historical=historical1 - if either of `broker1`, `historical1` subdirectory contains jvm.config, - exception is thrown since `totalMemory` argument is specified - start-druid baseConfigPath=../conf/druid/profile broker=broker1 historical=historical1 - exception is thrown if either of `broker1`, `historical1` - subdirectory contains jvm.config but not both - If none of the subdirectory contains jvm.config, memory distribution is computed """ print(text) @@ -316,26 +314,26 @@ def main(): print("Druid quickstart\n") - base_config_path, total_memory, service_config, run_zk, compute_only = parse_arguments() + root_config_path, total_memory, service_config, run_zk, compute_only = parse_arguments() # change directory to bin os.chdir(os.path.dirname(sys.argv[0])) - print(f'Arguments passed: baseConfigPath: "{base_config_path}", totalMemory: "{total_memory}"\n') - print_service_config(service_config, base_config_path, run_zk) + print(f'Arguments passed: rootConfigPath: "{root_config_path}", totalMemory: "{total_memory}"\n') + print_service_config(service_config, root_config_path, run_zk) service_memory_config = {} - if (should_compute_memory(base_config_path, total_memory, service_config)): + if (should_compute_memory(root_config_path, total_memory, service_config)): memory_in_mega_bytes = convert_total_memory_string(total_memory) service_memory_config = distribute_memory_over_services(service_config, memory_in_mega_bytes) - if compute_only: - return + if root_config_path == "": + root_config_path = QUICKSTART_BASE_CONFIG_PATH - if base_config_path == "": - base_config_path = QUICKSTART_BASE_CONFIG_PATH + script_arguments = build_supervise_script_arguments(service_config, service_memory_config, root_config_path, run_zk) - script_arguments = build_supervise_script_arguments(service_config, service_memory_config, base_config_path, run_zk) + if compute_only: + return os.system(f'exec ./supervise -a {script_arguments}') diff --git a/examples/bin/run-druid b/examples/bin/run-druid index 005c116e6100..2726477684df 100755 --- a/examples/bin/run-druid +++ b/examples/bin/run-druid @@ -56,11 +56,13 @@ fi if [ "$#" -eq 3 ] then + # arguments for this case is cd "$WHEREAMI/.." exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$3"/jvm.config | xargs` \ -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" org.apache.druid.cli.Main server $SERVER_NAME elif [ "$#" -eq 4 ] then + # arguments for this case is JVMARGS=`cat "$CONFDIR/_common/common.jvm.config" | xargs` JVMARGS+=' ' JVMARGS+=$4 diff --git a/examples/conf/druid/single-server/medium/broker/runtime.properties b/examples/conf/druid/single-server/medium/broker/runtime.properties index d46d6de70514..32dff403fa10 100644 --- a/examples/conf/druid/single-server/medium/broker/runtime.properties +++ b/examples/conf/druid/single-server/medium/broker/runtime.properties @@ -20,16 +20,6 @@ druid.service=druid/broker druid.plaintextPort=8082 -# HTTP server settings -druid.server.http.numThreads=60 - -# HTTP client settings -druid.broker.http.numConnections=50 -druid.broker.http.maxQueuedBytes=10MiB - -# Processing threads and buffers -druid.processing.buffer.sizeBytes=500MiB -druid.processing.numMergeBuffers=4 druid.processing.tmpDir=var/druid/processing # Query cache disabled -- push down caching and merging instead diff --git a/examples/conf/druid/single-server/medium/historical/runtime.properties b/examples/conf/druid/single-server/medium/historical/runtime.properties index a7a640b8061d..82443c1c43c2 100644 --- a/examples/conf/druid/single-server/medium/historical/runtime.properties +++ b/examples/conf/druid/single-server/medium/historical/runtime.properties @@ -20,13 +20,6 @@ druid.service=druid/historical druid.plaintextPort=8083 -# HTTP server threads -druid.server.http.numThreads=60 - -# Processing threads and buffers -druid.processing.buffer.sizeBytes=500MiB -druid.processing.numMergeBuffers=4 -druid.processing.numThreads=15 druid.processing.tmpDir=var/druid/processing # Segment storage @@ -36,4 +29,3 @@ druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g" druid.historical.cache.useCache=true druid.historical.cache.populateCache=true druid.cache.type=caffeine -druid.cache.sizeInBytes=256MiB diff --git a/examples/conf/druid/single-server/medium/router/runtime.properties b/examples/conf/druid/single-server/medium/router/runtime.properties index 497d3b4d3aa1..3858dec044bd 100644 --- a/examples/conf/druid/single-server/medium/router/runtime.properties +++ b/examples/conf/druid/single-server/medium/router/runtime.properties @@ -20,12 +20,6 @@ druid.service=druid/router druid.plaintextPort=8888 -# HTTP proxy -druid.router.http.numConnections=50 -druid.router.http.readTimeout=PT5M -druid.router.http.numMaxThreads=100 -druid.server.http.numThreads=100 - # Service discovery druid.router.defaultBrokerServiceName=druid/broker druid.router.coordinatorServiceName=druid/coordinator From 68a1e664b225f750a42b57b5ae27683960404418 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sat, 19 Nov 2022 01:05:25 +0530 Subject: [PATCH 09/82] simplify script arguments --- examples/bin/druid-quickstart.py | 228 +++++++++++++++---------------- examples/bin/run-druid | 25 ++-- 2 files changed, 121 insertions(+), 132 deletions(-) diff --git a/examples/bin/druid-quickstart.py b/examples/bin/druid-quickstart.py index 30ca0f60aad0..0b2da6658771 100644 --- a/examples/bin/druid-quickstart.py +++ b/examples/bin/druid-quickstart.py @@ -1,6 +1,7 @@ import sys import os import psutil +import pathlib from pathlib import Path QUICKSTART_BASE_CONFIG_PATH = "conf/druid/single-server/quickstart" @@ -8,10 +9,12 @@ COMPUTE_ONLY_ARG_IDENTIFIER = "computeOnly" RUN_ZK_IDENTIFIER = "runZk" ROOT_CONFIG_PATH_IDENTIFIER = "rootConfigPath" +SERVICES_IDENTIFIER = "services" MEMORY_ARG_IDENTIFIER = "totalMemory" MEMORY_GIGABYTES_IDENTIFIER = "g" MEMORY_MEGABYTES_IDENTIFIER = "m" ARG_SEPARATOR = "=" +SERVICE_SEPARATOR = "," BROKER_SERVICE_NAME = "broker" ROUTER_SERVICE_NAME = "router" @@ -58,7 +61,6 @@ def get_argument_value(argument): def parse_arguments(): service_list = [] - service_path_list = [] root_config_path = "" total_memory = "" compute_only = False @@ -71,54 +73,52 @@ def parse_arguments(): run_zk = True elif check_argument_type(argument, ROOT_CONFIG_PATH_IDENTIFIER): root_config_path = os.path.join(os.getcwd(), get_argument_value(argument)) + if os.path.exists(root_config_path) is False: + raise Exception(f'rootConfigPath `{root_config_path}` doesn\'t exist') elif (check_argument_type(argument, MEMORY_ARG_IDENTIFIER)): total_memory = get_argument_value(argument) - else: + elif (check_argument_type(argument, SERVICES_IDENTIFIER)): split_args = argument.split(ARG_SEPARATOR) - service = split_args[0] - - if service not in DEFAULT_SERVICES: - raise Exception(f'{service} is not a valid service name, should be one of {DEFAULT_SERVICES}') - - if service in service_list: - raise Exception(f'{service} is specified multiple times') + services_arg = split_args[1] + services = services_arg.split(SERVICE_SEPARATOR) - subdirectory = "" + for service in services: + if service not in DEFAULT_SERVICES: + raise Exception(f'{service} is not a valid service name, should be one of {DEFAULT_SERVICES}') - if len(split_args) == 2: - subdirectory = split_args[1] + if service in service_list: + raise Exception(f'{service} is specified multiple times') - if subdirectory != "": - complete_path = os.path.join(root_config_path, subdirectory) - if os.path.exists(os.path.join(complete_path)) is False: - raise Exception(f'Path `{complete_path}` specified for service `{service}` doesn\'t exist') - - service_list.append(service) - service_path_list.append(subdirectory) + service_list.append(service) if len(service_list) == 0: # start all services service_list = DEFAULT_SERVICES - service_path_list = [""] * len(DEFAULT_SERVICES) run_zk = True - return root_config_path, total_memory, list(zip(service_list, service_path_list)), run_zk, compute_only + return root_config_path, total_memory, service_list, run_zk, compute_only + +def print_startup_config(service_list, root_config_path, run_zk): + print(f'starting {service_list}, using config from {root_config_path}') + if run_zk: + zk_config_path = pathlib.Path(f'{os.getcwd()}/../conf/zk').resolve() + print(f'starting zk, using default config from {zk_config_path}') + print('\n') -def should_compute_memory(root_config_path, total_memory, service_config): +def should_compute_memory(root_config_path, total_memory, service_list): # if jvm file is present for any of the services # it should be present for all services and totalMemory should not be specified # if totalMemory is given, jvm file shouldn't be present for any service jvm_config_count = 0 - for item in service_config: - if item[1] != "": - if Path(f'{root_config_path}/{item[1]}/jvm.config').is_file(): - jvm_config_count += 1 - elif jvm_config_count > 0: - raise Exception('jvm.config file is missing for service {item[0]}, jvm.config should be specified for all the services or none') - - if jvm_config_count > 0 and (jvm_config_count != len(service_config) or total_memory != ""): - if jvm_config_count != len(service_config): + for service in service_list: + if Path(f'{root_config_path}/{service}/jvm.config').is_file(): + jvm_config_count += 1 + elif jvm_config_count > 0: + raise Exception('jvm.config file is missing for service {service}, jvm.config should be specified for all the services or none') + + if jvm_config_count > 0 and (jvm_config_count != len(service_list) or total_memory != ""): + if jvm_config_count != len(service_list): raise Exception("jvm.config file should be present for all services or none") if total_memory != "": raise Exception("If jvm.config is given for services, `totalMemory` argument shouldn't be specified") @@ -140,27 +140,26 @@ def convert_total_memory_string(memory): elif memory.endswith(MEMORY_GIGABYTES_IDENTIFIER): return 1024 * int(memory[:-1]) else: - raise Exception('Incorrect format for totalMemory argument, expected format is m or g') + raise Exception('Incorrect format for totalMemory argument, expected format is ') def build_memory_config_string(heap_memory, direct_memory): if direct_memory == 0: return f'-Xms{heap_memory}m -Xmx{heap_memory}m' return f'-Xms{heap_memory}m -Xmx{heap_memory}m -XX:MaxDirectMemorySize={direct_memory}m' -def distribute_memory_over_services(service_config, total_memory): +def distribute_memory_over_services(service_list, total_memory): service_memory_config = {} memory_weight_sum = 0 - for item in service_config: - memory_weight_sum += SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(item[0]) + for service in service_list: + memory_weight_sum += SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) multiplier = total_memory / memory_weight_sum lower_bound_memory_allocation = 0 allocated_services = set() - for item in service_config: - service = item[0] + for service in service_list: allocated_memory = SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) * multiplier if service in SERVICE_MEMORY_LOWER_BOUND and allocated_memory < SERVICE_MEMORY_LOWER_BOUND.get(service): allocated_memory = SERVICE_MEMORY_LOWER_BOUND.get(service) @@ -173,15 +172,13 @@ def distribute_memory_over_services(service_config, total_memory): if lower_bound_memory_allocation > 0: # compute the multiplier again for remaing services memory_weight_sum = 0 - for item in service_config: - service = item[0] + for service in service_list: if service in allocated_services: continue - memory_weight_sum += SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) * value + memory_weight_sum += SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) multiplier = (total_memory - lower_bound_memory_allocation) / memory_weight_sum - for item in service_config: - service = item[0] + for service in service_list: if service in allocated_services: continue allocated_memory = SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) * multiplier @@ -199,7 +196,7 @@ def distribute_memory_over_services(service_config, total_memory): return service_memory_config -def build_supervise_script_arguments(service_config, service_memory_config, root_config_path, run_zk): +def build_supervise_script_arguments(service_list, service_memory_config, root_config_path, run_zk): argument_list = [] argument_list.append("\":verify bin/verify-java\"") @@ -210,23 +207,19 @@ def build_supervise_script_arguments(service_config, service_memory_config, root if run_zk: argument_list.append("\"!p10 zk bin/run-zk conf\"") - for item in service_config: - service = item[0] + for service in service_list: prefix = '' if service == MIDDLE_MANAGER_SERVICE_NAME: prefix = '!p90 ' - if item[1] == "": - service_path = item[0] - else: - service_path = item[1] - jvm_args = service_memory_config.get(item[0]) + + jvm_args = service_memory_config.get(service) if jvm_args is None: - argument_list.append(f'\"{prefix}{service} bin/run-druid {service} {root_config_path} {service_path}\"') + argument_list.append(f'\"{prefix}{service} bin/run-druid {service} {root_config_path}\"') else: - argument_list.append(f'\"{prefix}{service} bin/run-druid {service} {root_config_path} {service_path} \'{jvm_args}\'\"') + argument_list.append(f'\"{prefix}{service} bin/run-druid {service} {root_config_path} \'{jvm_args}\'\"') - print('Command for supervise script:') + print('Supervise script command:') for item in argument_list: print(item) @@ -234,73 +227,75 @@ def build_supervise_script_arguments(service_config, service_memory_config, root return ",".join(argument_list) -def print_service_config(service_config, root_config_path, run_zk): - print('Services to start:') - for item in service_config: - if item[1] == "": - print(f'{item[0]}, using default config from {os.getcwd()}/../{QUICKSTART_BASE_CONFIG_PATH}') - else: - print(f'{item[0]}, using config from {root_config_path}/{item[1]}') - if run_zk: - print(f'zk, using default config from {os.getcwd()}/../conf/zk') - print('\n') - def display_help(): text = """ Usage: start-druid [options] where options include: - totalMemory= - memory for druid cluster, if totalMemory is not specified + totalMemory= + Integer value is supported with 'm' or 'g' suffix. + Memory for druid services, if totalMemory is not specified 80 percent of system memory is used. Note, if service specific jvm config is present, - totalMemory shouldn't be specified. - Integer value is supported with `m` or `g` suffix. - Memory should be greater than equals 2g - rootConfigPath= - directory containing common and service specific - properties to be overridden, this directory must contain `_common` - directory with `common.jvm.config` & `common.runtime.properties` - if `rootConfigPath` is not specified, config from - conf/druid/single-server/quickstart directory is used - computeOnly - command dry-run, validates the arguments and - display the memory distribution for services + shouldn't be specified. + should be greater than equals 3g + rootConfigPath= + Directory containing common and service specific + properties to be overridden, this directory must contain '_common' + directory with 'common.jvm.config' & 'common.runtime.properties' + files. + If is not specified, config from + 'conf/druid/single-server/quickstart' directory is used. + This path is relative to current working directory + services= + Value is comma separated string. + List of services to be started, should be a subset of + [broker, router, middleManager, historical, coordinator-overlord]. + If runtime or jvm properties are to be overridden, they should be + kept within /. + If is not specified config files can also be + placed within `conf/druid/single-server/quickstart/` + directory. + Note, if jvm.config file is present for one of the services, + it must be present for all services. + If argument is not given, all services + alongwith zookeeper is started. runZk - specification to run zookeeper, zk config is picked up from conf/zk - [=subdirectory] - service_identifier is the service to be started, multiple services - can be specified, `service_identifier` should be one of - [broker, router, middleManager, historical, coordinator-overlord] - `subdirectory` is optional directory within `rootConfigPath` - containing runtime properties or/and jvm properties - Note, if jvm.config file is present for one of the service, - it must be present for all other services. - If service is not explicitly specified, all services - alongwith zookeeper is started - Note each service should be specified at most once + Specification to run zookeeper, zk config is picked up from conf/zk. + computeOnly + Validate the arguments and display memory distribution for services. sample usage: start-druid - start up all the services (including zk) using the default system memory + Start up all the services (including zk) + using 80% of system memory. start-druid totalMemory=100g - start up all the services using the given memory + Start up all the services (including zk) + using the given memory. start-druid totalMemory=100g computeOnly - compute memory distribution for all the services - start-druid totalMemory=100g broker router historical - starts `broker`, `router` and `historical` services, using `100g` of memory - start-druid totalMemory=100g rootConfigPath=../conf/druid/single-server/large broker router historical - start `broker`, `router` and `historical` service, using 100g of memory, - use common configs from specified `rootConfigPath` - start-druid totalMemory=100g rootConfigPath=../conf/druid/single-server/large broker=broker router=router historical=historical - starts `broker`, `router` and `historical` services, using 100g of memory, use common configs - from specified `rootConfigPath`, use service specific config from specified sub directories - Since totalMemory is specific, the service specific folder shouldn't contain jvm.config - start-druid rootConfigPath=../conf/druid/profile broker=broker historical=historical - exception is thrown if either of `broker`, `historical` - subdirectory contains jvm.config but not both. - If none of the service specific subdirectory contains jvm.config, - memory distribution is computed + Compute memory distribution and validate + arguments for starting all the services. + start-druid totalMemory=100g services=broker,router + Start `broker` & `router` service, using `100g` of memory. + Read config from conf/druid/single-server/quickstart. + start-druid totalMemory=100g rootConfigPath=conf/druid/single-server/custom services=broker,router + Start `broker` & `router` service, using 100g of memory. + Read config from . + Since is specified, exception is thrown if + jvm.config is present for any of the services. + start-druid rootConfigPath=conf/druid/single-server/custom services=broker,router + Start `broker` & `router` service, + using 80% of system memory. + If jvm.config is specified for both the + services within /, + memory distribution is not calculated. + If jvm.config is present for either of the services, + exception is thrown. + If jvm.config is not present for both of the services, + memory distribution is calculated. + start-druid totalMemory=100g rootConfigPath=conf/druid/single-server/custom services=broker,router runZk + Start zookeeper alongwith other services. + zk config is read from conf/zk. """ print(text) @@ -314,23 +309,24 @@ def main(): print("Druid quickstart\n") - root_config_path, total_memory, service_config, run_zk, compute_only = parse_arguments() + root_config_path, total_memory, service_list, run_zk, compute_only = parse_arguments() # change directory to bin os.chdir(os.path.dirname(sys.argv[0])) - print(f'Arguments passed: rootConfigPath: "{root_config_path}", totalMemory: "{total_memory}"\n') - print_service_config(service_config, root_config_path, run_zk) + if root_config_path == "": + root_config_path = pathlib.Path(f'{os.getcwd()}/../{QUICKSTART_BASE_CONFIG_PATH}').resolve() + + print_startup_config(service_list, root_config_path, run_zk) service_memory_config = {} - if (should_compute_memory(root_config_path, total_memory, service_config)): + if (should_compute_memory(root_config_path, total_memory, service_list)): memory_in_mega_bytes = convert_total_memory_string(total_memory) - service_memory_config = distribute_memory_over_services(service_config, memory_in_mega_bytes) - - if root_config_path == "": - root_config_path = QUICKSTART_BASE_CONFIG_PATH + service_memory_config = distribute_memory_over_services(service_list, memory_in_mega_bytes) + else: + print('not computing memory distribution, reading memory specification from service jvm.config\n') - script_arguments = build_supervise_script_arguments(service_config, service_memory_config, root_config_path, run_zk) + script_arguments = build_supervise_script_arguments(service_list, service_memory_config, root_config_path, run_zk) if compute_only: return diff --git a/examples/bin/run-druid b/examples/bin/run-druid index 2726477684df..4d050d3d5712 100755 --- a/examples/bin/run-druid +++ b/examples/bin/run-druid @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -if [ "$#" -gt 4 ] || [ "$#" -eq 0 ] +if [ "$#" -gt 3 ] || [ "$#" -eq 0 ] then >&2 echo "usage: $0 [conf-dir]" exit 1 @@ -47,25 +47,18 @@ if [ ! -d "$LOG_DIR" ]; then mkdir -p $LOG_DIR; fi echo "Running [$1], logging to [$LOG_DIR/$1.log] if no changes made to log4j2.xml" -if [ "$WHATAMI" = 'coordinator-overlord' ] -then - SERVER_NAME=coordinator -else - SERVER_NAME="$WHATAMI" -fi - if [ "$#" -eq 3 ] then - # arguments for this case is - cd "$WHEREAMI/.." - exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$3"/jvm.config | xargs` \ - -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" org.apache.druid.cli.Main server $SERVER_NAME -elif [ "$#" -eq 4 ] -then - # arguments for this case is JVMARGS=`cat "$CONFDIR/_common/common.jvm.config" | xargs` JVMARGS+=' ' - JVMARGS+=$4 + JVMARGS+=$3 + + if [ "$WHATAMI" = 'coordinator-overlord' ] + then + SERVER_NAME=coordinator + else + SERVER_NAME="$WHATAMI" + fi cd "$WHEREAMI/.." exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" $JVMARGS \ From e5706066c56527759c814a832eb226828cad90d9 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sat, 19 Nov 2022 01:40:57 +0530 Subject: [PATCH 10/82] restore changes in medium profile --- .../single-server/medium/broker/runtime.properties | 10 ++++++++++ .../single-server/medium/historical/runtime.properties | 8 ++++++++ .../single-server/medium/router/runtime.properties | 6 ++++++ 3 files changed, 24 insertions(+) diff --git a/examples/conf/druid/single-server/medium/broker/runtime.properties b/examples/conf/druid/single-server/medium/broker/runtime.properties index 32dff403fa10..d46d6de70514 100644 --- a/examples/conf/druid/single-server/medium/broker/runtime.properties +++ b/examples/conf/druid/single-server/medium/broker/runtime.properties @@ -20,6 +20,16 @@ druid.service=druid/broker druid.plaintextPort=8082 +# HTTP server settings +druid.server.http.numThreads=60 + +# HTTP client settings +druid.broker.http.numConnections=50 +druid.broker.http.maxQueuedBytes=10MiB + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500MiB +druid.processing.numMergeBuffers=4 druid.processing.tmpDir=var/druid/processing # Query cache disabled -- push down caching and merging instead diff --git a/examples/conf/druid/single-server/medium/historical/runtime.properties b/examples/conf/druid/single-server/medium/historical/runtime.properties index 82443c1c43c2..a7a640b8061d 100644 --- a/examples/conf/druid/single-server/medium/historical/runtime.properties +++ b/examples/conf/druid/single-server/medium/historical/runtime.properties @@ -20,6 +20,13 @@ druid.service=druid/historical druid.plaintextPort=8083 +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes=500MiB +druid.processing.numMergeBuffers=4 +druid.processing.numThreads=15 druid.processing.tmpDir=var/druid/processing # Segment storage @@ -29,3 +36,4 @@ druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g" druid.historical.cache.useCache=true druid.historical.cache.populateCache=true druid.cache.type=caffeine +druid.cache.sizeInBytes=256MiB diff --git a/examples/conf/druid/single-server/medium/router/runtime.properties b/examples/conf/druid/single-server/medium/router/runtime.properties index 3858dec044bd..497d3b4d3aa1 100644 --- a/examples/conf/druid/single-server/medium/router/runtime.properties +++ b/examples/conf/druid/single-server/medium/router/runtime.properties @@ -20,6 +20,12 @@ druid.service=druid/router druid.plaintextPort=8888 +# HTTP proxy +druid.router.http.numConnections=50 +druid.router.http.readTimeout=PT5M +druid.router.http.numMaxThreads=100 +druid.server.http.numThreads=100 + # Service discovery druid.router.defaultBrokerServiceName=druid/broker druid.router.coordinatorServiceName=druid/coordinator From b413d159ef0ec9b9cba3d01518bffbbd622da7a5 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 21 Nov 2022 11:53:52 +0530 Subject: [PATCH 11/82] run-druid refactor --- examples/bin/run-druid | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/examples/bin/run-druid b/examples/bin/run-druid index 4d050d3d5712..258f8376d0d2 100755 --- a/examples/bin/run-druid +++ b/examples/bin/run-druid @@ -47,25 +47,35 @@ if [ ! -d "$LOG_DIR" ]; then mkdir -p $LOG_DIR; fi echo "Running [$1], logging to [$LOG_DIR/$1.log] if no changes made to log4j2.xml" +if [ "$WHATAMI" = 'coordinator-overlord' ] +then + SERVER_NAME=coordinator +else + SERVER_NAME="$WHATAMI" +fi + + +if [ ! -f "$CONFDIR"/$WHATAMI/main.config ]; + then + MAIN_CLASS="org.apache.druid.cli.Main server $SERVER_NAME" + else + MAIN_CLASS=`cat "$CONFDIR"/$WHATAMI/main.config | xargs` +fi + +cd "$WHEREAMI/.." + if [ "$#" -eq 3 ] +then + +elif [ "$#" -eq 4 ] then JVMARGS=`cat "$CONFDIR/_common/common.jvm.config" | xargs` JVMARGS+=' ' JVMARGS+=$3 - if [ "$WHATAMI" = 'coordinator-overlord' ] - then - SERVER_NAME=coordinator - else - SERVER_NAME="$WHATAMI" - fi - - cd "$WHEREAMI/.." exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" $JVMARGS \ - -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" org.apache.druid.cli.Main server $SERVER_NAME + -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" $MAIN_CLASS else - cd "$WHEREAMI/.." exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ - -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" \ - `cat "$CONFDIR"/$WHATAMI/main.config | xargs` + -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" $MAIN_CLASS fi From 9b8cdc6197f9aeec7008c880e64f7d8916c10c9a Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 22 Nov 2022 11:09:37 +0530 Subject: [PATCH 12/82] compute and pass middle manager runtime properties to run-druid supervise script changes to process java opts array use argparse, leave free memory, logging --- examples/bin/druid-quickstart.py | 337 -------------- examples/bin/run-druid | 27 +- examples/bin/start-druid | 2 +- examples/bin/start-druid-main | 439 ++++++++++++++++++ examples/bin/supervise | 6 +- .../middleManager/runtime.properties | 6 +- 6 files changed, 467 insertions(+), 350 deletions(-) delete mode 100644 examples/bin/druid-quickstart.py create mode 100644 examples/bin/start-druid-main diff --git a/examples/bin/druid-quickstart.py b/examples/bin/druid-quickstart.py deleted file mode 100644 index 0b2da6658771..000000000000 --- a/examples/bin/druid-quickstart.py +++ /dev/null @@ -1,337 +0,0 @@ -import sys -import os -import psutil -import pathlib -from pathlib import Path - -QUICKSTART_BASE_CONFIG_PATH = "conf/druid/single-server/quickstart" -HELP_ARG_IDENTIFIER = "help" -COMPUTE_ONLY_ARG_IDENTIFIER = "computeOnly" -RUN_ZK_IDENTIFIER = "runZk" -ROOT_CONFIG_PATH_IDENTIFIER = "rootConfigPath" -SERVICES_IDENTIFIER = "services" -MEMORY_ARG_IDENTIFIER = "totalMemory" -MEMORY_GIGABYTES_IDENTIFIER = "g" -MEMORY_MEGABYTES_IDENTIFIER = "m" -ARG_SEPARATOR = "=" -SERVICE_SEPARATOR = "," - -BROKER_SERVICE_NAME = "broker" -ROUTER_SERVICE_NAME = "router" -COORDINATOR_SERVICE_NAME = "coordinator-overlord" -HISTORICAL_SERVICE_NAME = "historical" -MIDDLE_MANAGER_SERVICE_NAME = "middleManager" - -DEFAULT_SERVICES = [ - BROKER_SERVICE_NAME, - ROUTER_SERVICE_NAME, - COORDINATOR_SERVICE_NAME, - HISTORICAL_SERVICE_NAME, - MIDDLE_MANAGER_SERVICE_NAME -] - -SERVICE_MEMORY_DISTRIBUTION_WEIGHT = { - MIDDLE_MANAGER_SERVICE_NAME: 1, - ROUTER_SERVICE_NAME: 2, - COORDINATOR_SERVICE_NAME: 36, - BROKER_SERVICE_NAME: 56, - HISTORICAL_SERVICE_NAME: 90 -} - -SERVICE_MEMORY_LOWER_BOUND = { - MIDDLE_MANAGER_SERVICE_NAME: 64, - ROUTER_SERVICE_NAME: 128 -} - -SERVICE_MEMORY_HEAP_PERCENTAGE = { - MIDDLE_MANAGER_SERVICE_NAME: 1, - ROUTER_SERVICE_NAME: 1, - COORDINATOR_SERVICE_NAME: 1, - BROKER_SERVICE_NAME: 0.60, - HISTORICAL_SERVICE_NAME: 0.40 -} - -def check_argument_type(argument, type): - split_args = argument.split(ARG_SEPARATOR) - return split_args[0] == type - -def get_argument_value(argument): - split_args = argument.split(ARG_SEPARATOR) - return split_args[1] - -def parse_arguments(): - service_list = [] - root_config_path = "" - total_memory = "" - compute_only = False - run_zk = False - - for argument in sys.argv[1:]: - if check_argument_type(argument, COMPUTE_ONLY_ARG_IDENTIFIER): - compute_only = True - elif check_argument_type(argument, RUN_ZK_IDENTIFIER): - run_zk = True - elif check_argument_type(argument, ROOT_CONFIG_PATH_IDENTIFIER): - root_config_path = os.path.join(os.getcwd(), get_argument_value(argument)) - if os.path.exists(root_config_path) is False: - raise Exception(f'rootConfigPath `{root_config_path}` doesn\'t exist') - elif (check_argument_type(argument, MEMORY_ARG_IDENTIFIER)): - total_memory = get_argument_value(argument) - elif (check_argument_type(argument, SERVICES_IDENTIFIER)): - split_args = argument.split(ARG_SEPARATOR) - services_arg = split_args[1] - services = services_arg.split(SERVICE_SEPARATOR) - - for service in services: - if service not in DEFAULT_SERVICES: - raise Exception(f'{service} is not a valid service name, should be one of {DEFAULT_SERVICES}') - - if service in service_list: - raise Exception(f'{service} is specified multiple times') - - service_list.append(service) - - if len(service_list) == 0: - # start all services - service_list = DEFAULT_SERVICES - run_zk = True - - return root_config_path, total_memory, service_list, run_zk, compute_only - -def print_startup_config(service_list, root_config_path, run_zk): - print(f'starting {service_list}, using config from {root_config_path}') - if run_zk: - zk_config_path = pathlib.Path(f'{os.getcwd()}/../conf/zk').resolve() - print(f'starting zk, using default config from {zk_config_path}') - print('\n') - -def should_compute_memory(root_config_path, total_memory, service_list): - # if jvm file is present for any of the services - # it should be present for all services and totalMemory should not be specified - # if totalMemory is given, jvm file shouldn't be present for any service - - jvm_config_count = 0 - for service in service_list: - if Path(f'{root_config_path}/{service}/jvm.config').is_file(): - jvm_config_count += 1 - elif jvm_config_count > 0: - raise Exception('jvm.config file is missing for service {service}, jvm.config should be specified for all the services or none') - - if jvm_config_count > 0 and (jvm_config_count != len(service_list) or total_memory != ""): - if jvm_config_count != len(service_list): - raise Exception("jvm.config file should be present for all services or none") - if total_memory != "": - raise Exception("If jvm.config is given for services, `totalMemory` argument shouldn't be specified") - - return jvm_config_count == 0 - -def compute_system_memory(): - system_memory = psutil.virtual_memory().total # mem in bytes - memory_for_druid = int((system_memory * 0.8) / (1024 * 1024)) - return memory_for_druid - -def convert_total_memory_string(memory): - if memory == "": - computed_memory = compute_system_memory() - print(f'`{MEMORY_ARG_IDENTIFIER}` argument is not specified, Druid will use 80% of system memory: {computed_memory}m') - return computed_memory - elif memory.endswith(MEMORY_MEGABYTES_IDENTIFIER): - return int(memory[:-1]) - elif memory.endswith(MEMORY_GIGABYTES_IDENTIFIER): - return 1024 * int(memory[:-1]) - else: - raise Exception('Incorrect format for totalMemory argument, expected format is ') - -def build_memory_config_string(heap_memory, direct_memory): - if direct_memory == 0: - return f'-Xms{heap_memory}m -Xmx{heap_memory}m' - return f'-Xms{heap_memory}m -Xmx{heap_memory}m -XX:MaxDirectMemorySize={direct_memory}m' - -def distribute_memory_over_services(service_list, total_memory): - service_memory_config = {} - - memory_weight_sum = 0 - for service in service_list: - memory_weight_sum += SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) - - multiplier = total_memory / memory_weight_sum - - lower_bound_memory_allocation = 0 - allocated_services = set() - - for service in service_list: - allocated_memory = SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) * multiplier - if service in SERVICE_MEMORY_LOWER_BOUND and allocated_memory < SERVICE_MEMORY_LOWER_BOUND.get(service): - allocated_memory = SERVICE_MEMORY_LOWER_BOUND.get(service) - heap_memory = SERVICE_MEMORY_HEAP_PERCENTAGE.get(service) * allocated_memory - direct_memory = allocated_memory - heap_memory - service_memory_config[service] = build_memory_config_string(int(heap_memory), int(direct_memory)) - lower_bound_memory_allocation += allocated_memory - allocated_services.add(service) - - if lower_bound_memory_allocation > 0: - # compute the multiplier again for remaing services - memory_weight_sum = 0 - for service in service_list: - if service in allocated_services: - continue - memory_weight_sum += SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) - multiplier = (total_memory - lower_bound_memory_allocation) / memory_weight_sum - - for service in service_list: - if service in allocated_services: - continue - allocated_memory = SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) * multiplier - if service in SERVICE_MEMORY_LOWER_BOUND and allocated_memory < SERVICE_MEMORY_LOWER_BOUND.get(service): - allocated_memory = SERVICE_MEMORY_LOWER_BOUND.get(service) - - heap_memory = SERVICE_MEMORY_HEAP_PERCENTAGE.get(service) * allocated_memory - direct_memory = allocated_memory - heap_memory - service_memory_config[service] = build_memory_config_string(int(heap_memory), int(direct_memory)) - - print(f'\nMemory distribution for services:') - for key, value in service_memory_config.items(): - print(f'{key}, memory_config: {value}') - print('\n') - - return service_memory_config - -def build_supervise_script_arguments(service_list, service_memory_config, root_config_path, run_zk): - argument_list = [] - - argument_list.append("\":verify bin/verify-java\"") - argument_list.append("\":verify bin/verify-default-ports\"") - argument_list.append("\":notify bin/greet\"") - argument_list.append("\":kill-timeout 10\"") - - if run_zk: - argument_list.append("\"!p10 zk bin/run-zk conf\"") - - for service in service_list: - prefix = '' - if service == MIDDLE_MANAGER_SERVICE_NAME: - prefix = '!p90 ' - - jvm_args = service_memory_config.get(service) - - if jvm_args is None: - argument_list.append(f'\"{prefix}{service} bin/run-druid {service} {root_config_path}\"') - else: - argument_list.append(f'\"{prefix}{service} bin/run-druid {service} {root_config_path} \'{jvm_args}\'\"') - - print('Supervise script command:') - for item in argument_list: - print(item) - - print('\n') - - return ",".join(argument_list) - -def display_help(): - text = """ - Usage: start-druid [options] - - where options include: - totalMemory= - Integer value is supported with 'm' or 'g' suffix. - Memory for druid services, if totalMemory is not specified - 80 percent of system memory is used. - Note, if service specific jvm config is present, - shouldn't be specified. - should be greater than equals 3g - rootConfigPath= - Directory containing common and service specific - properties to be overridden, this directory must contain '_common' - directory with 'common.jvm.config' & 'common.runtime.properties' - files. - If is not specified, config from - 'conf/druid/single-server/quickstart' directory is used. - This path is relative to current working directory - services= - Value is comma separated string. - List of services to be started, should be a subset of - [broker, router, middleManager, historical, coordinator-overlord]. - If runtime or jvm properties are to be overridden, they should be - kept within /. - If is not specified config files can also be - placed within `conf/druid/single-server/quickstart/` - directory. - Note, if jvm.config file is present for one of the services, - it must be present for all services. - If argument is not given, all services - alongwith zookeeper is started. - runZk - Specification to run zookeeper, zk config is picked up from conf/zk. - computeOnly - Validate the arguments and display memory distribution for services. - - sample usage: - start-druid - Start up all the services (including zk) - using 80% of system memory. - start-druid totalMemory=100g - Start up all the services (including zk) - using the given memory. - start-druid totalMemory=100g computeOnly - Compute memory distribution and validate - arguments for starting all the services. - start-druid totalMemory=100g services=broker,router - Start `broker` & `router` service, using `100g` of memory. - Read config from conf/druid/single-server/quickstart. - start-druid totalMemory=100g rootConfigPath=conf/druid/single-server/custom services=broker,router - Start `broker` & `router` service, using 100g of memory. - Read config from . - Since is specified, exception is thrown if - jvm.config is present for any of the services. - start-druid rootConfigPath=conf/druid/single-server/custom services=broker,router - Start `broker` & `router` service, - using 80% of system memory. - If jvm.config is specified for both the - services within /, - memory distribution is not calculated. - If jvm.config is present for either of the services, - exception is thrown. - If jvm.config is not present for both of the services, - memory distribution is calculated. - start-druid totalMemory=100g rootConfigPath=conf/druid/single-server/custom services=broker,router runZk - Start zookeeper alongwith other services. - zk config is read from conf/zk. - """ - - print(text) - - -def main(): - for argument in sys.argv[1:]: - if check_argument_type(argument, HELP_ARG_IDENTIFIER): - display_help() - return - - print("Druid quickstart\n") - - root_config_path, total_memory, service_list, run_zk, compute_only = parse_arguments() - - # change directory to bin - os.chdir(os.path.dirname(sys.argv[0])) - - if root_config_path == "": - root_config_path = pathlib.Path(f'{os.getcwd()}/../{QUICKSTART_BASE_CONFIG_PATH}').resolve() - - print_startup_config(service_list, root_config_path, run_zk) - - service_memory_config = {} - if (should_compute_memory(root_config_path, total_memory, service_list)): - memory_in_mega_bytes = convert_total_memory_string(total_memory) - service_memory_config = distribute_memory_over_services(service_list, memory_in_mega_bytes) - else: - print('not computing memory distribution, reading memory specification from service jvm.config\n') - - script_arguments = build_supervise_script_arguments(service_list, service_memory_config, root_config_path, run_zk) - - if compute_only: - return - - os.system(f'exec ./supervise -a {script_arguments}') - -if __name__ == '__main__': - main() diff --git a/examples/bin/run-druid b/examples/bin/run-druid index 258f8376d0d2..9e9f93d90deb 100755 --- a/examples/bin/run-druid +++ b/examples/bin/run-druid @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -if [ "$#" -gt 3 ] || [ "$#" -eq 0 ] +if [ "$#" -gt 5 ] || [ "$#" -eq 0 ] then >&2 echo "usage: $0 [conf-dir]" exit 1 @@ -64,18 +64,29 @@ fi cd "$WHEREAMI/.." -if [ "$#" -eq 3 ] -then - -elif [ "$#" -eq 4 ] +if [ "$#" -eq 3 ] || [ "$#" -eq 5 ] then + # args: or JVMARGS=`cat "$CONFDIR/_common/common.jvm.config" | xargs` JVMARGS+=' ' JVMARGS+=$3 - exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" $JVMARGS \ - -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" $MAIN_CLASS + if [ "$#" -eq 3 ] + then + exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 $3 "-Ddruid.log.path=$LOG_DIR" $JVMARGS \ + -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" org.apache.druid.cli.Main server $SERVER_NAME + else + exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 $3 "-Ddruid.log.path=$LOG_DIR" $4 $5 $JVMARGS \ + -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" org.apache.druid.cli.Main server $SERVER_NAME + fi +elif [ "$#" -eq 4 ] +then + # args: + exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 $3 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ + -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" org.apache.druid.cli.Main server $SERVER_NAME else + # args: exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ - -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" $MAIN_CLASS + -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" \ + $MAIN_CLASS fi diff --git a/examples/bin/start-druid b/examples/bin/start-druid index 41639d8cdce6..601df55ffef5 100644 --- a/examples/bin/start-druid +++ b/examples/bin/start-druid @@ -21,4 +21,4 @@ PWD="$(pwd)" WHEREAMI="$(dirname "$0")" WHEREAMI="$(cd "$WHEREAMI" && pwd)" -python3 "$WHEREAMI"/druid-quickstart.py $@ +exec "$WHEREAMI/start-druid-main" "$@" diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main new file mode 100644 index 000000000000..b74d43336c98 --- /dev/null +++ b/examples/bin/start-druid-main @@ -0,0 +1,439 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import sys +import os +import psutil +import pathlib +import multiprocessing +import argparse + +QUICKSTART_ROOT_CONFIG_PATH = "conf/druid/single-server/quickstart" + +MEMORY_GIGABYTES_IDENTIFIER = "g" +MEMORY_MEGABYTES_IDENTIFIER = "m" +SERVICE_SEPARATOR = "," + +MM_TASK_JAVAOPTS_ARRAY = ["-server", "-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] +MM_TASK_MEMORY_TYPE_LOW = "low" +MM_TASK_MEMORY_TYPE_HIGH = "high" +MM_TASK_MEM_MAP = { + MM_TASK_MEMORY_TYPE_LOW: ["-Xms256m", "-Xmx256m", "-XX:MaxDirectMemorySize=256g"], + MM_TASK_MEMORY_TYPE_HIGH: ["-Xms2g", "-Xmx2g", "-XX:MaxDirectMemorySize=2g"] +} + +MM_TASK_JAVAOPTS_PROP = "-Ddruid.indexer.runner.javaOptsArray" +MM_TASK_WORKER_CAPACITY_PROP = "-Ddruid.worker.capacity" + +BROKER = "broker" +ROUTER = "router" +COORDINATOR = "coordinator-overlord" +HISTORICAL = "historical" +MM = "middleManager" +MM_TASK = "middleManager-task" +MM_TASK_COUNT = "task-count" + +DEFAULT_SERVICES = [ + BROKER, + ROUTER, + COORDINATOR, + HISTORICAL, + MM +] + +SERVICE_MEMORY_DISTRIBUTION_WEIGHT = { + MM: 0.5, + ROUTER: 1, + COORDINATOR: 15, + BROKER: 23, + HISTORICAL: 40, + MM_TASK: 15 +} + +SERVICE_MEMORY_LOWER_BOUND = { + MM: 64, + ROUTER: 128, + MM_TASK: 1024, + BROKER: 900, + COORDINATOR: 256, + HISTORICAL: 900 +} + +SERVICE_MEMORY_HEAP_PERCENTAGE = { + MM: 1, + ROUTER: 1, + COORDINATOR: 1, + BROKER: 0.60, + HISTORICAL: 0.40, + MM_TASK: 0.50 +} + +LOGGING_ENABLED = False + +def custom_print(message): + if LOGGING_ENABLED: + print(message) + +def error_and_exit(message): + sys.stderr.write(message + '\n') + sys.exit(1) + +def configure_parser(): + parser = argparse.ArgumentParser( + prog='Druid quickstart', + formatter_class=argparse.RawTextHelpFormatter, + epilog= +""" +sample usage: + start-druid + Start up all the services (including zk). + 50 - 80 percent of system memory is used. + start-druid -m=100g + Start up all the services (including zk) + using the given memory. + start-druid -m=100g --compute_only + Compute memory distribution and validate + arguments. + start-druid -m=100g -sl=broker,router + Start broker & router service, using 100g of memory. + Read config from conf/druid/single-server/quickstart. + start-druid -m=100g --sl=broker,router \\ + -cp=conf/druid/single-server/custom + Start broker & router service, using 100g of memory. + Read config from . + Since is specified, exception is thrown if + jvm.config is present for any of the services. + start-druid -sl=broker,router \\ + -cp=conf/druid/single-server/custom + Start broker & router service, using system memory. + If jvm.config is specified for both the + services within /, + memory distribution is not calculated. + If jvm.config is present for either of the services, + exception is thrown. + If jvm.config is not present for both of the services, + memory distribution is calculated. + start-druid -m=100g \\ + -cp=conf/druid/single-server/custom \\ + -sl=broker,router \\ + --run_zk + Start broker, router and zookeeper. + zk config is read from conf/zk. +""" + ) + parser.add_argument('--memory', '-m', type=str, required=False, + help='Total memory for all processes (services and tasks, if any). \n' + 'This parameter is ignored if each service already has a jvm.config \n' + 'in the given conf directory. e.g. 500m, 4g, 6g\n') + parser.add_argument('--service_list', '-sl', type=str, required=False, + help='List of services to be started, subset of \n' + '{broker, router, middleManager, historical, coordinator-overlord}. \n' + 'If the argument is not given, all services \n' + 'and zookeeper is started. e.g. -sl=broker,historical') + parser.add_argument('--config_path', '-cp', type=str, required=False, + help='Relative path to the directory containing common and service \n' + 'specific properties to be overridden. \n' + 'This directory must contain \'_common\' directory with \n' + '\'common.jvm.config\' & \'common.runtime.properties\' files. \n' + 'If this argument is not given, config from \n' + '\'conf/druid/single-server/quickstart\' directory is used.\n') + parser.add_argument('--compute_only', action='store_true', + help='Validate the arguments and display memory distribution') + parser.add_argument('--run_zk', action='store_true', + help='Specification to run zookeeper, \n' + 'zk config is picked up from conf/zk.') + parser.add_argument('--verbose', action='store_true', help='Log details.') + + parser.set_defaults(run_zk=False) + parser.set_defaults(compute_only=False) + parser.set_defaults(verbose=False) + + return parser + +def parse_arguments(args): + service_list = [] + config_path = "" + total_memory = "" + compute_only = False + run_zk = False + + if args.compute_only: + compute_only = True + if args.run_zk: + run_zk = True + if args.config_path is not None: + config_path = os.path.join(os.getcwd(), args.config_path) + if os.path.exists(config_path) is False: + error_and_exit(f'config_path {config_path} doesn\'t exist') + if args.memory is not None: + total_memory = args.memory + if args.service_list is not None: + services = args.service_list.split(SERVICE_SEPARATOR) + + for service in services: + if service not in DEFAULT_SERVICES: + error_and_exit(f'Invalid service name {service}, should be one of {DEFAULT_SERVICES}') + + if service in service_list: + error_and_exit(f'{service} is specified multiple times') + + service_list.append(service) + + if len(service_list) == 0: + # start all services + service_list = DEFAULT_SERVICES + run_zk = True + + return config_path, total_memory, service_list, run_zk, compute_only + +def print_startup_config(service_list, config_path, run_zk): + custom_print(f'starting {service_list}') + custom_print(f'reading config from {config_path}') + if run_zk: + zk_config_path = pathlib.Path(f'{os.getcwd()}/../conf/zk').resolve() + custom_print(f'starting zk, reading default config from {zk_config_path}') + custom_print('\n') + +def should_compute_memory(config_path, total_memory, service_list): + # if jvm file is present for any of the services + # it should be present for all services and memory should not be specified + # if memory is given, jvm file shouldn't be present for any service + + jvm_config_count = 0 + for service in service_list: + if pathlib.Path(f'{config_path}/{service}/jvm.config').is_file(): + jvm_config_count += 1 + elif jvm_config_count > 0: + error_and_exit(f'jvm.config file is missing for service {service}, jvm.config should be specified for all the services or none') + + if jvm_config_count > 0 and (jvm_config_count != len(service_list) or total_memory != ""): + if jvm_config_count != len(service_list): + error_and_exit("jvm.config file should be present for all services or none") + if total_memory != "": + error_and_exit("If jvm.config is given for services, memory argument shouldn't be specified") + + return jvm_config_count == 0 + +def check_memory_constraint(total_memory, service_list): + # sum of lower bound service memory should be >= 80% of total memory + lower_bound_memory = 0 + + for service in service_list: + lower_bound_memory += SERVICE_MEMORY_LOWER_BOUND.get(service) + + required_memory = int(lower_bound_memory / 0.8) + + if total_memory < required_memory: + error_and_exit(f'Minimum memory required for starting services is {required_memory}m') + + if total_memory > 2 * required_memory: + return int(total_memory / 2) + else: + return required_memory + +def compute_system_memory(): + system_memory = psutil.virtual_memory().total # mem in bytes + memory_for_druid = int(system_memory / (1024 * 1024)) + return memory_for_druid + +def convert_total_memory_string(memory): + if memory == "": + computed_memory = compute_system_memory() + return computed_memory + elif memory.endswith(MEMORY_MEGABYTES_IDENTIFIER): + return int(memory[:-1]) + elif memory.endswith(MEMORY_GIGABYTES_IDENTIFIER): + return 1024 * int(memory[:-1]) + else: + error_and_exit('Incorrect format for memory argument, expected format is ') + +def build_mm_task_javaopts_array(memory_type): + task_memory = f'{MM_TASK_JAVAOPTS_PROP}=[' + + MEM_ARRAY = MM_TASK_MEM_MAP.get(memory_type) + + javaopts_list = MM_TASK_JAVAOPTS_ARRAY + MEM_ARRAY + + for item in javaopts_list: + task_memory += f'"\"{item}\";' + + task_memory = task_memory[:-1] + task_memory += ']' + return task_memory + +def build_memory_config_string(service, allocated_memory): + if service == MM_TASK: + if allocated_memory >= 2048: + task_count = int(allocated_memory / 2048) + memory_type = MM_TASK_MEMORY_TYPE_HIGH + task_memory = 2048 + else: + task_count = int(allocated_memory / 512) + memory_type = MM_TASK_MEMORY_TYPE_LOW + task_memory = 512 + task_count = min(task_count, multiprocessing.cpu_count()) + + javaopts_array = build_mm_task_javaopts_array(memory_type) + return [f'{MM_TASK_WORKER_CAPACITY_PROP}={task_count}', javaopts_array], task_memory * task_count + else: + heap_memory = SERVICE_MEMORY_HEAP_PERCENTAGE.get(service) * allocated_memory + direct_memory = int(allocated_memory - heap_memory) + heap_memory = int(heap_memory) + + if direct_memory == 0: + return f'-Xms{heap_memory}m -Xmx{heap_memory}m', allocated_memory + + return f'-Xms{heap_memory}m -Xmx{heap_memory}m -XX:MaxDirectMemorySize={direct_memory}m', allocated_memory + +def distribute_memory_over_services(service_list, total_memory): + service_memory_config = {} + + memory_weight_sum = 0 + for service in service_list: + memory_weight_sum += SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) + + multiplier = total_memory / memory_weight_sum + + lower_bound_memory_allocation = 0 + allocated_services = set() + + for service in service_list: + allocated_memory = SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) * multiplier + if service in SERVICE_MEMORY_LOWER_BOUND and allocated_memory < SERVICE_MEMORY_LOWER_BOUND.get(service): + allocated_memory = SERVICE_MEMORY_LOWER_BOUND.get(service) + service_memory_config[service], allocated_memory = build_memory_config_string(service, allocated_memory) + lower_bound_memory_allocation += allocated_memory + allocated_services.add(service) + + if lower_bound_memory_allocation > 0: + # compute the multiplier again for remaining services + memory_weight_sum = 0 + for service in service_list: + if service in allocated_services: + continue + memory_weight_sum += SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) + multiplier = (total_memory - lower_bound_memory_allocation) / memory_weight_sum + + for service in service_list: + if service in allocated_services: + continue + allocated_memory = SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) * multiplier + if service in SERVICE_MEMORY_LOWER_BOUND and allocated_memory < SERVICE_MEMORY_LOWER_BOUND.get(service): + allocated_memory = SERVICE_MEMORY_LOWER_BOUND.get(service) + + service_memory_config[service], allocated_memory = build_memory_config_string(service, allocated_memory) + + custom_print(f'\nMemory distribution for services:') + for key, value in service_memory_config.items(): + custom_print(f'{key}, memory_config: {value}') + custom_print('\n') + + return service_memory_config + +def build_supervise_script_arguments(service_list, service_memory_config, config_path, run_zk): + args = [] + commands = [] + args.append('supervise') + args.append('-a') + + commands.append(":verify bin/verify-java") + commands.append(":verify bin/verify-default-ports") + commands.append(":notify bin/greet") + commands.append(":kill-timeout 10") + + if run_zk: + commands.append("!p10 zk bin/run-zk conf") + + for service in service_list: + if service == MM_TASK: + continue + + prefix = '' + if service == MM: + prefix = '!p90 ' + + jvm_args = service_memory_config.get(service) + + if service == MM: + task_config = service_memory_config.get(MM_TASK) + task_count = task_config[0] + task_memory = task_config[1] + if jvm_args is None: + commands.append( + f'{prefix}{service} bin/run-druid {service} {config_path} \'{task_count}\' \'{task_memory}\'') + else: + commands.append( + f'{prefix}{service} bin/run-druid {service} {config_path} \'{task_count}\' \'{task_memory}\' \'{jvm_args}\'') + else: + if jvm_args is None: + commands.append(f'{prefix}{service} bin/run-druid {service} {config_path}') + else: + commands.append(f'{prefix}{service} bin/run-druid {service} {config_path} \'{jvm_args}\'') + + custom_print('Supervise script args:') + for item in commands: + custom_print(item) + + custom_print('\n') + + args.append(",".join(commands)) + return args + +def main(): + parser = configure_parser() + args = parser.parse_args() + + global LOGGING_ENABLED + LOGGING_ENABLED = args.verbose + + config_path, total_memory, service_list, run_zk, compute_only = parse_arguments(args) + + # change directory to bin + os.chdir(os.path.dirname(sys.argv[0])) + + if config_path == "": + config_path = pathlib.Path(f'{os.getcwd()}/../{QUICKSTART_ROOT_CONFIG_PATH}').resolve() + + print_startup_config(service_list, config_path, run_zk) + + service_memory_config = {} + + if MM in service_list: + service_list.append(MM_TASK) + + if (should_compute_memory(config_path, total_memory, service_list)): + memory_in_mega_bytes = convert_total_memory_string(total_memory) + custom_print(f'total memory is {memory_in_mega_bytes}m') + memory_to_be_used = check_memory_constraint(memory_in_mega_bytes, service_list) + custom_print(f'memory used for services & tasks {memory_to_be_used}m') + service_memory_config = distribute_memory_over_services(service_list, memory_to_be_used) + else: + custom_print('not computing memory distribution, reading memory specification from service jvm.config\n') + + script_arguments = build_supervise_script_arguments(service_list, service_memory_config, config_path, run_zk) + + if compute_only: + return + + os.execv('./supervise', script_arguments) + +try: + main() +except KeyboardInterrupt: + sys.exit(1) diff --git a/examples/bin/supervise b/examples/bin/supervise index 507795733c34..9a6809bacdc8 100755 --- a/examples/bin/supervise +++ b/examples/bin/supervise @@ -56,7 +56,6 @@ sub process_config for my $line (@lines) { if ($line =~ /^(:verify|:notify|:kill-timeout|(?:\!p[0-9]+\s+)?[^:]\S+)\s+(.+)$/) { - my $name = $1; my $order = 50; my $command = $2; @@ -211,6 +210,11 @@ if (not defined $opt{'conf'}) { @config_lines = read_config_file($opt{'conf'}); } +for my $alpha (@config_lines) +{ + $alpha =~ s/;/,/g; +} + my $config = process_config(@config_lines); @commands = @{$config->{commands}}; diff --git a/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties b/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties index e7262930f498..b0cf3ca3c43e 100644 --- a/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties @@ -21,15 +21,15 @@ druid.service=druid/middleManager druid.plaintextPort=8091 # Number of tasks per middleManager -druid.worker.capacity=2 +# druid.worker.capacity # Task launch parameters druid.indexer.runner.javaCommand=bin/run-java -druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] +#druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task # HTTP server threads -druid.server.http.numThreads=12 +#druid.server.http.numThreads=12 # Processing threads and buffers on Peons druid.indexer.fork.property.druid.processing.numMergeBuffers=2 From 75d169f15cbcd0a6fdc68f4068f14050c40909fd Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 22 Nov 2022 11:50:59 +0530 Subject: [PATCH 13/82] Remove extra quotes from mm task javaopts array --- examples/bin/start-druid-main | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index b74d43336c98..a1fe6af4985d 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -35,7 +35,7 @@ MM_TASK_MEMORY_TYPE_LOW = "low" MM_TASK_MEMORY_TYPE_HIGH = "high" MM_TASK_MEM_MAP = { MM_TASK_MEMORY_TYPE_LOW: ["-Xms256m", "-Xmx256m", "-XX:MaxDirectMemorySize=256g"], - MM_TASK_MEMORY_TYPE_HIGH: ["-Xms2g", "-Xmx2g", "-XX:MaxDirectMemorySize=2g"] + MM_TASK_MEMORY_TYPE_HIGH: ["-Xms1g", "-Xmx1g", "-XX:MaxDirectMemorySize=1g"] } MM_TASK_JAVAOPTS_PROP = "-Ddruid.indexer.runner.javaOptsArray" @@ -242,7 +242,7 @@ def check_memory_constraint(total_memory, service_list): if total_memory < required_memory: error_and_exit(f'Minimum memory required for starting services is {required_memory}m') - if total_memory > 2 * required_memory: + if total_memory >= 2 * required_memory: return int(total_memory / 2) else: return required_memory @@ -271,7 +271,7 @@ def build_mm_task_javaopts_array(memory_type): javaopts_list = MM_TASK_JAVAOPTS_ARRAY + MEM_ARRAY for item in javaopts_list: - task_memory += f'"\"{item}\";' + task_memory += f'\"{item}\";' task_memory = task_memory[:-1] task_memory += ']' From a891319e005eac2d09bbe9af8887536286ba9b60 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 22 Nov 2022 12:37:00 +0530 Subject: [PATCH 14/82] Update logic to compute minimum memory --- examples/bin/start-druid-main | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index a1fe6af4985d..ecd630c10501 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -230,23 +230,6 @@ def should_compute_memory(config_path, total_memory, service_list): return jvm_config_count == 0 -def check_memory_constraint(total_memory, service_list): - # sum of lower bound service memory should be >= 80% of total memory - lower_bound_memory = 0 - - for service in service_list: - lower_bound_memory += SERVICE_MEMORY_LOWER_BOUND.get(service) - - required_memory = int(lower_bound_memory / 0.8) - - if total_memory < required_memory: - error_and_exit(f'Minimum memory required for starting services is {required_memory}m') - - if total_memory >= 2 * required_memory: - return int(total_memory / 2) - else: - return required_memory - def compute_system_memory(): system_memory = psutil.virtual_memory().total # mem in bytes memory_for_druid = int(system_memory / (1024 * 1024)) @@ -263,6 +246,23 @@ def convert_total_memory_string(memory): else: error_and_exit('Incorrect format for memory argument, expected format is ') +def check_memory_constraint(total_memory, service_list): + # 80% of total memory >= sum of lower bound service memory should be + lower_bound_memory = 0 + + for service in service_list: + lower_bound_memory += SERVICE_MEMORY_LOWER_BOUND.get(service) + + required_memory = int(lower_bound_memory / 0.8) + + if total_memory < required_memory: + error_and_exit(f'Minimum memory required for starting services is {required_memory}m') + + if total_memory >= 2 * lower_bound_memory: + return int(total_memory / 2) + else: + return lower_bound_memory + def build_mm_task_javaopts_array(memory_type): task_memory = f'{MM_TASK_JAVAOPTS_PROP}=[' From 569275378487ddf6dd516a448f0c8f3edeb11a64 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 22 Nov 2022 15:29:04 +0530 Subject: [PATCH 15/82] simplify run-druid --- examples/bin/run-druid | 18 +++++------ examples/bin/start-druid-main | 59 +++++++++++++++++------------------ 2 files changed, 36 insertions(+), 41 deletions(-) diff --git a/examples/bin/run-druid b/examples/bin/run-druid index 9e9f93d90deb..73ce559ef481 100755 --- a/examples/bin/run-druid +++ b/examples/bin/run-druid @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -if [ "$#" -gt 5 ] || [ "$#" -eq 0 ] +if [ "$#" -gt 4 ] || [ "$#" -eq 0 ] then >&2 echo "usage: $0 [conf-dir]" exit 1 @@ -64,28 +64,26 @@ fi cd "$WHEREAMI/.." -if [ "$#" -eq 3 ] || [ "$#" -eq 5 ] +if [ "$#" -eq 3 ] || [ "$#" -eq 4 ] then - # args: or + # args: or JVMARGS=`cat "$CONFDIR/_common/common.jvm.config" | xargs` JVMARGS+=' ' JVMARGS+=$3 if [ "$#" -eq 3 ] then - exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 $3 "-Ddruid.log.path=$LOG_DIR" $JVMARGS \ + # args: + exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 $4 "-Ddruid.log.path=$LOG_DIR" $JVMARGS \ -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" org.apache.druid.cli.Main server $SERVER_NAME else - exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 $3 "-Ddruid.log.path=$LOG_DIR" $4 $5 $JVMARGS \ + # args: + exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 $4"-Ddruid.log.path=$LOG_DIR" -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 $JVMARGS \ -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" org.apache.druid.cli.Main server $SERVER_NAME fi -elif [ "$#" -eq 4 ] -then - # args: - exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 $3 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ - -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" org.apache.druid.cli.Main server $SERVER_NAME else # args: + echo "case parameters count 2, $#" exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" \ $MAIN_CLASS diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index ecd630c10501..86eec4b8eddf 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -158,7 +158,7 @@ sample usage: parser.add_argument('--run_zk', action='store_true', help='Specification to run zookeeper, \n' 'zk config is picked up from conf/zk.') - parser.add_argument('--verbose', action='store_true', help='Log details.') + parser.add_argument('--verbose', action='store_true', help='Log details') parser.set_defaults(run_zk=False) parser.set_defaults(compute_only=False) @@ -214,7 +214,6 @@ def should_compute_memory(config_path, total_memory, service_list): # if jvm file is present for any of the services # it should be present for all services and memory should not be specified # if memory is given, jvm file shouldn't be present for any service - jvm_config_count = 0 for service in service_list: if pathlib.Path(f'{config_path}/{service}/jvm.config').is_file(): @@ -236,14 +235,17 @@ def compute_system_memory(): return memory_for_druid def convert_total_memory_string(memory): - if memory == "": - computed_memory = compute_system_memory() - return computed_memory - elif memory.endswith(MEMORY_MEGABYTES_IDENTIFIER): - return int(memory[:-1]) - elif memory.endswith(MEMORY_GIGABYTES_IDENTIFIER): - return 1024 * int(memory[:-1]) - else: + try: + if memory == "": + computed_memory = compute_system_memory() + return computed_memory + elif memory.endswith(MEMORY_MEGABYTES_IDENTIFIER): + return int(memory[:-1]) + elif memory.endswith(MEMORY_GIGABYTES_IDENTIFIER): + return 1024 * int(memory[:-1]) + else: + error_and_exit('Incorrect format for memory argument, expected format is ') + except Exception: error_and_exit('Incorrect format for memory argument, expected format is ') def check_memory_constraint(total_memory, service_list): @@ -301,10 +303,15 @@ def build_memory_config_string(service, allocated_memory): return f'-Xms{heap_memory}m -Xmx{heap_memory}m -XX:MaxDirectMemorySize={direct_memory}m', allocated_memory -def distribute_memory_over_services(service_list, total_memory): +def distribute_memory_over_services(services, total_memory): service_memory_config = {} memory_weight_sum = 0 + + service_list = services.copy() + if MM in services: + service_list.append(MM_TASK) + for service in service_list: memory_weight_sum += SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) @@ -361,28 +368,21 @@ def build_supervise_script_arguments(service_list, service_memory_config, config commands.append("!p10 zk bin/run-zk conf") for service in service_list: - if service == MM_TASK: - continue + jvm_args = service_memory_config.get(service) prefix = '' if service == MM: prefix = '!p90 ' - jvm_args = service_memory_config.get(service) - - if service == MM: - task_config = service_memory_config.get(MM_TASK) - task_count = task_config[0] - task_memory = task_config[1] - if jvm_args is None: - commands.append( - f'{prefix}{service} bin/run-druid {service} {config_path} \'{task_count}\' \'{task_memory}\'') - else: - commands.append( - f'{prefix}{service} bin/run-druid {service} {config_path} \'{task_count}\' \'{task_memory}\' \'{jvm_args}\'') + if jvm_args is None: + commands.append(f'{prefix}{service} bin/run-druid {service} {config_path}') else: - if jvm_args is None: - commands.append(f'{prefix}{service} bin/run-druid {service} {config_path}') + if service == MM: + task_config = service_memory_config.get(MM_TASK) + task_count = task_config[0] + task_memory = task_config[1] + commands.append( + f'{prefix}{service} bin/run-druid {service} {config_path} \'{jvm_args}\' \'{task_count} {task_memory}\'') else: commands.append(f'{prefix}{service} bin/run-druid {service} {config_path} \'{jvm_args}\'') @@ -400,7 +400,7 @@ def main(): args = parser.parse_args() global LOGGING_ENABLED - LOGGING_ENABLED = args.verbose + LOGGING_ENABLED = args.verbose or args.compute_only config_path, total_memory, service_list, run_zk, compute_only = parse_arguments(args) @@ -414,9 +414,6 @@ def main(): service_memory_config = {} - if MM in service_list: - service_list.append(MM_TASK) - if (should_compute_memory(config_path, total_memory, service_list)): memory_in_mega_bytes = convert_total_memory_string(total_memory) custom_print(f'total memory is {memory_in_mega_bytes}m') From d111012eaee512cda1b92fb5621d21cfa12f69c5 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 22 Nov 2022 15:30:01 +0530 Subject: [PATCH 16/82] remove debug options from run-druid --- examples/bin/run-druid | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/bin/run-druid b/examples/bin/run-druid index 73ce559ef481..f6fd1cfd85bb 100755 --- a/examples/bin/run-druid +++ b/examples/bin/run-druid @@ -78,12 +78,11 @@ then -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" org.apache.druid.cli.Main server $SERVER_NAME else # args: - exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 $4"-Ddruid.log.path=$LOG_DIR" -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 $JVMARGS \ + exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 $4"-Ddruid.log.path=$LOG_DIR" $JVMARGS \ -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" org.apache.druid.cli.Main server $SERVER_NAME fi else # args: - echo "case parameters count 2, $#" exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" \ $MAIN_CLASS From a9b24e28ba1ace4a4abefa8fc1a4252d9b1a48fa Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 22 Nov 2022 15:49:21 +0530 Subject: [PATCH 17/82] resolve the config_path provided --- examples/bin/run-druid | 4 ++-- examples/bin/start-druid-main | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/bin/run-druid b/examples/bin/run-druid index f6fd1cfd85bb..c2fd8056ba79 100755 --- a/examples/bin/run-druid +++ b/examples/bin/run-druid @@ -74,11 +74,11 @@ then if [ "$#" -eq 3 ] then # args: - exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 $4 "-Ddruid.log.path=$LOG_DIR" $JVMARGS \ + exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" $JVMARGS \ -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" org.apache.druid.cli.Main server $SERVER_NAME else # args: - exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 $4"-Ddruid.log.path=$LOG_DIR" $JVMARGS \ + exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 $4 "-Ddruid.log.path=$LOG_DIR" $JVMARGS \ -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" org.apache.druid.cli.Main server $SERVER_NAME fi else diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index 86eec4b8eddf..e4d1b956883f 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -178,7 +178,7 @@ def parse_arguments(args): if args.run_zk: run_zk = True if args.config_path is not None: - config_path = os.path.join(os.getcwd(), args.config_path) + config_path = pathlib.Path(os.path.join(os.getcwd(), args.config_path)).resolve() if os.path.exists(config_path) is False: error_and_exit(f'config_path {config_path} doesn\'t exist') if args.memory is not None: From 71bfca1da0cc1d30103b14a4aee4e27b7bcd96d3 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 22 Nov 2022 16:02:11 +0530 Subject: [PATCH 18/82] comment out service specific runtime properties which are computed in the code --- .../single-server/quickstart/broker/runtime.properties | 10 +++++----- .../quickstart/historical/runtime.properties | 10 +++++----- .../quickstart/middleManager/runtime.properties | 6 +++--- .../single-server/quickstart/router/runtime.properties | 8 ++++---- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/examples/conf/druid/single-server/quickstart/broker/runtime.properties b/examples/conf/druid/single-server/quickstart/broker/runtime.properties index 92b85f8bc754..549117efa50d 100644 --- a/examples/conf/druid/single-server/quickstart/broker/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/broker/runtime.properties @@ -21,15 +21,15 @@ druid.service=druid/broker druid.plaintextPort=8082 # HTTP server settings -druid.server.http.numThreads=12 +#druid.server.http.numThreads=12 # HTTP client settings -druid.broker.http.numConnections=10 -druid.broker.http.maxQueuedBytes=5MiB +#druid.broker.http.numConnections=10 +#druid.broker.http.maxQueuedBytes=5MiB # Processing threads and buffers -druid.processing.buffer.sizeBytes=100MiB -druid.processing.numMergeBuffers=2 +#druid.processing.buffer.sizeBytes=100MiB +#druid.processing.numMergeBuffers=2 druid.processing.tmpDir=var/druid/processing # Query cache disabled -- push down caching and merging instead diff --git a/examples/conf/druid/single-server/quickstart/historical/runtime.properties b/examples/conf/druid/single-server/quickstart/historical/runtime.properties index 76eaa5d4b0cd..cf3e966cc30e 100644 --- a/examples/conf/druid/single-server/quickstart/historical/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/historical/runtime.properties @@ -21,12 +21,12 @@ druid.service=druid/historical druid.plaintextPort=8083 # HTTP server threads -druid.server.http.numThreads=12 +#druid.server.http.numThreads=12 # Processing threads and buffers -druid.processing.buffer.sizeBytes=200MiB -druid.processing.numMergeBuffers=2 -druid.processing.numThreads=2 +#druid.processing.buffer.sizeBytes=200MiB +#druid.processing.numMergeBuffers=2 +#druid.processing.numThreads=2 druid.processing.tmpDir=var/druid/processing # Segment storage @@ -36,4 +36,4 @@ druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g" druid.historical.cache.useCache=true druid.historical.cache.populateCache=true druid.cache.type=caffeine -druid.cache.sizeInBytes=10MiB +#druid.cache.sizeInBytes=10MiB diff --git a/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties b/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties index b0cf3ca3c43e..df9b790a69b0 100644 --- a/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties @@ -32,9 +32,9 @@ druid.indexer.task.baseTaskDir=var/druid/task #druid.server.http.numThreads=12 # Processing threads and buffers on Peons -druid.indexer.fork.property.druid.processing.numMergeBuffers=2 -druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB -druid.indexer.fork.property.druid.processing.numThreads=1 +#druid.indexer.fork.property.druid.processing.numMergeBuffers=2 +#druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB +#druid.indexer.fork.property.druid.processing.numThreads=1 # Hadoop indexing druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/examples/conf/druid/single-server/quickstart/router/runtime.properties b/examples/conf/druid/single-server/quickstart/router/runtime.properties index 497d3b4d3aa1..4c38ba88baab 100644 --- a/examples/conf/druid/single-server/quickstart/router/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/router/runtime.properties @@ -21,10 +21,10 @@ druid.service=druid/router druid.plaintextPort=8888 # HTTP proxy -druid.router.http.numConnections=50 -druid.router.http.readTimeout=PT5M -druid.router.http.numMaxThreads=100 -druid.server.http.numThreads=100 +#druid.router.http.numConnections=50 +#druid.router.http.readTimeout=PT5M +#druid.router.http.numMaxThreads=100 +#druid.server.http.numThreads=100 # Service discovery druid.router.defaultBrokerServiceName=druid/broker From 032bae0f20042f0ed4038345d878cd6e52fd2dc3 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 22 Nov 2022 16:13:31 +0530 Subject: [PATCH 19/82] simplify run-druid --- examples/bin/run-druid | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/examples/bin/run-druid b/examples/bin/run-druid index c2fd8056ba79..4be0afc5854b 100755 --- a/examples/bin/run-druid +++ b/examples/bin/run-druid @@ -64,6 +64,8 @@ fi cd "$WHEREAMI/.." +CLASS_PATH="$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" + if [ "$#" -eq 3 ] || [ "$#" -eq 4 ] then # args: or @@ -75,15 +77,15 @@ then then # args: exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" $JVMARGS \ - -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" org.apache.druid.cli.Main server $SERVER_NAME + -cp $CLASS_PATH $MAIN_CLASS else # args: exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 $4 "-Ddruid.log.path=$LOG_DIR" $JVMARGS \ - -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" org.apache.druid.cli.Main server $SERVER_NAME + -cp $CLASS_PATH $MAIN_CLASS fi else # args: - exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ - -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" \ - $MAIN_CLASS + exec "$WHEREAMI"/run-java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" \ + `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ + -cp $CLASS_PATH $MAIN_CLASS fi From f55c74929ab72965d0b772e8572edecf96f6b788 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 23 Nov 2022 09:56:20 +0530 Subject: [PATCH 20/82] clean up docs, naming changes --- examples/bin/start-druid-main | 200 +++++++++++++++++----------------- 1 file changed, 98 insertions(+), 102 deletions(-) diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index e4d1b956883f..1447d8bfba4b 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -26,67 +26,66 @@ import argparse QUICKSTART_ROOT_CONFIG_PATH = "conf/druid/single-server/quickstart" -MEMORY_GIGABYTES_IDENTIFIER = "g" -MEMORY_MEGABYTES_IDENTIFIER = "m" +MEM_GB_SUFFIX = "g" +MEM_MB_SUFFIX = "m" SERVICE_SEPARATOR = "," -MM_TASK_JAVAOPTS_ARRAY = ["-server", "-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] -MM_TASK_MEMORY_TYPE_LOW = "low" -MM_TASK_MEMORY_TYPE_HIGH = "high" -MM_TASK_MEM_MAP = { - MM_TASK_MEMORY_TYPE_LOW: ["-Xms256m", "-Xmx256m", "-XX:MaxDirectMemorySize=256g"], - MM_TASK_MEMORY_TYPE_HIGH: ["-Xms1g", "-Xmx1g", "-XX:MaxDirectMemorySize=1g"] +TASK_JAVA_OPTS_ARRAY = ["-server", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-XX:+ExitOnOutOfMemoryError", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] +TASK_JAVA_OPTS_PROPERTY = "-Ddruid.indexer.runner.javaOptsArray" +TASK_WORKER_CAPACITY_PROPERTY = "-Ddruid.worker.capacity" +TASK_COUNT = "task-count" +TASK_MEM_TYPE_LOW = "low" +TASK_MEM_TYPE_HIGH = "high" +TASK_MEM_MAP = { + TASK_MEM_TYPE_LOW: ["-Xms256m", "-Xmx256m", "-XX:MaxDirectMemorySize=256g"], + TASK_MEM_TYPE_HIGH: ["-Xms1g", "-Xmx1g", "-XX:MaxDirectMemorySize=1g"] } -MM_TASK_JAVAOPTS_PROP = "-Ddruid.indexer.runner.javaOptsArray" -MM_TASK_WORKER_CAPACITY_PROP = "-Ddruid.worker.capacity" - BROKER = "broker" ROUTER = "router" COORDINATOR = "coordinator-overlord" HISTORICAL = "historical" -MM = "middleManager" -MM_TASK = "middleManager-task" -MM_TASK_COUNT = "task-count" +MIDDLE_MANAGER = "middleManager" +TASK = "task" DEFAULT_SERVICES = [ BROKER, ROUTER, COORDINATOR, HISTORICAL, - MM + MIDDLE_MANAGER ] -SERVICE_MEMORY_DISTRIBUTION_WEIGHT = { - MM: 0.5, - ROUTER: 1, - COORDINATOR: 15, - BROKER: 23, - HISTORICAL: 40, - MM_TASK: 15 +SERVICE_MEMORY_RATIO = { + MIDDLE_MANAGER: 1, + ROUTER: 2, + COORDINATOR: 30, + BROKER: 46, + HISTORICAL: 80, + TASK: 30 } -SERVICE_MEMORY_LOWER_BOUND = { - MM: 64, +MINIMUM_MEMORY_MB = { + MIDDLE_MANAGER: 64, ROUTER: 128, - MM_TASK: 1024, + TASK: 1024, BROKER: 900, COORDINATOR: 256, HISTORICAL: 900 } -SERVICE_MEMORY_HEAP_PERCENTAGE = { - MM: 1, +HEAP_TO_TOTAL_MEM_RATIO = { + MIDDLE_MANAGER: 1, ROUTER: 1, COORDINATOR: 1, BROKER: 0.60, HISTORICAL: 0.40, - MM_TASK: 0.50 + TASK: 0.50 } LOGGING_ENABLED = False -def custom_print(message): +def print_if_verbose(message): if LOGGING_ENABLED: print(message) @@ -103,58 +102,55 @@ def configure_parser(): sample usage: start-druid Start up all the services (including zk). - 50 - 80 percent of system memory is used. start-druid -m=100g Start up all the services (including zk) - using the given memory. + using a total memory of 100GB. start-druid -m=100g --compute_only - Compute memory distribution and validate - arguments. - start-druid -m=100g -sl=broker,router - Start broker & router service, using 100g of memory. - Read config from conf/druid/single-server/quickstart. - start-druid -m=100g --sl=broker,router \\ - -cp=conf/druid/single-server/custom - Start broker & router service, using 100g of memory. - Read config from . - Since is specified, exception is thrown if - jvm.config is present for any of the services. - start-druid -sl=broker,router \\ + Compute memory distribution and validate arguments. + start-druid -m=100g -s=broker,router + Starts a broker and a router, using a total memory of 100GB. + Reads config from conf/druid/single-server/quickstart. + start-druid -m=100g --s=broker,router \\ -cp=conf/druid/single-server/custom - Start broker & router service, using system memory. - If jvm.config is specified for both the - services within /, - memory distribution is not calculated. - If jvm.config is present for either of the services, - exception is thrown. - If jvm.config is not present for both of the services, - memory distribution is calculated. + Starts a broker and a router, using a total memory of 100GB. + Reads configs from the given config path. + Throws an exception if there is a jvm.config present in any + of the service directories inside the given root conf directory. + start-druid -s=broker,router \\ + -c=conf/druid/single-server/custom + Starts a broker and a router service, using upto 80% of the + total system memory. Calculates the memory requirements for + each service if no jvm.config is present for any of the + services. Does not calculate memory requirements if jvm.config + is present in all of the service directories. Throws an exception + if jvm.config is present in only some of the service directories. start-druid -m=100g \\ - -cp=conf/druid/single-server/custom \\ - -sl=broker,router \\ + -s=broker,router \\ + -c=conf/druid/single-server/custom \\ --run_zk - Start broker, router and zookeeper. - zk config is read from conf/zk. + Starts broker, router and zookeeper. + zookeeper config is read from conf/zk. """ ) parser.add_argument('--memory', '-m', type=str, required=False, help='Total memory for all processes (services and tasks, if any). \n' 'This parameter is ignored if each service already has a jvm.config \n' 'in the given conf directory. e.g. 500m, 4g, 6g\n') - parser.add_argument('--service_list', '-sl', type=str, required=False, + parser.add_argument('--services', '-s', type=str, required=False, help='List of services to be started, subset of \n' '{broker, router, middleManager, historical, coordinator-overlord}. \n' 'If the argument is not given, all services \n' 'and zookeeper is started. e.g. -sl=broker,historical') - parser.add_argument('--config_path', '-cp', type=str, required=False, + parser.add_argument('--config', '-c', type=str, required=False, help='Relative path to the directory containing common and service \n' 'specific properties to be overridden. \n' 'This directory must contain \'_common\' directory with \n' '\'common.jvm.config\' & \'common.runtime.properties\' files. \n' 'If this argument is not given, config from \n' - '\'conf/druid/single-server/quickstart\' directory is used.\n') + 'conf/druid/single-server/quickstart directory is used.\n') parser.add_argument('--compute_only', action='store_true', - help='Validate the arguments and display memory distribution') + help='Does not start Druid, only displays the memory allocated \n' + 'to each service if started with the given total memory.\n') parser.add_argument('--run_zk', action='store_true', help='Specification to run zookeeper, \n' 'zk config is picked up from conf/zk.') @@ -177,14 +173,14 @@ def parse_arguments(args): compute_only = True if args.run_zk: run_zk = True - if args.config_path is not None: - config_path = pathlib.Path(os.path.join(os.getcwd(), args.config_path)).resolve() - if os.path.exists(config_path) is False: - error_and_exit(f'config_path {config_path} doesn\'t exist') + if args.config is not None: + config = pathlib.Path(os.path.join(os.getcwd(), args.config)).resolve() + if os.path.exists(config) is False: + error_and_exit(f'config {config} doesn\'t exist') if args.memory is not None: total_memory = args.memory - if args.service_list is not None: - services = args.service_list.split(SERVICE_SEPARATOR) + if args.services is not None: + services = args.services.split(SERVICE_SEPARATOR) for service in services: if service not in DEFAULT_SERVICES: @@ -203,12 +199,12 @@ def parse_arguments(args): return config_path, total_memory, service_list, run_zk, compute_only def print_startup_config(service_list, config_path, run_zk): - custom_print(f'starting {service_list}') - custom_print(f'reading config from {config_path}') + print_if_verbose(f'starting {service_list}') + print_if_verbose(f'reading config from {config_path}') if run_zk: zk_config_path = pathlib.Path(f'{os.getcwd()}/../conf/zk').resolve() - custom_print(f'starting zk, reading default config from {zk_config_path}') - custom_print('\n') + print_if_verbose(f'starting zk, reading default config from {zk_config_path}') + print_if_verbose('\n') def should_compute_memory(config_path, total_memory, service_list): # if jvm file is present for any of the services @@ -239,9 +235,9 @@ def convert_total_memory_string(memory): if memory == "": computed_memory = compute_system_memory() return computed_memory - elif memory.endswith(MEMORY_MEGABYTES_IDENTIFIER): + elif memory.endswith(MEM_MB_SUFFIX): return int(memory[:-1]) - elif memory.endswith(MEMORY_GIGABYTES_IDENTIFIER): + elif memory.endswith(MEM_GB_SUFFIX): return 1024 * int(memory[:-1]) else: error_and_exit('Incorrect format for memory argument, expected format is ') @@ -253,7 +249,7 @@ def check_memory_constraint(total_memory, service_list): lower_bound_memory = 0 for service in service_list: - lower_bound_memory += SERVICE_MEMORY_LOWER_BOUND.get(service) + lower_bound_memory += MINIMUM_MEMORY_MB.get(service) required_memory = int(lower_bound_memory / 0.8) @@ -266,11 +262,11 @@ def check_memory_constraint(total_memory, service_list): return lower_bound_memory def build_mm_task_javaopts_array(memory_type): - task_memory = f'{MM_TASK_JAVAOPTS_PROP}=[' + task_memory = f'{TASK_JAVA_OPTS_PROPERTY}=[' - MEM_ARRAY = MM_TASK_MEM_MAP.get(memory_type) + MEM_ARRAY = TASK_MEM_MAP.get(memory_type) - javaopts_list = MM_TASK_JAVAOPTS_ARRAY + MEM_ARRAY + javaopts_list = TASK_JAVA_OPTS_ARRAY + MEM_ARRAY for item in javaopts_list: task_memory += f'\"{item}\";' @@ -280,21 +276,21 @@ def build_mm_task_javaopts_array(memory_type): return task_memory def build_memory_config_string(service, allocated_memory): - if service == MM_TASK: + if service == TASK: if allocated_memory >= 2048: task_count = int(allocated_memory / 2048) - memory_type = MM_TASK_MEMORY_TYPE_HIGH + memory_type = TASK_MEM_TYPE_HIGH task_memory = 2048 else: task_count = int(allocated_memory / 512) - memory_type = MM_TASK_MEMORY_TYPE_LOW + memory_type = TASK_MEM_TYPE_LOW task_memory = 512 task_count = min(task_count, multiprocessing.cpu_count()) javaopts_array = build_mm_task_javaopts_array(memory_type) - return [f'{MM_TASK_WORKER_CAPACITY_PROP}={task_count}', javaopts_array], task_memory * task_count + return [f'{TASK_WORKER_CAPACITY_PROPERTY}={task_count}', javaopts_array], task_memory * task_count else: - heap_memory = SERVICE_MEMORY_HEAP_PERCENTAGE.get(service) * allocated_memory + heap_memory = HEAP_TO_TOTAL_MEM_RATIO.get(service) * allocated_memory direct_memory = int(allocated_memory - heap_memory) heap_memory = int(heap_memory) @@ -309,11 +305,11 @@ def distribute_memory_over_services(services, total_memory): memory_weight_sum = 0 service_list = services.copy() - if MM in services: - service_list.append(MM_TASK) + if MIDDLE_MANAGER in services: + service_list.append(TASK) for service in service_list: - memory_weight_sum += SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) + memory_weight_sum += SERVICE_MEMORY_RATIO.get(service) multiplier = total_memory / memory_weight_sum @@ -321,9 +317,9 @@ def distribute_memory_over_services(services, total_memory): allocated_services = set() for service in service_list: - allocated_memory = SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) * multiplier - if service in SERVICE_MEMORY_LOWER_BOUND and allocated_memory < SERVICE_MEMORY_LOWER_BOUND.get(service): - allocated_memory = SERVICE_MEMORY_LOWER_BOUND.get(service) + allocated_memory = SERVICE_MEMORY_RATIO.get(service) * multiplier + if service in MINIMUM_MEMORY_MB and allocated_memory < MINIMUM_MEMORY_MB.get(service): + allocated_memory = MINIMUM_MEMORY_MB.get(service) service_memory_config[service], allocated_memory = build_memory_config_string(service, allocated_memory) lower_bound_memory_allocation += allocated_memory allocated_services.add(service) @@ -334,22 +330,22 @@ def distribute_memory_over_services(services, total_memory): for service in service_list: if service in allocated_services: continue - memory_weight_sum += SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) + memory_weight_sum += SERVICE_MEMORY_RATIO.get(service) multiplier = (total_memory - lower_bound_memory_allocation) / memory_weight_sum for service in service_list: if service in allocated_services: continue - allocated_memory = SERVICE_MEMORY_DISTRIBUTION_WEIGHT.get(service) * multiplier - if service in SERVICE_MEMORY_LOWER_BOUND and allocated_memory < SERVICE_MEMORY_LOWER_BOUND.get(service): - allocated_memory = SERVICE_MEMORY_LOWER_BOUND.get(service) + allocated_memory = SERVICE_MEMORY_RATIO.get(service) * multiplier + if service in MINIMUM_MEMORY_MB and allocated_memory < MINIMUM_MEMORY_MB.get(service): + allocated_memory = MINIMUM_MEMORY_MB.get(service) service_memory_config[service], allocated_memory = build_memory_config_string(service, allocated_memory) - custom_print(f'\nMemory distribution for services:') + print_if_verbose(f'\nMemory distribution for services:') for key, value in service_memory_config.items(): - custom_print(f'{key}, memory_config: {value}') - custom_print('\n') + print_if_verbose(f'{key}, memory_config: {value}') + print_if_verbose('\n') return service_memory_config @@ -371,14 +367,14 @@ def build_supervise_script_arguments(service_list, service_memory_config, config jvm_args = service_memory_config.get(service) prefix = '' - if service == MM: + if service == MIDDLE_MANAGER: prefix = '!p90 ' if jvm_args is None: commands.append(f'{prefix}{service} bin/run-druid {service} {config_path}') else: - if service == MM: - task_config = service_memory_config.get(MM_TASK) + if service == MIDDLE_MANAGER: + task_config = service_memory_config.get(TASK) task_count = task_config[0] task_memory = task_config[1] commands.append( @@ -386,11 +382,11 @@ def build_supervise_script_arguments(service_list, service_memory_config, config else: commands.append(f'{prefix}{service} bin/run-druid {service} {config_path} \'{jvm_args}\'') - custom_print('Supervise script args:') + print_if_verbose('Supervise script args:') for item in commands: - custom_print(item) + print_if_verbose(item) - custom_print('\n') + print_if_verbose('\n') args.append(",".join(commands)) return args @@ -416,12 +412,12 @@ def main(): if (should_compute_memory(config_path, total_memory, service_list)): memory_in_mega_bytes = convert_total_memory_string(total_memory) - custom_print(f'total memory is {memory_in_mega_bytes}m') + print_if_verbose(f'total memory is {memory_in_mega_bytes}m') memory_to_be_used = check_memory_constraint(memory_in_mega_bytes, service_list) - custom_print(f'memory used for services & tasks {memory_to_be_used}m') + print_if_verbose(f'memory used for services & tasks {memory_to_be_used}m') service_memory_config = distribute_memory_over_services(service_list, memory_to_be_used) else: - custom_print('not computing memory distribution, reading memory specification from service jvm.config\n') + print_if_verbose('not computing memory distribution, reading memory specification from service jvm.config\n') script_arguments = build_supervise_script_arguments(service_list, service_memory_config, config_path, run_zk) From 2a0dd346cdaaf44754f6b7f50205a8214498bb0d Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 23 Nov 2022 10:03:22 +0530 Subject: [PATCH 21/82] Throw ValueError exception on illegal state --- examples/bin/start-druid-main | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index 1447d8bfba4b..11c06aa6bc3d 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -89,10 +89,6 @@ def print_if_verbose(message): if LOGGING_ENABLED: print(message) -def error_and_exit(message): - sys.stderr.write(message + '\n') - sys.exit(1) - def configure_parser(): parser = argparse.ArgumentParser( prog='Druid quickstart', @@ -176,7 +172,7 @@ def parse_arguments(args): if args.config is not None: config = pathlib.Path(os.path.join(os.getcwd(), args.config)).resolve() if os.path.exists(config) is False: - error_and_exit(f'config {config} doesn\'t exist') + raise ValueError(f'config {config} doesn\'t exist') if args.memory is not None: total_memory = args.memory if args.services is not None: @@ -184,10 +180,10 @@ def parse_arguments(args): for service in services: if service not in DEFAULT_SERVICES: - error_and_exit(f'Invalid service name {service}, should be one of {DEFAULT_SERVICES}') + raise ValueError(f'Invalid service name {service}, should be one of {DEFAULT_SERVICES}') if service in service_list: - error_and_exit(f'{service} is specified multiple times') + raise ValueError(f'{service} is specified multiple times') service_list.append(service) @@ -215,13 +211,13 @@ def should_compute_memory(config_path, total_memory, service_list): if pathlib.Path(f'{config_path}/{service}/jvm.config').is_file(): jvm_config_count += 1 elif jvm_config_count > 0: - error_and_exit(f'jvm.config file is missing for service {service}, jvm.config should be specified for all the services or none') + raise ValueError(f'jvm.config file is missing for service {service}, jvm.config should be specified for all the services or none') if jvm_config_count > 0 and (jvm_config_count != len(service_list) or total_memory != ""): if jvm_config_count != len(service_list): - error_and_exit("jvm.config file should be present for all services or none") + raise ValueError("jvm.config file should be present for all services or none") if total_memory != "": - error_and_exit("If jvm.config is given for services, memory argument shouldn't be specified") + raise ValueError("If jvm.config is given for services, memory argument shouldn't be specified") return jvm_config_count == 0 @@ -240,9 +236,9 @@ def convert_total_memory_string(memory): elif memory.endswith(MEM_GB_SUFFIX): return 1024 * int(memory[:-1]) else: - error_and_exit('Incorrect format for memory argument, expected format is ') + raise ValueError('Incorrect format for memory argument, expected format is ') except Exception: - error_and_exit('Incorrect format for memory argument, expected format is ') + raise ValueError('Incorrect format for memory argument, expected format is ') def check_memory_constraint(total_memory, service_list): # 80% of total memory >= sum of lower bound service memory should be @@ -254,7 +250,7 @@ def check_memory_constraint(total_memory, service_list): required_memory = int(lower_bound_memory / 0.8) if total_memory < required_memory: - error_and_exit(f'Minimum memory required for starting services is {required_memory}m') + raise ValueError(f'Minimum memory required for starting services is {required_memory}m') if total_memory >= 2 * lower_bound_memory: return int(total_memory / 2) @@ -427,6 +423,7 @@ def main(): os.execv('./supervise', script_arguments) try: - main() -except KeyboardInterrupt: - sys.exit(1) + main() +except (KeyboardInterrupt, ValueError) as error: + print(error) + sys.exit(1) From c35cc1e79dfcad3699b9a1ef1835c870e9af08ea Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 23 Nov 2022 10:25:34 +0530 Subject: [PATCH 22/82] update docs --- examples/bin/start-druid-main | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index 11c06aa6bc3d..6165a0e91e7f 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -241,7 +241,7 @@ def convert_total_memory_string(memory): raise ValueError('Incorrect format for memory argument, expected format is ') def check_memory_constraint(total_memory, service_list): - # 80% of total memory >= sum of lower bound service memory should be + # 80% of total memory >= sum of lower bound service memory lower_bound_memory = 0 for service in service_list: From 891eaaa1308b249acbb235d0d17309ca559eeed6 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 23 Nov 2022 11:36:40 +0530 Subject: [PATCH 23/82] rename args, compute_only -> compute, run_zk -> zk --- examples/bin/start-druid-main | 74 +++++++++++++++++------------------ 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index 6165a0e91e7f..5444ef4bab3c 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -101,13 +101,13 @@ sample usage: start-druid -m=100g Start up all the services (including zk) using a total memory of 100GB. - start-druid -m=100g --compute_only + start-druid -m=100g --compute Compute memory distribution and validate arguments. start-druid -m=100g -s=broker,router Starts a broker and a router, using a total memory of 100GB. Reads config from conf/druid/single-server/quickstart. start-druid -m=100g --s=broker,router \\ - -cp=conf/druid/single-server/custom + -c=conf/druid/single-server/custom Starts a broker and a router, using a total memory of 100GB. Reads configs from the given config path. Throws an exception if there is a jvm.config present in any @@ -123,7 +123,7 @@ sample usage: start-druid -m=100g \\ -s=broker,router \\ -c=conf/druid/single-server/custom \\ - --run_zk + --zk Starts broker, router and zookeeper. zookeeper config is read from conf/zk. """ @@ -144,31 +144,31 @@ sample usage: '\'common.jvm.config\' & \'common.runtime.properties\' files. \n' 'If this argument is not given, config from \n' 'conf/druid/single-server/quickstart directory is used.\n') - parser.add_argument('--compute_only', action='store_true', + parser.add_argument('--compute', action='store_true', help='Does not start Druid, only displays the memory allocated \n' 'to each service if started with the given total memory.\n') - parser.add_argument('--run_zk', action='store_true', + parser.add_argument('--zk', '-zk', action='store_true', help='Specification to run zookeeper, \n' 'zk config is picked up from conf/zk.') parser.add_argument('--verbose', action='store_true', help='Log details') - parser.set_defaults(run_zk=False) - parser.set_defaults(compute_only=False) + parser.set_defaults(zk=False) + parser.set_defaults(compute=False) parser.set_defaults(verbose=False) return parser def parse_arguments(args): service_list = [] - config_path = "" + config = "" total_memory = "" - compute_only = False - run_zk = False + compute = False + zk = False - if args.compute_only: - compute_only = True - if args.run_zk: - run_zk = True + if args.compute: + compute = True + if args.zk: + zk = True if args.config is not None: config = pathlib.Path(os.path.join(os.getcwd(), args.config)).resolve() if os.path.exists(config) is False: @@ -190,25 +190,25 @@ def parse_arguments(args): if len(service_list) == 0: # start all services service_list = DEFAULT_SERVICES - run_zk = True + zk = True - return config_path, total_memory, service_list, run_zk, compute_only + return config, total_memory, service_list, zk, compute -def print_startup_config(service_list, config_path, run_zk): +def print_startup_config(service_list, config, zk): print_if_verbose(f'starting {service_list}') - print_if_verbose(f'reading config from {config_path}') - if run_zk: - zk_config_path = pathlib.Path(f'{os.getcwd()}/../conf/zk').resolve() - print_if_verbose(f'starting zk, reading default config from {zk_config_path}') + print_if_verbose(f'reading config from {config}') + if zk: + zk_config = pathlib.Path(f'{os.getcwd()}/../conf/zk').resolve() + print_if_verbose(f'starting zk, reading default config from {zk_config}') print_if_verbose('\n') -def should_compute_memory(config_path, total_memory, service_list): +def should_compute_memory(config, total_memory, service_list): # if jvm file is present for any of the services # it should be present for all services and memory should not be specified # if memory is given, jvm file shouldn't be present for any service jvm_config_count = 0 for service in service_list: - if pathlib.Path(f'{config_path}/{service}/jvm.config').is_file(): + if pathlib.Path(f'{config}/{service}/jvm.config').is_file(): jvm_config_count += 1 elif jvm_config_count > 0: raise ValueError(f'jvm.config file is missing for service {service}, jvm.config should be specified for all the services or none') @@ -241,7 +241,7 @@ def convert_total_memory_string(memory): raise ValueError('Incorrect format for memory argument, expected format is ') def check_memory_constraint(total_memory, service_list): - # 80% of total memory >= sum of lower bound service memory + # 80% of total memory >= sum of lower bound service memory should be lower_bound_memory = 0 for service in service_list: @@ -345,7 +345,7 @@ def distribute_memory_over_services(services, total_memory): return service_memory_config -def build_supervise_script_arguments(service_list, service_memory_config, config_path, run_zk): +def build_supervise_script_arguments(service_list, service_memory_config, config, zk): args = [] commands = [] args.append('supervise') @@ -356,7 +356,7 @@ def build_supervise_script_arguments(service_list, service_memory_config, config commands.append(":notify bin/greet") commands.append(":kill-timeout 10") - if run_zk: + if zk: commands.append("!p10 zk bin/run-zk conf") for service in service_list: @@ -367,16 +367,16 @@ def build_supervise_script_arguments(service_list, service_memory_config, config prefix = '!p90 ' if jvm_args is None: - commands.append(f'{prefix}{service} bin/run-druid {service} {config_path}') + commands.append(f'{prefix}{service} bin/run-druid {service} {config}') else: if service == MIDDLE_MANAGER: task_config = service_memory_config.get(TASK) task_count = task_config[0] task_memory = task_config[1] commands.append( - f'{prefix}{service} bin/run-druid {service} {config_path} \'{jvm_args}\' \'{task_count} {task_memory}\'') + f'{prefix}{service} bin/run-druid {service} {config} \'{jvm_args}\' \'{task_count} {task_memory}\'') else: - commands.append(f'{prefix}{service} bin/run-druid {service} {config_path} \'{jvm_args}\'') + commands.append(f'{prefix}{service} bin/run-druid {service} {config} \'{jvm_args}\'') print_if_verbose('Supervise script args:') for item in commands: @@ -392,21 +392,21 @@ def main(): args = parser.parse_args() global LOGGING_ENABLED - LOGGING_ENABLED = args.verbose or args.compute_only + LOGGING_ENABLED = args.verbose or args.compute - config_path, total_memory, service_list, run_zk, compute_only = parse_arguments(args) + config, total_memory, service_list, zk, compute = parse_arguments(args) # change directory to bin os.chdir(os.path.dirname(sys.argv[0])) - if config_path == "": - config_path = pathlib.Path(f'{os.getcwd()}/../{QUICKSTART_ROOT_CONFIG_PATH}').resolve() + if config == "": + config = pathlib.Path(f'{os.getcwd()}/../{QUICKSTART_ROOT_CONFIG_PATH}').resolve() - print_startup_config(service_list, config_path, run_zk) + print_startup_config(service_list, config, zk) service_memory_config = {} - if (should_compute_memory(config_path, total_memory, service_list)): + if (should_compute_memory(config, total_memory, service_list)): memory_in_mega_bytes = convert_total_memory_string(total_memory) print_if_verbose(f'total memory is {memory_in_mega_bytes}m') memory_to_be_used = check_memory_constraint(memory_in_mega_bytes, service_list) @@ -415,9 +415,9 @@ def main(): else: print_if_verbose('not computing memory distribution, reading memory specification from service jvm.config\n') - script_arguments = build_supervise_script_arguments(service_list, service_memory_config, config_path, run_zk) + script_arguments = build_supervise_script_arguments(service_list, service_memory_config, config, zk) - if compute_only: + if compute: return os.execv('./supervise', script_arguments) From a06e86bcfa12bcec8b788239f62cbe96581f2020 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 23 Nov 2022 14:50:04 +0530 Subject: [PATCH 24/82] update help documentation --- examples/bin/start-druid-main | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index 5444ef4bab3c..081c90d67211 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -46,7 +46,7 @@ ROUTER = "router" COORDINATOR = "coordinator-overlord" HISTORICAL = "historical" MIDDLE_MANAGER = "middleManager" -TASK = "task" +TASKS = "tasks" DEFAULT_SERVICES = [ BROKER, @@ -62,13 +62,13 @@ SERVICE_MEMORY_RATIO = { COORDINATOR: 30, BROKER: 46, HISTORICAL: 80, - TASK: 30 + TASKS: 30 } MINIMUM_MEMORY_MB = { MIDDLE_MANAGER: 64, ROUTER: 128, - TASK: 1024, + TASKS: 1024, BROKER: 900, COORDINATOR: 256, HISTORICAL: 900 @@ -80,7 +80,7 @@ HEAP_TO_TOTAL_MEM_RATIO = { COORDINATOR: 1, BROKER: 0.60, HISTORICAL: 0.40, - TASK: 0.50 + TASKS: 0.50 } LOGGING_ENABLED = False @@ -105,21 +105,18 @@ sample usage: Compute memory distribution and validate arguments. start-druid -m=100g -s=broker,router Starts a broker and a router, using a total memory of 100GB. - Reads config from conf/druid/single-server/quickstart. start-druid -m=100g --s=broker,router \\ -c=conf/druid/single-server/custom Starts a broker and a router, using a total memory of 100GB. - Reads configs from the given config path. + Reads configs for each service (jvm.config, runtime.properties) + from respective folders inside the given root config path. Throws an exception if there is a jvm.config present in any of the service directories inside the given root conf directory. start-druid -s=broker,router \\ -c=conf/druid/single-server/custom - Starts a broker and a router service, using upto 80% of the - total system memory. Calculates the memory requirements for - each service if no jvm.config is present for any of the - services. Does not calculate memory requirements if jvm.config - is present in all of the service directories. Throws an exception - if jvm.config is present in only some of the service directories. + Starts a broker and a router service, reading service configs + from the given root directory. Calculates memory requirements for + each service, if required, using upto 80% of the total system memory. start-druid -m=100g \\ -s=broker,router \\ -c=conf/druid/single-server/custom \\ @@ -272,7 +269,7 @@ def build_mm_task_javaopts_array(memory_type): return task_memory def build_memory_config_string(service, allocated_memory): - if service == TASK: + if service == TASKS: if allocated_memory >= 2048: task_count = int(allocated_memory / 2048) memory_type = TASK_MEM_TYPE_HIGH @@ -302,7 +299,7 @@ def distribute_memory_over_services(services, total_memory): service_list = services.copy() if MIDDLE_MANAGER in services: - service_list.append(TASK) + service_list.append(TASKS) for service in service_list: memory_weight_sum += SERVICE_MEMORY_RATIO.get(service) @@ -370,7 +367,7 @@ def build_supervise_script_arguments(service_list, service_memory_config, config commands.append(f'{prefix}{service} bin/run-druid {service} {config}') else: if service == MIDDLE_MANAGER: - task_config = service_memory_config.get(TASK) + task_config = service_memory_config.get(TASKS) task_count = task_config[0] task_memory = task_config[1] commands.append( From dc8f2b23b04eed8a89376764754e6a90e7787b5a Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 23 Nov 2022 14:52:09 +0530 Subject: [PATCH 25/82] update help documentation --- examples/bin/start-druid-main | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index 081c90d67211..312557125934 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -110,8 +110,6 @@ sample usage: Starts a broker and a router, using a total memory of 100GB. Reads configs for each service (jvm.config, runtime.properties) from respective folders inside the given root config path. - Throws an exception if there is a jvm.config present in any - of the service directories inside the given root conf directory. start-druid -s=broker,router \\ -c=conf/druid/single-server/custom Starts a broker and a router service, reading service configs From 589586f8d0edcacf241452dd0cc068863ea875b9 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 23 Nov 2022 15:09:17 +0530 Subject: [PATCH 26/82] move task memory computation into separate method --- examples/bin/start-druid-main | 72 +++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index 312557125934..7605ee8a0097 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -30,7 +30,8 @@ MEM_GB_SUFFIX = "g" MEM_MB_SUFFIX = "m" SERVICE_SEPARATOR = "," -TASK_JAVA_OPTS_ARRAY = ["-server", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-XX:+ExitOnOutOfMemoryError", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] +TASK_JAVA_OPTS_ARRAY = ["-server", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-XX:+ExitOnOutOfMemoryError", + "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] TASK_JAVA_OPTS_PROPERTY = "-Ddruid.indexer.runner.javaOptsArray" TASK_WORKER_CAPACITY_PROPERTY = "-Ddruid.worker.capacity" TASK_COUNT = "task-count" @@ -85,16 +86,18 @@ HEAP_TO_TOTAL_MEM_RATIO = { LOGGING_ENABLED = False + def print_if_verbose(message): if LOGGING_ENABLED: print(message) + def configure_parser(): parser = argparse.ArgumentParser( prog='Druid quickstart', formatter_class=argparse.RawTextHelpFormatter, epilog= -""" + """ sample usage: start-druid Start up all the services (including zk). @@ -153,6 +156,7 @@ sample usage: return parser + def parse_arguments(args): service_list = [] config = "" @@ -167,7 +171,7 @@ def parse_arguments(args): if args.config is not None: config = pathlib.Path(os.path.join(os.getcwd(), args.config)).resolve() if os.path.exists(config) is False: - raise ValueError(f'config {config} doesn\'t exist') + raise ValueError(f'config {config} not found') if args.memory is not None: total_memory = args.memory if args.services is not None: @@ -189,6 +193,7 @@ def parse_arguments(args): return config, total_memory, service_list, zk, compute + def print_startup_config(service_list, config, zk): print_if_verbose(f'starting {service_list}') print_if_verbose(f'reading config from {config}') @@ -197,6 +202,7 @@ def print_startup_config(service_list, config, zk): print_if_verbose(f'starting zk, reading default config from {zk_config}') print_if_verbose('\n') + def should_compute_memory(config, total_memory, service_list): # if jvm file is present for any of the services # it should be present for all services and memory should not be specified @@ -206,7 +212,9 @@ def should_compute_memory(config, total_memory, service_list): if pathlib.Path(f'{config}/{service}/jvm.config').is_file(): jvm_config_count += 1 elif jvm_config_count > 0: - raise ValueError(f'jvm.config file is missing for service {service}, jvm.config should be specified for all the services or none') + raise ValueError( + f'jvm.config file is missing for service {service}, jvm.config should be specified for all the ' + f'services or none') if jvm_config_count > 0 and (jvm_config_count != len(service_list) or total_memory != ""): if jvm_config_count != len(service_list): @@ -216,11 +224,13 @@ def should_compute_memory(config, total_memory, service_list): return jvm_config_count == 0 + def compute_system_memory(): - system_memory = psutil.virtual_memory().total # mem in bytes + system_memory = psutil.virtual_memory().total # mem in bytes memory_for_druid = int(system_memory / (1024 * 1024)) return memory_for_druid + def convert_total_memory_string(memory): try: if memory == "": @@ -235,6 +245,7 @@ def convert_total_memory_string(memory): except Exception: raise ValueError('Incorrect format for memory argument, expected format is ') + def check_memory_constraint(total_memory, service_list): # 80% of total memory >= sum of lower bound service memory should be lower_bound_memory = 0 @@ -252,34 +263,41 @@ def check_memory_constraint(total_memory, service_list): else: return lower_bound_memory -def build_mm_task_javaopts_array(memory_type): + +def build_mm_task_java_opts_array(memory_type): task_memory = f'{TASK_JAVA_OPTS_PROPERTY}=[' - MEM_ARRAY = TASK_MEM_MAP.get(memory_type) + mem_array = TASK_MEM_MAP.get(memory_type) - javaopts_list = TASK_JAVA_OPTS_ARRAY + MEM_ARRAY + java_opts_list = TASK_JAVA_OPTS_ARRAY + mem_array - for item in javaopts_list: + for item in java_opts_list: task_memory += f'\"{item}\";' task_memory = task_memory[:-1] task_memory += ']' return task_memory -def build_memory_config_string(service, allocated_memory): - if service == TASKS: - if allocated_memory >= 2048: - task_count = int(allocated_memory / 2048) - memory_type = TASK_MEM_TYPE_HIGH - task_memory = 2048 - else: - task_count = int(allocated_memory / 512) - memory_type = TASK_MEM_TYPE_LOW - task_memory = 512 - task_count = min(task_count, multiprocessing.cpu_count()) - javaopts_array = build_mm_task_javaopts_array(memory_type) - return [f'{TASK_WORKER_CAPACITY_PROPERTY}={task_count}', javaopts_array], task_memory * task_count +def compute_tasks_memory(allocated_memory): + if allocated_memory >= 2048: + task_count = int(allocated_memory / 2048) + memory_type = TASK_MEM_TYPE_HIGH + task_memory_mb = 2048 + else: + task_count = int(allocated_memory / 512) + memory_type = TASK_MEM_TYPE_LOW + task_memory_mb = 512 + task_count = min(task_count, multiprocessing.cpu_count()) + + return memory_type, task_count, task_memory_mb + + +def build_memory_config(service, allocated_memory): + if service == TASKS: + memory_type, task_count, task_memory = compute_tasks_memory(allocated_memory) + java_opts_array = build_mm_task_java_opts_array(memory_type) + return [f'{TASK_WORKER_CAPACITY_PROPERTY}={task_count}', java_opts_array], task_memory * task_count else: heap_memory = HEAP_TO_TOTAL_MEM_RATIO.get(service) * allocated_memory direct_memory = int(allocated_memory - heap_memory) @@ -290,6 +308,7 @@ def build_memory_config_string(service, allocated_memory): return f'-Xms{heap_memory}m -Xmx{heap_memory}m -XX:MaxDirectMemorySize={direct_memory}m', allocated_memory + def distribute_memory_over_services(services, total_memory): service_memory_config = {} @@ -311,7 +330,7 @@ def distribute_memory_over_services(services, total_memory): allocated_memory = SERVICE_MEMORY_RATIO.get(service) * multiplier if service in MINIMUM_MEMORY_MB and allocated_memory < MINIMUM_MEMORY_MB.get(service): allocated_memory = MINIMUM_MEMORY_MB.get(service) - service_memory_config[service], allocated_memory = build_memory_config_string(service, allocated_memory) + service_memory_config[service], allocated_memory = build_memory_config(service, allocated_memory) lower_bound_memory_allocation += allocated_memory allocated_services.add(service) @@ -331,7 +350,7 @@ def distribute_memory_over_services(services, total_memory): if service in MINIMUM_MEMORY_MB and allocated_memory < MINIMUM_MEMORY_MB.get(service): allocated_memory = MINIMUM_MEMORY_MB.get(service) - service_memory_config[service], allocated_memory = build_memory_config_string(service, allocated_memory) + service_memory_config[service], allocated_memory = build_memory_config(service, allocated_memory) print_if_verbose(f'\nMemory distribution for services:') for key, value in service_memory_config.items(): @@ -340,6 +359,7 @@ def distribute_memory_over_services(services, total_memory): return service_memory_config + def build_supervise_script_arguments(service_list, service_memory_config, config, zk): args = [] commands = [] @@ -382,6 +402,7 @@ def build_supervise_script_arguments(service_list, service_memory_config, config args.append(",".join(commands)) return args + def main(): parser = configure_parser() args = parser.parse_args() @@ -401,7 +422,7 @@ def main(): service_memory_config = {} - if (should_compute_memory(config, total_memory, service_list)): + if should_compute_memory(config, total_memory, service_list): memory_in_mega_bytes = convert_total_memory_string(total_memory) print_if_verbose(f'total memory is {memory_in_mega_bytes}m') memory_to_be_used = check_memory_constraint(memory_in_mega_bytes, service_list) @@ -417,6 +438,7 @@ def main(): os.execv('./supervise', script_arguments) + try: main() except (KeyboardInterrupt, ValueError) as error: From b78b1164d81b9c15f814b902712bd5b1afedc47f Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 23 Nov 2022 19:05:01 +0530 Subject: [PATCH 27/82] Add validation checks --- examples/bin/start-druid-main | 71 ++++++++++++++++--- .../apache/druid/guice/StorageNodeModule.java | 2 +- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index 7605ee8a0097..07a23586aacc 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -28,12 +28,15 @@ QUICKSTART_ROOT_CONFIG_PATH = "conf/druid/single-server/quickstart" MEM_GB_SUFFIX = "g" MEM_MB_SUFFIX = "m" +XMX_PARAMETER = "-Xmx" +XMS_PARAMETER = "-Xms" +DIRECT_MEM_PARAMETER = "-XX:MaxDirectMemorySize" SERVICE_SEPARATOR = "," TASK_JAVA_OPTS_ARRAY = ["-server", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-XX:+ExitOnOutOfMemoryError", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] -TASK_JAVA_OPTS_PROPERTY = "-Ddruid.indexer.runner.javaOptsArray" -TASK_WORKER_CAPACITY_PROPERTY = "-Ddruid.worker.capacity" +TASK_JAVA_OPTS_PROPERTY = "druid.indexer.runner.javaOptsArray" +TASK_WORKER_CAPACITY_PROPERTY = "druid.worker.capacity" TASK_COUNT = "task-count" TASK_MEM_TYPE_LOW = "low" TASK_MEM_TYPE_HIGH = "high" @@ -157,6 +160,18 @@ sample usage: return parser +def validate_common_directory(config): + if pathlib.Path(f'{config}/_common').is_dir() is False: + raise ValueError( + f'_common directory is missing in the root config, check {QUICKSTART_ROOT_CONFIG_PATH}/_common directory') + + if pathlib.Path(f'{config}/_common/common.runtime.properties').is_file() is False: + raise ValueError(f'_common/common.runtime.properties file is missing in the root config, check {QUICKSTART_ROOT_CONFIG_PATH}/_common directory') + + if pathlib.Path(f'{config}/_common/common.jvm.config').is_file() is False: + raise ValueError(f'_common/jvm.config file is missing in the root config, check {QUICKSTART_ROOT_CONFIG_PATH}/_common directory') + + def parse_arguments(args): service_list = [] config = "" @@ -172,6 +187,7 @@ def parse_arguments(args): config = pathlib.Path(os.path.join(os.getcwd(), args.config)).resolve() if os.path.exists(config) is False: raise ValueError(f'config {config} not found') + validate_common_directory(config) if args.memory is not None: total_memory = args.memory if args.services is not None: @@ -203,6 +219,42 @@ def print_startup_config(service_list, config, zk): print_if_verbose('\n') +def verify_service_config(service, config): + path = f'{config}/{service}/jvm.config' + + required_parameters = [XMX_PARAMETER, XMS_PARAMETER] + + if HEAP_TO_TOTAL_MEM_RATIO.get(service) != 1: + required_parameters.append(DIRECT_MEM_PARAMETER) + + with open(path) as file: + for line in file: + if line.startswith(XMX_PARAMETER): + required_parameters.remove(XMX_PARAMETER) + if line.startswith(XMS_PARAMETER): + required_parameters.remove(XMS_PARAMETER) + if line.startswith(DIRECT_MEM_PARAMETER) and DIRECT_MEM_PARAMETER in required_parameters: + required_parameters.remove(DIRECT_MEM_PARAMETER) + + if len(required_parameters) > 0: + params = ",".join(required_parameters) + raise ValueError(f'{params} missing in {service}/jvm.config') + + if service == MIDDLE_MANAGER: + if pathlib.Path(f'{config}/{service}/runtime.properties').is_file() is False: + raise ValueError(f'{service}/runtime.properties file is missing in the root config') + + java_opts_property_present = False + print(config) + with open(f'{config}/{service}/runtime.properties') as file: + for line in file: + if line.startswith(TASK_JAVA_OPTS_PROPERTY): + java_opts_property_present = True + + if java_opts_property_present is False: + raise ValueError(f'{TASK_JAVA_OPTS_PROPERTY} property missing in {service}/runtime.properties') + + def should_compute_memory(config, total_memory, service_list): # if jvm file is present for any of the services # it should be present for all services and memory should not be specified @@ -216,11 +268,13 @@ def should_compute_memory(config, total_memory, service_list): f'jvm.config file is missing for service {service}, jvm.config should be specified for all the ' f'services or none') - if jvm_config_count > 0 and (jvm_config_count != len(service_list) or total_memory != ""): + if jvm_config_count > 0: if jvm_config_count != len(service_list): raise ValueError("jvm.config file should be present for all services or none") if total_memory != "": raise ValueError("If jvm.config is given for services, memory argument shouldn't be specified") + for service in service_list: + verify_service_config(service, config) return jvm_config_count == 0 @@ -265,7 +319,7 @@ def check_memory_constraint(total_memory, service_list): def build_mm_task_java_opts_array(memory_type): - task_memory = f'{TASK_JAVA_OPTS_PROPERTY}=[' + task_memory = f'-D{TASK_JAVA_OPTS_PROPERTY}=[' mem_array = TASK_MEM_MAP.get(memory_type) @@ -297,7 +351,7 @@ def build_memory_config(service, allocated_memory): if service == TASKS: memory_type, task_count, task_memory = compute_tasks_memory(allocated_memory) java_opts_array = build_mm_task_java_opts_array(memory_type) - return [f'{TASK_WORKER_CAPACITY_PROPERTY}={task_count}', java_opts_array], task_memory * task_count + return [f'-D{TASK_WORKER_CAPACITY_PROPERTY}={task_count}', java_opts_array], task_memory * task_count else: heap_memory = HEAP_TO_TOTAL_MEM_RATIO.get(service) * allocated_memory direct_memory = int(allocated_memory - heap_memory) @@ -309,7 +363,7 @@ def build_memory_config(service, allocated_memory): return f'-Xms{heap_memory}m -Xmx{heap_memory}m -XX:MaxDirectMemorySize={direct_memory}m', allocated_memory -def distribute_memory_over_services(services, total_memory): +def distribute_memory(services, total_memory): service_memory_config = {} memory_weight_sum = 0 @@ -354,7 +408,7 @@ def distribute_memory_over_services(services, total_memory): print_if_verbose(f'\nMemory distribution for services:') for key, value in service_memory_config.items(): - print_if_verbose(f'{key}, memory_config: {value}') + print_if_verbose(f'{key}, {value}') print_if_verbose('\n') return service_memory_config @@ -427,7 +481,7 @@ def main(): print_if_verbose(f'total memory is {memory_in_mega_bytes}m') memory_to_be_used = check_memory_constraint(memory_in_mega_bytes, service_list) print_if_verbose(f'memory used for services & tasks {memory_to_be_used}m') - service_memory_config = distribute_memory_over_services(service_list, memory_to_be_used) + service_memory_config = distribute_memory(service_list, memory_to_be_used) else: print_if_verbose('not computing memory distribution, reading memory specification from service jvm.config\n') @@ -438,7 +492,6 @@ def main(): os.execv('./supervise', script_arguments) - try: main() except (KeyboardInterrupt, ValueError) as error: diff --git a/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java b/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java index a232f6d01aac..842abd0e5347 100644 --- a/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java +++ b/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java @@ -101,7 +101,7 @@ public DataNodeService getDataNodeService( serverTypeConfig.getServerType() ); if (ServerType.HISTORICAL.equals(serverTypeConfig.getServerType())) { - throw new ProvisionException("Segment cache locations must be set on historicals."); + throw new ProvisionException("druid.segmentCache.locations must be set on historicals."); } } From a2887049dfbbef8266cf31e93e9aa4972b9af689 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 23 Nov 2022 19:12:21 +0530 Subject: [PATCH 28/82] remove print --- examples/bin/start-druid-main | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index 07a23586aacc..3146a06a1290 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -245,7 +245,6 @@ def verify_service_config(service, config): raise ValueError(f'{service}/runtime.properties file is missing in the root config') java_opts_property_present = False - print(config) with open(f'{config}/{service}/runtime.properties') as file: for line in file: if line.startswith(TASK_JAVA_OPTS_PROPERTY): From 81134a6c0741aaa2c850aca3c374d5af0e8e0f21 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 24 Nov 2022 14:05:27 +0530 Subject: [PATCH 29/82] Add validations --- examples/bin/start-druid-main | 100 ++++++++++++++++++++++++---------- 1 file changed, 71 insertions(+), 29 deletions(-) diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index 3146a06a1290..f5acbc5df6b9 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -160,16 +160,20 @@ sample usage: return parser +def validate_common_jvm_args(config): + if pathlib.Path(f'{config}/_common/common.jvm.config').is_file() is False: + raise ValueError(f'_common/common.jvm.config file is missing in the root config, ' + f'check {QUICKSTART_ROOT_CONFIG_PATH}/_common directory') + + def validate_common_directory(config): if pathlib.Path(f'{config}/_common').is_dir() is False: raise ValueError( f'_common directory is missing in the root config, check {QUICKSTART_ROOT_CONFIG_PATH}/_common directory') if pathlib.Path(f'{config}/_common/common.runtime.properties').is_file() is False: - raise ValueError(f'_common/common.runtime.properties file is missing in the root config, check {QUICKSTART_ROOT_CONFIG_PATH}/_common directory') - - if pathlib.Path(f'{config}/_common/common.jvm.config').is_file() is False: - raise ValueError(f'_common/jvm.config file is missing in the root config, check {QUICKSTART_ROOT_CONFIG_PATH}/_common directory') + raise ValueError(f'_common/common.runtime.properties file is missing in the root config, ' + f'check {QUICKSTART_ROOT_CONFIG_PATH}/_common directory') def parse_arguments(args): @@ -211,14 +215,29 @@ def parse_arguments(args): def print_startup_config(service_list, config, zk): - print_if_verbose(f'starting {service_list}') - print_if_verbose(f'reading config from {config}') + print_if_verbose(f'Starting {service_list}') + print_if_verbose(f'Reading config from {config}') if zk: zk_config = pathlib.Path(f'{os.getcwd()}/../conf/zk').resolve() - print_if_verbose(f'starting zk, reading default config from {zk_config}') + print_if_verbose(f'Starting zk, reading default config from {zk_config}') print_if_verbose('\n') +def middle_manager_task_memory_params_present(config): + java_opts_property_present = False + worker_capacity_property_present = False + + if pathlib.Path(f'{config}/middleManager/runtime.properties').is_file(): + with open(f'{config}/middleManager/runtime.properties') as file: + for line in file: + if line.startswith(TASK_JAVA_OPTS_PROPERTY): + java_opts_property_present = True + elif line.startswith(TASK_WORKER_CAPACITY_PROPERTY): + worker_capacity_property_present = True + + return java_opts_property_present, worker_capacity_property_present + + def verify_service_config(service, config): path = f'{config}/{service}/jvm.config' @@ -229,9 +248,9 @@ def verify_service_config(service, config): with open(path) as file: for line in file: - if line.startswith(XMX_PARAMETER): + if line.startswith(XMX_PARAMETER) and XMX_PARAMETER in required_parameters: required_parameters.remove(XMX_PARAMETER) - if line.startswith(XMS_PARAMETER): + if line.startswith(XMS_PARAMETER) and XMS_PARAMETER in required_parameters: required_parameters.remove(XMS_PARAMETER) if line.startswith(DIRECT_MEM_PARAMETER) and DIRECT_MEM_PARAMETER in required_parameters: required_parameters.remove(DIRECT_MEM_PARAMETER) @@ -244,38 +263,57 @@ def verify_service_config(service, config): if pathlib.Path(f'{config}/{service}/runtime.properties').is_file() is False: raise ValueError(f'{service}/runtime.properties file is missing in the root config') - java_opts_property_present = False - with open(f'{config}/{service}/runtime.properties') as file: - for line in file: - if line.startswith(TASK_JAVA_OPTS_PROPERTY): - java_opts_property_present = True + mm_task_java_opts_property, mm_task_worker_capacity_prop = middle_manager_task_memory_params_present(config) - if java_opts_property_present is False: + if mm_task_java_opts_property is False: raise ValueError(f'{TASK_JAVA_OPTS_PROPERTY} property missing in {service}/runtime.properties') def should_compute_memory(config, total_memory, service_list): - # if jvm file is present for any of the services - # it should be present for all services and memory should not be specified - # if memory is given, jvm file shouldn't be present for any service + """ + if memory argument is given, memory for services and tasks is computed, jvm.config file + or runtime.properties with task memory specification shouldn't be present + Alternatively, all memory related parameters are specified + which implies following should be present: + jvm.config file for all services with -Xmx=***, Xms=*** parameters + -XX:MaxDirectMemorySize=** in jvm.config for broker and historical + druid.indexer.runner.javaOptsArray (optionally druid.worker.capacity) in + rootDirectory/middleManager/runtime.properties + """ + jvm_config_count = 0 for service in service_list: if pathlib.Path(f'{config}/{service}/jvm.config').is_file(): jvm_config_count += 1 - elif jvm_config_count > 0: - raise ValueError( - f'jvm.config file is missing for service {service}, jvm.config should be specified for all the ' - f'services or none') - if jvm_config_count > 0: - if jvm_config_count != len(service_list): - raise ValueError("jvm.config file should be present for all services or none") + mm_task_property_present = False + if MIDDLE_MANAGER in service_list: + mm_task_java_opts_property, mm_task_worker_capacity_prop = middle_manager_task_memory_params_present(config) + mm_task_property_present = mm_task_java_opts_property or mm_task_worker_capacity_prop + + # possible error states + # 1. memory argument is specified, also jvm.config or middleManger/runtime.properties having + # druid.indexer.runner.javaOptsArray or druid.worker.capacity parameters is present + # 2. jvm.config is not present for any service, but middleManger/runtime.properties has + # druid.indexer.runner.javaOptsArray or druid.worker.capacity parameters + # 3. jvm.config present for some but not all services + # 4. jvm.config file is present for all services, but it doesn't contain required parameters + # 5. lastly, if middleManager is to be started, and it is missing task memory properties + if jvm_config_count > 0 or mm_task_property_present: if total_memory != "": raise ValueError("If jvm.config is given for services, memory argument shouldn't be specified") + if jvm_config_count == 0: + raise ValueError("druid.indexer.runner.javaOptsArray or druid.worker.capacity is present in " + "middleManager/runtime.properties, \n " + "add jvm.config for all other services") + if jvm_config_count != len(service_list): + raise ValueError("jvm.config file should be present for all services or none") for service in service_list: verify_service_config(service, config) - return jvm_config_count == 0 + # compute memory only when none of the specified services contains jvm.config, + # if middleManager is to be started it doesn't contain task memory properties + return jvm_config_count == 0 and mm_task_property_present is False def compute_system_memory(): @@ -476,13 +514,16 @@ def main(): service_memory_config = {} if should_compute_memory(config, total_memory, service_list): + # if memory is to be computed, _common directory should contain common.jvm.config + validate_common_jvm_args(config) memory_in_mega_bytes = convert_total_memory_string(total_memory) - print_if_verbose(f'total memory is {memory_in_mega_bytes}m') + print_if_verbose(f'Total memory is {memory_in_mega_bytes}m\n') memory_to_be_used = check_memory_constraint(memory_in_mega_bytes, service_list) - print_if_verbose(f'memory used for services & tasks {memory_to_be_used}m') + print_if_verbose(f'Memory used for services & tasks {memory_to_be_used}m\n') service_memory_config = distribute_memory(service_list, memory_to_be_used) else: - print_if_verbose('not computing memory distribution, reading memory specification from service jvm.config\n') + print_if_verbose('Not computing memory distribution, reading memory specification from service jvm.config & ' + 'middleManager/runtime.properties\n') script_arguments = build_supervise_script_arguments(service_list, service_memory_config, config, zk) @@ -491,6 +532,7 @@ def main(): os.execv('./supervise', script_arguments) + try: main() except (KeyboardInterrupt, ValueError) as error: From b8f01cc60caaa29423a864ad9ce2f8b61fc632b2 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 24 Nov 2022 14:06:26 +0530 Subject: [PATCH 30/82] remove start-druid bash script, rename start-druid-main --- examples/bin/start-druid | 526 ++++++++++++++++++++++++++++++++- examples/bin/start-druid-main | 540 ---------------------------------- 2 files changed, 521 insertions(+), 545 deletions(-) delete mode 100644 examples/bin/start-druid-main diff --git a/examples/bin/start-druid b/examples/bin/start-druid index 601df55ffef5..f5acbc5df6b9 100644 --- a/examples/bin/start-druid +++ b/examples/bin/start-druid @@ -1,4 +1,4 @@ -#!/bin/bash -eu +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -17,8 +17,524 @@ # specific language governing permissions and limitations # under the License. -PWD="$(pwd)" -WHEREAMI="$(dirname "$0")" -WHEREAMI="$(cd "$WHEREAMI" && pwd)" +import sys +import os +import psutil +import pathlib +import multiprocessing +import argparse -exec "$WHEREAMI/start-druid-main" "$@" +QUICKSTART_ROOT_CONFIG_PATH = "conf/druid/single-server/quickstart" + +MEM_GB_SUFFIX = "g" +MEM_MB_SUFFIX = "m" +XMX_PARAMETER = "-Xmx" +XMS_PARAMETER = "-Xms" +DIRECT_MEM_PARAMETER = "-XX:MaxDirectMemorySize" +SERVICE_SEPARATOR = "," + +TASK_JAVA_OPTS_ARRAY = ["-server", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-XX:+ExitOnOutOfMemoryError", + "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] +TASK_JAVA_OPTS_PROPERTY = "druid.indexer.runner.javaOptsArray" +TASK_WORKER_CAPACITY_PROPERTY = "druid.worker.capacity" +TASK_COUNT = "task-count" +TASK_MEM_TYPE_LOW = "low" +TASK_MEM_TYPE_HIGH = "high" +TASK_MEM_MAP = { + TASK_MEM_TYPE_LOW: ["-Xms256m", "-Xmx256m", "-XX:MaxDirectMemorySize=256g"], + TASK_MEM_TYPE_HIGH: ["-Xms1g", "-Xmx1g", "-XX:MaxDirectMemorySize=1g"] +} + +BROKER = "broker" +ROUTER = "router" +COORDINATOR = "coordinator-overlord" +HISTORICAL = "historical" +MIDDLE_MANAGER = "middleManager" +TASKS = "tasks" + +DEFAULT_SERVICES = [ + BROKER, + ROUTER, + COORDINATOR, + HISTORICAL, + MIDDLE_MANAGER +] + +SERVICE_MEMORY_RATIO = { + MIDDLE_MANAGER: 1, + ROUTER: 2, + COORDINATOR: 30, + BROKER: 46, + HISTORICAL: 80, + TASKS: 30 +} + +MINIMUM_MEMORY_MB = { + MIDDLE_MANAGER: 64, + ROUTER: 128, + TASKS: 1024, + BROKER: 900, + COORDINATOR: 256, + HISTORICAL: 900 +} + +HEAP_TO_TOTAL_MEM_RATIO = { + MIDDLE_MANAGER: 1, + ROUTER: 1, + COORDINATOR: 1, + BROKER: 0.60, + HISTORICAL: 0.40, + TASKS: 0.50 +} + +LOGGING_ENABLED = False + + +def print_if_verbose(message): + if LOGGING_ENABLED: + print(message) + + +def configure_parser(): + parser = argparse.ArgumentParser( + prog='Druid quickstart', + formatter_class=argparse.RawTextHelpFormatter, + epilog= + """ +sample usage: + start-druid + Start up all the services (including zk). + start-druid -m=100g + Start up all the services (including zk) + using a total memory of 100GB. + start-druid -m=100g --compute + Compute memory distribution and validate arguments. + start-druid -m=100g -s=broker,router + Starts a broker and a router, using a total memory of 100GB. + start-druid -m=100g --s=broker,router \\ + -c=conf/druid/single-server/custom + Starts a broker and a router, using a total memory of 100GB. + Reads configs for each service (jvm.config, runtime.properties) + from respective folders inside the given root config path. + start-druid -s=broker,router \\ + -c=conf/druid/single-server/custom + Starts a broker and a router service, reading service configs + from the given root directory. Calculates memory requirements for + each service, if required, using upto 80% of the total system memory. + start-druid -m=100g \\ + -s=broker,router \\ + -c=conf/druid/single-server/custom \\ + --zk + Starts broker, router and zookeeper. + zookeeper config is read from conf/zk. +""" + ) + parser.add_argument('--memory', '-m', type=str, required=False, + help='Total memory for all processes (services and tasks, if any). \n' + 'This parameter is ignored if each service already has a jvm.config \n' + 'in the given conf directory. e.g. 500m, 4g, 6g\n') + parser.add_argument('--services', '-s', type=str, required=False, + help='List of services to be started, subset of \n' + '{broker, router, middleManager, historical, coordinator-overlord}. \n' + 'If the argument is not given, all services \n' + 'and zookeeper is started. e.g. -sl=broker,historical') + parser.add_argument('--config', '-c', type=str, required=False, + help='Relative path to the directory containing common and service \n' + 'specific properties to be overridden. \n' + 'This directory must contain \'_common\' directory with \n' + '\'common.jvm.config\' & \'common.runtime.properties\' files. \n' + 'If this argument is not given, config from \n' + 'conf/druid/single-server/quickstart directory is used.\n') + parser.add_argument('--compute', action='store_true', + help='Does not start Druid, only displays the memory allocated \n' + 'to each service if started with the given total memory.\n') + parser.add_argument('--zk', '-zk', action='store_true', + help='Specification to run zookeeper, \n' + 'zk config is picked up from conf/zk.') + parser.add_argument('--verbose', action='store_true', help='Log details') + + parser.set_defaults(zk=False) + parser.set_defaults(compute=False) + parser.set_defaults(verbose=False) + + return parser + + +def validate_common_jvm_args(config): + if pathlib.Path(f'{config}/_common/common.jvm.config').is_file() is False: + raise ValueError(f'_common/common.jvm.config file is missing in the root config, ' + f'check {QUICKSTART_ROOT_CONFIG_PATH}/_common directory') + + +def validate_common_directory(config): + if pathlib.Path(f'{config}/_common').is_dir() is False: + raise ValueError( + f'_common directory is missing in the root config, check {QUICKSTART_ROOT_CONFIG_PATH}/_common directory') + + if pathlib.Path(f'{config}/_common/common.runtime.properties').is_file() is False: + raise ValueError(f'_common/common.runtime.properties file is missing in the root config, ' + f'check {QUICKSTART_ROOT_CONFIG_PATH}/_common directory') + + +def parse_arguments(args): + service_list = [] + config = "" + total_memory = "" + compute = False + zk = False + + if args.compute: + compute = True + if args.zk: + zk = True + if args.config is not None: + config = pathlib.Path(os.path.join(os.getcwd(), args.config)).resolve() + if os.path.exists(config) is False: + raise ValueError(f'config {config} not found') + validate_common_directory(config) + if args.memory is not None: + total_memory = args.memory + if args.services is not None: + services = args.services.split(SERVICE_SEPARATOR) + + for service in services: + if service not in DEFAULT_SERVICES: + raise ValueError(f'Invalid service name {service}, should be one of {DEFAULT_SERVICES}') + + if service in service_list: + raise ValueError(f'{service} is specified multiple times') + + service_list.append(service) + + if len(service_list) == 0: + # start all services + service_list = DEFAULT_SERVICES + zk = True + + return config, total_memory, service_list, zk, compute + + +def print_startup_config(service_list, config, zk): + print_if_verbose(f'Starting {service_list}') + print_if_verbose(f'Reading config from {config}') + if zk: + zk_config = pathlib.Path(f'{os.getcwd()}/../conf/zk').resolve() + print_if_verbose(f'Starting zk, reading default config from {zk_config}') + print_if_verbose('\n') + + +def middle_manager_task_memory_params_present(config): + java_opts_property_present = False + worker_capacity_property_present = False + + if pathlib.Path(f'{config}/middleManager/runtime.properties').is_file(): + with open(f'{config}/middleManager/runtime.properties') as file: + for line in file: + if line.startswith(TASK_JAVA_OPTS_PROPERTY): + java_opts_property_present = True + elif line.startswith(TASK_WORKER_CAPACITY_PROPERTY): + worker_capacity_property_present = True + + return java_opts_property_present, worker_capacity_property_present + + +def verify_service_config(service, config): + path = f'{config}/{service}/jvm.config' + + required_parameters = [XMX_PARAMETER, XMS_PARAMETER] + + if HEAP_TO_TOTAL_MEM_RATIO.get(service) != 1: + required_parameters.append(DIRECT_MEM_PARAMETER) + + with open(path) as file: + for line in file: + if line.startswith(XMX_PARAMETER) and XMX_PARAMETER in required_parameters: + required_parameters.remove(XMX_PARAMETER) + if line.startswith(XMS_PARAMETER) and XMS_PARAMETER in required_parameters: + required_parameters.remove(XMS_PARAMETER) + if line.startswith(DIRECT_MEM_PARAMETER) and DIRECT_MEM_PARAMETER in required_parameters: + required_parameters.remove(DIRECT_MEM_PARAMETER) + + if len(required_parameters) > 0: + params = ",".join(required_parameters) + raise ValueError(f'{params} missing in {service}/jvm.config') + + if service == MIDDLE_MANAGER: + if pathlib.Path(f'{config}/{service}/runtime.properties').is_file() is False: + raise ValueError(f'{service}/runtime.properties file is missing in the root config') + + mm_task_java_opts_property, mm_task_worker_capacity_prop = middle_manager_task_memory_params_present(config) + + if mm_task_java_opts_property is False: + raise ValueError(f'{TASK_JAVA_OPTS_PROPERTY} property missing in {service}/runtime.properties') + + +def should_compute_memory(config, total_memory, service_list): + """ + if memory argument is given, memory for services and tasks is computed, jvm.config file + or runtime.properties with task memory specification shouldn't be present + Alternatively, all memory related parameters are specified + which implies following should be present: + jvm.config file for all services with -Xmx=***, Xms=*** parameters + -XX:MaxDirectMemorySize=** in jvm.config for broker and historical + druid.indexer.runner.javaOptsArray (optionally druid.worker.capacity) in + rootDirectory/middleManager/runtime.properties + """ + + jvm_config_count = 0 + for service in service_list: + if pathlib.Path(f'{config}/{service}/jvm.config').is_file(): + jvm_config_count += 1 + + mm_task_property_present = False + if MIDDLE_MANAGER in service_list: + mm_task_java_opts_property, mm_task_worker_capacity_prop = middle_manager_task_memory_params_present(config) + mm_task_property_present = mm_task_java_opts_property or mm_task_worker_capacity_prop + + # possible error states + # 1. memory argument is specified, also jvm.config or middleManger/runtime.properties having + # druid.indexer.runner.javaOptsArray or druid.worker.capacity parameters is present + # 2. jvm.config is not present for any service, but middleManger/runtime.properties has + # druid.indexer.runner.javaOptsArray or druid.worker.capacity parameters + # 3. jvm.config present for some but not all services + # 4. jvm.config file is present for all services, but it doesn't contain required parameters + # 5. lastly, if middleManager is to be started, and it is missing task memory properties + if jvm_config_count > 0 or mm_task_property_present: + if total_memory != "": + raise ValueError("If jvm.config is given for services, memory argument shouldn't be specified") + if jvm_config_count == 0: + raise ValueError("druid.indexer.runner.javaOptsArray or druid.worker.capacity is present in " + "middleManager/runtime.properties, \n " + "add jvm.config for all other services") + if jvm_config_count != len(service_list): + raise ValueError("jvm.config file should be present for all services or none") + for service in service_list: + verify_service_config(service, config) + + # compute memory only when none of the specified services contains jvm.config, + # if middleManager is to be started it doesn't contain task memory properties + return jvm_config_count == 0 and mm_task_property_present is False + + +def compute_system_memory(): + system_memory = psutil.virtual_memory().total # mem in bytes + memory_for_druid = int(system_memory / (1024 * 1024)) + return memory_for_druid + + +def convert_total_memory_string(memory): + try: + if memory == "": + computed_memory = compute_system_memory() + return computed_memory + elif memory.endswith(MEM_MB_SUFFIX): + return int(memory[:-1]) + elif memory.endswith(MEM_GB_SUFFIX): + return 1024 * int(memory[:-1]) + else: + raise ValueError('Incorrect format for memory argument, expected format is ') + except Exception: + raise ValueError('Incorrect format for memory argument, expected format is ') + + +def check_memory_constraint(total_memory, service_list): + # 80% of total memory >= sum of lower bound service memory should be + lower_bound_memory = 0 + + for service in service_list: + lower_bound_memory += MINIMUM_MEMORY_MB.get(service) + + required_memory = int(lower_bound_memory / 0.8) + + if total_memory < required_memory: + raise ValueError(f'Minimum memory required for starting services is {required_memory}m') + + if total_memory >= 2 * lower_bound_memory: + return int(total_memory / 2) + else: + return lower_bound_memory + + +def build_mm_task_java_opts_array(memory_type): + task_memory = f'-D{TASK_JAVA_OPTS_PROPERTY}=[' + + mem_array = TASK_MEM_MAP.get(memory_type) + + java_opts_list = TASK_JAVA_OPTS_ARRAY + mem_array + + for item in java_opts_list: + task_memory += f'\"{item}\";' + + task_memory = task_memory[:-1] + task_memory += ']' + return task_memory + + +def compute_tasks_memory(allocated_memory): + if allocated_memory >= 2048: + task_count = int(allocated_memory / 2048) + memory_type = TASK_MEM_TYPE_HIGH + task_memory_mb = 2048 + else: + task_count = int(allocated_memory / 512) + memory_type = TASK_MEM_TYPE_LOW + task_memory_mb = 512 + task_count = min(task_count, multiprocessing.cpu_count()) + + return memory_type, task_count, task_memory_mb + + +def build_memory_config(service, allocated_memory): + if service == TASKS: + memory_type, task_count, task_memory = compute_tasks_memory(allocated_memory) + java_opts_array = build_mm_task_java_opts_array(memory_type) + return [f'-D{TASK_WORKER_CAPACITY_PROPERTY}={task_count}', java_opts_array], task_memory * task_count + else: + heap_memory = HEAP_TO_TOTAL_MEM_RATIO.get(service) * allocated_memory + direct_memory = int(allocated_memory - heap_memory) + heap_memory = int(heap_memory) + + if direct_memory == 0: + return f'-Xms{heap_memory}m -Xmx{heap_memory}m', allocated_memory + + return f'-Xms{heap_memory}m -Xmx{heap_memory}m -XX:MaxDirectMemorySize={direct_memory}m', allocated_memory + + +def distribute_memory(services, total_memory): + service_memory_config = {} + + memory_weight_sum = 0 + + service_list = services.copy() + if MIDDLE_MANAGER in services: + service_list.append(TASKS) + + for service in service_list: + memory_weight_sum += SERVICE_MEMORY_RATIO.get(service) + + multiplier = total_memory / memory_weight_sum + + lower_bound_memory_allocation = 0 + allocated_services = set() + + for service in service_list: + allocated_memory = SERVICE_MEMORY_RATIO.get(service) * multiplier + if service in MINIMUM_MEMORY_MB and allocated_memory < MINIMUM_MEMORY_MB.get(service): + allocated_memory = MINIMUM_MEMORY_MB.get(service) + service_memory_config[service], allocated_memory = build_memory_config(service, allocated_memory) + lower_bound_memory_allocation += allocated_memory + allocated_services.add(service) + + if lower_bound_memory_allocation > 0: + # compute the multiplier again for remaining services + memory_weight_sum = 0 + for service in service_list: + if service in allocated_services: + continue + memory_weight_sum += SERVICE_MEMORY_RATIO.get(service) + multiplier = (total_memory - lower_bound_memory_allocation) / memory_weight_sum + + for service in service_list: + if service in allocated_services: + continue + allocated_memory = SERVICE_MEMORY_RATIO.get(service) * multiplier + if service in MINIMUM_MEMORY_MB and allocated_memory < MINIMUM_MEMORY_MB.get(service): + allocated_memory = MINIMUM_MEMORY_MB.get(service) + + service_memory_config[service], allocated_memory = build_memory_config(service, allocated_memory) + + print_if_verbose(f'\nMemory distribution for services:') + for key, value in service_memory_config.items(): + print_if_verbose(f'{key}, {value}') + print_if_verbose('\n') + + return service_memory_config + + +def build_supervise_script_arguments(service_list, service_memory_config, config, zk): + args = [] + commands = [] + args.append('supervise') + args.append('-a') + + commands.append(":verify bin/verify-java") + commands.append(":verify bin/verify-default-ports") + commands.append(":notify bin/greet") + commands.append(":kill-timeout 10") + + if zk: + commands.append("!p10 zk bin/run-zk conf") + + for service in service_list: + jvm_args = service_memory_config.get(service) + + prefix = '' + if service == MIDDLE_MANAGER: + prefix = '!p90 ' + + if jvm_args is None: + commands.append(f'{prefix}{service} bin/run-druid {service} {config}') + else: + if service == MIDDLE_MANAGER: + task_config = service_memory_config.get(TASKS) + task_count = task_config[0] + task_memory = task_config[1] + commands.append( + f'{prefix}{service} bin/run-druid {service} {config} \'{jvm_args}\' \'{task_count} {task_memory}\'') + else: + commands.append(f'{prefix}{service} bin/run-druid {service} {config} \'{jvm_args}\'') + + print_if_verbose('Supervise script args:') + for item in commands: + print_if_verbose(item) + + print_if_verbose('\n') + + args.append(",".join(commands)) + return args + + +def main(): + parser = configure_parser() + args = parser.parse_args() + + global LOGGING_ENABLED + LOGGING_ENABLED = args.verbose or args.compute + + config, total_memory, service_list, zk, compute = parse_arguments(args) + + # change directory to bin + os.chdir(os.path.dirname(sys.argv[0])) + + if config == "": + config = pathlib.Path(f'{os.getcwd()}/../{QUICKSTART_ROOT_CONFIG_PATH}').resolve() + + print_startup_config(service_list, config, zk) + + service_memory_config = {} + + if should_compute_memory(config, total_memory, service_list): + # if memory is to be computed, _common directory should contain common.jvm.config + validate_common_jvm_args(config) + memory_in_mega_bytes = convert_total_memory_string(total_memory) + print_if_verbose(f'Total memory is {memory_in_mega_bytes}m\n') + memory_to_be_used = check_memory_constraint(memory_in_mega_bytes, service_list) + print_if_verbose(f'Memory used for services & tasks {memory_to_be_used}m\n') + service_memory_config = distribute_memory(service_list, memory_to_be_used) + else: + print_if_verbose('Not computing memory distribution, reading memory specification from service jvm.config & ' + 'middleManager/runtime.properties\n') + + script_arguments = build_supervise_script_arguments(service_list, service_memory_config, config, zk) + + if compute: + return + + os.execv('./supervise', script_arguments) + + +try: + main() +except (KeyboardInterrupt, ValueError) as error: + print(error) + sys.exit(1) diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main deleted file mode 100644 index f5acbc5df6b9..000000000000 --- a/examples/bin/start-druid-main +++ /dev/null @@ -1,540 +0,0 @@ -#!/usr/bin/env python3 - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import sys -import os -import psutil -import pathlib -import multiprocessing -import argparse - -QUICKSTART_ROOT_CONFIG_PATH = "conf/druid/single-server/quickstart" - -MEM_GB_SUFFIX = "g" -MEM_MB_SUFFIX = "m" -XMX_PARAMETER = "-Xmx" -XMS_PARAMETER = "-Xms" -DIRECT_MEM_PARAMETER = "-XX:MaxDirectMemorySize" -SERVICE_SEPARATOR = "," - -TASK_JAVA_OPTS_ARRAY = ["-server", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-XX:+ExitOnOutOfMemoryError", - "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] -TASK_JAVA_OPTS_PROPERTY = "druid.indexer.runner.javaOptsArray" -TASK_WORKER_CAPACITY_PROPERTY = "druid.worker.capacity" -TASK_COUNT = "task-count" -TASK_MEM_TYPE_LOW = "low" -TASK_MEM_TYPE_HIGH = "high" -TASK_MEM_MAP = { - TASK_MEM_TYPE_LOW: ["-Xms256m", "-Xmx256m", "-XX:MaxDirectMemorySize=256g"], - TASK_MEM_TYPE_HIGH: ["-Xms1g", "-Xmx1g", "-XX:MaxDirectMemorySize=1g"] -} - -BROKER = "broker" -ROUTER = "router" -COORDINATOR = "coordinator-overlord" -HISTORICAL = "historical" -MIDDLE_MANAGER = "middleManager" -TASKS = "tasks" - -DEFAULT_SERVICES = [ - BROKER, - ROUTER, - COORDINATOR, - HISTORICAL, - MIDDLE_MANAGER -] - -SERVICE_MEMORY_RATIO = { - MIDDLE_MANAGER: 1, - ROUTER: 2, - COORDINATOR: 30, - BROKER: 46, - HISTORICAL: 80, - TASKS: 30 -} - -MINIMUM_MEMORY_MB = { - MIDDLE_MANAGER: 64, - ROUTER: 128, - TASKS: 1024, - BROKER: 900, - COORDINATOR: 256, - HISTORICAL: 900 -} - -HEAP_TO_TOTAL_MEM_RATIO = { - MIDDLE_MANAGER: 1, - ROUTER: 1, - COORDINATOR: 1, - BROKER: 0.60, - HISTORICAL: 0.40, - TASKS: 0.50 -} - -LOGGING_ENABLED = False - - -def print_if_verbose(message): - if LOGGING_ENABLED: - print(message) - - -def configure_parser(): - parser = argparse.ArgumentParser( - prog='Druid quickstart', - formatter_class=argparse.RawTextHelpFormatter, - epilog= - """ -sample usage: - start-druid - Start up all the services (including zk). - start-druid -m=100g - Start up all the services (including zk) - using a total memory of 100GB. - start-druid -m=100g --compute - Compute memory distribution and validate arguments. - start-druid -m=100g -s=broker,router - Starts a broker and a router, using a total memory of 100GB. - start-druid -m=100g --s=broker,router \\ - -c=conf/druid/single-server/custom - Starts a broker and a router, using a total memory of 100GB. - Reads configs for each service (jvm.config, runtime.properties) - from respective folders inside the given root config path. - start-druid -s=broker,router \\ - -c=conf/druid/single-server/custom - Starts a broker and a router service, reading service configs - from the given root directory. Calculates memory requirements for - each service, if required, using upto 80% of the total system memory. - start-druid -m=100g \\ - -s=broker,router \\ - -c=conf/druid/single-server/custom \\ - --zk - Starts broker, router and zookeeper. - zookeeper config is read from conf/zk. -""" - ) - parser.add_argument('--memory', '-m', type=str, required=False, - help='Total memory for all processes (services and tasks, if any). \n' - 'This parameter is ignored if each service already has a jvm.config \n' - 'in the given conf directory. e.g. 500m, 4g, 6g\n') - parser.add_argument('--services', '-s', type=str, required=False, - help='List of services to be started, subset of \n' - '{broker, router, middleManager, historical, coordinator-overlord}. \n' - 'If the argument is not given, all services \n' - 'and zookeeper is started. e.g. -sl=broker,historical') - parser.add_argument('--config', '-c', type=str, required=False, - help='Relative path to the directory containing common and service \n' - 'specific properties to be overridden. \n' - 'This directory must contain \'_common\' directory with \n' - '\'common.jvm.config\' & \'common.runtime.properties\' files. \n' - 'If this argument is not given, config from \n' - 'conf/druid/single-server/quickstart directory is used.\n') - parser.add_argument('--compute', action='store_true', - help='Does not start Druid, only displays the memory allocated \n' - 'to each service if started with the given total memory.\n') - parser.add_argument('--zk', '-zk', action='store_true', - help='Specification to run zookeeper, \n' - 'zk config is picked up from conf/zk.') - parser.add_argument('--verbose', action='store_true', help='Log details') - - parser.set_defaults(zk=False) - parser.set_defaults(compute=False) - parser.set_defaults(verbose=False) - - return parser - - -def validate_common_jvm_args(config): - if pathlib.Path(f'{config}/_common/common.jvm.config').is_file() is False: - raise ValueError(f'_common/common.jvm.config file is missing in the root config, ' - f'check {QUICKSTART_ROOT_CONFIG_PATH}/_common directory') - - -def validate_common_directory(config): - if pathlib.Path(f'{config}/_common').is_dir() is False: - raise ValueError( - f'_common directory is missing in the root config, check {QUICKSTART_ROOT_CONFIG_PATH}/_common directory') - - if pathlib.Path(f'{config}/_common/common.runtime.properties').is_file() is False: - raise ValueError(f'_common/common.runtime.properties file is missing in the root config, ' - f'check {QUICKSTART_ROOT_CONFIG_PATH}/_common directory') - - -def parse_arguments(args): - service_list = [] - config = "" - total_memory = "" - compute = False - zk = False - - if args.compute: - compute = True - if args.zk: - zk = True - if args.config is not None: - config = pathlib.Path(os.path.join(os.getcwd(), args.config)).resolve() - if os.path.exists(config) is False: - raise ValueError(f'config {config} not found') - validate_common_directory(config) - if args.memory is not None: - total_memory = args.memory - if args.services is not None: - services = args.services.split(SERVICE_SEPARATOR) - - for service in services: - if service not in DEFAULT_SERVICES: - raise ValueError(f'Invalid service name {service}, should be one of {DEFAULT_SERVICES}') - - if service in service_list: - raise ValueError(f'{service} is specified multiple times') - - service_list.append(service) - - if len(service_list) == 0: - # start all services - service_list = DEFAULT_SERVICES - zk = True - - return config, total_memory, service_list, zk, compute - - -def print_startup_config(service_list, config, zk): - print_if_verbose(f'Starting {service_list}') - print_if_verbose(f'Reading config from {config}') - if zk: - zk_config = pathlib.Path(f'{os.getcwd()}/../conf/zk').resolve() - print_if_verbose(f'Starting zk, reading default config from {zk_config}') - print_if_verbose('\n') - - -def middle_manager_task_memory_params_present(config): - java_opts_property_present = False - worker_capacity_property_present = False - - if pathlib.Path(f'{config}/middleManager/runtime.properties').is_file(): - with open(f'{config}/middleManager/runtime.properties') as file: - for line in file: - if line.startswith(TASK_JAVA_OPTS_PROPERTY): - java_opts_property_present = True - elif line.startswith(TASK_WORKER_CAPACITY_PROPERTY): - worker_capacity_property_present = True - - return java_opts_property_present, worker_capacity_property_present - - -def verify_service_config(service, config): - path = f'{config}/{service}/jvm.config' - - required_parameters = [XMX_PARAMETER, XMS_PARAMETER] - - if HEAP_TO_TOTAL_MEM_RATIO.get(service) != 1: - required_parameters.append(DIRECT_MEM_PARAMETER) - - with open(path) as file: - for line in file: - if line.startswith(XMX_PARAMETER) and XMX_PARAMETER in required_parameters: - required_parameters.remove(XMX_PARAMETER) - if line.startswith(XMS_PARAMETER) and XMS_PARAMETER in required_parameters: - required_parameters.remove(XMS_PARAMETER) - if line.startswith(DIRECT_MEM_PARAMETER) and DIRECT_MEM_PARAMETER in required_parameters: - required_parameters.remove(DIRECT_MEM_PARAMETER) - - if len(required_parameters) > 0: - params = ",".join(required_parameters) - raise ValueError(f'{params} missing in {service}/jvm.config') - - if service == MIDDLE_MANAGER: - if pathlib.Path(f'{config}/{service}/runtime.properties').is_file() is False: - raise ValueError(f'{service}/runtime.properties file is missing in the root config') - - mm_task_java_opts_property, mm_task_worker_capacity_prop = middle_manager_task_memory_params_present(config) - - if mm_task_java_opts_property is False: - raise ValueError(f'{TASK_JAVA_OPTS_PROPERTY} property missing in {service}/runtime.properties') - - -def should_compute_memory(config, total_memory, service_list): - """ - if memory argument is given, memory for services and tasks is computed, jvm.config file - or runtime.properties with task memory specification shouldn't be present - Alternatively, all memory related parameters are specified - which implies following should be present: - jvm.config file for all services with -Xmx=***, Xms=*** parameters - -XX:MaxDirectMemorySize=** in jvm.config for broker and historical - druid.indexer.runner.javaOptsArray (optionally druid.worker.capacity) in - rootDirectory/middleManager/runtime.properties - """ - - jvm_config_count = 0 - for service in service_list: - if pathlib.Path(f'{config}/{service}/jvm.config').is_file(): - jvm_config_count += 1 - - mm_task_property_present = False - if MIDDLE_MANAGER in service_list: - mm_task_java_opts_property, mm_task_worker_capacity_prop = middle_manager_task_memory_params_present(config) - mm_task_property_present = mm_task_java_opts_property or mm_task_worker_capacity_prop - - # possible error states - # 1. memory argument is specified, also jvm.config or middleManger/runtime.properties having - # druid.indexer.runner.javaOptsArray or druid.worker.capacity parameters is present - # 2. jvm.config is not present for any service, but middleManger/runtime.properties has - # druid.indexer.runner.javaOptsArray or druid.worker.capacity parameters - # 3. jvm.config present for some but not all services - # 4. jvm.config file is present for all services, but it doesn't contain required parameters - # 5. lastly, if middleManager is to be started, and it is missing task memory properties - if jvm_config_count > 0 or mm_task_property_present: - if total_memory != "": - raise ValueError("If jvm.config is given for services, memory argument shouldn't be specified") - if jvm_config_count == 0: - raise ValueError("druid.indexer.runner.javaOptsArray or druid.worker.capacity is present in " - "middleManager/runtime.properties, \n " - "add jvm.config for all other services") - if jvm_config_count != len(service_list): - raise ValueError("jvm.config file should be present for all services or none") - for service in service_list: - verify_service_config(service, config) - - # compute memory only when none of the specified services contains jvm.config, - # if middleManager is to be started it doesn't contain task memory properties - return jvm_config_count == 0 and mm_task_property_present is False - - -def compute_system_memory(): - system_memory = psutil.virtual_memory().total # mem in bytes - memory_for_druid = int(system_memory / (1024 * 1024)) - return memory_for_druid - - -def convert_total_memory_string(memory): - try: - if memory == "": - computed_memory = compute_system_memory() - return computed_memory - elif memory.endswith(MEM_MB_SUFFIX): - return int(memory[:-1]) - elif memory.endswith(MEM_GB_SUFFIX): - return 1024 * int(memory[:-1]) - else: - raise ValueError('Incorrect format for memory argument, expected format is ') - except Exception: - raise ValueError('Incorrect format for memory argument, expected format is ') - - -def check_memory_constraint(total_memory, service_list): - # 80% of total memory >= sum of lower bound service memory should be - lower_bound_memory = 0 - - for service in service_list: - lower_bound_memory += MINIMUM_MEMORY_MB.get(service) - - required_memory = int(lower_bound_memory / 0.8) - - if total_memory < required_memory: - raise ValueError(f'Minimum memory required for starting services is {required_memory}m') - - if total_memory >= 2 * lower_bound_memory: - return int(total_memory / 2) - else: - return lower_bound_memory - - -def build_mm_task_java_opts_array(memory_type): - task_memory = f'-D{TASK_JAVA_OPTS_PROPERTY}=[' - - mem_array = TASK_MEM_MAP.get(memory_type) - - java_opts_list = TASK_JAVA_OPTS_ARRAY + mem_array - - for item in java_opts_list: - task_memory += f'\"{item}\";' - - task_memory = task_memory[:-1] - task_memory += ']' - return task_memory - - -def compute_tasks_memory(allocated_memory): - if allocated_memory >= 2048: - task_count = int(allocated_memory / 2048) - memory_type = TASK_MEM_TYPE_HIGH - task_memory_mb = 2048 - else: - task_count = int(allocated_memory / 512) - memory_type = TASK_MEM_TYPE_LOW - task_memory_mb = 512 - task_count = min(task_count, multiprocessing.cpu_count()) - - return memory_type, task_count, task_memory_mb - - -def build_memory_config(service, allocated_memory): - if service == TASKS: - memory_type, task_count, task_memory = compute_tasks_memory(allocated_memory) - java_opts_array = build_mm_task_java_opts_array(memory_type) - return [f'-D{TASK_WORKER_CAPACITY_PROPERTY}={task_count}', java_opts_array], task_memory * task_count - else: - heap_memory = HEAP_TO_TOTAL_MEM_RATIO.get(service) * allocated_memory - direct_memory = int(allocated_memory - heap_memory) - heap_memory = int(heap_memory) - - if direct_memory == 0: - return f'-Xms{heap_memory}m -Xmx{heap_memory}m', allocated_memory - - return f'-Xms{heap_memory}m -Xmx{heap_memory}m -XX:MaxDirectMemorySize={direct_memory}m', allocated_memory - - -def distribute_memory(services, total_memory): - service_memory_config = {} - - memory_weight_sum = 0 - - service_list = services.copy() - if MIDDLE_MANAGER in services: - service_list.append(TASKS) - - for service in service_list: - memory_weight_sum += SERVICE_MEMORY_RATIO.get(service) - - multiplier = total_memory / memory_weight_sum - - lower_bound_memory_allocation = 0 - allocated_services = set() - - for service in service_list: - allocated_memory = SERVICE_MEMORY_RATIO.get(service) * multiplier - if service in MINIMUM_MEMORY_MB and allocated_memory < MINIMUM_MEMORY_MB.get(service): - allocated_memory = MINIMUM_MEMORY_MB.get(service) - service_memory_config[service], allocated_memory = build_memory_config(service, allocated_memory) - lower_bound_memory_allocation += allocated_memory - allocated_services.add(service) - - if lower_bound_memory_allocation > 0: - # compute the multiplier again for remaining services - memory_weight_sum = 0 - for service in service_list: - if service in allocated_services: - continue - memory_weight_sum += SERVICE_MEMORY_RATIO.get(service) - multiplier = (total_memory - lower_bound_memory_allocation) / memory_weight_sum - - for service in service_list: - if service in allocated_services: - continue - allocated_memory = SERVICE_MEMORY_RATIO.get(service) * multiplier - if service in MINIMUM_MEMORY_MB and allocated_memory < MINIMUM_MEMORY_MB.get(service): - allocated_memory = MINIMUM_MEMORY_MB.get(service) - - service_memory_config[service], allocated_memory = build_memory_config(service, allocated_memory) - - print_if_verbose(f'\nMemory distribution for services:') - for key, value in service_memory_config.items(): - print_if_verbose(f'{key}, {value}') - print_if_verbose('\n') - - return service_memory_config - - -def build_supervise_script_arguments(service_list, service_memory_config, config, zk): - args = [] - commands = [] - args.append('supervise') - args.append('-a') - - commands.append(":verify bin/verify-java") - commands.append(":verify bin/verify-default-ports") - commands.append(":notify bin/greet") - commands.append(":kill-timeout 10") - - if zk: - commands.append("!p10 zk bin/run-zk conf") - - for service in service_list: - jvm_args = service_memory_config.get(service) - - prefix = '' - if service == MIDDLE_MANAGER: - prefix = '!p90 ' - - if jvm_args is None: - commands.append(f'{prefix}{service} bin/run-druid {service} {config}') - else: - if service == MIDDLE_MANAGER: - task_config = service_memory_config.get(TASKS) - task_count = task_config[0] - task_memory = task_config[1] - commands.append( - f'{prefix}{service} bin/run-druid {service} {config} \'{jvm_args}\' \'{task_count} {task_memory}\'') - else: - commands.append(f'{prefix}{service} bin/run-druid {service} {config} \'{jvm_args}\'') - - print_if_verbose('Supervise script args:') - for item in commands: - print_if_verbose(item) - - print_if_verbose('\n') - - args.append(",".join(commands)) - return args - - -def main(): - parser = configure_parser() - args = parser.parse_args() - - global LOGGING_ENABLED - LOGGING_ENABLED = args.verbose or args.compute - - config, total_memory, service_list, zk, compute = parse_arguments(args) - - # change directory to bin - os.chdir(os.path.dirname(sys.argv[0])) - - if config == "": - config = pathlib.Path(f'{os.getcwd()}/../{QUICKSTART_ROOT_CONFIG_PATH}').resolve() - - print_startup_config(service_list, config, zk) - - service_memory_config = {} - - if should_compute_memory(config, total_memory, service_list): - # if memory is to be computed, _common directory should contain common.jvm.config - validate_common_jvm_args(config) - memory_in_mega_bytes = convert_total_memory_string(total_memory) - print_if_verbose(f'Total memory is {memory_in_mega_bytes}m\n') - memory_to_be_used = check_memory_constraint(memory_in_mega_bytes, service_list) - print_if_verbose(f'Memory used for services & tasks {memory_to_be_used}m\n') - service_memory_config = distribute_memory(service_list, memory_to_be_used) - else: - print_if_verbose('Not computing memory distribution, reading memory specification from service jvm.config & ' - 'middleManager/runtime.properties\n') - - script_arguments = build_supervise_script_arguments(service_list, service_memory_config, config, zk) - - if compute: - return - - os.execv('./supervise', script_arguments) - - -try: - main() -except (KeyboardInterrupt, ValueError) as error: - print(error) - sys.exit(1) From 6e6305f5ec165e0dc26bcb1312c7ff7f7b7be80f Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 24 Nov 2022 14:56:34 +0530 Subject: [PATCH 31/82] Include tasks in lower bound memory calculation --- examples/bin/start-druid | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/bin/start-druid b/examples/bin/start-druid index f5acbc5df6b9..07f3fdee673e 100644 --- a/examples/bin/start-druid +++ b/examples/bin/start-druid @@ -191,7 +191,6 @@ def parse_arguments(args): config = pathlib.Path(os.path.join(os.getcwd(), args.config)).resolve() if os.path.exists(config) is False: raise ValueError(f'config {config} not found') - validate_common_directory(config) if args.memory is not None: total_memory = args.memory if args.services is not None: @@ -337,10 +336,14 @@ def convert_total_memory_string(memory): raise ValueError('Incorrect format for memory argument, expected format is ') -def check_memory_constraint(total_memory, service_list): +def check_memory_constraint(total_memory, services): # 80% of total memory >= sum of lower bound service memory should be lower_bound_memory = 0 + service_list = services.copy() + if MIDDLE_MANAGER in services: + service_list.append(TASKS) + for service in service_list: lower_bound_memory += MINIMUM_MEMORY_MB.get(service) @@ -509,6 +512,8 @@ def main(): if config == "": config = pathlib.Path(f'{os.getcwd()}/../{QUICKSTART_ROOT_CONFIG_PATH}').resolve() + validate_common_directory(config) + print_startup_config(service_list, config, zk) service_memory_config = {} From a14af0f3cc19bef189b977d548b58eef35419125 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 24 Nov 2022 15:02:21 +0530 Subject: [PATCH 32/82] Fix test --- .../test/java/org/apache/druid/guice/StorageNodeModuleTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/apache/druid/guice/StorageNodeModuleTest.java b/server/src/test/java/org/apache/druid/guice/StorageNodeModuleTest.java index e849bb6910e8..082361a1c885 100644 --- a/server/src/test/java/org/apache/druid/guice/StorageNodeModuleTest.java +++ b/server/src/test/java/org/apache/druid/guice/StorageNodeModuleTest.java @@ -112,7 +112,7 @@ public void getDataNodeServiceWithNoServerTypeConfigShouldThrowProvisionExceptio public void getDataNodeServiceWithNoSegmentCacheConfiguredThrowProvisionException() { exceptionRule.expect(ProvisionException.class); - exceptionRule.expectMessage("Segment cache locations must be set on historicals."); + exceptionRule.expectMessage("druid.segmentCache.locations must be set on historicals."); Mockito.doReturn(ServerType.HISTORICAL).when(serverTypeConfig).getServerType(); mockSegmentCacheNotConfigured(); injector.getInstance(DataNodeService.class); From 3a13e39b32943d888124fec4cdbf7be8c7cd7931 Mon Sep 17 00:00:00 2001 From: Rishabh Singh <6513075+findingrish@users.noreply.github.com> Date: Mon, 28 Nov 2022 23:13:19 +0530 Subject: [PATCH 33/82] 256m instead of 256g --- examples/bin/start-druid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/bin/start-druid b/examples/bin/start-druid index 07f3fdee673e..1bdd0c58a249 100644 --- a/examples/bin/start-druid +++ b/examples/bin/start-druid @@ -41,7 +41,7 @@ TASK_COUNT = "task-count" TASK_MEM_TYPE_LOW = "low" TASK_MEM_TYPE_HIGH = "high" TASK_MEM_MAP = { - TASK_MEM_TYPE_LOW: ["-Xms256m", "-Xmx256m", "-XX:MaxDirectMemorySize=256g"], + TASK_MEM_TYPE_LOW: ["-Xms256m", "-Xmx256m", "-XX:MaxDirectMemorySize=256m"], TASK_MEM_TYPE_HIGH: ["-Xms1g", "-Xmx1g", "-XX:MaxDirectMemorySize=1g"] } From af0d4458664ebcb0bbefef57953e3414d4dd6179 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 29 Nov 2022 15:37:39 +0530 Subject: [PATCH 34/82] caffeine cache uses 5% of heap --- .../main/java/org/apache/druid/client/cache/CaffeineCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/druid/client/cache/CaffeineCache.java b/server/src/main/java/org/apache/druid/client/cache/CaffeineCache.java index b7a1ea826c58..362a387231a6 100644 --- a/server/src/main/java/org/apache/druid/client/cache/CaffeineCache.java +++ b/server/src/main/java/org/apache/druid/client/cache/CaffeineCache.java @@ -70,7 +70,7 @@ public static CaffeineCache create(final CaffeineCacheConfig config, final Execu if (config.getSizeInBytes() >= 0) { builder.maximumWeight(config.getSizeInBytes()); } else { - builder.maximumWeight(Math.min(MAX_DEFAULT_BYTES, JvmUtils.getRuntimeInfo().getMaxHeapSizeBytes() / 10)); + builder.maximumWeight(Math.min(MAX_DEFAULT_BYTES, JvmUtils.getRuntimeInfo().getMaxHeapSizeBytes() / 20)); } builder .weigher((NamedKey key, byte[] value) -> value.length From 6397eefcd235009ebbdb9d9a32f163d2f4fc1601 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 29 Nov 2022 15:38:02 +0530 Subject: [PATCH 35/82] ensure min task count is 2, task count is monotonic --- examples/bin/start-druid | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/bin/start-druid b/examples/bin/start-druid index 1bdd0c58a249..25c282f461c5 100644 --- a/examples/bin/start-druid +++ b/examples/bin/start-druid @@ -97,7 +97,7 @@ def print_if_verbose(message): def configure_parser(): parser = argparse.ArgumentParser( - prog='Druid quickstart', + prog='start-druid', formatter_class=argparse.RawTextHelpFormatter, epilog= """ @@ -300,9 +300,12 @@ def should_compute_memory(config, total_memory, service_list): # 5. lastly, if middleManager is to be started, and it is missing task memory properties if jvm_config_count > 0 or mm_task_property_present: if total_memory != "": - raise ValueError("If jvm.config is given for services, memory argument shouldn't be specified") + raise ValueError( + "If jvm.config for services and/or middleManager configs " + "(druid.worker.capacity, druid.indexer.runner.javaOptsArray) is present, " + "memory argument shouldn't be specified") if jvm_config_count == 0: - raise ValueError("druid.indexer.runner.javaOptsArray or druid.worker.capacity is present in " + raise ValueError("middleManger configs (druid.indexer.runner.javaOptsArray or druid.worker.capacity) is present in " "middleManager/runtime.properties, \n " "add jvm.config for all other services") if jvm_config_count != len(service_list): @@ -374,12 +377,12 @@ def build_mm_task_java_opts_array(memory_type): def compute_tasks_memory(allocated_memory): - if allocated_memory >= 2048: + if allocated_memory >= 4096: task_count = int(allocated_memory / 2048) memory_type = TASK_MEM_TYPE_HIGH task_memory_mb = 2048 else: - task_count = int(allocated_memory / 512) + task_count = 2 memory_type = TASK_MEM_TYPE_LOW task_memory_mb = 512 task_count = min(task_count, multiprocessing.cpu_count()) From a3d442bbaf3de1d71df938546853281ae1ed3ec8 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 29 Nov 2022 15:57:49 +0530 Subject: [PATCH 36/82] update configs and documentation for runtime props in conf/druid/single-server/quickstart --- .../quickstart/broker/runtime.properties | 14 +++++++++----- .../quickstart/historical/runtime.properties | 12 ++++++------ .../quickstart/middleManager/runtime.properties | 9 +++------ .../quickstart/router/runtime.properties | 6 ------ 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/examples/conf/druid/single-server/quickstart/broker/runtime.properties b/examples/conf/druid/single-server/quickstart/broker/runtime.properties index 549117efa50d..f4c494019933 100644 --- a/examples/conf/druid/single-server/quickstart/broker/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/broker/runtime.properties @@ -21,15 +21,19 @@ druid.service=druid/broker druid.plaintextPort=8082 # HTTP server settings -#druid.server.http.numThreads=12 +# HTTP server thread pool size. Higher values increase peak load on the Broker, but +# may be useful for high-concurrency workloads. +# Default is max(10, (Number of processors * 17) / 16 + 2) + 30. +# druid.server.http.numThreads=N # HTTP client settings -#druid.broker.http.numConnections=10 -#druid.broker.http.maxQueuedBytes=5MiB +# Connection pool size from the Broker to each data server. May be useful to +# raise this for high-concurrency workloads. +# druid.broker.http.numConnections=20 # Processing threads and buffers -#druid.processing.buffer.sizeBytes=100MiB -#druid.processing.numMergeBuffers=2 +# Determined automatically based on available memory. For details on how to manually set parameters: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers druid.processing.tmpDir=var/druid/processing # Query cache disabled -- push down caching and merging instead diff --git a/examples/conf/druid/single-server/quickstart/historical/runtime.properties b/examples/conf/druid/single-server/quickstart/historical/runtime.properties index cf3e966cc30e..d9ce1850e4fc 100644 --- a/examples/conf/druid/single-server/quickstart/historical/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/historical/runtime.properties @@ -20,13 +20,14 @@ druid.service=druid/historical druid.plaintextPort=8083 -# HTTP server threads -#druid.server.http.numThreads=12 +# HTTP server thread pool size. Higher values increase peak load on the Broker, but +# may be useful for high-concurrency workloads. +# Default is max(10, (Number of processors * 17) / 16 + 2) + 30. +# druid.server.http.numThreads=N # Processing threads and buffers -#druid.processing.buffer.sizeBytes=200MiB -#druid.processing.numMergeBuffers=2 -#druid.processing.numThreads=2 +# Determined automatically based on available memory. For details on how to manually set parameters: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers druid.processing.tmpDir=var/druid/processing # Segment storage @@ -36,4 +37,3 @@ druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g" druid.historical.cache.useCache=true druid.historical.cache.populateCache=true druid.cache.type=caffeine -#druid.cache.sizeInBytes=10MiB diff --git a/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties b/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties index df9b790a69b0..a90e62579676 100644 --- a/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties @@ -20,17 +20,14 @@ druid.service=druid/middleManager druid.plaintextPort=8091 -# Number of tasks per middleManager -# druid.worker.capacity +# Number of tasks (druid.worker.capacity) and memory usage per task (druid.indexer.runner.javaOptsArray) is automatically +# determined based on available memory. For details on how to manually set parameters, see: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#middlemanager # Task launch parameters druid.indexer.runner.javaCommand=bin/run-java -#druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task -# HTTP server threads -#druid.server.http.numThreads=12 - # Processing threads and buffers on Peons #druid.indexer.fork.property.druid.processing.numMergeBuffers=2 #druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB diff --git a/examples/conf/druid/single-server/quickstart/router/runtime.properties b/examples/conf/druid/single-server/quickstart/router/runtime.properties index 4c38ba88baab..3858dec044bd 100644 --- a/examples/conf/druid/single-server/quickstart/router/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/router/runtime.properties @@ -20,12 +20,6 @@ druid.service=druid/router druid.plaintextPort=8888 -# HTTP proxy -#druid.router.http.numConnections=50 -#druid.router.http.readTimeout=PT5M -#druid.router.http.numMaxThreads=100 -#druid.server.http.numThreads=100 - # Service discovery druid.router.defaultBrokerServiceName=druid/broker druid.router.coordinatorServiceName=druid/coordinator From b332729455dcf2c4b6e636ed4d40ef321240c29e Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 29 Nov 2022 17:06:02 +0530 Subject: [PATCH 37/82] Update docs --- docs/operations/single-server.md | 36 +++++++++++++++---------- docs/tutorials/index.md | 34 ++++++++++++----------- docs/tutorials/tutorial-batch-hadoop.md | 2 +- docs/tutorials/tutorial-kafka.md | 2 +- 4 files changed, 42 insertions(+), 32 deletions(-) diff --git a/docs/operations/single-server.md b/docs/operations/single-server.md index 35413952e222..7ce252379e9c 100644 --- a/docs/operations/single-server.md +++ b/docs/operations/single-server.md @@ -31,6 +31,7 @@ Druid includes a set of reference configurations and launch scripts for single-m - `medium` - `large` - `xlarge` +- `start-druid` The `micro-quickstart` is sized for small machines like laptops and is intended for quick evaluation use-cases. @@ -42,37 +43,44 @@ The startup scripts for these example configurations run a single ZK instance al The example configurations run the Druid Coordinator and Overlord together in a single process using the optional configuration `druid.coordinator.asOverlord.enabled=true`, described in the [Coordinator configuration documentation](../configuration/index.md#coordinator-operation). +The `start-druid` is a generic launch script for starting druid services on single server, it accepts optional arguments like services, memory and config. +All reference configurations can be acheived by passing appropriate arguments to this script. +If memory argument isn't specified, services will use upto 80% system memory. +Existing launch scripts are deprecated and will be removed in the next release. + + While example configurations are provided for very large single machines, at higher scales we recommend running Druid in a [clustered deployment](../tutorials/cluster.md), for fault-tolerance and reduced resource contention. ## Single server reference configurations -### Nano-Quickstart: 1 CPU, 4GiB RAM +### Nano: 1 CPU, 4GiB RAM -- Launch command: `bin/start-nano-quickstart` -- Configuration directory: `conf/druid/single-server/nano-quickstart` +- Launch command: `bin/start-druid` +- Configuration directory: `conf/druid/single-server/quickstart` -### Micro-Quickstart: 4 CPU, 16GiB RAM +### Micro: 4 CPU, 16GiB RAM -- Launch command: `bin/start-micro-quickstart` -- Configuration directory: `conf/druid/single-server/micro-quickstart` +- Launch command: `bin/start-druid` +- Configuration directory: `conf/druid/single-server/quickstart` ### Small: 8 CPU, 64GiB RAM (~i3.2xlarge) -- Launch command: `bin/start-small` -- Configuration directory: `conf/druid/single-server/small` +- Launch command: `bin/start-druid` +- Configuration directory: `conf/druid/single-server/quickstart` ### Medium: 16 CPU, 128GiB RAM (~i3.4xlarge) -- Launch command: `bin/start-medium` -- Configuration directory: `conf/druid/single-server/medium` +- Launch command: `bin/start-druid` +- Configuration directory: `conf/druid/single-server/quickstart` ### Large: 32 CPU, 256GiB RAM (~i3.8xlarge) -- Launch command: `bin/start-large` -- Configuration directory: `conf/druid/single-server/large` +- Launch command: `bin/start-druid` +- Configuration directory: `conf/druid/single-server/quickstart` ### X-Large: 64 CPU, 512GiB RAM (~i3.16xlarge) -- Launch command: `bin/start-xlarge` -- Configuration directory: `conf/druid/single-server/xlarge` +- Launch command: `bin/start-druid` +- Configuration directory: `conf/druid/single-server/quickstart` +The amount of memory used for Druid can be limited by passing memory argument in the launch command, `bin/start-druid --memory=value` \ No newline at end of file diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index 7d3d2ab68ec1..8d029d568189 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -23,7 +23,7 @@ title: "Quickstart (local)" --> -This quickstart gets you started with Apache Druid using the [`micro-quickstart`](../operations/single-server.md#micro-quickstart-4-cpu-16gib-ram) configuration, and introduces you to Druid ingestion and query features. +This quickstart gets you started with Apache Druid using the [`micro`](../operations/single-server.md#micro-4-cpu-16gib-ram) configuration, and introduces you to Druid ingestion and query features. In this quickstart, you'll do the following: - install Druid @@ -37,7 +37,8 @@ Druid supports a variety of ingestion options. Once you're done with this tutori You can follow these steps on a relatively modest machine, such as a workstation or virtual server with 16 GiB of RAM. -Druid comes equipped with several [startup configuration profiles](../operations/single-server.md) for a +Druid comes equipped with a single launch script that can be used to run several +[startup configuration profiles](../operations/single-server.md) for a range of machine sizes. These range from `nano` (1 CPU, 4GiB RAM) to `x-large` (64 CPU, 512GiB RAM). For more information, see [Single server deployment](../operations/single-server.md). For information on deploying Druid services across clustered machines, see [Clustered deployment](./cluster.md). @@ -72,30 +73,31 @@ The distribution directory contains `LICENSE` and `NOTICE` files and subdirector ## Start up Druid services -Start up Druid services using the `micro-quickstart` single-machine configuration. +Start up Druid services using the `micro` single-machine configuration. This configuration includes default settings that are appropriate for this tutorial, such as loading the `druid-multi-stage-query` extension by default so that you can use the MSQ task engine. -You can view that setting and others in the configuration files in the `conf/druid/single-server/micro-quickstart/`. +You can view that setting and others in the configuration files in the `conf/druid/single-server/quickstart/`. From the apache-druid-{{DRUIDVERSION}} package root, run the following command: ```bash -./bin/start-micro-quickstart +./bin/start-druid --memory=16g ``` This brings up instances of ZooKeeper and the Druid services: ```bash $ ./bin/start-micro-quickstart -[Thu Sep 8 18:30:00 2022] Starting Apache Druid. -[Thu Sep 8 18:30:00 2022] Open http://localhost:8888/ in your browser to access the web console. -[Thu Sep 8 18:30:00 2022] Or, if you have enabled TLS, use https on port 9088. -[Thu Sep 8 18:30:00 2022] Running command[zk], logging to[/apache-druid-{{DRUIDVERSION}}/var/sv/zk.log]: bin/run-zk conf -[Thu Sep 8 18:30:00 2022] Running command[coordinator-overlord], logging to[/apache-druid-{{DRUIDVERSION}}/var/sv/coordinator-overlord.log]: bin/run-druid coordinator-overlord conf/druid/single-server/micro-quickstart -[Thu Sep 8 18:30:00 2022] Running command[broker], logging to[/apache-druid-{{DRUIDVERSION}}/var/sv/broker.log]: bin/run-druid broker conf/druid/single-server/micro-quickstart -[Thu Sep 8 18:30:00 2022] Running command[router], logging to[/apache-druid-{{DRUIDVERSION}}/var/sv/router.log]: bin/run-druid router conf/druid/single-server/micro-quickstart -[Thu Sep 8 18:30:00 2022] Running command[historical], logging to[/apache-druid-{{DRUIDVERSION}}/var/sv/historical.log]: bin/run-druid historical conf/druid/single-server/micro-quickstart -[Thu Sep 8 18:30:00 2022] Running command[middleManager], logging to[/apache-druid-{{DRUIDVERSION}}/var/sv/middleManager.log]: bin/run-druid middleManager conf/druid/single-server/micro-quickstart +[Tue Nov 29 16:31:06 2022] Starting Apache Druid. +[Tue Nov 29 16:31:06 2022] Open http://localhost:8888/ in your browser to access the web console. +[Tue Nov 29 16:31:06 2022] Or, if you have enabled TLS, use https on port 9088. +[Tue Nov 29 16:31:06 2022] Starting services with log directory [/apache-druid-{{DRUIDVERSION}}/log]. +[Tue Nov 29 16:31:06 2022] Running command[zk]: bin/run-zk conf +[Tue Nov 29 16:31:06 2022] Running command[broker]: bin/run-druid broker /apache-druid-{{DRUIDVERSION}}/conf/druid/single-server/quickstart '-Xms1187m -Xmx1187m -XX:MaxDirectMemorySize=791m' +[Tue Nov 29 16:31:06 2022] Running command[router]: bin/run-druid router /apache-druid-{{DRUIDVERSION}}/conf/druid/single-server/quickstart '-Xms128m -Xmx128m' +[Tue Nov 29 16:31:06 2022] Running command[coordinator-overlord]: bin/run-druid coordinator-overlord /apache-druid-{{DRUIDVERSION}}/conf/druid/single-server/quickstart '-Xms1290m -Xmx1290m' +[Tue Nov 29 16:31:06 2022] Running command[historical]: bin/run-druid historical /apache-druid-{{DRUIDVERSION}}/conf/druid/single-server/quickstart '-Xms1376m -Xmx1376m -XX:MaxDirectMemorySize=2064m' +[Tue Nov 29 16:31:06 2022] Running command[middleManager]: bin/run-druid middleManager /apache-druid-{{DRUIDVERSION}}/conf/druid/single-server/quickstart '-Xms64m -Xmx64m' '-Ddruid.worker.capacity=2 -Ddruid.indexer.runner.javaOptsArray=["-server","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager","-Xms256m","-Xmx256m","-XX:MaxDirectMemorySize=256m"]' ``` All persistent state, such as the cluster metadata store and segments for the services, are kept in the `var` directory under @@ -103,7 +105,7 @@ the Druid root directory, apache-druid-{{DRUIDVERSION}}. Each service writes to At any time, you can revert Druid to its original, post-installation state by deleting the entire `var` directory. You may want to do this, for example, between Druid tutorials or after experimentation, to start with a fresh instance. -To stop Druid at any time, use CTRL+C in the terminal. This exits the `bin/start-micro-quickstart` script and terminates all Druid processes. +To stop Druid at any time, use CTRL+C in the terminal. This exits the `bin/start-druid` script and terminates all Druid processes. ## Open the web console @@ -222,4 +224,4 @@ See the following topics for more information: * [Tutorial: Load stream data from Apache Kafka](./tutorial-kafka.md) to load streaming data from a Kafka topic. * [Extensions](../development/extensions.md) for details on Druid extensions. -Remember that after stopping Druid services, you can start clean next time by deleting the `var` directory from the Druid root directory and running the `bin/start-micro-quickstart` script again. You may want to do this before using other data ingestion tutorials, since they use the same Wikipedia datasource. +Remember that after stopping Druid services, you can start clean next time by deleting the `var` directory from the Druid root directory and running the `bin/start-druid` script again. You may want to do this before using other data ingestion tutorials, since they use the same Wikipedia datasource. diff --git a/docs/tutorials/tutorial-batch-hadoop.md b/docs/tutorials/tutorial-batch-hadoop.md index 47cd2d6bcbe5..bff033a28c41 100644 --- a/docs/tutorials/tutorial-batch-hadoop.md +++ b/docs/tutorials/tutorial-batch-hadoop.md @@ -28,7 +28,7 @@ This tutorial shows you how to load data files into Apache Druid using a remote For this tutorial, we'll assume that you've already completed the previous [batch ingestion tutorial](tutorial-batch.md) using Druid's native batch ingestion system and are using the -`micro-quickstart` single-machine configuration as described in the [quickstart](index.md). +`micro` single-machine configuration as described in the [quickstart](../operations/single-server.md#micro-4-cpu-16gib-ram). ## Install Docker diff --git a/docs/tutorials/tutorial-kafka.md b/docs/tutorials/tutorial-kafka.md index eb06f4239f95..bbdf6af72ea0 100644 --- a/docs/tutorials/tutorial-kafka.md +++ b/docs/tutorials/tutorial-kafka.md @@ -30,7 +30,7 @@ The tutorial guides you through the steps to load sample nested clickstream data ## Prerequisites -Before you follow the steps in this tutorial, download Druid as described in the [quickstart](index.md) using the [micro-quickstart](../operations/single-server.md#micro-quickstart-4-cpu-16gib-ram) single-machine configuration and have it running on your local machine. You don't need to have loaded any data. +Before you follow the steps in this tutorial, download Druid as described in the [quickstart](index.md) using the [micro](../operations/single-server.md#micro-4-cpu-16gib-ram) single-machine configuration and have it running on your local machine. You don't need to have loaded any data. ## Download and start Kafka From b8d34c423c0e56af21e3197d01c983e65697a8ae Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 29 Nov 2022 17:11:47 +0530 Subject: [PATCH 38/82] Specify memory argument for each profile in single-server.md --- docs/operations/single-server.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/operations/single-server.md b/docs/operations/single-server.md index 7ce252379e9c..34dd2dc0e006 100644 --- a/docs/operations/single-server.md +++ b/docs/operations/single-server.md @@ -44,8 +44,7 @@ The startup scripts for these example configurations run a single ZK instance al The example configurations run the Druid Coordinator and Overlord together in a single process using the optional configuration `druid.coordinator.asOverlord.enabled=true`, described in the [Coordinator configuration documentation](../configuration/index.md#coordinator-operation). The `start-druid` is a generic launch script for starting druid services on single server, it accepts optional arguments like services, memory and config. -All reference configurations can be acheived by passing appropriate arguments to this script. -If memory argument isn't specified, services will use upto 80% system memory. +All reference configurations can be acheived by passing appropriate arguments to this script. Existing launch scripts are deprecated and will be removed in the next release. @@ -55,32 +54,32 @@ While example configurations are provided for very large single machines, at hig ### Nano: 1 CPU, 4GiB RAM -- Launch command: `bin/start-druid` +- Launch command: `bin/start-druid --memory=4g` - Configuration directory: `conf/druid/single-server/quickstart` ### Micro: 4 CPU, 16GiB RAM -- Launch command: `bin/start-druid` +- Launch command: `bin/start-druid --memory=16g` - Configuration directory: `conf/druid/single-server/quickstart` ### Small: 8 CPU, 64GiB RAM (~i3.2xlarge) -- Launch command: `bin/start-druid` +- Launch command: `bin/start-druid --memory=64g` - Configuration directory: `conf/druid/single-server/quickstart` ### Medium: 16 CPU, 128GiB RAM (~i3.4xlarge) -- Launch command: `bin/start-druid` +- Launch command: `bin/start-druid --memory=128g` - Configuration directory: `conf/druid/single-server/quickstart` ### Large: 32 CPU, 256GiB RAM (~i3.8xlarge) -- Launch command: `bin/start-druid` +- Launch command: `bin/start-druid --memory=256g` - Configuration directory: `conf/druid/single-server/quickstart` ### X-Large: 64 CPU, 512GiB RAM (~i3.16xlarge) -- Launch command: `bin/start-druid` +- Launch command: `bin/start-druid --memory=512g` - Configuration directory: `conf/druid/single-server/quickstart` -The amount of memory used for Druid can be limited by passing memory argument in the launch command, `bin/start-druid --memory=value` \ No newline at end of file +Memory argument (`--memory`) in the above launch command is optional, if not specified Druid will use upto 80% of system memory. \ No newline at end of file From cafdc83e85c35438fdced27f8756567fe2b37125 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 29 Nov 2022 17:30:15 +0530 Subject: [PATCH 39/82] Update middleManager runtime.properties --- .../quickstart/middleManager/runtime.properties | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties b/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties index a90e62579676..08c58bae6de0 100644 --- a/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties +++ b/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties @@ -29,9 +29,8 @@ druid.indexer.runner.javaCommand=bin/run-java druid.indexer.task.baseTaskDir=var/druid/task # Processing threads and buffers on Peons -#druid.indexer.fork.property.druid.processing.numMergeBuffers=2 -#druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB -#druid.indexer.fork.property.druid.processing.numThreads=1 +# Determined automatically based on available memory. For details on how to manually set parameters: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers # Hadoop indexing druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp From 0da21ab640e7c146c9ec8cfa8b58a6ba752b5cc3 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 30 Nov 2022 17:38:59 +0530 Subject: [PATCH 40/82] Move quickstart configs to conf/druid/base, add bash launch script, support python2 --- examples/bin/start-druid | 89 ++++++++++--------- examples/bin/start-druid-base | 32 +++++++ .../_common/common.jvm.config | 0 .../_common/common.runtime.properties | 0 .../quickstart => base}/_common/log4j2.xml | 0 .../broker/runtime.properties | 14 ++- .../coordinator-overlord/runtime.properties | 0 .../historical/runtime.properties | 12 +-- .../middleManager/runtime.properties | 15 ++-- .../router/runtime.properties | 6 ++ 10 files changed, 104 insertions(+), 64 deletions(-) create mode 100644 examples/bin/start-druid-base rename examples/conf/druid/{single-server/quickstart => base}/_common/common.jvm.config (100%) rename examples/conf/druid/{single-server/quickstart => base}/_common/common.runtime.properties (100%) rename examples/conf/druid/{single-server/quickstart => base}/_common/log4j2.xml (100%) rename examples/conf/druid/{single-server/quickstart => base}/broker/runtime.properties (64%) rename examples/conf/druid/{single-server/quickstart => base}/coordinator-overlord/runtime.properties (100%) rename examples/conf/druid/{single-server/quickstart => base}/historical/runtime.properties (71%) rename examples/conf/druid/{single-server/quickstart => base}/middleManager/runtime.properties (57%) rename examples/conf/druid/{single-server/quickstart => base}/router/runtime.properties (87%) diff --git a/examples/bin/start-druid b/examples/bin/start-druid index 25c282f461c5..37fe506adec5 100644 --- a/examples/bin/start-druid +++ b/examples/bin/start-druid @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -24,7 +24,7 @@ import pathlib import multiprocessing import argparse -QUICKSTART_ROOT_CONFIG_PATH = "conf/druid/single-server/quickstart" +BASE_CONFIG_PATH = "conf/druid/base" MEM_GB_SUFFIX = "g" MEM_MB_SUFFIX = "m" @@ -161,19 +161,19 @@ sample usage: def validate_common_jvm_args(config): - if pathlib.Path(f'{config}/_common/common.jvm.config').is_file() is False: - raise ValueError(f'_common/common.jvm.config file is missing in the root config, ' - f'check {QUICKSTART_ROOT_CONFIG_PATH}/_common directory') + if pathlib.Path('{0}/_common/common.jvm.config'.format(config)).is_file() is False: + raise ValueError('_common/common.jvm.config file is missing in the root config, ' + 'check {0}/_common directory'.format(BASE_CONFIG_PATH)) def validate_common_directory(config): - if pathlib.Path(f'{config}/_common').is_dir() is False: + if pathlib.Path('{0}/_common'.format(config)).is_dir() is False: raise ValueError( - f'_common directory is missing in the root config, check {QUICKSTART_ROOT_CONFIG_PATH}/_common directory') + '_common directory is missing in the root config, check {0}/_common directory'.format(BASE_CONFIG_PATH)) - if pathlib.Path(f'{config}/_common/common.runtime.properties').is_file() is False: - raise ValueError(f'_common/common.runtime.properties file is missing in the root config, ' - f'check {QUICKSTART_ROOT_CONFIG_PATH}/_common directory') + if pathlib.Path('{0}/_common/common.runtime.properties'.format(config)).is_file() is False: + raise ValueError('_common/common.runtime.properties file is missing in the root config, ' + 'check {0}/_common directory'.format(BASE_CONFIG_PATH)) def parse_arguments(args): @@ -189,8 +189,8 @@ def parse_arguments(args): zk = True if args.config is not None: config = pathlib.Path(os.path.join(os.getcwd(), args.config)).resolve() - if os.path.exists(config) is False: - raise ValueError(f'config {config} not found') + if config.is_dir() == False: + raise ValueError('config {0} not found'.format(config)) if args.memory is not None: total_memory = args.memory if args.services is not None: @@ -198,10 +198,10 @@ def parse_arguments(args): for service in services: if service not in DEFAULT_SERVICES: - raise ValueError(f'Invalid service name {service}, should be one of {DEFAULT_SERVICES}') + raise ValueError('Invalid service name {0}, should be one of {1}'.format(service, DEFAULT_SERVICES)) if service in service_list: - raise ValueError(f'{service} is specified multiple times') + raise ValueError('{0} is specified multiple times'.format(service)) service_list.append(service) @@ -214,11 +214,11 @@ def parse_arguments(args): def print_startup_config(service_list, config, zk): - print_if_verbose(f'Starting {service_list}') - print_if_verbose(f'Reading config from {config}') + print_if_verbose('Starting {0}'.format(service_list)) + print_if_verbose('Reading config from {0}'.format(config)) if zk: - zk_config = pathlib.Path(f'{os.getcwd()}/../conf/zk').resolve() - print_if_verbose(f'Starting zk, reading default config from {zk_config}') + zk_config = pathlib.Path('{0}/../conf/zk'.format(os.getcwd())).resolve() + print_if_verbose('Starting zk, reading default config from {0}'.format(zk_config)) print_if_verbose('\n') @@ -226,8 +226,8 @@ def middle_manager_task_memory_params_present(config): java_opts_property_present = False worker_capacity_property_present = False - if pathlib.Path(f'{config}/middleManager/runtime.properties').is_file(): - with open(f'{config}/middleManager/runtime.properties') as file: + if pathlib.Path('{0}/middleManager/runtime.properties'.format(config)).is_file(): + with open('{0}/middleManager/runtime.properties'.format(config)) as file: for line in file: if line.startswith(TASK_JAVA_OPTS_PROPERTY): java_opts_property_present = True @@ -238,7 +238,7 @@ def middle_manager_task_memory_params_present(config): def verify_service_config(service, config): - path = f'{config}/{service}/jvm.config' + path = '{0}/{1}/jvm.config'.format(config, service) required_parameters = [XMX_PARAMETER, XMS_PARAMETER] @@ -256,16 +256,16 @@ def verify_service_config(service, config): if len(required_parameters) > 0: params = ",".join(required_parameters) - raise ValueError(f'{params} missing in {service}/jvm.config') + raise ValueError('{0} missing in {1}/jvm.config'.format(params, service)) if service == MIDDLE_MANAGER: - if pathlib.Path(f'{config}/{service}/runtime.properties').is_file() is False: - raise ValueError(f'{service}/runtime.properties file is missing in the root config') + if pathlib.Path('{0}/{1}/runtime.properties'.format(config, service)).is_file() is False: + raise ValueError('{0}/runtime.properties file is missing in the root config'.format(service)) mm_task_java_opts_property, mm_task_worker_capacity_prop = middle_manager_task_memory_params_present(config) if mm_task_java_opts_property is False: - raise ValueError(f'{TASK_JAVA_OPTS_PROPERTY} property missing in {service}/runtime.properties') + raise ValueError('{0} property missing in {1}/runtime.properties'.format(TASK_JAVA_OPTS_PROPERTY, service)) def should_compute_memory(config, total_memory, service_list): @@ -282,7 +282,7 @@ def should_compute_memory(config, total_memory, service_list): jvm_config_count = 0 for service in service_list: - if pathlib.Path(f'{config}/{service}/jvm.config').is_file(): + if pathlib.Path('{0}/{1}/jvm.config'.format(config, service)).is_file(): jvm_config_count += 1 mm_task_property_present = False @@ -305,8 +305,8 @@ def should_compute_memory(config, total_memory, service_list): "(druid.worker.capacity, druid.indexer.runner.javaOptsArray) is present, " "memory argument shouldn't be specified") if jvm_config_count == 0: - raise ValueError("middleManger configs (druid.indexer.runner.javaOptsArray or druid.worker.capacity) is present in " - "middleManager/runtime.properties, \n " + raise ValueError("middleManger configs (druid.indexer.runner.javaOptsArray or druid.worker.capacity) " + "is present in middleManager/runtime.properties, " "add jvm.config for all other services") if jvm_config_count != len(service_list): raise ValueError("jvm.config file should be present for all services or none") @@ -343,7 +343,7 @@ def check_memory_constraint(total_memory, services): # 80% of total memory >= sum of lower bound service memory should be lower_bound_memory = 0 - service_list = services.copy() + service_list = list(services) if MIDDLE_MANAGER in services: service_list.append(TASKS) @@ -353,7 +353,7 @@ def check_memory_constraint(total_memory, services): required_memory = int(lower_bound_memory / 0.8) if total_memory < required_memory: - raise ValueError(f'Minimum memory required for starting services is {required_memory}m') + raise ValueError('Minimum memory required for starting services is {0}m'.format(required_memory)) if total_memory >= 2 * lower_bound_memory: return int(total_memory / 2) @@ -362,14 +362,14 @@ def check_memory_constraint(total_memory, services): def build_mm_task_java_opts_array(memory_type): - task_memory = f'-D{TASK_JAVA_OPTS_PROPERTY}=[' + task_memory = '-D{0}=['.format(TASK_JAVA_OPTS_PROPERTY) mem_array = TASK_MEM_MAP.get(memory_type) java_opts_list = TASK_JAVA_OPTS_ARRAY + mem_array for item in java_opts_list: - task_memory += f'\"{item}\";' + task_memory += '\"{0}\";'.format(item) task_memory = task_memory[:-1] task_memory += ']' @@ -394,16 +394,16 @@ def build_memory_config(service, allocated_memory): if service == TASKS: memory_type, task_count, task_memory = compute_tasks_memory(allocated_memory) java_opts_array = build_mm_task_java_opts_array(memory_type) - return [f'-D{TASK_WORKER_CAPACITY_PROPERTY}={task_count}', java_opts_array], task_memory * task_count + return ['-D{0}={1}'.format(TASK_WORKER_CAPACITY_PROPERTY, task_count), java_opts_array], task_memory * task_count else: heap_memory = HEAP_TO_TOTAL_MEM_RATIO.get(service) * allocated_memory direct_memory = int(allocated_memory - heap_memory) heap_memory = int(heap_memory) if direct_memory == 0: - return f'-Xms{heap_memory}m -Xmx{heap_memory}m', allocated_memory + return '-Xms{0}m -Xmx{0}m'.format(heap_memory), allocated_memory - return f'-Xms{heap_memory}m -Xmx{heap_memory}m -XX:MaxDirectMemorySize={direct_memory}m', allocated_memory + return '-Xms{0}m -Xmx{0}m -XX:MaxDirectMemorySize={1}m'.format(heap_memory, direct_memory), allocated_memory def distribute_memory(services, total_memory): @@ -411,7 +411,7 @@ def distribute_memory(services, total_memory): memory_weight_sum = 0 - service_list = services.copy() + service_list = list(services) if MIDDLE_MANAGER in services: service_list.append(TASKS) @@ -449,9 +449,9 @@ def distribute_memory(services, total_memory): service_memory_config[service], allocated_memory = build_memory_config(service, allocated_memory) - print_if_verbose(f'\nMemory distribution for services:') + print_if_verbose('\nMemory distribution for services:') for key, value in service_memory_config.items(): - print_if_verbose(f'{key}, {value}') + print_if_verbose('{0}, {1}'.format(key, value)) print_if_verbose('\n') return service_memory_config @@ -479,16 +479,17 @@ def build_supervise_script_arguments(service_list, service_memory_config, config prefix = '!p90 ' if jvm_args is None: - commands.append(f'{prefix}{service} bin/run-druid {service} {config}') + commands.append('{0}{1} bin/run-druid {1} {2}'.format(prefix, service, config)) else: if service == MIDDLE_MANAGER: task_config = service_memory_config.get(TASKS) task_count = task_config[0] task_memory = task_config[1] commands.append( - f'{prefix}{service} bin/run-druid {service} {config} \'{jvm_args}\' \'{task_count} {task_memory}\'') + '{0}{1} bin/run-druid {1} {2} \'{3}\' \'{4} {5}\'' + .format(prefix, service, config, jvm_args, task_count, task_memory)) else: - commands.append(f'{prefix}{service} bin/run-druid {service} {config} \'{jvm_args}\'') + commands.append('{0}{1} bin/run-druid {1} {2} \'{3}\''.format(prefix, service, config, jvm_args)) print_if_verbose('Supervise script args:') for item in commands: @@ -513,7 +514,7 @@ def main(): os.chdir(os.path.dirname(sys.argv[0])) if config == "": - config = pathlib.Path(f'{os.getcwd()}/../{QUICKSTART_ROOT_CONFIG_PATH}').resolve() + config = pathlib.Path('{0}/../{1}'.format(os.getcwd(), BASE_CONFIG_PATH)).resolve() validate_common_directory(config) @@ -525,9 +526,9 @@ def main(): # if memory is to be computed, _common directory should contain common.jvm.config validate_common_jvm_args(config) memory_in_mega_bytes = convert_total_memory_string(total_memory) - print_if_verbose(f'Total memory is {memory_in_mega_bytes}m\n') + print_if_verbose('Total memory is {0}m\n'.format(memory_in_mega_bytes)) memory_to_be_used = check_memory_constraint(memory_in_mega_bytes, service_list) - print_if_verbose(f'Memory used for services & tasks {memory_to_be_used}m\n') + print_if_verbose('Memory used for services & tasks {0}m\n'.format(memory_to_be_used)) service_memory_config = distribute_memory(service_list, memory_to_be_used) else: print_if_verbose('Not computing memory distribution, reading memory specification from service jvm.config & ' diff --git a/examples/bin/start-druid-base b/examples/bin/start-druid-base new file mode 100644 index 000000000000..38f7a5f9b0a5 --- /dev/null +++ b/examples/bin/start-druid-base @@ -0,0 +1,32 @@ +#!/bin/bash -eu + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +PWD="$(pwd)" +WHEREAMI="$(dirname "$0")" +WHEREAMI="$(cd "$WHEREAMI" && pwd)" + +if [ -x "$(command -v python3)" ] +then + exec python3 "$WHEREAMI/start-druid" "$@" +elif [ -x "$(command -v python2)" ] +then + exec python2 "$WHEREAMI/start-druid" "$@" +else + exec "$WHEREAMI/start-druid" "$@" +fi diff --git a/examples/conf/druid/single-server/quickstart/_common/common.jvm.config b/examples/conf/druid/base/_common/common.jvm.config similarity index 100% rename from examples/conf/druid/single-server/quickstart/_common/common.jvm.config rename to examples/conf/druid/base/_common/common.jvm.config diff --git a/examples/conf/druid/single-server/quickstart/_common/common.runtime.properties b/examples/conf/druid/base/_common/common.runtime.properties similarity index 100% rename from examples/conf/druid/single-server/quickstart/_common/common.runtime.properties rename to examples/conf/druid/base/_common/common.runtime.properties diff --git a/examples/conf/druid/single-server/quickstart/_common/log4j2.xml b/examples/conf/druid/base/_common/log4j2.xml similarity index 100% rename from examples/conf/druid/single-server/quickstart/_common/log4j2.xml rename to examples/conf/druid/base/_common/log4j2.xml diff --git a/examples/conf/druid/single-server/quickstart/broker/runtime.properties b/examples/conf/druid/base/broker/runtime.properties similarity index 64% rename from examples/conf/druid/single-server/quickstart/broker/runtime.properties rename to examples/conf/druid/base/broker/runtime.properties index f4c494019933..549117efa50d 100644 --- a/examples/conf/druid/single-server/quickstart/broker/runtime.properties +++ b/examples/conf/druid/base/broker/runtime.properties @@ -21,19 +21,15 @@ druid.service=druid/broker druid.plaintextPort=8082 # HTTP server settings -# HTTP server thread pool size. Higher values increase peak load on the Broker, but -# may be useful for high-concurrency workloads. -# Default is max(10, (Number of processors * 17) / 16 + 2) + 30. -# druid.server.http.numThreads=N +#druid.server.http.numThreads=12 # HTTP client settings -# Connection pool size from the Broker to each data server. May be useful to -# raise this for high-concurrency workloads. -# druid.broker.http.numConnections=20 +#druid.broker.http.numConnections=10 +#druid.broker.http.maxQueuedBytes=5MiB # Processing threads and buffers -# Determined automatically based on available memory. For details on how to manually set parameters: -# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers +#druid.processing.buffer.sizeBytes=100MiB +#druid.processing.numMergeBuffers=2 druid.processing.tmpDir=var/druid/processing # Query cache disabled -- push down caching and merging instead diff --git a/examples/conf/druid/single-server/quickstart/coordinator-overlord/runtime.properties b/examples/conf/druid/base/coordinator-overlord/runtime.properties similarity index 100% rename from examples/conf/druid/single-server/quickstart/coordinator-overlord/runtime.properties rename to examples/conf/druid/base/coordinator-overlord/runtime.properties diff --git a/examples/conf/druid/single-server/quickstart/historical/runtime.properties b/examples/conf/druid/base/historical/runtime.properties similarity index 71% rename from examples/conf/druid/single-server/quickstart/historical/runtime.properties rename to examples/conf/druid/base/historical/runtime.properties index d9ce1850e4fc..cf3e966cc30e 100644 --- a/examples/conf/druid/single-server/quickstart/historical/runtime.properties +++ b/examples/conf/druid/base/historical/runtime.properties @@ -20,14 +20,13 @@ druid.service=druid/historical druid.plaintextPort=8083 -# HTTP server thread pool size. Higher values increase peak load on the Broker, but -# may be useful for high-concurrency workloads. -# Default is max(10, (Number of processors * 17) / 16 + 2) + 30. -# druid.server.http.numThreads=N +# HTTP server threads +#druid.server.http.numThreads=12 # Processing threads and buffers -# Determined automatically based on available memory. For details on how to manually set parameters: -# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers +#druid.processing.buffer.sizeBytes=200MiB +#druid.processing.numMergeBuffers=2 +#druid.processing.numThreads=2 druid.processing.tmpDir=var/druid/processing # Segment storage @@ -37,3 +36,4 @@ druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g" druid.historical.cache.useCache=true druid.historical.cache.populateCache=true druid.cache.type=caffeine +#druid.cache.sizeInBytes=10MiB diff --git a/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties b/examples/conf/druid/base/middleManager/runtime.properties similarity index 57% rename from examples/conf/druid/single-server/quickstart/middleManager/runtime.properties rename to examples/conf/druid/base/middleManager/runtime.properties index 08c58bae6de0..15fa486c7098 100644 --- a/examples/conf/druid/single-server/quickstart/middleManager/runtime.properties +++ b/examples/conf/druid/base/middleManager/runtime.properties @@ -20,17 +20,22 @@ druid.service=druid/middleManager druid.plaintextPort=8091 -# Number of tasks (druid.worker.capacity) and memory usage per task (druid.indexer.runner.javaOptsArray) is automatically -# determined based on available memory. For details on how to manually set parameters, see: -# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#middlemanager +# Number of tasks per middleManager +# druid.worker.capacity # Task launch parameters druid.indexer.runner.javaCommand=bin/run-java +#druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task +#druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] + +# HTTP server threads +#druid.server.http.numThreads=12 # Processing threads and buffers on Peons -# Determined automatically based on available memory. For details on how to manually set parameters: -# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers +#druid.indexer.fork.property.druid.processing.numMergeBuffers=2 +#druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB +#druid.indexer.fork.property.druid.processing.numThreads=1 # Hadoop indexing druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/examples/conf/druid/single-server/quickstart/router/runtime.properties b/examples/conf/druid/base/router/runtime.properties similarity index 87% rename from examples/conf/druid/single-server/quickstart/router/runtime.properties rename to examples/conf/druid/base/router/runtime.properties index 3858dec044bd..4c38ba88baab 100644 --- a/examples/conf/druid/single-server/quickstart/router/runtime.properties +++ b/examples/conf/druid/base/router/runtime.properties @@ -20,6 +20,12 @@ druid.service=druid/router druid.plaintextPort=8888 +# HTTP proxy +#druid.router.http.numConnections=50 +#druid.router.http.readTimeout=PT5M +#druid.router.http.numMaxThreads=100 +#druid.server.http.numThreads=100 + # Service discovery druid.router.defaultBrokerServiceName=druid/broker druid.router.coordinatorServiceName=druid/coordinator From b03d873fe3e0a0e2c73f9bb6ce285a0a75b7fc54 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 30 Nov 2022 23:50:50 +0530 Subject: [PATCH 41/82] Update supervise script --- examples/bin/supervise | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/examples/bin/supervise b/examples/bin/supervise index 9a6809bacdc8..e9cdca5ce925 100755 --- a/examples/bin/supervise +++ b/examples/bin/supervise @@ -196,25 +196,20 @@ usage() unless GetOptions( 'kill-timeout|t=i', 'chdir=s', 'svlogd:s', - 'array|a=s{,}' + 'command=s@' => \@commands ); -usage() unless (($opt{'array'} || $opt{'conf'}) && $opt{'vardir'}); +usage() unless ((@commands || $opt{'conf'}) && $opt{'vardir'}); my @config_lines; # get commands to execute either from reading the config file or command line if (not defined $opt{'conf'}) { - @config_lines = split(',', $opt{'array'}); + @config_lines = @commands } else { @config_lines = read_config_file($opt{'conf'}); } -for my $alpha (@config_lines) -{ - $alpha =~ s/;/,/g; -} - my $config = process_config(@config_lines); @commands = @{$config->{commands}}; From aabc53c86ea18360ca758bea509e0be098950469 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 03:21:23 +0530 Subject: [PATCH 42/82] rename base config directory to auto --- .../{start-druid-base => start-druid-auto} | 0 .../bin/{start-druid => start-druid-main} | 0 .../conf/druid/auto/_common/common.jvm.config | 7 + .../auto/_common/common.runtime.properties | 158 ++++++++++++++++++ examples/conf/druid/auto/_common/log4j2.xml | 86 ++++++++++ .../conf/druid/auto/broker/runtime.properties | 37 ++++ .../coordinator-overlord/runtime.properties | 33 ++++ .../druid/auto/historical/runtime.properties | 39 +++++ .../auto/middleManager/runtime.properties | 41 +++++ .../conf/druid/auto/router/runtime.properties | 34 ++++ 10 files changed, 435 insertions(+) rename examples/bin/{start-druid-base => start-druid-auto} (100%) rename examples/bin/{start-druid => start-druid-main} (100%) create mode 100644 examples/conf/druid/auto/_common/common.jvm.config create mode 100644 examples/conf/druid/auto/_common/common.runtime.properties create mode 100644 examples/conf/druid/auto/_common/log4j2.xml create mode 100644 examples/conf/druid/auto/broker/runtime.properties create mode 100644 examples/conf/druid/auto/coordinator-overlord/runtime.properties create mode 100644 examples/conf/druid/auto/historical/runtime.properties create mode 100644 examples/conf/druid/auto/middleManager/runtime.properties create mode 100644 examples/conf/druid/auto/router/runtime.properties diff --git a/examples/bin/start-druid-base b/examples/bin/start-druid-auto similarity index 100% rename from examples/bin/start-druid-base rename to examples/bin/start-druid-auto diff --git a/examples/bin/start-druid b/examples/bin/start-druid-main similarity index 100% rename from examples/bin/start-druid rename to examples/bin/start-druid-main diff --git a/examples/conf/druid/auto/_common/common.jvm.config b/examples/conf/druid/auto/_common/common.jvm.config new file mode 100644 index 000000000000..fd74cf358979 --- /dev/null +++ b/examples/conf/druid/auto/_common/common.jvm.config @@ -0,0 +1,7 @@ +-server +-XX:+ExitOnOutOfMemoryError +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/auto/_common/common.runtime.properties b/examples/conf/druid/auto/_common/common.runtime.properties new file mode 100644 index 000000000000..b0adb0695cd7 --- /dev/null +++ b/examples/conf/druid/auto/_common/common.runtime.properties @@ -0,0 +1,158 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Extensions specified in the load list will be loaded by Druid +# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead +# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead + +# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. +# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. +# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query"] + +# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory +# and uncomment the line below to point to your directory. +#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies + + +# +# Hostname +# +druid.host=localhost + +# +# Logging +# + +# Log all runtime properties on startup. Disable to avoid logging properties on startup: +druid.startup.logging.logProperties=true + +# +# Zookeeper +# + +druid.zk.service.host=localhost +druid.zk.paths.base=/druid + +# +# Metadata storage +# + +# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): +druid.metadata.storage.type=derby +druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true +druid.metadata.storage.connector.host=localhost +druid.metadata.storage.connector.port=1527 + +# For MySQL (make sure to include the MySQL JDBC driver on the classpath): +#druid.metadata.storage.type=mysql +#druid.metadata.storage.connector.connectURI=jdbc:mysql://db.example.com:3306/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# For PostgreSQL: +#druid.metadata.storage.type=postgresql +#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# +# Deep storage +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.storage.type=local +druid.storage.storageDirectory=var/druid/segments + +# For HDFS: +#druid.storage.type=hdfs +#druid.storage.storageDirectory=/druid/segments + +# For S3: +#druid.storage.type=s3 +#druid.storage.bucket=your-bucket +#druid.storage.baseKey=druid/segments +#druid.s3.accessKey=... +#druid.s3.secretKey=... + +# +# Indexing service logs +# + +# For local disk (only viable in a cluster if this is a network mount): +druid.indexer.logs.type=file +druid.indexer.logs.directory=var/druid/indexing-logs + +# For HDFS: +#druid.indexer.logs.type=hdfs +#druid.indexer.logs.directory=/druid/indexing-logs + +# For S3: +#druid.indexer.logs.type=s3 +#druid.indexer.logs.s3Bucket=your-bucket +#druid.indexer.logs.s3Prefix=druid/indexing-logs + +# +# Service discovery +# + +druid.selectors.indexing.serviceName=druid/overlord +druid.selectors.coordinator.serviceName=druid/coordinator + +# +# Monitoring +# + +druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] +druid.emitter=noop +druid.emitter.logging.logLevel=info + +# Storage type of double columns +# ommiting this will lead to index double as float at the storage layer + +druid.indexing.doubleStorage=double + +# +# Security +# +druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password", "password", "key", "token", "pwd"] + + +# +# SQL +# +druid.sql.enable=true + +# Planning SQL query when there is aggregate distinct in the statement +druid.sql.planner.useGroupingSetForExactDistinct=true + +# +# Lookups +# +druid.lookup.enableLookupSyncOnStartup=false + +# +# Expression processing config +# +druid.expressions.useStrictBooleans=true + +# +# Http client +# +druid.global.http.eagerInitialization=false diff --git a/examples/conf/druid/auto/_common/log4j2.xml b/examples/conf/druid/auto/_common/log4j2.xml new file mode 100644 index 000000000000..66dc13da4c5e --- /dev/null +++ b/examples/conf/druid/auto/_common/log4j2.xml @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/conf/druid/auto/broker/runtime.properties b/examples/conf/druid/auto/broker/runtime.properties new file mode 100644 index 000000000000..549117efa50d --- /dev/null +++ b/examples/conf/druid/auto/broker/runtime.properties @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/broker +druid.plaintextPort=8082 + +# HTTP server settings +#druid.server.http.numThreads=12 + +# HTTP client settings +#druid.broker.http.numConnections=10 +#druid.broker.http.maxQueuedBytes=5MiB + +# Processing threads and buffers +#druid.processing.buffer.sizeBytes=100MiB +#druid.processing.numMergeBuffers=2 +druid.processing.tmpDir=var/druid/processing + +# Query cache disabled -- push down caching and merging instead +druid.broker.cache.useCache=false +druid.broker.cache.populateCache=false diff --git a/examples/conf/druid/auto/coordinator-overlord/runtime.properties b/examples/conf/druid/auto/coordinator-overlord/runtime.properties new file mode 100644 index 000000000000..c053823f8ed8 --- /dev/null +++ b/examples/conf/druid/auto/coordinator-overlord/runtime.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/coordinator +druid.plaintextPort=8081 + +druid.coordinator.startDelay=PT10S +druid.coordinator.period=PT5S +druid.manager.segments.pollDuration=PT5S + +# Run the overlord service in the coordinator process +druid.coordinator.asOverlord.enabled=true +druid.coordinator.asOverlord.overlordService=druid/overlord + +druid.indexer.queue.startDelay=PT5S + +druid.indexer.storage.type=metadata diff --git a/examples/conf/druid/auto/historical/runtime.properties b/examples/conf/druid/auto/historical/runtime.properties new file mode 100644 index 000000000000..cf3e966cc30e --- /dev/null +++ b/examples/conf/druid/auto/historical/runtime.properties @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/historical +druid.plaintextPort=8083 + +# HTTP server threads +#druid.server.http.numThreads=12 + +# Processing threads and buffers +#druid.processing.buffer.sizeBytes=200MiB +#druid.processing.numMergeBuffers=2 +#druid.processing.numThreads=2 +druid.processing.tmpDir=var/druid/processing + +# Segment storage +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g"}] + +# Query cache +druid.historical.cache.useCache=true +druid.historical.cache.populateCache=true +druid.cache.type=caffeine +#druid.cache.sizeInBytes=10MiB diff --git a/examples/conf/druid/auto/middleManager/runtime.properties b/examples/conf/druid/auto/middleManager/runtime.properties new file mode 100644 index 000000000000..15fa486c7098 --- /dev/null +++ b/examples/conf/druid/auto/middleManager/runtime.properties @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/middleManager +druid.plaintextPort=8091 + +# Number of tasks per middleManager +# druid.worker.capacity + +# Task launch parameters +druid.indexer.runner.javaCommand=bin/run-java +#druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] +druid.indexer.task.baseTaskDir=var/druid/task +#druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] + +# HTTP server threads +#druid.server.http.numThreads=12 + +# Processing threads and buffers on Peons +#druid.indexer.fork.property.druid.processing.numMergeBuffers=2 +#druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB +#druid.indexer.fork.property.druid.processing.numThreads=1 + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/examples/conf/druid/auto/router/runtime.properties b/examples/conf/druid/auto/router/runtime.properties new file mode 100644 index 000000000000..4c38ba88baab --- /dev/null +++ b/examples/conf/druid/auto/router/runtime.properties @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/router +druid.plaintextPort=8888 + +# HTTP proxy +#druid.router.http.numConnections=50 +#druid.router.http.readTimeout=PT5M +#druid.router.http.numMaxThreads=100 +#druid.server.http.numThreads=100 + +# Service discovery +druid.router.defaultBrokerServiceName=druid/broker +druid.router.coordinatorServiceName=druid/coordinator + +# Management proxy to coordinator / overlord: required for unified web console. +druid.router.managementProxy.enabled=true From 2d19c54cccd0b82c66c295c1f3306f90844255f4 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 03:22:48 +0530 Subject: [PATCH 43/82] rename python script, changes to pass repeated args to supervise --- examples/bin/start-druid-main | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index 37fe506adec5..e4e727e04748 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -24,7 +24,7 @@ import pathlib import multiprocessing import argparse -BASE_CONFIG_PATH = "conf/druid/base" +BASE_CONFIG_PATH = "conf/druid/auto" MEM_GB_SUFFIX = "g" MEM_MB_SUFFIX = "m" @@ -456,20 +456,21 @@ def distribute_memory(services, total_memory): return service_memory_config +def append_command(commands, command): + commands.append('--command') + commands.append(command) def build_supervise_script_arguments(service_list, service_memory_config, config, zk): - args = [] commands = [] - args.append('supervise') - args.append('-a') + commands.append('supervise') - commands.append(":verify bin/verify-java") - commands.append(":verify bin/verify-default-ports") - commands.append(":notify bin/greet") - commands.append(":kill-timeout 10") + append_command(commands, ":verify bin/verify-java") + append_command(commands, ":verify bin/verify-default-ports") + append_command(commands, ":notify bin/greet") + append_command(commands, ":kill-timeout 10") if zk: - commands.append("!p10 zk bin/run-zk conf") + append_command(commands, "!p10 zk bin/run-zk conf") for service in service_list: jvm_args = service_memory_config.get(service) @@ -479,17 +480,18 @@ def build_supervise_script_arguments(service_list, service_memory_config, config prefix = '!p90 ' if jvm_args is None: - commands.append('{0}{1} bin/run-druid {1} {2}'.format(prefix, service, config)) + append_command(commands, '{0}{1} bin/run-druid {1} {2}'.format(prefix, service, config)) else: if service == MIDDLE_MANAGER: task_config = service_memory_config.get(TASKS) task_count = task_config[0] task_memory = task_config[1] - commands.append( + append_command( + commands, '{0}{1} bin/run-druid {1} {2} \'{3}\' \'{4} {5}\'' .format(prefix, service, config, jvm_args, task_count, task_memory)) else: - commands.append('{0}{1} bin/run-druid {1} {2} \'{3}\''.format(prefix, service, config, jvm_args)) + append_command(commands, '{0}{1} bin/run-druid {1} {2} \'{3}\''.format(prefix, service, config, jvm_args)) print_if_verbose('Supervise script args:') for item in commands: @@ -497,8 +499,7 @@ def build_supervise_script_arguments(service_list, service_memory_config, config print_if_verbose('\n') - args.append(",".join(commands)) - return args + return commands def main(): From 657b5135e46a2b73cf1b86298f7f9004ec552cb6 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 03:23:42 +0530 Subject: [PATCH 44/82] remove exmaples/conf/druid/base dir --- .../conf/druid/base/_common/common.jvm.config | 7 - .../base/_common/common.runtime.properties | 158 ------------------ examples/conf/druid/base/_common/log4j2.xml | 86 ---------- .../conf/druid/base/broker/runtime.properties | 37 ---- .../coordinator-overlord/runtime.properties | 33 ---- .../druid/base/historical/runtime.properties | 39 ----- .../base/middleManager/runtime.properties | 41 ----- .../conf/druid/base/router/runtime.properties | 34 ---- 8 files changed, 435 deletions(-) delete mode 100644 examples/conf/druid/base/_common/common.jvm.config delete mode 100644 examples/conf/druid/base/_common/common.runtime.properties delete mode 100644 examples/conf/druid/base/_common/log4j2.xml delete mode 100644 examples/conf/druid/base/broker/runtime.properties delete mode 100644 examples/conf/druid/base/coordinator-overlord/runtime.properties delete mode 100644 examples/conf/druid/base/historical/runtime.properties delete mode 100644 examples/conf/druid/base/middleManager/runtime.properties delete mode 100644 examples/conf/druid/base/router/runtime.properties diff --git a/examples/conf/druid/base/_common/common.jvm.config b/examples/conf/druid/base/_common/common.jvm.config deleted file mode 100644 index fd74cf358979..000000000000 --- a/examples/conf/druid/base/_common/common.jvm.config +++ /dev/null @@ -1,7 +0,0 @@ --server --XX:+ExitOnOutOfMemoryError --XX:+UseG1GC --Duser.timezone=UTC --Dfile.encoding=UTF-8 --Djava.io.tmpdir=var/tmp --Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/base/_common/common.runtime.properties b/examples/conf/druid/base/_common/common.runtime.properties deleted file mode 100644 index b0adb0695cd7..000000000000 --- a/examples/conf/druid/base/_common/common.runtime.properties +++ /dev/null @@ -1,158 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Extensions specified in the load list will be loaded by Druid -# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead -# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead - -# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. -# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. -# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html -druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query"] - -# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory -# and uncomment the line below to point to your directory. -#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies - - -# -# Hostname -# -druid.host=localhost - -# -# Logging -# - -# Log all runtime properties on startup. Disable to avoid logging properties on startup: -druid.startup.logging.logProperties=true - -# -# Zookeeper -# - -druid.zk.service.host=localhost -druid.zk.paths.base=/druid - -# -# Metadata storage -# - -# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): -druid.metadata.storage.type=derby -druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true -druid.metadata.storage.connector.host=localhost -druid.metadata.storage.connector.port=1527 - -# For MySQL (make sure to include the MySQL JDBC driver on the classpath): -#druid.metadata.storage.type=mysql -#druid.metadata.storage.connector.connectURI=jdbc:mysql://db.example.com:3306/druid -#druid.metadata.storage.connector.user=... -#druid.metadata.storage.connector.password=... - -# For PostgreSQL: -#druid.metadata.storage.type=postgresql -#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid -#druid.metadata.storage.connector.user=... -#druid.metadata.storage.connector.password=... - -# -# Deep storage -# - -# For local disk (only viable in a cluster if this is a network mount): -druid.storage.type=local -druid.storage.storageDirectory=var/druid/segments - -# For HDFS: -#druid.storage.type=hdfs -#druid.storage.storageDirectory=/druid/segments - -# For S3: -#druid.storage.type=s3 -#druid.storage.bucket=your-bucket -#druid.storage.baseKey=druid/segments -#druid.s3.accessKey=... -#druid.s3.secretKey=... - -# -# Indexing service logs -# - -# For local disk (only viable in a cluster if this is a network mount): -druid.indexer.logs.type=file -druid.indexer.logs.directory=var/druid/indexing-logs - -# For HDFS: -#druid.indexer.logs.type=hdfs -#druid.indexer.logs.directory=/druid/indexing-logs - -# For S3: -#druid.indexer.logs.type=s3 -#druid.indexer.logs.s3Bucket=your-bucket -#druid.indexer.logs.s3Prefix=druid/indexing-logs - -# -# Service discovery -# - -druid.selectors.indexing.serviceName=druid/overlord -druid.selectors.coordinator.serviceName=druid/coordinator - -# -# Monitoring -# - -druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] -druid.emitter=noop -druid.emitter.logging.logLevel=info - -# Storage type of double columns -# ommiting this will lead to index double as float at the storage layer - -druid.indexing.doubleStorage=double - -# -# Security -# -druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password", "password", "key", "token", "pwd"] - - -# -# SQL -# -druid.sql.enable=true - -# Planning SQL query when there is aggregate distinct in the statement -druid.sql.planner.useGroupingSetForExactDistinct=true - -# -# Lookups -# -druid.lookup.enableLookupSyncOnStartup=false - -# -# Expression processing config -# -druid.expressions.useStrictBooleans=true - -# -# Http client -# -druid.global.http.eagerInitialization=false diff --git a/examples/conf/druid/base/_common/log4j2.xml b/examples/conf/druid/base/_common/log4j2.xml deleted file mode 100644 index 66dc13da4c5e..000000000000 --- a/examples/conf/druid/base/_common/log4j2.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/examples/conf/druid/base/broker/runtime.properties b/examples/conf/druid/base/broker/runtime.properties deleted file mode 100644 index 549117efa50d..000000000000 --- a/examples/conf/druid/base/broker/runtime.properties +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -druid.service=druid/broker -druid.plaintextPort=8082 - -# HTTP server settings -#druid.server.http.numThreads=12 - -# HTTP client settings -#druid.broker.http.numConnections=10 -#druid.broker.http.maxQueuedBytes=5MiB - -# Processing threads and buffers -#druid.processing.buffer.sizeBytes=100MiB -#druid.processing.numMergeBuffers=2 -druid.processing.tmpDir=var/druid/processing - -# Query cache disabled -- push down caching and merging instead -druid.broker.cache.useCache=false -druid.broker.cache.populateCache=false diff --git a/examples/conf/druid/base/coordinator-overlord/runtime.properties b/examples/conf/druid/base/coordinator-overlord/runtime.properties deleted file mode 100644 index c053823f8ed8..000000000000 --- a/examples/conf/druid/base/coordinator-overlord/runtime.properties +++ /dev/null @@ -1,33 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -druid.service=druid/coordinator -druid.plaintextPort=8081 - -druid.coordinator.startDelay=PT10S -druid.coordinator.period=PT5S -druid.manager.segments.pollDuration=PT5S - -# Run the overlord service in the coordinator process -druid.coordinator.asOverlord.enabled=true -druid.coordinator.asOverlord.overlordService=druid/overlord - -druid.indexer.queue.startDelay=PT5S - -druid.indexer.storage.type=metadata diff --git a/examples/conf/druid/base/historical/runtime.properties b/examples/conf/druid/base/historical/runtime.properties deleted file mode 100644 index cf3e966cc30e..000000000000 --- a/examples/conf/druid/base/historical/runtime.properties +++ /dev/null @@ -1,39 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -druid.service=druid/historical -druid.plaintextPort=8083 - -# HTTP server threads -#druid.server.http.numThreads=12 - -# Processing threads and buffers -#druid.processing.buffer.sizeBytes=200MiB -#druid.processing.numMergeBuffers=2 -#druid.processing.numThreads=2 -druid.processing.tmpDir=var/druid/processing - -# Segment storage -druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g"}] - -# Query cache -druid.historical.cache.useCache=true -druid.historical.cache.populateCache=true -druid.cache.type=caffeine -#druid.cache.sizeInBytes=10MiB diff --git a/examples/conf/druid/base/middleManager/runtime.properties b/examples/conf/druid/base/middleManager/runtime.properties deleted file mode 100644 index 15fa486c7098..000000000000 --- a/examples/conf/druid/base/middleManager/runtime.properties +++ /dev/null @@ -1,41 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -druid.service=druid/middleManager -druid.plaintextPort=8091 - -# Number of tasks per middleManager -# druid.worker.capacity - -# Task launch parameters -druid.indexer.runner.javaCommand=bin/run-java -#druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] -druid.indexer.task.baseTaskDir=var/druid/task -#druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] - -# HTTP server threads -#druid.server.http.numThreads=12 - -# Processing threads and buffers on Peons -#druid.indexer.fork.property.druid.processing.numMergeBuffers=2 -#druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB -#druid.indexer.fork.property.druid.processing.numThreads=1 - -# Hadoop indexing -druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/examples/conf/druid/base/router/runtime.properties b/examples/conf/druid/base/router/runtime.properties deleted file mode 100644 index 4c38ba88baab..000000000000 --- a/examples/conf/druid/base/router/runtime.properties +++ /dev/null @@ -1,34 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -druid.service=druid/router -druid.plaintextPort=8888 - -# HTTP proxy -#druid.router.http.numConnections=50 -#druid.router.http.readTimeout=PT5M -#druid.router.http.numMaxThreads=100 -#druid.server.http.numThreads=100 - -# Service discovery -druid.router.defaultBrokerServiceName=druid/broker -druid.router.coordinatorServiceName=druid/coordinator - -# Management proxy to coordinator / overlord: required for unified web console. -druid.router.managementProxy.enabled=true From a4c34e7dffa5d0f0a934d9aa17ebde2b3b22f838 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 03:23:51 +0530 Subject: [PATCH 45/82] add docs --- docs/operations/python.md | 54 +++++++++++++++++++++ docs/operations/single-server.md | 64 ++++++++++++++----------- docs/tutorials/cluster.md | 5 +- docs/tutorials/index.md | 25 +++++----- docs/tutorials/tutorial-batch-hadoop.md | 8 ++-- docs/tutorials/tutorial-kafka.md | 2 +- 6 files changed, 112 insertions(+), 46 deletions(-) create mode 100644 docs/operations/python.md diff --git a/docs/operations/python.md b/docs/operations/python.md new file mode 100644 index 000000000000..2bfb8f0a0615 --- /dev/null +++ b/docs/operations/python.md @@ -0,0 +1,54 @@ +--- +id: python +title: "Python Installation" +--- + + + +Apache Druid startup script requires python. This page has steps to install python and additional libraries required by the startup script. + +## Python installation instruction + +### MacOS +MacOS comes with a version of Python2, which has been deprecated. Python2 and Python3 can coexist on the same machine without problems. + +#### Install the official Python release +* Browse to the [Python Downloads Page](https://www.python.org/downloads/) and download the latest version (3.x.x) +* Verify if python3 and pip3 (python package manager) is installed by issuing `python3` and `pip3 -V` commands. + +#### Install with Homebrew +Refer [Installing Python 3 on Mac OS X](https://docs.python-guide.org/starting/install3/osx/) + +### Linux + +#### Debian + - `sudo apt update` + - `sudo apt install -y python3-pip` +#### RHEL + - `sudo yum install -y epel-release` + - `sudo yum install -y python3-pip` + +## Additional libraries + +### psutil +- pip3 install psutil + +### pathlib +- pip3 install pathlib \ No newline at end of file diff --git a/docs/operations/single-server.md b/docs/operations/single-server.md index 34dd2dc0e006..852e1597a99f 100644 --- a/docs/operations/single-server.md +++ b/docs/operations/single-server.md @@ -25,13 +25,15 @@ title: "Single server deployment" Druid includes a set of reference configurations and launch scripts for single-machine deployments: -- `nano-quickstart` -- `micro-quickstart` -- `small` -- `medium` -- `large` -- `xlarge` -- `start-druid` +- `auto` (run script: `bin/start-druid-auto`) +- `nano-quickstart` (run script: `bin/start-nano-quickstart`) +- `micro-quickstart` (run script: `bin/start-micro-quickstart`) +- `small` (run script: `bin/start-single-server-small`) +- `medium` (run script: `bin/start-single-server-medium`) +- `large` (run script: `bin/start-single-server-large`) +- `xlarge` (run script: `bin/start-single-server-xlarge`) + +The `auto` can be used to spin up Druid cluster on any machine. Read more details about this configuration [below](#Druid-auto-start). The `micro-quickstart` is sized for small machines like laptops and is intended for quick evaluation use-cases. @@ -43,43 +45,51 @@ The startup scripts for these example configurations run a single ZK instance al The example configurations run the Druid Coordinator and Overlord together in a single process using the optional configuration `druid.coordinator.asOverlord.enabled=true`, described in the [Coordinator configuration documentation](../configuration/index.md#coordinator-operation). -The `start-druid` is a generic launch script for starting druid services on single server, it accepts optional arguments like services, memory and config. -All reference configurations can be acheived by passing appropriate arguments to this script. -Existing launch scripts are deprecated and will be removed in the next release. +While example configurations are provided for very large single machines, at higher scales we recommend running Druid in a [clustered deployment](../tutorials/cluster.md), for fault-tolerance and reduced resource contention. + +## Druid auto start + +Setting up a new Druid cluster can sometimes be complicated as there are several runtime properties required by each service. Each process must also be given the appropriate jvm arguments so that the system can perform optimally. + +`start-druid-auto` is a generic launch script capable of starting any set of Druid services on a server. +It accepts optional arguments such as list of services, total memory and a config directory to override default jvm arguments and service-specific runtime properties. +All other reference configurations (e.g. `micro`, `small`, `xlarge`) can be obtained by passing the +correct memory to this script. +Druid services will use all processors and upto 80% memory on the system. +For details about possible arguments, run `bin/start-druid-auto --help`. + +The corresponding launch scripts (e.g. `start-micro-quickstart`) are now deprecated. -While example configurations are provided for very large single machines, at higher scales we recommend running Druid in a [clustered deployment](../tutorials/cluster.md), for fault-tolerance and reduced resource contention. ## Single server reference configurations -### Nano: 1 CPU, 4GiB RAM +### Nano-Quickstart: 1 CPU, 4GiB RAM -- Launch command: `bin/start-druid --memory=4g` -- Configuration directory: `conf/druid/single-server/quickstart` +- Launch command: `bin/start-nano-quickstart` +- Configuration directory: `conf/druid/single-server/nano-quickstart` -### Micro: 4 CPU, 16GiB RAM +### Micro-Quickstart: 4 CPU, 16GiB RAM -- Launch command: `bin/start-druid --memory=16g` -- Configuration directory: `conf/druid/single-server/quickstart` +- Launch command: `bin/start-micro-quickstart` +- Configuration directory: `conf/druid/single-server/micro-quickstart` ### Small: 8 CPU, 64GiB RAM (~i3.2xlarge) -- Launch command: `bin/start-druid --memory=64g` -- Configuration directory: `conf/druid/single-server/quickstart` +- Launch command: `bin/start-small` +- Configuration directory: `conf/druid/single-server/small` ### Medium: 16 CPU, 128GiB RAM (~i3.4xlarge) -- Launch command: `bin/start-druid --memory=128g` -- Configuration directory: `conf/druid/single-server/quickstart` +- Launch command: `bin/start-medium` +- Configuration directory: `conf/druid/single-server/medium` ### Large: 32 CPU, 256GiB RAM (~i3.8xlarge) -- Launch command: `bin/start-druid --memory=256g` -- Configuration directory: `conf/druid/single-server/quickstart` +- Launch command: `bin/start-large` +- Configuration directory: `conf/druid/single-server/large` ### X-Large: 64 CPU, 512GiB RAM (~i3.16xlarge) -- Launch command: `bin/start-druid --memory=512g` -- Configuration directory: `conf/druid/single-server/quickstart` - -Memory argument (`--memory`) in the above launch command is optional, if not specified Druid will use upto 80% of system memory. \ No newline at end of file +- Launch command: `bin/start-xlarge` +- Configuration directory: `conf/druid/single-server/xlarge` \ No newline at end of file diff --git a/docs/tutorials/cluster.md b/docs/tutorials/cluster.md index b61953c2f427..5ee68e2cd13e 100644 --- a/docs/tutorials/cluster.md +++ b/docs/tutorials/cluster.md @@ -130,7 +130,10 @@ The [basic cluster tuning guide](../operations/basic-cluster-tuning.md) has info ## Select OS -We recommend running your favorite Linux distribution. You will also need [Java 8 or 11](../operations/java.md). +We recommend running your favorite Linux distribution. You will also need + +* [Java 8 or 11](../operations/java.md). +* [Python2 or Python3](../operations/python.md) > If needed, you can specify where to find Java using the environment variables > `DRUID_JAVA_HOME` or `JAVA_HOME`. For more details run the `bin/verify-java` script. diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index 8d029d568189..c611fde67cc7 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -22,8 +22,7 @@ title: "Quickstart (local)" ~ under the License. --> - -This quickstart gets you started with Apache Druid using the [`micro`](../operations/single-server.md#micro-4-cpu-16gib-ram) configuration, and introduces you to Druid ingestion and query features. +This quickstart gets you started with Apache Druid and introduces you to Druid ingestion and query features. For this tutorial, we recommend a machine with at least 6 GB of RAM. In this quickstart, you'll do the following: - install Druid @@ -37,16 +36,16 @@ Druid supports a variety of ingestion options. Once you're done with this tutori You can follow these steps on a relatively modest machine, such as a workstation or virtual server with 16 GiB of RAM. -Druid comes equipped with a single launch script that can be used to run several -[startup configuration profiles](../operations/single-server.md) for a -range of machine sizes. These range from `nano` (1 CPU, 4GiB RAM) to `x-large` (64 CPU, 512GiB RAM). For more -information, see [Single server deployment](../operations/single-server.md). For information on deploying Druid services -across clustered machines, see [Clustered deployment](./cluster.md). +Druid comes equipped with launch scripts that can be used to start all processes on a single server. Here, we will use [`auto`](../operations/single-server.md#Druid-auto-start), which automatically sets various runtime properties based on available processors and memory. + +In addition, Druid includes several [bundled non-automatic profiles](../operations/single-server.md) for a range of machine sizes. These range from nano (1 CPU, 4GiB RAM) to x-large (64 CPU, 512GiB RAM). +We won't use those here, but for more information, see [Single server deployment](../operations/single-server.md). For additional information on deploying Druid services across clustered machines, see [Clustered deployment](./cluster.md). The software requirements for the installation machine are: * Linux, Mac OS X, or other Unix-like OS. (Windows is not supported.) * Java 8u92+ or Java 11. +* [Python2 or Python3](../operations/python.md) > Druid relies on the environment variables `JAVA_HOME` or `DRUID_JAVA_HOME` to find Java on the machine. You can set `DRUID_JAVA_HOME` if there is more than one instance of Java. To verify Java requirements for your environment, run the @@ -73,21 +72,21 @@ The distribution directory contains `LICENSE` and `NOTICE` files and subdirector ## Start up Druid services -Start up Druid services using the `micro` single-machine configuration. +Start up Druid services using the `auto` single-machine configuration. This configuration includes default settings that are appropriate for this tutorial, such as loading the `druid-multi-stage-query` extension by default so that you can use the MSQ task engine. -You can view that setting and others in the configuration files in the `conf/druid/single-server/quickstart/`. +You can view that setting and others in the configuration files in the `conf/druid/auto`. From the apache-druid-{{DRUIDVERSION}} package root, run the following command: ```bash -./bin/start-druid --memory=16g +./bin/start-druid-main-auto ``` This brings up instances of ZooKeeper and the Druid services: ```bash -$ ./bin/start-micro-quickstart +$ ./bin/start-druid-main-auto [Tue Nov 29 16:31:06 2022] Starting Apache Druid. [Tue Nov 29 16:31:06 2022] Open http://localhost:8888/ in your browser to access the web console. [Tue Nov 29 16:31:06 2022] Or, if you have enabled TLS, use https on port 9088. @@ -105,7 +104,7 @@ the Druid root directory, apache-druid-{{DRUIDVERSION}}. Each service writes to At any time, you can revert Druid to its original, post-installation state by deleting the entire `var` directory. You may want to do this, for example, between Druid tutorials or after experimentation, to start with a fresh instance. -To stop Druid at any time, use CTRL+C in the terminal. This exits the `bin/start-druid` script and terminates all Druid processes. +To stop Druid at any time, use CTRL+C in the terminal. This exits the `bin/start-druid-auto` script and terminates all Druid processes. ## Open the web console @@ -224,4 +223,4 @@ See the following topics for more information: * [Tutorial: Load stream data from Apache Kafka](./tutorial-kafka.md) to load streaming data from a Kafka topic. * [Extensions](../development/extensions.md) for details on Druid extensions. -Remember that after stopping Druid services, you can start clean next time by deleting the `var` directory from the Druid root directory and running the `bin/start-druid` script again. You may want to do this before using other data ingestion tutorials, since they use the same Wikipedia datasource. +Remember that after stopping Druid services, you can start clean next time by deleting the `var` directory from the Druid root directory and running the `bin/start-druid-auto` script again. You may want to do this before using other data ingestion tutorials, since they use the same Wikipedia datasource. diff --git a/docs/tutorials/tutorial-batch-hadoop.md b/docs/tutorials/tutorial-batch-hadoop.md index bff033a28c41..e8ea8a19052f 100644 --- a/docs/tutorials/tutorial-batch-hadoop.md +++ b/docs/tutorials/tutorial-batch-hadoop.md @@ -28,7 +28,7 @@ This tutorial shows you how to load data files into Apache Druid using a remote For this tutorial, we'll assume that you've already completed the previous [batch ingestion tutorial](tutorial-batch.md) using Druid's native batch ingestion system and are using the -`micro` single-machine configuration as described in the [quickstart](../operations/single-server.md#micro-4-cpu-16gib-ram). +`auto` single-machine configuration as described in the [quickstart](../operations/single-server.md#Druid-auto-start). ## Install Docker @@ -156,7 +156,7 @@ cp /tmp/shared/hadoop_xml/*.xml {PATH_TO_DRUID}/conf/druid/single-server/micro-q ### Update Druid segment and log storage -In your favorite text editor, open `conf/druid/single-server/micro-quickstart/_common/common.runtime.properties`, and make the following edits: +In your favorite text editor, open `conf/druid/auto/_common/common.runtime.properties`, and make the following edits: #### Disable local deep storage and enable HDFS deep storage @@ -196,7 +196,7 @@ druid.indexer.logs.directory=/druid/indexing-logs Once the Hadoop .xml files have been copied to the Druid cluster and the segment/log storage configuration has been updated to use HDFS, the Druid cluster needs to be restarted for the new configurations to take effect. -If the cluster is still running, CTRL-C to terminate the `bin/start-micro-quickstart` script, and re-run it to bring the Druid services back up. +If the cluster is still running, CTRL-C to terminate the `bin/start-druid-auto` script, and re-run it to bring the Druid services back up. ## Load batch data @@ -221,7 +221,7 @@ This tutorial is only meant to be used together with the [query tutorial](../tut If you wish to go through any of the other tutorials, you will need to: * Shut down the cluster and reset the cluster state by removing the contents of the `var` directory under the druid package. -* Revert the deep storage and task storage config back to local types in `conf/druid/single-server/micro-quickstart/_common/common.runtime.properties` +* Revert the deep storage and task storage config back to local types in `conf/druid/auto/_common/common.runtime.properties` * Restart the cluster This is necessary because the other ingestion tutorials will write to the same "wikipedia" datasource, and later tutorials expect the cluster to use local deep storage. diff --git a/docs/tutorials/tutorial-kafka.md b/docs/tutorials/tutorial-kafka.md index bbdf6af72ea0..73feeaf65901 100644 --- a/docs/tutorials/tutorial-kafka.md +++ b/docs/tutorials/tutorial-kafka.md @@ -30,7 +30,7 @@ The tutorial guides you through the steps to load sample nested clickstream data ## Prerequisites -Before you follow the steps in this tutorial, download Druid as described in the [quickstart](index.md) using the [micro](../operations/single-server.md#micro-4-cpu-16gib-ram) single-machine configuration and have it running on your local machine. You don't need to have loaded any data. +Before you follow the steps in this tutorial, download Druid as described in the [quickstart](index.md) using the [auto](../operations/single-server.md#Druid-auto-start) single-machine configuration and have it running on your local machine. You don't need to have loaded any data. ## Download and start Kafka From 0fd9b09f61acce6104364a5cbda2207257b4569c Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 03:33:14 +0530 Subject: [PATCH 46/82] restore changes in conf dir --- .../conf/druid/auto/broker/runtime.properties | 14 +++++++++----- .../druid/auto/historical/runtime.properties | 16 ++++++---------- .../druid/auto/middleManager/runtime.properties | 15 +++++---------- .../conf/druid/auto/router/runtime.properties | 6 ------ 4 files changed, 20 insertions(+), 31 deletions(-) diff --git a/examples/conf/druid/auto/broker/runtime.properties b/examples/conf/druid/auto/broker/runtime.properties index 549117efa50d..f4c494019933 100644 --- a/examples/conf/druid/auto/broker/runtime.properties +++ b/examples/conf/druid/auto/broker/runtime.properties @@ -21,15 +21,19 @@ druid.service=druid/broker druid.plaintextPort=8082 # HTTP server settings -#druid.server.http.numThreads=12 +# HTTP server thread pool size. Higher values increase peak load on the Broker, but +# may be useful for high-concurrency workloads. +# Default is max(10, (Number of processors * 17) / 16 + 2) + 30. +# druid.server.http.numThreads=N # HTTP client settings -#druid.broker.http.numConnections=10 -#druid.broker.http.maxQueuedBytes=5MiB +# Connection pool size from the Broker to each data server. May be useful to +# raise this for high-concurrency workloads. +# druid.broker.http.numConnections=20 # Processing threads and buffers -#druid.processing.buffer.sizeBytes=100MiB -#druid.processing.numMergeBuffers=2 +# Determined automatically based on available memory. For details on how to manually set parameters: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers druid.processing.tmpDir=var/druid/processing # Query cache disabled -- push down caching and merging instead diff --git a/examples/conf/druid/auto/historical/runtime.properties b/examples/conf/druid/auto/historical/runtime.properties index cf3e966cc30e..6c241aa7cf06 100644 --- a/examples/conf/druid/auto/historical/runtime.properties +++ b/examples/conf/druid/auto/historical/runtime.properties @@ -17,23 +17,19 @@ # under the License. # -druid.service=druid/historical -druid.plaintextPort=8083 - -# HTTP server threads -#druid.server.http.numThreads=12 +# HTTP server thread pool size. Higher values increase peak load on the Broker, but +# may be useful for high-concurrency workloads. +# Default is max(10, (Number of processors * 17) / 16 + 2) + 30. +# druid.server.http.numThreads=N # Processing threads and buffers -#druid.processing.buffer.sizeBytes=200MiB -#druid.processing.numMergeBuffers=2 -#druid.processing.numThreads=2 +# Determined automatically based on available memory. For details on how to manually set parameters: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers druid.processing.tmpDir=var/druid/processing # Segment storage druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":"300g"}] - # Query cache druid.historical.cache.useCache=true druid.historical.cache.populateCache=true druid.cache.type=caffeine -#druid.cache.sizeInBytes=10MiB diff --git a/examples/conf/druid/auto/middleManager/runtime.properties b/examples/conf/druid/auto/middleManager/runtime.properties index 15fa486c7098..08c58bae6de0 100644 --- a/examples/conf/druid/auto/middleManager/runtime.properties +++ b/examples/conf/druid/auto/middleManager/runtime.properties @@ -20,22 +20,17 @@ druid.service=druid/middleManager druid.plaintextPort=8091 -# Number of tasks per middleManager -# druid.worker.capacity +# Number of tasks (druid.worker.capacity) and memory usage per task (druid.indexer.runner.javaOptsArray) is automatically +# determined based on available memory. For details on how to manually set parameters, see: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#middlemanager # Task launch parameters druid.indexer.runner.javaCommand=bin/run-java -#druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] druid.indexer.task.baseTaskDir=var/druid/task -#druid.indexer.runner.javaOptsArray=["-server","-Xms1g","-Xmx1g","-XX:MaxDirectMemorySize=1g","-Duser.timezone=UTC","-Dfile.encoding=UTF-8","-XX:+ExitOnOutOfMemoryError","-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] - -# HTTP server threads -#druid.server.http.numThreads=12 # Processing threads and buffers on Peons -#druid.indexer.fork.property.druid.processing.numMergeBuffers=2 -#druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB -#druid.indexer.fork.property.druid.processing.numThreads=1 +# Determined automatically based on available memory. For details on how to manually set parameters: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers # Hadoop indexing druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/examples/conf/druid/auto/router/runtime.properties b/examples/conf/druid/auto/router/runtime.properties index 4c38ba88baab..3858dec044bd 100644 --- a/examples/conf/druid/auto/router/runtime.properties +++ b/examples/conf/druid/auto/router/runtime.properties @@ -20,12 +20,6 @@ druid.service=druid/router druid.plaintextPort=8888 -# HTTP proxy -#druid.router.http.numConnections=50 -#druid.router.http.readTimeout=PT5M -#druid.router.http.numMaxThreads=100 -#druid.server.http.numThreads=100 - # Service discovery druid.router.defaultBrokerServiceName=druid/broker druid.router.coordinatorServiceName=druid/coordinator From c0da3cdbed645aef569d912fd44fcb3682eeb13a Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 03:44:54 +0530 Subject: [PATCH 47/82] update start-druid-auto --- examples/bin/start-druid-auto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/bin/start-druid-auto b/examples/bin/start-druid-auto index 38f7a5f9b0a5..b6dc9cfbf57b 100644 --- a/examples/bin/start-druid-auto +++ b/examples/bin/start-druid-auto @@ -23,10 +23,10 @@ WHEREAMI="$(cd "$WHEREAMI" && pwd)" if [ -x "$(command -v python3)" ] then - exec python3 "$WHEREAMI/start-druid" "$@" + exec python3 "$WHEREAMI/start-druid-main" "$@" elif [ -x "$(command -v python2)" ] then - exec python2 "$WHEREAMI/start-druid" "$@" + exec python2 "$WHEREAMI/start-druid-main" "$@" else - exec "$WHEREAMI/start-druid" "$@" + exec "$WHEREAMI/start-druid-main" "$@" fi From 7b2132373ffb0ee3c8ba5c578d85e3d0bf34d60c Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 10:51:53 +0530 Subject: [PATCH 48/82] remove hashref for commands in supervise script --- examples/bin/supervise | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/bin/supervise b/examples/bin/supervise index e9cdca5ce925..29aca9eeddb8 100755 --- a/examples/bin/supervise +++ b/examples/bin/supervise @@ -196,16 +196,16 @@ usage() unless GetOptions( 'kill-timeout|t=i', 'chdir=s', 'svlogd:s', - 'command=s@' => \@commands + 'command=s@' ); -usage() unless ((@commands || $opt{'conf'}) && $opt{'vardir'}); +usage() unless ((@{$opt{'command'}} || $opt{'conf'}) && $opt{'vardir'}); my @config_lines; # get commands to execute either from reading the config file or command line if (not defined $opt{'conf'}) { - @config_lines = @commands + @config_lines = @{$opt{'command'}} } else { @config_lines = read_config_file($opt{'conf'}); } From 7535ec87e90c138322b2d7a8a8eb1a292523fb05 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 10:53:30 +0530 Subject: [PATCH 49/82] start-druid-main java_opts array is comma separated --- examples/bin/start-druid-main | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index e4e727e04748..17e74ff9d8f5 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -369,7 +369,7 @@ def build_mm_task_java_opts_array(memory_type): java_opts_list = TASK_JAVA_OPTS_ARRAY + mem_array for item in java_opts_list: - task_memory += '\"{0}\";'.format(item) + task_memory += '\"{0}\",'.format(item) task_memory = task_memory[:-1] task_memory += ']' From 144042f7c0bf1a645f944cfcbb4779a23f3755a6 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 11:04:46 +0530 Subject: [PATCH 50/82] update entry point script name in python script --- examples/bin/start-druid-main | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index 17e74ff9d8f5..6889fdb5e8f8 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -97,31 +97,31 @@ def print_if_verbose(message): def configure_parser(): parser = argparse.ArgumentParser( - prog='start-druid', + prog='start-druid-auto', formatter_class=argparse.RawTextHelpFormatter, epilog= """ sample usage: - start-druid + start-druid-auto Start up all the services (including zk). - start-druid -m=100g + start-druid-auto -m=100g Start up all the services (including zk) using a total memory of 100GB. - start-druid -m=100g --compute + start-druid-auto -m=100g --compute Compute memory distribution and validate arguments. - start-druid -m=100g -s=broker,router + start-druid-auto -m=100g -s=broker,router Starts a broker and a router, using a total memory of 100GB. - start-druid -m=100g --s=broker,router \\ + start-druid-auto -m=100g --s=broker,router \\ -c=conf/druid/single-server/custom Starts a broker and a router, using a total memory of 100GB. Reads configs for each service (jvm.config, runtime.properties) from respective folders inside the given root config path. - start-druid -s=broker,router \\ + start-druid-auto -s=broker,router \\ -c=conf/druid/single-server/custom Starts a broker and a router service, reading service configs from the given root directory. Calculates memory requirements for each service, if required, using upto 80% of the total system memory. - start-druid -m=100g \\ + start-druid-auto -m=100g \\ -s=broker,router \\ -c=conf/druid/single-server/custom \\ --zk From f5b225f836cc63e87b95ab2d2eff36402931208a Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 11:10:43 +0530 Subject: [PATCH 51/82] Update help docs --- examples/bin/start-druid-main | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main index 6889fdb5e8f8..9e80fd22bb98 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main @@ -144,7 +144,7 @@ sample usage: 'This directory must contain \'_common\' directory with \n' '\'common.jvm.config\' & \'common.runtime.properties\' files. \n' 'If this argument is not given, config from \n' - 'conf/druid/single-server/quickstart directory is used.\n') + 'conf/druid/auto directory is used.\n') parser.add_argument('--compute', action='store_true', help='Does not start Druid, only displays the memory allocated \n' 'to each service if started with the given total memory.\n') From 42be20aef9ab9f790820c4c6d664285dac6002fd Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 12:43:24 +0530 Subject: [PATCH 52/82] documentation changes --- docs/operations/single-server.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/operations/single-server.md b/docs/operations/single-server.md index 852e1597a99f..ce7bfe5752c1 100644 --- a/docs/operations/single-server.md +++ b/docs/operations/single-server.md @@ -23,7 +23,10 @@ title: "Single server deployment" --> -Druid includes a set of reference configurations and launch scripts for single-machine deployments: +Druid includes a set of reference configurations and launch scripts for single-machine deployments. +These configuration bundles are located in `conf/druid/single-server/`. + +The `auto` configuration sizes runtime parameters based on available processors and memory. Other configurations include hard-coded runtime parameters for various server sizes. Most users should stick with `auto`. - `auto` (run script: `bin/start-druid-auto`) - `nano-quickstart` (run script: `bin/start-nano-quickstart`) @@ -33,8 +36,6 @@ Druid includes a set of reference configurations and launch scripts for single-m - `large` (run script: `bin/start-single-server-large`) - `xlarge` (run script: `bin/start-single-server-xlarge`) -The `auto` can be used to spin up Druid cluster on any machine. Read more details about this configuration [below](#Druid-auto-start). - The `micro-quickstart` is sized for small machines like laptops and is intended for quick evaluation use-cases. The `nano-quickstart` is an even smaller configuration, targeting a machine with 1 CPU and 4GiB memory. It is meant for limited evaluations in resource constrained environments, such as small Docker containers. From eed55bed984a5789ed269ac1294d55f8037e90cb Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 12:43:35 +0530 Subject: [PATCH 53/82] docs changes --- docs/operations/single-server.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/operations/single-server.md b/docs/operations/single-server.md index ce7bfe5752c1..8ea6e8b5ae55 100644 --- a/docs/operations/single-server.md +++ b/docs/operations/single-server.md @@ -26,8 +26,7 @@ title: "Single server deployment" Druid includes a set of reference configurations and launch scripts for single-machine deployments. These configuration bundles are located in `conf/druid/single-server/`. -The `auto` configuration sizes runtime parameters based on available processors and memory. Other configurations include hard-coded runtime parameters for various server sizes. Most users should stick with `auto`. - +The `auto` configuration sizes runtime parameters based on available processors and memory. Other configurations include hard-coded runtime parameters for various server sizes. Most users should stick with `auto`. Refer below [Druid auto start](#Druid-auto-start) - `auto` (run script: `bin/start-druid-auto`) - `nano-quickstart` (run script: `bin/start-nano-quickstart`) - `micro-quickstart` (run script: `bin/start-micro-quickstart`) @@ -48,7 +47,7 @@ The example configurations run the Druid Coordinator and Overlord together in a While example configurations are provided for very large single machines, at higher scales we recommend running Druid in a [clustered deployment](../tutorials/cluster.md), for fault-tolerance and reduced resource contention. -## Druid auto start +## Druid auto start Setting up a new Druid cluster can sometimes be complicated as there are several runtime properties required by each service. Each process must also be given the appropriate jvm arguments so that the system can perform optimally. From 9063884bf9c874331b23b9ac7906b707a22199b9 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 15:05:59 +0530 Subject: [PATCH 54/82] update docs --- docs/tutorials/index.md | 4 ++-- .../{start-druid-main => start-druid-main.py} | 24 +++++++++++++++---- 2 files changed, 21 insertions(+), 7 deletions(-) rename examples/bin/{start-druid-main => start-druid-main.py} (95%) diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index c611fde67cc7..ed749c7efcfe 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -80,13 +80,13 @@ You can view that setting and others in the configuration files in the `conf/dru From the apache-druid-{{DRUIDVERSION}} package root, run the following command: ```bash -./bin/start-druid-main-auto +./bin/start-druid-auto ``` This brings up instances of ZooKeeper and the Druid services: ```bash -$ ./bin/start-druid-main-auto +$ ./bin/start-druid-auto [Tue Nov 29 16:31:06 2022] Starting Apache Druid. [Tue Nov 29 16:31:06 2022] Open http://localhost:8888/ in your browser to access the web console. [Tue Nov 29 16:31:06 2022] Or, if you have enabled TLS, use https on port 9088. diff --git a/examples/bin/start-druid-main b/examples/bin/start-druid-main.py similarity index 95% rename from examples/bin/start-druid-main rename to examples/bin/start-druid-main.py index 9e80fd22bb98..1201437479da 100644 --- a/examples/bin/start-druid-main +++ b/examples/bin/start-druid-main.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -51,6 +49,7 @@ HISTORICAL = "historical" MIDDLE_MANAGER = "middleManager" TASKS = "tasks" +INDEXER = "indexer" DEFAULT_SERVICES = [ BROKER, @@ -135,8 +134,8 @@ def configure_parser(): 'in the given conf directory. e.g. 500m, 4g, 6g\n') parser.add_argument('--services', '-s', type=str, required=False, help='List of services to be started, subset of \n' - '{broker, router, middleManager, historical, coordinator-overlord}. \n' - 'If the argument is not given, all services \n' + '{broker, router, middleManager, historical, coordinator-overlord, indexer}. \n' + 'If the argument is not given, broker, router, middleManager, historical, coordinator-overlord \n' 'and zookeeper is started. e.g. -sl=broker,historical') parser.add_argument('--config', '-c', type=str, required=False, help='Relative path to the directory containing common and service \n' @@ -290,6 +289,21 @@ def should_compute_memory(config, total_memory, service_list): mm_task_java_opts_property, mm_task_worker_capacity_prop = middle_manager_task_memory_params_present(config) mm_task_property_present = mm_task_java_opts_property or mm_task_worker_capacity_prop + # if indexer has to be run, all the memory related parameters need to be specified + if INDEXER in service_list: + if MIDDLE_MANAGER in service_list: + raise ValueError("one of indexer or middleManager can run") + if total_memory != "": + raise ValueError( + "If service list includes indexer, jvm.config should be specified for " + "each service and memory argument shouldn't be specified") + if jvm_config_count != len(service_list): + raise ValueError("If service list includes indexer, jvm.config should be specified for each service") + for service in service_list: + verify_service_config(service, config) + + return False + # possible error states # 1. memory argument is specified, also jvm.config or middleManger/runtime.properties having # druid.indexer.runner.javaOptsArray or druid.worker.capacity parameters is present @@ -298,7 +312,7 @@ def should_compute_memory(config, total_memory, service_list): # 3. jvm.config present for some but not all services # 4. jvm.config file is present for all services, but it doesn't contain required parameters # 5. lastly, if middleManager is to be started, and it is missing task memory properties - if jvm_config_count > 0 or mm_task_property_present: + if INDEXER in service_list or jvm_config_count > 0 or mm_task_property_present: if total_memory != "": raise ValueError( "If jvm.config for services and/or middleManager configs " From fe5c7733b6ea7bc7c07a76bde817c359ba75edf3 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 15:07:22 +0530 Subject: [PATCH 55/82] add support for running indexer --- examples/bin/start-druid-auto | 6 +-- examples/conf/druid/auto/indexer/jvm.config | 9 +++++ .../druid/auto/indexer/runtime.properties | 38 +++++++++++++++++++ examples/conf/druid/indexer/jvm.config | 9 +++++ examples/conf/druid/indexer/main.config | 1 + .../conf/druid/indexer/runtime.properties | 38 +++++++++++++++++++ 6 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 examples/conf/druid/auto/indexer/jvm.config create mode 100644 examples/conf/druid/auto/indexer/runtime.properties create mode 100644 examples/conf/druid/indexer/jvm.config create mode 100644 examples/conf/druid/indexer/main.config create mode 100644 examples/conf/druid/indexer/runtime.properties diff --git a/examples/bin/start-druid-auto b/examples/bin/start-druid-auto index b6dc9cfbf57b..c0447e6eb939 100644 --- a/examples/bin/start-druid-auto +++ b/examples/bin/start-druid-auto @@ -23,10 +23,10 @@ WHEREAMI="$(cd "$WHEREAMI" && pwd)" if [ -x "$(command -v python3)" ] then - exec python3 "$WHEREAMI/start-druid-main" "$@" + exec python3 "$WHEREAMI/start-druid-main.py" "$@" elif [ -x "$(command -v python2)" ] then - exec python2 "$WHEREAMI/start-druid-main" "$@" + exec python2 "$WHEREAMI/start-druid-main.py" "$@" else - exec "$WHEREAMI/start-druid-main" "$@" + exec "$WHEREAMI/start-druid-main.py" "$@" fi diff --git a/examples/conf/druid/auto/indexer/jvm.config b/examples/conf/druid/auto/indexer/jvm.config new file mode 100644 index 000000000000..4611a65196a2 --- /dev/null +++ b/examples/conf/druid/auto/indexer/jvm.config @@ -0,0 +1,9 @@ +-server +-Xms4g +-Xmx4g +-XX:MaxDirectMemorySize=4g +-XX:+ExitOnOutOfMemoryError +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/auto/indexer/runtime.properties b/examples/conf/druid/auto/indexer/runtime.properties new file mode 100644 index 000000000000..b36c9eb42872 --- /dev/null +++ b/examples/conf/druid/auto/indexer/runtime.properties @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/indexer +druid.plaintextPort=8091 + +# Number of tasks per indexer +druid.worker.capacity=4 + +# Task launch parameters +druid.indexer.task.baseTaskDir=var/druid/task + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers on Indexer +druid.processing.numMergeBuffers=2 +druid.processing.buffer.sizeBytes=100MiB +druid.processing.numThreads=4 + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/examples/conf/druid/indexer/jvm.config b/examples/conf/druid/indexer/jvm.config new file mode 100644 index 000000000000..4611a65196a2 --- /dev/null +++ b/examples/conf/druid/indexer/jvm.config @@ -0,0 +1,9 @@ +-server +-Xms4g +-Xmx4g +-XX:MaxDirectMemorySize=4g +-XX:+ExitOnOutOfMemoryError +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/indexer/main.config b/examples/conf/druid/indexer/main.config new file mode 100644 index 000000000000..5183399415cf --- /dev/null +++ b/examples/conf/druid/indexer/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server indexer diff --git a/examples/conf/druid/indexer/runtime.properties b/examples/conf/druid/indexer/runtime.properties new file mode 100644 index 000000000000..b36c9eb42872 --- /dev/null +++ b/examples/conf/druid/indexer/runtime.properties @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/indexer +druid.plaintextPort=8091 + +# Number of tasks per indexer +druid.worker.capacity=4 + +# Task launch parameters +druid.indexer.task.baseTaskDir=var/druid/task + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers on Indexer +druid.processing.numMergeBuffers=2 +druid.processing.buffer.sizeBytes=100MiB +druid.processing.numThreads=4 + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp From 62912bb4f71173fac8eb0caecce0b1bbaa054da6 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 15:17:24 +0530 Subject: [PATCH 56/82] update supported services list --- examples/bin/start-druid-main.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/examples/bin/start-druid-main.py b/examples/bin/start-druid-main.py index 1201437479da..b617786d0125 100644 --- a/examples/bin/start-druid-main.py +++ b/examples/bin/start-druid-main.py @@ -59,6 +59,15 @@ MIDDLE_MANAGER ] +SUPPORTED_SERVICES = [ + BROKER, + ROUTER, + COORDINATOR, + HISTORICAL, + MIDDLE_MANAGER, + INDEXER +] + SERVICE_MEMORY_RATIO = { MIDDLE_MANAGER: 1, ROUTER: 2, @@ -196,7 +205,7 @@ def parse_arguments(args): services = args.services.split(SERVICE_SEPARATOR) for service in services: - if service not in DEFAULT_SERVICES: + if service not in SUPPORTED_SERVICES: raise ValueError('Invalid service name {0}, should be one of {1}'.format(service, DEFAULT_SERVICES)) if service in service_list: From 9b08b04941e947a532e11a4747eb70d28f1e9fbc Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 15:20:12 +0530 Subject: [PATCH 57/82] update help --- examples/bin/start-druid-main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/bin/start-druid-main.py b/examples/bin/start-druid-main.py index b617786d0125..d6e26e8aaabe 100644 --- a/examples/bin/start-druid-main.py +++ b/examples/bin/start-druid-main.py @@ -145,7 +145,7 @@ def configure_parser(): help='List of services to be started, subset of \n' '{broker, router, middleManager, historical, coordinator-overlord, indexer}. \n' 'If the argument is not given, broker, router, middleManager, historical, coordinator-overlord \n' - 'and zookeeper is started. e.g. -sl=broker,historical') + 'and zookeeper is started. e.g. -s=broker,historical') parser.add_argument('--config', '-c', type=str, required=False, help='Relative path to the directory containing common and service \n' 'specific properties to be overridden. \n' From 622a7ffb39716d5e3b08ec1211b5b3637ec034ea Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 15:37:34 +0530 Subject: [PATCH 58/82] Update python.md --- docs/operations/python.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/operations/python.md b/docs/operations/python.md index 2bfb8f0a0615..ecbfafa4e7c7 100644 --- a/docs/operations/python.md +++ b/docs/operations/python.md @@ -22,9 +22,9 @@ title: "Python Installation" ~ under the License. --> -Apache Druid startup script requires python. This page has steps to install python and additional libraries required by the startup script. +Apache Druid startup script requires a python interpreter. This page has steps to install python interpreter and required libraries. -## Python installation instruction +## Python interpreter installation instructions ### MacOS MacOS comes with a version of Python2, which has been deprecated. Python2 and Python3 can coexist on the same machine without problems. @@ -45,7 +45,7 @@ Refer [Installing Python 3 on Mac OS X](https://docs.python-guide.org/starting/i - `sudo yum install -y epel-release` - `sudo yum install -y python3-pip` -## Additional libraries +## Required libraries ### psutil - pip3 install psutil From 6da787c67766cc4e32ae2a072ba9104c04443023 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 15:51:04 +0530 Subject: [PATCH 59/82] remove dir --- examples/conf/druid/indexer/jvm.config | 9 ----- examples/conf/druid/indexer/main.config | 1 - .../conf/druid/indexer/runtime.properties | 38 ------------------- 3 files changed, 48 deletions(-) delete mode 100644 examples/conf/druid/indexer/jvm.config delete mode 100644 examples/conf/druid/indexer/main.config delete mode 100644 examples/conf/druid/indexer/runtime.properties diff --git a/examples/conf/druid/indexer/jvm.config b/examples/conf/druid/indexer/jvm.config deleted file mode 100644 index 4611a65196a2..000000000000 --- a/examples/conf/druid/indexer/jvm.config +++ /dev/null @@ -1,9 +0,0 @@ --server --Xms4g --Xmx4g --XX:MaxDirectMemorySize=4g --XX:+ExitOnOutOfMemoryError --Duser.timezone=UTC --Dfile.encoding=UTF-8 --Djava.io.tmpdir=var/tmp --Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/indexer/main.config b/examples/conf/druid/indexer/main.config deleted file mode 100644 index 5183399415cf..000000000000 --- a/examples/conf/druid/indexer/main.config +++ /dev/null @@ -1 +0,0 @@ -org.apache.druid.cli.Main server indexer diff --git a/examples/conf/druid/indexer/runtime.properties b/examples/conf/druid/indexer/runtime.properties deleted file mode 100644 index b36c9eb42872..000000000000 --- a/examples/conf/druid/indexer/runtime.properties +++ /dev/null @@ -1,38 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -druid.service=druid/indexer -druid.plaintextPort=8091 - -# Number of tasks per indexer -druid.worker.capacity=4 - -# Task launch parameters -druid.indexer.task.baseTaskDir=var/druid/task - -# HTTP server threads -druid.server.http.numThreads=60 - -# Processing threads and buffers on Indexer -druid.processing.numMergeBuffers=2 -druid.processing.buffer.sizeBytes=100MiB -druid.processing.numThreads=4 - -# Hadoop indexing -druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp From 0677d0ca070effef52b146009f300340f54bec6a Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 1 Dec 2022 21:11:54 +0530 Subject: [PATCH 60/82] update .spelling --- docs/operations/python.md | 4 ++-- website/.spelling | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/operations/python.md b/docs/operations/python.md index ecbfafa4e7c7..4a4a065b0d24 100644 --- a/docs/operations/python.md +++ b/docs/operations/python.md @@ -31,7 +31,7 @@ MacOS comes with a version of Python2, which has been deprecated. Python2 and Py #### Install the official Python release * Browse to the [Python Downloads Page](https://www.python.org/downloads/) and download the latest version (3.x.x) -* Verify if python3 and pip3 (python package manager) is installed by issuing `python3` and `pip3 -V` commands. +* Verify if Python3 and Pip3 (python package manager) is installed by issuing `python3` and `pip3 -V` commands. #### Install with Homebrew Refer [Installing Python 3 on Mac OS X](https://docs.python-guide.org/starting/install3/osx/) @@ -51,4 +51,4 @@ Refer [Installing Python 3 on Mac OS X](https://docs.python-guide.org/starting/i - pip3 install psutil ### pathlib -- pip3 install pathlib \ No newline at end of file +- pip3 install pathlib diff --git a/website/.spelling b/website/.spelling index c2eb66104ee8..91fd5ff36234 100644 --- a/website/.spelling +++ b/website/.spelling @@ -2294,3 +2294,14 @@ Czechia Zeelund - ../docs/tutorials/docker.md nano +MacOS +python2 +python3 +Python2 +Python3 +pip3 +RHEL +psutil +pathlib +jvm +upto \ No newline at end of file From 43802354a6e4a92348f981af2d0d67c9cc06d94d Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 2 Dec 2022 01:32:01 +0530 Subject: [PATCH 61/82] Remove dependency on psutil and pathlib --- examples/bin/start-druid-main.py | 103 ++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 23 deletions(-) diff --git a/examples/bin/start-druid-main.py b/examples/bin/start-druid-main.py index d6e26e8aaabe..657d71f6fdaa 100644 --- a/examples/bin/start-druid-main.py +++ b/examples/bin/start-druid-main.py @@ -17,10 +17,10 @@ import sys import os -import psutil -import pathlib import multiprocessing import argparse +import subprocess +import platform BASE_CONFIG_PATH = "conf/druid/auto" @@ -168,18 +168,30 @@ def configure_parser(): return parser +def is_file(path): + return os.path.isfile(path) + + +def is_dir(path): + return os.path.isdir(path) + + +def resolve_path(path): + return os.path.abspath(path) + + def validate_common_jvm_args(config): - if pathlib.Path('{0}/_common/common.jvm.config'.format(config)).is_file() is False: + if is_file('{0}/_common/common.jvm.config'.format(config)) is False: raise ValueError('_common/common.jvm.config file is missing in the root config, ' 'check {0}/_common directory'.format(BASE_CONFIG_PATH)) def validate_common_directory(config): - if pathlib.Path('{0}/_common'.format(config)).is_dir() is False: + if is_dir('{0}/_common'.format(config)) is False: raise ValueError( '_common directory is missing in the root config, check {0}/_common directory'.format(BASE_CONFIG_PATH)) - if pathlib.Path('{0}/_common/common.runtime.properties'.format(config)).is_file() is False: + if is_file('{0}/_common/common.runtime.properties'.format(config)) is False: raise ValueError('_common/common.runtime.properties file is missing in the root config, ' 'check {0}/_common directory'.format(BASE_CONFIG_PATH)) @@ -196,8 +208,8 @@ def parse_arguments(args): if args.zk: zk = True if args.config is not None: - config = pathlib.Path(os.path.join(os.getcwd(), args.config)).resolve() - if config.is_dir() == False: + config = resolve_path(os.path.join(os.getcwd(), args.config)) + if is_dir(config) is False: raise ValueError('config {0} not found'.format(config)) if args.memory is not None: total_memory = args.memory @@ -225,7 +237,7 @@ def print_startup_config(service_list, config, zk): print_if_verbose('Starting {0}'.format(service_list)) print_if_verbose('Reading config from {0}'.format(config)) if zk: - zk_config = pathlib.Path('{0}/../conf/zk'.format(os.getcwd())).resolve() + zk_config = resolve_path('{0}/../conf/zk'.format(os.getcwd())) print_if_verbose('Starting zk, reading default config from {0}'.format(zk_config)) print_if_verbose('\n') @@ -234,7 +246,7 @@ def middle_manager_task_memory_params_present(config): java_opts_property_present = False worker_capacity_property_present = False - if pathlib.Path('{0}/middleManager/runtime.properties'.format(config)).is_file(): + if is_file('{0}/middleManager/runtime.properties'.format(config)): with open('{0}/middleManager/runtime.properties'.format(config)) as file: for line in file: if line.startswith(TASK_JAVA_OPTS_PROPERTY): @@ -267,7 +279,7 @@ def verify_service_config(service, config): raise ValueError('{0} missing in {1}/jvm.config'.format(params, service)) if service == MIDDLE_MANAGER: - if pathlib.Path('{0}/{1}/runtime.properties'.format(config, service)).is_file() is False: + if is_file('{0}/{1}/runtime.properties'.format(config, service)) is False: raise ValueError('{0}/runtime.properties file is missing in the root config'.format(service)) mm_task_java_opts_property, mm_task_worker_capacity_prop = middle_manager_task_memory_params_present(config) @@ -290,7 +302,7 @@ def should_compute_memory(config, total_memory, service_list): jvm_config_count = 0 for service in service_list: - if pathlib.Path('{0}/{1}/jvm.config'.format(config, service)).is_file(): + if is_file('{0}/{1}/jvm.config'.format(config, service)): jvm_config_count += 1 mm_task_property_present = False @@ -304,8 +316,8 @@ def should_compute_memory(config, total_memory, service_list): raise ValueError("one of indexer or middleManager can run") if total_memory != "": raise ValueError( - "If service list includes indexer, jvm.config should be specified for " - "each service and memory argument shouldn't be specified") + "If service list includes indexer, jvm.config should be specified for " + "each service and memory argument shouldn't be specified") if jvm_config_count != len(service_list): raise ValueError("If service list includes indexer, jvm.config should be specified for each service") for service in service_list: @@ -324,9 +336,9 @@ def should_compute_memory(config, total_memory, service_list): if INDEXER in service_list or jvm_config_count > 0 or mm_task_property_present: if total_memory != "": raise ValueError( - "If jvm.config for services and/or middleManager configs " - "(druid.worker.capacity, druid.indexer.runner.javaOptsArray) is present, " - "memory argument shouldn't be specified") + "If jvm.config for services and/or middleManager configs " + "(druid.worker.capacity, druid.indexer.runner.javaOptsArray) is present, " + "memory argument shouldn't be specified") if jvm_config_count == 0: raise ValueError("middleManger configs (druid.indexer.runner.javaOptsArray or druid.worker.capacity) " "is present in middleManager/runtime.properties, " @@ -341,15 +353,56 @@ def should_compute_memory(config, total_memory, service_list): return jvm_config_count == 0 and mm_task_property_present is False +def estimate_memory_linux(): + mems = {} + + def get_procfs_path(): + return sys.modules['psutil'].PROCFS_PATH + + def open_binary(fname): + FILE_READ_BUFFER_SIZE = 32 * 1024 + return open(fname, "rb", buffering=FILE_READ_BUFFER_SIZE) + + with open_binary('%s/meminfo' % get_procfs_path()) as f: + for line in f: + fields = line.split() + mems[fields[0]] = int(fields[1]) * 1024 + + return mems[b'MemTotal:'] + + +def estimate_memory_osx(): + p1 = subprocess.Popen(['sysctl', '-a'], stdout=subprocess.PIPE) + p2 = subprocess.check_output(['grep', 'hw.memsize'], stdin=p1.stdout) + p2 = p2.decode('utf-8') + fields = p2.split(':') + + mem = int(fields[1]) / (1024 * 1024) + + return mem + + def compute_system_memory(): - system_memory = psutil.virtual_memory().total # mem in bytes - memory_for_druid = int(system_memory / (1024 * 1024)) - return memory_for_druid + operating_system = platform.system() + print_if_verbose('operating system is {0}'.format(operating_system)) + + system_memory = None + + try: + if operating_system == 'Darwin': + system_memory = estimate_memory_osx() + elif operating_system == 'Linux': + system_memory = estimate_memory_linux() + except (Exception) as error: + print(error) + raise ValueError('Please specify memory argument') + + return system_memory def convert_total_memory_string(memory): try: - if memory == "": + if memory == '': computed_memory = compute_system_memory() return computed_memory elif memory.endswith(MEM_MB_SUFFIX): @@ -417,7 +470,8 @@ def build_memory_config(service, allocated_memory): if service == TASKS: memory_type, task_count, task_memory = compute_tasks_memory(allocated_memory) java_opts_array = build_mm_task_java_opts_array(memory_type) - return ['-D{0}={1}'.format(TASK_WORKER_CAPACITY_PROPERTY, task_count), java_opts_array], task_memory * task_count + return ['-D{0}={1}'.format(TASK_WORKER_CAPACITY_PROPERTY, task_count), + java_opts_array], task_memory * task_count else: heap_memory = HEAP_TO_TOTAL_MEM_RATIO.get(service) * allocated_memory direct_memory = int(allocated_memory - heap_memory) @@ -479,10 +533,12 @@ def distribute_memory(services, total_memory): return service_memory_config + def append_command(commands, command): commands.append('--command') commands.append(command) + def build_supervise_script_arguments(service_list, service_memory_config, config, zk): commands = [] commands.append('supervise') @@ -514,7 +570,8 @@ def build_supervise_script_arguments(service_list, service_memory_config, config '{0}{1} bin/run-druid {1} {2} \'{3}\' \'{4} {5}\'' .format(prefix, service, config, jvm_args, task_count, task_memory)) else: - append_command(commands, '{0}{1} bin/run-druid {1} {2} \'{3}\''.format(prefix, service, config, jvm_args)) + append_command(commands, + '{0}{1} bin/run-druid {1} {2} \'{3}\''.format(prefix, service, config, jvm_args)) print_if_verbose('Supervise script args:') for item in commands: @@ -538,7 +595,7 @@ def main(): os.chdir(os.path.dirname(sys.argv[0])) if config == "": - config = pathlib.Path('{0}/../{1}'.format(os.getcwd(), BASE_CONFIG_PATH)).resolve() + config = resolve_path('{0}/../{1}'.format(os.getcwd(), BASE_CONFIG_PATH)) validate_common_directory(config) From 77febf8a3e85276f18247cdb6e94314cd6f40ec7 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 2 Dec 2022 01:36:41 +0530 Subject: [PATCH 62/82] update docs --- docs/operations/python.md | 10 +--------- docs/operations/single-server.md | 6 +++--- docs/tutorials/index.md | 4 ++-- examples/bin/{start-druid-auto => start-druid} | 0 4 files changed, 6 insertions(+), 14 deletions(-) rename examples/bin/{start-druid-auto => start-druid} (100%) mode change 100644 => 100755 diff --git a/docs/operations/python.md b/docs/operations/python.md index 4a4a065b0d24..14a0f5e7bcd8 100644 --- a/docs/operations/python.md +++ b/docs/operations/python.md @@ -22,7 +22,7 @@ title: "Python Installation" ~ under the License. --> -Apache Druid startup script requires a python interpreter. This page has steps to install python interpreter and required libraries. +Apache Druid startup script requires a python interpreter. ## Python interpreter installation instructions @@ -44,11 +44,3 @@ Refer [Installing Python 3 on Mac OS X](https://docs.python-guide.org/starting/i #### RHEL - `sudo yum install -y epel-release` - `sudo yum install -y python3-pip` - -## Required libraries - -### psutil -- pip3 install psutil - -### pathlib -- pip3 install pathlib diff --git a/docs/operations/single-server.md b/docs/operations/single-server.md index 8ea6e8b5ae55..01a3f5c7d3f6 100644 --- a/docs/operations/single-server.md +++ b/docs/operations/single-server.md @@ -27,7 +27,7 @@ Druid includes a set of reference configurations and launch scripts for single-m These configuration bundles are located in `conf/druid/single-server/`. The `auto` configuration sizes runtime parameters based on available processors and memory. Other configurations include hard-coded runtime parameters for various server sizes. Most users should stick with `auto`. Refer below [Druid auto start](#Druid-auto-start) -- `auto` (run script: `bin/start-druid-auto`) +- `auto` (run script: `bin/start-druid`) - `nano-quickstart` (run script: `bin/start-nano-quickstart`) - `micro-quickstart` (run script: `bin/start-micro-quickstart`) - `small` (run script: `bin/start-single-server-small`) @@ -51,12 +51,12 @@ While example configurations are provided for very large single machines, at hig Setting up a new Druid cluster can sometimes be complicated as there are several runtime properties required by each service. Each process must also be given the appropriate jvm arguments so that the system can perform optimally. -`start-druid-auto` is a generic launch script capable of starting any set of Druid services on a server. +`start-druid` is a generic launch script capable of starting any set of Druid services on a server. It accepts optional arguments such as list of services, total memory and a config directory to override default jvm arguments and service-specific runtime properties. All other reference configurations (e.g. `micro`, `small`, `xlarge`) can be obtained by passing the correct memory to this script. Druid services will use all processors and upto 80% memory on the system. -For details about possible arguments, run `bin/start-druid-auto --help`. +For details about possible arguments, run `bin/start-druid --help`. The corresponding launch scripts (e.g. `start-micro-quickstart`) are now deprecated. diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index ed749c7efcfe..9f31be652660 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -80,13 +80,13 @@ You can view that setting and others in the configuration files in the `conf/dru From the apache-druid-{{DRUIDVERSION}} package root, run the following command: ```bash -./bin/start-druid-auto +./bin/start-druid ``` This brings up instances of ZooKeeper and the Druid services: ```bash -$ ./bin/start-druid-auto +$ ./bin/start-druid [Tue Nov 29 16:31:06 2022] Starting Apache Druid. [Tue Nov 29 16:31:06 2022] Open http://localhost:8888/ in your browser to access the web console. [Tue Nov 29 16:31:06 2022] Or, if you have enabled TLS, use https on port 9088. diff --git a/examples/bin/start-druid-auto b/examples/bin/start-druid old mode 100644 new mode 100755 similarity index 100% rename from examples/bin/start-druid-auto rename to examples/bin/start-druid From 1c8bd246ccfb9170589f6839137601468a30523a Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 2 Dec 2022 01:51:03 +0530 Subject: [PATCH 63/82] Update get_physical_memory method --- examples/bin/start-druid-main.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/examples/bin/start-druid-main.py b/examples/bin/start-druid-main.py index 657d71f6fdaa..35bbd4b427d4 100644 --- a/examples/bin/start-druid-main.py +++ b/examples/bin/start-druid-main.py @@ -353,7 +353,7 @@ def should_compute_memory(config, total_memory, service_list): return jvm_config_count == 0 and mm_task_property_present is False -def estimate_memory_linux(): +def get_physical_memory_linux(): mems = {} def get_procfs_path(): @@ -371,18 +371,18 @@ def open_binary(fname): return mems[b'MemTotal:'] -def estimate_memory_osx(): +def get_physical_memory_osx(): p1 = subprocess.Popen(['sysctl', '-a'], stdout=subprocess.PIPE) p2 = subprocess.check_output(['grep', 'hw.memsize'], stdin=p1.stdout) p2 = p2.decode('utf-8') fields = p2.split(':') - mem = int(fields[1]) / (1024 * 1024) + mem = int(int(fields[1]) / (1024 * 1024)) return mem -def compute_system_memory(): +def get_physical_memory(): operating_system = platform.system() print_if_verbose('operating system is {0}'.format(operating_system)) @@ -390,12 +390,11 @@ def compute_system_memory(): try: if operating_system == 'Darwin': - system_memory = estimate_memory_osx() + system_memory = get_physical_memory_osx() elif operating_system == 'Linux': - system_memory = estimate_memory_linux() - except (Exception) as error: - print(error) - raise ValueError('Please specify memory argument') + system_memory = get_physical_memory_linux() + except Exception: + pass return system_memory @@ -403,14 +402,20 @@ def compute_system_memory(): def convert_total_memory_string(memory): try: if memory == '': - computed_memory = compute_system_memory() - return computed_memory + physical_memory = get_physical_memory() + + if physical_memory == None: + raise ValueError('Please specify memory argument') + + return physical_memory elif memory.endswith(MEM_MB_SUFFIX): return int(memory[:-1]) elif memory.endswith(MEM_GB_SUFFIX): return 1024 * int(memory[:-1]) else: raise ValueError('Incorrect format for memory argument, expected format is ') + except ValueError as e: + raise e except Exception: raise ValueError('Incorrect format for memory argument, expected format is ') From 9e495b36e016b8088bc6f9e71ee87cf6ae6d024a Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 2 Dec 2022 01:53:42 +0530 Subject: [PATCH 64/82] Update help docs --- examples/bin/start-druid-main.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/bin/start-druid-main.py b/examples/bin/start-druid-main.py index 35bbd4b427d4..e47b92c77b4b 100644 --- a/examples/bin/start-druid-main.py +++ b/examples/bin/start-druid-main.py @@ -105,31 +105,31 @@ def print_if_verbose(message): def configure_parser(): parser = argparse.ArgumentParser( - prog='start-druid-auto', + prog='start-druid', formatter_class=argparse.RawTextHelpFormatter, epilog= """ sample usage: - start-druid-auto + start-druid Start up all the services (including zk). - start-druid-auto -m=100g + start-druid -m=100g Start up all the services (including zk) using a total memory of 100GB. - start-druid-auto -m=100g --compute + start-druid -m=100g --compute Compute memory distribution and validate arguments. - start-druid-auto -m=100g -s=broker,router + start-druid -m=100g -s=broker,router Starts a broker and a router, using a total memory of 100GB. - start-druid-auto -m=100g --s=broker,router \\ + start-druid -m=100g --s=broker,router \\ -c=conf/druid/single-server/custom Starts a broker and a router, using a total memory of 100GB. Reads configs for each service (jvm.config, runtime.properties) from respective folders inside the given root config path. - start-druid-auto -s=broker,router \\ + start-druid -s=broker,router \\ -c=conf/druid/single-server/custom Starts a broker and a router service, reading service configs from the given root directory. Calculates memory requirements for each service, if required, using upto 80% of the total system memory. - start-druid-auto -m=100g \\ + start-druid -m=100g \\ -s=broker,router \\ -c=conf/druid/single-server/custom \\ --zk From e04af93a7964b9e864e4a18f550edd24e365a5dc Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 2 Dec 2022 01:58:05 +0530 Subject: [PATCH 65/82] update docs --- docs/tutorials/index.md | 4 ++-- docs/tutorials/tutorial-batch-hadoop.md | 2 +- website/.spelling | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index 9f31be652660..5672fe4e3e6d 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -104,7 +104,7 @@ the Druid root directory, apache-druid-{{DRUIDVERSION}}. Each service writes to At any time, you can revert Druid to its original, post-installation state by deleting the entire `var` directory. You may want to do this, for example, between Druid tutorials or after experimentation, to start with a fresh instance. -To stop Druid at any time, use CTRL+C in the terminal. This exits the `bin/start-druid-auto` script and terminates all Druid processes. +To stop Druid at any time, use CTRL+C in the terminal. This exits the `bin/start-druid` script and terminates all Druid processes. ## Open the web console @@ -223,4 +223,4 @@ See the following topics for more information: * [Tutorial: Load stream data from Apache Kafka](./tutorial-kafka.md) to load streaming data from a Kafka topic. * [Extensions](../development/extensions.md) for details on Druid extensions. -Remember that after stopping Druid services, you can start clean next time by deleting the `var` directory from the Druid root directory and running the `bin/start-druid-auto` script again. You may want to do this before using other data ingestion tutorials, since they use the same Wikipedia datasource. +Remember that after stopping Druid services, you can start clean next time by deleting the `var` directory from the Druid root directory and running the `bin/start-druid` script again. You may want to do this before using other data ingestion tutorials, since they use the same Wikipedia datasource. diff --git a/docs/tutorials/tutorial-batch-hadoop.md b/docs/tutorials/tutorial-batch-hadoop.md index e8ea8a19052f..ee0861b210d2 100644 --- a/docs/tutorials/tutorial-batch-hadoop.md +++ b/docs/tutorials/tutorial-batch-hadoop.md @@ -196,7 +196,7 @@ druid.indexer.logs.directory=/druid/indexing-logs Once the Hadoop .xml files have been copied to the Druid cluster and the segment/log storage configuration has been updated to use HDFS, the Druid cluster needs to be restarted for the new configurations to take effect. -If the cluster is still running, CTRL-C to terminate the `bin/start-druid-auto` script, and re-run it to bring the Druid services back up. +If the cluster is still running, CTRL-C to terminate the `bin/start-druid` script, and re-run it to bring the Druid services back up. ## Load batch data diff --git a/website/.spelling b/website/.spelling index 91fd5ff36234..fc88d5f84af2 100644 --- a/website/.spelling +++ b/website/.spelling @@ -2304,4 +2304,4 @@ RHEL psutil pathlib jvm -upto \ No newline at end of file +upto From ca9d5ea5cb98882a8f3462d1c8c78813df52a783 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 2 Dec 2022 12:41:58 +0530 Subject: [PATCH 66/82] update method to get physical memory on python --- examples/bin/start-druid-main.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/examples/bin/start-druid-main.py b/examples/bin/start-druid-main.py index e47b92c77b4b..51ca126713f5 100644 --- a/examples/bin/start-druid-main.py +++ b/examples/bin/start-druid-main.py @@ -354,21 +354,9 @@ def should_compute_memory(config, total_memory, service_list): def get_physical_memory_linux(): - mems = {} - - def get_procfs_path(): - return sys.modules['psutil'].PROCFS_PATH - - def open_binary(fname): - FILE_READ_BUFFER_SIZE = 32 * 1024 - return open(fname, "rb", buffering=FILE_READ_BUFFER_SIZE) - - with open_binary('%s/meminfo' % get_procfs_path()) as f: - for line in f: - fields = line.split() - mems[fields[0]] = int(fields[1]) * 1024 - - return mems[b'MemTotal:'] + mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') + mem_mbs = int(mem_bytes / (1024 * 1024)) + return mem_mbs def get_physical_memory_osx(): @@ -377,9 +365,9 @@ def get_physical_memory_osx(): p2 = p2.decode('utf-8') fields = p2.split(':') - mem = int(int(fields[1]) / (1024 * 1024)) + mem_mbs = int(int(fields[1]) / (1024 * 1024)) - return mem + return mem_mbs def get_physical_memory(): From 01fb4b20573d6f2e75f7bfc382f5d73bf19c225d Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 2 Dec 2022 12:42:06 +0530 Subject: [PATCH 67/82] udpate spelling --- website/.spelling | 1 + 1 file changed, 1 insertion(+) diff --git a/website/.spelling b/website/.spelling index fc88d5f84af2..451797e6862b 100644 --- a/website/.spelling +++ b/website/.spelling @@ -2300,6 +2300,7 @@ python3 Python2 Python3 pip3 +Pip3 RHEL psutil pathlib From d0c3b21859a811a6da1d5b8f39c7bb29a14fae76 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 2 Dec 2022 12:49:09 +0530 Subject: [PATCH 68/82] update .spelling --- website/.spelling | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/website/.spelling b/website/.spelling index 451797e6862b..2637f62b1ca8 100644 --- a/website/.spelling +++ b/website/.spelling @@ -421,6 +421,10 @@ programmatically proto proxied proxyConfig +python2 +python3 +Python2 +Python3 QPS quantile quantiles @@ -2294,15 +2298,13 @@ Czechia Zeelund - ../docs/tutorials/docker.md nano +- ../docs/operations/python.md MacOS -python2 -python3 -Python2 -Python3 pip3 Pip3 RHEL psutil pathlib +- ../docs/operatons/single-server.md jvm upto From c516320d08d0ea8f5381fcdb4d055c88165569f1 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 2 Dec 2022 14:41:49 +0530 Subject: [PATCH 69/82] minor change --- examples/bin/start-druid-main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/bin/start-druid-main.py b/examples/bin/start-druid-main.py index 51ca126713f5..03d5c51de90e 100644 --- a/examples/bin/start-druid-main.py +++ b/examples/bin/start-druid-main.py @@ -392,7 +392,7 @@ def convert_total_memory_string(memory): if memory == '': physical_memory = get_physical_memory() - if physical_memory == None: + if physical_memory is None: raise ValueError('Please specify memory argument') return physical_memory From 9aeb55152dc42e224caa0bec1ff978fdf0149a3a Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 2 Dec 2022 15:29:22 +0530 Subject: [PATCH 70/82] Minor change --- examples/bin/start-druid-main.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/bin/start-druid-main.py b/examples/bin/start-druid-main.py index 03d5c51de90e..25d7e93afc59 100644 --- a/examples/bin/start-druid-main.py +++ b/examples/bin/start-druid-main.py @@ -333,7 +333,7 @@ def should_compute_memory(config, total_memory, service_list): # 3. jvm.config present for some but not all services # 4. jvm.config file is present for all services, but it doesn't contain required parameters # 5. lastly, if middleManager is to be started, and it is missing task memory properties - if INDEXER in service_list or jvm_config_count > 0 or mm_task_property_present: + if jvm_config_count > 0 or mm_task_property_present: if total_memory != "": raise ValueError( "If jvm.config for services and/or middleManager configs " @@ -348,9 +348,11 @@ def should_compute_memory(config, total_memory, service_list): for service in service_list: verify_service_config(service, config) + return False + # compute memory only when none of the specified services contains jvm.config, - # if middleManager is to be started it doesn't contain task memory properties - return jvm_config_count == 0 and mm_task_property_present is False + # if middleManager is to be started it shouldn't contain task memory properties + return True def get_physical_memory_linux(): From 71977d69e6cf57b6b13ab71b8caff0fddfb2e96a Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 6 Dec 2022 18:21:21 +0530 Subject: [PATCH 71/82] memory comptuation for indexer --- examples/bin/start-druid-main.py | 74 +++++++++++-------- examples/conf/druid/auto/indexer/jvm.config | 9 --- .../druid/auto/indexer/runtime.properties | 12 +-- 3 files changed, 49 insertions(+), 46 deletions(-) delete mode 100644 examples/conf/druid/auto/indexer/jvm.config diff --git a/examples/bin/start-druid-main.py b/examples/bin/start-druid-main.py index 25d7e93afc59..d1f4e6114918 100644 --- a/examples/bin/start-druid-main.py +++ b/examples/bin/start-druid-main.py @@ -74,7 +74,8 @@ COORDINATOR: 30, BROKER: 46, HISTORICAL: 80, - TASKS: 30 + TASKS: 30, + INDEXER: 32 } MINIMUM_MEMORY_MB = { @@ -83,7 +84,8 @@ TASKS: 1024, BROKER: 900, COORDINATOR: 256, - HISTORICAL: 900 + HISTORICAL: 900, + INDEXER: 1124 } HEAP_TO_TOTAL_MEM_RATIO = { @@ -92,7 +94,8 @@ COORDINATOR: 1, BROKER: 0.60, HISTORICAL: 0.40, - TASKS: 0.50 + TASKS: 0.50, + INDEXER: 0.50 } LOGGING_ENABLED = False @@ -225,6 +228,9 @@ def parse_arguments(args): service_list.append(service) + if INDEXER in services and MIDDLE_MANAGER in services: + raise ValueError('one of indexer and middleManager can run') + if len(service_list) == 0: # start all services service_list = DEFAULT_SERVICES @@ -242,12 +248,12 @@ def print_startup_config(service_list, config, zk): print_if_verbose('\n') -def middle_manager_task_memory_params_present(config): +def task_memory_params_present(config, service): java_opts_property_present = False worker_capacity_property_present = False - if is_file('{0}/middleManager/runtime.properties'.format(config)): - with open('{0}/middleManager/runtime.properties'.format(config)) as file: + if is_file('{0}/{1}/runtime.properties'.format(config, service)): + with open('{0}/{1}/runtime.properties'.format(config, service)) as file: for line in file: if line.startswith(TASK_JAVA_OPTS_PROPERTY): java_opts_property_present = True @@ -282,7 +288,7 @@ def verify_service_config(service, config): if is_file('{0}/{1}/runtime.properties'.format(config, service)) is False: raise ValueError('{0}/runtime.properties file is missing in the root config'.format(service)) - mm_task_java_opts_property, mm_task_worker_capacity_prop = middle_manager_task_memory_params_present(config) + mm_task_java_opts_prop, mm_task_worker_capacity_prop = task_memory_params_present(config, MIDDLE_MANAGER) if mm_task_java_opts_property is False: raise ValueError('{0} property missing in {1}/runtime.properties'.format(TASK_JAVA_OPTS_PROPERTY, service)) @@ -307,42 +313,36 @@ def should_compute_memory(config, total_memory, service_list): mm_task_property_present = False if MIDDLE_MANAGER in service_list: - mm_task_java_opts_property, mm_task_worker_capacity_prop = middle_manager_task_memory_params_present(config) - mm_task_property_present = mm_task_java_opts_property or mm_task_worker_capacity_prop + mm_task_java_opts_prop, mm_task_worker_capacity_prop = task_memory_params_present(config, MIDDLE_MANAGER) + mm_task_property_present = mm_task_java_opts_prop or mm_task_worker_capacity_prop - # if indexer has to be run, all the memory related parameters need to be specified + indexer_task_worker_capacity_prop = False if INDEXER in service_list: - if MIDDLE_MANAGER in service_list: - raise ValueError("one of indexer or middleManager can run") - if total_memory != "": - raise ValueError( - "If service list includes indexer, jvm.config should be specified for " - "each service and memory argument shouldn't be specified") - if jvm_config_count != len(service_list): - raise ValueError("If service list includes indexer, jvm.config should be specified for each service") - for service in service_list: - verify_service_config(service, config) - - return False + indexer_task_java_opts_prop, indexer_task_worker_capacity_prop = task_memory_params_present(config, INDEXER) # possible error states # 1. memory argument is specified, also jvm.config or middleManger/runtime.properties having # druid.indexer.runner.javaOptsArray or druid.worker.capacity parameters is present # 2. jvm.config is not present for any service, but middleManger/runtime.properties has # druid.indexer.runner.javaOptsArray or druid.worker.capacity parameters + # or indexer/runtime.properties has druid.worker.capacity # 3. jvm.config present for some but not all services # 4. jvm.config file is present for all services, but it doesn't contain required parameters # 5. lastly, if middleManager is to be started, and it is missing task memory properties - if jvm_config_count > 0 or mm_task_property_present: + if jvm_config_count > 0 or mm_task_property_present or indexer_task_worker_capacity_prop: if total_memory != "": raise ValueError( - "If jvm.config for services and/or middleManager configs " + "If jvm.config for services and/or middleManager/indexer configs " "(druid.worker.capacity, druid.indexer.runner.javaOptsArray) is present, " "memory argument shouldn't be specified") - if jvm_config_count == 0: + if jvm_config_count == 0 and mm_task_property_present: raise ValueError("middleManger configs (druid.indexer.runner.javaOptsArray or druid.worker.capacity) " "is present in middleManager/runtime.properties, " "add jvm.config for all other services") + if jvm_config_count == 0 and indexer_task_worker_capacity_prop: + raise ValueError("indexer configs (druid.worker.capacity) " + "is present in indexer/runtime.properties, " + "add jvm.config for all other services") if jvm_config_count != len(service_list): raise ValueError("jvm.config file should be present for all services or none") for service in service_list: @@ -352,6 +352,7 @@ def should_compute_memory(config, total_memory, service_list): # compute memory only when none of the specified services contains jvm.config, # if middleManager is to be started it shouldn't contain task memory properties + # if indexer is present it shouldn't contain task memory properties return True @@ -467,6 +468,14 @@ def build_memory_config(service, allocated_memory): java_opts_array = build_mm_task_java_opts_array(memory_type) return ['-D{0}={1}'.format(TASK_WORKER_CAPACITY_PROPERTY, task_count), java_opts_array], task_memory * task_count + elif service == INDEXER: + heap_memory = HEAP_TO_TOTAL_MEM_RATIO.get(service) * allocated_memory + direct_memory = int(allocated_memory - heap_memory) + heap_memory = int(heap_memory) + memory_type, task_count, task_memory = compute_tasks_memory(allocated_memory) + return ['-D{0}={1}'.format(TASK_WORKER_CAPACITY_PROPERTY, task_count), + '-Xms{0}m -Xmx{0}m -XX:MaxDirectMemorySize={1}m'.format(heap_memory, direct_memory)], \ + task_memory * task_count else: heap_memory = HEAP_TO_TOTAL_MEM_RATIO.get(service) * allocated_memory direct_memory = int(allocated_memory - heap_memory) @@ -547,13 +556,13 @@ def build_supervise_script_arguments(service_list, service_memory_config, config append_command(commands, "!p10 zk bin/run-zk conf") for service in service_list: - jvm_args = service_memory_config.get(service) + memory_config = service_memory_config.get(service) prefix = '' if service == MIDDLE_MANAGER: prefix = '!p90 ' - if jvm_args is None: + if memory_config is None: append_command(commands, '{0}{1} bin/run-druid {1} {2}'.format(prefix, service, config)) else: if service == MIDDLE_MANAGER: @@ -563,10 +572,17 @@ def build_supervise_script_arguments(service_list, service_memory_config, config append_command( commands, '{0}{1} bin/run-druid {1} {2} \'{3}\' \'{4} {5}\'' - .format(prefix, service, config, jvm_args, task_count, task_memory)) + .format(prefix, service, config, memory_config, task_count, task_memory)) + elif service == INDEXER: + task_count = memory_config[0] + jvm_args = memory_config[1] + append_command( + commands, + '{0}{1} bin/run-druid {1} {2} \'{3}\' \'{4}\'' + .format(prefix, service, config, jvm_args, task_count)) else: append_command(commands, - '{0}{1} bin/run-druid {1} {2} \'{3}\''.format(prefix, service, config, jvm_args)) + '{0}{1} bin/run-druid {1} {2} \'{3}\''.format(prefix, service, config, memory_config)) print_if_verbose('Supervise script args:') for item in commands: diff --git a/examples/conf/druid/auto/indexer/jvm.config b/examples/conf/druid/auto/indexer/jvm.config deleted file mode 100644 index 4611a65196a2..000000000000 --- a/examples/conf/druid/auto/indexer/jvm.config +++ /dev/null @@ -1,9 +0,0 @@ --server --Xms4g --Xmx4g --XX:MaxDirectMemorySize=4g --XX:+ExitOnOutOfMemoryError --Duser.timezone=UTC --Dfile.encoding=UTF-8 --Djava.io.tmpdir=var/tmp --Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/auto/indexer/runtime.properties b/examples/conf/druid/auto/indexer/runtime.properties index b36c9eb42872..5aef64535b67 100644 --- a/examples/conf/druid/auto/indexer/runtime.properties +++ b/examples/conf/druid/auto/indexer/runtime.properties @@ -20,19 +20,15 @@ druid.service=druid/indexer druid.plaintextPort=8091 -# Number of tasks per indexer -druid.worker.capacity=4 +# Number of tasks (druid.worker.capacity) is automatically +# determined based on available processor. # Task launch parameters druid.indexer.task.baseTaskDir=var/druid/task -# HTTP server threads -druid.server.http.numThreads=60 - # Processing threads and buffers on Indexer -druid.processing.numMergeBuffers=2 -druid.processing.buffer.sizeBytes=100MiB -druid.processing.numThreads=4 +# Determined automatically based on available memory. For details on how to manually set parameters: +# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers # Hadoop indexing druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp From d5d81f43391b0fb1be57ac9d9c1e0fe7dca2b3e9 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 8 Dec 2022 09:50:07 +0530 Subject: [PATCH 72/82] update start-druid --- examples/bin/start-druid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/bin/start-druid b/examples/bin/start-druid index c0447e6eb939..e098a04edc1c 100755 --- a/examples/bin/start-druid +++ b/examples/bin/start-druid @@ -28,5 +28,5 @@ elif [ -x "$(command -v python2)" ] then exec python2 "$WHEREAMI/start-druid-main.py" "$@" else - exec "$WHEREAMI/start-druid-main.py" "$@" + exec python "$WHEREAMI/start-druid-main.py" "$@" fi From 5942eff0215f55033b22f88408da88f081a3ab59 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 8 Dec 2022 10:11:39 +0530 Subject: [PATCH 73/82] Update python.md --- docs/operations/python.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/operations/python.md b/docs/operations/python.md index 14a0f5e7bcd8..3f6d03c69c3d 100644 --- a/docs/operations/python.md +++ b/docs/operations/python.md @@ -22,12 +22,21 @@ title: "Python Installation" ~ under the License. --> -Apache Druid startup script requires a python interpreter. +Apache Druid startup script requires Python2 or Python3 interpreter. +Since Python2 is deprecated, this document has instructions to install Python3 interpreter. -## Python interpreter installation instructions +## Python3 interpreter installation instructions + +### Linux + +#### Debian or Ubuntu + - `sudo apt update` + - `sudo apt install -y python3-pip` +#### RHEL + - `sudo yum install -y epel-release` + - `sudo yum install -y python3-pip` ### MacOS -MacOS comes with a version of Python2, which has been deprecated. Python2 and Python3 can coexist on the same machine without problems. #### Install the official Python release * Browse to the [Python Downloads Page](https://www.python.org/downloads/) and download the latest version (3.x.x) @@ -35,12 +44,3 @@ MacOS comes with a version of Python2, which has been deprecated. Python2 and Py #### Install with Homebrew Refer [Installing Python 3 on Mac OS X](https://docs.python-guide.org/starting/install3/osx/) - -### Linux - -#### Debian - - `sudo apt update` - - `sudo apt install -y python3-pip` -#### RHEL - - `sudo yum install -y epel-release` - - `sudo yum install -y python3-pip` From a4d14f762045c4127d11e92287e0e0f5a525d548 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 8 Dec 2022 10:25:29 +0530 Subject: [PATCH 74/82] Update single-server.md --- docs/operations/single-server.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/operations/single-server.md b/docs/operations/single-server.md index 01a3f5c7d3f6..11217c809841 100644 --- a/docs/operations/single-server.md +++ b/docs/operations/single-server.md @@ -49,19 +49,16 @@ While example configurations are provided for very large single machines, at hig ## Druid auto start -Setting up a new Druid cluster can sometimes be complicated as there are several runtime properties required by each service. Each process must also be given the appropriate jvm arguments so that the system can perform optimally. +Druid includes a launch script, `bin/start-druid` that automatically sets various memory-related parameters based on available processors and memory. It accepts optional arguments such as list of services, total memory and a config directory to override default jvm arguments and service-specific runtime properties. `start-druid` is a generic launch script capable of starting any set of Druid services on a server. It accepts optional arguments such as list of services, total memory and a config directory to override default jvm arguments and service-specific runtime properties. -All other reference configurations (e.g. `micro`, `small`, `xlarge`) can be obtained by passing the -correct memory to this script. -Druid services will use all processors and upto 80% memory on the system. +Druid services will use all processors and up to 80% memory on the system. For details about possible arguments, run `bin/start-druid --help`. The corresponding launch scripts (e.g. `start-micro-quickstart`) are now deprecated. - ## Single server reference configurations ### Nano-Quickstart: 1 CPU, 4GiB RAM From 41ad0bd2d8aa44e6829469f45d12474ce479fa10 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 8 Dec 2022 11:33:04 +0530 Subject: [PATCH 75/82] Update python.md --- docs/operations/python.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/operations/python.md b/docs/operations/python.md index 3f6d03c69c3d..4bcc07aee350 100644 --- a/docs/operations/python.md +++ b/docs/operations/python.md @@ -38,9 +38,12 @@ Since Python2 is deprecated, this document has instructions to install Python3 i ### MacOS +#### Install with Homebrew +Refer [Installing Python 3 on Mac OS X](https://docs.python-guide.org/starting/install3/osx/) + #### Install the official Python release * Browse to the [Python Downloads Page](https://www.python.org/downloads/) and download the latest version (3.x.x) -* Verify if Python3 and Pip3 (python package manager) is installed by issuing `python3` and `pip3 -V` commands. -#### Install with Homebrew -Refer [Installing Python 3 on Mac OS X](https://docs.python-guide.org/starting/install3/osx/) +Verify if Python3 is installed by issuing `python3` command. + + From 08507b4dabe2692eb7c2428fe5436872876a449b Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 8 Dec 2022 11:45:00 +0530 Subject: [PATCH 76/82] run python3 --version to check if python is installed --- docs/operations/python.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/python.md b/docs/operations/python.md index 4bcc07aee350..9f5b9c34909e 100644 --- a/docs/operations/python.md +++ b/docs/operations/python.md @@ -44,6 +44,6 @@ Refer [Installing Python 3 on Mac OS X](https://docs.python-guide.org/starting/i #### Install the official Python release * Browse to the [Python Downloads Page](https://www.python.org/downloads/) and download the latest version (3.x.x) -Verify if Python3 is installed by issuing `python3` command. +Verify if Python3 is installed by issuing `python3 --version` command. From 14b8679c05d8f34e083d5882fb742d571e30bd7d Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 9 Dec 2022 09:21:09 +0530 Subject: [PATCH 77/82] Update supervise script --- examples/bin/supervise | 2 +- web-console/src/variables.scss | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/bin/supervise b/examples/bin/supervise index 29aca9eeddb8..98dadbdb30a6 100755 --- a/examples/bin/supervise +++ b/examples/bin/supervise @@ -199,7 +199,7 @@ usage() unless GetOptions( 'command=s@' ); -usage() unless ((@{$opt{'command'}} || $opt{'conf'}) && $opt{'vardir'}); +usage() unless (($opt{'conf'} || @{$opt{'command'}}) && $opt{'vardir'}); my @config_lines; diff --git a/web-console/src/variables.scss b/web-console/src/variables.scss index 1dc4d0124cf3..fec32add0008 100644 --- a/web-console/src/variables.scss +++ b/web-console/src/variables.scss @@ -25,7 +25,7 @@ $view-control-bar-height: 30px; $standard-padding: 15px; $thin-padding: 10px; -// various style variables +// various style Variables $druid-brand: #2ceefb; $druid-brand2: #00b6bf; From 2837dd397650f99bc138b00835a055282e83207e Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 9 Dec 2022 09:57:04 +0530 Subject: [PATCH 78/82] start-druid: echo message if python not found --- examples/bin/start-druid | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/bin/start-druid b/examples/bin/start-druid index e098a04edc1c..2c36a6f8bca0 100755 --- a/examples/bin/start-druid +++ b/examples/bin/start-druid @@ -27,6 +27,10 @@ then elif [ -x "$(command -v python2)" ] then exec python2 "$WHEREAMI/start-druid-main.py" "$@" -else +elif [ -x "$(command -v python)" ] +then exec python "$WHEREAMI/start-druid-main.py" "$@" +else +then + echo "python interepreter not found" fi From 511583ef63ec5097d3913fa3e8f937d2cfdf7826 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 9 Dec 2022 10:51:02 +0530 Subject: [PATCH 79/82] update anchor text --- docs/operations/single-server.md | 2 +- docs/tutorials/index.md | 2 +- docs/tutorials/tutorial-batch-hadoop.md | 2 +- docs/tutorials/tutorial-kafka.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/operations/single-server.md b/docs/operations/single-server.md index 11217c809841..69565f77b6e2 100644 --- a/docs/operations/single-server.md +++ b/docs/operations/single-server.md @@ -26,7 +26,7 @@ title: "Single server deployment" Druid includes a set of reference configurations and launch scripts for single-machine deployments. These configuration bundles are located in `conf/druid/single-server/`. -The `auto` configuration sizes runtime parameters based on available processors and memory. Other configurations include hard-coded runtime parameters for various server sizes. Most users should stick with `auto`. Refer below [Druid auto start](#Druid-auto-start) +The `auto` configuration sizes runtime parameters based on available processors and memory. Other configurations include hard-coded runtime parameters for various server sizes. Most users should stick with `auto`. Refer below [Druid auto start](#druid-auto-start) - `auto` (run script: `bin/start-druid`) - `nano-quickstart` (run script: `bin/start-nano-quickstart`) - `micro-quickstart` (run script: `bin/start-micro-quickstart`) diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index 5672fe4e3e6d..d854a691e6b9 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -36,7 +36,7 @@ Druid supports a variety of ingestion options. Once you're done with this tutori You can follow these steps on a relatively modest machine, such as a workstation or virtual server with 16 GiB of RAM. -Druid comes equipped with launch scripts that can be used to start all processes on a single server. Here, we will use [`auto`](../operations/single-server.md#Druid-auto-start), which automatically sets various runtime properties based on available processors and memory. +Druid comes equipped with launch scripts that can be used to start all processes on a single server. Here, we will use [`auto`](../operations/single-server.md#druid-auto-start), which automatically sets various runtime properties based on available processors and memory. In addition, Druid includes several [bundled non-automatic profiles](../operations/single-server.md) for a range of machine sizes. These range from nano (1 CPU, 4GiB RAM) to x-large (64 CPU, 512GiB RAM). We won't use those here, but for more information, see [Single server deployment](../operations/single-server.md). For additional information on deploying Druid services across clustered machines, see [Clustered deployment](./cluster.md). diff --git a/docs/tutorials/tutorial-batch-hadoop.md b/docs/tutorials/tutorial-batch-hadoop.md index ee0861b210d2..234e8426b064 100644 --- a/docs/tutorials/tutorial-batch-hadoop.md +++ b/docs/tutorials/tutorial-batch-hadoop.md @@ -28,7 +28,7 @@ This tutorial shows you how to load data files into Apache Druid using a remote For this tutorial, we'll assume that you've already completed the previous [batch ingestion tutorial](tutorial-batch.md) using Druid's native batch ingestion system and are using the -`auto` single-machine configuration as described in the [quickstart](../operations/single-server.md#Druid-auto-start). +`auto` single-machine configuration as described in the [quickstart](../operations/single-server.md#druid-auto-start). ## Install Docker diff --git a/docs/tutorials/tutorial-kafka.md b/docs/tutorials/tutorial-kafka.md index 73feeaf65901..0a47d3237fa6 100644 --- a/docs/tutorials/tutorial-kafka.md +++ b/docs/tutorials/tutorial-kafka.md @@ -30,7 +30,7 @@ The tutorial guides you through the steps to load sample nested clickstream data ## Prerequisites -Before you follow the steps in this tutorial, download Druid as described in the [quickstart](index.md) using the [auto](../operations/single-server.md#Druid-auto-start) single-machine configuration and have it running on your local machine. You don't need to have loaded any data. +Before you follow the steps in this tutorial, download Druid as described in the [quickstart](index.md) using the [auto](../operations/single-server.md#druid-auto-start) single-machine configuration and have it running on your local machine. You don't need to have loaded any data. ## Download and start Kafka From d569693175fbb9749f830cbab67f86e8cd2038a8 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 9 Dec 2022 10:58:30 +0530 Subject: [PATCH 80/82] minor change --- examples/bin/start-druid | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/bin/start-druid b/examples/bin/start-druid index 2c36a6f8bca0..81d8938adbb2 100755 --- a/examples/bin/start-druid +++ b/examples/bin/start-druid @@ -31,6 +31,5 @@ elif [ -x "$(command -v python)" ] then exec python "$WHEREAMI/start-druid-main.py" "$@" else -then echo "python interepreter not found" fi From d0e737461bc631bf60606185fdd99c72b54f3c5f Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 9 Dec 2022 13:19:19 +0530 Subject: [PATCH 81/82] Update condition in supervise script --- examples/bin/supervise | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/bin/supervise b/examples/bin/supervise index 98dadbdb30a6..ba336c31aa66 100755 --- a/examples/bin/supervise +++ b/examples/bin/supervise @@ -199,7 +199,7 @@ usage() unless GetOptions( 'command=s@' ); -usage() unless (($opt{'conf'} || @{$opt{'command'}}) && $opt{'vardir'}); +usage() unless (($opt{'command'} && @{$opt{'command'}}) || $opt{'conf'}) && $opt{'vardir'}; my @config_lines; From e3d36577561d721be4eac5c83af230fcbd134c29 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 9 Dec 2022 16:33:14 +0530 Subject: [PATCH 82/82] JVM not jvm in docs --- docs/operations/single-server.md | 4 ++-- website/.spelling | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/docs/operations/single-server.md b/docs/operations/single-server.md index 69565f77b6e2..48459a286042 100644 --- a/docs/operations/single-server.md +++ b/docs/operations/single-server.md @@ -49,10 +49,10 @@ While example configurations are provided for very large single machines, at hig ## Druid auto start -Druid includes a launch script, `bin/start-druid` that automatically sets various memory-related parameters based on available processors and memory. It accepts optional arguments such as list of services, total memory and a config directory to override default jvm arguments and service-specific runtime properties. +Druid includes a launch script, `bin/start-druid` that automatically sets various memory-related parameters based on available processors and memory. It accepts optional arguments such as list of services, total memory and a config directory to override default JVM arguments and service-specific runtime properties. `start-druid` is a generic launch script capable of starting any set of Druid services on a server. -It accepts optional arguments such as list of services, total memory and a config directory to override default jvm arguments and service-specific runtime properties. +It accepts optional arguments such as list of services, total memory and a config directory to override default JVM arguments and service-specific runtime properties. Druid services will use all processors and up to 80% memory on the system. For details about possible arguments, run `bin/start-druid --help`. diff --git a/website/.spelling b/website/.spelling index 65b51cde7873..e52ecebbc532 100644 --- a/website/.spelling +++ b/website/.spelling @@ -2302,13 +2302,8 @@ Czechia Zeelund - ../docs/tutorials/docker.md nano -- ../docs/operations/python.md + - ../docs/operations/python.md MacOS -pip3 -Pip3 RHEL psutil pathlib -- ../docs/operatons/single-server.md -jvm -upto