From 638cda8022e8cf7636d07187a67b2f780a53021c Mon Sep 17 00:00:00 2001 From: Amatya Date: Thu, 15 Sep 2022 17:50:15 +0530 Subject: [PATCH 1/4] Make httpRemote the default task runner --- docs/configuration/index.md | 2 +- .../cluster/master/coordinator-overlord/runtime.properties | 2 +- .../single-server/large/coordinator-overlord/runtime.properties | 2 +- .../medium/coordinator-overlord/runtime.properties | 2 +- .../micro-quickstart/coordinator-overlord/runtime.properties | 2 +- .../nano-quickstart/coordinator-overlord/runtime.properties | 2 +- .../single-server/small/coordinator-overlord/runtime.properties | 2 +- .../xlarge/coordinator-overlord/runtime.properties | 2 +- services/src/main/java/org/apache/druid/cli/CliOverlord.java | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 49ed052fbcbc..e9f3fda2b9e2 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -1097,7 +1097,7 @@ These Overlord static configurations can be defined in the `overlord/runtime.pro |Property|Description|Default| |--------|-----------|-------| -|`druid.indexer.runner.type`|Choices "local" or "remote". Indicates whether tasks should be run locally or in a distributed environment. Experimental task runner "httpRemote" is also available which is same as "remote" but uses HTTP to interact with Middle Managers instead of ZooKeeper.|local| +|`druid.indexer.runner.type`|Indicates whether tasks should be run locally using "local" or in a distributed environment using "remote". The recommended option is "httpRemote", which is similar to "remote" but uses HTTP to interact with Middle Managers instead of ZooKeeper.|httpRemote| |`druid.indexer.storage.type`|Choices are "local" or "metadata". Indicates whether incoming tasks should be stored locally (in heap) or in metadata storage. "local" is mainly for internal testing while "metadata" is recommended in production because storing incoming tasks in metadata storage allows for tasks to be resumed if the Overlord should fail.|local| |`druid.indexer.storage.recentlyFinishedThreshold`|Duration of time to store task results. Default is 24 hours. If you have hundreds of tasks running in a day, consider increasing this threshold.|PT24H| |`druid.indexer.tasklock.forceTimeChunkLock`|_**Setting this to false is still experimental**_
If set, all tasks are enforced to use time chunk lock. If not set, each task automatically chooses a lock type to use. This configuration can be overwritten by setting `forceTimeChunkLock` in the [task context](../ingestion/tasks.md#context). See [Task Locking & Priority](../ingestion/tasks.md#context) for more details about locking in tasks.|true| diff --git a/examples/conf/druid/cluster/master/coordinator-overlord/runtime.properties b/examples/conf/druid/cluster/master/coordinator-overlord/runtime.properties index 00071a83d7dc..693c3a7b91d4 100644 --- a/examples/conf/druid/cluster/master/coordinator-overlord/runtime.properties +++ b/examples/conf/druid/cluster/master/coordinator-overlord/runtime.properties @@ -30,5 +30,5 @@ druid.coordinator.asOverlord.overlordService=druid/overlord druid.indexer.queue.startDelay=PT5S -druid.indexer.runner.type=remote +druid.indexer.runner.type=httpRemote druid.indexer.storage.type=metadata diff --git a/examples/conf/druid/single-server/large/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/large/coordinator-overlord/runtime.properties index 00071a83d7dc..693c3a7b91d4 100644 --- a/examples/conf/druid/single-server/large/coordinator-overlord/runtime.properties +++ b/examples/conf/druid/single-server/large/coordinator-overlord/runtime.properties @@ -30,5 +30,5 @@ druid.coordinator.asOverlord.overlordService=druid/overlord druid.indexer.queue.startDelay=PT5S -druid.indexer.runner.type=remote +druid.indexer.runner.type=httpRemote druid.indexer.storage.type=metadata diff --git a/examples/conf/druid/single-server/medium/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/medium/coordinator-overlord/runtime.properties index 00071a83d7dc..693c3a7b91d4 100644 --- a/examples/conf/druid/single-server/medium/coordinator-overlord/runtime.properties +++ b/examples/conf/druid/single-server/medium/coordinator-overlord/runtime.properties @@ -30,5 +30,5 @@ druid.coordinator.asOverlord.overlordService=druid/overlord druid.indexer.queue.startDelay=PT5S -druid.indexer.runner.type=remote +druid.indexer.runner.type=httpRemote druid.indexer.storage.type=metadata diff --git a/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/runtime.properties index 00071a83d7dc..693c3a7b91d4 100644 --- a/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/runtime.properties +++ b/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/runtime.properties @@ -30,5 +30,5 @@ druid.coordinator.asOverlord.overlordService=druid/overlord druid.indexer.queue.startDelay=PT5S -druid.indexer.runner.type=remote +druid.indexer.runner.type=httpRemote druid.indexer.storage.type=metadata diff --git a/examples/conf/druid/single-server/nano-quickstart/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/nano-quickstart/coordinator-overlord/runtime.properties index 00071a83d7dc..693c3a7b91d4 100644 --- a/examples/conf/druid/single-server/nano-quickstart/coordinator-overlord/runtime.properties +++ b/examples/conf/druid/single-server/nano-quickstart/coordinator-overlord/runtime.properties @@ -30,5 +30,5 @@ druid.coordinator.asOverlord.overlordService=druid/overlord druid.indexer.queue.startDelay=PT5S -druid.indexer.runner.type=remote +druid.indexer.runner.type=httpRemote druid.indexer.storage.type=metadata diff --git a/examples/conf/druid/single-server/small/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/small/coordinator-overlord/runtime.properties index 00071a83d7dc..693c3a7b91d4 100644 --- a/examples/conf/druid/single-server/small/coordinator-overlord/runtime.properties +++ b/examples/conf/druid/single-server/small/coordinator-overlord/runtime.properties @@ -30,5 +30,5 @@ druid.coordinator.asOverlord.overlordService=druid/overlord druid.indexer.queue.startDelay=PT5S -druid.indexer.runner.type=remote +druid.indexer.runner.type=httpRemote druid.indexer.storage.type=metadata diff --git a/examples/conf/druid/single-server/xlarge/coordinator-overlord/runtime.properties b/examples/conf/druid/single-server/xlarge/coordinator-overlord/runtime.properties index 00071a83d7dc..693c3a7b91d4 100644 --- a/examples/conf/druid/single-server/xlarge/coordinator-overlord/runtime.properties +++ b/examples/conf/druid/single-server/xlarge/coordinator-overlord/runtime.properties @@ -30,5 +30,5 @@ druid.coordinator.asOverlord.overlordService=druid/overlord druid.indexer.queue.startDelay=PT5S -druid.indexer.runner.type=remote +druid.indexer.runner.type=httpRemote druid.indexer.storage.type=metadata diff --git a/services/src/main/java/org/apache/druid/cli/CliOverlord.java b/services/src/main/java/org/apache/druid/cli/CliOverlord.java index 3a45975bf9d7..cb9b68788b4f 100644 --- a/services/src/main/java/org/apache/druid/cli/CliOverlord.java +++ b/services/src/main/java/org/apache/druid/cli/CliOverlord.java @@ -300,7 +300,7 @@ private void configureRunners(Binder binder) binder, "druid.indexer.runner.type", Key.get(TaskRunnerFactory.class), - Key.get(ForkingTaskRunnerFactory.class) + Key.get(HttpRemoteTaskRunnerFactory.class) ); final MapBinder biddy = PolyBind.optionBinder( binder, From 4c28fd18eff27c36b27789be017b4ecc29fb0cbb Mon Sep 17 00:00:00 2001 From: Amatya Date: Thu, 15 Sep 2022 18:08:28 +0530 Subject: [PATCH 2/4] Make http based inventory and segment management default --- dev/intellij-setup.md | 4 ++-- docs/configuration/index.md | 6 +++--- .../druid/client/FilteredServerInventoryViewProvider.java | 2 +- .../apache/druid/client/ServerInventoryViewProvider.java | 2 +- .../druid/server/coordinator/DruidCoordinatorConfig.java | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dev/intellij-setup.md b/dev/intellij-setup.md index c2536e7b1316..5d6cc281db95 100644 --- a/dev/intellij-setup.md +++ b/dev/intellij-setup.md @@ -93,7 +93,7 @@ You can configure application definitions in XML for import into IntelliJ. Below