From 9c9fda1b0e96aca30d4725d7112add075ccb0fe8 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 14 Dec 2022 16:25:33 +0530 Subject: [PATCH 1/3] update task memory computation --- examples/bin/start-druid-main.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/examples/bin/start-druid-main.py b/examples/bin/start-druid-main.py index 844933b830f6..cda5117a2610 100644 --- a/examples/bin/start-druid-main.py +++ b/examples/bin/start-druid-main.py @@ -37,9 +37,11 @@ TASK_WORKER_CAPACITY_PROPERTY = "druid.worker.capacity" TASK_COUNT = "task-count" TASK_MEM_TYPE_LOW = "low" +TASK_MEM_TYPE_LOW = "medium" TASK_MEM_TYPE_HIGH = "high" TASK_MEM_MAP = { TASK_MEM_TYPE_LOW: ["-Xms256m", "-Xmx256m", "-XX:MaxDirectMemorySize=256m"], + TASK_MEM_TYPE_MEDIUM: ["-Xms512m", "-Xmx512m", "-XX:MaxDirectMemorySize=512m"], TASK_MEM_TYPE_HIGH: ["-Xms1g", "-Xmx1g", "-XX:MaxDirectMemorySize=1g"] } @@ -431,10 +433,7 @@ def check_memory_constraint(total_memory, services): if total_memory < required_memory: raise ValueError('Minimum memory required for starting services is {0}m'.format(required_memory)) - if total_memory >= 2 * lower_bound_memory: - return int(total_memory / 2) - else: - return lower_bound_memory + return int(total_memory * 0.8) def build_mm_task_java_opts_array(memory_type): @@ -457,6 +456,10 @@ def compute_tasks_memory(allocated_memory): task_count = int(allocated_memory / 2048) memory_type = TASK_MEM_TYPE_HIGH task_memory_mb = 2048 + elif allocated_memory >= 2048: + task_count = int(allocated_memory / 1024) + memory_type = TASK_MEM_TYPE_MEDIUM + task_memory = 1024 else: task_count = 2 memory_type = TASK_MEM_TYPE_LOW From 5191d972b8df40d6b84dd873d71c0d01ed2bcac8 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 14 Dec 2022 17:40:19 +0530 Subject: [PATCH 2/3] Minor changes --- examples/bin/start-druid-main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/bin/start-druid-main.py b/examples/bin/start-druid-main.py index cda5117a2610..7f11b2e5cf07 100644 --- a/examples/bin/start-druid-main.py +++ b/examples/bin/start-druid-main.py @@ -37,7 +37,7 @@ TASK_WORKER_CAPACITY_PROPERTY = "druid.worker.capacity" TASK_COUNT = "task-count" TASK_MEM_TYPE_LOW = "low" -TASK_MEM_TYPE_LOW = "medium" +TASK_MEM_TYPE_MEDIUM = "medium" TASK_MEM_TYPE_HIGH = "high" TASK_MEM_MAP = { TASK_MEM_TYPE_LOW: ["-Xms256m", "-Xmx256m", "-XX:MaxDirectMemorySize=256m"], @@ -459,7 +459,7 @@ def compute_tasks_memory(allocated_memory): elif allocated_memory >= 2048: task_count = int(allocated_memory / 1024) memory_type = TASK_MEM_TYPE_MEDIUM - task_memory = 1024 + task_memory_mb = 1024 else: task_count = 2 memory_type = TASK_MEM_TYPE_LOW From a4608d8cd749b3f854d195cc691765430a5dfaf7 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 14 Dec 2022 18:09:43 +0530 Subject: [PATCH 3/3] Direct mem for router --- examples/bin/start-druid-main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/bin/start-druid-main.py b/examples/bin/start-druid-main.py index 7f11b2e5cf07..5b7e293ccc25 100644 --- a/examples/bin/start-druid-main.py +++ b/examples/bin/start-druid-main.py @@ -84,7 +84,7 @@ MINIMUM_MEMORY_MB = { MIDDLE_MANAGER: 64, - ROUTER: 128, + ROUTER: 256, TASKS: 1024, BROKER: 900, COORDINATOR: 256, @@ -486,6 +486,8 @@ def build_memory_config(service, allocated_memory): else: heap_memory = HEAP_TO_TOTAL_MEM_RATIO.get(service) * allocated_memory direct_memory = int(allocated_memory - heap_memory) + if service == ROUTER: + direct_memory = 128 heap_memory = int(heap_memory) if direct_memory == 0: