From bbd48f1447a35422ebd1c1e6ce6049e5dbf0277b Mon Sep 17 00:00:00 2001 From: deardeng Date: Mon, 3 Mar 2025 21:12:34 +0800 Subject: [PATCH 1/2] [fix](table) Breaking up dynamic partition table scheduling consuming peak CPU consumption --- .../main/java/org/apache/doris/common/Config.java | 8 ++++++++ .../doris/clone/DynamicPartitionScheduler.java | 13 +++++++++++++ 2 files changed, 21 insertions(+) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 2b69b18bec5d9c..8d20b2b1168899 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1345,6 +1345,14 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static long dynamic_partition_check_interval_seconds = 600; + /** + * When scheduling dynamic partition tables, + * the execution interval of each table to prevent excessive consumption of FE CPU at the same time + * default is 0 + */ + @ConfField(mutable = true, masterOnly = true) + public static long dynamic_partition_step_interval_ms = 0; + /** * If set to true, dynamic partition feature will open */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index dcd3631b117f7b..e3221c64fcebbe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -672,6 +672,19 @@ public void executeDynamicPartition(Collection> dynamicPartitio } cloudBatchAfterCreatePartitions(executeFirstTime, partsInfo, addPartitionClauses, db, olapTable, indexIds, tableName); + + // ATTN: Breaking up dynamic partition table scheduling, consuming peak CPU consumption + if (!executeFirstTime && Config.dynamic_partition_step_interval_ms > 0) { + long sleep = Config.dynamic_partition_step_interval_ms; + if (sleep > 1800 * 1000) { + LOG.warn("fe conf dynamic_partition_step_interval_ms bigger than 1800s, plz check it"); + } + try { + Thread.sleep(sleep); + } catch (InterruptedException e) { + LOG.warn("sleep err", e); + } + } } } } From 0fa89d60c2d29739e651c97242657dc6114542a5 Mon Sep 17 00:00:00 2001 From: deardeng Date: Tue, 4 Mar 2025 11:27:57 +0800 Subject: [PATCH 2/2] fix --- .../java/org/apache/doris/clone/DynamicPartitionScheduler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index e3221c64fcebbe..d40cc702eb3946 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -674,7 +674,8 @@ public void executeDynamicPartition(Collection> dynamicPartitio addPartitionClauses, db, olapTable, indexIds, tableName); // ATTN: Breaking up dynamic partition table scheduling, consuming peak CPU consumption - if (!executeFirstTime && Config.dynamic_partition_step_interval_ms > 0) { + if (!executeFirstTime && !addPartitionClauses.isEmpty() + && Config.dynamic_partition_step_interval_ms > 0) { long sleep = Config.dynamic_partition_step_interval_ms; if (sleep > 1800 * 1000) { LOG.warn("fe conf dynamic_partition_step_interval_ms bigger than 1800s, plz check it");