From 5515c71a9b626cc1d5e5cf255c36d6e00b1ff7e0 Mon Sep 17 00:00:00 2001 From: deardeng Date: Mon, 10 Mar 2025 20:16:20 +0800 Subject: [PATCH] [fix](table) Breaking up dynamic partition table scheduling consuming peak CPU consumption (#48577) --- .../main/java/org/apache/doris/common/Config.java | 8 ++++++++ .../doris/clone/DynamicPartitionScheduler.java | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index cda1b914041e9e..dbd4cda4a99321 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1324,6 +1324,14 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static long dynamic_partition_check_interval_seconds = 600; + /** + * When scheduling dynamic partition tables, + * the execution interval of each table to prevent excessive consumption of FE CPU at the same time + * default is 0 + */ + @ConfField(mutable = true, masterOnly = true) + public static long dynamic_partition_step_interval_ms = 0; + /** * If set to true, dynamic partition feature will open */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index e875bcf47d8d27..8e5f46573ee0ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -671,6 +671,20 @@ public void executeDynamicPartition(Collection> dynamicPartitio } cloudBatchAfterCreatePartitions(executeFirstTime, partsInfo, addPartitionClauses, db, olapTable, indexIds, tableName); + + // ATTN: Breaking up dynamic partition table scheduling, consuming peak CPU consumption + if (!executeFirstTime && !addPartitionClauses.isEmpty() + && Config.dynamic_partition_step_interval_ms > 0) { + long sleep = Config.dynamic_partition_step_interval_ms; + if (sleep > 1800 * 1000) { + LOG.warn("fe conf dynamic_partition_step_interval_ms bigger than 1800s, plz check it"); + } + try { + Thread.sleep(sleep); + } catch (InterruptedException e) { + LOG.warn("sleep err", e); + } + } } } }