From c860d2d17237cd29d4d1580f34c5c36da0484bfd Mon Sep 17 00:00:00 2001 From: guanzhaoyun Date: Mon, 23 Oct 2023 15:40:33 +0800 Subject: [PATCH 1/7] [fix](meta) add sync new image timeout (#25764) Add timeout config for sync --- .../src/main/java/org/apache/doris/common/Config.java | 9 +++++++++ .../src/main/java/org/apache/doris/catalog/Env.java | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index fef83a5f62a521..e942963dc7fcff 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2259,4 +2259,13 @@ public class Config extends ConfigBase { }) public static boolean ignore_unknown_metadata_module = false; + @ConfField(mutable = true, masterOnly = true, description = { + "FE扩容时，从主节点同步元数据的timeout时间，根据${meta_dir}/image文件夹下面的image文件大小和节点间的网络环境调整，" + + "单位为毫秒，默认值30000", + "The timeout for new FE Follower/Observer synchronizing metadata from the FE Master, " + + "adjust by the size of image file in the ${meta_dir}/image and the network environment between nodes. " + + "The default values is 3000 ms." + }) + public static int sync_new_image_timeout = 30000; + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 8192713d7bdaab..ea3e2f9f01ec39 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -1726,7 +1726,7 @@ private void getNewImage(HostInfo helperNode) throws IOException { String url = "http://" + hostPort + "/image?version=" + version; String filename = Storage.IMAGE + "." + version; File dir = new File(this.imageDir); - MetaHelper.getRemoteFile(url, HTTP_TIMEOUT_SECOND * 1000, MetaHelper.getFile(filename, dir)); + MetaHelper.getRemoteFile(url, Config.sync_new_image_timeout, MetaHelper.getFile(filename, dir)); MetaHelper.complete(filename, dir); } else { LOG.warn("get an image with a lower version, localImageVersion: {}, got version: {}", From ee3ba7114a92dd9b1f74e5a0a5486ab9de1772d3 Mon Sep 17 00:00:00 2001 From: guanzhaoyun Date: Mon, 23 Oct 2023 19:06:57 +0800 Subject: [PATCH 2/7] [fix](meta) change config name (#25764) Add timeout config for sync --- .../src/main/java/org/apache/doris/common/Config.java | 6 +++--- fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index e942963dc7fcff..4ab927e0b84b9f 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2261,11 +2261,11 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true, description = { "FE扩容时，从主节点同步元数据的timeout时间，根据${meta_dir}/image文件夹下面的image文件大小和节点间的网络环境调整，" + - "单位为毫秒，默认值30000", + "单位为秒，默认值30", "The timeout for new FE Follower/Observer synchronizing metadata from the FE Master, " + "adjust by the size of image file in the ${meta_dir}/image and the network environment between nodes. " + - "The default values is 3000 ms." + "The default values is 30s." }) - public static int sync_new_image_timeout = 30000; + public static int sync_new_image_timeout_s = 30000; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index ea3e2f9f01ec39..9eabb287ce78d5 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -1726,7 +1726,7 @@ private void getNewImage(HostInfo helperNode) throws IOException { String url = "http://" + hostPort + "/image?version=" + version; String filename = Storage.IMAGE + "." + version; File dir = new File(this.imageDir); - MetaHelper.getRemoteFile(url, Config.sync_new_image_timeout, MetaHelper.getFile(filename, dir)); + MetaHelper.getRemoteFile(url, Config.sync_new_image_timeout_s * 1000, MetaHelper.getFile(filename, dir)); MetaHelper.complete(filename, dir); } else { LOG.warn("get an image with a lower version, localImageVersion: {}, got version: {}", From 58eab3b687dbdfdc3182274ce7ad9968de73d1ae Mon Sep 17 00:00:00 2001 From: guanzhaoyun Date: Mon, 23 Oct 2023 19:34:51 +0800 Subject: [PATCH 3/7] [fix](meta) change config name (#25764) Add timeout config for sync --- .../src/main/java/org/apache/doris/common/Config.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 4ab927e0b84b9f..af524ad3f19139 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2261,11 +2261,11 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true, description = { "FE扩容时，从主节点同步元数据的timeout时间，根据${meta_dir}/image文件夹下面的image文件大小和节点间的网络环境调整，" + - "单位为秒，默认值30", + "单位为秒，默认值300", "The timeout for new FE Follower/Observer synchronizing metadata from the FE Master, " + "adjust by the size of image file in the ${meta_dir}/image and the network environment between nodes. " + - "The default values is 30s." + "The default values is 300s." }) - public static int sync_new_image_timeout_s = 30000; + public static int sync_new_image_timeout_s = 300; } From da024b352bbfb9690ee5c52d2119edc6f3bf9132 Mon Sep 17 00:00:00 2001 From: guanzhaoyun Date: Mon, 23 Oct 2023 19:51:01 +0800 Subject: [PATCH 4/7] [fix](meta) fix checkstyle problems (#25764) Add timeout config for sync --- .../src/main/java/org/apache/doris/common/Config.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index af524ad3f19139..1926126835d061 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2260,11 +2260,11 @@ public class Config extends ConfigBase { public static boolean ignore_unknown_metadata_module = false; @ConfField(mutable = true, masterOnly = true, description = { - "FE扩容时，从主节点同步元数据的timeout时间，根据${meta_dir}/image文件夹下面的image文件大小和节点间的网络环境调整，" + - "单位为秒，默认值300", - "The timeout for new FE Follower/Observer synchronizing metadata from the FE Master, " + - "adjust by the size of image file in the ${meta_dir}/image and the network environment between nodes. " + - "The default values is 300s." + "FE扩容时，从主节点同步元数据的timeout时间，根据${meta_dir}/image文件夹下面的image文件大小和节点间的网络环境调整，" + + "单位为秒，默认值300", + "The timeout for new FE Follower/Observer synchronizing metadata from the FE Master, " + + "adjust by the size of image file in the ${meta_dir}/image and the network environment between nodes. " + + "The default values is 300s." }) public static int sync_new_image_timeout_s = 300; From e38d5b42721107a09d2dbf4de37a1da626e8935b Mon Sep 17 00:00:00 2001 From: guanzhaoyun Date: Tue, 24 Oct 2023 15:50:31 +0800 Subject: [PATCH 5/7] [fix](meta) change config name (#25764) --- fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 2 +- fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 1926126835d061..bbb064339f7293 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2266,6 +2266,6 @@ public class Config extends ConfigBase { + "adjust by the size of image file in the ${meta_dir}/image and the network environment between nodes. " + "The default values is 300s." }) - public static int sync_new_image_timeout_s = 300; + public static int sync_image_timeout_second = 300; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 9eabb287ce78d5..37fb8f7d7101ec 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -1726,7 +1726,7 @@ private void getNewImage(HostInfo helperNode) throws IOException { String url = "http://" + hostPort + "/image?version=" + version; String filename = Storage.IMAGE + "." + version; File dir = new File(this.imageDir); - MetaHelper.getRemoteFile(url, Config.sync_new_image_timeout_s * 1000, MetaHelper.getFile(filename, dir)); + MetaHelper.getRemoteFile(url, Config.sync_image_timeout_second * 1000, MetaHelper.getFile(filename, dir)); MetaHelper.complete(filename, dir); } else { LOG.warn("get an image with a lower version, localImageVersion: {}, got version: {}", From 1a078bedd65a8750fdec495a91520e129df4dd96 Mon Sep 17 00:00:00 2001 From: guanzhaoyun Date: Tue, 24 Oct 2023 18:59:11 +0800 Subject: [PATCH 6/7] [fix](meta) add sync new image timeout (#25764) fix checkstyle --- fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 37fb8f7d7101ec..fd74cd08dd7ecb 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -1726,7 +1726,8 @@ private void getNewImage(HostInfo helperNode) throws IOException { String url = "http://" + hostPort + "/image?version=" + version; String filename = Storage.IMAGE + "." + version; File dir = new File(this.imageDir); - MetaHelper.getRemoteFile(url, Config.sync_image_timeout_second * 1000, MetaHelper.getFile(filename, dir)); + MetaHelper.getRemoteFile(url, Config.sync_image_timeout_second * 1000, + MetaHelper.getFile(filename, dir)); MetaHelper.complete(filename, dir); } else { LOG.warn("get an image with a lower version, localImageVersion: {}, got version: {}", From 5f1ec3264d60a5d4fcc96db96b8337a66c5c0000 Mon Sep 17 00:00:00 2001 From: guanzhaoyun Date: Tue, 24 Oct 2023 21:03:16 +0800 Subject: [PATCH 7/7] [fix](meta) add sync new image timeout (#25764) fix checkstyle --- fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index fd74cd08dd7ecb..8e46daaf936b95 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -1727,7 +1727,7 @@ private void getNewImage(HostInfo helperNode) throws IOException { String filename = Storage.IMAGE + "." + version; File dir = new File(this.imageDir); MetaHelper.getRemoteFile(url, Config.sync_image_timeout_second * 1000, - MetaHelper.getFile(filename, dir)); + MetaHelper.getFile(filename, dir)); MetaHelper.complete(filename, dir); } else { LOG.warn("get an image with a lower version, localImageVersion: {}, got version: {}",