From 4d835df0e7566eb388bef55d9cd0eb6e270fefcf Mon Sep 17 00:00:00 2001 From: YangJiaqi Date: Sat, 10 Aug 2024 12:34:39 +0800 Subject: [PATCH 1/9] fix raft maxEntriesSize config not take effect --- .../hugegraph/store/options/HgStoreEngineOptions.java | 2 +- .../org/apache/hugegraph/store/node/AppConfig.java | 2 ++ .../hugegraph/store/node/grpc/HgStoreNodeService.java | 11 ++++++----- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/HgStoreEngineOptions.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/HgStoreEngineOptions.java index 52888c1d18..6a194f2de5 100644 --- a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/HgStoreEngineOptions.java +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/HgStoreEngineOptions.java @@ -98,7 +98,7 @@ public static class RaftOptions { /** * The maximum number of entries in AppendEntriesRequest */ - private final int maxEntriesSize = 256; + private int maxEntriesSize = 256; /** * Raft集群发生数据积压后，限速等待时间 单位毫秒 **/ diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java index 6c561f4c02..c656912235 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java @@ -182,6 +182,8 @@ public class Raft { private int maxSegmentFileSize; @Value("${raft.maxReplicatorInflightMsgs:256}") private int maxReplicatorInflightMsgs; + @Value("${raft.maxEntriesSize:256}") + private int maxEntriesSize; } diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java index 4492f37b2c..00b2b92a0c 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java @@ -100,6 +100,7 @@ public void init() { .isUseRocksDBSegmentLogStorage()); setMaxSegmentFileSize(appConfig.getRaft().getMaxSegmentFileSize()); setMaxReplicatorInflightMsgs(appConfig.getRaft().getMaxReplicatorInflightMsgs()); + setMaxEntriesSize(appConfig.getRaft().getMaxEntriesSize()); }}); setFakePdOptions(new FakePdOptions() {{ setStoreList(appConfig.getFakePdConfig().getStoreList()); @@ -125,9 +126,9 @@ public List getGraphLeaderPartitionIds(String graphName) { } /** - * 添加raft 任务，转发数据给raft + * 添加 raft 任务，转发数据给 raft * - * @return true 表示数据已被提交，false表示未提交，用于单副本入库减少批次拆分 + * @return true 表示数据已被提交，false 表示未提交，用于单副本入库减少批次拆分 */ public void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, @@ -147,7 +148,7 @@ void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, req.writeTo(output); output.checkNoSpaceLeft(); output.flush(); - // 传送给raft + // 传送给 raft storeEngine.addRaftTask(graphName, partitionId, RaftOperation.create(methodId, buffer, req), closure); @@ -159,7 +160,7 @@ void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, } /** - * 来自日志的任务，一般是follower 或者 日志回滚的任务 + * 来自日志的任务，一般是 follower 或者 日志回滚的任务 */ @Override public boolean invoke(int partId, byte[] request, RaftClosure response) throws @@ -190,7 +191,7 @@ public boolean invoke(int partId, byte[] request, RaftClosure response) throws } /** - * 处理raft传送过来的数据 + * 处理 raft 传送过来的数据 */ @Override public boolean invoke(int partId, byte methodId, Object req, RaftClosure response) throws From 4127b57ac675f4025e2323699f72435e3c8bcd0d Mon Sep 17 00:00:00 2001 From: YangJiaqi Date: Sat, 10 Aug 2024 12:41:12 +0800 Subject: [PATCH 2/9] roll back format --- .../hugegraph/store/node/grpc/HgStoreNodeService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java index 00b2b92a0c..373b2951f9 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java @@ -126,7 +126,7 @@ public List getGraphLeaderPartitionIds(String graphName) { } /** - * 添加 raft 任务，转发数据给 raft + * 添加raft 任务，转发数据给raft * * @return true 表示数据已被提交，false 表示未提交，用于单副本入库减少批次拆分 */ @@ -148,7 +148,7 @@ void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, req.writeTo(output); output.checkNoSpaceLeft(); output.flush(); - // 传送给 raft + // 传送给raft storeEngine.addRaftTask(graphName, partitionId, RaftOperation.create(methodId, buffer, req), closure); @@ -160,7 +160,7 @@ void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, } /** - * 来自日志的任务，一般是 follower 或者 日志回滚的任务 + * 来自日志的任务，一般是follower 或者 日志回滚的任务 */ @Override public boolean invoke(int partId, byte[] request, RaftClosure response) throws @@ -191,7 +191,7 @@ public boolean invoke(int partId, byte[] request, RaftClosure response) throws } /** - * 处理 raft 传送过来的数据 + * 处理raft传送过来的数据 */ @Override public boolean invoke(int partId, byte methodId, Object req, RaftClosure response) throws From f69af8067843fc5b1a2b0b31ffbcf180f8817d3c Mon Sep 17 00:00:00 2001 From: YangJiaqi Date: Sat, 10 Aug 2024 12:42:53 +0800 Subject: [PATCH 3/9] roll back format --- .../hugegraph/store/node/grpc/HgStoreNodeService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java index 373b2951f9..00b2b92a0c 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java @@ -126,7 +126,7 @@ public List getGraphLeaderPartitionIds(String graphName) { } /** - * 添加raft 任务，转发数据给raft + * 添加 raft 任务，转发数据给 raft * * @return true 表示数据已被提交，false 表示未提交，用于单副本入库减少批次拆分 */ @@ -148,7 +148,7 @@ void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, req.writeTo(output); output.checkNoSpaceLeft(); output.flush(); - // 传送给raft + // 传送给 raft storeEngine.addRaftTask(graphName, partitionId, RaftOperation.create(methodId, buffer, req), closure); @@ -160,7 +160,7 @@ void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, } /** - * 来自日志的任务，一般是follower 或者 日志回滚的任务 + * 来自日志的任务，一般是 follower 或者 日志回滚的任务 */ @Override public boolean invoke(int partId, byte[] request, RaftClosure response) throws @@ -191,7 +191,7 @@ public boolean invoke(int partId, byte[] request, RaftClosure response) throws } /** - * 处理raft传送过来的数据 + * 处理 raft 传送过来的数据 */ @Override public boolean invoke(int partId, byte methodId, Object req, RaftClosure response) throws From 64f6d9a5e45c5fcb4904a1b39b49e18cbcf8e14e Mon Sep 17 00:00:00 2001 From: YangJiaqi Date: Sat, 10 Aug 2024 12:34:39 +0800 Subject: [PATCH 4/9] fix raft maxEntriesSize config not take effect --- .../hugegraph/store/options/HgStoreEngineOptions.java | 2 +- .../org/apache/hugegraph/store/node/AppConfig.java | 2 ++ .../hugegraph/store/node/grpc/HgStoreNodeService.java | 11 ++++++----- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/HgStoreEngineOptions.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/HgStoreEngineOptions.java index 3b3ff9bc78..18d145fb40 100644 --- a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/HgStoreEngineOptions.java +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/HgStoreEngineOptions.java @@ -98,7 +98,7 @@ public static class RaftOptions { /** * The maximum number of entries in AppendEntriesRequest */ - private final int maxEntriesSize = 256; + private int maxEntriesSize = 256; /** * Raft cluster data backlog occurs, rate limiting wait time in milliseconds. **/ diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java index 6c561f4c02..c656912235 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java @@ -182,6 +182,8 @@ public class Raft { private int maxSegmentFileSize; @Value("${raft.maxReplicatorInflightMsgs:256}") private int maxReplicatorInflightMsgs; + @Value("${raft.maxEntriesSize:256}") + private int maxEntriesSize; } diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java index 4492f37b2c..00b2b92a0c 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java @@ -100,6 +100,7 @@ public void init() { .isUseRocksDBSegmentLogStorage()); setMaxSegmentFileSize(appConfig.getRaft().getMaxSegmentFileSize()); setMaxReplicatorInflightMsgs(appConfig.getRaft().getMaxReplicatorInflightMsgs()); + setMaxEntriesSize(appConfig.getRaft().getMaxEntriesSize()); }}); setFakePdOptions(new FakePdOptions() {{ setStoreList(appConfig.getFakePdConfig().getStoreList()); @@ -125,9 +126,9 @@ public List getGraphLeaderPartitionIds(String graphName) { } /** - * 添加raft 任务，转发数据给raft + * 添加 raft 任务，转发数据给 raft * - * @return true 表示数据已被提交，false表示未提交，用于单副本入库减少批次拆分 + * @return true 表示数据已被提交，false 表示未提交，用于单副本入库减少批次拆分 */ public void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, @@ -147,7 +148,7 @@ void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, req.writeTo(output); output.checkNoSpaceLeft(); output.flush(); - // 传送给raft + // 传送给 raft storeEngine.addRaftTask(graphName, partitionId, RaftOperation.create(methodId, buffer, req), closure); @@ -159,7 +160,7 @@ void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, } /** - * 来自日志的任务，一般是follower 或者 日志回滚的任务 + * 来自日志的任务，一般是 follower 或者 日志回滚的任务 */ @Override public boolean invoke(int partId, byte[] request, RaftClosure response) throws @@ -190,7 +191,7 @@ public boolean invoke(int partId, byte[] request, RaftClosure response) throws } /** - * 处理raft传送过来的数据 + * 处理 raft 传送过来的数据 */ @Override public boolean invoke(int partId, byte methodId, Object req, RaftClosure response) throws From 969108b6b8f37d1495d7fe9656f3e9aee4c0754b Mon Sep 17 00:00:00 2001 From: YangJiaqi Date: Sat, 10 Aug 2024 12:41:12 +0800 Subject: [PATCH 5/9] roll back format --- .../hugegraph/store/node/grpc/HgStoreNodeService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java index 00b2b92a0c..373b2951f9 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java @@ -126,7 +126,7 @@ public List getGraphLeaderPartitionIds(String graphName) { } /** - * 添加 raft 任务，转发数据给 raft + * 添加raft 任务，转发数据给raft * * @return true 表示数据已被提交，false 表示未提交，用于单副本入库减少批次拆分 */ @@ -148,7 +148,7 @@ void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, req.writeTo(output); output.checkNoSpaceLeft(); output.flush(); - // 传送给 raft + // 传送给raft storeEngine.addRaftTask(graphName, partitionId, RaftOperation.create(methodId, buffer, req), closure); @@ -160,7 +160,7 @@ void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, } /** - * 来自日志的任务，一般是 follower 或者 日志回滚的任务 + * 来自日志的任务，一般是follower 或者 日志回滚的任务 */ @Override public boolean invoke(int partId, byte[] request, RaftClosure response) throws @@ -191,7 +191,7 @@ public boolean invoke(int partId, byte[] request, RaftClosure response) throws } /** - * 处理 raft 传送过来的数据 + * 处理raft传送过来的数据 */ @Override public boolean invoke(int partId, byte methodId, Object req, RaftClosure response) throws From c5f5aedacc126c55f96c3e274abee93ef00455a5 Mon Sep 17 00:00:00 2001 From: YangJiaqi Date: Sat, 10 Aug 2024 12:42:53 +0800 Subject: [PATCH 6/9] roll back format --- .../hugegraph/store/node/grpc/HgStoreNodeService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java index 373b2951f9..00b2b92a0c 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java @@ -126,7 +126,7 @@ public List getGraphLeaderPartitionIds(String graphName) { } /** - * 添加raft 任务，转发数据给raft + * 添加 raft 任务，转发数据给 raft * * @return true 表示数据已被提交，false 表示未提交，用于单副本入库减少批次拆分 */ @@ -148,7 +148,7 @@ void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, req.writeTo(output); output.checkNoSpaceLeft(); output.flush(); - // 传送给raft + // 传送给 raft storeEngine.addRaftTask(graphName, partitionId, RaftOperation.create(methodId, buffer, req), closure); @@ -160,7 +160,7 @@ void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, } /** - * 来自日志的任务，一般是follower 或者 日志回滚的任务 + * 来自日志的任务，一般是 follower 或者 日志回滚的任务 */ @Override public boolean invoke(int partId, byte[] request, RaftClosure response) throws @@ -191,7 +191,7 @@ public boolean invoke(int partId, byte[] request, RaftClosure response) throws } /** - * 处理raft传送过来的数据 + * 处理 raft 传送过来的数据 */ @Override public boolean invoke(int partId, byte methodId, Object req, RaftClosure response) throws From 83ad40bac715ff1a8bfba0ab3e6618d958947f25 Mon Sep 17 00:00:00 2001 From: YangJiaqi Date: Sat, 10 Aug 2024 12:55:44 +0800 Subject: [PATCH 7/9] translate CJK comments to English --- .../store/node/grpc/HgStoreNodeService.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java index 00b2b92a0c..bd433e383b 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java @@ -126,9 +126,9 @@ public List getGraphLeaderPartitionIds(String graphName) { } /** - * 添加 raft 任务，转发数据给 raft + * Add raft task, forward data to raft * - * @return true 表示数据已被提交，false 表示未提交，用于单副本入库减少批次拆分 + * @return true means the data has been submitted, false means not submitted, used to reduce batch splitting for single-replica storage */ public void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, @@ -141,14 +141,14 @@ void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, } // try { - // 序列化， + // Serialization final byte[] buffer = new byte[req.getSerializedSize() + 1]; final CodedOutputStream output = CodedOutputStream.newInstance(buffer); output.write(methodId); req.writeTo(output); output.checkNoSpaceLeft(); output.flush(); - // 传送给 raft + //Add raft task storeEngine.addRaftTask(graphName, partitionId, RaftOperation.create(methodId, buffer, req), closure); @@ -160,7 +160,7 @@ void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, } /** - * 来自日志的任务，一般是 follower 或者 日志回滚的任务 + * Tasks from logs, generally tasks from followers or log rollbacks */ @Override public boolean invoke(int partId, byte[] request, RaftClosure response) throws @@ -191,7 +191,8 @@ public boolean invoke(int partId, byte[] request, RaftClosure response) throws } /** - * 处理 raft 传送过来的数据 + * Process the data sent by raft + * */ @Override public boolean invoke(int partId, byte methodId, Object req, RaftClosure response) throws From 3234d9c6b1c13cb5ffdc8b490bdfd4013ddce054 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sat, 10 Aug 2024 18:02:43 +0800 Subject: [PATCH 8/9] Update .asf.yaml --- .asf.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.asf.yaml b/.asf.yaml index 6c78530425..af53e6cf04 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -28,7 +28,8 @@ github: del_branch_on_merge: true #labels: enabled_merge_buttons: - merge: false + # TODO: disable it after common merged + merge: true rebase: true squash: true protected_branches: From bf06270e6cb70c40476fd9e8f3b41352754af4c0 Mon Sep 17 00:00:00 2001 From: YangJiaqi Date: Sun, 11 Aug 2024 11:51:55 +0800 Subject: [PATCH 9/9] reformat --- .../apache/hugegraph/store/node/grpc/HgStoreNodeService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java index bd433e383b..6c88555160 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java @@ -148,7 +148,7 @@ void addRaftTask(byte methodId, String graphName, Integer partitionId, Req req, req.writeTo(output); output.checkNoSpaceLeft(); output.flush(); - //Add raft task + // Add raft task storeEngine.addRaftTask(graphName, partitionId, RaftOperation.create(methodId, buffer, req), closure); @@ -192,7 +192,6 @@ public boolean invoke(int partId, byte[] request, RaftClosure response) throws /** * Process the data sent by raft - * */ @Override public boolean invoke(int partId, byte methodId, Object req, RaftClosure response) throws