From 904202b07f13c1bee18489251ccec33797d704e0 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Thu, 14 Jan 2021 15:13:45 +0800 Subject: [PATCH 1/3] fix gremlin job result size gt cassandra limit and lt hugegraph limit Change-Id: Iade74c66259214a6895ae369374c7243761cb0e0 --- .../com/baidu/hugegraph/task/TaskCallable.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskCallable.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskCallable.java index ce3d26c644..8cf0a8a9e2 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskCallable.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskCallable.java @@ -35,8 +35,9 @@ public abstract class TaskCallable implements Callable { private static final Logger LOG = Log.logger(HugeTask.class); - private static final String ERROR_MAX_LEN = "Failed to commit changes: " + - "The max length of bytes is"; + private static final String ERROR_COMMIT = "Failed to commit changes: "; + private static final String ERROR_MAX_LEN = "The max length of bytes is"; + private static final String ERROR_TOO_LARGE_BATCH = "Batch too large"; private HugeTask task = null; private HugeGraph graph = null; @@ -98,7 +99,15 @@ protected void save() { */ LOG.error("Failed to save task with error \"{}\": {}", e, task.asMap(false)); - if (e.getMessage().contains(ERROR_MAX_LEN)) { + String message = e.getMessage(); + /* + * ERROR_TOO_LARGE_BATCH occurs when using cassandra store + * and result size is in + * [batch_size_fail_threshold_in_kb, TASK_RESULT_SIZE_LIMIT) + */ + if (message.contains(ERROR_COMMIT) && + (message.contains(ERROR_MAX_LEN) || + message.contains(ERROR_TOO_LARGE_BATCH))) { task.failSave(e); this.graph().taskScheduler().save(task); return; From 7a0c7bfc6cc32cf49eda15eeb4818a0a7bcc1f61 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Thu, 14 Jan 2021 17:02:25 +0800 Subject: [PATCH 2/3] improve Change-Id: Iead22b315ac6959d8a9492ed36fc97c9b5d88345 --- .../baidu/hugegraph/task/TaskCallable.java | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskCallable.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskCallable.java index 8cf0a8a9e2..11f75a6df8 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskCallable.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskCallable.java @@ -20,6 +20,7 @@ package com.baidu.hugegraph.task; import java.util.Date; +import java.util.Set; import java.util.concurrent.Callable; import org.apache.tinkerpop.gremlin.structure.Transaction; @@ -30,14 +31,29 @@ import com.baidu.hugegraph.HugeGraphParams; import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.Log; +import com.google.common.collect.ImmutableSet; public abstract class TaskCallable implements Callable { private static final Logger LOG = Log.logger(HugeTask.class); private static final String ERROR_COMMIT = "Failed to commit changes: "; - private static final String ERROR_MAX_LEN = "The max length of bytes is"; - private static final String ERROR_TOO_LARGE_BATCH = "Batch too large"; + private static final Set ERROR_MESSAGES = ImmutableSet.of( + /* + * "The max length of bytes is" exception message occurs when + * task input size exceeds TASK_INPUT_SIZE_LIMIT or task result size + * exceeds TASK_RESULT_SIZE_LIMIT + */ + "The max length of bytes is", + /* + * "Batch too large" exception message occurs when using + * cassandra store and task input size is in + * [batch_size_fail_threshold_in_kb, TASK_INPUT_SIZE_LIMIT) or + * task result size is in + * [batch_size_fail_threshold_in_kb, TASK_RESULT_SIZE_LIMIT) + */ + "Batch too large" + ); private HugeTask task = null; private HugeGraph graph = null; @@ -100,14 +116,7 @@ protected void save() { LOG.error("Failed to save task with error \"{}\": {}", e, task.asMap(false)); String message = e.getMessage(); - /* - * ERROR_TOO_LARGE_BATCH occurs when using cassandra store - * and result size is in - * [batch_size_fail_threshold_in_kb, TASK_RESULT_SIZE_LIMIT) - */ - if (message.contains(ERROR_COMMIT) && - (message.contains(ERROR_MAX_LEN) || - message.contains(ERROR_TOO_LARGE_BATCH))) { + if (message.contains(ERROR_COMMIT) && needSaveWithEx(message)) { task.failSave(e); this.graph().taskScheduler().save(task); return; @@ -117,6 +126,15 @@ protected void save() { } } + private static boolean needSaveWithEx(String message) { + for (String error : ERROR_MESSAGES) { + if (message.contains(error)) { + return true; + } + } + return false; + } + protected void graph(HugeGraph graph) { this.graph = graph; } From f81a3cf579f26b2e7f3d1a03362ec1109f5cbb49 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Thu, 14 Jan 2021 17:18:21 +0800 Subject: [PATCH 3/3] improve Change-Id: Ic2a246facffca683052bb56bf76ad21b5413d94a --- .../com/baidu/hugegraph/task/TaskCallable.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskCallable.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskCallable.java index 11f75a6df8..bb21ac8554 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskCallable.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskCallable.java @@ -126,15 +126,6 @@ protected void save() { } } - private static boolean needSaveWithEx(String message) { - for (String error : ERROR_MESSAGES) { - if (message.contains(error)) { - return true; - } - } - return false; - } - protected void graph(HugeGraph graph) { this.graph = graph; } @@ -165,6 +156,15 @@ public static TaskCallable fromClass(String className) { } } + private static boolean needSaveWithEx(String message) { + for (String error : ERROR_MESSAGES) { + if (message.contains(error)) { + return true; + } + } + return false; + } + public static TaskCallable empty(Exception e) { return new TaskCallable() { @Override