From e9383a7054661596d17ed698b1fafb2ce588b8b2 Mon Sep 17 00:00:00 2001 From: liningrui Date: Fri, 2 Apr 2021 13:14:41 +0800 Subject: [PATCH 1/2] Let TaskScheduler pause when under loading mode Change-Id: I593ee46a2a14af18f9105d84249da68592556dde --- .../com/baidu/hugegraph/config/ServerOptions.java | 2 +- hugegraph-core/pom.xml | 2 +- .../com/baidu/hugegraph/StandardHugeGraph.java | 9 +++++++++ .../java/com/baidu/hugegraph/task/TaskManager.java | 14 +++++++++++--- .../assembly/static/conf/rest-server.properties | 4 ++++ 5 files changed, 26 insertions(+), 5 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java index b15b0030ca..f017d965a5 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java @@ -173,7 +173,7 @@ public static synchronized ServerOptions instance() { "batch.max_write_threads", "The maximum threads for batch writing, " + "if the value is 0, the actual value will be set to " + - "batch.max_write_ratio * total-rest-threads.", + "batch.max_write_ratio * restserver.max_worker_threads.", nonNegativeInt(), 0); diff --git a/hugegraph-core/pom.xml b/hugegraph-core/pom.xml index 61b26a5981..d9fb1a1700 100644 --- a/hugegraph-core/pom.xml +++ b/hugegraph-core/pom.xml @@ -19,7 +19,7 @@ com.baidu.hugegraph hugegraph-common - 1.8.4 + 1.8.5 diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java index 3a16deb59e..11aa6a0361 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java @@ -280,6 +280,15 @@ public GraphMode mode() { public void mode(GraphMode mode) { LOG.info("Graph {} will work in {} mode", this, mode); this.mode = mode; + if (mode.loading()) { + /* + * NOTE: This may block tasks submit and lead the queue to be full, + * so don't submit gremlin job when loading data + */ + this.taskManager.pauseScheduledThreadPool(); + } else { + this.taskManager.resumeScheduledThreadPool(); + } } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskManager.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskManager.java index 84322941a2..a95707dc54 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskManager.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskManager.java @@ -24,7 +24,6 @@ import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -33,6 +32,7 @@ import com.baidu.hugegraph.HugeException; import com.baidu.hugegraph.HugeGraphParams; +import com.baidu.hugegraph.concurrent.PausableScheduledThreadPool; import com.baidu.hugegraph.util.Consumers; import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.ExecutorUtil; @@ -59,7 +59,7 @@ public final class TaskManager { private final ExecutorService taskExecutor; private final ExecutorService taskDbExecutor; private final ExecutorService serverInfoDbExecutor; - private final ScheduledExecutorService schedulerExecutor; + private final PausableScheduledThreadPool schedulerExecutor; public static TaskManager instance() { return MANAGER; @@ -76,7 +76,7 @@ private TaskManager(int pool) { this.serverInfoDbExecutor = ExecutorUtil.newFixedThreadPool( 1, SERVER_INFO_DB_WORKER); // For schedule task to run, just one thread is ok - this.schedulerExecutor = ExecutorUtil.newScheduledThreadPool( + this.schedulerExecutor = ExecutorUtil.newPausableScheduledThreadPool( 1, TASK_SCHEDULER); // Start after 10s waiting for HugeGraphServer startup this.schedulerExecutor.scheduleWithFixedDelay(this::scheduleOrExecuteJob, @@ -155,6 +155,14 @@ private void closeSchedulerTx(HugeGraphParams graph) { } } + public void pauseScheduledThreadPool() { + this.schedulerExecutor.pauseSchedule(); + } + + public void resumeScheduledThreadPool() { + this.schedulerExecutor.resumeSchedule(); + } + public TaskScheduler getScheduler(HugeGraphParams graph) { return this.schedulers.get(graph); } diff --git a/hugegraph-dist/src/assembly/static/conf/rest-server.properties b/hugegraph-dist/src/assembly/static/conf/rest-server.properties index 30bc288287..4b29180fb8 100644 --- a/hugegraph-dist/src/assembly/static/conf/rest-server.properties +++ b/hugegraph-dist/src/assembly/static/conf/rest-server.properties @@ -6,6 +6,10 @@ restserver.url=http://127.0.0.1:8080 # graphs list with pair NAME:CONF_PATH graphs=[hugegraph:conf/hugegraph.properties] +# The maximum thread ratio for batch writing, only take effect if the batch.max_write_threads is 0 +batch.max_write_ratio=80 +batch.max_write_threads=0 + # authentication #auth.authenticator= #auth.admin_token= From ecb6d4ce2fcccf10bf9b3946c416743332da8a34 Mon Sep 17 00:00:00 2001 From: liningrui Date: Fri, 16 Apr 2021 19:29:55 +0800 Subject: [PATCH 2/2] Replace addProperty with setProperty Change-Id: Ie4e9645686e11be58b0870de8643c638d3f54f50 --- .../src/main/java/com/baidu/hugegraph/server/RestServer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/server/RestServer.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/server/RestServer.java index f7b1c9be80..3a796df0c3 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/server/RestServer.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/server/RestServer.java @@ -203,7 +203,8 @@ private void calcMaxWriteThreads() { } LOG.info("The maximum batch writing threads is {} (total threads {})", maxWriteThreads, maxWorkerThreads); - this.conf.addProperty(ServerOptions.MAX_WRITE_THREADS.name(), + // NOTE: addProperty will make exist option's value become List + this.conf.setProperty(ServerOptions.MAX_WRITE_THREADS.name(), String.valueOf(maxWriteThreads)); }