From 9c50ec82ede635e7b42e166030c1be8b6412c16b Mon Sep 17 00:00:00 2001 From: liningrui Date: Tue, 29 Jun 2021 17:26:31 +0800 Subject: [PATCH 01/14] Improve some action for install snapshot and add peer 1. Fix bug that relative path need normalize when load snapshot 2. Use zip decompresser since decompress tar file may lead dead loop 3. Move raft.endpoint and raft.group_peers into rest-server.properties 4. Pass raft.endpoint and raft.group_peers from server to core 5. Let RestServer start before GremlinServer 6. Use read-index to ensure the raft log caught up for new node 7. Let add_peer and remove_peer API work in async job 8. Let truncateBackend don't truncate system store Change-Id: I7de3d98cb51c3b7c32a4f19a0d99ed622f78d331 --- .../com/baidu/hugegraph/api/raft/RaftAPI.java | 58 +++++++++---- .../hugegraph/auth/HugeGraphAuthProxy.java | 8 +- .../baidu/hugegraph/config/ServerOptions.java | 16 ++++ .../baidu/hugegraph/core/GraphManager.java | 24 ++++-- .../java/com/baidu/hugegraph/HugeGraph.java | 4 +- .../baidu/hugegraph/StandardHugeGraph.java | 8 +- .../store/AbstractBackendStoreProvider.java | 9 +- .../backend/store/BackendStoreProvider.java | 2 +- .../backend/store/raft/RaftAddPeerJob.java | 50 +++++++++++ .../store/raft/RaftBackendStoreProvider.java | 28 ++++-- .../backend/store/raft/RaftClosure.java | 1 + .../backend/store/raft/RaftNode.java | 68 ++++++++------- .../backend/store/raft/RaftRemovePeerJob.java | 50 +++++++++++ .../backend/store/raft/RaftSharedContext.java | 86 +++++++++---------- .../backend/store/raft/StoreSnapshotFile.java | 50 +++++++++-- .../backend/store/raft/StoreStateMachine.java | 9 +- .../store/raft/rpc/ListPeersProcessor.java | 3 +- .../store/raft/rpc/SetLeaderProcessor.java | 3 +- .../baidu/hugegraph/config/CoreOptions.java | 37 +++----- .../baidu/hugegraph/util/CompressUtil.java | 9 +- .../baidu/hugegraph/dist/HugeGraphServer.java | 27 ++++-- .../hugegraph/unit/util/CompressUtilTest.java | 3 +- 22 files changed, 381 insertions(+), 172 deletions(-) create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftAddPeerJob.java create mode 100644 hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftRemovePeerJob.java diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/raft/RaftAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/raft/RaftAPI.java index 9f06852c6f..5d39adcf50 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/raft/RaftAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/raft/RaftAPI.java @@ -19,6 +19,7 @@ package com.baidu.hugegraph.api.raft; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -40,8 +41,14 @@ import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.api.API; import com.baidu.hugegraph.api.filter.StatusFilter.Status; +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.store.raft.RaftAddPeerJob; import com.baidu.hugegraph.backend.store.raft.RaftGroupManager; +import com.baidu.hugegraph.backend.store.raft.RaftRemovePeerJob; import com.baidu.hugegraph.core.GraphManager; +import com.baidu.hugegraph.job.JobBuilder; +import com.baidu.hugegraph.util.DateUtil; +import com.baidu.hugegraph.util.JsonUtil; import com.baidu.hugegraph.util.Log; import com.codahale.metrics.annotation.Timed; import com.google.common.collect.ImmutableMap; @@ -144,19 +151,26 @@ public Map setLeader(@Context GraphManager manager, @Consumes(APPLICATION_JSON) @Produces(APPLICATION_JSON_WITH_CHARSET) @RolesAllowed({"admin"}) - public Map addPeer(@Context GraphManager manager, - @PathParam("graph") String graph, - @QueryParam("group") - @DefaultValue("default") - String group, - @QueryParam("endpoint") - String endpoint) { + public Map addPeer(@Context GraphManager manager, + @PathParam("graph") String graph, + @QueryParam("group") @DefaultValue("default") + String group, + @QueryParam("endpoint") String endpoint) { LOG.debug("Graph [{}] prepare to add peer: {}", graph, endpoint); HugeGraph g = graph(manager, graph); RaftGroupManager raftManager = raftGroupManager(g, group, "add_peer"); - String peerId = raftManager.addPeer(endpoint); - return ImmutableMap.of(raftManager.group(), peerId); + + JobBuilder builder = JobBuilder.of(g); + String name = String.format("raft-group-[%s]-add-peer-[%s]-at-[%s]", + raftManager.group(), endpoint, + DateUtil.now()); + Map inputs = new HashMap<>(); + inputs.put("endpoint", endpoint); + builder.name(name) + .input(JsonUtil.toJson(inputs)) + .job(new RaftAddPeerJob()); + return ImmutableMap.of("task_id", builder.schedule().id()); } @POST @@ -166,26 +180,32 @@ public Map addPeer(@Context GraphManager manager, @Consumes(APPLICATION_JSON) @Produces(APPLICATION_JSON_WITH_CHARSET) @RolesAllowed({"admin"}) - public Map removePeer(@Context GraphManager manager, - @PathParam("graph") String graph, - @QueryParam("group") - @DefaultValue("default") - String group, - @QueryParam("endpoint") - String endpoint) { + public Map removePeer(@Context GraphManager manager, + @PathParam("graph") String graph, + @QueryParam("group") + @DefaultValue("default") String group, + @QueryParam("endpoint") String endpoint) { LOG.debug("Graph [{}] prepare to remove peer: {}", graph, endpoint); HugeGraph g = graph(manager, graph); RaftGroupManager raftManager = raftGroupManager(g, group, "remove_peer"); - String peerId = raftManager.removePeer(endpoint); - return ImmutableMap.of(raftManager.group(), peerId); + JobBuilder builder = JobBuilder.of(g); + String name = String.format("raft-group-[%s]-remove-peer-[%s]-at-[%s]", + raftManager.group(), endpoint, + DateUtil.now()); + Map inputs = new HashMap<>(); + inputs.put("endpoint", endpoint); + builder.name(name) + .input(JsonUtil.toJson(inputs)) + .job(new RaftRemovePeerJob()); + return ImmutableMap.of("task_id", builder.schedule().id()); } private static RaftGroupManager raftGroupManager(HugeGraph graph, String group, String operation) { - RaftGroupManager raftManager = graph.raftGroupManager(group); + RaftGroupManager raftManager = graph.raftGroupManager(); if (raftManager == null) { throw new HugeException("Allowed %s operation only when " + "working on raft mode", operation); diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java index 3add4ab17e..760299d44a 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java @@ -639,9 +639,9 @@ public void readMode(GraphReadMode readMode) { } @Override - public void waitStarted() { + public void waitReady() { this.verifyAnyPermission(); - this.hugegraph.waitStarted(); + this.hugegraph.waitReady(); } @Override @@ -688,9 +688,9 @@ public void switchAuthManager(AuthManager authManager) { } @Override - public RaftGroupManager raftGroupManager(String group) { + public RaftGroupManager raftGroupManager() { this.verifyAdminPermission(); - return this.hugegraph.raftGroupManager(group); + return this.hugegraph.raftGroupManager(); } @Override diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java index a0c4b073fa..fc02c1958e 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java @@ -177,6 +177,22 @@ public static synchronized ServerOptions instance() { nonNegativeInt(), 0); + public static final ConfigOption RAFT_ENDPOINT = + new ConfigOption<>( + "raft.endpoint", + "The peerid of current raft node.", + disallowEmpty(), + "127.0.0.1:8281" + ); + + public static final ConfigOption RAFT_GROUP_PEERS = + new ConfigOption<>( + "raft.group_peers", + "The initial peers of current raft group.", + disallowEmpty(), + "127.0.0.1:8281" + ); + public static final ConfigOption ALLOW_TRACE = new ConfigOption<>( "exception.allow_trace", diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java index eed83c0cb7..47a41ed7cd 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java @@ -100,30 +100,33 @@ public GraphManager(HugeConfig conf, EventHub hub) { this.loadGraphs(ConfigUtil.scanGraphsDir(this.graphsDir)); // this.installLicense(conf, ""); // Raft will load snapshot firstly then launch election and replay log - this.waitGraphsStarted(); + this.waitGraphsReady(); + this.checkBackendVersionOrExit(conf); this.startRpcServer(); this.serverStarted(conf); this.addMetrics(conf); } - public void loadGraphs(final Map graphConfs) { + public void loadGraphs(HugeConfig serverConfig) { + Map graphConfs = serverConfig.getMap( + ServerOptions.GRAPHS); for (Map.Entry conf : graphConfs.entrySet()) { String name = conf.getKey(); String path = conf.getValue(); HugeFactory.checkGraphName(name, "rest-server.properties"); try { - this.loadGraph(name, path); + this.loadGraph(serverConfig, name, path); } catch (RuntimeException e) { LOG.error("Graph '{}' can't be loaded: '{}'", name, path, e); } } } - public void waitGraphsStarted() { + public void waitGraphsReady() { this.graphs.keySet().forEach(name -> { HugeGraph graph = this.graph(name); - graph.waitStarted(); + graph.waitReady(); }); } @@ -331,8 +334,14 @@ private void closeTx(final Set graphSourceNamesToCloseTxOn, }); } - private void loadGraph(String name, String path) { - final Graph graph = GraphFactory.open(path); + private void loadGraph(HugeConfig serverConfig, String name, String path) { + HugeConfig config = new HugeConfig(path); + String raftEndpoint = serverConfig.get(ServerOptions.RAFT_ENDPOINT); + String raftGroupPeers = serverConfig.get(ServerOptions.RAFT_GROUP_PEERS); + config.addProperty(ServerOptions.RAFT_ENDPOINT.name(), raftEndpoint); + config.addProperty(ServerOptions.RAFT_GROUP_PEERS.name(), raftGroupPeers); + + final Graph graph = GraphFactory.open(config); this.graphs.put(name, graph); LOG.info("Graph '{}' was successfully configured via '{}'", name, path); @@ -344,6 +353,7 @@ private void loadGraph(String name, String path) { } private void checkBackendVersionOrExit(HugeConfig config) { + LOG.info("Check backend version"); for (String graph : this.graphs()) { // TODO: close tx from main thread HugeGraph hugegraph = this.graph(graph); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java index 69e495f56d..01b489c26d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java @@ -144,7 +144,7 @@ public interface HugeGraph extends Graph { public GraphReadMode readMode(); public void readMode(GraphReadMode readMode); - public void waitStarted(); + public void waitReady(); public void serverStarted(Id serverId, NodeRole serverRole); public boolean started(); public boolean closed(); @@ -169,7 +169,7 @@ public interface HugeGraph extends Graph { public AuthManager authManager(); public void switchAuthManager(AuthManager authManager); public TaskScheduler taskScheduler(); - public RaftGroupManager raftGroupManager(String group); + public RaftGroupManager raftGroupManager(); public void proxy(HugeGraph graph); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java index 1336fd98d4..af432bdf82 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java @@ -206,7 +206,7 @@ public StandardHugeGraph(HugeConfig config) { LockUtil.destroy(this.name); String message = "Failed to load backend store provider"; LOG.error("{}: {}", message, e.getMessage()); - throw new HugeException(message); + throw new HugeException(message, e); } try { @@ -310,7 +310,7 @@ public void readMode(GraphReadMode readMode) { } @Override - public void waitStarted() { + public void waitReady() { // Just for trigger Tx.getOrNewTransaction, then load 3 stores this.schemaTransaction(); this.storeProvider.waitStoreStarted(); @@ -1003,13 +1003,13 @@ public void switchAuthManager(AuthManager authManager) { } @Override - public RaftGroupManager raftGroupManager(String group) { + public RaftGroupManager raftGroupManager() { if (!(this.storeProvider instanceof RaftBackendStoreProvider)) { return null; } RaftBackendStoreProvider provider = ((RaftBackendStoreProvider) this.storeProvider); - return provider.raftNodeManager(group); + return provider.raftNodeManager(); } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java index c9f0c6e996..88759f82cd 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java @@ -134,10 +134,15 @@ public void clear() throws BackendException { } @Override - public void truncate() { + public void truncate(HugeGraph graph) { this.checkOpened(); + HugeConfig config = (HugeConfig) graph.configuration(); + String systemStoreName = config.get(CoreOptions.STORE_SYSTEM); for (BackendStore store : this.stores.values()) { - store.truncate(); + // Don't truncate system store + if (!store.store().equals(systemStoreName)) { + store.truncate(); + } } this.notifyAndWaitEvent(Events.STORE_TRUNCATE); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java index a4027e17ac..f248ed1743 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java @@ -52,7 +52,7 @@ public interface BackendStoreProvider { public void clear(); - public void truncate(); + public void truncate(HugeGraph graph); public void initSystemInfo(HugeGraph graph); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftAddPeerJob.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftAddPeerJob.java new file mode 100644 index 0000000000..0fd1d49c37 --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftAddPeerJob.java @@ -0,0 +1,50 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.store.raft; + +import java.util.Map; + +import com.baidu.hugegraph.job.SysJob; +import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.util.JsonUtil; + +public class RaftAddPeerJob extends SysJob { + + public static final String TASK_TYPE = "raft_add_peer"; + + @Override + public String type() { + return TASK_TYPE; + } + + @Override + public String execute() throws Exception { + String input = this.task().input(); + E.checkArgumentNotNull(input, "The input can't be null"); + @SuppressWarnings("unchecked") + Map map = JsonUtil.fromJson(input, Map.class); + + Object value = map.get("endpoint"); + E.checkArgument(value instanceof String, + "Invalid endpoint value '%s'", value); + String endpoint = (String) value; + return this.graph().raftGroupManager().addPeer(endpoint); + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java index 73d8c200c0..dbc1fa4e3d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java @@ -22,16 +22,17 @@ import java.util.Set; import java.util.concurrent.Future; -import com.baidu.hugegraph.config.HugeConfig; import org.slf4j.Logger; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.HugeGraphParams; +import com.baidu.hugegraph.backend.BackendException; import com.baidu.hugegraph.backend.store.BackendStore; import com.baidu.hugegraph.backend.store.BackendStoreProvider; import com.baidu.hugegraph.backend.store.BackendStoreSystemInfo; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreAction; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreType; +import com.baidu.hugegraph.config.CoreOptions; import com.baidu.hugegraph.config.HugeConfig; import com.baidu.hugegraph.event.EventHub; import com.baidu.hugegraph.event.EventListener; @@ -59,8 +60,8 @@ public RaftBackendStoreProvider(BackendStoreProvider provider, this.systemStore = null; } - public RaftGroupManager raftNodeManager(String group) { - return this.context.raftNodeManager(group); + public RaftGroupManager raftNodeManager() { + return this.context.raftNodeManager(); } private Set stores() { @@ -181,10 +182,15 @@ public void clear() { } @Override - public void truncate() { + public void truncate(HugeGraph graph) { this.checkOpened(); + HugeConfig config = (HugeConfig) graph.configuration(); + String systemStoreName = config.get(CoreOptions.STORE_SYSTEM); for (RaftBackendStore store : this.stores()) { - store.truncate(); + // Don't truncate system store + if (!store.store().equals(systemStoreName)) { + store.truncate(); + } } this.notifyAndWaitEvent(Events.STORE_TRUNCATE); @@ -219,8 +225,16 @@ public void createSnapshot() { StoreCommand command = new StoreCommand(StoreType.GRAPH, StoreAction.SNAPSHOT, null); RaftStoreClosure closure = new RaftStoreClosure(command); - this.context.node().submitAndWait(command, closure); - LOG.debug("Graph '{}' has writed snapshot", this.graph()); + RaftClosure future = this.context.node().submitAndWait(command, + closure); + if (future != null) { + try { + future.waitFinished(); + LOG.debug("Graph '{}' has writed snapshot", this.graph()); + } catch (Throwable e) { + throw new BackendException("Failed to create snapshot", e); + } + } } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftClosure.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftClosure.java index eb337c72cc..a3d64a003d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftClosure.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftClosure.java @@ -69,6 +69,7 @@ private RaftResult get() { } public void complete(Status status, Supplier callback) { + // This callback is called by consumer thread(like grizzly) this.future.complete(new RaftResult<>(status, callback)); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java index d3dd65b0ab..84411fc236 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Random; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -28,6 +29,7 @@ import org.slf4j.Logger; import com.alipay.sofa.jraft.Node; +import com.alipay.sofa.jraft.RaftGroupService; import com.alipay.sofa.jraft.RaftServiceFactory; import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.closure.ReadIndexClosure; @@ -46,6 +48,7 @@ public final class RaftNode { private static final Logger LOG = Log.logger(RaftNode.class); private final RaftSharedContext context; + private RaftGroupService raftGroupService; private final Node node; private final StoreStateMachine stateMachine; private final AtomicReference leaderInfo; @@ -98,14 +101,11 @@ public void shutdown() { this.node.shutdown(); } - public void snapshot() { - if (!this.context.useSnapshot()) { - return; - } + public RaftClosure snapshot() { RaftClosure future = new RaftClosure<>(); try { this.node().snapshot(future); - future.waitFinished(); + return future; } catch (Throwable e) { throw new BackendException("Failed to create snapshot", e); } @@ -114,7 +114,10 @@ public void snapshot() { private Node initRaftNode() throws IOException { NodeOptions nodeOptions = this.context.nodeOptions(); nodeOptions.setFsm(this.stateMachine); - // TODO: When support sharding, groupId needs to be bound to shard Id + /* + * TODO: the groupId is same as graph name now, when support sharding, + * groupId needs to be bound to shard Id + */ String groupId = this.context.group(); PeerId endpoint = this.context.endpoint(); /* @@ -130,7 +133,7 @@ private Node initRaftNode() throws IOException { private void submitCommand(StoreCommand command, RaftStoreClosure closure) { // Wait leader elected LeaderInfo leaderInfo = this.waitLeaderElected( - RaftSharedContext.NO_TIMEOUT); + RaftSharedContext.WAIT_LEADER_TIMEOUT); if (!leaderInfo.selfIsLeader) { this.context.rpcForwarder().forwardToLeader(leaderInfo.leaderId, command, closure); @@ -153,14 +156,16 @@ private void submitCommand(StoreCommand command, RaftStoreClosure closure) { this.node.apply(task); } - public Object submitAndWait(StoreCommand command, RaftStoreClosure future) { + public T submitAndWait(StoreCommand command, RaftStoreClosure future) { this.submitCommand(command, future); try { /* * Here will wait future complete, actually the follower has waited * in forwardToLeader, written like this to simplify the code */ - return future.waitFinished(); + @SuppressWarnings("unchecked") + T result = (T) future.waitFinished(); + return result; } catch (Throwable e) { throw new BackendException("Failed to wait store command %s", e, command); @@ -173,13 +178,14 @@ protected LeaderInfo waitLeaderElected(int timeout) { if (leaderInfo.leaderId != null) { return leaderInfo; } + LOG.info("Waiting for raft group '{}' leader elected", group); long beginTime = System.currentTimeMillis(); while (leaderInfo.leaderId == null) { try { Thread.sleep(RaftSharedContext.POLL_INTERVAL); } catch (InterruptedException e) { - LOG.info("Waiting for raft group '{}' election is " + - "interrupted: {}", group, e); + throw new BackendException("Interrupted while waiting for " + + "raft group '%s' election", group, e); } long consumedTime = System.currentTimeMillis() - beginTime; if (timeout > 0 && consumedTime >= timeout) { @@ -187,42 +193,38 @@ protected LeaderInfo waitLeaderElected(int timeout) { "Waiting for raft group '%s' election timeout(%sms)", group, consumedTime); } - LOG.warn("Waiting for raft group '{}' election cost {}s", - group, consumedTime / 1000.0); leaderInfo = this.leaderInfo.get(); assert leaderInfo != null; } + LOG.info("Waited for raft group '{}' leader elected successfully", group); return leaderInfo; } - protected void waitStarted(int timeout) { + protected void waitRaftLogSynced(int timeout) { String group = this.context.group(); - ReadIndexClosure readIndexClosure = new ReadIndexClosure() { - @Override - public void run(Status status, long index, byte[] reqCtx) { - RaftNode.this.started.set(status.isOk()); - } - }; + LOG.info("Waiting for raft group '{}' log synced", group); long beginTime = System.currentTimeMillis(); - while (true) { - this.node.readIndex(BytesUtil.EMPTY_BYTES, readIndexClosure); - if (this.started.get()) { - break; - } + while (!this.started.get()) { + this.node.readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { + @Override + public void run(Status status, long index, byte[] reqCtx) { + RaftNode.this.started.set(status.isOk()); + } + }); try { Thread.sleep(RaftSharedContext.POLL_INTERVAL); } catch (InterruptedException e) { - LOG.info("Waiting for heartbeat is interrupted: {}", e); + throw new BackendException("Interrupted while waiting for " + + "raft group '%s' log sync", group, e); } long consumedTime = System.currentTimeMillis() - beginTime; if (timeout > 0 && consumedTime >= timeout) { throw new BackendException( - "Waiting for raft group '%s' heartbeat timeout(%sms)", + "Waiting for raft group '%s' log sync timeout(%sms)", group, consumedTime); } - LOG.warn("Waiting for raft group '{}' heartbeat cost {}s", - group, consumedTime / 1000.0); } + LOG.info("Waited for raft group '{}' log synced successfully", group); } private void waitIfBusy() { @@ -231,8 +233,12 @@ private void waitIfBusy() { return; } // It may lead many thread sleep, but this is exactly what I want - long time = counter * RaftSharedContext.BUSY_SLEEP_FACTOR; - LOG.info("The node {} will try to sleep {} ms", this.node, time); + int delta = RaftSharedContext.BUSY_MAX_SLEEP_FACTOR - + RaftSharedContext.BUSY_MIN_SLEEP_FACTOR; + Random random = new Random(); + int timeout = random.nextInt(delta) + + RaftSharedContext.BUSY_MIN_SLEEP_FACTOR; + int time = counter * timeout; try { Thread.sleep(time); } catch (InterruptedException e) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftRemovePeerJob.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftRemovePeerJob.java new file mode 100644 index 0000000000..eed62515dc --- /dev/null +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftRemovePeerJob.java @@ -0,0 +1,50 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.store.raft; + +import java.util.Map; + +import com.baidu.hugegraph.job.SysJob; +import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.util.JsonUtil; + +public class RaftRemovePeerJob extends SysJob { + + public static final String TASK_TYPE = "raft_remove_peer"; + + @Override + public String type() { + return TASK_TYPE; + } + + @Override + public String execute() throws Exception { + String input = this.task().input(); + E.checkArgumentNotNull(input, "The input can't be null"); + @SuppressWarnings("unchecked") + Map map = JsonUtil.fromJson(input, Map.class); + + Object value = map.get("endpoint"); + E.checkArgument(value instanceof String, + "Invalid endpoint value '%s'", value); + String endpoint = (String) value; + return this.graph().raftGroupManager().removePeer(endpoint); + } +} diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftSharedContext.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftSharedContext.java index 8342d08a0e..d319538ebe 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftSharedContext.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftSharedContext.java @@ -31,6 +31,8 @@ import java.util.concurrent.RejectedExecutionHandler; import java.util.concurrent.ThreadPoolExecutor; +import javax.ws.rs.HEAD; + import org.apache.commons.io.FileUtils; import org.slf4j.Logger; @@ -73,10 +75,11 @@ public final class RaftSharedContext { // unit is ms public static final int NO_TIMEOUT = -1; - public static final int POLL_INTERVAL = 3000; + public static final int POLL_INTERVAL = 5000; public static final int WAIT_RAFTLOG_TIMEOUT = 30 * 60 * 1000; - public static final int WAIT_LEADER_TIMEOUT = 5 * 60 * 1000; - public static final int BUSY_SLEEP_FACTOR = 3 * 1000; + public static final int WAIT_LEADER_TIMEOUT = 10 * 60 * 1000; + public static final int BUSY_MIN_SLEEP_FACTOR = 3 * 1000; + public static final int BUSY_MAX_SLEEP_FACTOR = 5 * 1000; public static final int WAIT_RPC_TIMEOUT = 30 * 60 * 1000; public static final int LOG_WARN_INTERVAL = 60 * 1000; @@ -85,15 +88,15 @@ public final class RaftSharedContext { // work queue size public static final int QUEUE_SIZE = CoreOptions.CPUS; - public static final long KEEP_ALIVE_SEC = 300L; - - public static final String DEFAULT_GROUP = "default"; + public static final long KEEP_ALIVE_SECOND = 300L; private final HugeGraphParams params; private final String schemaStoreName; private final String graphStoreName; private final String systemStoreName; private final RaftBackendStore[] stores; + private final PeerId endpoint; + private final Configuration groupPeers; private final RpcServer rpcServer; @SuppressWarnings("unused") private final ExecutorService readIndexExecutor; @@ -112,6 +115,20 @@ public RaftSharedContext(HugeGraphParams params) { this.graphStoreName = config.get(CoreOptions.STORE_GRAPH); this.systemStoreName = config.get(CoreOptions.STORE_SYSTEM); this.stores = new RaftBackendStore[StoreType.ALL.getNumber()]; + + // TODO: 依赖了ServerOptions的配置项名，需要打通ServerConfig和CoreConfig + this.endpoint = new PeerId(); + String endpointStr = config.getString("raft.endpoint"); + if (!this.endpoint.parse(endpointStr)) { + throw new HugeException("Failed to parse endpoint %s", endpointStr); + } + this.groupPeers = new Configuration(); + String groupPeersStr = config.getString("raft.group_peers"); + if (!this.groupPeers.parse(groupPeersStr)) { + throw new HugeException("Failed to parse group peers %s", + groupPeersStr); + } + this.rpcServer = this.initAndStartRpcServer(); if (config.get(CoreOptions.RAFT_SAFE_READ)) { int threads = config.get(CoreOptions.RAFT_READ_INDEX_THREADS); @@ -119,11 +136,7 @@ public RaftSharedContext(HugeGraphParams params) { } else { this.readIndexExecutor = null; } - if (config.get(CoreOptions.RAFT_USE_SNAPSHOT)) { - this.snapshotExecutor = this.createSnapshotExecutor(4); - } else { - this.snapshotExecutor = null; - } + this.snapshotExecutor = this.createSnapshotExecutor(4); int backendThreads = config.get(CoreOptions.RAFT_BACKEND_THREADS); this.backendExecutor = this.createBackendExecutor(backendThreads); @@ -143,9 +156,7 @@ public void initRaftNode() { public void waitRaftNodeStarted() { RaftNode node = this.node(); node.waitLeaderElected(RaftSharedContext.WAIT_LEADER_TIMEOUT); - if (node.selfIsLeader()) { - node.waitStarted(RaftSharedContext.NO_TIMEOUT); - } + node.waitRaftLogSynced(RaftSharedContext.NO_TIMEOUT); } public void close() { @@ -167,15 +178,13 @@ public RpcForwarder rpcForwarder() { return this.rpcForwarder; } - public RaftGroupManager raftNodeManager(String group) { - E.checkArgument(DEFAULT_GROUP.equals(group), - "The group must be '%s' now, actual is '%s'", - DEFAULT_GROUP, group); + public RaftGroupManager raftNodeManager() { return this.raftGroupManager; } public String group() { - return DEFAULT_GROUP; + // Use graph name as group name + return this.params.name(); } public void addStore(StoreType type, RaftBackendStore store) { @@ -206,8 +215,6 @@ public BackendStore originStore(StoreType storeType) { public NodeOptions nodeOptions() throws IOException { HugeConfig config = this.config(); - PeerId selfId = new PeerId(); - selfId.parse(config.get(CoreOptions.RAFT_ENDPOINT)); NodeOptions nodeOptions = new NodeOptions(); nodeOptions.setEnableMetrics(false); @@ -217,6 +224,8 @@ public NodeOptions nodeOptions() throws IOException { config.get(CoreOptions.RAFT_RPC_CONNECT_TIMEOUT)); nodeOptions.setRpcDefaultTimeout( config.get(CoreOptions.RAFT_RPC_TIMEOUT)); + nodeOptions.setRpcInstallSnapshotTimeout( + config.get(CoreOptions.RAFT_INSTALL_SNAPSHOT_TIMEOUT)); int electionTimeout = config.get(CoreOptions.RAFT_ELECTION_TIMEOUT); nodeOptions.setElectionTimeoutMs(electionTimeout); @@ -224,14 +233,7 @@ public NodeOptions nodeOptions() throws IOException { int snapshotInterval = config.get(CoreOptions.RAFT_SNAPSHOT_INTERVAL); nodeOptions.setSnapshotIntervalSecs(snapshotInterval); - - Configuration groupPeers = new Configuration(); - String groupPeersStr = config.get(CoreOptions.RAFT_GROUP_PEERS); - if (!groupPeers.parse(groupPeersStr)) { - throw new HugeException("Failed to parse group peers %s", - groupPeersStr); - } - nodeOptions.setInitialConf(groupPeers); + nodeOptions.setInitialConf(this.groupPeers); String raftPath = config.get(CoreOptions.RAFT_PATH); String logUri = Paths.get(raftPath, "log").toString(); @@ -242,11 +244,9 @@ public NodeOptions nodeOptions() throws IOException { FileUtils.forceMkdir(new File(metaUri)); nodeOptions.setRaftMetaUri(metaUri); - if (config.get(CoreOptions.RAFT_USE_SNAPSHOT)) { - String snapshotUri = Paths.get(raftPath, "snapshot").toString(); - FileUtils.forceMkdir(new File(snapshotUri)); - nodeOptions.setSnapshotUri(snapshotUri); - } + String snapshotUri = Paths.get(raftPath, "snapshot").toString(); + FileUtils.forceMkdir(new File(snapshotUri)); + nodeOptions.setSnapshotUri(snapshotUri); RaftOptions raftOptions = nodeOptions.getRaftOptions(); /* @@ -329,22 +329,13 @@ protected void notifyCache(String action, HugeType type, List ids) { } public PeerId endpoint() { - PeerId endpoint = new PeerId(); - String endpointStr = this.config().get(CoreOptions.RAFT_ENDPOINT); - if (!endpoint.parse(endpointStr)) { - throw new HugeException("Failed to parse endpoint %s", endpointStr); - } - return endpoint; + return this.endpoint; } public boolean isSafeRead() { return this.config().get(CoreOptions.RAFT_SAFE_READ); } - public boolean useSnapshot() { - return this.config().get(CoreOptions.RAFT_USE_SNAPSHOT); - } - public ExecutorService snapshotExecutor() { return this.snapshotExecutor; } @@ -353,6 +344,10 @@ public ExecutorService backendExecutor() { return this.backendExecutor; } + public ExecutorService readIndexExecutor() { + return this.readIndexExecutor; + } + public GraphMode graphMode() { return this.params.mode(); } @@ -375,6 +370,7 @@ private RpcServer initAndStartRpcServer() { NodeManager.getInstance().addAddress(endpoint.getEndpoint()); RpcServer rpcServer = RaftRpcServerFactory.createAndStartRaftRpcServer( endpoint.getEndpoint()); + LOG.info("RPC server is started successfully"); return rpcServer; } @@ -421,7 +417,7 @@ private static ExecutorService newPool(int coreThreads, int maxThreads, .enableMetric(false) .coreThreads(coreThreads) .maximumThreads(maxThreads) - .keepAliveSeconds(KEEP_ALIVE_SEC) + .keepAliveSeconds(KEEP_ALIVE_SECOND) .workQueue(queue) .threadFactory(new NamedThreadFactory(name, true)) .rejectedHandler(handler) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreSnapshotFile.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreSnapshotFile.java index 54a8b9c32b..c7fb2a9524 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreSnapshotFile.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreSnapshotFile.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.zip.Checksum; import org.apache.commons.io.FileUtils; @@ -52,10 +53,11 @@ public class StoreSnapshotFile { private static final Logger LOG = Log.logger(StoreSnapshotFile.class); public static final String SNAPSHOT_DIR = "snapshot"; - private static final String TAR = ".tar"; + private static final String TAR = ".zip"; private final RaftBackendStore[] stores; private final Map dataDisks; + private final AtomicBoolean compressing; public StoreSnapshotFile(RaftBackendStore[] stores) { this.stores = stores; @@ -65,6 +67,7 @@ public StoreSnapshotFile(RaftBackendStore[] stores) { this.dataDisks.putAll(Whitebox.invoke(raftStore, "store", "reportDiskMapping")); } + this.compressing = new AtomicBoolean(false); /* * Like that: * general=/parent_path/rocksdb-data @@ -79,7 +82,16 @@ public void save(SnapshotWriter writer, Closure done, // Write snapshot to real directory Map snapshotDirMaps = this.doSnapshotSave(); executor.execute(() -> { + if (this.compressing.get()) { + LOG.info("Last compress task doesn't finish, skipped it"); + done.run(new Status(RaftError.EBUSY, + "Last compress task doesn't finish, " + + "skipped it")); + return; + } + try { + this.compressing.set(true); this.compressSnapshotDir(writer, snapshotDirMaps); this.deleteSnapshotDirs(snapshotDirMaps.keySet()); done.run(Status.OK()); @@ -88,6 +100,8 @@ public void save(SnapshotWriter writer, Closure done, done.run(new Status(RaftError.EIO, "Failed to compress snapshot, " + "error is %s", e.getMessage())); + } finally { + this.compressing.set(false); } }); } catch (Throwable e) { @@ -102,15 +116,23 @@ public boolean load(SnapshotReader reader) { Set snapshotDirTars = reader.listFiles(); LOG.info("The snapshot tar files to be loaded are {}", snapshotDirTars); Set snapshotDirs = new HashSet<>(); - for (String snapshotDirTar : snapshotDirTars) { - try { + if (this.compressing.get()) { + LOG.info("Last decompress task doesn't finish, skipped it"); + return false; + } + + try { + this.compressing.set(true); + for (String snapshotDirTar : snapshotDirTars) { String snapshotDir = this.decompressSnapshot(reader, snapshotDirTar); snapshotDirs.add(snapshotDir); - } catch (Throwable e) { - LOG.error("Failed to decompress snapshot tar", e); - return false; } + } catch (Throwable e) { + LOG.error("Failed to decompress snapshot tar", e); + return false; + } finally { + this.compressing.set(false); } try { @@ -151,7 +173,13 @@ private void compressSnapshotDir(SnapshotWriter writer, .toString(); Checksum checksum = new CRC64(); try { - CompressUtil.compressTar(snapshotDir, outputFile, checksum); + LOG.info("Prepare to compress dir {} to {}", + snapshotDir, outputFile); + long begin = System.currentTimeMillis(); + CompressUtil.compressZip(snapshotDir, outputFile, checksum); + long end = System.currentTimeMillis(); + LOG.info("Compressed dir {} to {}, took {} seconds", + snapshotDir, outputFile, (end - begin) / 1000.0F); } catch (Throwable e) { throw new RaftException( "Failed to compress snapshot, path=%s, files=%s", @@ -195,7 +223,13 @@ private String decompressSnapshot(SnapshotReader reader, Checksum checksum = new CRC64(); String archiveFile = Paths.get(reader.getPath(), snapshotDirTar) .toString(); - CompressUtil.decompressTar(archiveFile, parentPath, checksum); + LOG.info("Prepare to decompress snapshot zip {} to {}", + archiveFile, parentPath); + long begin = System.currentTimeMillis(); + CompressUtil.decompressZip(archiveFile, parentPath, checksum); + long end = System.currentTimeMillis(); + LOG.info("Decompress snapshot zip {} to {}, took {} seconds", + archiveFile, parentPath, (end - begin) / 1000.0F); if (meta.hasChecksum()) { String expected = meta.getChecksum(); String actual = Long.toHexString(checksum.getValue()); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreStateMachine.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreStateMachine.java index 73b1cd7c24..a8d3de5f44 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreStateMachine.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreStateMachine.java @@ -25,6 +25,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; +import javax.ws.rs.HEAD; + import org.slf4j.Logger; import com.alipay.sofa.jraft.Closure; @@ -45,6 +47,8 @@ import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreAction; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreType; import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.type.HugeType; +import com.baidu.hugegraph.type.define.GraphMode; import com.baidu.hugegraph.util.LZ4Util; import com.baidu.hugegraph.util.Log; @@ -72,7 +76,7 @@ private RaftNode node() { public void onApply(Iterator iter) { LOG.debug("Node role: {}", this.node().selfIsLeader() ? "leader" : "follower"); - List> futures = new ArrayList<>(); + List> futures = new ArrayList<>(64); try { // Apply all the logs while (iter.hasNext()) { @@ -161,8 +165,7 @@ private Object applyCommand(StoreType type, StoreAction action, break; case SNAPSHOT: assert store == null; - this.node().snapshot(); - break; + return this.node().snapshot(); case BEGIN_TX: store.beginTx(); break; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/ListPeersProcessor.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/ListPeersProcessor.java index 328cf3c108..88a641dd0b 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/ListPeersProcessor.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/ListPeersProcessor.java @@ -48,8 +48,7 @@ public ListPeersProcessor(RaftSharedContext context) { public Message processRequest(ListPeersRequest request, RpcRequestClosure done) { LOG.debug("Processing ListPeersRequest {}", request.getClass()); - RaftGroupManager nodeManager = this.context.raftNodeManager( - RaftSharedContext.DEFAULT_GROUP); + RaftGroupManager nodeManager = this.context.raftNodeManager(); try { CommonResponse common = CommonResponse.newBuilder() .setStatus(true) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/SetLeaderProcessor.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/SetLeaderProcessor.java index f110537ffb..3b7c2b337a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/SetLeaderProcessor.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/SetLeaderProcessor.java @@ -47,8 +47,7 @@ public SetLeaderProcessor(RaftSharedContext context) { public Message processRequest(SetLeaderRequest request, RpcRequestClosure done) { LOG.debug("Processing SetLeaderRequest {}", request.getClass()); - RaftGroupManager nodeManager = this.context.raftNodeManager( - RaftSharedContext.DEFAULT_GROUP); + RaftGroupManager nodeManager = this.context.raftNodeManager(); try { nodeManager.setLeader(request.getEndpoint()); CommonResponse common = CommonResponse.newBuilder() diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java index e5be7e6347..02ddfa8432 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java @@ -120,30 +120,6 @@ public static synchronized CoreOptions instance() { false ); - public static final ConfigOption RAFT_USE_SNAPSHOT = - new ConfigOption<>( - "raft.use_snapshot", - "Whether to use snapshot.", - disallowEmpty(), - true - ); - - public static final ConfigOption RAFT_ENDPOINT = - new ConfigOption<>( - "raft.endpoint", - "The peerid of current raft node.", - disallowEmpty(), - "127.0.0.1:8281" - ); - - public static final ConfigOption RAFT_GROUP_PEERS = - new ConfigOption<>( - "raft.group_peers", - "The peers of current raft group.", - disallowEmpty(), - "127.0.0.1:8281,127.0.0.1:8282,127.0.0.1:8283" - ); - public static final ConfigOption RAFT_PATH = new ConfigOption<>( "raft.path", @@ -252,10 +228,19 @@ public static synchronized CoreOptions instance() { public static final ConfigOption RAFT_RPC_TIMEOUT = new ConfigOption<>( "raft.rpc_timeout", - "The rpc timeout for jraft rpc.", + "The general rpc timeout for jraft rpc.", positiveInt(), // jraft default value is 5000(ms) - 60000 + 60 * 1000 + ); + + public static final ConfigOption RAFT_INSTALL_SNAPSHOT_TIMEOUT = + new ConfigOption<>( + "raft.rpc_timeout", + "The install snapshot rpc timeout for jraft rpc.", + positiveInt(), + // jraft default value is 5 * 60 * 1000 + 24 * 60 * 60 * 1000 ); public static final ConfigOption RAFT_RPC_BUF_LOW_WATER_MARK = diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/CompressUtil.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/CompressUtil.java index 2d28333147..5e586cd143 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/CompressUtil.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/CompressUtil.java @@ -170,13 +170,20 @@ private static Path zipSlipProtect(ArchiveEntry entry, Path targetDir) * else throws exception */ Path normalizePath = targetDirResolved.normalize(); - if (!normalizePath.startsWith(targetDir)) { + if (!normalizePath.startsWith(targetDir.normalize())) { throw new IOException(String.format("Bad entry: %s", entry.getName())); } return normalizePath; } + public static void compressZip(String inputDir, String outputFile, + Checksum checksum) throws IOException { + String rootDir = Paths.get(inputDir).getParent().toString(); + String sourceDir = Paths.get(inputDir).getFileName().toString(); + compressZip(rootDir, sourceDir, outputFile, checksum); + } + public static void compressZip(String rootDir, String sourceDir, String outputFile, Checksum checksum) throws IOException { diff --git a/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java b/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java index 476e5420ea..86fe9ffca4 100644 --- a/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java +++ b/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java @@ -37,8 +37,8 @@ public class HugeGraphServer { private static final Logger LOG = Log.logger(HugeGraphServer.class); - private final GremlinServer gremlinServer; private final RestServer restServer; + private final GremlinServer gremlinServer; public static void register() { RegisterUtil.registerBackends(); @@ -48,6 +48,14 @@ public static void register() { public HugeGraphServer(String gremlinServerConf, String restServerConf) throws Exception { + try { + // Start HugeRestServer + this.restServer = HugeRestServer.start(restServerConf); + } catch (Throwable e) { + LOG.error("HugeRestServer start error: ", e); + throw e; + } + // Only switch on security manager after HugeGremlinServer started SecurityManager securityManager = System.getSecurityManager(); System.setSecurityManager(null); @@ -62,6 +70,11 @@ public HugeGraphServer(String gremlinServerConf, String restServerConf) graphsDir, hub); } catch (Throwable e) { LOG.error("HugeGremlinServer start error: ", e); + try { + this.restServer.shutdown().get(); + } catch (Throwable t) { + LOG.error("HugeRestServer stop error: ", t); + } HugeFactory.shutdown(30L); throw e; } finally { @@ -85,17 +98,17 @@ public HugeGraphServer(String gremlinServerConf, String restServerConf) public void stop() { try { - this.restServer.shutdown().get(); - LOG.info("HugeRestServer stopped"); + this.gremlinServer.stop().get(); + LOG.info("HugeGremlinServer stopped"); } catch (Throwable e) { - LOG.error("HugeRestServer stop error: ", e); + LOG.error("HugeGremlinServer stop error: ", e); } try { - this.gremlinServer.stop().get(); - LOG.info("HugeGremlinServer stopped"); + this.restServer.shutdown().get(); + LOG.info("HugeRestServer stopped"); } catch (Throwable e) { - LOG.error("HugeGremlinServer stop error: ", e); + LOG.error("HugeRestServer stop error: ", e); } try { diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/CompressUtilTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/CompressUtilTest.java index b91f66dee1..c3b8f7c736 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/CompressUtilTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/util/CompressUtilTest.java @@ -50,7 +50,8 @@ public void testZipCompress() throws IOException { prepareFiles(Paths.get(rootDir, sourceDir).toString()); Checksum checksum = new CRC64(); - CompressUtil.compressZip(rootDir, sourceDir, zipFile, checksum); + CompressUtil.compressZip(Paths.get(rootDir, sourceDir).toString(), + zipFile, checksum); CompressUtil.decompressZip(zipFile, output, checksum); assertDirEquals(rootDir, output); From 6af7a3ce7ee3d947d52a6df82aa992d29ad1705e Mon Sep 17 00:00:00 2001 From: liningrui Date: Thu, 1 Jul 2021 17:43:24 +0800 Subject: [PATCH 02/14] Use shared rpc server Change-Id: Iba6e9cbfab15f28cab4126b7ef8ab7210370a0d6 --- .../hugegraph/auth/HugeGraphAuthProxy.java | 5 +- .../baidu/hugegraph/config/ServerOptions.java | 14 +-- .../baidu/hugegraph/core/GraphManager.java | 91 ++++++++++++++----- .../java/com/baidu/hugegraph/HugeGraph.java | 3 +- .../baidu/hugegraph/StandardHugeGraph.java | 8 +- .../baidu/hugegraph/auth/EntityManager.java | 1 + .../backend/store/BackendProviderFactory.java | 2 +- .../backend/store/raft/RaftBackendStore.java | 4 +- .../store/raft/RaftBackendStoreProvider.java | 47 ++++++++-- .../backend/store/raft/RaftClosure.java | 2 +- ...aftSharedContext.java => RaftContext.java} | 37 ++++---- .../store/raft/RaftGroupManagerImpl.java | 2 +- .../backend/store/raft/RaftNode.java | 22 ++--- .../backend/store/raft/StoreStateMachine.java | 10 +- .../store/raft/rpc/ListPeersProcessor.java | 6 +- .../backend/store/raft/rpc/RpcForwarder.java | 2 +- .../store/raft/rpc/SetLeaderProcessor.java | 6 +- .../store/raft/rpc/StoreCommandProcessor.java | 6 +- .../backend/tx/SchemaTransaction.java | 2 +- .../baidu/hugegraph/util/CompressUtil.java | 4 +- 20 files changed, 180 insertions(+), 94 deletions(-) rename hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/{RaftSharedContext.java => RaftContext.java} (95%) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java index 760299d44a..523f4eb280 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java @@ -57,6 +57,7 @@ import org.apache.tinkerpop.gremlin.structure.io.Io; import org.slf4j.Logger; +import com.alipay.remoting.rpc.RpcServer; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.auth.HugeAuthenticator.RolePerm; import com.baidu.hugegraph.auth.HugeAuthenticator.User; @@ -639,9 +640,9 @@ public void readMode(GraphReadMode readMode) { } @Override - public void waitReady() { + public void waitReady(RpcServer rpcServer) { this.verifyAnyPermission(); - this.hugegraph.waitReady(); + this.hugegraph.waitReady(rpcServer); } @Override diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java index fc02c1958e..8190c081cc 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java @@ -177,13 +177,13 @@ public static synchronized ServerOptions instance() { nonNegativeInt(), 0); - public static final ConfigOption RAFT_ENDPOINT = - new ConfigOption<>( - "raft.endpoint", - "The peerid of current raft node.", - disallowEmpty(), - "127.0.0.1:8281" - ); +// public static final ConfigOption RAFT_ENDPOINT = +// new ConfigOption<>( +// "raft.endpoint", +// "The peerid of current raft node.", +// disallowEmpty(), +// "127.0.0.1:8281" +// ); public static final ConfigOption RAFT_GROUP_PEERS = new ConfigOption<>( diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java index 47a41ed7cd..eeeade2b34 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java @@ -37,6 +37,10 @@ import org.apache.tinkerpop.gremlin.structure.util.GraphFactory; import org.slf4j.Logger; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory; +import com.alipay.sofa.rpc.config.ServerConfig; +import com.baidu.hugegraph.HugeException; import com.baidu.hugegraph.HugeFactory; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.auth.AuthManager; @@ -51,6 +55,7 @@ import com.baidu.hugegraph.backend.store.BackendStoreSystemInfo; import com.baidu.hugegraph.config.CoreOptions; import com.baidu.hugegraph.config.HugeConfig; +import com.baidu.hugegraph.config.RpcOptions; import com.baidu.hugegraph.config.ServerOptions; import com.baidu.hugegraph.config.TypedOption; import com.baidu.hugegraph.event.EventHub; @@ -66,6 +71,7 @@ import com.baidu.hugegraph.serializer.Serializer; import com.baidu.hugegraph.server.RestServer; import com.baidu.hugegraph.task.TaskManager; +import com.baidu.hugegraph.testutil.Whitebox; import com.baidu.hugegraph.type.define.NodeRole; import com.baidu.hugegraph.util.ConfigUtil; import com.baidu.hugegraph.util.E; @@ -100,12 +106,32 @@ public GraphManager(HugeConfig conf, EventHub hub) { this.loadGraphs(ConfigUtil.scanGraphsDir(this.graphsDir)); // this.installLicense(conf, ""); // Raft will load snapshot firstly then launch election and replay log - this.waitGraphsReady(); + this.startRpcServer(); + + initAllSystemSchema(); + + ServerConfig serverConfig = Whitebox.getInternalState(this.rpcServer, + "serverConfig"); + com.alipay.remoting.rpc.RpcServer remotingRpcServer; + remotingRpcServer = Whitebox.getInternalState(serverConfig.getServer(), + "remotingServer"); + this.waitGraphsReady(remotingRpcServer); this.checkBackendVersionOrExit(conf); - this.startRpcServer(); + this.serverStarted(conf); this.addMetrics(conf); + } + + public { + // + register(-1, ~task, xx, xx, xx,); + + + + + + } public void loadGraphs(HugeConfig serverConfig) { @@ -123,10 +149,10 @@ public void loadGraphs(HugeConfig serverConfig) { } } - public void waitGraphsReady() { + public void waitGraphsReady(com.alipay.remoting.rpc.RpcServer rpcServer) { this.graphs.keySet().forEach(name -> { HugeGraph graph = this.graph(name); - graph.waitReady(); + graph.waitReady(rpcServer); }); } @@ -260,6 +286,45 @@ public void close() { this.unlistenChanges(); } + private void loadGraph(HugeConfig serverConfig, String name, String path) { + HugeConfig config = new HugeConfig(path); + String raftGroupPeers = serverConfig.get(ServerOptions.RAFT_GROUP_PEERS); + config.addProperty(ServerOptions.RAFT_GROUP_PEERS.name(), raftGroupPeers); + + final Graph graph = GraphFactory.open(config); + this.graphs.put(name, graph); + LOG.info("Graph '{}' was successfully configured via '{}'", name, path); + + if (this.requireAuthentication() && + !(graph instanceof HugeGraphAuthProxy)) { + LOG.warn("You may need to support access control for '{}' with {}", + path, HugeFactoryAuthProxy.GRAPH_FACTORY); + } + } + +// private com.alipay.sofa.jraft.rpc.RpcServer startRaftRpcServer(HugeConfig +// config) { +// Integer lowWaterMark = config.get( +// CoreOptions.RAFT_RPC_BUF_LOW_WATER_MARK); +// System.setProperty("bolt.channel_write_buf_low_water_mark", +// String.valueOf(lowWaterMark)); +// Integer highWaterMark = config.get( +// CoreOptions.RAFT_RPC_BUF_HIGH_WATER_MARK); +// System.setProperty("bolt.channel_write_buf_high_water_mark", +// String.valueOf(highWaterMark)); +// +// PeerId endpoint = new PeerId(); +// String endpointStr = config.get(ServerOptions.RAFT_ENDPOINT); +// if (!endpoint.parse(endpointStr)) { +// throw new HugeException("Failed to parse endpoint %s", endpointStr); +// } +// com.alipay.sofa.jraft.rpc.RpcServer rpcServer; +// rpcServer = RaftRpcServerFactory.createAndStartRaftRpcServer( +// endpoint.getEndpoint()); +// LOG.info("Raft RPC server is started successfully"); +// return rpcServer; +// } + private void startRpcServer() { if (!this.rpcServer.enabled()) { LOG.info("RpcServer is not enabled, skip starting rpc service"); @@ -334,24 +399,6 @@ private void closeTx(final Set graphSourceNamesToCloseTxOn, }); } - private void loadGraph(HugeConfig serverConfig, String name, String path) { - HugeConfig config = new HugeConfig(path); - String raftEndpoint = serverConfig.get(ServerOptions.RAFT_ENDPOINT); - String raftGroupPeers = serverConfig.get(ServerOptions.RAFT_GROUP_PEERS); - config.addProperty(ServerOptions.RAFT_ENDPOINT.name(), raftEndpoint); - config.addProperty(ServerOptions.RAFT_GROUP_PEERS.name(), raftGroupPeers); - - final Graph graph = GraphFactory.open(config); - this.graphs.put(name, graph); - LOG.info("Graph '{}' was successfully configured via '{}'", name, path); - - if (this.requireAuthentication() && - !(graph instanceof HugeGraphAuthProxy)) { - LOG.warn("You may need to support access control for '{}' with {}", - path, HugeFactoryAuthProxy.GRAPH_FACTORY); - } - } - private void checkBackendVersionOrExit(HugeConfig config) { LOG.info("Check backend version"); for (String graph : this.graphs()) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java index 01b489c26d..775b0da58a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java @@ -31,6 +31,7 @@ import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.VertexProperty; +import com.alipay.remoting.rpc.RpcServer; import com.baidu.hugegraph.auth.AuthManager; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.query.Query; @@ -144,7 +145,7 @@ public interface HugeGraph extends Graph { public GraphReadMode readMode(); public void readMode(GraphReadMode readMode); - public void waitReady(); + public void waitReady(RpcServer rpcServer); public void serverStarted(Id serverId, NodeRole serverRole); public boolean started(); public boolean closed(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java index af432bdf82..a72726c482 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java @@ -39,6 +39,7 @@ import org.apache.tinkerpop.gremlin.structure.util.StringFactory; import org.slf4j.Logger; +import com.alipay.remoting.rpc.RpcServer; import com.baidu.hugegraph.analyzer.Analyzer; import com.baidu.hugegraph.analyzer.AnalyzerFactory; import com.baidu.hugegraph.auth.AuthManager; @@ -310,7 +311,12 @@ public void readMode(GraphReadMode readMode) { } @Override - public void waitReady() { + public void waitReady(RpcServer rpcServer) { + if (this.storeProvider instanceof RaftBackendStoreProvider) { + ((RaftBackendStoreProvider) this.storeProvider).initRaftContext( + this.params, + rpcServer); + } // Just for trigger Tx.getOrNewTransaction, then load 3 stores this.schemaTransaction(); this.storeProvider.waitStoreStarted(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/auth/EntityManager.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/auth/EntityManager.java index c4b668e619..35e6a99722 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/auth/EntityManager.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/auth/EntityManager.java @@ -174,6 +174,7 @@ private Id save(T entity, boolean expectExists) { // Add or update user in backend store, stale index might exist vertex = this.tx().addVertex(vertex); + addVertex(, System.User) this.commitOrRollback(); return vertex.id(); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendProviderFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendProviderFactory.java index deac4254d8..18f21b23cd 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendProviderFactory.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendProviderFactory.java @@ -52,7 +52,7 @@ public static BackendStoreProvider open(HugeGraphParams params) { if (raftMode) { LOG.info("Opening backend store '{}' in raft mode for graph '{}'", backend, graph); - provider = new RaftBackendStoreProvider(provider, params); + provider = new RaftBackendStoreProvider(provider); } provider.open(graph); return provider; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStore.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStore.java index 5f9dfb2d8a..94aad3e95f 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStore.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStore.java @@ -48,11 +48,11 @@ public class RaftBackendStore implements BackendStore { private static final Logger LOG = Log.logger(RaftBackendStore.class); private final BackendStore store; - private final RaftSharedContext context; + private final RaftContext context; private final ThreadLocal mutationBatch; private final boolean isSafeRead; - public RaftBackendStore(BackendStore store, RaftSharedContext context) { + public RaftBackendStore(BackendStore store, RaftContext context) { this.store = store; this.context = context; this.mutationBatch = new ThreadLocal<>(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java index dbc1fa4e3d..2c24c0d9c1 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java @@ -24,6 +24,11 @@ import org.slf4j.Logger; +import com.alipay.remoting.rpc.RpcServer; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory; +import com.alipay.sofa.jraft.rpc.impl.BoltRpcServer; +import com.baidu.hugegraph.HugeException; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.HugeGraphParams; import com.baidu.hugegraph.backend.BackendException; @@ -46,18 +51,47 @@ public class RaftBackendStoreProvider implements BackendStoreProvider { private static final Logger LOG = Log.logger(RaftBackendStoreProvider.class); private final BackendStoreProvider provider; - private final RaftSharedContext context; private RaftBackendStore schemaStore; private RaftBackendStore graphStore; private RaftBackendStore systemStore; + private RaftContext context; - public RaftBackendStoreProvider(BackendStoreProvider provider, - HugeGraphParams params) { + public RaftBackendStoreProvider(BackendStoreProvider provider) { this.provider = provider; - this.context = new RaftSharedContext(params); this.schemaStore = null; this.graphStore = null; this.systemStore = null; + this.context = null; + } + + public void initRaftContext(HugeGraphParams params, RpcServer rpcServer) { + HugeConfig config = params.configuration(); + Integer lowWaterMark = config.get( + CoreOptions.RAFT_RPC_BUF_LOW_WATER_MARK); + System.setProperty("bolt.channel_write_buf_low_water_mark", + String.valueOf(lowWaterMark)); + Integer highWaterMark = config.get( + CoreOptions.RAFT_RPC_BUF_HIGH_WATER_MARK); + System.setProperty("bolt.channel_write_buf_high_water_mark", + String.valueOf(highWaterMark)); + +// PeerId endpoint = new PeerId(); +// String endpointStr = config.get(ServerOptions.RAFT_ENDPOINT); +// if (!endpoint.parse(endpointStr)) { +// throw new HugeException("Failed to parse endpoint %s", endpointStr); +// } + + // Reference from RaftRpcServerFactory.createAndStartRaftRpcServer + com.alipay.sofa.jraft.rpc.RpcServer raftRpcServer = + new BoltRpcServer(rpcServer); + RaftRpcServerFactory.addRaftRequestProcessors(raftRpcServer); + raftRpcServer.init(null); + + PeerId endpoint = new PeerId(rpcServer.ip(), rpcServer.port()); + this.context = new RaftContext(params, raftRpcServer, endpoint); + + // + this.context.addStore(StoreType.SYSTEM, this.systemStore); } public RaftGroupManager raftNodeManager() { @@ -129,7 +163,6 @@ public synchronized BackendStore loadSystemStore(HugeConfig config, String name) BackendStore store = this.provider.loadSystemStore(config, name); this.checkNonSharedStore(store); this.systemStore = new RaftBackendStore(store, this.context); - this.context.addStore(StoreType.SYSTEM, this.systemStore); } return this.systemStore; } @@ -203,7 +236,9 @@ public void initSystemInfo(HugeGraph graph) { BackendStoreSystemInfo info = graph.backendStoreSystemInfo(); info.init(); - this.notifyAndWaitEvent(Events.STORE_INITED); + // 创建系统schema，保存到pool里面 + init(); +// this.notifyAndWaitEvent(Events.STORE_INITED); LOG.debug("Graph '{}' system info has been initialized", this.graph()); /* * Take the initiative to generate a snapshot, it can avoid this diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftClosure.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftClosure.java index a3d64a003d..b38d895fd5 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftClosure.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftClosure.java @@ -57,7 +57,7 @@ public Status status() { private RaftResult get() { try { - return this.future.get(RaftSharedContext.WAIT_RAFTLOG_TIMEOUT, + return this.future.get(RaftContext.WAIT_RAFTLOG_TIMEOUT, TimeUnit.MILLISECONDS); } catch (ExecutionException e) { throw new BackendException("ExecutionException", e); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftSharedContext.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java similarity index 95% rename from hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftSharedContext.java rename to hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java index d319538ebe..eb2f1403eb 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftSharedContext.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java @@ -31,8 +31,6 @@ import java.util.concurrent.RejectedExecutionHandler; import java.util.concurrent.ThreadPoolExecutor; -import javax.ws.rs.HEAD; - import org.apache.commons.io.FileUtils; import org.slf4j.Logger; @@ -69,9 +67,9 @@ import com.baidu.hugegraph.util.Events; import com.baidu.hugegraph.util.Log; -public final class RaftSharedContext { +public final class RaftContext { - private static final Logger LOG = Log.logger(RaftSharedContext.class); + private static final Logger LOG = Log.logger(RaftContext.class); // unit is ms public static final int NO_TIMEOUT = -1; @@ -91,13 +89,14 @@ public final class RaftSharedContext { public static final long KEEP_ALIVE_SECOND = 300L; private final HugeGraphParams params; + private final RpcServer rpcServer; + private final PeerId endpoint; + private final String schemaStoreName; private final String graphStoreName; private final String systemStoreName; private final RaftBackendStore[] stores; - private final PeerId endpoint; private final Configuration groupPeers; - private final RpcServer rpcServer; @SuppressWarnings("unused") private final ExecutorService readIndexExecutor; private final ExecutorService snapshotExecutor; @@ -107,21 +106,24 @@ public final class RaftSharedContext { private RaftGroupManager raftGroupManager; private RpcForwarder rpcForwarder; - public RaftSharedContext(HugeGraphParams params) { + public RaftContext(HugeGraphParams params, RpcServer rpcServer, + PeerId endpoint) { this.params = params; - HugeConfig config = this.config(); + this.rpcServer = rpcServer; + this.endpoint = endpoint; + HugeConfig config = params.configuration(); this.schemaStoreName = config.get(CoreOptions.STORE_SCHEMA); this.graphStoreName = config.get(CoreOptions.STORE_GRAPH); this.systemStoreName = config.get(CoreOptions.STORE_SYSTEM); this.stores = new RaftBackendStore[StoreType.ALL.getNumber()]; - // TODO: 依赖了ServerOptions的配置项名，需要打通ServerConfig和CoreConfig - this.endpoint = new PeerId(); - String endpointStr = config.getString("raft.endpoint"); - if (!this.endpoint.parse(endpointStr)) { - throw new HugeException("Failed to parse endpoint %s", endpointStr); - } +// // TODO: 依赖了ServerOptions的配置项名，需要打通ServerConfig和CoreConfig +// this.endpoint = new PeerId(); +// String endpointStr = config.getString("raft.endpoint"); +// if (!this.endpoint.parse(endpointStr)) { +// throw new HugeException("Failed to parse endpoint %s", endpointStr); +// } this.groupPeers = new Configuration(); String groupPeersStr = config.getString("raft.group_peers"); if (!this.groupPeers.parse(groupPeersStr)) { @@ -129,7 +131,6 @@ public RaftSharedContext(HugeGraphParams params) { groupPeersStr); } - this.rpcServer = this.initAndStartRpcServer(); if (config.get(CoreOptions.RAFT_SAFE_READ)) { int threads = config.get(CoreOptions.RAFT_READ_INDEX_THREADS); this.readIndexExecutor = this.createReadIndexExecutor(threads); @@ -155,8 +156,8 @@ public void initRaftNode() { public void waitRaftNodeStarted() { RaftNode node = this.node(); - node.waitLeaderElected(RaftSharedContext.WAIT_LEADER_TIMEOUT); - node.waitRaftLogSynced(RaftSharedContext.NO_TIMEOUT); + node.waitLeaderElected(RaftContext.WAIT_LEADER_TIMEOUT); + node.waitRaftLogSynced(RaftContext.NO_TIMEOUT); } public void close() { @@ -355,7 +356,6 @@ public GraphMode graphMode() { private HugeConfig config() { return this.params.configuration(); } - private RpcServer initAndStartRpcServer() { Integer lowWaterMark = this.config().get( CoreOptions.RAFT_RPC_BUF_LOW_WATER_MARK); @@ -385,7 +385,6 @@ private void registerRpcRequestProcessors() { this.rpcServer.registerProcessor(new SetLeaderProcessor(this)); this.rpcServer.registerProcessor(new ListPeersProcessor(this)); } - private ExecutorService createReadIndexExecutor(int coreThreads) { int maxThreads = coreThreads << 2; String name = "store-read-index-callback"; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftGroupManagerImpl.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftGroupManagerImpl.java index d1b15b634e..a000403711 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftGroupManagerImpl.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftGroupManagerImpl.java @@ -40,7 +40,7 @@ public class RaftGroupManagerImpl implements RaftGroupManager { private final RaftNode raftNode; private final RpcForwarder rpcForwarder; - public RaftGroupManagerImpl(RaftSharedContext context) { + public RaftGroupManagerImpl(RaftContext context) { this.group = context.group(); this.raftNode = context.node(); this.rpcForwarder = context.rpcForwarder(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java index 84411fc236..095f602268 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java @@ -47,7 +47,7 @@ public final class RaftNode { private static final Logger LOG = Log.logger(RaftNode.class); - private final RaftSharedContext context; + private final RaftContext context; private RaftGroupService raftGroupService; private final Node node; private final StoreStateMachine stateMachine; @@ -55,7 +55,7 @@ public final class RaftNode { private final AtomicBoolean started; private final AtomicInteger busyCounter; - public RaftNode(RaftSharedContext context) { + public RaftNode(RaftContext context) { this.context = context; this.stateMachine = new StoreStateMachine(context); try { @@ -70,7 +70,7 @@ public RaftNode(RaftSharedContext context) { this.busyCounter = new AtomicInteger(); } - public RaftSharedContext context() { + public RaftContext context() { return this.context; } @@ -133,7 +133,7 @@ private Node initRaftNode() throws IOException { private void submitCommand(StoreCommand command, RaftStoreClosure closure) { // Wait leader elected LeaderInfo leaderInfo = this.waitLeaderElected( - RaftSharedContext.WAIT_LEADER_TIMEOUT); + RaftContext.WAIT_LEADER_TIMEOUT); if (!leaderInfo.selfIsLeader) { this.context.rpcForwarder().forwardToLeader(leaderInfo.leaderId, command, closure); @@ -146,7 +146,7 @@ private void submitCommand(StoreCommand command, RaftStoreClosure closure) { task.setDone(closure); // compress return BytesBuffer ByteBuffer buffer = LZ4Util.compress(command.data(), - RaftSharedContext.BLOCK_SIZE) + RaftContext.BLOCK_SIZE) .forReadWritten() .asByteBuffer(); LOG.debug("The bytes size of command(compressed) {} is {}", @@ -182,7 +182,7 @@ protected LeaderInfo waitLeaderElected(int timeout) { long beginTime = System.currentTimeMillis(); while (leaderInfo.leaderId == null) { try { - Thread.sleep(RaftSharedContext.POLL_INTERVAL); + Thread.sleep(RaftContext.POLL_INTERVAL); } catch (InterruptedException e) { throw new BackendException("Interrupted while waiting for " + "raft group '%s' election", group, e); @@ -212,7 +212,7 @@ public void run(Status status, long index, byte[] reqCtx) { } }); try { - Thread.sleep(RaftSharedContext.POLL_INTERVAL); + Thread.sleep(RaftContext.POLL_INTERVAL); } catch (InterruptedException e) { throw new BackendException("Interrupted while waiting for " + "raft group '%s' log sync", group, e); @@ -233,11 +233,11 @@ private void waitIfBusy() { return; } // It may lead many thread sleep, but this is exactly what I want - int delta = RaftSharedContext.BUSY_MAX_SLEEP_FACTOR - - RaftSharedContext.BUSY_MIN_SLEEP_FACTOR; + int delta = RaftContext.BUSY_MAX_SLEEP_FACTOR - + RaftContext.BUSY_MIN_SLEEP_FACTOR; Random random = new Random(); int timeout = random.nextInt(delta) + - RaftSharedContext.BUSY_MIN_SLEEP_FACTOR; + RaftContext.BUSY_MIN_SLEEP_FACTOR; int time = counter * timeout; try { Thread.sleep(time); @@ -283,7 +283,7 @@ public void onDestroyed(PeerId peer) { public void onError(PeerId peer, Status status) { long now = System.currentTimeMillis(); long interval = now - this.lastPrintTime; - if (interval >= RaftSharedContext.LOG_WARN_INTERVAL) { + if (interval >= RaftContext.LOG_WARN_INTERVAL) { LOG.warn("Replicator meet error: {}", status); this.lastPrintTime = now; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreStateMachine.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreStateMachine.java index a8d3de5f44..d06481bf72 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreStateMachine.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreStateMachine.java @@ -25,8 +25,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; -import javax.ws.rs.HEAD; - import org.slf4j.Logger; import com.alipay.sofa.jraft.Closure; @@ -47,8 +45,6 @@ import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreAction; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreType; import com.baidu.hugegraph.util.E; -import com.baidu.hugegraph.type.HugeType; -import com.baidu.hugegraph.type.define.GraphMode; import com.baidu.hugegraph.util.LZ4Util; import com.baidu.hugegraph.util.Log; @@ -56,10 +52,10 @@ public final class StoreStateMachine extends StateMachineAdapter { private static final Logger LOG = Log.logger(StoreStateMachine.class); - private final RaftSharedContext context; + private final RaftContext context; private final StoreSnapshotFile snapshotFile; - public StoreStateMachine(RaftSharedContext context) { + public StoreStateMachine(RaftContext context) { this.context = context; this.snapshotFile = new StoreSnapshotFile(context.stores()); } @@ -134,7 +130,7 @@ private Future onApplyFollower(ByteBuffer data) { // Let the backend thread do it directly return this.context.backendExecutor().submit(() -> { BytesBuffer buffer = LZ4Util.decompress(bytes, - RaftSharedContext.BLOCK_SIZE); + RaftContext.BLOCK_SIZE); buffer.forReadWritten(); StoreType type = StoreType.valueOf(buffer.read()); StoreAction action = StoreAction.valueOf(buffer.read()); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/ListPeersProcessor.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/ListPeersProcessor.java index 88a641dd0b..a79a075872 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/ListPeersProcessor.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/ListPeersProcessor.java @@ -24,7 +24,7 @@ import com.alipay.sofa.jraft.rpc.RpcRequestClosure; import com.alipay.sofa.jraft.rpc.RpcRequestProcessor; import com.baidu.hugegraph.backend.store.raft.RaftGroupManager; -import com.baidu.hugegraph.backend.store.raft.RaftSharedContext; +import com.baidu.hugegraph.backend.store.raft.RaftContext; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.CommonResponse; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.ListPeersRequest; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.ListPeersResponse; @@ -37,9 +37,9 @@ public class ListPeersProcessor private static final Logger LOG = Log.logger(ListPeersProcessor.class); - private final RaftSharedContext context; + private final RaftContext context; - public ListPeersProcessor(RaftSharedContext context) { + public ListPeersProcessor(RaftContext context) { super(null, null); this.context = context; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/RpcForwarder.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/RpcForwarder.java index 0151f5040d..110fbbb086 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/RpcForwarder.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/RpcForwarder.java @@ -19,7 +19,7 @@ package com.baidu.hugegraph.backend.store.raft.rpc; -import static com.baidu.hugegraph.backend.store.raft.RaftSharedContext.WAIT_RPC_TIMEOUT; +import static com.baidu.hugegraph.backend.store.raft.RaftContext.WAIT_RPC_TIMEOUT; import java.util.concurrent.ExecutionException; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/SetLeaderProcessor.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/SetLeaderProcessor.java index 3b7c2b337a..56518f73f1 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/SetLeaderProcessor.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/SetLeaderProcessor.java @@ -24,7 +24,7 @@ import com.alipay.sofa.jraft.rpc.RpcRequestClosure; import com.alipay.sofa.jraft.rpc.RpcRequestProcessor; import com.baidu.hugegraph.backend.store.raft.RaftGroupManager; -import com.baidu.hugegraph.backend.store.raft.RaftSharedContext; +import com.baidu.hugegraph.backend.store.raft.RaftContext; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.CommonResponse; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.SetLeaderRequest; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.SetLeaderResponse; @@ -36,9 +36,9 @@ public class SetLeaderProcessor private static final Logger LOG = Log.logger(SetLeaderProcessor.class); - private final RaftSharedContext context; + private final RaftContext context; - public SetLeaderProcessor(RaftSharedContext context) { + public SetLeaderProcessor(RaftContext context) { super(null, null); this.context = context; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/StoreCommandProcessor.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/StoreCommandProcessor.java index 13ce2cfb8e..447f990137 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/StoreCommandProcessor.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/StoreCommandProcessor.java @@ -24,7 +24,7 @@ import com.alipay.sofa.jraft.rpc.RpcRequestClosure; import com.alipay.sofa.jraft.rpc.RpcRequestProcessor; import com.baidu.hugegraph.backend.store.raft.RaftNode; -import com.baidu.hugegraph.backend.store.raft.RaftSharedContext; +import com.baidu.hugegraph.backend.store.raft.RaftContext; import com.baidu.hugegraph.backend.store.raft.RaftStoreClosure; import com.baidu.hugegraph.backend.store.raft.StoreCommand; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreAction; @@ -40,9 +40,9 @@ public class StoreCommandProcessor private static final Logger LOG = Log.logger( StoreCommandProcessor.class); - private final RaftSharedContext context; + private final RaftContext context; - public StoreCommandProcessor(RaftSharedContext context) { + public StoreCommandProcessor(RaftContext context) { super(null, null); this.context = context; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/SchemaTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/SchemaTransaction.java index af567b384f..14594ef16d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/SchemaTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/SchemaTransaction.java @@ -69,7 +69,7 @@ public class SchemaTransaction extends IndexableTransaction { - private SchemaIndexTransaction indexTx; + private final SchemaIndexTransaction indexTx; public SchemaTransaction(HugeGraphParams graph, BackendStore store) { super(graph, store); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/CompressUtil.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/CompressUtil.java index 5e586cd143..72034a6ea3 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/CompressUtil.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/CompressUtil.java @@ -46,7 +46,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.io.output.NullOutputStream; -import com.baidu.hugegraph.backend.store.raft.RaftSharedContext; +import com.baidu.hugegraph.backend.store.raft.RaftContext; import net.jpountz.lz4.LZ4BlockInputStream; import net.jpountz.lz4.LZ4BlockOutputStream; @@ -63,7 +63,7 @@ public static void compressTar(String inputDir, String outputFile, Checksum checksum) throws IOException { LZ4Factory factory = LZ4Factory.fastestInstance(); LZ4Compressor compressor = factory.fastCompressor(); - int blockSize = RaftSharedContext.BLOCK_SIZE; + int blockSize = RaftContext.BLOCK_SIZE; try (FileOutputStream fos = new FileOutputStream(outputFile); CheckedOutputStream cos = new CheckedOutputStream(fos, checksum); BufferedOutputStream bos = new BufferedOutputStream(cos); From 435edfe47b7d05f0114eca69dfebd4c100d531fc Mon Sep 17 00:00:00 2001 From: liningrui Date: Sat, 16 Apr 2022 12:06:52 +0800 Subject: [PATCH 03/14] rebase and address some comments Change-Id: Iefe83f4781ad77ece88939f59eb81552175189cd --- .../baidu/hugegraph/config/ServerOptions.java | 14 ++++---- .../baidu/hugegraph/StandardHugeGraph.java | 2 +- .../baidu/hugegraph/auth/EntityManager.java | 1 - .../backend/store/raft/RaftContext.java | 4 +++ .../backend/store/raft/RaftNode.java | 36 +++++++++++++------ .../backend/store/raft/StoreSnapshotFile.java | 18 +++++----- .../baidu/hugegraph/config/CoreOptions.java | 2 +- 7 files changed, 47 insertions(+), 30 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java index 8190c081cc..bd3f4bcfac 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java @@ -177,13 +177,13 @@ public static synchronized ServerOptions instance() { nonNegativeInt(), 0); -// public static final ConfigOption RAFT_ENDPOINT = -// new ConfigOption<>( -// "raft.endpoint", -// "The peerid of current raft node.", -// disallowEmpty(), -// "127.0.0.1:8281" -// ); + public static final ConfigOption RAFT_ENDPOINT = + new ConfigOption<>( + "raft.endpoint", + "The endpoint of current raft node.", + disallowEmpty(), + "127.0.0.1:8281" + ); public static final ConfigOption RAFT_GROUP_PEERS = new ConfigOption<>( diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java index a72726c482..510c590de7 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java @@ -369,7 +369,7 @@ public void truncateBackend() { LockUtil.lock(this.name, LockUtil.GRAPH_LOCK); try { - this.storeProvider.truncate(); + this.storeProvider.truncate(this); this.storeProvider.initSystemInfo(this); this.serverStarted(this.serverInfoManager().selfServerId(), this.serverInfoManager().selfServerRole()); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/auth/EntityManager.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/auth/EntityManager.java index 35e6a99722..c4b668e619 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/auth/EntityManager.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/auth/EntityManager.java @@ -174,7 +174,6 @@ private Id save(T entity, boolean expectExists) { // Add or update user in backend store, stale index might exist vertex = this.tx().addVertex(vertex); - addVertex(, System.User) this.commitOrRollback(); return vertex.id(); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java index eb2f1403eb..df166172a9 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java @@ -175,6 +175,10 @@ public RaftNode node() { return this.raftNode; } + public RpcServer rpcServer() { + return this.rpcServer; + } + public RpcForwarder rpcForwarder() { return this.rpcForwarder; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java index 095f602268..586c457db9 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java @@ -38,6 +38,7 @@ import com.alipay.sofa.jraft.entity.Task; import com.alipay.sofa.jraft.error.RaftError; import com.alipay.sofa.jraft.option.NodeOptions; +import com.alipay.sofa.jraft.rpc.RpcServer; import com.alipay.sofa.jraft.util.BytesUtil; import com.baidu.hugegraph.backend.BackendException; import com.baidu.hugegraph.util.LZ4Util; @@ -48,7 +49,7 @@ public final class RaftNode { private static final Logger LOG = Log.logger(RaftNode.class); private final RaftContext context; - private RaftGroupService raftGroupService; + private final RaftGroupService raftGroupService; private final Node node; private final StoreStateMachine stateMachine; private final AtomicReference leaderInfo; @@ -59,7 +60,9 @@ public RaftNode(RaftContext context) { this.context = context; this.stateMachine = new StoreStateMachine(context); try { - this.node = this.initRaftNode(); + this.raftGroupService = this.initRaftNode(); + // Start node + this.node = this.raftGroupService.start(false); LOG.info("Start raft node: {}", this); } catch (IOException e) { throw new BackendException("Failed to init raft node", e); @@ -111,7 +114,7 @@ public RaftClosure snapshot() { } } - private Node initRaftNode() throws IOException { + private RaftGroupService initRaftNode() throws IOException { NodeOptions nodeOptions = this.context.nodeOptions(); nodeOptions.setFsm(this.stateMachine); /* @@ -122,12 +125,25 @@ private Node initRaftNode() throws IOException { PeerId endpoint = this.context.endpoint(); /* * Start raft node with shared rpc server: - * return new RaftGroupService(groupId, endpoint, nodeOptions, - * this.context.rpcServer(), true) - * .start(false) */ - return RaftServiceFactory.createAndInitRaftNode(groupId, endpoint, - nodeOptions); + RpcServer rpcServer = this.context.rpcServer(); + LOG.info("The raft endpoint '{}', initial group peers [{}]", + endpoint, nodeOptions.getInitialConf()); + // Shared rpc server + return new RaftGroupService(groupId, endpoint, nodeOptions, + rpcServer, true); + } + + public void close() { + if (this.raftGroupService != null) { + this.raftGroupService.shutdown(); + try { + this.raftGroupService.join(); + } catch (final InterruptedException e) { + throw new RaftException("Interrupted while shutdown " + + "raftGroupService"); + } + } } private void submitCommand(StoreCommand command, RaftStoreClosure closure) { @@ -215,12 +231,12 @@ public void run(Status status, long index, byte[] reqCtx) { Thread.sleep(RaftContext.POLL_INTERVAL); } catch (InterruptedException e) { throw new BackendException("Interrupted while waiting for " + - "raft group '%s' log sync", group, e); + "raft group '%s' log synced", group, e); } long consumedTime = System.currentTimeMillis() - beginTime; if (timeout > 0 && consumedTime >= timeout) { throw new BackendException( - "Waiting for raft group '%s' log sync timeout(%sms)", + "Waiting for raft group '%s' log synced timeout(%sms)", group, consumedTime); } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreSnapshotFile.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreSnapshotFile.java index c7fb2a9524..e54fef62f8 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreSnapshotFile.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreSnapshotFile.java @@ -82,7 +82,7 @@ public void save(SnapshotWriter writer, Closure done, // Write snapshot to real directory Map snapshotDirMaps = this.doSnapshotSave(); executor.execute(() -> { - if (this.compressing.get()) { + if (!this.compressing.compareAndSet(false, true)) { LOG.info("Last compress task doesn't finish, skipped it"); done.run(new Status(RaftError.EBUSY, "Last compress task doesn't finish, " + @@ -91,7 +91,6 @@ public void save(SnapshotWriter writer, Closure done, } try { - this.compressing.set(true); this.compressSnapshotDir(writer, snapshotDirMaps); this.deleteSnapshotDirs(snapshotDirMaps.keySet()); done.run(Status.OK()); @@ -101,7 +100,7 @@ public void save(SnapshotWriter writer, Closure done, "Failed to compress snapshot, " + "error is %s", e.getMessage())); } finally { - this.compressing.set(false); + this.compressing.compareAndSet(true, false); } }); } catch (Throwable e) { @@ -116,13 +115,12 @@ public boolean load(SnapshotReader reader) { Set snapshotDirTars = reader.listFiles(); LOG.info("The snapshot tar files to be loaded are {}", snapshotDirTars); Set snapshotDirs = new HashSet<>(); - if (this.compressing.get()) { + if (!this.compressing.compareAndSet(false, true)) { LOG.info("Last decompress task doesn't finish, skipped it"); return false; } try { - this.compressing.set(true); for (String snapshotDirTar : snapshotDirTars) { String snapshotDir = this.decompressSnapshot(reader, snapshotDirTar); @@ -132,7 +130,7 @@ public boolean load(SnapshotReader reader) { LOG.error("Failed to decompress snapshot tar", e); return false; } finally { - this.compressing.set(false); + this.compressing.compareAndSet(true, false); } try { @@ -173,12 +171,12 @@ private void compressSnapshotDir(SnapshotWriter writer, .toString(); Checksum checksum = new CRC64(); try { - LOG.info("Prepare to compress dir {} to {}", + LOG.info("Prepare to compress dir '{}' to '{}'", snapshotDir, outputFile); long begin = System.currentTimeMillis(); CompressUtil.compressZip(snapshotDir, outputFile, checksum); long end = System.currentTimeMillis(); - LOG.info("Compressed dir {} to {}, took {} seconds", + LOG.info("Compressed dir '{}' to '{}', took {} seconds", snapshotDir, outputFile, (end - begin) / 1000.0F); } catch (Throwable e) { throw new RaftException( @@ -223,12 +221,12 @@ private String decompressSnapshot(SnapshotReader reader, Checksum checksum = new CRC64(); String archiveFile = Paths.get(reader.getPath(), snapshotDirTar) .toString(); - LOG.info("Prepare to decompress snapshot zip {} to {}", + LOG.info("Prepare to decompress snapshot zip '{}' to '{}'", archiveFile, parentPath); long begin = System.currentTimeMillis(); CompressUtil.decompressZip(archiveFile, parentPath, checksum); long end = System.currentTimeMillis(); - LOG.info("Decompress snapshot zip {} to {}, took {} seconds", + LOG.info("Decompress snapshot zip '{}' to '{}', took {} seconds", archiveFile, parentPath, (end - begin) / 1000.0F); if (meta.hasChecksum()) { String expected = meta.getChecksum(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java index 02ddfa8432..6c18a78123 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java @@ -236,7 +236,7 @@ public static synchronized CoreOptions instance() { public static final ConfigOption RAFT_INSTALL_SNAPSHOT_TIMEOUT = new ConfigOption<>( - "raft.rpc_timeout", + "raft.install_snapshot_rpc_timeout", "The install snapshot rpc timeout for jraft rpc.", positiveInt(), // jraft default value is 5 * 60 * 1000 From c49999a683fb1ae7b8d93409cf849c256f0f570e Mon Sep 17 00:00:00 2001 From: liningrui Date: Sun, 15 May 2022 16:50:15 +0800 Subject: [PATCH 04/14] address some comments Change-Id: Ic27656f2fb8793d02baf40e720b609b4a6fb9a12 --- .../baidu/hugegraph/core/GraphManager.java | 46 ++++--------------- .../store/raft/RaftBackendStoreProvider.java | 8 +--- .../backend/store/raft/RaftContext.java | 15 +++--- .../backend/store/raft/RaftNode.java | 2 +- 4 files changed, 20 insertions(+), 51 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java index eeeade2b34..ae6495b9f2 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java @@ -108,30 +108,16 @@ public GraphManager(HugeConfig conf, EventHub hub) { // Raft will load snapshot firstly then launch election and replay log this.startRpcServer(); - initAllSystemSchema(); + this.initAllSystemSchema(); - ServerConfig serverConfig = Whitebox.getInternalState(this.rpcServer, - "serverConfig"); com.alipay.remoting.rpc.RpcServer remotingRpcServer; - remotingRpcServer = Whitebox.getInternalState(serverConfig.getServer(), - "remotingServer"); + remotingRpcServer = this.remotingRpcServer(); this.waitGraphsReady(remotingRpcServer); this.checkBackendVersionOrExit(conf); this.serverStarted(conf); this.addMetrics(conf); - } - - public { - // - register(-1, ~task, xx, xx, xx,); - - - - - - } public void loadGraphs(HugeConfig serverConfig) { @@ -302,28 +288,12 @@ private void loadGraph(HugeConfig serverConfig, String name, String path) { } } -// private com.alipay.sofa.jraft.rpc.RpcServer startRaftRpcServer(HugeConfig -// config) { -// Integer lowWaterMark = config.get( -// CoreOptions.RAFT_RPC_BUF_LOW_WATER_MARK); -// System.setProperty("bolt.channel_write_buf_low_water_mark", -// String.valueOf(lowWaterMark)); -// Integer highWaterMark = config.get( -// CoreOptions.RAFT_RPC_BUF_HIGH_WATER_MARK); -// System.setProperty("bolt.channel_write_buf_high_water_mark", -// String.valueOf(highWaterMark)); -// -// PeerId endpoint = new PeerId(); -// String endpointStr = config.get(ServerOptions.RAFT_ENDPOINT); -// if (!endpoint.parse(endpointStr)) { -// throw new HugeException("Failed to parse endpoint %s", endpointStr); -// } -// com.alipay.sofa.jraft.rpc.RpcServer rpcServer; -// rpcServer = RaftRpcServerFactory.createAndStartRaftRpcServer( -// endpoint.getEndpoint()); -// LOG.info("Raft RPC server is started successfully"); -// return rpcServer; -// } + private com.alipay.remoting.rpc.RpcServer remotingRpcServer() { + ServerConfig serverConfig = Whitebox.getInternalState(this.rpcServer, + "serverConfig"); + return Whitebox.getInternalState(serverConfig.getServer(), + "remotingServer"); + } private void startRpcServer() { if (!this.rpcServer.enabled()) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java index 2c24c0d9c1..abf237bedf 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java @@ -28,7 +28,6 @@ import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory; import com.alipay.sofa.jraft.rpc.impl.BoltRpcServer; -import com.baidu.hugegraph.HugeException; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.HugeGraphParams; import com.baidu.hugegraph.backend.BackendException; @@ -75,6 +74,7 @@ public void initRaftContext(HugeGraphParams params, RpcServer rpcServer) { System.setProperty("bolt.channel_write_buf_high_water_mark", String.valueOf(highWaterMark)); + // TODO: pass ServerOptions object to core context // PeerId endpoint = new PeerId(); // String endpointStr = config.get(ServerOptions.RAFT_ENDPOINT); // if (!endpoint.parse(endpointStr)) { @@ -89,8 +89,6 @@ public void initRaftContext(HugeGraphParams params, RpcServer rpcServer) { PeerId endpoint = new PeerId(rpcServer.ip(), rpcServer.port()); this.context = new RaftContext(params, raftRpcServer, endpoint); - - // this.context.addStore(StoreType.SYSTEM, this.systemStore); } @@ -236,9 +234,7 @@ public void initSystemInfo(HugeGraph graph) { BackendStoreSystemInfo info = graph.backendStoreSystemInfo(); info.init(); - // 创建系统schema，保存到pool里面 - init(); -// this.notifyAndWaitEvent(Events.STORE_INITED); + this.init(); LOG.debug("Graph '{}' system info has been initialized", this.graph()); /* * Take the initiative to generate a snapshot, it can avoid this diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java index df166172a9..6a5fbc8d08 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java @@ -118,12 +118,15 @@ public RaftContext(HugeGraphParams params, RpcServer rpcServer, this.systemStoreName = config.get(CoreOptions.STORE_SYSTEM); this.stores = new RaftBackendStore[StoreType.ALL.getNumber()]; -// // TODO: 依赖了ServerOptions的配置项名，需要打通ServerConfig和CoreConfig -// this.endpoint = new PeerId(); -// String endpointStr = config.getString("raft.endpoint"); -// if (!this.endpoint.parse(endpointStr)) { -// throw new HugeException("Failed to parse endpoint %s", endpointStr); -// } + /* + * TODO Depending on the name of the config item for server options, + * need to get through ServerConfig and CoreConfig + */ + // this.endpoint = new PeerId(); + // String endpointStr = config.getString("raft.endpoint"); + // if (!this.endpoint.parse(endpointStr)) { + // throw new HugeException("Failed to parse endpoint %s", endpointStr); + // } this.groupPeers = new Configuration(); String groupPeersStr = config.getString("raft.group_peers"); if (!this.groupPeers.parse(groupPeersStr)) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java index 586c457db9..6f2858cf04 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java @@ -149,7 +149,7 @@ public void close() { private void submitCommand(StoreCommand command, RaftStoreClosure closure) { // Wait leader elected LeaderInfo leaderInfo = this.waitLeaderElected( - RaftContext.WAIT_LEADER_TIMEOUT); + RaftContext.WAIT_LEADER_TIMEOUT); if (!leaderInfo.selfIsLeader) { this.context.rpcForwarder().forwardToLeader(leaderInfo.leaderId, command, closure); From 9df37f0015d41fd58908ae26c82c170f88126bb9 Mon Sep 17 00:00:00 2001 From: Zhangmei Li Date: Sun, 15 May 2022 23:24:53 +0800 Subject: [PATCH 05/14] small fix Change-Id: Ie16b77460133a9b7a5c27922efc82de171b775c7 --- .../baidu/hugegraph/core/GraphManager.java | 56 +++++++++---------- .../store/AbstractBackendStoreProvider.java | 10 +--- .../backend/store/BackendStoreProvider.java | 2 +- .../store/raft/RaftBackendStoreProvider.java | 30 +++------- .../backend/store/raft/RaftContext.java | 36 ++++++------ .../backend/store/raft/RaftNode.java | 49 ++++++++-------- .../backend/store/raft/rpc/RpcForwarder.java | 6 +- .../baidu/hugegraph/config/CoreOptions.java | 20 +++++-- 8 files changed, 97 insertions(+), 112 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java index ae6495b9f2..282f01524a 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java @@ -37,10 +37,7 @@ import org.apache.tinkerpop.gremlin.structure.util.GraphFactory; import org.slf4j.Logger; -import com.alipay.sofa.jraft.entity.PeerId; -import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory; import com.alipay.sofa.rpc.config.ServerConfig; -import com.baidu.hugegraph.HugeException; import com.baidu.hugegraph.HugeFactory; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.auth.AuthManager; @@ -55,7 +52,6 @@ import com.baidu.hugegraph.backend.store.BackendStoreSystemInfo; import com.baidu.hugegraph.config.CoreOptions; import com.baidu.hugegraph.config.HugeConfig; -import com.baidu.hugegraph.config.RpcOptions; import com.baidu.hugegraph.config.ServerOptions; import com.baidu.hugegraph.config.TypedOption; import com.baidu.hugegraph.event.EventHub; @@ -110,9 +106,7 @@ public GraphManager(HugeConfig conf, EventHub hub) { this.initAllSystemSchema(); - com.alipay.remoting.rpc.RpcServer remotingRpcServer; - remotingRpcServer = this.remotingRpcServer(); - this.waitGraphsReady(remotingRpcServer); + this.waitGraphsReady(); this.checkBackendVersionOrExit(conf); @@ -135,13 +129,6 @@ public void loadGraphs(HugeConfig serverConfig) { } } - public void waitGraphsReady(com.alipay.remoting.rpc.RpcServer rpcServer) { - this.graphs.keySet().forEach(name -> { - HugeGraph graph = this.graph(name); - graph.waitReady(rpcServer); - }); - } - public HugeGraph cloneGraph(String name, String newName, String configText) { /* @@ -272,22 +259,6 @@ public void close() { this.unlistenChanges(); } - private void loadGraph(HugeConfig serverConfig, String name, String path) { - HugeConfig config = new HugeConfig(path); - String raftGroupPeers = serverConfig.get(ServerOptions.RAFT_GROUP_PEERS); - config.addProperty(ServerOptions.RAFT_GROUP_PEERS.name(), raftGroupPeers); - - final Graph graph = GraphFactory.open(config); - this.graphs.put(name, graph); - LOG.info("Graph '{}' was successfully configured via '{}'", name, path); - - if (this.requireAuthentication() && - !(graph instanceof HugeGraphAuthProxy)) { - LOG.warn("You may need to support access control for '{}' with {}", - path, HugeFactoryAuthProxy.GRAPH_FACTORY); - } - } - private com.alipay.remoting.rpc.RpcServer remotingRpcServer() { ServerConfig serverConfig = Whitebox.getInternalState(this.rpcServer, "serverConfig"); @@ -369,6 +340,31 @@ private void closeTx(final Set graphSourceNamesToCloseTxOn, }); } + private void loadGraph(HugeConfig serverConfig, String name, String path) { + HugeConfig config = new HugeConfig(path); + String raftGroupPeers = serverConfig.get(ServerOptions.RAFT_GROUP_PEERS); + config.addProperty(ServerOptions.RAFT_GROUP_PEERS.name(), raftGroupPeers); + + final Graph graph = GraphFactory.open(config); + this.graphs.put(name, graph); + LOG.info("Graph '{}' was successfully configured via '{}'", name, path); + + if (this.requireAuthentication() && + !(graph instanceof HugeGraphAuthProxy)) { + LOG.warn("You may need to support access control for '{}' with {}", + path, HugeFactoryAuthProxy.GRAPH_FACTORY); + } + } + + private void waitGraphsReady() { + com.alipay.remoting.rpc.RpcServer remotingRpcServer = + this.remotingRpcServer(); + this.graphs.keySet().forEach(name -> { + HugeGraph graph = this.graph(name); + graph.waitReady(remotingRpcServer); + }); + } + private void checkBackendVersionOrExit(HugeConfig config) { LOG.info("Check backend version"); for (String graph : this.graphs()) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java index 88759f82cd..005d133ba4 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java @@ -23,7 +23,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Future; -import com.baidu.hugegraph.config.HugeConfig; import org.slf4j.Logger; import com.baidu.hugegraph.HugeGraph; @@ -134,15 +133,10 @@ public void clear() throws BackendException { } @Override - public void truncate(HugeGraph graph) { + public void truncate() { this.checkOpened(); - HugeConfig config = (HugeConfig) graph.configuration(); - String systemStoreName = config.get(CoreOptions.STORE_SYSTEM); for (BackendStore store : this.stores.values()) { - // Don't truncate system store - if (!store.store().equals(systemStoreName)) { - store.truncate(); - } + store.truncate(); } this.notifyAndWaitEvent(Events.STORE_TRUNCATE); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java index f248ed1743..a4027e17ac 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java @@ -52,7 +52,7 @@ public interface BackendStoreProvider { public void clear(); - public void truncate(HugeGraph graph); + public void truncate(); public void initSystemInfo(HugeGraph graph); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java index abf237bedf..30e8f751ba 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java @@ -64,6 +64,7 @@ public RaftBackendStoreProvider(BackendStoreProvider provider) { } public void initRaftContext(HugeGraphParams params, RpcServer rpcServer) { + // TODO: pass ServerOptions instead of CoreOptions, to share by graphs HugeConfig config = params.configuration(); Integer lowWaterMark = config.get( CoreOptions.RAFT_RPC_BUF_LOW_WATER_MARK); @@ -74,13 +75,6 @@ public void initRaftContext(HugeGraphParams params, RpcServer rpcServer) { System.setProperty("bolt.channel_write_buf_high_water_mark", String.valueOf(highWaterMark)); - // TODO: pass ServerOptions object to core context -// PeerId endpoint = new PeerId(); -// String endpointStr = config.get(ServerOptions.RAFT_ENDPOINT); -// if (!endpoint.parse(endpointStr)) { -// throw new HugeException("Failed to parse endpoint %s", endpointStr); -// } - // Reference from RaftRpcServerFactory.createAndStartRaftRpcServer com.alipay.sofa.jraft.rpc.RpcServer raftRpcServer = new BoltRpcServer(rpcServer); @@ -213,15 +207,10 @@ public void clear() { } @Override - public void truncate(HugeGraph graph) { + public void truncate() { this.checkOpened(); - HugeConfig config = (HugeConfig) graph.configuration(); - String systemStoreName = config.get(CoreOptions.STORE_SYSTEM); for (RaftBackendStore store : this.stores()) { - // Don't truncate system store - if (!store.store().equals(systemStoreName)) { - store.truncate(); - } + store.truncate(); } this.notifyAndWaitEvent(Events.STORE_TRUNCATE); @@ -258,13 +247,12 @@ public void createSnapshot() { RaftStoreClosure closure = new RaftStoreClosure(command); RaftClosure future = this.context.node().submitAndWait(command, closure); - if (future != null) { - try { - future.waitFinished(); - LOG.debug("Graph '{}' has writed snapshot", this.graph()); - } catch (Throwable e) { - throw new BackendException("Failed to create snapshot", e); - } + E.checkState(future != null, "The snapshot future can't be null"); + try { + future.waitFinished(); + LOG.debug("Graph '{}' has writed snapshot", this.graph()); + } catch (Throwable e) { + throw new BackendException("Failed to create snapshot", e); } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java index 6a5fbc8d08..381a37f6bd 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java @@ -97,7 +97,6 @@ public final class RaftContext { private final String systemStoreName; private final RaftBackendStore[] stores; private final Configuration groupPeers; - @SuppressWarnings("unused") private final ExecutorService readIndexExecutor; private final ExecutorService snapshotExecutor; private final ExecutorService backendExecutor; @@ -119,19 +118,14 @@ public RaftContext(HugeGraphParams params, RpcServer rpcServer, this.stores = new RaftBackendStore[StoreType.ALL.getNumber()]; /* - * TODO Depending on the name of the config item for server options, - * need to get through ServerConfig and CoreConfig + * NOTE: `raft.group_peers` option is transfered from ServerConfig + * to CoreConfig, since it's shared by all graphs. */ - // this.endpoint = new PeerId(); - // String endpointStr = config.getString("raft.endpoint"); - // if (!this.endpoint.parse(endpointStr)) { - // throw new HugeException("Failed to parse endpoint %s", endpointStr); - // } this.groupPeers = new Configuration(); - String groupPeersStr = config.getString("raft.group_peers"); - if (!this.groupPeers.parse(groupPeersStr)) { - throw new HugeException("Failed to parse group peers %s", - groupPeersStr); + String groupPeersString = config.getString("raft.group_peers"); + if (!this.groupPeers.parse(groupPeersString)) { + throw new HugeException("Failed to parse raft.group_peers: '%s'", + groupPeersString); } if (config.get(CoreOptions.RAFT_SAFE_READ)) { @@ -140,9 +134,12 @@ public RaftContext(HugeGraphParams params, RpcServer rpcServer, } else { this.readIndexExecutor = null; } - this.snapshotExecutor = this.createSnapshotExecutor(4); - int backendThreads = config.get(CoreOptions.RAFT_BACKEND_THREADS); - this.backendExecutor = this.createBackendExecutor(backendThreads); + + int threads = config.get(CoreOptions.RAFT_SNAPSHOT_THREADS); + this.snapshotExecutor = this.createSnapshotExecutor(threads); + + threads = config.get(CoreOptions.RAFT_BACKEND_THREADS); + this.backendExecutor = this.createBackendExecutor(threads); this.raftNode = null; this.raftGroupManager = null; @@ -231,9 +228,9 @@ public NodeOptions nodeOptions() throws IOException { nodeOptions.setRpcConnectTimeoutMs( config.get(CoreOptions.RAFT_RPC_CONNECT_TIMEOUT)); nodeOptions.setRpcDefaultTimeout( - config.get(CoreOptions.RAFT_RPC_TIMEOUT)); + 1000 * config.get(CoreOptions.RAFT_RPC_TIMEOUT)); nodeOptions.setRpcInstallSnapshotTimeout( - config.get(CoreOptions.RAFT_INSTALL_SNAPSHOT_TIMEOUT)); + 1000 * config.get(CoreOptions.RAFT_INSTALL_SNAPSHOT_TIMEOUT)); int electionTimeout = config.get(CoreOptions.RAFT_ELECTION_TIMEOUT); nodeOptions.setElectionTimeoutMs(electionTimeout); @@ -363,6 +360,8 @@ public GraphMode graphMode() { private HugeConfig config() { return this.params.configuration(); } + + @SuppressWarnings("unused") private RpcServer initAndStartRpcServer() { Integer lowWaterMark = this.config().get( CoreOptions.RAFT_RPC_BUF_LOW_WATER_MARK); @@ -377,7 +376,7 @@ private RpcServer initAndStartRpcServer() { NodeManager.getInstance().addAddress(endpoint.getEndpoint()); RpcServer rpcServer = RaftRpcServerFactory.createAndStartRaftRpcServer( endpoint.getEndpoint()); - LOG.info("RPC server is started successfully"); + LOG.info("Raft-RPC server is started successfully"); return rpcServer; } @@ -392,6 +391,7 @@ private void registerRpcRequestProcessors() { this.rpcServer.registerProcessor(new SetLeaderProcessor(this)); this.rpcServer.registerProcessor(new ListPeersProcessor(this)); } + private ExecutorService createReadIndexExecutor(int coreThreads) { int maxThreads = coreThreads << 2; String name = "store-read-index-callback"; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java index 6f2858cf04..e1d0fe7c2f 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftNode.java @@ -30,7 +30,6 @@ import com.alipay.sofa.jraft.Node; import com.alipay.sofa.jraft.RaftGroupService; -import com.alipay.sofa.jraft.RaftServiceFactory; import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.closure.ReadIndexClosure; import com.alipay.sofa.jraft.core.Replicator.ReplicatorStateListener; @@ -49,7 +48,7 @@ public final class RaftNode { private static final Logger LOG = Log.logger(RaftNode.class); private final RaftContext context; - private final RaftGroupService raftGroupService; + private RaftGroupService raftGroupService; private final Node node; private final StoreStateMachine stateMachine; private final AtomicReference leaderInfo; @@ -60,9 +59,8 @@ public RaftNode(RaftContext context) { this.context = context; this.stateMachine = new StoreStateMachine(context); try { - this.raftGroupService = this.initRaftNode(); - // Start node - this.node = this.raftGroupService.start(false); + // Start raft node + this.node = this.initRaftNode(); LOG.info("Start raft node: {}", this); } catch (IOException e) { throw new BackendException("Failed to init raft node", e); @@ -102,6 +100,16 @@ public void onLeaderInfoChange(PeerId leaderId, boolean selfIsLeader) { public void shutdown() { LOG.info("Shutdown raft node: {}", this); this.node.shutdown(); + + if (this.raftGroupService != null) { + this.raftGroupService.shutdown(); + try { + this.raftGroupService.join(); + } catch (final InterruptedException e) { + throw new RaftException( + "Interrupted while shutdown raftGroupService"); + } + } } public RaftClosure snapshot() { @@ -114,36 +122,27 @@ public RaftClosure snapshot() { } } - private RaftGroupService initRaftNode() throws IOException { + private Node initRaftNode() throws IOException { NodeOptions nodeOptions = this.context.nodeOptions(); nodeOptions.setFsm(this.stateMachine); /* * TODO: the groupId is same as graph name now, when support sharding, - * groupId needs to be bound to shard Id + * groupId needs to be bound to shard Id */ String groupId = this.context.group(); PeerId endpoint = this.context.endpoint(); /* - * Start raft node with shared rpc server: + * Create RaftGroupService with shared rpc-server, then start raft node + * TODO: don't create + hold RaftGroupService and just share rpc-server + * and create Node by RaftServiceFactory.createAndInitRaftNode() */ RpcServer rpcServer = this.context.rpcServer(); - LOG.info("The raft endpoint '{}', initial group peers [{}]", - endpoint, nodeOptions.getInitialConf()); - // Shared rpc server - return new RaftGroupService(groupId, endpoint, nodeOptions, - rpcServer, true); - } - - public void close() { - if (this.raftGroupService != null) { - this.raftGroupService.shutdown(); - try { - this.raftGroupService.join(); - } catch (final InterruptedException e) { - throw new RaftException("Interrupted while shutdown " + - "raftGroupService"); - } - } + LOG.debug("Start raft node with endpoint '{}', initial conf [{}]", + endpoint, nodeOptions.getInitialConf()); + this.raftGroupService = new RaftGroupService(groupId, endpoint, + nodeOptions, + rpcServer, true); + return this.raftGroupService.start(false); } private void submitCommand(StoreCommand command, RaftStoreClosure closure) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/RpcForwarder.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/RpcForwarder.java index 110fbbb086..551a0b1156 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/RpcForwarder.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/rpc/RpcForwarder.java @@ -19,8 +19,6 @@ package com.baidu.hugegraph.backend.store.raft.rpc; -import static com.baidu.hugegraph.backend.store.raft.RaftContext.WAIT_RPC_TIMEOUT; - import java.util.concurrent.ExecutionException; import org.slf4j.Logger; @@ -34,6 +32,7 @@ import com.alipay.sofa.jraft.util.Endpoint; import com.baidu.hugegraph.backend.BackendException; import com.baidu.hugegraph.backend.store.raft.RaftClosure; +import com.baidu.hugegraph.backend.store.raft.RaftContext; import com.baidu.hugegraph.backend.store.raft.RaftNode; import com.baidu.hugegraph.backend.store.raft.RaftStoreClosure; import com.baidu.hugegraph.backend.store.raft.StoreCommand; @@ -151,7 +150,8 @@ private void waitRpc(Endpoint endpoint, Message request, E.checkNotNull(endpoint, "leader endpoint"); try { this.rpcClient.invokeWithDone(endpoint, request, done, - WAIT_RPC_TIMEOUT).get(); + RaftContext.WAIT_RPC_TIMEOUT) + .get(); } catch (InterruptedException e) { throw new BackendException("Invoke rpc request was interrupted, " + "please try again later", e); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java index 6c18a78123..cf8adaca0d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java @@ -155,6 +155,14 @@ public static synchronized CoreOptions instance() { 3600 ); + public static final ConfigOption RAFT_SNAPSHOT_THREADS = + new ConfigOption<>( + "raft.snapshot_threads", + "The thread number used to do snapshot.", + rangeInt(0, Integer.MAX_VALUE), + 4 + ); + public static final ConfigOption RAFT_BACKEND_THREADS = new ConfigOption<>( "raft.backend_threads", @@ -228,19 +236,19 @@ public static synchronized CoreOptions instance() { public static final ConfigOption RAFT_RPC_TIMEOUT = new ConfigOption<>( "raft.rpc_timeout", - "The general rpc timeout for jraft rpc.", + "The general rpc timeout in seconds for jraft rpc.", positiveInt(), - // jraft default value is 5000(ms) - 60 * 1000 + // jraft default value is 5s + 60 ); public static final ConfigOption RAFT_INSTALL_SNAPSHOT_TIMEOUT = new ConfigOption<>( "raft.install_snapshot_rpc_timeout", - "The install snapshot rpc timeout for jraft rpc.", + "The install snapshot rpc timeout in seconds for jraft rpc.", positiveInt(), - // jraft default value is 5 * 60 * 1000 - 24 * 60 * 60 * 1000 + // jraft default value is 5 minutes + 24 * 60 * 60 ); public static final ConfigOption RAFT_RPC_BUF_LOW_WATER_MARK = From 03903e36cd76c4931c098b06dc4b6c94fe2ce5e7 Mon Sep 17 00:00:00 2001 From: Zhangmei Li Date: Mon, 16 May 2022 00:21:38 +0800 Subject: [PATCH 06/14] fix truncate Change-Id: I86348c0ce7be613150ec097adfe8276f0cd8a22a --- .../src/main/java/com/baidu/hugegraph/StandardHugeGraph.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java index d00a695469..00a2242345 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java @@ -369,7 +369,7 @@ public void truncateBackend() { LockUtil.lock(this.name, LockUtil.GRAPH_LOCK); try { - this.storeProvider.truncate(this); + this.storeProvider.truncate(); this.storeProvider.initSystemInfo(this); this.serverStarted(this.serverInfoManager().selfServerId(), this.serverInfoManager().selfServerRole()); From d5aed6565ff30d2b4b026a81e0e48ef1c22e6f70 Mon Sep 17 00:00:00 2001 From: Zhangmei Li Date: Mon, 16 May 2022 00:22:20 +0800 Subject: [PATCH 07/14] improve initRaftContext() Change-Id: Ida94cbd278b340e784baf0d85439789236c300d4 --- .../store/raft/RaftBackendStoreProvider.java | 25 +----------- .../backend/store/raft/RaftContext.java | 40 ++++++++++++++----- 2 files changed, 32 insertions(+), 33 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java index 30e8f751ba..e48da80da8 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java @@ -25,9 +25,6 @@ import org.slf4j.Logger; import com.alipay.remoting.rpc.RpcServer; -import com.alipay.sofa.jraft.entity.PeerId; -import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory; -import com.alipay.sofa.jraft.rpc.impl.BoltRpcServer; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.HugeGraphParams; import com.baidu.hugegraph.backend.BackendException; @@ -36,7 +33,6 @@ import com.baidu.hugegraph.backend.store.BackendStoreSystemInfo; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreAction; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreType; -import com.baidu.hugegraph.config.CoreOptions; import com.baidu.hugegraph.config.HugeConfig; import com.baidu.hugegraph.event.EventHub; import com.baidu.hugegraph.event.EventListener; @@ -64,25 +60,8 @@ public RaftBackendStoreProvider(BackendStoreProvider provider) { } public void initRaftContext(HugeGraphParams params, RpcServer rpcServer) { - // TODO: pass ServerOptions instead of CoreOptions, to share by graphs - HugeConfig config = params.configuration(); - Integer lowWaterMark = config.get( - CoreOptions.RAFT_RPC_BUF_LOW_WATER_MARK); - System.setProperty("bolt.channel_write_buf_low_water_mark", - String.valueOf(lowWaterMark)); - Integer highWaterMark = config.get( - CoreOptions.RAFT_RPC_BUF_HIGH_WATER_MARK); - System.setProperty("bolt.channel_write_buf_high_water_mark", - String.valueOf(highWaterMark)); - - // Reference from RaftRpcServerFactory.createAndStartRaftRpcServer - com.alipay.sofa.jraft.rpc.RpcServer raftRpcServer = - new BoltRpcServer(rpcServer); - RaftRpcServerFactory.addRaftRequestProcessors(raftRpcServer); - raftRpcServer.init(null); - - PeerId endpoint = new PeerId(rpcServer.ip(), rpcServer.port()); - this.context = new RaftContext(params, raftRpcServer, endpoint); + this.context = new RaftContext(params, rpcServer); + // NOTE: loadSystemStore() did not addStore() this.context.addStore(StoreType.SYSTEM, this.systemStore); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java index 5699fbfeac..8ee931e3f4 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java @@ -42,6 +42,7 @@ import com.alipay.sofa.jraft.option.ReadOnlyOption; import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory; import com.alipay.sofa.jraft.rpc.RpcServer; +import com.alipay.sofa.jraft.rpc.impl.BoltRpcServer; import com.alipay.sofa.jraft.util.NamedThreadFactory; import com.alipay.sofa.jraft.util.ThreadPoolUtil; import com.baidu.hugegraph.HugeException; @@ -89,7 +90,7 @@ public final class RaftContext { public static final long KEEP_ALIVE_SECOND = 300L; private final HugeGraphParams params; - private final RpcServer rpcServer; + private final RpcServer raftRpcServer; private final PeerId endpoint; private final String schemaStoreName; @@ -105,11 +106,11 @@ public final class RaftContext { private RaftGroupManager raftGroupManager; private RpcForwarder rpcForwarder; - public RaftContext(HugeGraphParams params, RpcServer rpcServer, - PeerId endpoint) { + public RaftContext(HugeGraphParams params, + com.alipay.remoting.rpc.RpcServer rpcServer) { this.params = params; - this.rpcServer = rpcServer; - this.endpoint = endpoint; + this.raftRpcServer = this.wrapRpcServer(rpcServer); + this.endpoint = new PeerId(rpcServer.ip(), rpcServer.port());; HugeConfig config = params.configuration(); this.schemaStoreName = config.get(CoreOptions.STORE_SCHEMA); @@ -176,7 +177,7 @@ public RaftNode node() { } public RpcServer rpcServer() { - return this.rpcServer; + return this.raftRpcServer; } public RpcForwarder rpcForwarder() { @@ -380,16 +381,35 @@ private RpcServer initAndStartRpcServer() { return rpcServer; } + private RpcServer wrapRpcServer(com.alipay.remoting.rpc.RpcServer rpcServer) { + // TODO: pass ServerOptions instead of CoreOptions, to share by graphs + Integer lowWaterMark = this.config().get( + CoreOptions.RAFT_RPC_BUF_LOW_WATER_MARK); + System.setProperty("bolt.channel_write_buf_low_water_mark", + String.valueOf(lowWaterMark)); + Integer highWaterMark = this.config().get( + CoreOptions.RAFT_RPC_BUF_HIGH_WATER_MARK); + System.setProperty("bolt.channel_write_buf_high_water_mark", + String.valueOf(highWaterMark)); + + // Reference from RaftRpcServerFactory.createAndStartRaftRpcServer + RpcServer raftRpcServer = new BoltRpcServer(rpcServer); + RaftRpcServerFactory.addRaftRequestProcessors(raftRpcServer); + raftRpcServer.init(null); + + return raftRpcServer; + } + private void shutdownRpcServer() { - this.rpcServer.shutdown(); + this.raftRpcServer.shutdown(); PeerId endpoint = this.endpoint(); NodeManager.getInstance().removeAddress(endpoint.getEndpoint()); } private void registerRpcRequestProcessors() { - this.rpcServer.registerProcessor(new StoreCommandProcessor(this)); - this.rpcServer.registerProcessor(new SetLeaderProcessor(this)); - this.rpcServer.registerProcessor(new ListPeersProcessor(this)); + this.raftRpcServer.registerProcessor(new StoreCommandProcessor(this)); + this.raftRpcServer.registerProcessor(new SetLeaderProcessor(this)); + this.raftRpcServer.registerProcessor(new ListPeersProcessor(this)); } private ExecutorService createReadIndexExecutor(int coreThreads) { From c1ce75f398906c5b87ef0e00dde442cfc2ea8698 Mon Sep 17 00:00:00 2001 From: Zhangmei Li Date: Mon, 16 May 2022 02:18:48 +0800 Subject: [PATCH 08/14] fix loadGraphs/loadGraph Change-Id: Ib86e892929bfa1a1d88ec1ceb1bf43f962437794 --- .../baidu/hugegraph/core/GraphManager.java | 57 ++++++++++--------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java index 079ff4f5b1..bf3bc8d7ed 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java @@ -98,33 +98,33 @@ public GraphManager(HugeConfig conf, EventHub hub) { this.rpcClient = new RpcClientProvider(conf); this.eventHub = hub; this.conf = conf; + this.listenChanges(); this.loadGraphs(ConfigUtil.scanGraphsDir(this.graphsDir)); // this.installLicense(conf, ""); - // Raft will load snapshot firstly then launch election and replay log - this.startRpcServer(); - this.initAllSystemSchema(); + // Start RPC-Server for raft-rpc/auth-rpc/cache-notify-rpc... + this.startRpcServer(); + // Raft will load snapshot firstly then launch election and replay log this.waitGraphsReady(); this.checkBackendVersionOrExit(conf); - this.serverStarted(conf); + this.addMetrics(conf); } - public void loadGraphs(HugeConfig serverConfig) { - Map graphConfs = serverConfig.getMap( - ServerOptions.GRAPHS); + public void loadGraphs(Map graphConfs) { for (Map.Entry conf : graphConfs.entrySet()) { String name = conf.getKey(); - String path = conf.getValue(); + String graphConfPath = conf.getValue(); HugeFactory.checkGraphName(name, "rest-server.properties"); try { - this.loadGraph(serverConfig, name, path); + this.loadGraph(name, graphConfPath); } catch (RuntimeException e) { - LOG.error("Graph '{}' can't be loaded: '{}'", name, path, e); + LOG.error("Graph '{}' can't be loaded: '{}'", + name, graphConfPath, e); } } } @@ -259,13 +259,6 @@ public void close() { this.unlistenChanges(); } - private com.alipay.remoting.rpc.RpcServer remotingRpcServer() { - ServerConfig serverConfig = Whitebox.getInternalState(this.rpcServer, - "serverConfig"); - return Whitebox.getInternalState(serverConfig.getServer(), - "remotingServer"); - } - private void startRpcServer() { if (!this.rpcServer.enabled()) { LOG.info("RpcServer is not enabled, skip starting rpc service"); @@ -298,6 +291,13 @@ private void startRpcServer() { } } + private com.alipay.remoting.rpc.RpcServer remotingRpcServer() { + ServerConfig serverConfig = Whitebox.getInternalState(this.rpcServer, + "serverConfig"); + return Whitebox.getInternalState(serverConfig.getServer(), + "remotingServer"); + } + private void destroyRpcServer() { try { this.rpcClient.destroy(); @@ -340,21 +340,26 @@ private void closeTx(final Set graphSourceNamesToCloseTxOn, }); } - private void loadGraph(HugeConfig serverConfig, String name, String path) { - HugeConfig config = new HugeConfig(path); - String raftGroupPeers = serverConfig.get(ServerOptions.RAFT_GROUP_PEERS); - config.addProperty(ServerOptions.RAFT_GROUP_PEERS.name(), raftGroupPeers); + private void loadGraph(String name, String graphConfPath) { + HugeConfig config = new HugeConfig(graphConfPath); - final Graph graph = GraphFactory.open(config); + String raftGroupPeers = this.conf.get(ServerOptions.RAFT_GROUP_PEERS); + config.addProperty(ServerOptions.RAFT_GROUP_PEERS.name(), + raftGroupPeers); + + Graph graph = GraphFactory.open(config); this.graphs.put(name, graph); - HugeConfig config = (HugeConfig) graph.configuration(); - config.file(path); - LOG.info("Graph '{}' was successfully configured via '{}'", name, path); + + HugeConfig graphConfig = (HugeConfig) graph.configuration(); + assert graphConfPath.equals(graphConfig.file().getPath()); + + LOG.info("Graph '{}' was successfully configured via '{}'", + name, graphConfPath); if (this.requireAuthentication() && !(graph instanceof HugeGraphAuthProxy)) { LOG.warn("You may need to support access control for '{}' with {}", - path, HugeFactoryAuthProxy.GRAPH_FACTORY); + graphConfPath, HugeFactoryAuthProxy.GRAPH_FACTORY); } } From 75b75d33259a2a0b5a7683ae10890b9b2db7d76d Mon Sep 17 00:00:00 2001 From: Zhangmei Li Date: Mon, 16 May 2022 02:19:43 +0800 Subject: [PATCH 09/14] resume Events.STORE_INITED Change-Id: I628bcb4dc3545bb702b4cac3ebe151e5fd9f8e54 --- .../hugegraph/backend/store/raft/RaftBackendStoreProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java index e48da80da8..2157911e83 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java @@ -202,7 +202,7 @@ public void initSystemInfo(HugeGraph graph) { BackendStoreSystemInfo info = graph.backendStoreSystemInfo(); info.init(); - this.init(); + this.notifyAndWaitEvent(Events.STORE_INITED); LOG.debug("Graph '{}' system info has been initialized", this.graph()); /* * Take the initiative to generate a snapshot, it can avoid this From d9a5a9f5600f95a6d96192fc8da321261942f695 Mon Sep 17 00:00:00 2001 From: Zhangmei Li Date: Mon, 16 May 2022 02:20:42 +0800 Subject: [PATCH 10/14] fix HugeGraphServer start Change-Id: I9f4506a1bc4115d010255b584e8bbfa7d20fe67c --- .../baidu/hugegraph/dist/HugeGraphServer.java | 35 ++++++------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java b/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java index 86fe9ffca4..602f20ecdc 100644 --- a/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java +++ b/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/HugeGraphServer.java @@ -19,6 +19,8 @@ package com.baidu.hugegraph.dist; +import java.util.concurrent.CompletableFuture; + import org.apache.tinkerpop.gremlin.server.GremlinServer; import org.slf4j.Logger; @@ -31,8 +33,6 @@ import com.baidu.hugegraph.util.ConfigUtil; import com.baidu.hugegraph.util.Log; -import java.util.concurrent.CompletableFuture; - public class HugeGraphServer { private static final Logger LOG = Log.logger(HugeGraphServer.class); @@ -48,14 +48,6 @@ public static void register() { public HugeGraphServer(String gremlinServerConf, String restServerConf) throws Exception { - try { - // Start HugeRestServer - this.restServer = HugeRestServer.start(restServerConf); - } catch (Throwable e) { - LOG.error("HugeRestServer start error: ", e); - throw e; - } - // Only switch on security manager after HugeGremlinServer started SecurityManager securityManager = System.getSecurityManager(); System.setSecurityManager(null); @@ -64,6 +56,15 @@ public HugeGraphServer(String gremlinServerConf, String restServerConf) HugeConfig restServerConfig = new HugeConfig(restServerConf); String graphsDir = restServerConfig.get(ServerOptions.GRAPHS); EventHub hub = new EventHub("gremlin=>hub<=rest"); + + try { + // Start HugeRestServer + this.restServer = HugeRestServer.start(restServerConf, hub); + } catch (Throwable e) { + LOG.error("HugeRestServer start error: ", e); + throw e; + } + try { // Start GremlinServer this.gremlinServer = HugeGremlinServer.start(gremlinServerConf, @@ -80,20 +81,6 @@ public HugeGraphServer(String gremlinServerConf, String restServerConf) } finally { System.setSecurityManager(securityManager); } - - try { - // Start HugeRestServer - this.restServer = HugeRestServer.start(restServerConf, hub); - } catch (Throwable e) { - LOG.error("HugeRestServer start error: ", e); - try { - this.gremlinServer.stop().get(); - } catch (Throwable t) { - LOG.error("GremlinServer stop error: ", t); - } - HugeFactory.shutdown(30L); - throw e; - } } public void stop() { From 1814a756899194371ffcedb3eb1767ada1bc23a3 Mon Sep 17 00:00:00 2001 From: Zhangmei Li Date: Mon, 16 May 2022 02:27:59 +0800 Subject: [PATCH 11/14] improve options Change-Id: I68517141d8bbc6a9b2991849fda5fc686f23b513 --- .../com/baidu/hugegraph/config/ServerOptions.java | 12 ++---------- .../java/com/baidu/hugegraph/config/CoreOptions.java | 2 +- .../assembly/static/conf/graphs/hugegraph.properties | 7 +++---- .../src/assembly/static/conf/rest-server.properties | 10 ++++++---- 4 files changed, 12 insertions(+), 19 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java index bd3f4bcfac..c132edcd22 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java @@ -177,20 +177,12 @@ public static synchronized ServerOptions instance() { nonNegativeInt(), 0); - public static final ConfigOption RAFT_ENDPOINT = - new ConfigOption<>( - "raft.endpoint", - "The endpoint of current raft node.", - disallowEmpty(), - "127.0.0.1:8281" - ); - public static final ConfigOption RAFT_GROUP_PEERS = new ConfigOption<>( "raft.group_peers", - "The initial peers of current raft group.", + "The rpc address of raft group initial peers.", disallowEmpty(), - "127.0.0.1:8281" + "127.0.0.1:8090" ); public static final ConfigOption ALLOW_TRACE = diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java index cf8adaca0d..e158a4e3f2 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java @@ -248,7 +248,7 @@ public static synchronized CoreOptions instance() { "The install snapshot rpc timeout in seconds for jraft rpc.", positiveInt(), // jraft default value is 5 minutes - 24 * 60 * 60 + 10 * 60 * 60 ); public static final ConfigOption RAFT_RPC_BUF_LOW_WATER_MARK = diff --git a/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties b/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties index 9dc75f7244..df7963fa89 100644 --- a/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties +++ b/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties @@ -26,22 +26,21 @@ store=hugegraph raft.mode=false raft.safe_read=false -raft.use_snapshot=false -raft.endpoint=127.0.0.1:8281 -raft.group_peers=127.0.0.1:8281,127.0.0.1:8282,127.0.0.1:8283 raft.path=./raft-log raft.use_replicator_pipeline=true raft.election_timeout=10000 raft.snapshot_interval=3600 raft.backend_threads=48 raft.read_index_threads=8 +raft.snapshot_threads=4 raft.read_strategy=ReadOnlyLeaseBased raft.queue_size=16384 raft.queue_publish_timeout=60 raft.apply_batch=1 raft.rpc_threads=80 raft.rpc_connect_timeout=5000 -raft.rpc_timeout=60000 +raft.rpc_timeout=60 +raft.install_snapshot_rpc_timeout=36000 search.text_analyzer=jieba search.text_analyzer_mode=INDEX diff --git a/hugegraph-dist/src/assembly/static/conf/rest-server.properties b/hugegraph-dist/src/assembly/static/conf/rest-server.properties index 1666946dda..f7eda0010c 100644 --- a/hugegraph-dist/src/assembly/static/conf/rest-server.properties +++ b/hugegraph-dist/src/assembly/static/conf/rest-server.properties @@ -22,20 +22,22 @@ batch.max_write_threads=0 #auth.admin_token= #auth.user_tokens=[] -# rpc group configs of multi graph servers -# rpc server configs +# rpc server configs for multi graph-servers or raft-servers rpc.server_host=127.0.0.1 -rpc.server_port=8090 +rpc.server_port=8091 #rpc.server_timeout=30 # rpc client configs (like enable to keep cache consistency) -rpc.remote_url=127.0.0.1:8090 +#rpc.remote_url=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093 #rpc.client_connect_timeout=20 #rpc.client_reconnect_period=10 #rpc.client_read_timeout=40 #rpc.client_retries=3 #rpc.client_load_balancer=consistentHash +# raft group initial peers +#raft.group_peers=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093 + # lightweight load balancing (beta) server.id=server-1 server.role=master From 6ae318585775c94fadd82e1a842ecb0918ff6e65 Mon Sep 17 00:00:00 2001 From: Zhangmei Li Date: Mon, 16 May 2022 02:54:02 +0800 Subject: [PATCH 12/14] fix RaftBackendStoreProvider.close() context is null Change-Id: I6be1a445412693701e8f10f5f3e8597000497530 --- .../store/raft/RaftBackendStoreProvider.java | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java index 2157911e83..2b159b309e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java @@ -66,7 +66,14 @@ public void initRaftContext(HugeGraphParams params, RpcServer rpcServer) { } public RaftGroupManager raftNodeManager() { - return this.context.raftNodeManager(); + return this.context().raftNodeManager(); + } + + private RaftContext context() { + if (this.context == null) { + E.checkState(false, "Please ensure init raft context"); + } + return this.context; } private Set stores() { @@ -109,8 +116,8 @@ public synchronized BackendStore loadSchemaStore(HugeConfig config, String name) LOG.info("Init raft backend schema store"); BackendStore store = this.provider.loadSchemaStore(config, name); this.checkNonSharedStore(store); - this.schemaStore = new RaftBackendStore(store, this.context); - this.context.addStore(StoreType.SCHEMA, this.schemaStore); + this.schemaStore = new RaftBackendStore(store, this.context()); + this.context().addStore(StoreType.SCHEMA, this.schemaStore); } return this.schemaStore; } @@ -121,8 +128,8 @@ public synchronized BackendStore loadGraphStore(HugeConfig config, String name) LOG.info("Init raft backend graph store"); BackendStore store = this.provider.loadGraphStore(config, name); this.checkNonSharedStore(store); - this.graphStore = new RaftBackendStore(store, this.context); - this.context.addStore(StoreType.GRAPH, this.graphStore); + this.graphStore = new RaftBackendStore(store, this.context()); + this.context().addStore(StoreType.GRAPH, this.graphStore); } return this.graphStore; } @@ -133,7 +140,7 @@ public synchronized BackendStore loadSystemStore(HugeConfig config, String name) LOG.info("Init raft backend system store"); BackendStore store = this.provider.loadSystemStore(config, name); this.checkNonSharedStore(store); - this.systemStore = new RaftBackendStore(store, this.context); + this.systemStore = new RaftBackendStore(store, this.context()); } return this.systemStore; } @@ -145,17 +152,19 @@ public void open(String name) { @Override public void waitStoreStarted() { - this.context.initRaftNode(); + this.context().initRaftNode(); LOG.info("The raft node is initialized"); - this.context.waitRaftNodeStarted(); + this.context().waitRaftNodeStarted(); LOG.info("The raft store is started"); } @Override public void close() { this.provider.close(); - this.context.close(); + if (this.context != null) { + this.context.close(); + } } @Override @@ -224,8 +233,8 @@ public void createSnapshot() { StoreCommand command = new StoreCommand(StoreType.GRAPH, StoreAction.SNAPSHOT, null); RaftStoreClosure closure = new RaftStoreClosure(command); - RaftClosure future = this.context.node().submitAndWait(command, - closure); + RaftClosure future = this.context().node().submitAndWait(command, + closure); E.checkState(future != null, "The snapshot future can't be null"); try { future.waitFinished(); From 0f4d154706513df3be88a562c25f7ab87c278a50 Mon Sep 17 00:00:00 2001 From: Zhangmei Li Date: Mon, 16 May 2022 02:54:48 +0800 Subject: [PATCH 13/14] fix test: raft.group_peers Change-Id: Ib128b3221ee3acafa757069409625888790e14cf --- .../static/conf/graphs/hugegraph.properties | 2 +- .../conf-raft1/graphs/hugegraph.properties | 17 +---------------- .../travis/conf-raft1/rest-server.properties | 2 ++ .../conf-raft2/graphs/hugegraph.properties | 17 +---------------- .../travis/conf-raft2/rest-server.properties | 2 ++ .../conf-raft3/graphs/hugegraph.properties | 17 +---------------- .../travis/conf-raft3/rest-server.properties | 2 ++ .../assembly/travis/run-api-test-for-raft.sh | 2 +- 8 files changed, 11 insertions(+), 50 deletions(-) diff --git a/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties b/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties index df7963fa89..94af261455 100644 --- a/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties +++ b/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties @@ -25,8 +25,8 @@ serializer=binary store=hugegraph raft.mode=false -raft.safe_read=false raft.path=./raft-log +raft.safe_read=true raft.use_replicator_pipeline=true raft.election_timeout=10000 raft.snapshot_interval=3600 diff --git a/hugegraph-dist/src/assembly/travis/conf-raft1/graphs/hugegraph.properties b/hugegraph-dist/src/assembly/travis/conf-raft1/graphs/hugegraph.properties index 0f043b44dc..33150b467b 100644 --- a/hugegraph-dist/src/assembly/travis/conf-raft1/graphs/hugegraph.properties +++ b/hugegraph-dist/src/assembly/travis/conf-raft1/graphs/hugegraph.properties @@ -10,20 +10,5 @@ rocksdb.data_path=rocksdb-data-raft1 rocksdb.wal_path=rocksdb-data-raft1 raft.mode=true -raft.safe_read=true -raft.use_snapshot=false -raft.endpoint=127.0.0.1:8281 -raft.group_peers=127.0.0.1:8281,127.0.0.1:8282,127.0.0.1:8283 raft.path=rocksdb-raftlog1 -raft.use_replicator_pipeline=true -raft.election_timeout=10000 -raft.snapshot_interval=3600 -raft.backend_threads=48 -raft.read_index_threads=8 -raft.read_strategy=ReadOnlyLeaseBased -raft.queue_size=16384 -raft.queue_publish_timeout=60 -raft.apply_batch=1 -raft.rpc_threads=8 -raft.rpc_connect_timeout=5000 -raft.rpc_timeout=60000 +raft.safe_read=true diff --git a/hugegraph-dist/src/assembly/travis/conf-raft1/rest-server.properties b/hugegraph-dist/src/assembly/travis/conf-raft1/rest-server.properties index 25b6b283d3..a94463487c 100644 --- a/hugegraph-dist/src/assembly/travis/conf-raft1/rest-server.properties +++ b/hugegraph-dist/src/assembly/travis/conf-raft1/rest-server.properties @@ -7,6 +7,8 @@ rpc.server_host=127.0.0.1 rpc.server_port=8091 rpc.remote_url=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093 +raft.group_peers=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093 + server.id=server1 server.role=master diff --git a/hugegraph-dist/src/assembly/travis/conf-raft2/graphs/hugegraph.properties b/hugegraph-dist/src/assembly/travis/conf-raft2/graphs/hugegraph.properties index 9aaf5fdfc4..13deeae4f0 100644 --- a/hugegraph-dist/src/assembly/travis/conf-raft2/graphs/hugegraph.properties +++ b/hugegraph-dist/src/assembly/travis/conf-raft2/graphs/hugegraph.properties @@ -10,20 +10,5 @@ rocksdb.data_path=rocksdb-data-raft2 rocksdb.wal_path=rocksdb-data-raft2 raft.mode=true -raft.safe_read=true -raft.use_snapshot=false -raft.endpoint=127.0.0.1:8282 -raft.group_peers=127.0.0.1:8281,127.0.0.1:8282,127.0.0.1:8283 raft.path=rocksdb-raftlog2 -raft.use_replicator_pipeline=true -raft.election_timeout=10000 -raft.snapshot_interval=3600 -raft.backend_threads=48 -raft.read_index_threads=8 -raft.read_strategy=ReadOnlyLeaseBased -raft.queue_size=16384 -raft.queue_publish_timeout=60 -raft.apply_batch=1 -raft.rpc_threads=8 -raft.rpc_connect_timeout=5000 -raft.rpc_timeout=60000 +raft.safe_read=true diff --git a/hugegraph-dist/src/assembly/travis/conf-raft2/rest-server.properties b/hugegraph-dist/src/assembly/travis/conf-raft2/rest-server.properties index f1eb1b4584..2a02d20b14 100644 --- a/hugegraph-dist/src/assembly/travis/conf-raft2/rest-server.properties +++ b/hugegraph-dist/src/assembly/travis/conf-raft2/rest-server.properties @@ -7,6 +7,8 @@ rpc.server_host=127.0.0.1 rpc.server_port=8092 rpc.remote_url=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093 +raft.group_peers=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093 + server.id=server2 server.role=worker diff --git a/hugegraph-dist/src/assembly/travis/conf-raft3/graphs/hugegraph.properties b/hugegraph-dist/src/assembly/travis/conf-raft3/graphs/hugegraph.properties index 0fb5a203cd..38f859c0a4 100644 --- a/hugegraph-dist/src/assembly/travis/conf-raft3/graphs/hugegraph.properties +++ b/hugegraph-dist/src/assembly/travis/conf-raft3/graphs/hugegraph.properties @@ -10,20 +10,5 @@ rocksdb.data_path=rocksdb-data-raft3 rocksdb.wal_path=rocksdb-data-raft3 raft.mode=true -raft.safe_read=true -raft.use_snapshot=false -raft.endpoint=127.0.0.1:8283 -raft.group_peers=127.0.0.1:8281,127.0.0.1:8282,127.0.0.1:8283 raft.path=rocksdb-raftlog3 -raft.use_replicator_pipeline=true -raft.election_timeout=10000 -raft.snapshot_interval=3600 -raft.backend_threads=48 -raft.read_index_threads=8 -raft.read_strategy=ReadOnlyLeaseBased -raft.queue_size=16384 -raft.queue_publish_timeout=60 -raft.apply_batch=1 -raft.rpc_threads=8 -raft.rpc_connect_timeout=5000 -raft.rpc_timeout=60000 +raft.safe_read=true diff --git a/hugegraph-dist/src/assembly/travis/conf-raft3/rest-server.properties b/hugegraph-dist/src/assembly/travis/conf-raft3/rest-server.properties index e31d3d286f..1fa980f4e8 100644 --- a/hugegraph-dist/src/assembly/travis/conf-raft3/rest-server.properties +++ b/hugegraph-dist/src/assembly/travis/conf-raft3/rest-server.properties @@ -7,6 +7,8 @@ rpc.server_host=127.0.0.1 rpc.server_port=8093 rpc.remote_url=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093 +raft.group_peers=127.0.0.1:8091,127.0.0.1:8092,127.0.0.1:8093 + server.id=server3 server.role=worker diff --git a/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh b/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh index 9c58e7170b..75819ee13c 100755 --- a/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh +++ b/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh @@ -18,7 +18,7 @@ GREMLIN_SERVER_CONF=$SERVER_DIR/conf/gremlin-server.yaml JACOCO_PORT=36320 RAFT_TOOLS=$RAFT1_DIR/bin/raft-tools.sh -RAFT_LEADER="127.0.0.1:8281" +RAFT_LEADER="127.0.0.1:8091" mvn package -DskipTests From 7592b23ba1082255fe92e5c7fe6d842802f4a73f Mon Sep 17 00:00:00 2001 From: Zhangmei Li Date: Mon, 16 May 2022 12:49:22 +0800 Subject: [PATCH 14/14] fix raft-server can't start Change-Id: I34abe845ba28d91a05e96065413a6148d39fa5a6 --- .../hugegraph/auth/StandardAuthenticator.java | 6 +++ .../baidu/hugegraph/core/GraphManager.java | 3 ++ .../baidu/hugegraph/StandardHugeGraph.java | 7 +-- .../store/AbstractBackendStoreProvider.java | 5 ++- .../backend/store/BackendProviderFactory.java | 2 +- .../backend/store/BackendStoreProvider.java | 3 +- .../store/raft/RaftBackendStoreProvider.java | 20 ++++----- .../backend/store/raft/RaftContext.java | 45 ++++++++++++------- 8 files changed, 52 insertions(+), 39 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/StandardAuthenticator.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/StandardAuthenticator.java index 5d56169f0e..b8e559680f 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/StandardAuthenticator.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/StandardAuthenticator.java @@ -118,6 +118,12 @@ public void setup(HugeConfig config) { // Forced set RAFT_MODE to false when initializing backend graphConfig.setProperty(CoreOptions.RAFT_MODE.name(), "false"); } + + // Transfer `raft.group_peers` from server config to graph config + String raftGroupPeers = config.get(ServerOptions.RAFT_GROUP_PEERS); + graphConfig.addProperty(ServerOptions.RAFT_GROUP_PEERS.name(), + raftGroupPeers); + this.graph = (HugeGraph) GraphFactory.open(graphConfig); String remoteUrl = config.get(ServerOptions.AUTH_REMOTE_URL); diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java index bf3bc8d7ed..6a4217f4d5 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java @@ -100,7 +100,9 @@ public GraphManager(HugeConfig conf, EventHub hub) { this.conf = conf; this.listenChanges(); + this.loadGraphs(ConfigUtil.scanGraphsDir(this.graphsDir)); + // this.installLicense(conf, ""); // Start RPC-Server for raft-rpc/auth-rpc/cache-notify-rpc... @@ -343,6 +345,7 @@ private void closeTx(final Set graphSourceNamesToCloseTxOn, private void loadGraph(String name, String graphConfPath) { HugeConfig config = new HugeConfig(graphConfPath); + // Transfer `raft.group_peers` from server config to graph config String raftGroupPeers = this.conf.get(ServerOptions.RAFT_GROUP_PEERS); config.addProperty(ServerOptions.RAFT_GROUP_PEERS.name(), raftGroupPeers); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java index 00a2242345..569d3d9e27 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java @@ -312,14 +312,9 @@ public void readMode(GraphReadMode readMode) { @Override public void waitReady(RpcServer rpcServer) { - if (this.storeProvider instanceof RaftBackendStoreProvider) { - ((RaftBackendStoreProvider) this.storeProvider).initRaftContext( - this.params, - rpcServer); - } // Just for trigger Tx.getOrNewTransaction, then load 3 stores this.schemaTransaction(); - this.storeProvider.waitStoreStarted(); + this.storeProvider.waitReady(rpcServer); } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java index 005d133ba4..b7e6951d64 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java @@ -25,6 +25,7 @@ import org.slf4j.Logger; +import com.alipay.remoting.rpc.RpcServer; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.backend.BackendException; import com.baidu.hugegraph.backend.store.raft.StoreSnapshotFile; @@ -94,8 +95,8 @@ public void open(String graph) { } @Override - public void waitStoreStarted() { - // pass + public void waitReady(RpcServer rpcServer) { + // passs } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendProviderFactory.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendProviderFactory.java index 18f21b23cd..c8d22450a0 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendProviderFactory.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendProviderFactory.java @@ -52,7 +52,7 @@ public static BackendStoreProvider open(HugeGraphParams params) { if (raftMode) { LOG.info("Opening backend store '{}' in raft mode for graph '{}'", backend, graph); - provider = new RaftBackendStoreProvider(provider); + provider = new RaftBackendStoreProvider(params, provider); } provider.open(graph); return provider; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java index 3d794d2c9d..2b84853267 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java @@ -19,6 +19,7 @@ package com.baidu.hugegraph.backend.store; +import com.alipay.remoting.rpc.RpcServer; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.config.HugeConfig; import com.baidu.hugegraph.event.EventHub; @@ -43,7 +44,7 @@ public interface BackendStoreProvider { void open(String name); - void waitStoreStarted(); + void waitReady(RpcServer rpcServer); void close(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java index 2b159b309e..145951604f 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java @@ -46,23 +46,18 @@ public class RaftBackendStoreProvider implements BackendStoreProvider { private static final Logger LOG = Log.logger(RaftBackendStoreProvider.class); private final BackendStoreProvider provider; + private final RaftContext context; + private RaftBackendStore schemaStore; private RaftBackendStore graphStore; private RaftBackendStore systemStore; - private RaftContext context; - - public RaftBackendStoreProvider(BackendStoreProvider provider) { + public RaftBackendStoreProvider(HugeGraphParams params, + BackendStoreProvider provider) { this.provider = provider; this.schemaStore = null; this.graphStore = null; this.systemStore = null; - this.context = null; - } - - public void initRaftContext(HugeGraphParams params, RpcServer rpcServer) { - this.context = new RaftContext(params, rpcServer); - // NOTE: loadSystemStore() did not addStore() - this.context.addStore(StoreType.SYSTEM, this.systemStore); + this.context = new RaftContext(params); } public RaftGroupManager raftNodeManager() { @@ -141,6 +136,7 @@ public synchronized BackendStore loadSystemStore(HugeConfig config, String name) BackendStore store = this.provider.loadSystemStore(config, name); this.checkNonSharedStore(store); this.systemStore = new RaftBackendStore(store, this.context()); + this.context().addStore(StoreType.SYSTEM, this.systemStore); } return this.systemStore; } @@ -151,8 +147,8 @@ public void open(String name) { } @Override - public void waitStoreStarted() { - this.context().initRaftNode(); + public void waitReady(RpcServer rpcServer) { + this.context().initRaftNode(rpcServer); LOG.info("The raft node is initialized"); this.context().waitRaftNodeStarted(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java index 8ee931e3f4..7197dd4614 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftContext.java @@ -90,45 +90,50 @@ public final class RaftContext { public static final long KEEP_ALIVE_SECOND = 300L; private final HugeGraphParams params; - private final RpcServer raftRpcServer; - private final PeerId endpoint; + + private final Configuration groupPeers; private final String schemaStoreName; private final String graphStoreName; private final String systemStoreName; + private final RaftBackendStore[] stores; - private final Configuration groupPeers; + private final ExecutorService readIndexExecutor; private final ExecutorService snapshotExecutor; private final ExecutorService backendExecutor; + private RpcServer raftRpcServer; + private PeerId endpoint; + private RaftNode raftNode; private RaftGroupManager raftGroupManager; private RpcForwarder rpcForwarder; - public RaftContext(HugeGraphParams params, - com.alipay.remoting.rpc.RpcServer rpcServer) { + public RaftContext(HugeGraphParams params) { this.params = params; - this.raftRpcServer = this.wrapRpcServer(rpcServer); - this.endpoint = new PeerId(rpcServer.ip(), rpcServer.port());; HugeConfig config = params.configuration(); - this.schemaStoreName = config.get(CoreOptions.STORE_SCHEMA); - this.graphStoreName = config.get(CoreOptions.STORE_GRAPH); - this.systemStoreName = config.get(CoreOptions.STORE_SYSTEM); - this.stores = new RaftBackendStore[StoreType.ALL.getNumber()]; /* * NOTE: `raft.group_peers` option is transfered from ServerConfig * to CoreConfig, since it's shared by all graphs. */ + String groupPeersString = this.config().getString("raft.group_peers"); + E.checkArgument(groupPeersString != null, + "Please ensure config `raft.group_peers` in raft mode"); this.groupPeers = new Configuration(); - String groupPeersString = config.getString("raft.group_peers"); if (!this.groupPeers.parse(groupPeersString)) { throw new HugeException("Failed to parse raft.group_peers: '%s'", groupPeersString); } + this.schemaStoreName = config.get(CoreOptions.STORE_SCHEMA); + this.graphStoreName = config.get(CoreOptions.STORE_GRAPH); + this.systemStoreName = config.get(CoreOptions.STORE_SYSTEM); + + this.stores = new RaftBackendStore[StoreType.ALL.getNumber()]; + if (config.get(CoreOptions.RAFT_SAFE_READ)) { int threads = config.get(CoreOptions.RAFT_READ_INDEX_THREADS); this.readIndexExecutor = this.createReadIndexExecutor(threads); @@ -142,14 +147,21 @@ public RaftContext(HugeGraphParams params, threads = config.get(CoreOptions.RAFT_BACKEND_THREADS); this.backendExecutor = this.createBackendExecutor(threads); + this.raftRpcServer = null; + this.endpoint = null; + this.raftNode = null; this.raftGroupManager = null; this.rpcForwarder = null; + } + + public void initRaftNode(com.alipay.remoting.rpc.RpcServer rpcServer) { + this.raftRpcServer = this.wrapRpcServer(rpcServer); + this.endpoint = new PeerId(rpcServer.ip(), rpcServer.port()); + this.registerRpcRequestProcessors(); LOG.info("Start raft server successfully: {}", this.endpoint()); - } - public void initRaftNode() { this.raftNode = new RaftNode(this); this.rpcForwarder = new RpcForwarder(this.raftNode.node()); this.raftGroupManager = new RaftGroupManagerImpl(this); @@ -176,11 +188,11 @@ public RaftNode node() { return this.raftNode; } - public RpcServer rpcServer() { + protected RpcServer rpcServer() { return this.raftRpcServer; } - public RpcForwarder rpcForwarder() { + protected RpcForwarder rpcForwarder() { return this.rpcForwarder; } @@ -395,7 +407,6 @@ private RpcServer wrapRpcServer(com.alipay.remoting.rpc.RpcServer rpcServer) { // Reference from RaftRpcServerFactory.createAndStartRaftRpcServer RpcServer raftRpcServer = new BoltRpcServer(rpcServer); RaftRpcServerFactory.addRaftRequestProcessors(raftRpcServer); - raftRpcServer.init(null); return raftRpcServer; }