From 2bdf897d972a6ad5c6f7792dab6ecc437e60eb90 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Sat, 24 Feb 2024 18:44:52 +0800 Subject: [PATCH 01/17] git add hugegraph-pd/hg-pd-grpc/ hugegraph-pd/hg-pd-common/ hugegraph-pd/hg-pd-client/ pom.xml hugegraph-pd/pom.xml --- hugegraph-pd/hg-pd-client/pom.xml | 73 + .../hugegraph/pd/client/AbstractClient.java | 265 ++++ .../pd/client/AbstractClientStubProxy.java | 72 + .../apache/hugegraph/pd/client/Channels.java | 44 + .../hugegraph/pd/client/ClientCache.java | 338 +++++ .../hugegraph/pd/client/Discoverable.java | 30 + .../hugegraph/pd/client/DiscoveryClient.java | 221 +++ .../pd/client/DiscoveryClientImpl.java | 140 ++ .../apache/hugegraph/pd/client/KvClient.java | 352 +++++ .../hugegraph/pd/client/LicenseClient.java | 71 + .../apache/hugegraph/pd/client/PDClient.java | 1345 +++++++++++++++++ .../apache/hugegraph/pd/client/PDConfig.java | 82 + .../apache/hugegraph/pd/client/PDPulse.java | 152 ++ .../hugegraph/pd/client/PDPulseImpl.java | 196 +++ .../apache/hugegraph/pd/client/PDWatch.java | 137 ++ .../hugegraph/pd/client/PDWatchImpl.java | 202 +++ .../hugegraph/pd/pulse/PartitionNotice.java | 49 + .../hugegraph/pd/pulse/PulseServerNotice.java | 35 + .../apache/hugegraph/pd/watch/NodeEvent.java | 99 ++ .../apache/hugegraph/pd/watch/PDWatcher.java | 22 + .../hugegraph/pd/watch/PartitionEvent.java | 93 ++ .../apache/hugegraph/pd/watch/WatchType.java | 30 + hugegraph-pd/hg-pd-common/pom.xml | 54 + .../hugegraph/pd/common/GraphCache.java | 62 + .../apache/hugegraph/pd/common/HgAssert.java | 117 ++ .../apache/hugegraph/pd/common/KVPair.java | 132 ++ .../hugegraph/pd/common/PDException.java | 47 + .../pd/common/PDRuntimeException.java | 49 + .../hugegraph/pd/common/PartitionCache.java | 460 ++++++ .../hugegraph/pd/common/PartitionUtils.java | 47 + hugegraph-pd/hg-pd-grpc/pom.xml | 138 ++ .../hg-pd-grpc/src/main/proto/discovery.proto | 71 + .../hg-pd-grpc/src/main/proto/kv.proto | 143 ++ .../hg-pd-grpc/src/main/proto/metaTask.proto | 64 + .../hg-pd-grpc/src/main/proto/metapb.proto | 394 +++++ .../hg-pd-grpc/src/main/proto/pd_common.proto | 53 + .../hg-pd-grpc/src/main/proto/pd_pulse.proto | 172 +++ .../hg-pd-grpc/src/main/proto/pd_watch.proto | 103 ++ .../hg-pd-grpc/src/main/proto/pdpb.proto | 607 ++++++++ hugegraph-pd/pom.xml | 272 ++++ pom.xml | 9 +- 41 files changed, 7038 insertions(+), 4 deletions(-) create mode 100644 hugegraph-pd/hg-pd-client/pom.xml create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClient.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClientStubProxy.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/Channels.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/ClientCache.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/Discoverable.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClient.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImpl.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/KvClient.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/LicenseClient.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDClient.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDConfig.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulse.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulseImpl.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatch.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatchImpl.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PartitionNotice.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PulseServerNotice.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/NodeEvent.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/PDWatcher.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/PartitionEvent.java create mode 100644 hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/WatchType.java create mode 100644 hugegraph-pd/hg-pd-common/pom.xml create mode 100644 hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/GraphCache.java create mode 100644 hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/HgAssert.java create mode 100644 hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/KVPair.java create mode 100644 hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PDException.java create mode 100644 hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PDRuntimeException.java create mode 100644 hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionCache.java create mode 100644 hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionUtils.java create mode 100644 hugegraph-pd/hg-pd-grpc/pom.xml create mode 100644 hugegraph-pd/hg-pd-grpc/src/main/proto/discovery.proto create mode 100644 hugegraph-pd/hg-pd-grpc/src/main/proto/kv.proto create mode 100644 hugegraph-pd/hg-pd-grpc/src/main/proto/metaTask.proto create mode 100644 hugegraph-pd/hg-pd-grpc/src/main/proto/metapb.proto create mode 100644 hugegraph-pd/hg-pd-grpc/src/main/proto/pd_common.proto create mode 100644 hugegraph-pd/hg-pd-grpc/src/main/proto/pd_pulse.proto create mode 100644 hugegraph-pd/hg-pd-grpc/src/main/proto/pd_watch.proto create mode 100644 hugegraph-pd/hg-pd-grpc/src/main/proto/pdpb.proto create mode 100644 hugegraph-pd/pom.xml diff --git a/hugegraph-pd/hg-pd-client/pom.xml b/hugegraph-pd/hg-pd-client/pom.xml new file mode 100644 index 0000000000..a64756fe94 --- /dev/null +++ b/hugegraph-pd/hg-pd-client/pom.xml @@ -0,0 +1,73 @@ + + + + + + 4.0.0 + + + org.apache.hugegraph + hugegraph-pd + ${revision} + ../pom.xml + + hg-pd-client + + + + org.projectlombok + lombok + 1.18.20 + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.17.0 + + + org.apache.hugegraph + hg-pd-grpc + ${revision} + + + org.apache.hugegraph + hg-pd-common + ${revision} + compile + + + junit + junit + 4.13.2 + test + + + commons-io + commons-io + 2.8.0 + + + org.yaml + snakeyaml + 1.28 + test + + + diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClient.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClient.java new file mode 100644 index 0000000000..874ef6f67c --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClient.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import java.io.Closeable; +import java.util.LinkedList; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; +import java.util.stream.Stream; + +import org.apache.hugegraph.pd.common.KVPair; +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.pd.grpc.PDGrpc; +import org.apache.hugegraph.pd.grpc.PDGrpc.PDBlockingStub; +import org.apache.hugegraph.pd.grpc.Pdpb; +import org.apache.hugegraph.pd.grpc.Pdpb.GetMembersRequest; +import org.apache.hugegraph.pd.grpc.Pdpb.GetMembersResponse; + +import io.grpc.Channel; +import io.grpc.ClientCall; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.MethodDescriptor; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.AbstractBlockingStub; +import io.grpc.stub.AbstractStub; +import io.grpc.stub.ClientCalls; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class AbstractClient implements Closeable { + + private static final ConcurrentHashMap chs = new ConcurrentHashMap<>(); + public static Pdpb.ResponseHeader okHeader = Pdpb.ResponseHeader.newBuilder().setError( + Pdpb.Error.newBuilder().setType(Pdpb.ErrorType.OK)).build(); + protected final Pdpb.RequestHeader header; + protected final AbstractClientStubProxy stubProxy; + protected final PDConfig config; + protected ManagedChannel channel = null; + protected volatile ConcurrentMap stubs = null; + + protected AbstractClient(PDConfig config) { + String[] hosts = config.getServerHost().split(","); + this.stubProxy = new AbstractClientStubProxy(hosts); + this.header = Pdpb.RequestHeader.getDefaultInstance(); + this.config = config; + } + + public static Pdpb.ResponseHeader newErrorHeader(int errorCode, String errorMsg) { + Pdpb.ResponseHeader header = Pdpb.ResponseHeader.newBuilder().setError( + Pdpb.Error.newBuilder().setTypeValue(errorCode).setMessage(errorMsg)).build(); + return header; + } + + protected static void handleErrors(Pdpb.ResponseHeader header) throws PDException { + if (header.hasError() && header.getError().getType() != Pdpb.ErrorType.OK) { + throw new PDException(header.getError().getTypeValue(), + String.format("PD request error, error code = %d, msg = %s", + header.getError().getTypeValue(), + header.getError().getMessage())); + } + } + + protected AbstractBlockingStub getBlockingStub() throws PDException { + if (stubProxy.getBlockingStub() == null) { + synchronized (this) { + if (stubProxy.getBlockingStub() == null) { + String host = resetStub(); + if (host.isEmpty()) { + throw new PDException(Pdpb.ErrorType.PD_UNREACHABLE_VALUE, + "PD unreachable, pd.peers=" + + config.getServerHost()); + } + } + } + } + return (AbstractBlockingStub) stubProxy.getBlockingStub() + .withDeadlineAfter(config.getGrpcTimeOut(), + TimeUnit.MILLISECONDS); + } + + protected AbstractStub getStub() throws PDException { + if (stubProxy.getStub() == null) { + synchronized (this) { + if (stubProxy.getStub() == null) { + String host = resetStub(); + if (host.isEmpty()) { + throw new PDException(Pdpb.ErrorType.PD_UNREACHABLE_VALUE, + "PD unreachable, pd.peers=" + + config.getServerHost()); + } + } + } + } + return stubProxy.getStub(); + } + + protected abstract AbstractStub createStub(); + + protected abstract AbstractBlockingStub createBlockingStub(); + + private String resetStub() { + String leaderHost = ""; + for (int i = 0; i < stubProxy.getHostCount(); i++) { + String host = stubProxy.nextHost(); + channel = ManagedChannelBuilder.forTarget(host).usePlaintext().build(); + PDBlockingStub blockingStub = PDGrpc.newBlockingStub(channel) + .withDeadlineAfter(config.getGrpcTimeOut(), + TimeUnit.MILLISECONDS); + try { + GetMembersRequest request = Pdpb.GetMembersRequest.newBuilder() + .setHeader(header).build(); + GetMembersResponse members = blockingStub.getMembers(request); + Metapb.Member leader = members.getLeader(); + leaderHost = leader.getGrpcUrl(); + close(); + channel = ManagedChannelBuilder.forTarget(leaderHost).usePlaintext().build(); + stubProxy.setBlockingStub(createBlockingStub()); + stubProxy.setStub(createStub()); + log.info("PDClient connect to host = {} success", leaderHost); + break; + } catch (Exception e) { + log.error("PDClient connect to {} exception {}, {}", host, e.getMessage(), + e.getCause() != null ? e.getCause().getMessage() : ""); + } + } + return leaderHost; + } + + protected > RespT blockingUnaryCall( + MethodDescriptor method, ReqT req) throws PDException { + return blockingUnaryCall(method, req, 5); + } + + protected > RespT blockingUnaryCall( + MethodDescriptor method, ReqT req, int retry) throws PDException { + AbstractBlockingStub stub = getBlockingStub(); + try { + RespT resp = + ClientCalls.blockingUnaryCall(stub.getChannel(), method, stub.getCallOptions(), + req); + return resp; + } catch (Exception e) { + log.error(method.getFullMethodName() + " exception, {}", e.getMessage()); + if (e instanceof StatusRuntimeException) { + if (retry < stubProxy.getHostCount()) { + // 网络不通，关掉之前连接，换host重新连接 + synchronized (this) { + stubProxy.setBlockingStub(null); + } + return blockingUnaryCall(method, req, ++retry); + } + } + } + return null; + } + + // this.stubs = new ConcurrentHashMap(hosts.length); + private AbstractBlockingStub getConcurrentBlockingStub(String address) { + AbstractBlockingStub stub = stubs.get(address); + if (stub != null) { + return stub; + } + Channel ch = ManagedChannelBuilder.forTarget(address).usePlaintext().build(); + PDBlockingStub blockingStub = + PDGrpc.newBlockingStub(ch).withDeadlineAfter(config.getGrpcTimeOut(), + TimeUnit.MILLISECONDS); + stubs.put(address, blockingStub); + return blockingStub; + + } + + protected KVPair concurrentBlockingUnaryCall( + MethodDescriptor method, ReqT req, Predicate predicate) { + LinkedList hostList = this.stubProxy.getHostList(); + if (this.stubs == null) { + synchronized (this) { + if (this.stubs == null) { + this.stubs = new ConcurrentHashMap<>(hostList.size()); + } + } + } + Stream respTStream = hostList.parallelStream().map((address) -> { + AbstractBlockingStub stub = getConcurrentBlockingStub(address); + RespT resp = ClientCalls.blockingUnaryCall(stub.getChannel(), + method, stub.getCallOptions(), req); + return resp; + }); + KVPair pair; + AtomicReference response = new AtomicReference<>(); + boolean result = respTStream.anyMatch((r) -> { + response.set(r); + return predicate.test(r); + }); + if (result) { + pair = new KVPair<>(true, null); + } else { + pair = new KVPair<>(false, response.get()); + } + return pair; + } + + protected void streamingCall(MethodDescriptor method, ReqT request, + StreamObserver responseObserver, + int retry) throws PDException { + AbstractStub stub = getStub(); + try { + ClientCall call = stub.getChannel().newCall(method, stub.getCallOptions()); + ClientCalls.asyncServerStreamingCall(call, request, responseObserver); + } catch (Exception e) { + if (e instanceof StatusRuntimeException) { + if (retry < stubProxy.getHostCount()) { + synchronized (this) { + stubProxy.setStub(null); + } + streamingCall(method, request, responseObserver, ++retry); + return; + } + } + log.error("rpc call with exception, {}", e.getMessage()); + } + } + + @Override + public void close() { + closeChannel(channel); + if (stubs != null) { + for (AbstractBlockingStub stub : stubs.values()) { + closeChannel((ManagedChannel) stub.getChannel()); + } + } + + } + + private void closeChannel(ManagedChannel channel) { + try { + while (channel != null && + !channel.shutdownNow().awaitTermination(100, TimeUnit.MILLISECONDS)) { + continue; + } + } catch (Exception e) { + log.info("Close channel with error : ", e); + } + } +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClientStubProxy.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClientStubProxy.java new file mode 100644 index 0000000000..6ee3fcb625 --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/AbstractClientStubProxy.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import java.util.LinkedList; + +import io.grpc.stub.AbstractBlockingStub; +import io.grpc.stub.AbstractStub; + +public class AbstractClientStubProxy { + + private final LinkedList hostList = new LinkedList<>(); + private AbstractBlockingStub blockingStub; + private AbstractStub stub; + + public AbstractClientStubProxy(String[] hosts) { + for (String host : hosts) { + if (!host.isEmpty()) { + hostList.offer(host); + } + } + } + + public LinkedList getHostList() { + return hostList; + } + + public String nextHost() { + String host = hostList.poll(); + hostList.offer(host); //移到尾部 + return host; + } + + public AbstractBlockingStub getBlockingStub() { + return this.blockingStub; + } + + public void setBlockingStub(AbstractBlockingStub stub) { + this.blockingStub = stub; + } + + public String getHost() { + return hostList.peek(); + } + + public int getHostCount() { + return hostList.size(); + } + + public AbstractStub getStub() { + return stub; + } + + public void setStub(AbstractStub stub) { + this.stub = stub; + } +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/Channels.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/Channels.java new file mode 100644 index 0000000000..34616e6374 --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/Channels.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import java.util.concurrent.ConcurrentHashMap; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; + +public class Channels { + + private static final ConcurrentHashMap chs = new ConcurrentHashMap<>(); + + public static ManagedChannel getChannel(String target) { + + ManagedChannel channel; + if ((channel = chs.get(target)) == null || channel.isShutdown() || channel.isTerminated()) { + synchronized (chs) { + if ((channel = chs.get(target)) == null || channel.isShutdown() || + channel.isTerminated()) { + channel = ManagedChannelBuilder.forTarget(target).usePlaintext().build(); + chs.put(target, channel); + } + } + } + + return channel; + } +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/ClientCache.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/ClientCache.java new file mode 100644 index 0000000000..d4fd50ffe9 --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/ClientCache.java @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hugegraph.pd.common.GraphCache; +import org.apache.hugegraph.pd.common.KVPair; +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.common.PartitionUtils; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.pd.grpc.Metapb.Partition; +import org.apache.hugegraph.pd.grpc.Metapb.Shard; +import org.apache.hugegraph.pd.grpc.Metapb.ShardGroup; +import org.apache.hugegraph.pd.grpc.Pdpb.CachePartitionResponse; +import org.apache.hugegraph.pd.grpc.Pdpb.CacheResponse; + +import com.google.common.collect.Range; +import com.google.common.collect.RangeMap; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ClientCache { + + private final AtomicBoolean initialized = new AtomicBoolean(false); + private final org.apache.hugegraph.pd.client.PDClient client; + private volatile Map> groups; + private volatile Map stores; + private volatile Map caches = new ConcurrentHashMap<>(); + + public ClientCache(org.apache.hugegraph.pd.client.PDClient pdClient) { + groups = new ConcurrentHashMap<>(); + stores = new ConcurrentHashMap<>(); + client = pdClient; + } + + private GraphCache getGraphCache(String graphName) { + GraphCache graph; + if ((graph = caches.get(graphName)) == null) { + synchronized (caches) { + if ((graph = caches.get(graphName)) == null) { + graph = new GraphCache(); + caches.put(graphName, graph); + } + } + } + return graph; + } + + public KVPair getPartitionById(String graphName, int partId) { + try { + GraphCache graph = initGraph(graphName); + Partition partition = graph.getPartition(partId); + Shard shard = groups.get(partId).getValue(); + if (partition == null || shard == null) { + return null; + } + return new KVPair<>(partition, shard); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private KVPair getPair(int partId, GraphCache graph) { + Partition p = graph.getPartition(partId); + KVPair pair = groups.get(partId); + if (p != null && pair != null) { + Shard s = pair.getValue(); + if (s == null) { + pair.setValue(getLeader(partId)); + return new KVPair<>(p, pair.getValue()); + } else { + return new KVPair<>(p, s); + } + } + return null; + } + + /** + * 根据key的hashcode返回分区信息 + * + * @param graphName + * @param code + * @return + */ + public KVPair getPartitionByCode(String graphName, long code) { + try { + GraphCache graph = initGraph(graphName); + RangeMap range = graph.getRange(); + Integer pId = range.get(code); + if (pId != null) { + return getPair(pId, graph); + } + return null; + } catch (PDException e) { + throw new RuntimeException(e); + } + } + + private GraphCache initGraph(String graphName) throws PDException { + initCache(); + GraphCache graph = getGraphCache(graphName); + if (!graph.getInitialized().get()) { + synchronized (graph) { + if (!graph.getInitialized().get()) { + CachePartitionResponse pc = client.getPartitionCache(graphName); + RangeMap range = graph.getRange(); + List ps = pc.getPartitionsList(); + HashMap gps = new HashMap<>(ps.size(), 1); + for (Partition p : ps) { + gps.put(p.getId(), p); + range.put(Range.closedOpen(p.getStartKey(), p.getEndKey()), p.getId()); + } + graph.setPartitions(gps); + graph.getInitialized().set(true); + } + } + } + return graph; + } + + private void initCache() throws PDException { + if (!initialized.get()) { + synchronized (this) { + if (!initialized.get()) { + CacheResponse cache = client.getClientCache(); + List shardGroups = cache.getShardsList(); + for (ShardGroup s : shardGroups) { + this.groups.put(s.getId(), new KVPair<>(s, getLeader(s.getId()))); + } + List stores = cache.getStoresList(); + for (Metapb.Store store : stores) { + this.stores.put(store.getId(), store); + } + List graphs = cache.getGraphsList(); + for (Metapb.Graph g : graphs) { + GraphCache c = new GraphCache(g); + caches.put(g.getGraphName(), c); + } + initialized.set(true); + } + } + } + } + + /** + * 返回key所在的分区信息 + * + * @param key + * @return + */ + public KVPair getPartitionByKey(String graphName, byte[] key) { + int code = PartitionUtils.calcHashcode(key); + return getPartitionByCode(graphName, code); + } + + public boolean update(String graphName, int partId, Partition partition) { + GraphCache graph = getGraphCache(graphName); + try { + Partition p = graph.getPartition(partId); + if (p != null && p.equals(partition)) { + return false; + } + RangeMap range = graph.getRange(); + graph.addPartition(partId, partition); + if (p != null) { + // old [1-3) 被 [2-3)覆盖了。当 [1-3) 变成[1-2) 不应该删除原先的[1-3) + // 当确认老的 start, end 都是自己的时候，才可以删除老的. (即还没覆盖） + if (Objects.equals(partition.getId(), range.get(partition.getStartKey())) && + Objects.equals(partition.getId(), range.get(partition.getEndKey() - 1))) { + range.remove(range.getEntry(partition.getStartKey()).getKey()); + } + } + range.put(Range.closedOpen(partition.getStartKey(), partition.getEndKey()), partId); + } catch (Exception e) { + throw new RuntimeException(e); + } + return true; + } + + public void removePartition(String graphName, int partId) { + GraphCache graph = getGraphCache(graphName); + Partition p = graph.removePartition(partId); + if (p != null) { + RangeMap range = graph.getRange(); + if (Objects.equals(p.getId(), range.get(p.getStartKey())) && + Objects.equals(p.getId(), range.get(p.getEndKey() - 1))) { + range.remove(range.getEntry(p.getStartKey()).getKey()); + } + } + } + + /** + * remove all partitions + */ + public void removePartitions() { + for (Entry entry : caches.entrySet()) { + removePartitions(entry.getValue()); + } + } + + private void removePartitions(GraphCache graph) { + graph.getState().clear(); + graph.getRange().clear(); + } + + /** + * remove partition cache of graphName + * + * @param graphName + */ + public void removeAll(String graphName) { + GraphCache graph = caches.get(graphName); + if (graph != null) { + removePartitions(graph); + } + } + + public boolean updateShardGroup(ShardGroup shardGroup) { + KVPair old = groups.get(shardGroup.getId()); + Shard leader = getLeader(shardGroup); + if (old != null) { + old.setKey(shardGroup); + old.setValue(leader); + return false; + } + groups.put(shardGroup.getId(), new KVPair<>(shardGroup, leader)); + return true; + } + + public void deleteShardGroup(int shardGroupId) { + groups.remove(shardGroupId); + } + + public ShardGroup getShardGroup(int groupId) { + KVPair pair = groups.get(groupId); + if (pair != null) { + return pair.getKey(); + } + return null; + } + + public boolean addStore(Long storeId, Metapb.Store store) { + Metapb.Store oldStore = stores.get(storeId); + if (oldStore != null && oldStore.equals(store)) { + return false; + } + stores.put(storeId, store); + return true; + } + + public Metapb.Store getStoreById(Long storeId) { + return stores.get(storeId); + } + + public void removeStore(Long storeId) { + stores.remove(storeId); + } + + public void reset() { + groups = new ConcurrentHashMap<>(); + stores = new ConcurrentHashMap<>(); + caches = new ConcurrentHashMap<>(); + } + + public Shard getLeader(int partitionId) { + KVPair pair = groups.get(partitionId); + if (pair != null) { + if (pair.getValue() != null) { + return pair.getValue(); + } + for (Shard shard : pair.getKey().getShardsList()) { + if (shard.getRole() == Metapb.ShardRole.Leader) { + pair.setValue(shard); + return shard; + } + } + } + + return null; + } + + public Shard getLeader(ShardGroup shardGroup) { + if (shardGroup != null) { + for (Shard shard : shardGroup.getShardsList()) { + if (shard.getRole() == Metapb.ShardRole.Leader) { + return shard; + } + } + } + + return null; + } + + public void updateLeader(int partitionId, Shard leader) { + KVPair pair = groups.get(partitionId); + if (pair != null && leader != null) { + Shard l = getLeader(partitionId); + if (l == null || leader.getStoreId() != l.getStoreId()) { + ShardGroup shardGroup = pair.getKey(); + ShardGroup.Builder builder = ShardGroup.newBuilder(shardGroup).clearShards(); + for (var shard : shardGroup.getShardsList()) { + builder.addShards( + Shard.newBuilder() + .setStoreId(shard.getStoreId()) + .setRole(shard.getStoreId() == leader.getStoreId() ? + Metapb.ShardRole.Leader : Metapb.ShardRole.Follower) + .build() + ); + } + pair.setKey(builder.build()); + pair.setValue(leader); + } + } + } +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/Discoverable.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/Discoverable.java new file mode 100644 index 0000000000..abdcac414c --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/Discoverable.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import org.apache.hugegraph.pd.grpc.discovery.NodeInfos; +import org.apache.hugegraph.pd.grpc.discovery.Query; + +public interface Discoverable { + + NodeInfos getNodeInfos(Query query); + + void scheduleTask(); + + void cancelTask(); +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClient.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClient.java new file mode 100644 index 0000000000..890e9e5864 --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClient.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import java.io.Closeable; +import java.util.LinkedList; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Consumer; +import java.util.function.Function; + +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.grpc.discovery.DiscoveryServiceGrpc; +import org.apache.hugegraph.pd.grpc.discovery.NodeInfo; +import org.apache.hugegraph.pd.grpc.discovery.NodeInfos; +import org.apache.hugegraph.pd.grpc.discovery.Query; +import org.apache.hugegraph.pd.grpc.discovery.RegisterInfo; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class DiscoveryClient implements Closeable, Discoverable { + + private final Timer timer = new Timer("serverHeartbeat", true); + private final AtomicBoolean requireResetStub = new AtomicBoolean(false); + protected int period; //心跳周期 + LinkedList pdAddresses = new LinkedList<>(); + ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(); + private volatile int currentIndex; // 当前在用pd地址位置 + private int maxTime = 6; + private ManagedChannel channel = null; + private DiscoveryServiceGrpc.DiscoveryServiceBlockingStub registerStub; + private DiscoveryServiceGrpc.DiscoveryServiceBlockingStub blockingStub; + + public DiscoveryClient(String centerAddress, int delay) { + String[] addresses = centerAddress.split(","); + for (int i = 0; i < addresses.length; i++) { + String singleAddress = addresses[i]; + if (singleAddress == null || singleAddress.length() <= 0) { + continue; + } + pdAddresses.add(addresses[i]); + } + this.period = delay; + if (maxTime < addresses.length) { + maxTime = addresses.length; + } + } + + private R tryWithTimes(Function function, V v) { + R r; + Exception ex = null; + for (int i = 0; i < maxTime; i++) { + try { + r = function.apply(v); + return r; + } catch (Exception e) { + requireResetStub.set(true); + resetStub(); + ex = e; + } + } + if (ex != null) { + log.error("Try discovery method with error: {}", ex.getMessage()); + } + return null; + } + + /*** + * 按照pd列表重置stub + */ + private void resetStub() { + String errLog = null; + for (int i = currentIndex + 1; i <= pdAddresses.size() + currentIndex; i++) { + currentIndex = i % pdAddresses.size(); + String singleAddress = pdAddresses.get(currentIndex); + try { + if (requireResetStub.get()) { + resetChannel(singleAddress); + } + errLog = null; + break; + } catch (Exception e) { + requireResetStub.set(true); + if (errLog == null) { + errLog = e.getMessage(); + } + continue; + } + } + if (errLog != null) { + log.error(errLog); + } + } + + /*** + * 按照某个pd的地址重置channel和stub + * @param singleAddress + * @throws PDException + */ + private void resetChannel(String singleAddress) throws PDException { + + readWriteLock.writeLock().lock(); + try { + if (requireResetStub.get()) { + while (channel != null && !channel.shutdownNow().awaitTermination( + 100, TimeUnit.MILLISECONDS)) { + continue; + } + channel = ManagedChannelBuilder.forTarget( + singleAddress).usePlaintext().build(); + this.registerStub = DiscoveryServiceGrpc.newBlockingStub( + channel); + this.blockingStub = DiscoveryServiceGrpc.newBlockingStub( + channel); + requireResetStub.set(false); + } + } catch (Exception e) { + throw new PDException(-1, String.format( + "Reset channel with error : %s.", e.getMessage())); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + /*** + * 获取注册节点信息 + * @param query + * @return + */ + @Override + public NodeInfos getNodeInfos(Query query) { + return tryWithTimes((q) -> { + this.readWriteLock.readLock().lock(); + NodeInfos nodes; + try { + nodes = this.blockingStub.getNodes(q); + } catch (Exception e) { + throw e; + } finally { + this.readWriteLock.readLock().unlock(); + } + return nodes; + }, query); + } + + /*** + * 启动心跳任务 + */ + @Override + public void scheduleTask() { + timer.schedule(new TimerTask() { + @Override + public void run() { + NodeInfo nodeInfo = getRegisterNode(); + tryWithTimes((t) -> { + RegisterInfo register; + readWriteLock.readLock().lock(); + try { + register = registerStub.register(t); + log.debug("Discovery Client work done."); + Consumer consumer = getRegisterConsumer(); + if (consumer != null) { + consumer.accept(register); + } + } catch (Exception e) { + throw e; + } finally { + readWriteLock.readLock().unlock(); + } + return register; + }, nodeInfo); + } + }, 0, period); + } + + abstract NodeInfo getRegisterNode(); + + abstract Consumer getRegisterConsumer(); + + @Override + public void cancelTask() { + this.timer.cancel(); + } + + @Override + public void close() { + this.timer.cancel(); + readWriteLock.writeLock().lock(); + try { + while (channel != null && !channel.shutdownNow().awaitTermination( + 100, TimeUnit.MILLISECONDS)) { + continue; + } + } catch (Exception e) { + log.info("Close channel with error : {}.", e); + } finally { + readWriteLock.writeLock().unlock(); + } + } +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImpl.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImpl.java new file mode 100644 index 0000000000..1208370d3f --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImpl.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import java.util.Map; +import java.util.function.Consumer; + +import org.apache.hugegraph.pd.grpc.discovery.NodeInfo; +import org.apache.hugegraph.pd.grpc.discovery.RegisterType; + +public class DiscoveryClientImpl extends DiscoveryClient { + + private final String id; + private final RegisterType type; // 心跳类型，备用 + private final String version; + private final String appName; + private final int times; // 心跳过期次数，备用 + private final String address; + private final Map labels; + private final Consumer registerConsumer; + + + private DiscoveryClientImpl(Builder builder) { + super(builder.centerAddress, builder.delay); + period = builder.delay; + id = builder.id; + type = builder.type; + version = builder.version; + appName = builder.appName; + times = builder.times; + address = builder.address; + labels = builder.labels; + registerConsumer = builder.registerConsumer; + } + + public static Builder newBuilder() { + return new Builder(); + } + + + @Override + NodeInfo getRegisterNode() { + return NodeInfo.newBuilder().setAddress(this.address) + .setVersion(this.version) + .setAppName(this.appName).setInterval(this.period) + .setId(this.id).putAllLabels(labels).build(); + } + + @Override + Consumer getRegisterConsumer() { + return registerConsumer; + } + + + public static final class Builder { + + private int delay; + private String centerAddress; + private String id; + private RegisterType type; + private String address; + private Map labels; + private String version; + private String appName; + private int times; + private Consumer registerConsumer; + + private Builder() { + } + + public Builder setDelay(int val) { + delay = val; + return this; + } + + public Builder setCenterAddress(String val) { + centerAddress = val; + return this; + } + + public Builder setId(String val) { + id = val; + return this; + } + + public Builder setType(RegisterType val) { + type = val; + return this; + } + + public Builder setAddress(String val) { + address = val; + return this; + } + + public Builder setLabels(Map val) { + labels = val; + return this; + } + + public Builder setVersion(String val) { + version = val; + return this; + } + + public Builder setAppName(String val) { + appName = val; + return this; + } + + public Builder setTimes(int val) { + times = val; + return this; + } + + public Builder setRegisterConsumer(Consumer registerConsumer) { + this.registerConsumer = registerConsumer; + return this; + } + + public DiscoveryClientImpl build() { + return new DiscoveryClientImpl(this); + } + } +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/KvClient.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/KvClient.java new file mode 100644 index 0000000000..3e5c2359ed --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/KvClient.java @@ -0,0 +1,352 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import java.io.Closeable; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +import org.apache.hugegraph.pd.client.AbstractClient; +import org.apache.hugegraph.pd.client.PDConfig; +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.grpc.kv.K; +import org.apache.hugegraph.pd.grpc.kv.KResponse; +import org.apache.hugegraph.pd.grpc.kv.Kv; +import org.apache.hugegraph.pd.grpc.kv.KvResponse; +import org.apache.hugegraph.pd.grpc.kv.KvServiceGrpc; +import org.apache.hugegraph.pd.grpc.kv.LockRequest; +import org.apache.hugegraph.pd.grpc.kv.LockResponse; +import org.apache.hugegraph.pd.grpc.kv.ScanPrefixResponse; +import org.apache.hugegraph.pd.grpc.kv.TTLRequest; +import org.apache.hugegraph.pd.grpc.kv.TTLResponse; +import org.apache.hugegraph.pd.grpc.kv.WatchEvent; +import org.apache.hugegraph.pd.grpc.kv.WatchKv; +import org.apache.hugegraph.pd.grpc.kv.WatchRequest; +import org.apache.hugegraph.pd.grpc.kv.WatchResponse; +import org.apache.hugegraph.pd.grpc.kv.WatchType; + +import io.grpc.stub.AbstractBlockingStub; +import io.grpc.stub.AbstractStub; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class KvClient extends AbstractClient implements Closeable { + + private final AtomicLong clientId = new AtomicLong(0); + private final Semaphore semaphore = new Semaphore(1); + private final ConcurrentHashMap observers = new ConcurrentHashMap<>(); + + public KvClient(PDConfig pdConfig) { + super(pdConfig); + } + + @Override + protected AbstractStub createStub() { + return KvServiceGrpc.newStub(channel); + } + + @Override + protected AbstractBlockingStub createBlockingStub() { + return KvServiceGrpc.newBlockingStub(channel); + } + + public KvResponse put(String key, String value) throws PDException { + Kv kv = Kv.newBuilder().setKey(key).setValue(value).build(); + KvResponse response = blockingUnaryCall(KvServiceGrpc.getPutMethod(), kv); + handleErrors(response.getHeader()); + return response; + } + + + public KResponse get(String key) throws PDException { + K k = K.newBuilder().setKey(key).build(); + KResponse response = blockingUnaryCall(KvServiceGrpc.getGetMethod(), k); + handleErrors(response.getHeader()); + return response; + } + + + public KvResponse delete(String key) throws PDException { + K k = K.newBuilder().setKey(key).build(); + KvResponse response = blockingUnaryCall(KvServiceGrpc.getDeleteMethod(), k); + handleErrors(response.getHeader()); + return response; + } + + + public KvResponse deletePrefix(String prefix) throws PDException { + K k = K.newBuilder().setKey(prefix).build(); + KvResponse response = blockingUnaryCall(KvServiceGrpc.getDeletePrefixMethod(), k); + handleErrors(response.getHeader()); + return response; + } + + + public ScanPrefixResponse scanPrefix(String prefix) throws PDException { + K k = K.newBuilder().setKey(prefix).build(); + ScanPrefixResponse response = blockingUnaryCall(KvServiceGrpc.getScanPrefixMethod(), k); + handleErrors(response.getHeader()); + return response; + } + + public TTLResponse keepTTLAlive(String key) throws PDException { + TTLRequest request = TTLRequest.newBuilder().setKey(key).build(); + TTLResponse response = blockingUnaryCall(KvServiceGrpc.getKeepTTLAliveMethod(), request); + handleErrors(response.getHeader()); + return response; + } + + public TTLResponse putTTL(String key, String value, long ttl) throws PDException { + TTLRequest request = + TTLRequest.newBuilder().setKey(key).setValue(value).setTtl(ttl).build(); + TTLResponse response = blockingUnaryCall(KvServiceGrpc.getPutTTLMethod(), request); + handleErrors(response.getHeader()); + return response; + } + + private void onEvent(WatchResponse value, Consumer consumer) { + log.info("receive message for {},event Count:{}", value, value.getEventsCount()); + clientId.compareAndSet(0L, value.getClientId()); + if (value.getEventsCount() != 0) { + consumer.accept((T) value); + } + } + + private StreamObserver getObserver(String key, Consumer consumer, + BiConsumer listenWrapper, + long client) { + StreamObserver observer; + if ((observer = observers.get(client)) == null) { + synchronized (this) { + if ((observer = observers.get(client)) == null) { + observer = getObserver(key, consumer, listenWrapper); + observers.put(client, observer); + } + } + } + return observer; + } + + private StreamObserver getObserver(String key, Consumer consumer, + BiConsumer listenWrapper) { + return new StreamObserver() { + @Override + public void onNext(WatchResponse value) { + switch (value.getState()) { + case Starting: + boolean b = clientId.compareAndSet(0, value.getClientId()); + if (b) { + observers.put(value.getClientId(), this); + log.info("set watch client id to :{}", value.getClientId()); + } + semaphore.release(); + break; + case Started: + onEvent(value, consumer); + break; + case Leader_Changed: + listenWrapper.accept(key, consumer); + break; + case Alive: + // only for check client is alive, do nothing + break; + default: + break; + } + } + + @Override + public void onError(Throwable t) { + listenWrapper.accept(key, consumer); + } + + + @Override + public void onCompleted() { + + } + }; + } + + public void listen(String key, Consumer consumer) throws PDException { + long value = clientId.get(); + StreamObserver observer = getObserver(key, consumer, listenWrapper, value); + acquire(); + WatchRequest k = WatchRequest.newBuilder().setClientId(value).setKey(key).build(); + streamingCall(KvServiceGrpc.getWatchMethod(), k, observer, 1); + } + + public void listenPrefix(String prefix, Consumer consumer) throws PDException { + long value = clientId.get(); + StreamObserver observer = + getObserver(prefix, consumer, prefixListenWrapper, value); + acquire(); + WatchRequest k = + WatchRequest.newBuilder().setClientId(clientId.get()).setKey(prefix).build(); + streamingCall(KvServiceGrpc.getWatchPrefixMethod(), k, observer, 1); + } + + private void acquire() { + if (clientId.get() == 0L) { + try { + semaphore.acquire(); + if (clientId.get() != 0L) { + semaphore.release(); + } + } catch (InterruptedException e) { + log.error("get semaphore with error:", e); + } + } + } + + public List getWatchList(T response) { + List values = new LinkedList<>(); + List eventsList = response.getEventsList(); + for (WatchEvent event : eventsList) { + if (event.getType() != WatchType.Put) { + return null; + } + String value = event.getCurrent().getValue(); + values.add(value); + } + return values; + } + + public Map getWatchMap(T response) { + Map values = new HashMap<>(); + List eventsList = response.getEventsList(); + for (WatchEvent event : eventsList) { + if (event.getType() != WatchType.Put) { + return null; + } + WatchKv current = event.getCurrent(); + String key = current.getKey(); + String value = current.getValue(); + values.put(key, value); + } + return values; + } + + public LockResponse lock(String key, long ttl) throws PDException { + acquire(); + LockResponse response; + try { + LockRequest k = + LockRequest.newBuilder().setKey(key).setClientId(clientId.get()).setTtl(ttl) + .build(); + response = blockingUnaryCall(KvServiceGrpc.getLockMethod(), k); + handleErrors(response.getHeader()); + if (clientId.compareAndSet(0L, response.getClientId())) { + semaphore.release(); + } + } catch (Exception e) { + if (clientId.get() == 0L) { + semaphore.release(); + } + throw e; + } + return response; + } BiConsumer listenWrapper = (key, consumer) -> { + try { + listen(key, consumer); + } catch (PDException e) { + try { + log.warn("start listen with warning:", e); + Thread.sleep(1000); + } catch (InterruptedException ex) { + } + } + }; + + public LockResponse lockWithoutReentrant(String key, long ttl) throws PDException { + acquire(); + LockResponse response; + try { + LockRequest k = + LockRequest.newBuilder().setKey(key).setClientId(clientId.get()).setTtl(ttl) + .build(); + response = blockingUnaryCall(KvServiceGrpc.getLockWithoutReentrantMethod(), k); + handleErrors(response.getHeader()); + if (clientId.compareAndSet(0L, response.getClientId())) { + semaphore.release(); + } + } catch (Exception e) { + if (clientId.get() == 0L) { + semaphore.release(); + } + throw e; + } + return response; + } + + public LockResponse isLocked(String key) throws PDException { + LockRequest k = LockRequest.newBuilder().setKey(key).setClientId(clientId.get()).build(); + LockResponse response = blockingUnaryCall(KvServiceGrpc.getIsLockedMethod(), k); + handleErrors(response.getHeader()); + return response; + } + + public LockResponse unlock(String key) throws PDException { + assert clientId.get() != 0; + LockRequest k = LockRequest.newBuilder().setKey(key).setClientId(clientId.get()).build(); + LockResponse response = blockingUnaryCall(KvServiceGrpc.getUnlockMethod(), k); + handleErrors(response.getHeader()); + clientId.compareAndSet(0L, response.getClientId()); + assert clientId.get() == response.getClientId(); + return response; + } + + public LockResponse keepAlive(String key) throws PDException { + assert clientId.get() != 0; + LockRequest k = LockRequest.newBuilder().setKey(key).setClientId(clientId.get()).build(); + LockResponse response = blockingUnaryCall(KvServiceGrpc.getKeepAliveMethod(), k); + handleErrors(response.getHeader()); + clientId.compareAndSet(0L, response.getClientId()); + assert clientId.get() == response.getClientId(); + return response; + } + + @Override + public void close() { + super.close(); + } + + + + BiConsumer prefixListenWrapper = (key, consumer) -> { + try { + listenPrefix(key, consumer); + } catch (PDException e) { + try { + log.warn("start listenPrefix with warning:", e); + Thread.sleep(1000); + } catch (InterruptedException ex) { + } + } + }; + + +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/LicenseClient.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/LicenseClient.java new file mode 100644 index 0000000000..a96185e5af --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/LicenseClient.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import org.apache.hugegraph.pd.common.KVPair; +import org.apache.hugegraph.pd.grpc.PDGrpc; +import org.apache.hugegraph.pd.grpc.Pdpb; + +import com.google.protobuf.ByteString; + +import io.grpc.stub.AbstractBlockingStub; +import io.grpc.stub.AbstractStub; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class LicenseClient extends AbstractClient { + + public LicenseClient(PDConfig config) { + super(config); + } + + @Override + protected AbstractStub createStub() { + return PDGrpc.newStub(channel); + } + + @Override + protected AbstractBlockingStub createBlockingStub() { + return PDGrpc.newBlockingStub(channel); + } + + public Pdpb.PutLicenseResponse putLicense(byte[] content) { + Pdpb.PutLicenseRequest request = Pdpb.PutLicenseRequest.newBuilder() + .setContent( + ByteString.copyFrom(content)) + .build(); + try { + KVPair pair = concurrentBlockingUnaryCall( + PDGrpc.getPutLicenseMethod(), request, + (rs) -> rs.getHeader().getError().getType().equals(Pdpb.ErrorType.OK)); + if (pair.getKey()) { + Pdpb.PutLicenseResponse.Builder builder = Pdpb.PutLicenseResponse.newBuilder(); + builder.setHeader(okHeader); + return builder.build(); + } else { + return pair.getValue(); + } + } catch (Exception e) { + e.printStackTrace(); + log.debug("put license with error:{} ", e); + Pdpb.ResponseHeader rh = + newErrorHeader(Pdpb.ErrorType.LICENSE_ERROR_VALUE, e.getMessage()); + return Pdpb.PutLicenseResponse.newBuilder().setHeader(rh).build(); + } + } +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDClient.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDClient.java new file mode 100644 index 0000000000..fa1e595264 --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDClient.java @@ -0,0 +1,1345 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import static org.apache.hugegraph.pd.watch.NodeEvent.EventType.NODE_PD_LEADER_CHANGE; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; + +import org.apache.hugegraph.pd.common.KVPair; +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.common.PartitionUtils; +import org.apache.hugegraph.pd.grpc.MetaTask; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.pd.grpc.Metapb.ShardGroup; +import org.apache.hugegraph.pd.grpc.PDGrpc; +import org.apache.hugegraph.pd.grpc.Pdpb; +import org.apache.hugegraph.pd.grpc.Pdpb.CachePartitionResponse; +import org.apache.hugegraph.pd.grpc.Pdpb.CacheResponse; +import org.apache.hugegraph.pd.grpc.Pdpb.GetGraphRequest; +import org.apache.hugegraph.pd.grpc.Pdpb.GetPartitionByCodeRequest; +import org.apache.hugegraph.pd.grpc.Pdpb.GetPartitionRequest; +import org.apache.hugegraph.pd.grpc.Pdpb.GetPartitionResponse; +import org.apache.hugegraph.pd.grpc.watch.WatchResponse; +import org.apache.hugegraph.pd.watch.NodeEvent; +import org.apache.hugegraph.pd.watch.PartitionEvent; + +import com.google.protobuf.ByteString; + +import io.grpc.ManagedChannel; +import io.grpc.MethodDescriptor; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.AbstractBlockingStub; +import lombok.extern.slf4j.Slf4j; + +/** + * PD客户端实现类 + */ +@Slf4j +public class PDClient { + private final PDConfig config; + private final Pdpb.RequestHeader header; + private final ClientCache cache; + private final StubProxy stubProxy; + private final List eventListeners; + private PDWatch.Watcher partitionWatcher; + private PDWatch.Watcher storeWatcher; + private PDWatch.Watcher graphWatcher; + private PDWatch.Watcher shardGroupWatcher; + private PDWatch pdWatch; + + private PDClient(PDConfig config) { + this.config = config; + this.header = Pdpb.RequestHeader.getDefaultInstance(); + this.stubProxy = new StubProxy(config.getServerHost().split(",")); + this.eventListeners = new CopyOnWriteArrayList<>(); + this.cache = new ClientCache(this); + } + + /** + * 创建PDClient对象，并初始化stub + * + * @param config + * @return + */ + public static PDClient create(PDConfig config) { + return new PDClient(config); + } + + private synchronized void newBlockingStub() throws PDException { + if (stubProxy.get() != null) { + return; + } + + String host = newLeaderStub(); + if (host.isEmpty()) { + throw new PDException(Pdpb.ErrorType.PD_UNREACHABLE_VALUE, + "PD unreachable, pd.peers=" + config.getServerHost()); + } + + log.info("PDClient enable cache, init PDWatch object"); + connectPdWatch(host); + } + + public void connectPdWatch(String leader) { + + if (pdWatch != null && Objects.equals(pdWatch.getCurrentHost(), leader) && + pdWatch.checkChannel()) { + return; + } + + log.info("PDWatch client connect host:{}", leader); + pdWatch = new PDWatchImpl(leader); + + partitionWatcher = pdWatch.watchPartition(new PDWatch.Listener<>() { + @Override + public void onNext(PartitionEvent response) { + // log.info("PDClient receive partition event {}-{} {}", + // response.getGraph(), response.getPartitionId(), response.getChangeType()); + invalidPartitionCache(response.getGraph(), response.getPartitionId()); + + if (response.getChangeType() == PartitionEvent.ChangeType.DEL) { + cache.removeAll(response.getGraph()); + } + + eventListeners.forEach(listener -> { + listener.onPartitionChanged(response); + }); + } + + @Override + public void onError(Throwable throwable) { + log.error("watchPartition exception {}", throwable.getMessage()); + closeStub(false); + } + }); + + storeWatcher = pdWatch.watchNode(new PDWatch.Listener<>() { + @Override + public void onNext(NodeEvent response) { + log.info("PDClient receive store event {} {}", + response.getEventType(), Long.toHexString(response.getNodeId())); + + if (response.getEventType() == NODE_PD_LEADER_CHANGE) { + // pd raft change + var leaderIp = response.getGraph(); + log.info("watchNode: pd leader changed to {}, current watch:{}", + leaderIp, pdWatch.getCurrentHost()); + closeStub(!Objects.equals(pdWatch.getCurrentHost(), leaderIp)); + connectPdWatch(leaderIp); + } + + invalidStoreCache(response.getNodeId()); + eventListeners.forEach(listener -> { + listener.onStoreChanged(response); + }); + } + + @Override + public void onError(Throwable throwable) { + log.error("watchNode exception {}", throwable.getMessage()); + closeStub(false); + } + + }); + + graphWatcher = pdWatch.watchGraph(new PDWatch.Listener<>() { + @Override + public void onNext(WatchResponse response) { + eventListeners.forEach(listener -> { + listener.onGraphChanged(response); + }); + } + + @Override + public void onError(Throwable throwable) { + log.warn("graphWatcher exception {}", throwable.getMessage()); + } + }); + + shardGroupWatcher = pdWatch.watchShardGroup(new PDWatch.Listener<>() { + @Override + public void onNext(WatchResponse response) { + var shardResponse = response.getShardGroupResponse(); + // log.info("PDClient receive shard group event: raft {}-{}", shardResponse + // .getShardGroupId(), + // shardResponse.getType()); + if (config.isEnableCache()) { + switch (shardResponse.getType()) { + case WATCH_CHANGE_TYPE_DEL: + cache.deleteShardGroup(shardResponse.getShardGroupId()); + break; + case WATCH_CHANGE_TYPE_ALTER: + cache.updateShardGroup( + response.getShardGroupResponse().getShardGroup()); + break; + default: + break; + } + } + eventListeners.forEach(listener -> listener.onShardGroupChanged(response)); + } + + @Override + public void onError(Throwable throwable) { + log.warn("shardGroupWatcher exception {}", throwable.getMessage()); + } + }); + + } + + private synchronized void closeStub(boolean closeWatcher) { + // TODO ManagedChannel 没有正常关闭 + stubProxy.set(null); + cache.reset(); + + if (closeWatcher) { + if (partitionWatcher != null) { + partitionWatcher.close(); + partitionWatcher = null; + } + if (storeWatcher != null) { + storeWatcher.close(); + storeWatcher = null; + } + if (graphWatcher != null) { + graphWatcher.close(); + graphWatcher = null; + } + + if (shardGroupWatcher != null) { + shardGroupWatcher.close(); + shardGroupWatcher = null; + } + + pdWatch = null; + } + } + + private PDGrpc.PDBlockingStub getStub() throws PDException { + if (stubProxy.get() == null) { + newBlockingStub(); + } + return stubProxy.get().withDeadlineAfter(config.getGrpcTimeOut(), TimeUnit.MILLISECONDS); + } + + private PDGrpc.PDBlockingStub newStub() throws PDException { + if (stubProxy.get() == null) { + newBlockingStub(); + } + return PDGrpc.newBlockingStub(stubProxy.get().getChannel()) + .withDeadlineAfter(config.getGrpcTimeOut(), + TimeUnit.MILLISECONDS); + } + + private String newLeaderStub() { + String leaderHost = ""; + for (int i = 0; i < stubProxy.getHostCount(); i++) { + String host = stubProxy.nextHost(); + ManagedChannel channel = Channels.getChannel(host); + + PDGrpc.PDBlockingStub stub = PDGrpc.newBlockingStub(channel) + .withDeadlineAfter(config.getGrpcTimeOut(), + TimeUnit.MILLISECONDS); + try { + var leaderIp = getLeaderIp(stub); + if (!leaderIp.equalsIgnoreCase(host)) { + leaderHost = leaderIp; + stubProxy.set(PDGrpc.newBlockingStub(channel) + .withDeadlineAfter(config.getGrpcTimeOut(), + TimeUnit.MILLISECONDS)); + } else { + stubProxy.set(stub); + leaderHost = host; + } + stubProxy.setLeader(leaderIp); + + log.info("PDClient connect to host = {} success", leaderHost); + break; + } catch (Exception e) { + log.error("PDClient connect to {} exception {}, {}", host, e.getMessage(), + e.getCause() != null ? e.getCause().getMessage() : ""); + } + } + return leaderHost; + } + + public String getLeaderIp() { + + return getLeaderIp(stubProxy.get()); + } + + private String getLeaderIp(PDGrpc.PDBlockingStub stub) { + if (stub == null) { + try { + getStub(); + return stubProxy.getLeader(); + } catch (PDException e) { + throw new RuntimeException(e); + } + } + + Pdpb.GetMembersRequest request = Pdpb.GetMembersRequest.newBuilder() + .setHeader(header) + .build(); + Metapb.Member leader = stub.getMembers(request).getLeader(); + return leader.getGrpcUrl(); + } + + /** + * Store注册，返回storeID，初次注册会返回新ID + * + * @param store + * @return + */ + public long registerStore(Metapb.Store store) throws PDException { + Pdpb.RegisterStoreRequest request = Pdpb.RegisterStoreRequest.newBuilder() + .setHeader(header) + .setStore(store).build(); + + Pdpb.RegisterStoreResponse response = + blockingUnaryCall(PDGrpc.getRegisterStoreMethod(), request); + handleResponseError(response.getHeader()); + return response.getStoreId(); + } + + /** + * 根据storeId返回Store对象 + * + * @param storeId + * @return + * @throws PDException + */ + public Metapb.Store getStore(long storeId) throws PDException { + Metapb.Store store = cache.getStoreById(storeId); + if (store == null) { + Pdpb.GetStoreRequest request = Pdpb.GetStoreRequest.newBuilder() + .setHeader(header) + .setStoreId(storeId).build(); + Pdpb.GetStoreResponse response = getStub().getStore(request); + handleResponseError(response.getHeader()); + store = response.getStore(); + if (config.isEnableCache()) { + cache.addStore(storeId, store); + } + } + return store; + } + + /** + * 更新Store信息，包括上下线等 + * + * @param store + * @return + */ + public Metapb.Store updateStore(Metapb.Store store) throws PDException { + Pdpb.SetStoreRequest request = Pdpb.SetStoreRequest.newBuilder() + .setHeader(header) + .setStore(store).build(); + + Pdpb.SetStoreResponse response = getStub().setStore(request); + handleResponseError(response.getHeader()); + store = response.getStore(); + if (config.isEnableCache()) { + cache.addStore(store.getId(), store); + } + return store; + } + + /** + * 返回活跃的Store + * + * @param graphName + * @return + */ + public List getActiveStores(String graphName) throws PDException { + List stores = new ArrayList<>(); + KVPair ptShard = this.getPartitionByCode(graphName, 0); + while (ptShard != null) { + stores.add(this.getStore(ptShard.getValue().getStoreId())); + if (ptShard.getKey().getEndKey() < PartitionUtils.MAX_VALUE) { + ptShard = this.getPartitionByCode(graphName, ptShard.getKey().getEndKey()); + } else { + ptShard = null; + } + } + return stores; + } + + public List getActiveStores() throws PDException { + Pdpb.GetAllStoresRequest request = Pdpb.GetAllStoresRequest.newBuilder() + .setHeader(header) + .setGraphName("") + .setExcludeOfflineStores(true) + .build(); + Pdpb.GetAllStoresResponse response = getStub().getAllStores(request); + handleResponseError(response.getHeader()); + return response.getStoresList(); + + } + + /** + * 返回活跃的Store + * + * @param graphName + * @return + */ + public List getAllStores(String graphName) throws PDException { + Pdpb.GetAllStoresRequest request = Pdpb.GetAllStoresRequest.newBuilder() + .setHeader(header) + .setGraphName(graphName) + .setExcludeOfflineStores(false) + .build(); + Pdpb.GetAllStoresResponse response = getStub().getAllStores(request); + handleResponseError(response.getHeader()); + return response.getStoresList(); + + } + + /** + * Store心跳，定期调用，保持在线状态 + * + * @param stats + * @throws PDException + */ + public Metapb.ClusterStats storeHeartbeat(Metapb.StoreStats stats) throws PDException { + Pdpb.StoreHeartbeatRequest request = Pdpb.StoreHeartbeatRequest.newBuilder() + .setHeader(header) + .setStats(stats).build(); + Pdpb.StoreHeartbeatResponse response = getStub().storeHeartbeat(request); + handleResponseError(response.getHeader()); + return response.getClusterStats(); + } + + private KVPair getKvPair(String graphName, byte[] key, + KVPair partShard) throws + PDException { + if (partShard == null) { + GetPartitionRequest request = GetPartitionRequest.newBuilder() + .setHeader(header) + .setGraphName(graphName) + .setKey(ByteString.copyFrom(key)) + .build(); + GetPartitionResponse response = + blockingUnaryCall(PDGrpc.getGetPartitionMethod(), request); + handleResponseError(response.getHeader()); + partShard = new KVPair<>(response.getPartition(), response.getLeader()); + cache.update(graphName, partShard.getKey().getId(), partShard.getKey()); + } + return partShard; + } + + /** + * 查询Key所属分区信息 + * + * @param graphName + * @param key + * @return + * @throws PDException + */ + public KVPair getPartition(String graphName, byte[] key) throws + PDException { + // 先查cache，cache没有命中，在调用PD + KVPair partShard = cache.getPartitionByKey(graphName, key); + partShard = getKvPair(graphName, key, partShard); + return partShard; + } + + public KVPair getPartition(String graphName, byte[] key, + int code) throws + PDException { + KVPair partShard = + cache.getPartitionByCode(graphName, code); + partShard = getKvPair(graphName, key, partShard); + return partShard; + } + + /** + * 根据hashcode查询所属分区信息 + * + * @param graphName + * @param hashCode + * @return + * @throws PDException + */ + public KVPair getPartitionByCode(String graphName, + long hashCode) + throws PDException { + // 先查cache，cache没有命中，在调用PD + KVPair partShard = + cache.getPartitionByCode(graphName, hashCode); + if (partShard == null) { + GetPartitionByCodeRequest request = GetPartitionByCodeRequest.newBuilder() + .setHeader(header) + .setGraphName(graphName) + .setCode(hashCode).build(); + GetPartitionResponse response = + blockingUnaryCall(PDGrpc.getGetPartitionByCodeMethod(), request); + handleResponseError(response.getHeader()); + partShard = new KVPair<>(response.getPartition(), response.getLeader()); + cache.update(graphName, partShard.getKey().getId(), partShard.getKey()); + cache.updateShardGroup(getShardGroup(partShard.getKey().getId())); + } + + if (partShard.getValue() == null) { + ShardGroup shardGroup = getShardGroup(partShard.getKey().getId()); + if (shardGroup != null) { + for (var shard : shardGroup.getShardsList()) { + if (shard.getRole() == Metapb.ShardRole.Leader) { + partShard.setValue(shard); + } + } + } else { + log.error("getPartitionByCode: get shard group failed, {}", + partShard.getKey().getId()); + } + } + return partShard; + } + + /** + * 获取Key的哈希值 + */ + public int keyToCode(String graphName, byte[] key) { + return PartitionUtils.calcHashcode(key); + } + + /** + * 根据分区id返回分区信息, RPC请求 + * + * @param graphName + * @param partId + * @return + * @throws PDException + */ + public KVPair getPartitionById(String graphName, + int partId) throws PDException { + KVPair partShard = + cache.getPartitionById(graphName, partId); + if (partShard == null) { + Pdpb.GetPartitionByIDRequest request = Pdpb.GetPartitionByIDRequest.newBuilder() + .setHeader(header) + .setGraphName( + graphName) + .setPartitionId( + partId) + .build(); + GetPartitionResponse response = + blockingUnaryCall(PDGrpc.getGetPartitionByIDMethod(), request); + handleResponseError(response.getHeader()); + partShard = new KVPair<>(response.getPartition(), response.getLeader()); + if (config.isEnableCache()) { + cache.update(graphName, partShard.getKey().getId(), partShard.getKey()); + cache.updateShardGroup(getShardGroup(partShard.getKey().getId())); + } + } + if (partShard.getValue() == null) { + var shardGroup = getShardGroup(partShard.getKey().getId()); + if (shardGroup != null) { + for (var shard : shardGroup.getShardsList()) { + if (shard.getRole() == Metapb.ShardRole.Leader) { + partShard.setValue(shard); + } + } + } else { + log.error("getPartitionById: get shard group failed, {}", + partShard.getKey().getId()); + } + } + return partShard; + } + + public ShardGroup getShardGroup(int partId) throws PDException { + ShardGroup group = cache.getShardGroup(partId); + if (group == null) { + Pdpb.GetShardGroupRequest request = Pdpb.GetShardGroupRequest.newBuilder() + .setHeader(header) + .setGroupId(partId) + .build(); + Pdpb.GetShardGroupResponse response = + blockingUnaryCall(PDGrpc.getGetShardGroupMethod(), request); + handleResponseError(response.getHeader()); + group = response.getShardGroup(); + if (config.isEnableCache()) { + cache.updateShardGroup(group); + } + } + return group; + } + + public void updateShardGroup(ShardGroup shardGroup) throws PDException { + Pdpb.UpdateShardGroupRequest request = Pdpb.UpdateShardGroupRequest.newBuilder() + .setHeader(header) + .setShardGroup( + shardGroup) + .build(); + Pdpb.UpdateShardGroupResponse response = + blockingUnaryCall(PDGrpc.getUpdateShardGroupMethod(), request); + handleResponseError(response.getHeader()); + + if (config.isEnableCache()) { + cache.updateShardGroup(shardGroup); + } + } + + /** + * 返回startKey和endKey跨越的所有分区信息 + * + * @param graphName + * @param startKey + * @param endKey + * @return + * @throws PDException + */ + public List> scanPartitions(String graphName, + byte[] startKey, + byte[] endKey) throws + PDException { + List> partitions = new ArrayList<>(); + KVPair startPartShard = getPartition(graphName, startKey); + KVPair endPartShard = getPartition(graphName, endKey); + if (startPartShard == null || endPartShard == null) { + return null; + } + + partitions.add(startPartShard); + while (startPartShard.getKey().getEndKey() < endPartShard.getKey().getEndKey() + && startPartShard.getKey().getEndKey() < + PartitionUtils.MAX_VALUE /*排除最后一个分区*/) { + startPartShard = getPartitionByCode(graphName, startPartShard.getKey().getEndKey()); + partitions.add(startPartShard); + } + return partitions; + } + + /** + * 根据条件查询分区信息 + * + * @return + * @throws PDException + */ + public List getPartitionsByStore(long storeId) throws PDException { + + Metapb.PartitionQuery query = Metapb.PartitionQuery.newBuilder() + .setStoreId(storeId) + .build(); + Pdpb.QueryPartitionsRequest request = Pdpb.QueryPartitionsRequest.newBuilder() + .setQuery(query).build(); + Pdpb.QueryPartitionsResponse response = + blockingUnaryCall(PDGrpc.getQueryPartitionsMethod(), request); + + handleResponseError(response.getHeader()); + return response.getPartitionsList(); + } + + /** + * 查找指定store上的指定partitionId + * + * @return + * @throws PDException + */ + public List queryPartitions(long storeId, int partitionId) throws + PDException { + + Metapb.PartitionQuery query = Metapb.PartitionQuery.newBuilder() + .setStoreId(storeId) + .setPartitionId(partitionId) + .build(); + Pdpb.QueryPartitionsRequest request = Pdpb.QueryPartitionsRequest.newBuilder() + .setQuery(query).build(); + Pdpb.QueryPartitionsResponse response = + blockingUnaryCall(PDGrpc.getQueryPartitionsMethod(), request); + + handleResponseError(response.getHeader()); + return response.getPartitionsList(); + } + + public List getPartitions(long storeId, String graphName) throws PDException { + + Metapb.PartitionQuery query = Metapb.PartitionQuery.newBuilder() + .setStoreId(storeId) + .setGraphName(graphName).build(); + Pdpb.QueryPartitionsRequest request = Pdpb.QueryPartitionsRequest.newBuilder() + .setQuery(query).build(); + Pdpb.QueryPartitionsResponse response = + blockingUnaryCall(PDGrpc.getQueryPartitionsMethod(), request); + + handleResponseError(response.getHeader()); + return response.getPartitionsList(); + + } + + public Metapb.Graph setGraph(Metapb.Graph graph) throws PDException { + Pdpb.SetGraphRequest request = Pdpb.SetGraphRequest.newBuilder() + .setGraph(graph) + .build(); + Pdpb.SetGraphResponse response = + blockingUnaryCall(PDGrpc.getSetGraphMethod(), request); + + handleResponseError(response.getHeader()); + return response.getGraph(); + } + + public Metapb.Graph getGraph(String graphName) throws PDException { + GetGraphRequest request = GetGraphRequest.newBuilder() + .setGraphName(graphName) + .build(); + Pdpb.GetGraphResponse response = + blockingUnaryCall(PDGrpc.getGetGraphMethod(), request); + + handleResponseError(response.getHeader()); + return response.getGraph(); + } + + public Metapb.Graph getGraphWithOutException(String graphName) throws + PDException { + GetGraphRequest request = GetGraphRequest.newBuilder() + .setGraphName( + graphName) + .build(); + Pdpb.GetGraphResponse response = blockingUnaryCall( + PDGrpc.getGetGraphMethod(), request); + return response.getGraph(); + } + + public Metapb.Graph delGraph(String graphName) throws PDException { + Pdpb.DelGraphRequest request = Pdpb.DelGraphRequest.newBuilder() + .setGraphName(graphName) + .build(); + Pdpb.DelGraphResponse response = + blockingUnaryCall(PDGrpc.getDelGraphMethod(), request); + + handleResponseError(response.getHeader()); + return response.getGraph(); + } + + public List updatePartition(List partitions) throws + PDException { + + Pdpb.UpdatePartitionRequest request = Pdpb.UpdatePartitionRequest.newBuilder() + .addAllPartition( + partitions) + .build(); + Pdpb.UpdatePartitionResponse response = + blockingUnaryCall(PDGrpc.getUpdatePartitionMethod(), request); + handleResponseError(response.getHeader()); + invalidPartitionCache(); + + return response.getPartitionList(); + } + + public Metapb.Partition delPartition(String graphName, int partitionId) throws PDException { + + Pdpb.DelPartitionRequest request = Pdpb.DelPartitionRequest.newBuilder() + .setGraphName(graphName) + .setPartitionId(partitionId) + .build(); + Pdpb.DelPartitionResponse response = + blockingUnaryCall(PDGrpc.getDelPartitionMethod(), request); + + handleResponseError(response.getHeader()); + invalidPartitionCache(graphName, partitionId); + return response.getPartition(); + } + + /** + * 删除分区缓存 + */ + public void invalidPartitionCache(String graphName, int partitionId) { + // 检查是否存在缓存 + if (null != cache.getPartitionById(graphName, partitionId)) { + cache.removePartition(graphName, partitionId); + } + } + + /** + * 删除分区缓存 + */ + public void invalidPartitionCache() { + // 检查是否存在缓存 + cache.removePartitions(); + } + + /** + * 删除分区缓存 + */ + public void invalidStoreCache(long storeId) { + cache.removeStore(storeId); + } + + /** + * Hugegraph server 调用，Leader发生改变，更新缓存 + */ + public void updatePartitionLeader(String graphName, int partId, long leaderStoreId) { + KVPair partShard = null; + try { + partShard = this.getPartitionById(graphName, partId); + + if (partShard != null && partShard.getValue().getStoreId() != leaderStoreId) { + var shardGroup = this.getShardGroup(partId); + Metapb.Shard shard = null; + List shards = new ArrayList<>(); + + for (Metapb.Shard s : shardGroup.getShardsList()) { + if (s.getStoreId() == leaderStoreId) { + shard = s; + shards.add(Metapb.Shard.newBuilder(s) + .setStoreId(s.getStoreId()) + .setRole(Metapb.ShardRole.Leader).build()); + } else { + shards.add(Metapb.Shard.newBuilder(s) + .setStoreId(s.getStoreId()) + .setRole(Metapb.ShardRole.Follower).build()); + } + } + + if (config.isEnableCache()) { + if (shard == null) { + // 分区的shard中未找到leader，说明分区发生了迁移 + cache.removePartition(graphName, partId); + } + } + } + } catch (PDException e) { + log.error("getPartitionException: {}", e.getMessage()); + } + } + + /** + * Hugegraph-store调用，更新缓存 + * + * @param partition + */ + public void updatePartitionCache(Metapb.Partition partition, Metapb.Shard leader) { + if (config.isEnableCache()) { + cache.update(partition.getGraphName(), partition.getId(), partition); + cache.updateLeader(partition.getId(), leader); + } + } + + public Pdpb.GetIdResponse getIdByKey(String key, int delta) throws PDException { + Pdpb.GetIdRequest request = Pdpb.GetIdRequest.newBuilder() + .setHeader(header) + .setKey(key) + .setDelta(delta) + .build(); + Pdpb.GetIdResponse response = blockingUnaryCall(PDGrpc.getGetIdMethod(), request); + handleResponseError(response.getHeader()); + return response; + } + + public Pdpb.ResetIdResponse resetIdByKey(String key) throws PDException { + Pdpb.ResetIdRequest request = Pdpb.ResetIdRequest.newBuilder() + .setHeader(header) + .setKey(key) + .build(); + Pdpb.ResetIdResponse response = blockingUnaryCall(PDGrpc.getResetIdMethod(), request); + handleResponseError(response.getHeader()); + return response; + } + + public Metapb.Member getLeader() throws PDException { + Pdpb.GetMembersRequest request = Pdpb.GetMembersRequest.newBuilder() + .setHeader(header) + .build(); + Pdpb.GetMembersResponse response = blockingUnaryCall(PDGrpc.getGetMembersMethod(), request); + handleResponseError(response.getHeader()); + return response.getLeader(); + } + + public Pdpb.GetMembersResponse getMembers() throws PDException { + Pdpb.GetMembersRequest request = Pdpb.GetMembersRequest.newBuilder() + .setHeader(header) + .build(); + Pdpb.GetMembersResponse response = blockingUnaryCall(PDGrpc.getGetMembersMethod(), request); + handleResponseError(response.getHeader()); + return response; + } + + public Metapb.ClusterStats getClusterStats() throws PDException { + Pdpb.GetClusterStatsRequest request = Pdpb.GetClusterStatsRequest.newBuilder() + .setHeader(header) + .build(); + Pdpb.GetClusterStatsResponse response = + blockingUnaryCall(PDGrpc.getGetClusterStatsMethod(), request); + handleResponseError(response.getHeader()); + return response.getCluster(); + } + + private > RespT + blockingUnaryCall(MethodDescriptor method, ReqT req) throws PDException { + return blockingUnaryCall(method, req, 1); + } + + private > RespT + blockingUnaryCall(MethodDescriptor method, ReqT req, int retry) throws + PDException { + io.grpc.stub.AbstractBlockingStub stub = (AbstractBlockingStub) getStub(); + try { + RespT resp = io.grpc.stub.ClientCalls.blockingUnaryCall(stub.getChannel(), method, + stub.getCallOptions(), req); + return resp; + } catch (Exception e) { + log.error(method.getFullMethodName() + " exception, {}", e.getMessage()); + if (e instanceof StatusRuntimeException) { + StatusRuntimeException se = (StatusRuntimeException) e; + //se.getStatus() == Status.UNAVAILABLE && + if (retry < stubProxy.getHostCount()) { + // 网络不通，关掉之前连接，换host重新连接 + closeStub(true); + return blockingUnaryCall(method, req, ++retry); + } + } + } + return null; + } + + private void handleResponseError(Pdpb.ResponseHeader header) throws + PDException { + var errorType = header.getError().getType(); + if (header.hasError() && errorType != Pdpb.ErrorType.OK) { + + throw new PDException(header.getError().getTypeValue(), + String.format( + "PD request error, error code = %d, msg = %s", + header.getError().getTypeValue(), + header.getError().getMessage())); + } + } + + public void addEventListener(PDEventListener listener) { + eventListeners.add(listener); + } + + public PDWatch getWatchClient() { + return new PDWatchImpl(stubProxy.getHost()); + } + + /** + * 返回Store状态信息 + */ + public List getStoreStatus(boolean offlineExcluded) throws PDException { + Pdpb.GetAllStoresRequest request = Pdpb.GetAllStoresRequest.newBuilder() + .setHeader(header) + .setExcludeOfflineStores( + offlineExcluded) + .build(); + Pdpb.GetAllStoresResponse response = getStub().getStoreStatus(request); + handleResponseError(response.getHeader()); + List stores = response.getStoresList(); + return stores; + } + + public void setGraphSpace(String graphSpaceName, long storageLimit) throws PDException { + Metapb.GraphSpace graphSpace = Metapb.GraphSpace.newBuilder().setName(graphSpaceName) + .setStorageLimit(storageLimit) + .setTimestamp(System.currentTimeMillis()) + .build(); + Pdpb.SetGraphSpaceRequest request = Pdpb.SetGraphSpaceRequest.newBuilder() + .setHeader(header) + .setGraphSpace(graphSpace) + .build(); + Pdpb.SetGraphSpaceResponse response = getStub().setGraphSpace(request); + handleResponseError(response.getHeader()); + } + + public List getGraphSpace(String graphSpaceName) throws + PDException { + Pdpb.GetGraphSpaceRequest.Builder builder = Pdpb.GetGraphSpaceRequest.newBuilder(); + Pdpb.GetGraphSpaceRequest request; + builder.setHeader(header); + if (graphSpaceName != null && graphSpaceName.length() > 0) { + builder.setGraphSpaceName(graphSpaceName); + } + request = builder.build(); + Pdpb.GetGraphSpaceResponse response = getStub().getGraphSpace(request); + List graphSpaceList = response.getGraphSpaceList(); + handleResponseError(response.getHeader()); + return graphSpaceList; + } + + public void setPDConfig(int partitionCount, String peerList, int shardCount, + long version) throws PDException { + Metapb.PDConfig pdConfig = Metapb.PDConfig.newBuilder().setPartitionCount(partitionCount) + .setPeersList(peerList).setShardCount(shardCount) + .setVersion(version) + .setTimestamp(System.currentTimeMillis()) + .build(); + Pdpb.SetPDConfigRequest request = Pdpb.SetPDConfigRequest.newBuilder() + .setHeader(header) + .setPdConfig(pdConfig) + .build(); + Pdpb.SetPDConfigResponse response = getStub().setPDConfig(request); + handleResponseError(response.getHeader()); + } + + public Metapb.PDConfig getPDConfig() throws PDException { + Pdpb.GetPDConfigRequest request = Pdpb.GetPDConfigRequest.newBuilder() + .setHeader(header) + .build(); + Pdpb.GetPDConfigResponse response = getStub().getPDConfig(request); + handleResponseError(response.getHeader()); + return response.getPdConfig(); + } + + public void setPDConfig(Metapb.PDConfig pdConfig) throws PDException { + Pdpb.SetPDConfigRequest request = Pdpb.SetPDConfigRequest.newBuilder() + .setHeader(header) + .setPdConfig(pdConfig) + .build(); + Pdpb.SetPDConfigResponse response = getStub().setPDConfig(request); + handleResponseError(response.getHeader()); + } + + public Metapb.PDConfig getPDConfig(long version) throws PDException { + Pdpb.GetPDConfigRequest request = Pdpb.GetPDConfigRequest.newBuilder().setHeader( + header).setVersion(version).build(); + Pdpb.GetPDConfigResponse response = getStub().getPDConfig(request); + handleResponseError(response.getHeader()); + return response.getPdConfig(); + } + + public void changePeerList(String peerList) throws PDException { + Pdpb.ChangePeerListRequest request = Pdpb.ChangePeerListRequest.newBuilder() + .setPeerList(peerList) + .setHeader(header).build(); + Pdpb.getChangePeerListResponse response = + blockingUnaryCall(PDGrpc.getChangePeerListMethod(), request); + handleResponseError(response.getHeader()); + } + + /** + * 工作模式 + * Auto：自动分裂，每个Store上分区数达到最大值 + * + * @throws PDException + */ + public void splitData() throws PDException { + Pdpb.SplitDataRequest request = Pdpb.SplitDataRequest.newBuilder() + .setHeader(header) + .setMode(Pdpb.OperationMode.Auto) + .build(); + Pdpb.SplitDataResponse response = getStub().splitData(request); + handleResponseError(response.getHeader()); + } + + /** + * 工作模式 + * Auto：自动分裂，每个Store上分区数达到最大值 + * Expert:专家模式，需要指定splitParams + * + * @param mode + * @param params + * @throws PDException + */ + public void splitData(Pdpb.OperationMode mode, List params) throws + PDException { + Pdpb.SplitDataRequest request = Pdpb.SplitDataRequest.newBuilder() + .setHeader(header) + .setMode(mode) + .addAllParam(params).build(); + Pdpb.SplitDataResponse response = getStub().splitData(request); + handleResponseError(response.getHeader()); + } + + public void splitGraphData(String graphName, int toCount) throws PDException { + Pdpb.SplitGraphDataRequest request = Pdpb.SplitGraphDataRequest.newBuilder() + .setHeader(header) + .setGraphName(graphName) + .setToCount(toCount) + .build(); + Pdpb.SplitDataResponse response = getStub().splitGraphData(request); + handleResponseError(response.getHeader()); + } + + /** + * 自动转移，达到每个Store上分区数量相同 + * + * @throws PDException + */ + public void balancePartition() throws PDException { + Pdpb.MovePartitionRequest request = Pdpb.MovePartitionRequest.newBuilder() + .setHeader(header) + .setMode( + Pdpb.OperationMode.Auto) + .build(); + Pdpb.MovePartitionResponse response = getStub().movePartition(request); + handleResponseError(response.getHeader()); + } + + /** + * //工作模式 + * // Auto：自动转移，达到每个Store上分区数量相同 + * // Expert:专家模式，需要指定transferParams + * + * @param mode + * @param params + * @throws PDException + */ + public void movePartition(Pdpb.OperationMode mode, List params) throws + PDException { + Pdpb.MovePartitionRequest request = Pdpb.MovePartitionRequest.newBuilder() + .setHeader(header) + .setMode(mode) + .addAllParam(params).build(); + Pdpb.MovePartitionResponse response = getStub().movePartition(request); + handleResponseError(response.getHeader()); + } + + public void reportTask(MetaTask.Task task) throws PDException { + Pdpb.ReportTaskRequest request = Pdpb.ReportTaskRequest.newBuilder() + .setHeader(header) + .setTask(task).build(); + Pdpb.ReportTaskResponse response = blockingUnaryCall(PDGrpc.getReportTaskMethod(), request); + handleResponseError(response.getHeader()); + } + + public Metapb.PartitionStats getPartitionsStats(String graph, int partId) throws PDException { + Pdpb.GetPartitionStatsRequest request = Pdpb.GetPartitionStatsRequest.newBuilder() + .setHeader(header) + .setGraphName(graph) + .setPartitionId(partId) + .build(); + Pdpb.GetPartitionStatsResponse response = getStub().getPartitionStats(request); + handleResponseError(response.getHeader()); + return response.getPartitionStats(); + } + + /** + * 平衡不同store中leader的数量 + */ + public void balanceLeaders() throws PDException { + Pdpb.BalanceLeadersRequest request = Pdpb.BalanceLeadersRequest.newBuilder() + .setHeader(header) + .build(); + Pdpb.BalanceLeadersResponse response = getStub().balanceLeaders(request); + handleResponseError(response.getHeader()); + } + + /** + * 从pd中删除store + */ + public Metapb.Store delStore(long storeId) throws PDException { + Pdpb.DetStoreRequest request = Pdpb.DetStoreRequest.newBuilder() + .setHeader(header) + .setStoreId(storeId) + .build(); + Pdpb.DetStoreResponse response = getStub().delStore(request); + handleResponseError(response.getHeader()); + return response.getStore(); + } + + /** + * 对rocksdb整体进行compaction + * + * @throws PDException + */ + public void dbCompaction() throws PDException { + Pdpb.DbCompactionRequest request = Pdpb.DbCompactionRequest + .newBuilder() + .setHeader(header) + .build(); + Pdpb.DbCompactionResponse response = getStub().dbCompaction(request); + handleResponseError(response.getHeader()); + } + + /** + * 对rocksdb指定表进行compaction + * + * @param tableName + * @throws PDException + */ + public void dbCompaction(String tableName) throws PDException { + Pdpb.DbCompactionRequest request = Pdpb.DbCompactionRequest + .newBuilder() + .setHeader(header) + .setTableName(tableName) + .build(); + Pdpb.DbCompactionResponse response = getStub().dbCompaction(request); + handleResponseError(response.getHeader()); + } + + /** + * 分区合并，把当前的分区缩容至toCount个 + * + * @param toCount 缩容到分区的个数 + * @throws PDException + */ + public void combineCluster(int toCount) throws PDException { + Pdpb.CombineClusterRequest request = Pdpb.CombineClusterRequest + .newBuilder() + .setHeader(header) + .setToCount(toCount) + .build(); + Pdpb.CombineClusterResponse response = getStub().combineCluster(request); + handleResponseError(response.getHeader()); + } + + /** + * 将单图缩容到 toCount个 + * + * @param graphName graph name + * @param toCount target count + * @throws PDException + */ + public void combineGraph(String graphName, int toCount) throws PDException { + Pdpb.CombineGraphRequest request = Pdpb.CombineGraphRequest + .newBuilder() + .setHeader(header) + .setGraphName(graphName) + .setToCount(toCount) + .build(); + Pdpb.CombineGraphResponse response = getStub().combineGraph(request); + handleResponseError(response.getHeader()); + } + + public void deleteShardGroup(int groupId) throws PDException { + Pdpb.DeleteShardGroupRequest request = Pdpb.DeleteShardGroupRequest + .newBuilder() + .setHeader(header) + .setGroupId(groupId) + .build(); + Pdpb.DeleteShardGroupResponse response = + blockingUnaryCall(PDGrpc.getDeleteShardGroupMethod(), request); + + handleResponseError(response.getHeader()); + } + + /** + * 用于 store的 shard list重建 + * + * @param groupId shard group id + * @param shards shard list，delete when shards size is 0 + */ + public void updateShardGroupOp(int groupId, List shards) throws PDException { + Pdpb.ChangeShardRequest request = Pdpb.ChangeShardRequest.newBuilder() + .setHeader(header) + .setGroupId(groupId) + .addAllShards(shards) + .build(); + Pdpb.ChangeShardResponse response = getStub().updateShardGroupOp(request); + handleResponseError(response.getHeader()); + } + + /** + * invoke fireChangeShard command + * + * @param groupId shard group id + * @param shards shard list + */ + public void changeShard(int groupId, List shards) throws PDException { + Pdpb.ChangeShardRequest request = Pdpb.ChangeShardRequest.newBuilder() + .setHeader(header) + .setGroupId(groupId) + .addAllShards(shards) + .build(); + Pdpb.ChangeShardResponse response = getStub().changeShard(request); + handleResponseError(response.getHeader()); + } + + public ClientCache getCache() { + return cache; + } + + public CacheResponse getClientCache() throws PDException { + GetGraphRequest request = GetGraphRequest.newBuilder().setHeader(header).build(); + CacheResponse cache = getStub().getCache(request); + handleResponseError(cache.getHeader()); + return cache; + } + + public CachePartitionResponse getPartitionCache(String graph) throws PDException { + GetGraphRequest request = + GetGraphRequest.newBuilder().setHeader(header).setGraphName(graph).build(); + CachePartitionResponse ps = getStub().getPartitions(request); + handleResponseError(ps.getHeader()); + return ps; + } + + public void updatePdRaft(String raftConfig) throws PDException { + Pdpb.UpdatePdRaftRequest request = Pdpb.UpdatePdRaftRequest.newBuilder() + .setHeader(header) + .setConfig(raftConfig) + .build(); + Pdpb.UpdatePdRaftResponse response = getStub().updatePdRaft(request); + handleResponseError(response.getHeader()); + } + + public interface PDEventListener { + void onStoreChanged(NodeEvent event); + + void onPartitionChanged(PartitionEvent event); + + void onGraphChanged(WatchResponse event); + + default void onShardGroupChanged(WatchResponse event) { + } + + } + + static class StubProxy { + + private final LinkedList hostList = new LinkedList<>(); + private volatile PDGrpc.PDBlockingStub stub; + private String leader; + + public StubProxy(String[] hosts) { + for (String host : hosts) { + if (!host.isEmpty()) { + hostList.offer(host); + } + } + } + + public String nextHost() { + String host = hostList.poll(); + hostList.offer(host); //移到尾部 + return host; + } + + public void set(PDGrpc.PDBlockingStub stub) { + this.stub = stub; + } + + public PDGrpc.PDBlockingStub get() { + return this.stub; + } + + public String getHost() { + return hostList.peek(); + } + + public int getHostCount() { + return hostList.size(); + } + + public String getLeader() { + return leader; + } + + public void setLeader(String leader) { + this.leader = leader; + } + } +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDConfig.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDConfig.java new file mode 100644 index 0000000000..16686a265b --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDConfig.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +public final class PDConfig { + //TODO multi-server + private String serverHost = "localhost:9000"; + private long grpcTimeOut = 60000; // grpc调用超时时间 10秒 + + // 是否接收PD异步通知 + private boolean enablePDNotify = false; + + private boolean enableCache = false; + + private PDConfig() { + } + + public static PDConfig of() { + return new PDConfig(); + } + + public static PDConfig of(String serverHost) { + PDConfig config = new PDConfig(); + config.serverHost = serverHost; + return config; + } + + public static PDConfig of(String serverHost, long timeOut) { + PDConfig config = new PDConfig(); + config.serverHost = serverHost; + config.grpcTimeOut = timeOut; + return config; + } + + public String getServerHost() { + return serverHost; + } + + public long getGrpcTimeOut() { + return grpcTimeOut; + } + + @Deprecated + public PDConfig setEnablePDNotify(boolean enablePDNotify) { + this.enablePDNotify = enablePDNotify; + + // TODO 临时代码，hugegraph修改完后删除 + this.enableCache = enablePDNotify; + return this; + } + + public boolean isEnableCache() { + return enableCache; + } + + public PDConfig setEnableCache(boolean enableCache) { + this.enableCache = enableCache; + return this; + } + + @Override + public String toString() { + return "PDConfig{" + + "serverHost='" + serverHost + '\'' + + '}'; + } +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulse.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulse.java new file mode 100644 index 0000000000..4b9bf32c4e --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulse.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import java.io.Closeable; +import java.util.function.Consumer; + +import org.apache.hugegraph.pd.grpc.pulse.PartitionHeartbeatRequest; +import org.apache.hugegraph.pd.grpc.pulse.PulseResponse; +import org.apache.hugegraph.pd.pulse.PulseServerNotice; + +/** + * Bidirectional communication interface of pd-client and pd-server + */ +public interface PDPulse { + + /*** inner static methods ***/ + static Listener listener(Consumer onNext) { + return listener(onNext, t -> { + }, () -> { + }); + } + + static Listener listener(Consumer onNext, Consumer onError) { + return listener(onNext, onError, () -> { + }); + } + + static Listener listener(Consumer onNext, Runnable onCompleted) { + return listener(onNext, t -> { + }, onCompleted); + } + + static Listener listener(Consumer onNext, Consumer onError, + Runnable onCompleted) { + return new Listener<>() { + @Override + public void onNext(T response) { + onNext.accept(response); + } + + @Override + public void onNotice(PulseServerNotice notice) { + + } + + @Override + public void onError(Throwable throwable) { + onError.accept(throwable); + } + + @Override + public void onCompleted() { + onCompleted.run(); + } + }; + } + + /** + * @param listener + * @return + */ + Notifier connectPartition(Listener listener); + + /** + * 切换成新的host。做 channel/host的检查，如果需要关闭，notifier调用close方法。 + * + * @param host new host + * @param notifier notifier + * @return true if create new stub, otherwise false + */ + boolean resetStub(String host, Notifier notifier); + + /** + * Interface of pulse. + */ + interface Listener { + /** + * Invoked on new events. + * + * @param response the response. + */ + @Deprecated + default void onNext(T response) { + } + + /** + * Invoked on new events. + * + * @param notice a wrapper of response + */ + default void onNotice(PulseServerNotice notice) { + notice.ack(); + } + + /** + * Invoked on errors. + * + * @param throwable the error. + */ + void onError(Throwable throwable); + + /** + * Invoked on completion. + */ + void onCompleted(); + + } + + /** + * Interface of notifier that can send notice to server. + * + * @param + */ + interface Notifier extends Closeable { + /** + * closes this watcher and all its resources. + */ + @Override + void close(); + + /** + * Send notice to pd-server. + * + * @return + */ + void notifyServer(T t); + + /** + * Send an error report to pd-server. + * + * @param error + */ + void crash(String error); + + } +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulseImpl.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulseImpl.java new file mode 100644 index 0000000000..d01cd42d70 --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulseImpl.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.apache.hugegraph.pd.grpc.pulse.HgPdPulseGrpc; +import org.apache.hugegraph.pd.grpc.pulse.PartitionHeartbeatRequest; +import org.apache.hugegraph.pd.grpc.pulse.PulseAckRequest; +import org.apache.hugegraph.pd.grpc.pulse.PulseCreateRequest; +import org.apache.hugegraph.pd.grpc.pulse.PulseNoticeRequest; +import org.apache.hugegraph.pd.grpc.pulse.PulseRequest; +import org.apache.hugegraph.pd.grpc.pulse.PulseResponse; +import org.apache.hugegraph.pd.grpc.pulse.PulseType; +import org.apache.hugegraph.pd.pulse.PartitionNotice; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +import io.grpc.ManagedChannel; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public final class PDPulseImpl implements PDPulse { + + private static final ConcurrentHashMap chs = new ConcurrentHashMap<>(); + private final ExecutorService threadPool; + private HgPdPulseGrpc.HgPdPulseStub stub; + private String pdServerAddress; + + // TODO: support several servers. + public PDPulseImpl(String pdServerAddress) { + this.pdServerAddress = pdServerAddress; + this.stub = HgPdPulseGrpc.newStub(Channels.getChannel(pdServerAddress)); + var namedThreadFactory = + new ThreadFactoryBuilder().setNameFormat("ack-notice-pool-%d").build(); + threadPool = Executors.newSingleThreadExecutor(namedThreadFactory); + } + + + private String getCurrentHost() { + return this.pdServerAddress; + } + + private boolean checkChannel() { + return stub != null && !((ManagedChannel) stub.getChannel()).isShutdown(); + } + + /* TODO: handle this override problem */ + @Override + public Notifier connectPartition(Listener + listener) { + return new PartitionHeartbeat(listener); + } + + @Override + public boolean resetStub(String host, Notifier notifier) { + log.info("reset stub: current, {}, new: {}, channel state:{}", getCurrentHost(), host, + checkChannel()); + if (Objects.equals(host, getCurrentHost()) && checkChannel()) { + return false; + } + + if (notifier != null) { + notifier.close(); + } + + this.stub = HgPdPulseGrpc.newStub(Channels.getChannel(host)); + log.info("pd pulse connect to {}", host); + this.pdServerAddress = host; + return true; + } + + /*** PartitionHeartbeat's implement ***/ + private class PartitionHeartbeat extends + AbstractConnector { + private long observerId = -1; + + PartitionHeartbeat(Listener listener) { + super(listener, PulseType.PULSE_TYPE_PARTITION_HEARTBEAT); + } + + private void setObserverId(long observerId) { + if (this.observerId == -1) { + this.observerId = observerId; + } + } + + @Override + public void notifyServer(PartitionHeartbeatRequest.Builder requestBuilder) { + this.reqStream.onNext(PulseRequest.newBuilder() + .setNoticeRequest( + PulseNoticeRequest.newBuilder() + .setPartitionHeartbeatRequest( + requestBuilder.build() + ).build() + ).build() + ); + } + + @Override + public void onNext(PulseResponse pulseResponse) { + this.setObserverId(pulseResponse.getObserverId()); + long noticeId = pulseResponse.getNoticeId(); + this.listener.onNext(pulseResponse); + this.listener.onNotice(new PartitionNotice(noticeId, + e -> super.ackNotice(e, observerId), + pulseResponse)); + } + + } + + private abstract class AbstractConnector implements Notifier, + StreamObserver { + Listener listener; + StreamObserver reqStream; + PulseType pulseType; + PulseRequest.Builder reqBuilder = PulseRequest.newBuilder(); + PulseAckRequest.Builder ackBuilder = PulseAckRequest.newBuilder(); + + private AbstractConnector(Listener listener, PulseType pulseType) { + this.listener = listener; + this.pulseType = pulseType; + this.init(); + } + + void init() { + PulseCreateRequest.Builder builder = PulseCreateRequest.newBuilder() + .setPulseType(this.pulseType); + + this.reqStream = PDPulseImpl.this.stub.pulse(this); + this.reqStream.onNext(reqBuilder.clear().setCreateRequest(builder).build()); + } + + /*** notifier ***/ + @Override + public void close() { + this.reqStream.onCompleted(); + } + + @Override + public abstract void notifyServer(N t); + + @Override + public void crash(String error) { + this.reqStream.onError(new Throwable(error)); + } + + /*** listener ***/ + @Override + public abstract void onNext(PulseResponse pulseResponse); + + @Override + public void onError(Throwable throwable) { + this.listener.onError(throwable); + } + + @Override + public void onCompleted() { + this.listener.onCompleted(); + } + + protected void ackNotice(long noticeId, long observerId) { + threadPool.execute(() -> { + // log.info("send ack: {}, ts: {}", noticeId, System.currentTimeMillis()); + this.reqStream.onNext(reqBuilder.clear() + .setAckRequest( + this.ackBuilder.clear() + .setNoticeId(noticeId) + .setObserverId(observerId) + .build() + ).build() + ); + }); + } + } +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatch.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatch.java new file mode 100644 index 0000000000..1428fe762d --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatch.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import java.io.Closeable; +import java.util.function.Consumer; + +import org.apache.hugegraph.pd.grpc.watch.WatchResponse; +import org.apache.hugegraph.pd.watch.NodeEvent; +import org.apache.hugegraph.pd.watch.PartitionEvent; + +public interface PDWatch { + + /** + * Watch the events of all store-nodes registered in the remote PD-Server. + * + * @param listener + * @return + */ + //PDWatcher watchNode(Listener listener); + + /** + * Watch the events of the store-nodes assigned to a specified graph. + * + * @param graph the graph name which you want to watch + * @param listener + * @return + */ + //PDWatcher watchNode(String graph, Listener listener); + + String getCurrentHost(); + + boolean checkChannel(); + + /*** inner static methods ***/ + static Listener listener(Consumer onNext) { + return listener(onNext, t -> { + }, () -> { + }); + } + + static Listener listener(Consumer onNext, Consumer onError) { + return listener(onNext, onError, () -> { + }); + } + + static Listener listener(Consumer onNext, Runnable onCompleted) { + return listener(onNext, t -> { + }, onCompleted); + } + + static Listener listener(Consumer onNext, Consumer onError, + Runnable onCompleted) { + return new Listener() { + @Override + public void onNext(T response) { + onNext.accept(response); + } + + @Override + public void onError(Throwable throwable) { + onError.accept(throwable); + } + + @Override + public void onCompleted() { + onCompleted.run(); + } + }; + } + + /** + * @param listener + * @return + */ + Watcher watchPartition(Listener listener); + + Watcher watchNode(Listener listener); + + Watcher watchGraph(Listener listener); + + Watcher watchShardGroup(Listener listener); + + + /** + * Interface of Watcher. + */ + interface Listener { + /** + * Invoked on new events. + * + * @param response the response. + */ + void onNext(T response); + + /** + * Invoked on errors. + * + * @param throwable the error. + */ + void onError(Throwable throwable); + + /** + * Invoked on completion. + */ + default void onCompleted() {}; + } + + interface Watcher extends Closeable { + /** + * closes this watcher and all its resources. + */ + @Override + void close(); + + /** + * Requests the latest revision processed and propagates it to listeners + */ + // TODO: what's it for? + //void requestProgress(); + } +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatchImpl.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatchImpl.java new file mode 100644 index 0000000000..a35893a33a --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatchImpl.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import java.util.function.Supplier; + +import org.apache.hugegraph.pd.grpc.watch.HgPdWatchGrpc; +import org.apache.hugegraph.pd.grpc.watch.WatchCreateRequest; +import org.apache.hugegraph.pd.grpc.watch.WatchNodeResponse; +import org.apache.hugegraph.pd.grpc.watch.WatchPartitionResponse; +import org.apache.hugegraph.pd.grpc.watch.WatchRequest; +import org.apache.hugegraph.pd.grpc.watch.WatchResponse; +import org.apache.hugegraph.pd.grpc.watch.WatchType; +import org.apache.hugegraph.pd.watch.NodeEvent; +import org.apache.hugegraph.pd.watch.PartitionEvent; + +import io.grpc.ManagedChannel; +import io.grpc.stub.StreamObserver; + +final class PDWatchImpl implements PDWatch { + + private final HgPdWatchGrpc.HgPdWatchStub stub; + + private final String pdServerAddress; + + // TODO: support several servers. + PDWatchImpl(String pdServerAddress) { + this.pdServerAddress = pdServerAddress; + this.stub = HgPdWatchGrpc.newStub(Channels.getChannel(pdServerAddress)); + } + + @Override + public String getCurrentHost() { + return this.pdServerAddress; + } + + @Override + public boolean checkChannel() { + return stub != null && !((ManagedChannel) stub.getChannel()).isShutdown(); + } + + /** + * Get Partition change watcher. + * + * @param listener + * @return + */ + @Override + public Watcher watchPartition(Listener listener) { + return new PartitionWatcher(listener); + } + + /** + * Get Store-Node change watcher. + * + * @param listener + * @return + */ + @Override + public Watcher watchNode(Listener listener) { + return new NodeWatcher(listener); + } + + @Override + public Watcher watchGraph(Listener listener) { + return new GraphWatcher(listener); + } + + @Override + public Watcher watchShardGroup(Listener listener) { + return new ShardGroupWatcher(listener); + } + + private class GraphWatcher extends AbstractWatcher { + + private GraphWatcher(Listener listener) { + super(listener, + () -> WatchCreateRequest + .newBuilder() + .setWatchType(WatchType.WATCH_TYPE_GRAPH_CHANGE) + .build() + ); + } + + @Override + public void onNext(WatchResponse watchResponse) { + this.listener.onNext(watchResponse); + } + } + + private class ShardGroupWatcher extends AbstractWatcher { + + private ShardGroupWatcher(Listener listener) { + super(listener, + () -> WatchCreateRequest + .newBuilder() + .setWatchType(WatchType.WATCH_TYPE_SHARD_GROUP_CHANGE) + .build() + ); + } + + @Override + public void onNext(WatchResponse watchResponse) { + this.listener.onNext(watchResponse); + } + } + + private class PartitionWatcher extends AbstractWatcher { + + private PartitionWatcher(Listener listener) { + super(listener, + () -> WatchCreateRequest + .newBuilder() + .setWatchType(WatchType.WATCH_TYPE_PARTITION_CHANGE) + .build() + ); + } + + @Override + public void onNext(WatchResponse watchResponse) { + WatchPartitionResponse res = watchResponse.getPartitionResponse(); + PartitionEvent event = new PartitionEvent(res.getGraph(), res.getPartitionId(), + PartitionEvent.ChangeType.grpcTypeOf( + res.getChangeType())); + this.listener.onNext(event); + } + } + + private class NodeWatcher extends AbstractWatcher { + private NodeWatcher(Listener listener) { + super(listener, + () -> WatchCreateRequest + .newBuilder() + .setWatchType(WatchType.WATCH_TYPE_STORE_NODE_CHANGE) + .build() + ); + } + + @Override + public void onNext(WatchResponse watchResponse) { + WatchNodeResponse res = watchResponse.getNodeResponse(); + NodeEvent event = new NodeEvent(res.getGraph(), res.getNodeId(), + NodeEvent.EventType.grpcTypeOf(res.getNodeEventType())); + this.listener.onNext(event); + } + } + + private abstract class AbstractWatcher implements Watcher, StreamObserver { + Listener listener; + StreamObserver reqStream; + Supplier requestSupplier; + + private AbstractWatcher(Listener listener, + Supplier requestSupplier) { + this.listener = listener; + this.requestSupplier = requestSupplier; + this.init(); + } + + void init() { + this.reqStream = PDWatchImpl.this.stub.watch(this); + this.reqStream.onNext(WatchRequest.newBuilder().setCreateRequest( + this.requestSupplier.get() + ).build()); + } + + @Override + public void close() { + this.reqStream.onCompleted(); + } + + @Override + public abstract void onNext(WatchResponse watchResponse); + + @Override + public void onError(Throwable throwable) { + + this.listener.onError(throwable); + } + + @Override + public void onCompleted() { + this.listener.onCompleted(); + } + } + +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PartitionNotice.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PartitionNotice.java new file mode 100644 index 0000000000..f8ee9b994c --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PartitionNotice.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.pulse; + +import java.util.function.Consumer; + +import org.apache.hugegraph.pd.grpc.pulse.PulseResponse; + +public class PartitionNotice implements PulseServerNotice { + private final long noticeId; + private final Consumer ackConsumer; + private final PulseResponse content; + + public PartitionNotice(long noticeId, Consumer ackConsumer, PulseResponse content) { + this.noticeId = noticeId; + this.ackConsumer = ackConsumer; + this.content = content; + } + + @Override + public void ack() { + this.ackConsumer.accept(this.noticeId); + } + + @Override + public long getNoticeId() { + return this.noticeId; + } + + @Override + public PulseResponse getContent() { + return this.content; + } +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PulseServerNotice.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PulseServerNotice.java new file mode 100644 index 0000000000..0c83f2b06e --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PulseServerNotice.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.pulse; + +public interface PulseServerNotice { + /** + * @throws RuntimeException when failed to send ack-message to pd-server + */ + void ack(); + + long getNoticeId(); + + /** + * Return a response object of gRPC stream. + * + * @return + */ + T getContent(); + +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/NodeEvent.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/NodeEvent.java new file mode 100644 index 0000000000..484d9a08c7 --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/NodeEvent.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.watch; + +import java.util.Objects; + +import org.apache.hugegraph.pd.grpc.watch.NodeEventType; + +public class NodeEvent { + private final String graph; + private final long nodeId; + private final EventType eventType; + + public NodeEvent(String graph, long nodeId, EventType eventType) { + this.graph = graph; + this.nodeId = nodeId; + this.eventType = eventType; + } + + public String getGraph() { + return graph; + } + + public long getNodeId() { + return nodeId; + } + + public EventType getEventType() { + return eventType; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + NodeEvent nodeEvent = (NodeEvent) o; + return nodeId == nodeEvent.nodeId && Objects.equals(graph, + nodeEvent.graph) && + eventType == nodeEvent.eventType; + } + + @Override + public int hashCode() { + return Objects.hash(graph, nodeId, eventType); + } + + @Override + public String toString() { + return "NodeEvent{" + + "graph='" + graph + '\'' + + ", nodeId=" + nodeId + + ", eventType=" + eventType + + '}'; + } + + public enum EventType { + UNKNOWN, + NODE_ONLINE, + NODE_OFFLINE, + NODE_RAFT_CHANGE, + NODE_PD_LEADER_CHANGE; + + public static EventType grpcTypeOf(NodeEventType grpcType) { + switch (grpcType) { + case NODE_EVENT_TYPE_NODE_ONLINE: + return NODE_ONLINE; + case NODE_EVENT_TYPE_NODE_OFFLINE: + return NODE_OFFLINE; + case NODE_EVENT_TYPE_NODE_RAFT_CHANGE: + return NODE_RAFT_CHANGE; + case NODE_EVENT_TYPE_PD_LEADER_CHANGE: + return NODE_PD_LEADER_CHANGE; + default: + return UNKNOWN; + } + + } + + } +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/PDWatcher.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/PDWatcher.java new file mode 100644 index 0000000000..d663f34a3c --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/PDWatcher.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.watch; + +public class PDWatcher { + +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/PartitionEvent.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/PartitionEvent.java new file mode 100644 index 0000000000..0211cf8ec9 --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/PartitionEvent.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.watch; + +import java.util.Objects; + +import org.apache.hugegraph.pd.grpc.watch.WatchChangeType; + +public class PartitionEvent { + private final String graph; + private final int partitionId; + private final ChangeType changeType; + + public PartitionEvent(String graph, int partitionId, ChangeType changeType) { + this.graph = graph; + this.partitionId = partitionId; + this.changeType = changeType; + } + + public String getGraph() { + return this.graph; + } + + public int getPartitionId() { + return this.partitionId; + } + + public ChangeType getChangeType() { + return this.changeType; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PartitionEvent that = (PartitionEvent) o; + return partitionId == that.partitionId && Objects.equals(graph, that.graph) && + changeType == that.changeType; + } + + @Override + public int hashCode() { + return Objects.hash(graph, partitionId, changeType); + } + + @Override + public String toString() { + return "PartitionEvent{" + + "graph='" + graph + '\'' + + ", partitionId=" + partitionId + + ", changeType=" + changeType + + '}'; + } + + public enum ChangeType { + UNKNOWN, + ADD, + ALTER, + DEL; + + public static ChangeType grpcTypeOf(WatchChangeType grpcType) { + switch (grpcType) { + case WATCH_CHANGE_TYPE_ADD: + return ADD; + case WATCH_CHANGE_TYPE_ALTER: + return ALTER; + case WATCH_CHANGE_TYPE_DEL: + return DEL; + default: + return UNKNOWN; + } + } + } +} diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/WatchType.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/WatchType.java new file mode 100644 index 0000000000..e537701936 --- /dev/null +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/WatchType.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.watch; + +enum WatchType { + + PARTITION_CHANGE(10); + + private final int value; + + WatchType(int value) { + this.value = value; + } + +} diff --git a/hugegraph-pd/hg-pd-common/pom.xml b/hugegraph-pd/hg-pd-common/pom.xml new file mode 100644 index 0000000000..918c8deab8 --- /dev/null +++ b/hugegraph-pd/hg-pd-common/pom.xml @@ -0,0 +1,54 @@ + + + + + + 4.0.0 + + org.apache.hugegraph + hugegraph-pd + ${revision} + ../pom.xml + + hg-pd-common + + + 11 + 11 + + + + + org.apache.hugegraph + hg-pd-grpc + ${revision} + + + org.projectlombok + lombok + 1.18.24 + + + org.apache.commons + commons-collections4 + 4.4 + + + diff --git a/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/GraphCache.java b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/GraphCache.java new file mode 100644 index 0000000000..07c7c332d9 --- /dev/null +++ b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/GraphCache.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.common; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.apache.hugegraph.pd.grpc.Metapb.Graph; +import org.apache.hugegraph.pd.grpc.Metapb.Partition; + +import com.google.common.collect.RangeMap; +import com.google.common.collect.TreeRangeMap; + +import lombok.Data; + +@Data +public class GraphCache { + + private Graph graph; + private AtomicBoolean initialized = new AtomicBoolean(false); + private AtomicBoolean writing = new AtomicBoolean(false); + private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private Map state = new ConcurrentHashMap<>(); + private Map partitions = new ConcurrentHashMap<>(); + private RangeMap range = TreeRangeMap.create(); + + public GraphCache(Graph graph) { + this.graph = graph; + } + + public GraphCache() { + } + + public Partition getPartition(Integer id) { + return partitions.get(id); + } + + public Partition addPartition(Integer id, Partition p) { + return partitions.put(id, p); + } + + public Partition removePartition(Integer id) { + return partitions.remove(id); + } +} diff --git a/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/HgAssert.java b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/HgAssert.java new file mode 100644 index 0000000000..710f96f28c --- /dev/null +++ b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/HgAssert.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.common; + +import java.util.Collection; +import java.util.Map; + +public final class HgAssert { + + public static void isTrue(boolean expression, String message) { + if (message == null) { + throw new IllegalArgumentException("message is null"); + } + + if (!expression) { + throw new IllegalArgumentException(message); + } + } + + public static void isFalse(boolean expression, String message) { + isTrue(!expression, message); + } + + public static void isArgumentValid(byte[] bytes, String parameter) { + isFalse(isInvalid(bytes), "The argument is invalid: " + parameter); + } + + public static void isArgumentValid(String str, String parameter) { + isFalse(isInvalid(str), "The argument is invalid: " + parameter); + } + + public static void isArgumentNotNull(Object obj, String parameter) { + isTrue(obj != null, "The argument is null: " + parameter); + } + + public static void istValid(byte[] bytes, String msg) { + isFalse(isInvalid(bytes), msg); + } + + public static void isValid(String str, String msg) { + isFalse(isInvalid(str), msg); + } + + public static void isNotNull(Object obj, String msg) { + isTrue(obj != null, msg); + } + + public static boolean isContains(Object[] objs, Object obj) { + if (objs == null || objs.length == 0 || obj == null) { + return false; + } + for (Object item : objs) { + if (obj.equals(item)) { + return true; + } + } + return false; + } + + public static boolean isInvalid(String... strs) { + if (strs == null || strs.length == 0) { + return true; + } + for (String item : strs) { + if (item == null || "".equals(item.trim())) { + return true; + } + } + return false; + } + + public static boolean isInvalid(byte[] bytes) { + return bytes == null || bytes.length == 0; + } + + public static boolean isInvalid(Map map) { + return map == null || map.isEmpty(); + } + + public static boolean isInvalid(Collection list) { + return list == null || list.isEmpty(); + } + + public static boolean isContains(Collection list, T item) { + if (list == null || item == null) { + return false; + } + return list.contains(item); + } + + public static boolean isNull(Object... objs) { + if (objs == null) { + return true; + } + for (Object item : objs) { + if (item == null) { + return true; + } + } + return false; + } +} diff --git a/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/KVPair.java b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/KVPair.java new file mode 100644 index 0000000000..b5e916c481 --- /dev/null +++ b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/KVPair.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.common; + +import java.io.Serializable; +import java.util.Objects; + +public class KVPair implements Serializable { + + /** + * Key of this Pair. + */ + private K key; + /** + * Value of this this Pair. + */ + private V value; + + /** + * Creates a new pair + * + * @param key The key for this pair + * @param value The value to use for this pair + */ + public KVPair(K key, V value) { + this.key = key; + this.value = value; + } + + /** + * Gets the key for this pair. + * + * @return key for this pair + */ + public K getKey() { + return key; + } + + public void setKey(K key) { + this.key = key; + } + + /** + * Gets the value for this pair. + * + * @return value for this pair + */ + public V getValue() { + return value; + } + + public void setValue(V value) { + this.value = value; + } + + /** + *

String representation of this + * Pair.

+ * + *

The default name/value delimiter '=' is always used.

+ * + * @return String representation of this Pair + */ + @Override + public String toString() { + return key + "=" + value; + } + + /** + *

Generate a hash code for this Pair.

+ * + *

The hash code is calculated using both the name and + * the value of the Pair.

+ * + * @return hash code for this Pair + */ + @Override + public int hashCode() { + // name's hashCode is multiplied by an arbitrary prime number (13) + // in order to make sure there is a difference in the hashCode between + // these two parameters: + // name: a value: aa + // name: aa value: a + return key.hashCode() * 13 + (value == null ? 0 : value.hashCode()); + } + + /** + *

Test this Pair for equality with another + * Object.

+ * + *

If the Object to be tested is not a + * Pair or is null, then this method + * returns false.

+ * + *

Two Pairs are considered equal if and only if + * both the names and values are equal.

+ * + * @param o the Object to test for + * equality with this Pair + * @return true if the given Object is + * equal to this Pair else false + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o instanceof KVPair) { + KVPair pair = (KVPair) o; + if (!Objects.equals(key, pair.key)) { + return false; + } + return Objects.equals(value, pair.value); + } + return false; + } +} diff --git a/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PDException.java b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PDException.java new file mode 100644 index 0000000000..b398137e82 --- /dev/null +++ b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PDException.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.common; + +public class PDException extends Exception { + + private final int errorCode; + + public PDException(int error) { + super(String.format("Error code = %d", error)); + this.errorCode = error; + } + + public PDException(int error, String msg) { + super(msg); + this.errorCode = error; + } + + public PDException(int error, Throwable e) { + super(e); + this.errorCode = error; + } + + public PDException(int error, String msg, Throwable e) { + super(msg, e); + this.errorCode = error; + } + + public int getErrorCode() { + return errorCode; + } +} diff --git a/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PDRuntimeException.java b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PDRuntimeException.java new file mode 100644 index 0000000000..0bd90241df --- /dev/null +++ b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PDRuntimeException.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.common; + +public class PDRuntimeException extends RuntimeException { + + // public static final int LICENSE_ERROR = -11; + + private int errorCode = 0; + + public PDRuntimeException(int error) { + super(String.format("Error code = %d", error)); + this.errorCode = error; + } + + public PDRuntimeException(int error, String msg) { + super(msg); + this.errorCode = error; + } + + public PDRuntimeException(int error, Throwable e) { + super(e); + this.errorCode = error; + } + + public PDRuntimeException(int error, String msg, Throwable e) { + super(msg, e); + this.errorCode = error; + } + + public int getErrorCode() { + return errorCode; + } +} diff --git a/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionCache.java b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionCache.java new file mode 100644 index 0000000000..8470faae0d --- /dev/null +++ b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionCache.java @@ -0,0 +1,460 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.common; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.apache.hugegraph.pd.grpc.Metapb; + +import com.google.common.collect.Range; +import com.google.common.collect.RangeMap; +import com.google.common.collect.TreeRangeMap; + +/** + * 放弃 copy on write 的方式 + * 1. 在 graph * partition 数量极多的时候，效率严重下降，不能用 + */ +public class PartitionCache { + + // 读写锁对象 + private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); + Lock writeLock = readWriteLock.writeLock(); + // 每张图一个缓存 + private volatile Map> keyToPartIdCache; + // graphName + PartitionID 组成 key + private volatile Map> partitionCache; + + private volatile Map shardGroupCache; + private volatile Map storeCache; + private volatile Map graphCache; + + private final Map locks = new HashMap<>(); + + public PartitionCache() { + keyToPartIdCache = new HashMap<>(); + partitionCache = new HashMap<>(); + shardGroupCache = new ConcurrentHashMap<>(); + storeCache = new ConcurrentHashMap<>(); + graphCache = new ConcurrentHashMap<>(); + } + + private AtomicBoolean getOrCreateGraphLock(String graphName) { + var lock = this.locks.get(graphName); + if (lock == null) { + try { + writeLock.lock(); + if ((lock = this.locks.get(graphName)) == null) { + lock = new AtomicBoolean(); + locks.put(graphName, lock); + } + } finally { + writeLock.unlock(); + } + } + return lock; + } + + public void waitGraphLock(String graphName) { + var lock = getOrCreateGraphLock(graphName); + while (lock.get()) { + Thread.onSpinWait(); + } + } + + public void lockGraph(String graphName) { + var lock = getOrCreateGraphLock(graphName); + while (lock.compareAndSet(false, true)) { + Thread.onSpinWait(); + } + } + + public void unlockGraph(String graphName) { + var lock = getOrCreateGraphLock(graphName); + lock.set(false); + } + + /** + * 根据 partitionId 返回分区信息 + * + * @param graphName + * @param partId + * @return + */ + public KVPair getPartitionById(String graphName, int partId) { + waitGraphLock(graphName); + var graphs = partitionCache.get(graphName); + if (graphs != null) { + var partition = graphs.get(partId); + if (partition != null) { + return new KVPair<>(partition, getLeaderShard(partId)); + } + } + + return null; + } + + /** + * 返回 key 所在的分区信息 + * + * @param key + * @return + */ + public KVPair getPartitionByKey(String graphName, byte[] key) { + int code = PartitionUtils.calcHashcode(key); + return getPartitionByCode(graphName, code); + } + + /** + * 根据 key 的 hashcode 返回分区信息 + * + * @param graphName + * @param code + * @return + */ + public KVPair getPartitionByCode(String graphName, long code) { + waitGraphLock(graphName); + RangeMap rangeMap = keyToPartIdCache.get(graphName); + if (rangeMap != null) { + Integer partId = rangeMap.get(code); + if (partId != null) { + return getPartitionById(graphName, partId); + } + } + return null; + } + + public List getPartitions(String graphName) { + waitGraphLock(graphName); + + List partitions = new ArrayList<>(); + if (!partitionCache.containsKey(graphName)) { + return partitions; + } + partitionCache.get(graphName).forEach((k, v) -> { + partitions.add(v); + }); + + return partitions; + } + + public boolean addPartition(String graphName, int partId, Metapb.Partition partition) { + waitGraphLock(graphName); + Metapb.Partition old = null; + + if (partitionCache.containsKey(graphName)) { + old = partitionCache.get(graphName).get(partId); + } + + if (old != null && old.equals(partition)) { + return false; + } + try { + + lockGraph(graphName); + + partitionCache.computeIfAbsent(graphName, k -> new HashMap<>()).put(partId, partition); + + if (old != null) { + // old [1-3) 被 [2-3) 覆盖了。当 [1-3) 变成 [1-2) 不应该删除原先的 [1-3) + // 当确认老的 start, end 都是自己的时候，才可以删除老的。(即还没覆盖） + var graphRange = keyToPartIdCache.get(graphName); + if (Objects.equals(partition.getId(), graphRange.get(partition.getStartKey())) && + Objects.equals(partition.getId(), graphRange.get(partition.getEndKey() - 1))) { + graphRange.remove(graphRange.getEntry(partition.getStartKey()).getKey()); + } + } + + keyToPartIdCache.computeIfAbsent(graphName, k -> TreeRangeMap.create()) + .put(Range.closedOpen(partition.getStartKey(), + partition.getEndKey()), partId); + } finally { + unlockGraph(graphName); + } + return true; + } + + public void updatePartition(String graphName, int partId, Metapb.Partition partition) { + try { + lockGraph(graphName); + Metapb.Partition old = null; + var graphs = partitionCache.get(graphName); + if (graphs != null) { + old = graphs.get(partId); + } + + if (old != null) { + var graphRange = keyToPartIdCache.get(graphName); + if (Objects.equals(partition.getId(), graphRange.get(partition.getStartKey())) && + Objects.equals(partition.getId(), graphRange.get(partition.getEndKey() - 1))) { + graphRange.remove(graphRange.getEntry(partition.getStartKey()).getKey()); + } + } + + partitionCache.computeIfAbsent(graphName, k -> new HashMap<>()).put(partId, partition); + keyToPartIdCache.computeIfAbsent(graphName, k -> TreeRangeMap.create()) + .put(Range.closedOpen(partition.getStartKey(), partition.getEndKey()), + partId); + } finally { + unlockGraph(graphName); + } + } + + public boolean updatePartition(Metapb.Partition partition) { + + var graphName = partition.getGraphName(); + var partitionId = partition.getId(); + + var old = getPartitionById(graphName, partitionId); + if (old != null && Objects.equals(partition, old.getKey())) { + return false; + } + + updatePartition(graphName, partitionId, partition); + return true; + } + + public void removePartition(String graphName, int partId) { + try { + lockGraph(graphName); + var partition = partitionCache.get(graphName).remove(partId); + if (partition != null) { + var graphRange = keyToPartIdCache.get(graphName); + + if (Objects.equals(partition.getId(), graphRange.get(partition.getStartKey())) && + Objects.equals(partition.getId(), graphRange.get(partition.getEndKey() - 1))) { + graphRange.remove(graphRange.getEntry(partition.getStartKey()).getKey()); + } + } + } finally { + unlockGraph(graphName); + } + } + + /** + * remove partition id of graph name + * + * @param graphName + * @param id + */ + public void remove(String graphName, int id) { + removePartition(graphName, id); + } + + /** + * remove all partitions + */ + public void removePartitions() { + writeLock.lock(); + try { + partitionCache = new HashMap<>(); + keyToPartIdCache = new HashMap<>(); + locks.clear(); + } finally { + writeLock.unlock(); + } + } + + /** + * remove partition cache of graphName + * + * @param graphName + */ + public void removeAll(String graphName) { + try { + lockGraph(graphName); + partitionCache.remove(graphName); + keyToPartIdCache.remove(graphName); + locks.remove(graphName); + } finally { + unlockGraph(graphName); + } + } + + private String makePartitionKey(String graphName, int partId) { + return graphName + "/" + partId; + } + + public boolean updateShardGroup(Metapb.ShardGroup shardGroup) { + Metapb.ShardGroup oldShardGroup = shardGroupCache.get(shardGroup.getId()); + if (oldShardGroup != null && oldShardGroup.equals(shardGroup)) { + return false; + } + shardGroupCache.put(shardGroup.getId(), shardGroup); + return true; + } + + public void deleteShardGroup(int shardGroupId) { + shardGroupCache.remove(shardGroupId); + } + + public Metapb.ShardGroup getShardGroup(int groupId) { + return shardGroupCache.get(groupId); + } + + public boolean addStore(Long storeId, Metapb.Store store) { + Metapb.Store oldStore = storeCache.get(storeId); + if (oldStore != null && oldStore.equals(store)) { + return false; + } + storeCache.put(storeId, store); + return true; + } + + public Metapb.Store getStoreById(Long storeId) { + return storeCache.get(storeId); + } + + public void removeStore(Long storeId) { + storeCache.remove(storeId); + } + + public boolean hasGraph(String graphName) { + return getPartitions(graphName).size() > 0; + } + + public void updateGraph(Metapb.Graph graph) { + if (Objects.equals(graph, getGraph(graph.getGraphName()))) { + return; + } + graphCache.put(graph.getGraphName(), graph); + } + + public Metapb.Graph getGraph(String graphName) { + return graphCache.get(graphName); + } + + public List getGraphs() { + List graphs = new ArrayList<>(); + graphCache.forEach((k, v) -> { + graphs.add(v); + }); + return graphs; + } + + public void reset() { + writeLock.lock(); + try { + partitionCache = new HashMap<>(); + keyToPartIdCache = new HashMap<>(); + shardGroupCache = new ConcurrentHashMap<>(); + storeCache = new ConcurrentHashMap<>(); + graphCache = new ConcurrentHashMap<>(); + locks.clear(); + } finally { + writeLock.unlock(); + } + } + + public void clear() { + reset(); + } + + public String debugCacheByGraphName(String graphName) { + StringBuilder builder = new StringBuilder(); + builder.append("Graph:").append(graphName).append(", cache info: range info: {"); + var rangeMap = keyToPartIdCache.get(graphName); + builder.append(rangeMap == null ? "" : rangeMap).append("}"); + + if (rangeMap != null) { + builder.append(", partition info : {"); + rangeMap.asMapOfRanges().forEach((k, v) -> { + var partition = partitionCache.get(graphName).get(v); + builder.append("[part_id:").append(v); + if (partition != null) { + builder.append(", start_key:").append(partition.getStartKey()) + .append(", end_key:").append(partition.getEndKey()) + .append(", state:").append(partition.getState().name()); + } + builder.append("], "); + }); + builder.append("}"); + } + + builder.append(", graph info:{"); + var graph = graphCache.get(graphName); + if (graph != null) { + builder.append("partition_count:").append(graph.getPartitionCount()) + .append(", state:").append(graph.getState().name()); + } + builder.append("}]"); + return builder.toString(); + } + + public Metapb.Shard getLeaderShard(int partitionId) { + var shardGroup = shardGroupCache.get(partitionId); + if (shardGroup != null) { + for (Metapb.Shard shard : shardGroup.getShardsList()) { + if (shard.getRole() == Metapb.ShardRole.Leader) { + return shard; + } + } + } + + return null; + } + + public void updateShardGroupLeader(int partitionId, Metapb.Shard leader) { + if (shardGroupCache.containsKey(partitionId) && leader != null) { + if (!Objects.equals(getLeaderShard(partitionId), leader)) { + var shardGroup = shardGroupCache.get(partitionId); + var builder = Metapb.ShardGroup.newBuilder(shardGroup).clearShards(); + for (var shard : shardGroup.getShardsList()) { + builder.addShards( + Metapb.Shard.newBuilder() + .setStoreId(shard.getStoreId()) + .setRole(shard.getStoreId() == leader.getStoreId() ? + Metapb.ShardRole.Leader : + Metapb.ShardRole.Follower) + .build() + ); + } + shardGroupCache.put(partitionId, builder.build()); + } + } + } + + public String debugShardGroup() { + StringBuilder builder = new StringBuilder(); + builder.append("shard group cache:{"); + shardGroupCache.forEach((partitionId, shardGroup) -> { + builder.append(partitionId).append("::{") + .append("version:").append(shardGroup.getVersion()) + .append(", conf_version:").append(shardGroup.getConfVer()) + .append(", state:").append(shardGroup.getState().name()) + .append(", shards:["); + + for (var shard : shardGroup.getShardsList()) { + builder.append("{store_id:").append(shard.getStoreId()) + .append(", role:").append(shard.getRole().name()) + .append("},"); + } + builder.append("], "); + }); + builder.append("}"); + return builder.toString(); + } +} diff --git a/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionUtils.java b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionUtils.java new file mode 100644 index 0000000000..0e35cc555e --- /dev/null +++ b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionUtils.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.common; + +public class PartitionUtils { + + public static final int MAX_VALUE = 0xffff; + + /** + * 计算key的hashcode + * + * @param key + * @return hashcode + */ + public static int calcHashcode(byte[] key) { + final int p = 16777619; + int hash = (int) 2166136261L; + for (byte element : key) { + hash = (hash ^ element) * p; + } + hash += hash << 13; + hash ^= hash >> 7; + hash += hash << 3; + hash ^= hash >> 17; + hash += hash << 5; + hash = hash & PartitionUtils.MAX_VALUE; + if (hash == PartitionUtils.MAX_VALUE) { + hash = PartitionUtils.MAX_VALUE - 1; + } + return hash; + } +} diff --git a/hugegraph-pd/hg-pd-grpc/pom.xml b/hugegraph-pd/hg-pd-grpc/pom.xml new file mode 100644 index 0000000000..cef49e957d --- /dev/null +++ b/hugegraph-pd/hg-pd-grpc/pom.xml @@ -0,0 +1,138 @@ + + + + + + 4.0.0 + + + org.apache.hugegraph + hugegraph-pd + ${revision} + ../pom.xml + + hg-pd-grpc + + + + 1.6.0 + 1.39.0 + 3.17.2 + 0.6.1 + + + + + io.grpc + grpc-netty-shaded + ${grpc.version} + + + io.grpc + grpc-protobuf + ${grpc.version} + + + io.grpc + grpc-stub + ${grpc.version} + + + javax.annotation + javax.annotation-api + 1.3.2 + + + + + ${basedir}/src/main/java + + + src/main/resources + + + src/main/proto + + + + + kr.motd.maven + os-maven-plugin + ${os.plugin.version} + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf.plugin.version} + true + + + com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier} + + grpc-java + + io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} + + + ${project.basedir}/src/main/proto + + + ${project.basedir}/src/main/java + + false + + + + + + generate-sources + + + compile + + compile-custom + + + + + + maven-clean-plugin + + + + src/main/java + + + + + + + initialize + + clean + + + + + + + diff --git a/hugegraph-pd/hg-pd-grpc/src/main/proto/discovery.proto b/hugegraph-pd/hg-pd-grpc/src/main/proto/discovery.proto new file mode 100644 index 0000000000..b434ab0e86 --- /dev/null +++ b/hugegraph-pd/hg-pd-grpc/src/main/proto/discovery.proto @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; +package discovery; +import "pdpb.proto"; + +option java_package = "org.apache.hugegraph.pd.grpc.discovery"; +option java_multiple_files = true; + + +service DiscoveryService { + rpc register(NodeInfo) returns (RegisterInfo); + rpc getNodes(Query) returns (NodeInfos); + // rpc getNodesByLabel(Conditions) returns (NodeInfos); +} + +/* requests */ +message NodeInfo { + string id = 1; + string appName = 2; + string version = 3; + string address = 4; + int64 interval = 5; + map labels = 6; +} +message Query { + string appName = 1; + string version = 2; + map labels = 3; +} +message LeaseInfo { + int64 registrationTs = 1; + int64 lastHeartbeatTs = 2; + int64 serverUpTs = 3; +} +message RegisterInfo { + NodeInfo nodeInfo = 1; + LeaseInfo leaseInfo = 2 ; + RegisterType type = 3 ; + pdpb.ResponseHeader header = 4; +} +enum RegisterType { + Register = 0; + Heartbeat = 1; + Dislodge = 2; +} +//message Condition{ +// string label = 1; +//} +//message Conditions{ +// string label = 1; +// string value = 2; +//} +message NodeInfos{ + repeated NodeInfo info = 1; +} diff --git a/hugegraph-pd/hg-pd-grpc/src/main/proto/kv.proto b/hugegraph-pd/hg-pd-grpc/src/main/proto/kv.proto new file mode 100644 index 0000000000..22007cda31 --- /dev/null +++ b/hugegraph-pd/hg-pd-grpc/src/main/proto/kv.proto @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; +package kv; +import "pdpb.proto"; +import "metapb.proto"; + +option java_package = "org.apache.hugegraph.pd.grpc.kv"; +option java_multiple_files = true; + + +service KvService { + rpc put(Kv) returns (KvResponse); + rpc get(K) returns (KResponse); + rpc delete(K) returns (KvResponse); + rpc deletePrefix(K) returns (KvResponse); + rpc scanPrefix(K) returns (ScanPrefixResponse); + rpc watch(WatchRequest) returns (stream WatchResponse); + rpc watchPrefix(WatchRequest) returns (stream WatchResponse); + rpc lock(LockRequest) returns (LockResponse); + rpc lockWithoutReentrant(LockRequest) returns (LockResponse); + rpc unlock(LockRequest) returns (LockResponse); + rpc keepAlive(LockRequest) returns (LockResponse); + rpc isLocked(LockRequest) returns (LockResponse); + rpc putTTL(TTLRequest) returns (TTLResponse); + rpc keepTTLAlive(TTLRequest) returns (TTLResponse); +} + +/* requests */ +message Kv { + pdpb.RequestHeader header = 1; + string key = 2; + string value = 3; +} +message KvResponse { + pdpb.ResponseHeader header = 1; +} + +message K{ + pdpb.RequestHeader header = 1; + string key = 2; +} + +message KResponse{ + pdpb.ResponseHeader header = 1; + string value = 2; +} + +message ScanPrefixResponse { + pdpb.ResponseHeader header = 1; + map kvs = 2; +} + +message LockRequest{ + pdpb.RequestHeader header = 1; + string key = 2; + int64 ttl = 3; + int64 clientId = 4; +} +message LockResponse{ + pdpb.ResponseHeader header = 1; + string key = 2; + int64 ttl = 3; + int64 clientId = 4; + bool succeed = 5; +} + +message LockAliveResponse{ + pdpb.ResponseHeader header = 1; + int64 clientId = 2; +} + + +message WatchKv { + string key = 1; + string value = 2; +} + +enum WatchType { + Put = 0; + Delete = 1; + Unrecognized = 2; +} + +message WatchEvent { + WatchKv current = 1; + WatchKv prev = 2; + WatchType type = 3; +} + +message WatchResponse { + pdpb.ResponseHeader header = 1; + repeated WatchEvent events = 2; + int64 clientId = 3; + WatchState state = 4; +} + +enum WatchState { + Starting = 0; + Started = 1; + Leader_Changed = 2; + Alive = 3; +} + +message WatchRequest { + pdpb.RequestHeader header = 1; + WatchState state = 2; + string key = 3; + int64 clientId = 4; +} + +message V{ + string value = 1; + int64 ttl = 2; + int64 st = 3; +} + +message TTLRequest{ + pdpb.RequestHeader header = 1; + string key = 2; + string value = 3; + int64 ttl = 4; +} + +message TTLResponse{ + pdpb.ResponseHeader header = 1; + bool succeed = 2; +} diff --git a/hugegraph-pd/hg-pd-grpc/src/main/proto/metaTask.proto b/hugegraph-pd/hg-pd-grpc/src/main/proto/metaTask.proto new file mode 100644 index 0000000000..c4bb8bde10 --- /dev/null +++ b/hugegraph-pd/hg-pd-grpc/src/main/proto/metaTask.proto @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; +package metaTask; +import "metapb.proto"; +import "pd_pulse.proto"; +option java_package = "org.apache.hugegraph.pd.grpc"; + +enum TaskType { + Unknown = 0; + Split_Partition = 1; + Change_Shard = 2; + Move_Partition = 3; + Clean_Partition = 4; + Change_KeyRange = 5; +} + +// 一条任务信息 +message Task { + uint64 id = 1; + TaskType type = 2; + TaskState state = 3; + int64 start_timestamp = 4; + metapb.Partition partition = 5; + string message = 6; + //每个shard执行的任务状态 + repeated ShardTaskState shardState = 7; + ChangeShard changeShard = 9; + SplitPartition splitPartition = 10; + MovePartition movePartition = 11; + CleanPartition cleanPartition = 12; + PartitionKeyRange partitionKeyRange = 13; +} + +enum TaskState{ + Task_Unknown = 0; + Task_Ready = 1; //任务就绪 + Task_Doing = 2; //执行中 + Task_Done = 3; //完成 + Task_Exit = 4; //退出 + Task_Stop = 10; + Task_Success = 11; + Task_Failure = 12; +} + +message ShardTaskState{ + uint64 store_id = 1; + TaskState state = 2; +} diff --git a/hugegraph-pd/hg-pd-grpc/src/main/proto/metapb.proto b/hugegraph-pd/hg-pd-grpc/src/main/proto/metapb.proto new file mode 100644 index 0000000000..a8a695be04 --- /dev/null +++ b/hugegraph-pd/hg-pd-grpc/src/main/proto/metapb.proto @@ -0,0 +1,394 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; +package metapb; +option java_package = "org.apache.hugegraph.pd.grpc"; +import "google/protobuf/any.proto"; + +enum ClusterState{ + // 集群健康 + Cluster_OK = 0; + // 分区警告，存在部分故障节点，短时间不影响读写 + Cluster_Warn = 2; + // 分区下线，可以读，无法写 + Cluster_Offline = 10; + // 分区故障，无法读写,需要尽快修复故障节点。 + Cluster_Fault = 11; + Cluster_Not_Ready = -1; +} +// 集群状态 +message ClusterStats{ + ClusterState state = 1; + string message = 2; + uint64 timestamp = 16; +} + +enum StoreState { + Unknown = 0; + // 未激活 + Pending = 4; + // 在线 + Up = 1; + // 离线 + Offline = 2; + // 下线中 + Exiting = 5; + // 已下线 + Tombstone = 3; +} + +// Store label for Storage grouping. +message StoreLabel { + string key = 1; + string value = 2; +} + +message Store { + uint64 id = 1; + // Address to handle client requests + string address = 2; + string raft_address = 3; + repeated StoreLabel labels = 4; + // Store软件版本号 + string version = 5; + StoreState state = 6; + // The start timestamp of the current store + int64 start_timestamp = 7; + string deploy_path = 8; + // The last heartbeat timestamp of the store. + int64 last_heartbeat = 9; + StoreStats stats = 10; + // 数据格式版本号 + int32 data_version = 11; + int32 cores = 12; + string data_path = 13; +} + +enum ShardRole { + None = 0; + Leader = 1; + Follower = 2; + // Learner/None -> Learner + Learner = 3; +} + +message Shard { + uint64 store_id = 2; + ShardRole role = 3; +} + +message ShardGroup{ + uint32 id = 1; + uint64 version = 2; + uint64 conf_ver = 3; + repeated Shard shards = 6; + PartitionState state = 10; + string message = 11; +} + +message Graph { + string graph_name = 2; + // 分区数量，0表示无效，不能大于raft分组总数 + int32 partition_count = 3; + // 当前工作状态 + PartitionState state = 10; + string message = 11; + GraphState graph_state = 12; +} +// 分区工作状态 +enum PartitionState{ + PState_None = 0; + // + PState_Normal = 1; + // 分区警告，存在部分故障节点，短时间不影响读写 + PState_Warn = 2; + // 分区下线，可以读，无法写 + PState_Offline = 10; + // 分区故障，无法读写,需要尽快修复故障节点。 + PState_Fault = 11; +} + +message PartitionV36 { + uint32 id = 1; + string graph_name = 3; + // 分区范围 [start_key, end_key). + uint64 start_key = 4; + uint64 end_key = 5; + repeated Shard shards = 6; + // Leader任期，leader切换后递增 + uint64 version = 7; + // shards版本号，每次改变后递增 + uint64 conf_ver = 8; + // 当前工作状态 + PartitionState state = 10; + string message = 11; +} + +message Partition { + uint32 id = 1; + string graph_name = 3; + // 分区范围 [start_key, end_key). + uint64 start_key = 4; + uint64 end_key = 5; + // Partition 对象不在保存 shard list（根据对应的shard group 去查询), version 和 conf version不再有实际的意义 + // repeated Shard shards = 6; + // key range 每次改变后递增 + uint64 version = 7; + // shards版本号，每次改变后递增 + // uint64 conf_ver = 8; + // 当前工作状态 + PartitionState state = 10; + string message = 11; +} + +message PartitionShard { + metapb.Partition partition = 1; + metapb.Shard leader = 2; + // 离线的Shard + repeated metapb.Shard offline_shards = 3; +} +// 记录分区所在的存储位置 +message PartitionStore { + uint32 partition_id = 1; + string graph_name = 3; + // 存储位置 + string store_location = 4; +} + +message PartitionRaft { + uint32 partition_id = 1; + string graph_name = 3; + // 存储位置 + string raft_location = 4; +} + +message ShardStats{ + uint64 store_id = 2; + ShardRole role = 3; + ShardState state = 4; + // 安装快照的进度 + uint32 progress = 5; +} +message PartitionStats{ + uint32 id = 1; + // raft分组的任期. + uint64 leader_term = 2; + repeated string graph_name = 3; + metapb.Shard leader = 4; + // 离线 shards + repeated metapb.Shard shard = 5; + repeated metapb.Shard learner = 6; + uint64 conf_ver = 7; + // 分区状态 + PartitionState state = 8; + repeated ShardStats shardStats = 9; + // 分区近似大小 + uint64 approximate_size = 10; + // 分区key的近似数量 + uint64 approximate_keys = 13; + // heartbeat timestamp + int64 timestamp = 16; +} + +message GraphStats{ + // 图名 + string graph_name = 1; + // 分区近似大小 + uint64 approximate_size = 2; + // 分区key的近似数量 + uint64 approximate_keys = 3; + // // committed index + // uint64 committed_index = 4; + uint32 partition_id = 5; + ShardRole role = 6; + // 当前工作状态 + PartitionState work_state = 8; +} + +message RaftStats { + // partition id + uint32 partition_id = 1; + // committed index + uint64 committed_index = 2; +} + +message TimeInterval { + // The unix timestamp in seconds of the start of this period. + uint64 start_timestamp = 1; + // The unix timestamp in seconds of the end of this period. + uint64 end_timestamp = 2; +} + +message RecordPair { + string key = 1; + uint64 value = 2; +} + + +message QueryStats { + uint64 GC = 1; + uint64 Get = 2; + uint64 Scan = 3; + uint64 Coprocessor = 4; + uint64 Delete = 5; + uint64 DeleteRange = 6; + uint64 Put = 7; +} + +enum ShardState{ + SState_None = 0; + // 正常 + SState_Normal = 1; + // 安装快照 + SState_Snapshot = 2; + // 离线 + SState_Offline = 10; +} + + +message StoreStats { + uint64 store_id = 1; + // Capacity for the store. + uint64 capacity = 2; + // Available size for the store. + uint64 available = 3; + // Total partition count in this store. + uint32 partition_count = 4; + // Current sending snapshot count. + uint32 sending_snap_count = 5; + // Current receiving snapshot count. + uint32 receiving_snap_count = 6; + // When the store is started (unix timestamp in seconds). + uint32 start_time = 7; + // How many partition is applying snapshot. + uint32 applying_snap_count = 8; + // If the store is busy + bool is_busy = 9; + // Actually used space by db + uint64 used_size = 10; + // Bytes written for the store during this period. + uint64 bytes_written = 11; + // Keys written for the store during this period. + uint64 keys_written = 12; + // Bytes read for the store during this period. + uint64 bytes_read = 13; + // Keys read for the store during this period. + uint64 keys_read = 14; + // Actually reported time interval + TimeInterval interval = 15; + // Threads' CPU usages in the store + repeated RecordPair cpu_usages = 16; + // Threads' read disk I/O rates in the store + repeated RecordPair read_io_rates = 17; + // Threads' write disk I/O rates in the store + repeated RecordPair write_io_rates = 18; + // Operations' latencies in the store + repeated RecordPair op_latencies = 19; + // Store query stats + QueryStats query_stats = 21; + // graph stats + repeated GraphStats graph_stats = 22; + // raft stats + repeated RaftStats raft_stats = 23; + int32 cores = 24; + // system metrics + repeated RecordPair system_metrics = 25; +} + +// 分区查询条件 +message PartitionQuery{ + optional uint64 store_id = 1; // 0 表示查询条件不包含store_id + optional string graph_name = 2; + optional uint32 partition_id = 4; +} + +//PD 节点信息 +message Member { + uint64 cluster_id = 1; + string raft_url = 3; + string grpc_url = 4; + string rest_url = 5; + string data_path = 6; + StoreState state = 7; + ShardRole role = 8; + string replicator_state = 9; +} + +// 图空间配置 +message GraphSpace{ + string name = 1; + // 最大占用存储 + uint64 storage_limit = 2; + // 已使用空间 + uint64 used_size = 3; + // 修改时间 + uint64 timestamp = 10; +} + +// PD 配置 +message PDConfig{ + uint64 version = 1; + // 分区数量, 初始化根据Store数量动态计算，分裂后进行修改 + int32 partition_count = 2; + // 每分区副本数量 + int32 shard_count = 3; + // pd集群列表 + string peers_list = 4; + // 集群中最少store数量 + int32 min_store_count = 6; + // 每个store最大副本数 + int32 max_Shards_Per_Store = 7; + // 修改时间 + uint64 timestamp = 10; +} + + + +//消息持久化 +message QueueItem{ + string item_id = 1; + string item_class = 2; + bytes item_content = 3; + int64 timestamp = 10; +} + +message LogRecord{ + string action = 1; + int64 timestamp = 2; + map labels = 3; + google.protobuf.Any object = 4; + string message = 5; +} + +message GraphState{ + GraphMode mode = 1; + GraphModeReason reason = 2; +} + +enum GraphMode{ + ReadWrite = 0; + ReadOnly = 1; + WriteOnly = 2; +} + +enum GraphModeReason{ + Empty = 0; // 空 + Initiative = 1; // 主动的状态设置 + Quota = 2; // 达到限额条件 + +} diff --git a/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_common.proto b/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_common.proto new file mode 100644 index 0000000000..c9eec81494 --- /dev/null +++ b/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_common.proto @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "org.apache.hugegraph.pd.grpc.common"; +option java_outer_classname = "HgPdCommonProto"; + +message RequestHeader { + // 集群 ID. + uint64 cluster_id = 1; + // 发送者 ID. + uint64 sender_id = 2; +} + +message ResponseHeader { + // cluster_id is the ID of the cluster which sent the response. + uint64 cluster_id = 1; + Error error = 2; +} + +enum ErrorType { + OK = 0; + UNKNOWN = 1; + STORE_NON_EXIST = 101; + STORE_TOMBSTONE = 103; + ALREADY_BOOTSTRAPPED = 4; + INCOMPATIBLE_VERSION = 5; + PARTITION_NOT_FOUND = 6; + + ETCD_READ_ERROR = 1000; + ETCD_WRITE_ERROR = 1001; +} + +message Error { + ErrorType type = 1; + string message = 2; +} diff --git a/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_pulse.proto b/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_pulse.proto new file mode 100644 index 0000000000..f6c5224536 --- /dev/null +++ b/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_pulse.proto @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +import "metapb.proto"; +import "pd_common.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.hugegraph.pd.grpc.pulse"; +option java_outer_classname = "HgPdPulseProto"; + +service HgPdPulse { + rpc Pulse(stream PulseRequest) returns (stream PulseResponse); +} + +/* requests */ +message PulseRequest { + PulseCreateRequest create_request = 1; + PulseCancelRequest cancel_request = 2; + PulseNoticeRequest notice_request = 3; + PulseAckRequest ack_request = 4; +} + +message PulseCreateRequest { + PulseType pulse_type = 1; +} + +message PulseCancelRequest { + int64 observer_id = 1; +} + +message PulseNoticeRequest { + int64 observer_id = 1; + oneof request_union { + PartitionHeartbeatRequest partition_heartbeat_request = 10; + } +} + +message PulseAckRequest { + int64 observer_id = 1; + int64 notice_id = 2; +} + +// 分区心跳，分区的peer增减、leader改变等事件发生时，由leader发送心跳。 +// 同时pd对分区进行shard增减通过Response发送给leader +message PartitionHeartbeatRequest { + RequestHeader header = 1; + // Leader Peer sending the heartbeat + metapb.PartitionStats states = 4; +} + +/* responses */ +message PulseResponse { + PulseType pulse_type = 1; + int64 observer_id = 2; + int32 status = 3; //0=ok,1=fail + int64 notice_id=4; + oneof response_union { + PartitionHeartbeatResponse partition_heartbeat_response = 10; + PdInstructionResponse instruction_response = 11; + } +} + +message PartitionHeartbeatResponse { + ResponseHeader header = 1; + uint64 id = 3; + metapb.Partition partition = 2; + ChangeShard change_shard = 4; + + TransferLeader transfer_leader = 5; + // 拆分成多个分区，第一个SplitPartition是原分区，从第二开始是新分区 + SplitPartition split_partition = 6; + // rocksdb compaction 指定的表，null是针对所有 + DbCompaction db_compaction = 7; + // 将partition的数据，迁移到 target + MovePartition move_partition = 8; + // 清理partition的graph的数据 + CleanPartition clean_partition = 9; + // partition key range 变化 + PartitionKeyRange key_range = 10; +} + +/* Date model */ +message ChangeShard { + repeated metapb.Shard shard = 1; + ConfChangeType change_type = 2; +} + +message TransferLeader { + metapb.Shard shard = 1; +} + +message SplitPartition { + repeated metapb.Partition new_partition = 1; +} + +message DbCompaction { + string table_name = 3; +} + +message MovePartition{ + // target partition的key range为，迁移后的新range + metapb.Partition target_partition = 1; + // partition 的 key start 和 key end的所有数据， + // 会迁移到 target partition 上 + uint64 key_start = 2; + uint64 key_end = 3; +} + +message CleanPartition { + uint64 key_start = 1; + uint64 key_end = 2; + CleanType clean_type = 3; + bool delete_partition = 4; //是否删除分区 +} + +message PartitionKeyRange{ + uint32 partition_id = 1; + uint64 key_start = 2; + uint64 key_end = 3; +} + +message PdInstructionResponse { + PdInstructionType instruction_type = 1; + string leader_ip = 2; +} + +/* enums */ +enum PulseType { + PULSE_TYPE_UNKNOWN = 0; + PULSE_TYPE_PARTITION_HEARTBEAT = 1; + PULSE_TYPE_PD_INSTRUCTION = 2; +} + +enum PulseChangeType { + PULSE_CHANGE_TYPE_UNKNOWN = 0; + PULSE_CHANGE_TYPE_ADD = 1; + PULSE_CHANGE_TYPE_ALTER = 2; + PULSE_CHANGE_TYPE_DEL = 3; +} + +enum ConfChangeType { + CONF_CHANGE_TYPE_UNKNOWN = 0; + CONF_CHANGE_TYPE_ADD_NODE = 1; + CONF_CHANGE_TYPE_REMOVE_NODE = 2; + CONF_CHANGE_TYPE_ADD_LEARNER_NODE = 3; + CONF_CHANGE_TYPE_ADJUST = 4; // 调整shard，leader根据新的配置动态增减。 +} + +enum CleanType { + CLEAN_TYPE_KEEP_RANGE = 0; // 仅保留这个range + CLEAN_TYPE_EXCLUDE_RANGE = 1; // 删除这个range +} + +enum PdInstructionType { + CHANGE_TO_FOLLOWER = 0; +} diff --git a/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_watch.proto b/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_watch.proto new file mode 100644 index 0000000000..4911307260 --- /dev/null +++ b/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_watch.proto @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +import "metapb.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.hugegraph.pd.grpc.watch"; +option java_outer_classname = "HgPdWatchProto"; + +service HgPdWatch { + rpc Watch(stream WatchRequest) returns (stream WatchResponse); +} + +message WatchRequest { + WatchCreateRequest create_request = 1; + WatchCancelRequest cancel_request = 2; +} + +message WatchCreateRequest { + WatchType watch_type = 1; +} + +message WatchCancelRequest { + int64 watcher_id = 1; +} + +message WatchResponse { + WatchType watch_type = 1; + int64 watcher_id = 2; + int32 status = 3; //0=ok,1=fail + int64 notice_id = 4; + string msg = 5; + oneof response_union { + WatchPartitionResponse partition_response = 10; + WatchNodeResponse node_response = 11; + WatchGraphResponse graph_response = 12; + WatchShardGroupResponse shard_group_response = 13; + } +} + +message WatchPartitionResponse { + string graph = 1; + int32 partition_id = 2; + WatchChangeType change_type = 3; +} + +message WatchNodeResponse { + string graph = 1; + uint64 node_id = 2; + NodeEventType node_event_type = 3; +} + +message WatchGraphResponse { + metapb.Graph graph = 1; + WatchType type = 2; +} + +message WatchShardGroupResponse { + metapb.ShardGroup shard_group = 1; + WatchChangeType type = 2; + int32 shard_group_id = 3; +} + +enum WatchType { + WATCH_TYPE_UNKNOWN = 0; + WATCH_TYPE_PARTITION_CHANGE = 1; + WATCH_TYPE_STORE_NODE_CHANGE = 2; + WATCH_TYPE_GRAPH_CHANGE = 3; + WATCH_TYPE_SHARD_GROUP_CHANGE = 4; +} + +enum WatchChangeType { + WATCH_CHANGE_TYPE_UNKNOWN = 0; + WATCH_CHANGE_TYPE_ADD = 1; + WATCH_CHANGE_TYPE_ALTER = 2; + WATCH_CHANGE_TYPE_DEL = 3; + WATCH_CHANGE_TYPE_SPECIAL1 = 4; +} + +enum NodeEventType { + NODE_EVENT_TYPE_UNKNOWN = 0; + NODE_EVENT_TYPE_NODE_ONLINE = 1; + NODE_EVENT_TYPE_NODE_OFFLINE = 2; + NODE_EVENT_TYPE_NODE_RAFT_CHANGE = 3; + // pd leader 变更 + NODE_EVENT_TYPE_PD_LEADER_CHANGE = 4; +} diff --git a/hugegraph-pd/hg-pd-grpc/src/main/proto/pdpb.proto b/hugegraph-pd/hg-pd-grpc/src/main/proto/pdpb.proto new file mode 100644 index 0000000000..4e293ca08e --- /dev/null +++ b/hugegraph-pd/hg-pd-grpc/src/main/proto/pdpb.proto @@ -0,0 +1,607 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; +package pdpb; + +import "metapb.proto"; +import "metaTask.proto"; + +option java_package = "org.apache.hugegraph.pd.grpc"; + +service PD { + // 注册store，首次注册会生成新的store_id， store_id是store唯一标识 + rpc RegisterStore(RegisterStoreRequest) returns (RegisterStoreResponse) {} + rpc GetStore(GetStoreRequest) returns (GetStoreResponse) {} + // 修改Store状态等信息. + rpc SetStore(SetStoreRequest) returns (SetStoreResponse) {} + // 根据可以查找所属分区 + rpc DelStore(DetStoreRequest) returns (DetStoreResponse) {} + rpc GetAllStores(GetAllStoresRequest) returns (GetAllStoresResponse) {} + rpc StoreHeartbeat(StoreHeartbeatRequest) returns (StoreHeartbeatResponse) {} + + // 根据可以查找所属分区 + rpc GetPartition(GetPartitionRequest) returns (GetPartitionResponse) {} + + // 根据HashCode查找所属分区 + rpc GetPartitionByCode(GetPartitionByCodeRequest) returns (GetPartitionResponse) {} + // 根据PartitionID返回分区 + rpc GetPartitionByID(GetPartitionByIDRequest) returns (GetPartitionResponse) {} + rpc ScanPartitions(ScanPartitionsRequest) returns (ScanPartitionsResponse) {} + // 更新分区信息，主要用来更新分区key范围，调用此接口需谨慎，否则会造成数据丢失。 + rpc UpdatePartition(UpdatePartitionRequest) returns (UpdatePartitionResponse) {} + // 根据可以查找所属分区 + rpc DelPartition(DelPartitionRequest) returns (DelPartitionResponse) {} + // 根据条件查询分区信息, 包括Store、Graph等条件 + rpc QueryPartitions(QueryPartitionsRequest) returns (QueryPartitionsResponse){} + // 读取图信息 + rpc GetGraph(GetGraphRequest) returns (GetGraphResponse){} + // 修改图信息 + rpc SetGraph(SetGraphRequest) returns (SetGraphResponse){} + rpc DelGraph(DelGraphRequest) returns (DelGraphResponse){} + // 全局唯一自增ID + rpc GetId(GetIdRequest) returns (GetIdResponse){} + rpc ResetId(ResetIdRequest) returns (ResetIdResponse){} + // PD的集群列表 + rpc GetMembers(GetMembersRequest) returns (GetMembersResponse) {} + rpc GetStoreStatus(GetAllStoresRequest) returns (GetAllStoresResponse) {} + rpc GetPDConfig(GetPDConfigRequest) returns (GetPDConfigResponse){} + rpc SetPDConfig(SetPDConfigRequest) returns (SetPDConfigResponse){} + rpc GetGraphSpace(GetGraphSpaceRequest) returns (GetGraphSpaceResponse){} + rpc SetGraphSpace(SetGraphSpaceRequest) returns (SetGraphSpaceResponse){} + // 获取集群健康状态 + rpc GetClusterStats(GetClusterStatsRequest) returns (GetClusterStatsResponse){} + // 替换PD的集群节点 + rpc ChangePeerList(ChangePeerListRequest) returns (getChangePeerListResponse) {} + // 数据分裂 + rpc SplitData(SplitDataRequest) returns (SplitDataResponse){} + + rpc SplitGraphData(SplitGraphDataRequest) returns (SplitDataResponse) {} + // 数据迁移 + rpc MovePartition(MovePartitionRequest) returns (MovePartitionResponse){} + // 汇报分区分裂等任务执行结果 + rpc ReportTask(ReportTaskRequest) returns (ReportTaskResponse){} + + rpc GetPartitionStats(GetPartitionStatsRequest) returns (GetPartitionStatsResponse){} + //平衡store中分区leader的数量 + rpc BalanceLeaders(BalanceLeadersRequest) returns (BalanceLeadersResponse){} + + // 替换license文件 + rpc PutLicense(PutLicenseRequest) returns (PutLicenseResponse){} + + // 通知rocksdb进行compaction + rpc DbCompaction(DbCompactionRequest) returns (DbCompactionResponse){} + + // 合并分区 + rpc CombineCluster(CombineClusterRequest) returns (CombineClusterResponse){} + // 单个图缩容 + rpc CombineGraph(CombineGraphRequest) returns (CombineGraphResponse) {} + + // shard group + rpc GetShardGroup(GetShardGroupRequest) returns (GetShardGroupResponse){} + rpc UpdateShardGroup(UpdateShardGroupRequest) returns (UpdateShardGroupResponse){} + // 删除掉shard group + rpc DeleteShardGroup(DeleteShardGroupRequest) returns (DeleteShardGroupResponse) {} + // shard group 运维相关的处理 + rpc UpdateShardGroupOp(ChangeShardRequest) returns (ChangeShardResponse){} + // change shard + rpc ChangeShard(ChangeShardRequest) returns (ChangeShardResponse) {} + // 更新pd raft + rpc updatePdRaft(UpdatePdRaftRequest) returns (UpdatePdRaftResponse) {} + + rpc getCache(GetGraphRequest) returns (CacheResponse) {} + rpc getPartitions(GetGraphRequest) returns (CachePartitionResponse) {} +} + +message RequestHeader { + // 集群 ID. + uint64 cluster_id = 1; + // 发送者 ID. + uint64 sender_id = 2; +} + +message ResponseHeader { + // cluster_id is the ID of the cluster which sent the response. + uint64 cluster_id = 1; + Error error = 2; +} + +enum ErrorType { + OK = 0; + UNKNOWN = 1; + + NOT_LEADER = 100; + STORE_ID_NOT_EXIST = 101; + NO_ACTIVE_STORE = 102; + NOT_FOUND = 103; + PD_UNREACHABLE = 104; + LESS_ACTIVE_STORE = 105; + STORE_HAS_BEEN_REMOVED = 106; + STORE_PROHIBIT_DELETION = 111; + SET_CONFIG_SHARD_COUNT_ERROR = 112; + UPDATE_STORE_STATE_ERROR = 113; + STORE_PROHIBIT_DUPLICATE = 114; + ROCKSDB_READ_ERROR = 1002; + ROCKSDB_WRITE_ERROR = 1003; + ROCKSDB_DEL_ERROR = 1004; + ROCKSDB_SAVE_SNAPSHOT_ERROR = 1005; + ROCKSDB_LOAD_SNAPSHOT_ERROR = 1006; + + // 当前集群状态禁止分裂 + Cluster_State_Forbid_Splitting = 1007; + // 正在分裂中 + Split_Partition_Doing = 1008; + // store上分区数量超过上限 + Too_Many_Partitions_Per_Store = 1009; + // license 错误 + LICENSE_ERROR = 107; + // license 认证错误 + LICENSE_VERIFY_ERROR = 108; + + //分区下线正在进行 + Store_Tombstone_Doing = 1010; + + // 不合法的分裂个数 + Invalid_Split_Partition_Count = 1011; +} + +message Error { + ErrorType type = 1; + string message = 2; +} +message GetStoreRequest { + RequestHeader header = 1; + uint64 store_id = 2; +} + +message GetStoreResponse { + ResponseHeader header = 1; + + metapb.Store store = 2; + metapb.StoreStats stats = 3; +} + +message DetStoreRequest { + RequestHeader header = 1; + uint64 store_id = 2; +} + +message DetStoreResponse { + ResponseHeader header = 1; + metapb.Store store = 2; +} + +message RegisterStoreRequest { + RequestHeader header = 1; + metapb.Store store = 2; +} + + +message RegisterStoreResponse { + ResponseHeader header = 1; + // 初次注册，返回新的store_id + uint64 store_id = 2; +} + +message SetStoreRequest { + RequestHeader header = 1; + metapb.Store store = 2; +} + +message SetStoreResponse { + ResponseHeader header = 1; + // 返回修改后的Store + metapb.Store store = 2; +} + + +// 返回graph_name所在的所有store，如果graph_name为空值，则返回系统所有的store +message GetAllStoresRequest { + RequestHeader header = 1; + string graph_name = 2; + // 是否返回离线的store + bool exclude_offline_stores = 3; +} + +message GetAllStoresResponse { + ResponseHeader header = 1; + + repeated metapb.Store stores = 2; +} + + +message StoreHeartbeatRequest { + RequestHeader header = 1; + + metapb.StoreStats stats = 2; +} + +message StoreHeartbeatResponse { + ResponseHeader header = 1; + string cluster_version = 3; + metapb.ClusterStats clusterStats = 4; +} + +message GetPartitionRequest { + RequestHeader header = 1; + string graph_name = 2; + bytes key = 3; +} + + +message GetPartitionByCodeRequest { + RequestHeader header = 1; + string graph_name = 2; + uint64 code = 3; +} + + +message GetPartitionResponse { + ResponseHeader header = 1; + metapb.Partition partition = 2; + metapb.Shard leader = 3; + // 离线的Shard + repeated metapb.Shard offline_shards = 4; +} + +message GetPartitionByIDRequest { + RequestHeader header = 1; + string graph_name = 2; + uint32 partition_id = 3; +} + +message DelPartitionRequest { + RequestHeader header = 1; + string graph_name = 2; + uint32 partition_id = 3; +} +message DelPartitionResponse { + ResponseHeader header = 1; + metapb.Partition partition = 2; +} + +message UpdatePartitionRequest{ + RequestHeader header = 1; + repeated metapb.Partition partition = 2; +} + +message UpdatePartitionResponse{ + ResponseHeader header = 1; + repeated metapb.Partition partition = 2; +} +// Use GetPartitionResponse as the response of GetPartitionByIDRequest. + +message ScanPartitionsRequest { + RequestHeader header = 1; + string graph_name = 2; + bytes start_key = 3; + bytes end_key = 4; // end_key is +inf when it is empty. +} + + + +message ScanPartitionsResponse { + ResponseHeader header = 1; + repeated metapb.PartitionShard partitions = 4; +} + + + +message QueryPartitionsRequest{ + RequestHeader header = 1; + metapb.PartitionQuery query = 2; +} + +message QueryPartitionsResponse { + ResponseHeader header = 1; + repeated metapb.Partition partitions = 4; +} + + + +message GetGraphRequest{ + RequestHeader header = 1; + string graph_name = 2; +} + +message GetGraphResponse{ + ResponseHeader header = 1; + metapb.Graph graph = 2; +} + +message SetGraphRequest{ + RequestHeader header = 1; + metapb.Graph graph = 2; +} + +message SetGraphResponse{ + ResponseHeader header = 1; + metapb.Graph graph = 2; +} + +message DelGraphRequest{ + RequestHeader header = 1; + string graph_name = 2; +} + +message DelGraphResponse{ + ResponseHeader header = 1; + metapb.Graph graph = 2; +} + +message GetIdRequest{ + RequestHeader header = 1; + string key = 2; + int32 delta = 3; +} + +message GetIdResponse{ + ResponseHeader header = 1; + int64 id = 2; + int32 delta = 3; +} + +message ResetIdRequest{ + RequestHeader header = 1; + string key = 2; +} + +message ResetIdResponse{ + ResponseHeader header = 1; + int32 result = 2; +} + +message GetMembersRequest{ + RequestHeader header = 1; +} + +message GetMembersResponse{ + ResponseHeader header = 1; + repeated metapb.Member members = 2; + metapb.Member leader = 3; +} + +message GetPDConfigRequest{ + RequestHeader header = 1; + uint64 version = 2 ; +} + +message GetPDConfigResponse{ + ResponseHeader header = 1; + metapb.PDConfig pd_config = 2; +} + +message SetPDConfigRequest{ + RequestHeader header = 1; + metapb.PDConfig pd_config = 2; +} + +message SetPDConfigResponse{ + ResponseHeader header = 1; +} + + +message GetGraphSpaceRequest{ + RequestHeader header = 1; + string graph_Space_Name = 2; +} + +message GetGraphSpaceResponse{ + ResponseHeader header = 1; + repeated metapb.GraphSpace graph_space = 2; +} + +message SetGraphSpaceRequest{ + RequestHeader header = 1; + metapb.GraphSpace graph_space = 2; +} + +message SetGraphSpaceResponse{ + ResponseHeader header = 1; +} + +message GetClusterStatsRequest{ + RequestHeader header = 1; +} + +message GetClusterStatsResponse{ + ResponseHeader header = 1; + metapb.ClusterStats cluster = 2; +} +message ChangePeerListRequest{ + RequestHeader header = 1; + string peer_List = 2; +} +message getChangePeerListResponse{ + ResponseHeader header = 1; +} + +enum OperationMode { + Auto = 0; + Expert = 1; +} + +message SplitDataParam{ + // 被分裂的源分区ID + uint32 partition_id = 1; + //目标分区数量 + uint32 count = 2; +} + +message SplitDataRequest{ + RequestHeader header = 1; + //工作模式 + // Auto：自动分裂，每个Store上分区数达到最大值 + // Expert:专家模式，需要指定splitParams + OperationMode mode = 2; + repeated SplitDataParam param = 3; +} + +message SplitGraphDataRequest{ + RequestHeader header = 1; + //工作模式 + string graph_name = 2; + uint32 to_count = 3; +} + +message SplitDataResponse{ + ResponseHeader header = 1; +} + +message MovePartitionParam{ + uint32 partition_id = 1; + uint64 src_store_id = 2; + uint64 dst_store_id = 3; +} + +message MovePartitionRequest{ + RequestHeader header = 1; + //工作模式 + // Auto：自动转移，达到每个Store上分区数量相同 + // Expert:专家模式，需要指定transferParams + OperationMode mode = 2; + repeated MovePartitionParam param = 3; +} + +message MovePartitionResponse{ + ResponseHeader header = 1; +} + +message ReportTaskRequest{ + RequestHeader header = 1; + metaTask.Task task = 2; +} + +message ReportTaskResponse{ + ResponseHeader header = 1; +} + +message GetPartitionStatsRequest{ + RequestHeader header = 1; + uint32 partition_id = 2; + // 如果未空，返回所有图的同一分区ID + string graph_name = 4; +} + +message GetPartitionStatsResponse{ + ResponseHeader header = 1; + metapb.PartitionStats partition_stats = 2; +} + +message BalanceLeadersRequest{ + RequestHeader header = 1; +} + +message BalanceLeadersResponse{ + ResponseHeader header = 1; +} + +message PutLicenseRequest{ + RequestHeader header = 1; + bytes content = 2; +} + +message PutLicenseResponse{ + ResponseHeader header = 1; +} + +message DbCompactionRequest{ + RequestHeader header = 1; + string tableName = 2; +} + +message DbCompactionResponse{ + ResponseHeader header = 1; +} + +message CombineClusterRequest { + RequestHeader header = 1; + uint32 toCount = 2; +} + +message CombineClusterResponse { + ResponseHeader header = 1; +} + +message CombineGraphRequest { + RequestHeader header = 1; + string graphName = 2; + uint32 toCount = 3; +} + +message CombineGraphResponse { + ResponseHeader header = 1; +} + +message DeleteShardGroupRequest { + RequestHeader header = 1; + uint32 groupId = 2; +} + +message DeleteShardGroupResponse { + ResponseHeader header = 1; +} + +message GetShardGroupRequest{ + RequestHeader header = 1; + uint32 group_id = 2 ; +} + +message GetShardGroupResponse{ + ResponseHeader header = 1; + metapb.ShardGroup shardGroup = 2; +} + +message UpdateShardGroupRequest{ + RequestHeader header = 1; + metapb.ShardGroup shardGroup = 2; +} + +message UpdateShardGroupResponse{ + ResponseHeader header = 1; +} + +message ChangeShardRequest{ + RequestHeader header = 1; + uint32 groupId = 2; + repeated metapb.Shard shards = 3; +} + +message ChangeShardResponse { + ResponseHeader header = 1; +} + +message UpdatePdRaftRequest{ + RequestHeader header = 1; + string config = 3; +} + +message UpdatePdRaftResponse{ + ResponseHeader header = 1; + string message = 2; +} +message CacheResponse { + ResponseHeader header = 1; + // 返回修改后的Store + repeated metapb.Store stores = 2; + repeated metapb.ShardGroup shards = 3; + repeated metapb.Graph graphs = 4; +} +message CachePartitionResponse { + ResponseHeader header = 1; + repeated metapb.Partition partitions = 2; +} diff --git a/hugegraph-pd/pom.xml b/hugegraph-pd/pom.xml new file mode 100644 index 0000000000..e7ccdce9b0 --- /dev/null +++ b/hugegraph-pd/pom.xml @@ -0,0 +1,272 @@ + + + + + 4.0.0 + hugegraph-pd + ${revision} + pom + + + org.apache.hugegraph + hugegraph + ${revision} + ../pom.xml + + + + hg-pd-grpc + hg-pd-client + hg-pd-core + hg-pd-service + hg-pd-common + hg-pd-dist + hg-pd-clitools + hg-pd-test + + + + + 11 + 11 + 2.17.0 + + + + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.17.0 + + + org.apache.hugegraph + hg-pd-grpc + ${revision} + + + org.apache.hugegraph + hg-pd-common + ${revision} + + + + + + + junit + junit + 4.13.2 + test + + + + + + + org.jacoco + jacoco-maven-plugin + 0.8.4 + + + **/grpc/**.* + **/config/**.* + + + + + + prepare-agent + + + + + + org.codehaus.mojo + flatten-maven-plugin + 1.2.7 + + true + resolveCiFriendliesOnly + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + + + org.apache.maven.plugins + maven-clean-plugin + + + + ${project.basedir}/ + + *.tar + *.tar.gz + .flattened-pom.xml + + dist/** + + false + + + + + + + + + + + + pd-client-test + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.20 + + + pd-client-test + + test + + test + + + + + + + + pd-core-test + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.20 + + + pd-core-test + + test + + test + + + + + + + + pd-cli-tools-test + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.20 + + + pd-cli-tools-test + + test + + test + + + + + + + + pd-common-test + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.20 + + + pd-common-test + + test + + test + + + + + + + + pd-service-test + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.20 + + + pd-service-test + + test + + test + + + + + + + + diff --git a/pom.xml b/pom.xml index 3de7393dba..7442ff1960 100644 --- a/pom.xml +++ b/pom.xml @@ -89,14 +89,15 @@ - 1.2.0 + + + 1.5.0.1 hugegraph-server - - - + hugegraph-pd + hugegraph-store From 2c4ff513c0006b24fc06107e1425bd685f3f26cb Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Sat, 24 Feb 2024 22:49:57 +0800 Subject: [PATCH 02/17] chore: compile --- hugegraph-pd/.gitignore | 1 + hugegraph-pd/pom.xml | 14 +++++++------- pom.xml | 5 ++++- 3 files changed, 12 insertions(+), 8 deletions(-) create mode 100644 hugegraph-pd/.gitignore diff --git a/hugegraph-pd/.gitignore b/hugegraph-pd/.gitignore new file mode 100644 index 0000000000..08bc7dd8af --- /dev/null +++ b/hugegraph-pd/.gitignore @@ -0,0 +1 @@ +hg-pd-grpc/src/main/java/ diff --git a/hugegraph-pd/pom.xml b/hugegraph-pd/pom.xml index e7ccdce9b0..72939bbce5 100644 --- a/hugegraph-pd/pom.xml +++ b/hugegraph-pd/pom.xml @@ -33,14 +33,14 @@ hg-pd-grpc - hg-pd-client - hg-pd-core - hg-pd-service hg-pd-common - hg-pd-dist - hg-pd-clitools - hg-pd-test - + hg-pd-client + + + + + + diff --git a/pom.xml b/pom.xml index 7442ff1960..452659e517 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,8 @@ hugegraph-server hugegraph-pd - hugegraph-store + + @@ -181,6 +182,8 @@ **/hbase-*/** **/apache-cassandra-*/** **/pid + + **/src/main/java/org/apache/hugegraph/pd/grpc/** true From c7f4a95c3f7030dd99d14bb3a31cbca044001487 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Mon, 26 Feb 2024 18:23:35 +0800 Subject: [PATCH 03/17] chore: remove JAVA 8 ci --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d439c31337..bae3434171 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: fail-fast: false matrix: BACKEND: [ memory, rocksdb, hbase, cassandra, mysql, postgresql, scylladb ] - JAVA_VERSION: [ '8', '11' ] + JAVA_VERSION: [ '11' ] steps: - name: Checkout From cd7d7fc30af525fee7917dd17bc06f0bb71efca3 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Mon, 26 Feb 2024 19:37:17 +0800 Subject: [PATCH 04/17] git add hugegraph-pd/hg-pd-test/ --- hugegraph-pd/hg-pd-test/pom.xml | 359 ++++++++ .../hugegraph/pd/MonitorServiceTest.java | 112 +++ .../hugegraph/pd/PartitionCacheTest.java | 102 +++ .../hugegraph/pd/PartitionServiceTest.java | 45 + .../hugegraph/pd/StoreNodeServiceTest.java | 485 ++++++++++ .../hugegraph/pd/StoreRegisterTest.java | 135 +++ .../org/apache/hugegraph/pd/UnitTestBase.java | 31 + .../hugegraph/pd/client/BaseClientTest.java | 44 + .../pd/client/DiscoveryClientImplTest.java | 147 ++++ .../pd/client/DiscoveryClientTest.java | 79 ++ .../hugegraph/pd/client/KvClientTest.java | 121 +++ .../pd/client/LicenseClientImplTest.java | 127 +++ .../pd/client/PDClientSuiteTest.java | 36 + .../hugegraph/pd/client/PDClientTest.java | 418 +++++++++ .../hugegraph/pd/client/PDPulseTest.java | 110 +++ .../hugegraph/pd/client/PDWatchTest.java | 86 ++ .../pd/client/test/HgPDTestUtil.java | 92 ++ .../pd/clitools/BaseCliToolsTest.java | 34 + .../pd/clitools/CliToolsSuiteTest.java | 35 + .../hugegraph/pd/clitools/MainTest.java | 89 ++ .../hugegraph/pd/common/BaseCommonTest.java | 33 + .../hugegraph/pd/common/CommonSuiteTest.java | 43 + .../hugegraph/pd/common/HgAssertTest.java | 134 +++ .../hugegraph/pd/common/KVPairTest.java | 73 ++ .../pd/common/MetadataKeyHelperTest.java | 217 +++++ .../pd/common/PartitionCacheTest.java | 392 +++++++++ .../pd/common/PartitionUtilsTest.java | 53 ++ .../hugegraph/pd/core/BaseCoreTest.java | 74 ++ .../hugegraph/pd/core/PDCoreSuiteTest.java | 37 + .../pd/core/StoreNodeServiceTest.java | 119 +++ .../pd/core/meta/MetadataKeyHelperTest.java | 34 + .../hugegraph/pd/grpc/BaseGrpcTest.java | 36 + .../hugegraph/pd/grpc/GrpcSuiteTest.java | 33 + .../hugegraph/pd/service/BaseServerTest.java | 57 ++ .../pd/service/ConfigServiceTest.java | 106 +++ .../hugegraph/pd/service/IdServiceTest.java | 109 +++ .../hugegraph/pd/service/KvServiceTest.java | 60 ++ .../hugegraph/pd/service/LogServiceTest.java | 54 ++ .../pd/service/PartitionServiceTest.java | 133 +++ .../hugegraph/pd/service/PdTestBase.java | 219 +++++ .../hugegraph/pd/service/RestApiTest.java | 120 +++ .../hugegraph/pd/service/ServerSuiteTest.java | 42 + .../service/StoreMonitorDataServiceTest.java | 82 ++ .../pd/service/StoreNodeServiceNewTest.java | 64 ++ .../pd/service/StoreServiceTest.java | 833 ++++++++++++++++++ .../pd/service/TaskScheduleServiceTest.java | 112 +++ .../hugegraph/pd/store/HgKVStoreImplTest.java | 105 +++ .../hg-pd-test/src/main/resources/log4j2.xml | 139 +++ hugegraph-pd/pom.xml | 2 +- 49 files changed, 6201 insertions(+), 1 deletion(-) create mode 100644 hugegraph-pd/hg-pd-test/pom.xml create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/MonitorServiceTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/PartitionCacheTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/PartitionServiceTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/StoreNodeServiceTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/StoreRegisterTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/UnitTestBase.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/BaseClientTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImplTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/KvClientTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/LicenseClientImplTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientSuiteTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDPulseTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDWatchTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/test/HgPDTestUtil.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/BaseCliToolsTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/CliToolsSuiteTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/MainTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/BaseCommonTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/CommonSuiteTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/HgAssertTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/KVPairTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/MetadataKeyHelperTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionCacheTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionUtilsTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/BaseCoreTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreSuiteTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/StoreNodeServiceTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/meta/MetadataKeyHelperTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/grpc/BaseGrpcTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/grpc/GrpcSuiteTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/BaseServerTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/ConfigServiceTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/IdServiceTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/KvServiceTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/LogServiceTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/PartitionServiceTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/PdTestBase.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/RestApiTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/ServerSuiteTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreMonitorDataServiceTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreNodeServiceNewTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreServiceTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/TaskScheduleServiceTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImplTest.java create mode 100644 hugegraph-pd/hg-pd-test/src/main/resources/log4j2.xml diff --git a/hugegraph-pd/hg-pd-test/pom.xml b/hugegraph-pd/hg-pd-test/pom.xml new file mode 100644 index 0000000000..19e1855d8c --- /dev/null +++ b/hugegraph-pd/hg-pd-test/pom.xml @@ -0,0 +1,359 @@ + + + + + + org.apache.hugegraph + hugegraph-pd + ${revision} + ../pom.xml + + 4.0.0 + + hg-pd-test + + + true + + 2.0.0-RC.3 + + + + + jacoco + + false + + + + + org.jacoco + jacoco-maven-plugin + 0.8.4 + + + **/grpc/**.* + **/config/**.* + + + + + + prepare-agent + + + + + + + + + + + + junit + junit + 4.13.2 + + + org.projectlombok + lombok + 1.18.24 + + + org.springframework + spring-context-support + 5.3.20 + + + org.springframework + spring-test + 5.3.20 + test + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j2.version} + + + + org.apache.hugegraph + hg-pd-client + ${revision} + + + + com.google.code.gson + gson + 2.8.9 + + + org.apache.hugegraph + hg-pd-grpc + ${revision} + + + commons-io + commons-io + 2.7 + + + com.fasterxml.jackson.core + jackson-databind + 2.13.0 + + + com.fasterxml.jackson.core + jackson-core + 2.13.0 + + + com.fasterxml.jackson.core + jackson-annotations + 2.13.0 + + + + + + + org.springframework.boot + spring-boot-starter-test + 2.5.14 + + + org.springframework.boot + spring-boot-starter-logging + + + + + org.apache.hugegraph + hg-pd-clitools + ${revision} + + + org.apache.hugegraph + hg-pd-common + ${revision} + + + + org.apache.hugegraph + hg-pd-core + ${revision} + + + org.apache.hugegraph + hg-pd-dist + ${revision} + compile + + + org.apache.hugegraph + hg-pd-service + ${revision} + + + + + + org.powermock + powermock-classloading-xstream + ${powermock.version} + + + org.powermock + powermock-module-junit4-rule + ${powermock.version} + + + org.powermock + powermock-api-support + ${powermock.version} + + + org.powermock + powermock-module-junit4 + 2.0.0-RC.3 + compile + + + org.powermock + powermock-api-mockito2 + 2.0.0-RC.3 + compile + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.20 + + + pd-client-test + + ${basedir}/src/main/java/ + + ${basedir}/target/classes/ + + + **/PDClientSuiteTest.java + + + + + pd-core-test + + ${basedir}/src/main/java/ + + ${basedir}/target/classes/ + + + **/PDCoreSuiteTest.java + + + + + pd-cli-tools-test + + ${basedir}/src/main/java/ + + ${basedir}/target/classes/ + + + **/CliToolsSuiteTest.java + + + + + pd-common-test + + ${basedir}/src/main/java/ + + ${basedir}/target/classes/ + + + **/CommonSuiteTest.java + + + + + pd-service-test + + ${basedir}/src/main/java/ + + ${basedir}/target/classes/ + + + **/ServerSuiteTest.java + + + + + + + org.jacoco + jacoco-maven-plugin + 0.8.4 + + + pre-test + + prepare-agent + + + + post-test + test + + report-aggregate + + + ${basedir}/target/site/jacoco + + + + + + org/apache/hugegraph/pd/rest/*.class + org/apache/hugegraph/pd/service/*.class + org/apache/hugegraph/pd/model/*.class + org/apache/hugegraph/pd/watch/*.class + org/apache/hugegraph/pd/pulse/*.class + org/apache/hugegraph/pd/license/*.class + org/apache/hugegraph/pd/notice/*.class + org/apache/hugegraph/pd/util/*.class + org/apache/hugegraph/pd/metrics/*.class + org/apache/hugegraph/pd/util/grpc/*.class + org/apache/hugegraph/pd/boot/*.class + org/apache/hugegraph/pd/grpc/**/*.class + org/apache/hugegraph/pd/raft/*.class + **/RaftKVStore.class + + + + + + + + src/main/resources/ + true + + + + + diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/MonitorServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/MonitorServiceTest.java new file mode 100644 index 0000000000..25272127fd --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/MonitorServiceTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd; + +import java.util.concurrent.ExecutionException; + +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.config.PDConfig; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.junit.Assert; +import org.junit.BeforeClass; + +// import org.junit.Test; + +public class MonitorServiceTest { + static PDConfig pdConfig; + + @BeforeClass + public static void init() throws ExecutionException, InterruptedException { + pdConfig = new PDConfig() {{ + this.setClusterId(100); + this.setPatrolInterval(1); + }}; + + //pdConfig.setEtcd(new PDConfig().new Etcd() {{ + // this.setAddress("http://localhost:2379"); + // + //}}); + pdConfig.setStore(new PDConfig().new Store() {{ + this.setMaxDownTime(1); + this.setKeepAliveTimeout(5); + }}); + + pdConfig.setPartition(new PDConfig().new Partition() {{ + this.setShardCount(3); + this.setTotalCount(10); + }}); + + clearClusterData(); + } + + public static void clearClusterData() throws ExecutionException, InterruptedException { + //Client client = Client.builder().endpoints(pdConfig.getEtcd().getAddress()).build(); + //KV kvClient = client.getKVClient(); + // + //ByteSequence key = ByteSequence.from("HUGEGRAPH/" + pdConfig.getClusterId(), Charset + // .forName("utf-8")); + //CompletableFuture rsp = kvClient.delete(key, DeleteOption.newBuilder() + // .isPrefix(true).build()); + //System.out.println("删除数量 : " + rsp.get().getDeleted()); + //kvClient.close(); + //client.close(); + } + + // @Test + public void testPatrolStores() throws PDException, InterruptedException { + StoreNodeService storeService = new StoreNodeService(pdConfig); + PartitionService partitionService = new PartitionService(pdConfig, storeService); + TaskScheduleService monitorService = + new TaskScheduleService(pdConfig, storeService, partitionService); + storeService.init(partitionService); + partitionService.init(); + monitorService.init(); + + int count = 6; + Metapb.Store[] stores = new Metapb.Store[count]; + for (int i = 0; i < count; i++) { + Metapb.Store store = Metapb.Store.newBuilder() + .setId(0) + .setAddress(String.valueOf(i)) + .setDeployPath("/data") + .addLabels(Metapb.StoreLabel.newBuilder() + .setKey("namespace") + .setValue("default") + .build()) + .build(); + stores[i] = storeService.register(store); + System.out.println("新注册store， id = " + Long.toHexString(stores[i].getId())); + } + Metapb.Graph graph = Metapb.Graph.newBuilder() + .setGraphName("defaultGH") + + .setPartitionCount(10) + .build(); + partitionService.updateGraph(graph); + Thread.sleep(10000); + count = 0; + count += storeService.getStores("").stream() + .filter(store -> store.getState() == Metapb.StoreState.Tombstone) + .count(); + + Assert.assertEquals(6, count); + + } + + +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/PartitionCacheTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/PartitionCacheTest.java new file mode 100644 index 0000000000..71efa70ecd --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/PartitionCacheTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hugegraph.pd.common.KVPair; +import org.apache.hugegraph.pd.common.PartitionCache; +import org.apache.hugegraph.pd.grpc.Metapb; + +import com.google.common.collect.Range; +import com.google.common.collect.RangeMap; +import com.google.common.collect.TreeRangeMap; + +public class PartitionCacheTest { + + // @Test + public void test() { + PartitionCache cache = new PartitionCache(); + for (int i = 0; i < 10; i++) { + KVPair partShards = + new KVPair<>(Metapb.Partition.newBuilder() + .setStartKey(i * 10) + .setEndKey((i + 1) * 10) + .build(), null); + cache.updatePartition("aa", i, partShards.getKey()); + } + + for (int i = 0; i < 100; i++) { + KVPair partShards = cache.getPartitionByCode("aa", i); + System.out.println(" " + i + " " + partShards.getKey().getStartKey()); + } + } + + + // @Test + public void test1() { + Map> keyToPartIdCache = new HashMap<>(); + // graphName + PartitionID组成key + Map> partitionCache = new HashMap<>(); + + // 缓存全部Store，用于全库查询，需要优化 + Map> allStoresCache = new HashMap<>(); + + keyToPartIdCache.put("a", TreeRangeMap.create()); + + keyToPartIdCache.get("a") + .put(Range.closedOpen(1L, 2L), 1); + + allStoresCache.put("a", new ArrayList<>()); + allStoresCache.get("a").add(Metapb.Store.newBuilder().setId(34).build()); + + + Map> keyToPartIdCache2 = + cloneKeyToPartIdCache(keyToPartIdCache); + System.out.println(keyToPartIdCache2.size()); + } + + public Map> cloneKeyToPartIdCache( + Map> cache) { + Map> cacheClone = new HashMap<>(); + cache.forEach((k1, v1) -> { + cacheClone.put(k1, TreeRangeMap.create()); + v1.asMapOfRanges().forEach((k2, v2) -> { + cacheClone.get(k1).put(k2, v2); + }); + }); + return cacheClone; + } + + public Map> + clonePartitionCache(Map> cache) { + Map> cacheClone = new HashMap<>(); + cacheClone.putAll(cache); + return cacheClone; + } + + public Map> + cloneStoreCache(Map> cache) { + Map> cacheClone = new HashMap<>(); + cacheClone.putAll(cache); + return cacheClone; + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/PartitionServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/PartitionServiceTest.java new file mode 100644 index 0000000000..82fd4c9bdb --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/PartitionServiceTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.hugegraph.pd.grpc.Metapb; +import org.junit.Test; + +public class PartitionServiceTest { + @Test + public void testPartitionHeartbeat() { + List shardList = new ArrayList<>(); + shardList.add(Metapb.Shard.newBuilder().setStoreId(1).build()); + shardList.add(Metapb.Shard.newBuilder().setStoreId(2).build()); + shardList.add(Metapb.Shard.newBuilder().setStoreId(3).build()); + shardList = new ArrayList<>(shardList); + Metapb.PartitionStats stats = Metapb.PartitionStats.newBuilder() + .addAllShard(shardList).build(); + List shardList2 = new ArrayList<>(stats.getShardList()); + Collections.shuffle(shardList2); + shardList2.forEach(shard -> { + System.out.println(shard.getStoreId()); + }); + + + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/StoreNodeServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/StoreNodeServiceTest.java new file mode 100644 index 0000000000..c8f0ce39e5 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/StoreNodeServiceTest.java @@ -0,0 +1,485 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.config.PDConfig; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.pd.grpc.pulse.ChangeShard; +import org.apache.hugegraph.pd.grpc.pulse.CleanPartition; +import org.apache.hugegraph.pd.grpc.pulse.DbCompaction; +import org.apache.hugegraph.pd.grpc.pulse.MovePartition; +import org.apache.hugegraph.pd.grpc.pulse.PartitionKeyRange; +import org.apache.hugegraph.pd.grpc.pulse.SplitPartition; +import org.apache.hugegraph.pd.grpc.pulse.TransferLeader; +import org.junit.Assert; +import org.junit.BeforeClass; + +public class StoreNodeServiceTest { + static PDConfig pdConfig; + + @BeforeClass + public static void init() throws Exception { + String path = "tmp/unitTest"; + deleteDirectory(new File(path)); + pdConfig = new PDConfig() {{ + this.setClusterId(100); + this.setInitialStoreList( + "127.0.0.1:8500,127.0.0.1:8501,127.0.0.1:8502,127.0.0.1:8503,127.0.0.1:8504," + + "127.0.0.1:8505"); + }}; + + pdConfig.setStore(new PDConfig().new Store() {{ + this.setMaxDownTime(3600); + this.setKeepAliveTimeout(3600); + }}); + + pdConfig.setPartition(new PDConfig().new Partition() {{ + this.setShardCount(3); + this.setMaxShardsPerStore(3); + }}); + pdConfig.setRaft(new PDConfig().new Raft() {{ + this.setEnable(false); + }}); + pdConfig.setDiscovery(new PDConfig().new Discovery()); + pdConfig.setDataPath(path); + ConfigService configService = new ConfigService(pdConfig); + pdConfig = configService.loadConfig(); + } + + public static byte[] intToByteArray(int i) { + byte[] result = new byte[4]; + result[0] = (byte) ((i >> 24) & 0xFF); + result[1] = (byte) ((i >> 16) & 0xFF); + result[2] = (byte) ((i >> 8) & 0xFF); + result[3] = (byte) (i & 0xFF); + return result; + } + + public static void deleteDirectory(File dir) { + try { + FileUtils.deleteDirectory(dir); + } catch (IOException e) { + System.out.printf("Failed to start ....,%s%n", e.getMessage()); + } + } + + // @Test + public void testStoreNodeService() throws PDException { + Assert.assertEquals(pdConfig.getPartition().getTotalCount(), + (long) pdConfig.getInitialStoreMap().size() * + pdConfig.getPartition().getMaxShardsPerStore() + / pdConfig.getPartition().getShardCount()); + StoreNodeService storeService = new StoreNodeService(pdConfig); + int count = 6; + Metapb.Store[] stores = new Metapb.Store[count]; + for (int i = 0; i < count; i++) { + Metapb.Store store = Metapb.Store.newBuilder() + .setId(0) + .setAddress("127.0.0.1:850" + i) + .setDeployPath("/data") + .addLabels(Metapb.StoreLabel.newBuilder() + .setKey("namespace") + .setValue("default") + .build()) + .build(); + stores[i] = storeService.register(store); + System.out.println("新注册store， id = " + stores[i].getId()); + } + Assert.assertEquals(count, storeService.getStores("").size()); + + for (Metapb.Store store : stores) { + Metapb.StoreStats stats = Metapb.StoreStats.newBuilder() + .setStoreId(store.getId()) + .build(); + storeService.heartBeat(stats); + } + + Assert.assertEquals(6, storeService.getActiveStores("").size()); + + Metapb.Graph graph = Metapb.Graph.newBuilder() + .setGraphName("defaultGH") + .setPartitionCount(10) + .build(); + // 分配shard + List shards = storeService.allocShards(graph, 1); + + + Assert.assertEquals(3, shards.size()); + + Assert.assertEquals(pdConfig.getPartition().getTotalCount(), + storeService.getShardGroups().size()); // 设置leader + Metapb.Shard leader = Metapb.Shard.newBuilder(shards.get(0)) + .setRole(Metapb.ShardRole.Leader).build(); + shards = new ArrayList<>(shards); + shards.set(0, leader); + // 增加shard + pdConfig.getPartition().setShardCount(5); + + Metapb.ShardGroup shardGroup = Metapb.ShardGroup.newBuilder() + .setId(1) + .addAllShards(shards).build(); + shards = storeService.reallocShards(shardGroup); + + Assert.assertEquals(5, shards.size()); + // 减少shard + pdConfig.getPartition().setShardCount(3); + shards = storeService.reallocShards(shardGroup); + Assert.assertEquals(3, shards.size()); + // 包含leader，leader不能被删除 + Assert.assertTrue(shards.contains(leader)); + + // 减少shard + pdConfig.getPartition().setShardCount(1); + graph = Metapb.Graph.newBuilder(graph).build(); + shards = storeService.reallocShards(shardGroup); + Assert.assertEquals(1, shards.size()); + // 包含leader，leader不能被删除 + Assert.assertTrue(shards.contains(leader)); + + for (Metapb.Store store : stores) { + storeService.removeStore(store.getId()); + } + Assert.assertEquals(0, storeService.getStores("").size()); + + + } + + // @Test + public void testSplitPartition() throws PDException { + StoreNodeService storeService = new StoreNodeService(pdConfig); + PartitionService partitionService = new PartitionService(pdConfig, storeService); + + storeService.init(partitionService); + partitionService.addInstructionListener(new PartitionInstructionListener() { + + @Override + public void changeShard(Metapb.Partition partition, ChangeShard changeShard) throws + PDException { + + } + + @Override + public void transferLeader(Metapb.Partition partition, + TransferLeader transferLeader) throws PDException { + + } + + @Override + public void splitPartition(Metapb.Partition partition, + SplitPartition splitPartition) throws PDException { + splitPartition.getNewPartitionList().forEach(p -> { + System.out.println("SplitPartition " + p.getId() + " " + p.getStartKey() + "," + + p.getEndKey()); + }); + } + + @Override + public void dbCompaction(Metapb.Partition partition, DbCompaction dbCompaction) throws + PDException { + + } + + @Override + public void movePartition(Metapb.Partition partition, + MovePartition movePartition) throws PDException { + + } + + @Override + public void cleanPartition(Metapb.Partition partition, + CleanPartition cleanPartition) throws PDException { + + } + + @Override + public void changePartitionKeyRange(Metapb.Partition partition, + PartitionKeyRange partitionKeyRange) throws + PDException { + + } + }); + int count = 6; + Metapb.Store[] stores = new Metapb.Store[count]; + for (int i = 0; i < count; i++) { + Metapb.Store store = Metapb.Store.newBuilder() + .setId(0) + .setAddress("127.0.0.1:850" + i) + .setDeployPath("/data") + .addLabels(Metapb.StoreLabel.newBuilder() + .setKey("namespace") + .setValue("default") + .build()) + .build(); + stores[i] = storeService.register(store); + System.out.println("新注册store， id = " + Long.toHexString(stores[i].getId())); + } + Assert.assertEquals(count, storeService.getStores().size()); + + Metapb.Graph graph = Metapb.Graph.newBuilder() + .setGraphName("defaultGH") + .build(); + Metapb.PartitionShard ptShard = + partitionService.getPartitionByCode(graph.getGraphName(), 0); + System.out.println(ptShard.getPartition().getId()); + { + Metapb.Partition pt = ptShard.getPartition(); + System.out.println(pt.getId() + " " + pt.getStartKey() + "," + pt.getEndKey()); + } + + Assert.assertEquals(6, storeService.getShardGroups().size()); + // storeService.splitShardGroups(ptShard.getPartition().getId(), 4); + Assert.assertEquals(9, storeService.getShardGroups().size()); + storeService.getShardGroups().forEach(shardGroup -> { + System.out.println("shardGroup id = " + shardGroup.getId()); + }); + } + + // @Test + public void testPartitionService() throws PDException, ExecutionException, + InterruptedException { + StoreNodeService storeService = new StoreNodeService(pdConfig); + int count = 6; + Metapb.Store[] stores = new Metapb.Store[count]; + for (int i = 0; i < count; i++) { + Metapb.Store store = Metapb.Store.newBuilder() + .setId(0) + .setAddress(String.valueOf(i)) + .setDeployPath("/data") + .addLabels(Metapb.StoreLabel.newBuilder() + .setKey("namespace") + .setValue("default") + .build()) + .build(); + stores[i] = storeService.register(store); + System.out.println("新注册store， id = " + Long.toHexString(stores[i].getId())); + } + Assert.assertEquals(count, storeService.getStores("").size()); + + + PartitionService partitionService = new PartitionService(pdConfig, storeService); + + Metapb.Graph graph = Metapb.Graph.newBuilder() + .setGraphName("defaultGH") + + .setPartitionCount(10) + .build(); + // 申请分区 + Metapb.PartitionShard[] partitions = new Metapb.PartitionShard[10]; + for (int i = 0; i < partitions.length; i++) { + partitions[i] = + partitionService.getPartitionShard(graph.getGraphName(), intToByteArray(i)); + Assert.assertEquals(3, storeService.getShardGroup(i).getShardsCount()); + } + System.out.println( + "分区数量： " + partitionService.getPartitions(graph.getGraphName()).size()); + + int[] caseNo = {0}; //1 测试增加shard, 2 //测试store下线 + + Metapb.Shard leader = null; + int[] finalCaseNo = caseNo; + + partitionService.addInstructionListener(new PartitionInstructionListener() { + + @Override + public void changeShard(Metapb.Partition partition, ChangeShard changeShard) throws + PDException { + switch (finalCaseNo[0]) { + case 2: + Assert.assertEquals(5, storeService.getShardGroup(partition.getId()) + .getShardsCount()); + break; + case 3: + storeService.getShardGroup(partition.getId()).getShardsList() + .forEach(shard -> { + Assert.assertNotEquals(shard.getStoreId(), + stores[0].getId()); + }); + break; + } + + } + + @Override + public void transferLeader(Metapb.Partition partition, TransferLeader transferLeader) { + + } + + @Override + public void splitPartition(Metapb.Partition partition, SplitPartition splitPartition) { + } + + @Override + public void dbCompaction(Metapb.Partition partition, DbCompaction dbCompaction) throws + PDException { + + } + + @Override + public void movePartition(Metapb.Partition partition, + MovePartition movePartition) throws PDException { + + } + + @Override + public void cleanPartition(Metapb.Partition partition, + CleanPartition cleanPartition) throws PDException { + + } + + @Override + public void changePartitionKeyRange(Metapb.Partition partition, + PartitionKeyRange partitionKeyRange) + throws PDException { + + } + }); + Metapb.Partition partition = partitions[0].getPartition(); + leader = Metapb.Shard.newBuilder( + storeService.getShardGroup(partition.getId()).getShardsList().get(0)).build(); + Metapb.Shard finalLeader = leader; + partitionService.addStatusListener(new PartitionStatusListener() { + @Override + public void onPartitionChanged(Metapb.Partition partition, + Metapb.Partition newPartition) { + + } + + @Override + public void onPartitionRemoved(Metapb.Partition partition) { + + } + }); + // 测试修改图 + caseNo[0] = 1; + partitionService.updateGraph(graph); + for (int i = 0; i < partitions.length; i++) { + partitions[i] = + partitionService.getPartitionShard(graph.getGraphName(), intToByteArray(i)); + Assert.assertEquals(3, storeService.getShardGroup(i).getShardsCount()); + } + + graph = Metapb.Graph.newBuilder(graph) + .setGraphName("defaultGH") + + .setPartitionCount(10) + .build(); + caseNo[0] = 2; + partitionService.updateGraph(graph); + + // 测试store离线 + caseNo[0] = 3; + partitionService.storeOffline(stores[0]); + + + Metapb.PartitionStats stats = Metapb.PartitionStats.newBuilder() + .addGraphName(partition.getGraphName()) + .setId(partition.getId()) + .setLeader( + Metapb.Shard.newBuilder(leader) + .setRole( + Metapb.ShardRole.Leader)) + .build(); + // 测试leader飘移 + caseNo[0] = 4; + partitionService.partitionHeartbeat(stats); + AtomicReference shard = new AtomicReference<>(); + Metapb.PartitionShard ss = + partitionService.getPartitionShardById(partition.getGraphName(), partition.getId()); + storeService.getShardList(partition.getId()).forEach(s -> { + if (s.getRole() == Metapb.ShardRole.Leader) { + Assert.assertNull(shard.get()); + shard.set(s); + } + }); + + Assert.assertEquals(leader.getStoreId(), shard.get().getStoreId()); + + } + + // @Test + public void testMergeGraphParams() throws PDException { + StoreNodeService storeService = new StoreNodeService(pdConfig); + PartitionService partitionService = new PartitionService(pdConfig, storeService); + + Metapb.Graph dfGraph = Metapb.Graph.newBuilder() + + .setPartitionCount( + pdConfig.getPartition().getTotalCount()) + + .build(); + + Metapb.Graph graph1 = Metapb.Graph.newBuilder() + .setGraphName("test") + .setPartitionCount(20) + + .build(); + + Metapb.Graph graph2 = Metapb.Graph.newBuilder() + .setGraphName("test") + .setPartitionCount(7).build(); + Metapb.Graph graph3 = Metapb.Graph.newBuilder() + .setGraphName("test") + .build(); + Metapb.Graph graph4 = Metapb.Graph.newBuilder() + .setGraphName("test") + .build(); + + Metapb.Graph graph = Metapb.Graph.newBuilder(dfGraph).mergeFrom(graph2).build(); + Assert.assertEquals(graph2.getGraphName(), graph.getGraphName()); + + Assert.assertEquals(graph2.getPartitionCount(), graph.getPartitionCount()); + + + graph = Metapb.Graph.newBuilder(dfGraph).mergeFrom(graph3).build(); + Assert.assertEquals(graph3.getGraphName(), graph.getGraphName()); + + Assert.assertEquals(dfGraph.getPartitionCount(), graph.getPartitionCount()); + + + graph = Metapb.Graph.newBuilder(dfGraph).mergeFrom(graph4).build(); + Assert.assertEquals(graph4.getGraphName(), graph.getGraphName()); + + Assert.assertEquals(dfGraph.getPartitionCount(), graph.getPartitionCount()); + + } + + // @Test + public void test() { + int[] n = new int[3]; + + + if (++n[2] > 1) { + System.out.println(n[2]); + } + if (++n[2] > 1) { + System.out.println(n[2]); + } + if (++n[2] > 1) { + System.out.println(n[2]); + } + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/StoreRegisterTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/StoreRegisterTest.java new file mode 100644 index 0000000000..48877273fc --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/StoreRegisterTest.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.apache.hugegraph.pd.client.PDClient; +import org.apache.hugegraph.pd.client.PDConfig; +import org.apache.hugegraph.pd.client.PDPulse; +import org.apache.hugegraph.pd.client.PDPulseImpl; +import org.apache.hugegraph.pd.common.KVPair; +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.pd.grpc.pulse.PartitionHeartbeatRequest; +import org.apache.hugegraph.pd.grpc.pulse.PulseResponse; +import org.apache.hugegraph.pd.pulse.PulseServerNotice; +import org.junit.Assert; +import org.junit.BeforeClass; + +public class StoreRegisterTest { + private static PDClient pdClient; + private final String storeAddr = "localhost"; + private final String graphName = "default/hugegraph/g"; + private long storeId = 0; + + @BeforeClass + public static void beforeClass() throws Exception { + PDConfig config = PDConfig.of("localhost:8686"); + config.setEnableCache(true); + pdClient = PDClient.create(config); + } + + // @Test + public void testRegisterStore() throws PDException { + Metapb.Store store = Metapb.Store.newBuilder().setAddress(storeAddr).build(); + try { + storeId = pdClient.registerStore(store); + } catch (Exception e) { + e.printStackTrace(); + } + Assert.assertTrue("RegisterStore store_id = " + storeId, storeId != 0); + } + + // @Test + public void testGetStore() throws PDException { + testRegisterStore(); + Metapb.Store store = pdClient.getStore(storeId); + Assert.assertEquals(storeAddr, store.getAddress()); + System.out.println(store); + } + + // @Test + public void testGetActiveStores() throws PDException { + testRegisterStore(); + List stores = pdClient.getActiveStores(graphName); + stores.forEach((e) -> { + System.out.println("-------------------------------------"); + System.out.println(e); + }); + } + + + // @Test + public void testStoreHeartbeat() throws PDException { + testRegisterStore(); + Metapb.StoreStats stats = Metapb.StoreStats.newBuilder().setStoreId(storeId).build(); + pdClient.storeHeartbeat(stats); + List stores = pdClient.getActiveStores(graphName); + boolean exist = false; + for (Metapb.Store store : stores) { + if (store.getId() == storeId) { + exist = true; + break; + } + } + Assert.assertTrue(exist); + } + + + // @Test + public void testPartitionHeartbeat() throws InterruptedException, PDException { + testRegisterStore(); + PDPulse pdPulse = new PDPulseImpl(pdClient.getLeaderIp()); + + PDPulse.Notifier notifier = pdPulse.connectPartition( + new PDPulse.Listener() { + + @Override + public void onNext(PulseResponse response) { + + } + + @Override + public void onNotice(PulseServerNotice notice) { + + } + + @Override + public void onError(Throwable throwable) { + + } + + @Override + public void onCompleted() { + + } + }); + KVPair partShard = + pdClient.getPartition("test", "1".getBytes(StandardCharsets.UTF_8)); + notifier.notifyServer(PartitionHeartbeatRequest.newBuilder().setStates( + Metapb.PartitionStats.newBuilder().addGraphName("test") + .setId(partShard.getKey().getId()) + .setLeader(Metapb.Shard.newBuilder().setStoreId(1).build()))); + + + Thread.sleep(10000); + } + +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/UnitTestBase.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/UnitTestBase.java new file mode 100644 index 0000000000..35ada84167 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/UnitTestBase.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd; + +import java.io.File; + +public class UnitTestBase { + public static boolean deleteDir(File dir) { + if (dir.isDirectory()) { + for (File file : dir.listFiles()) { + deleteDir(file); + } + } + return dir.delete(); + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/BaseClientTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/BaseClientTest.java new file mode 100644 index 0000000000..3fca39ade9 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/BaseClientTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.runner.RunWith; +import org.mockito.runners.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class BaseClientTest { + public static PDClient pdClient; + public final String storeAddr = "localhost"; + public final String graphName = "default/hugegraph/g"; + public long storeId = 0; + + @BeforeClass + public static void beforeClass() throws Exception { + PDConfig config = PDConfig.of("localhost:8686"); +// PDConfig config = PDConfig.of("10.81.116.77:8986"); + config.setEnableCache(true); + pdClient = PDClient.create(config); + } + + @After + public void teardown() throws Exception { + // pass + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImplTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImplTest.java new file mode 100644 index 0000000000..6d42c5ea73 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImplTest.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import java.util.HashMap; +import java.util.Map; +import java.util.Vector; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hugegraph.pd.grpc.discovery.NodeInfos; +import org.apache.hugegraph.pd.grpc.discovery.Query; +import org.junit.Assert; + +public class DiscoveryClientImplTest { + + private static final AtomicLong label = new AtomicLong(); + String address = "localhost:80"; + int delay = 1000; + int wait = delay * 3 + 500; + + // @Test + public void registerStore() throws InterruptedException { + + HashMap labels = new HashMap<>(); + + labels.put("metrics", "/actuator/prometheus"); + labels.put("target", "10.81.116.77:8520"); + labels.put("scheme", "http"); + labels.put("__relabeling", "http"); + labels.put("no_relabeling", "http"); + getClient("store", "address1", labels); + + labels.put("metrics", "/actuator/prometheus"); + labels.put("target", "10.81.116.78:8520"); + labels.put("scheme", "http"); + getClient("store", "address2", labels); + + labels.put("metrics", "/actuator/prometheus"); + labels.put("target", "10.81.116.79:8520"); + labels.put("scheme", "http"); + getClient("store", "address3", labels); + + labels.put("metrics", "/actuator/prometheus"); + labels.put("target", "10.81.116.78:8620"); + labels.put("scheme", "http"); + getClient("pd", "address1", labels); + + labels.put("metrics", "/graph/metrics"); + labels.put("target", "10.37.1.1:9200"); + labels.put("scheme", "https"); + getClient("hugegraph", "address1", labels); + } + + // @Test + public void testNodes() throws InterruptedException { + String appName = "hugegraph"; + register(appName, address); + } + + // @Test + public void testMultiNode() throws InterruptedException { + for (int i = 0; i < 2; i++) { + register("app" + i, address + i); + } + } + + // @Test + public void testParallelMultiNode() throws InterruptedException { + CountDownLatch latch = new CountDownLatch(30); + Vector exceptions = new Vector<>(); + for (int i = 0; i < 30; i++) { + int finalI = i; + new Thread(() -> { + try { + for (int j = 0; j < 3; j++) { + register("app" + finalI, address + j); + } + } catch (Exception e) { + exceptions.add(e); + } finally { + latch.countDown(); + } + }).start(); + } + latch.await(); + Assert.assertEquals(0, exceptions.size()); + } + + private void register(String appName, String address) throws InterruptedException { + + HashMap labels = new HashMap<>(); + String labelValue = String.valueOf(label.incrementAndGet()); + labels.put("address", labelValue); + labels.put("address1", labelValue); + Query query = Query.newBuilder().setAppName( + appName).setVersion("0.13.0").putAllLabels(labels).build(); + DiscoveryClientImpl discoveryClient = getClient(appName, address, labels); + Thread.sleep(10000); + NodeInfos nodeInfos1 = discoveryClient.getNodeInfos(query); + Assert.assertEquals(1, nodeInfos1.getInfoCount()); + DiscoveryClientImpl discoveryClient1 = getClient(appName, address + 0, labels); + Thread.sleep(10000); + Assert.assertEquals(2, discoveryClient.getNodeInfos(query).getInfoCount()); + Query query1 = Query.newBuilder().setAppName( + appName).setVersion("0.12.0").putAllLabels(labels).build(); + Assert.assertEquals(0, discoveryClient.getNodeInfos(query1).getInfoCount()); + discoveryClient.cancelTask(); + discoveryClient1.cancelTask(); + Thread.sleep(wait); + NodeInfos nodeInfos = discoveryClient.getNodeInfos(query); + System.out.println(nodeInfos); + Assert.assertEquals(0, nodeInfos.getInfoCount()); + discoveryClient.close(); + discoveryClient1.close(); + } + + private DiscoveryClientImpl getClient(String appName, String address, Map labels) { + DiscoveryClientImpl discoveryClient = null; + try { + discoveryClient = DiscoveryClientImpl.newBuilder().setCenterAddress( + "localhost:8687,localhost:8686,localhost:8688").setAddress(address).setAppName( + appName).setDelay(delay).setVersion("0.13.0").setId( + "0").setLabels(labels).build(); + discoveryClient.scheduleTask(); + } catch (Exception e) { + e.printStackTrace(); + } + + return discoveryClient; + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientTest.java new file mode 100644 index 0000000000..928f1dcba5 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +import org.apache.hugegraph.pd.client.DiscoveryClientImpl; +import org.apache.hugegraph.pd.grpc.discovery.NodeInfo; +import org.apache.hugegraph.pd.grpc.discovery.Query; +import org.junit.Before; +import org.junit.Test; + +public class DiscoveryClientTest { + + private DiscoveryClientImpl client; + + @Before + public void setUp() { + this.client = getClient("appName", "localhost:8654", new HashMap()); + } + + @Test + public void testGetRegisterNode() { + // Setup + try { + Consumer result = this.client.getRegisterConsumer(); + final NodeInfo expectedResult = NodeInfo.newBuilder() + .setAppName("appName") + .build(); + + Thread.sleep(3000); + Query query = Query.newBuilder().setAppName("appName") + .setVersion("0.13.0").build(); + + // Run the test + this.client.getNodeInfos(query); + } catch (InterruptedException e) { + e.printStackTrace(); + } finally { + this.client.close(); + } + + } + + private DiscoveryClientImpl getClient(String appName, String address, + Map labels) { + DiscoveryClientImpl discoveryClient = null; + try { + discoveryClient = DiscoveryClientImpl.newBuilder().setCenterAddress( + "localhost:8686").setAddress(address).setAppName(appName) + .setDelay(2000) + .setVersion("0.13.0") + .setId("0").setLabels(labels) + .build(); + discoveryClient.scheduleTask(); + } catch (Exception e) { + e.printStackTrace(); + } + + return discoveryClient; + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/KvClientTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/KvClientTest.java new file mode 100644 index 0000000000..c61413b8cd --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/KvClientTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +import java.util.function.Consumer; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hugegraph.pd.client.KvClient; +import org.apache.hugegraph.pd.client.PDConfig; +import org.apache.hugegraph.pd.grpc.kv.KResponse; +import org.apache.hugegraph.pd.grpc.kv.ScanPrefixResponse; +import org.apache.hugegraph.pd.grpc.kv.WatchEvent; +import org.apache.hugegraph.pd.grpc.kv.WatchKv; +import org.apache.hugegraph.pd.grpc.kv.WatchResponse; +import org.apache.hugegraph.pd.grpc.kv.WatchState; +import org.apache.hugegraph.pd.grpc.kv.WatchType; +import org.junit.Before; +import org.junit.Test; + +import io.grpc.stub.AbstractBlockingStub; +import io.grpc.stub.AbstractStub; + +public class KvClientTest extends BaseClientTest { + + String key = "key"; + String value = "value"; + private KvClient client; + + @Before + public void setUp() { + this.client = new KvClient<>(PDConfig.of("localhost:8686")); + } + + @Test + public void testCreateStub() { + // Setup + // Run the test + try { + final AbstractStub result = this.client.createStub(); + } catch (Exception e) { + + } + + + // Verify the results + } + + @Test + public void testCreateBlockingStub() { + // Setup + // Run the test + try { + final AbstractBlockingStub result = this.client.createBlockingStub(); + } catch (Exception e) { + + } + } + + @Test + public void testPutAndGet() throws Exception { + // Run the test + try { + this.client.put(this.key, this.value); + // Run the test + KResponse result = this.client.get(this.key); + + // Verify the results + assertThat(result.getValue()).isEqualTo(this.value); + this.client.delete(this.key); + result = this.client.get(this.key); + assertThat(StringUtils.isEmpty(result.getValue())); + this.client.deletePrefix(this.key); + this.client.put(this.key + "1", this.value); + this.client.put(this.key + "2", this.value); + ScanPrefixResponse response = this.client.scanPrefix(this.key); + assertThat(response.getKvsMap().size() == 2); + this.client.putTTL(this.key + "3", this.value, 1000); + this.client.keepTTLAlive(this.key + "3"); + final Consumer mockConsumer = mock(Consumer.class); + + // Run the test + this.client.listen(this.key + "3", mockConsumer); + this.client.listenPrefix(this.key + "4", mockConsumer); + WatchResponse r = WatchResponse.newBuilder().addEvents( + WatchEvent.newBuilder().setCurrent( + WatchKv.newBuilder().setKey(this.key).setValue("value") + .build()).setType(WatchType.Put).build()) + .setClientId(0L) + .setState(WatchState.Starting) + .build(); + this.client.getWatchList(r); + this.client.getWatchMap(r); + this.client.lock(this.key, 3000L); + this.client.isLocked(this.key); + this.client.unlock(this.key); + this.client.lock(this.key, 3000L); + this.client.keepAlive(this.key); + this.client.close(); + } catch (Exception e) { + + } + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/LicenseClientImplTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/LicenseClientImplTest.java new file mode 100644 index 0000000000..4ed11b9b21 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/LicenseClientImplTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import java.io.File; +import java.util.Iterator; +import java.util.Map; +import java.util.Properties; + +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.pd.grpc.Pdpb; +import org.apache.hugegraph.pd.grpc.kv.KResponse; +import org.apache.hugegraph.pd.grpc.kv.KvResponse; +import org.yaml.snakeyaml.Yaml; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class LicenseClientImplTest { + + // @Test + public void putLicense() { + PDConfig pdConfig = PDConfig.of("localhost:8686,localhost:8687,localhost:8688"); + //PDConfig pdConfig = PDConfig.of("localhost:8686"); + pdConfig.setEnableCache(true); + try (LicenseClient c = new LicenseClient(pdConfig)) { + File file = new File("../conf/hugegraph.license"); + byte[] bytes = FileUtils.readFileToByteArray(file); + Pdpb.PutLicenseResponse putLicenseResponse = c.putLicense(bytes); + Pdpb.Error error = putLicenseResponse.getHeader().getError(); + log.info(error.getMessage()); + assert error.getType().equals(Pdpb.ErrorType.OK); + } catch (Exception e) { + log.error("put license with error: ", e); + } + } + + // @Test + public void getKv() { + PDConfig pdConfig = PDConfig.of("10.157.12.36:8686"); + pdConfig.setEnableCache(true); + try (KvClient c = new KvClient(pdConfig)) { + KResponse kResponse = c.get("S:FS"); + Pdpb.Error error = kResponse.getHeader().getError(); + log.info(error.getMessage()); + assert error.getType().equals(Pdpb.ErrorType.OK); + Properties ymlConfig = getYmlConfig(kResponse.getValue()); + Object property = ymlConfig.get("rocksdb.write_buffer_size"); + assert property.toString().equals("32000000"); + } catch (Exception e) { + log.error("put license with error: ", e); + } + } + + // @Test + public void putKv() { + PDConfig pdConfig = PDConfig.of("127.0.0.1.70:8688"); + pdConfig.setEnableCache(true); + try (KvClient c = new KvClient(pdConfig)) { + long l = System.currentTimeMillis(); + KvResponse kvResponse = c.put("S:Timestamp", String.valueOf(l)); + Pdpb.Error error = kvResponse.getHeader().getError(); + log.info(error.getMessage()); + assert error.getType().equals(Pdpb.ErrorType.OK); + } catch (Exception e) { + log.error("put license with error: ", e); + } + } + + // @Test + public void putKvLocal() { + PDConfig pdConfig = PDConfig.of("localhost:8686"); + pdConfig.setEnableCache(true); + try (KvClient c = new KvClient(pdConfig)) { + long l = System.currentTimeMillis(); + KvResponse kvResponse = c.put("S:Timestamp", String.valueOf(l)); + Pdpb.Error error = kvResponse.getHeader().getError(); + log.info(error.getMessage()); + assert error.getType().equals(Pdpb.ErrorType.OK); + } catch (Exception e) { + log.error("put license with error: ", e); + } + } + + private Properties getYmlConfig(String yml) { + Yaml yaml = new Yaml(); + Iterable load = yaml.loadAll(yml); + Iterator iterator = load.iterator(); + Properties properties = new Properties(); + while (iterator.hasNext()) { + Map next = (Map) iterator.next(); + map2Properties(next, "", properties); + } + return properties; + } + + private void map2Properties(Map map, String prefix, Properties properties) { + + for (Map.Entry entry : map.entrySet()) { + String key = entry.getKey(); + String newPrefix = prefix == null || prefix.length() == 0 ? key : prefix + "." + key; + Object value = entry.getValue(); + if (!(value instanceof Map)) { + properties.put(newPrefix, value); + } else { + map2Properties((Map) value, newPrefix, properties); + } + + } + } + +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientSuiteTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientSuiteTest.java new file mode 100644 index 0000000000..a9d3ae1406 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientSuiteTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +import lombok.extern.slf4j.Slf4j; + + +@RunWith(Suite.class) +@Suite.SuiteClasses({ + PDClientTest.class, + KvClientTest.class, + DiscoveryClientTest.class +}) + +@Slf4j +public class PDClientSuiteTest { + +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientTest.java new file mode 100644 index 0000000000..5d068c02f8 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientTest.java @@ -0,0 +1,418 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.grpc.MetaTask; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.pd.grpc.Pdpb; +import org.junit.Test; +import org.mockito.Mockito; + +public class PDClientTest extends BaseClientTest { + @Test + public void testDbCompaction() { + System.out.println("testDbCompaction start"); + + try { + pdClient.dbCompaction(""); + pdClient.dbCompaction(); + } catch (PDException e) { + e.printStackTrace(); + } + + System.out.println("pdclienttest testDbCompaction end"); + } + + @Test + public void testRegisterStore() { + Metapb.Store store = Metapb.Store.newBuilder().build(); + try { + pdClient.registerStore(store); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testSetGraph() { + Metapb.Graph graph = Metapb.Graph.newBuilder().setGraphName("test").build(); + try { + pdClient.setGraph(graph); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testGetGraph() { + try { + pdClient.getGraph("test"); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testGetStore() { + try { + pdClient.getStore(0L); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testUpdateStore() { + Metapb.Store store = Metapb.Store.newBuilder().build(); + try { + pdClient.updateStore(store); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testGetActiveStores() { + try { + pdClient.getActiveStores("test"); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testGetAllStores() { + try { + pdClient.getAllStores("test"); + } catch (PDException e) { + e.printStackTrace(); + } + } + +// @Test +// public void testStoreHeartbeat(){ +// Metapb.StoreStats stats = Metapb.StoreStats.newBuilder().build(); +// try { +// pdClient.storeHeartbeat(stats); +// } catch (PDException e) { +// e.printStackTrace(); +// } +// } + + @Test + public void testKeyToCode() { + pdClient.keyToCode("test", "test".getBytes(StandardCharsets.UTF_8)); + } + + @Test + public void testScanPartitions() { + try { + pdClient.scanPartitions("test", "1".getBytes(StandardCharsets.UTF_8), + "9".getBytes(StandardCharsets.UTF_8)); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testGetPartitionsByStore() { + try { + pdClient.getPartitionsByStore(0L); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testQueryPartitions() { + try { + pdClient.queryPartitions(0L, 0); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testGetPartitions() { + try { + pdClient.getPartitions(0L, "test"); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testUpdatePartitionLeader() { + System.out.println("updatePartitionLeader start"); + + pdClient.updatePartitionLeader("aaa", 0, 0L); + } + + @Test + public void testInvalidPartitionCache() { + pdClient.invalidPartitionCache(); + } + + @Test + public void testInvalidStoreCache() { + pdClient.invalidStoreCache(0L); + } + + @Test + public void testUpdatePartitionCache() { + Metapb.Partition partition = Metapb.Partition.newBuilder().build(); + Metapb.Shard leader = Metapb.Shard.newBuilder().build(); + pdClient.updatePartitionCache(partition, leader); + } + + @Test + public void testGetIdByKey() { + try { + pdClient.getIdByKey("test", 1); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testResetIdByKey() { + try { + pdClient.resetIdByKey("test"); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testGetGetLeader() { + try { + pdClient.getLeader(); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testGetMembers() { + try { + pdClient.getMembers(); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testGetClusterStats() { + try { + pdClient.getClusterStats(); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testAddEventListener() { + PDClient.PDEventListener listener = Mockito.mock(PDClient.PDEventListener.class); + pdClient.addEventListener(listener); + } + + @Test + public void testGetWatchClient() { + pdClient.getWatchClient(); + } + + @Test + public void testGetPulseClient() { + // pdClient.getPulseClient(); + } + + @Test + public void testGetStoreStatus() { + try { + pdClient.getStoreStatus(true); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testGetPartition() { + try { + pdClient.getPartition("test", "test".getBytes(StandardCharsets.UTF_8)); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testSetGraphSpace() { + try { + pdClient.setGraphSpace("test", 1L); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testGetGraphSpace() { + try { + pdClient.getGraphSpace("test"); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testSetPDConfig() { + try { + pdClient.setPDConfig(0, "", 0, 0L); + } catch (PDException e) { + e.printStackTrace(); + } + Metapb.PDConfig pdConfig = Metapb.PDConfig.newBuilder().build(); + + try { + pdClient.setPDConfig(pdConfig); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testGetPDConfig() { + try { + pdClient.getPDConfig(0L); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testChangePeerList() { + try { + pdClient.changePeerList(""); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testSplitData() { + try { + Metapb.PDConfig config = pdClient.getPDConfig(); + pdClient.setPDConfig(config.toBuilder().setMaxShardsPerStore(12).build()); + System.out.println(pdClient.getPDConfig()); + + // 开始分区分裂 + pdClient.splitData(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Test + public void testBalancePartition() { + try { + pdClient.balancePartition(); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testMovePartition() { + Pdpb.OperationMode mode = Pdpb.OperationMode.Auto; + List params = new ArrayList<>(1); + try { + pdClient.movePartition(mode, params); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testReportTask() { + MetaTask.Task task = MetaTask.Task.newBuilder().build(); + try { + pdClient.reportTask(task); + } catch (PDException e) { + e.printStackTrace(); + } + } + + + @Test + public void testBalanceLeaders() { + try { + pdClient.balanceLeaders(); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testDelStore() { + try { + pdClient.delStore(0L); + } catch (PDException e) { + e.printStackTrace(); + } + } + +// @Test +// public void testgetQuota() { +// try { +// pdClient.getQuota(); +// } catch (PDException e) { +// e.printStackTrace(); +// } +// } + + @Test + public void testUpdatePartition() { + List partitions = new ArrayList<>(1); + try { + pdClient.updatePartition(partitions); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testDelPartition() { + try { + pdClient.delPartition("test", 0); + } catch (PDException e) { + e.printStackTrace(); + } + } + + @Test + public void testdelGraph() { + try { + pdClient.delGraph("test"); + } catch (PDException e) { + e.printStackTrace(); + } + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDPulseTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDPulseTest.java new file mode 100644 index 0000000000..d27c4b5e11 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDPulseTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.apache.hugegraph.pd.client.test.HgPDTestUtil; +import org.apache.hugegraph.pd.grpc.pulse.PartitionHeartbeatRequest; +import org.apache.hugegraph.pd.pulse.PulseServerNotice; +import org.junit.BeforeClass; +import org.junit.Test; + +public class PDPulseTest { + private static PDClient pdClient; + + private final long storeId = 0; + private final String storeAddress = "localhost"; + private final String graphName = "graph1"; + + @BeforeClass + public static void beforeClass() throws Exception { + PDConfig pdConfig = PDConfig.of("localhost:8686"); + pdConfig.setEnableCache(true); + pdClient = PDClient.create(pdConfig); + pdClient.getLeader(); + } + + @Test + public void listen() { + + PDPulse pulse = new PDPulseImpl(pdClient.getLeaderIp()); + CountDownLatch latch = new CountDownLatch(60); + + PDPulse.Notifier notifier1 = + pulse.connectPartition(new PulseListener(latch, "listener1")); + PDPulse.Notifier notifier2 = + pulse.connectPartition(new PulseListener(latch, "listener2")); + PDPulse.Notifier notifier3 = + pulse.connectPartition(new PulseListener(latch, "listener3")); + + try { + latch.await(120, TimeUnit.SECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); + } + PartitionHeartbeatRequest.Builder builder = PartitionHeartbeatRequest.newBuilder(); + + notifier1.notifyServer(builder); + + + notifier2.notifyServer(builder); + + notifier3.notifyServer(builder); + + notifier1.close(); + notifier2.close(); + notifier3.close(); + } + + + private class PulseListener implements PDPulse.Listener { + private final String listenerName; + CountDownLatch latch = new CountDownLatch(10); + + private PulseListener(CountDownLatch latch, String listenerName) { + this.latch = latch; + this.listenerName = listenerName; + } + + @Override + public void onNext(T response) { + // println(this.listenerName+" res: "+response); + // this.latch.countDown(); + } + + @Override + public void onNotice(PulseServerNotice notice) { + HgPDTestUtil.println(this.listenerName + " ---> res: " + notice.getContent()); + + notice.ack(); + this.latch.countDown(); + } + + @Override + public void onError(Throwable throwable) { + HgPDTestUtil.println(this.listenerName + " error: " + throwable.toString()); + } + + @Override + public void onCompleted() { + HgPDTestUtil.println(this.listenerName + " is completed"); + } + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDWatchTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDWatchTest.java new file mode 100644 index 0000000000..180b725555 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDWatchTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.apache.hugegraph.pd.client.test.HgPDTestUtil; +import org.junit.BeforeClass; +import org.junit.Test; + +@Deprecated +public class PDWatchTest { + private static PDClient pdClient; + + private final long storeId = 0; + private final String storeAddr = "localhost"; + private final String graphName = "graph1"; + + @BeforeClass + public static void beforeClass() { + pdClient = PDClient.create(PDConfig.of("localhost:9000")); + } + + @Test + public void watch() { + PDWatch watch = pdClient.getWatchClient(); + CountDownLatch latch = new CountDownLatch(10); + + PDWatch.Watcher watcher1 = watch.watchPartition(new WatchListener<>(latch, "watcher1")); + PDWatch.Watcher watcher2 = watch.watchPartition(new WatchListener<>(latch, "watcher2")); + PDWatch.Watcher watcher3 = watch.watchPartition(new WatchListener<>(latch, "watcher3")); + + PDWatch.Watcher nodeWatcher1 = watch.watchNode(new WatchListener<>(latch, "nodeWatcher1")); + + try { + latch.await(15, TimeUnit.SECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); + } + watcher1.close(); + watcher2.close(); + watcher3.close(); + } + + private class WatchListener implements PDWatch.Listener { + private final String watcherName; + CountDownLatch latch; + + private WatchListener(CountDownLatch latch, String watcherName) { + this.latch = latch; + this.watcherName = watcherName; + } + + @Override + public void onNext(T response) { + HgPDTestUtil.println(this.watcherName + " res: " + response); + this.latch.countDown(); + } + + @Override + public void onError(Throwable throwable) { + HgPDTestUtil.println(this.watcherName + " error: " + throwable.toString()); + } + + @Override + public void onCompleted() { + HgPDTestUtil.println(this.watcherName + " is completed"); + } + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/test/HgPDTestUtil.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/test/HgPDTestUtil.java new file mode 100644 index 0000000000..2c581ea6ff --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/test/HgPDTestUtil.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.client.test; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Iterator; +import java.util.List; + +public class HgPDTestUtil { + + public static void println(Object str) { + System.out.println(str); + } + + public static String toStr(byte[] b) { + if (b == null) return ""; + if (b.length == 0) return ""; + return new String(b, StandardCharsets.UTF_8); + } + + public static byte[] toBytes(String str) { + if (str == null) return null; + return str.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] toBytes(long l) { + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.putLong(l); + return buffer.array(); + } + + private static byte[] toBytes(final int i) { + ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES); + buffer.putInt(i); + return buffer.array(); + } + + public static long toLong(byte[] bytes) { + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.put(bytes); + buffer.flip();//need flip + return buffer.getLong(); + } + + public static long toInt(byte[] bytes) { + ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES); + buffer.put(bytes); + buffer.flip();//need flip + return buffer.getInt(); + } + + public static String padLeftZeros(String str, int n) { + return String.format("%1$" + n + "s", str).replace(' ', '0'); + } + + public static String toSuffix(int num, int length) { + return "-" + padLeftZeros(String.valueOf(num), length); + } + + public static int amountOf(List list) { + if (list == null) { + return 0; + } + return list.size(); + } + + public static int amountOf(Iterator iterator) { + if (iterator == null) return 0; + int count = 0; + while (iterator.hasNext()) { + iterator.next(); + count++; + } + return count; + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/BaseCliToolsTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/BaseCliToolsTest.java new file mode 100644 index 0000000000..146cffb139 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/BaseCliToolsTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.clitools; + +import org.junit.After; +import org.junit.BeforeClass; + + +public class BaseCliToolsTest { + @BeforeClass + public static void init() { + + } + + @After + public void teardown() { + // pass + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/CliToolsSuiteTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/CliToolsSuiteTest.java new file mode 100644 index 0000000000..abdcbea08f --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/CliToolsSuiteTest.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.clitools; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +import lombok.extern.slf4j.Slf4j; + + +@RunWith(Suite.class) +@Suite.SuiteClasses({ + MainTest.class +}) + +@Slf4j +public class CliToolsSuiteTest { + + +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/MainTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/MainTest.java new file mode 100644 index 0000000000..526558ac95 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/MainTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.clitools; + +import java.util.Arrays; +import java.util.List; + +import org.apache.hugegraph.pd.clitools.Main; +import org.apache.hugegraph.pd.common.PDException; +import org.junit.Test; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class MainTest extends BaseCliToolsTest { + + + public static boolean test2sup(List arrays, int tail, int res) { + System.out.printf("%d %d%n", tail, res); + if (tail == 0) { + System.out.printf("a = %d %d%n", tail, res); + return false; + } else if (tail == 1) { + System.out.printf("b = %d %d%n", arrays.get(0), res); + return (arrays.get(0) == res); + } else if (tail == 2) { + System.out.printf("c = %d %d %d%n", arrays.get(0), arrays.get(1), res); + return (arrays.get(0) + arrays.get(1) == Math.abs(res)) || + (Math.abs(arrays.get(0) - arrays.get(1)) == Math.abs(res)); + } else { + return test2sup(arrays, tail - 1, res + arrays.get(tail - 1)) || + test2sup(arrays, tail - 1, res - arrays.get(tail - 1)); + } + } + + @Test + public void getConfig() throws PDException { + Main.main(new String[]{"127.0.0.1:8686", "config", "enableBatchLoad"}); + } + + // @Test + public void setBatchTrue() throws PDException { + Main.main(new String[]{"127.0.0.1:8686", "config", "enableBatchLoad= true "}); + } + + // @Test + public void setBatchFalse() throws PDException { + Main.main(new String[]{"127.0.0.1:8686", "config", "enableBatchLoad=false"}); + } + + @Test + public void getConfig2() throws PDException { + Main.main(new String[]{"127.0.0.1:8686", "config", "shardCount"}); + } + + // @Test + public void setShardCount1() throws PDException { + Main.main(new String[]{"127.0.0.1:8686", "config", "shardCount=1"}); + } + + // @Test + public void setShardCount3() throws PDException { + Main.main(new String[]{"127.0.0.1:8686", "config", "shardCount=3"}); + } + + @Test + public void test2() { + Integer[] a = new Integer[]{1, 0, 3, 2}; + List aa = Arrays.asList(a); + System.out.printf(test2sup(aa, aa.size(), 0) ? "TRUE" : "FALSE"); + } + + +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/BaseCommonTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/BaseCommonTest.java new file mode 100644 index 0000000000..591779c0d5 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/BaseCommonTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.common; + +import org.junit.After; +import org.junit.BeforeClass; + +public class BaseCommonTest { + @BeforeClass + public static void init() { + + } + + @After + public void teardown() { + // pass + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/CommonSuiteTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/CommonSuiteTest.java new file mode 100644 index 0000000000..6f676c6068 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/CommonSuiteTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.common; + +import org.apache.hugegraph.pd.service.IdServiceTest; +import org.apache.hugegraph.pd.service.KvServiceTest; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +import lombok.extern.slf4j.Slf4j; + + +@RunWith(Suite.class) +@Suite.SuiteClasses({ + PartitionUtilsTest.class, + PartitionCacheTest.class, + MetadataKeyHelperTest.class, + KvServiceTest.class, + HgAssertTest.class, + KVPairTest.class, + IdServiceTest.class +}) + +@Slf4j +public class CommonSuiteTest { + + +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/HgAssertTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/HgAssertTest.java new file mode 100644 index 0000000000..253a9c67a0 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/HgAssertTest.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.common; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.HashMap; + +import org.apache.hugegraph.pd.common.HgAssert; +import org.junit.Test; + +public class HgAssertTest { + + @Test(expected = IllegalArgumentException.class) + public void testIsTrue() { + HgAssert.isTrue(false, ""); + } + + @Test(expected = IllegalArgumentException.class) + public void testIsTrue2() { + HgAssert.isTrue(true, null); + } + + @Test(expected = IllegalArgumentException.class) + public void testIsFalse() { + HgAssert.isFalse(true, ""); + } + + @Test(expected = IllegalArgumentException.class) + public void testIsFalse2() { + HgAssert.isTrue(false, null); + } + + @Test(expected = IllegalArgumentException.class) + public void isArgumentValid() { + HgAssert.isArgumentValid(new byte[0], ""); + } + + @Test(expected = IllegalArgumentException.class) + public void isArgumentValidStr() { + HgAssert.isArgumentValid("", ""); + } + + @Test(expected = IllegalArgumentException.class) + public void testIsArgumentNotNull() { + HgAssert.isArgumentNotNull(null, ""); + } + + @Test(expected = IllegalArgumentException.class) + public void testIstValid() { + HgAssert.istValid(new byte[0], ""); + } + + @Test(expected = IllegalArgumentException.class) + public void testIstValidStr() { + HgAssert.isValid("", ""); + } + + @Test(expected = IllegalArgumentException.class) + public void testIsNotNull() { + HgAssert.isNotNull(null, ""); + } + + + @Test + public void testIsInvalid() { + assertFalse(HgAssert.isInvalid("abc", "test")); + assertTrue(HgAssert.isInvalid("", null)); + } + + @Test + public void testIsInvalidByte() { + assertTrue(HgAssert.isInvalid(new byte[0])); + assertFalse(HgAssert.isInvalid(new byte[1])); + } + + @Test + public void testIsInvalidMap() { + assertTrue(HgAssert.isInvalid(new HashMap())); + assertFalse(HgAssert.isInvalid(new HashMap() {{ + put(1, 1); + }})); + } + + @Test + public void testIsInvalidCollection() { + assertTrue(HgAssert.isInvalid(new ArrayList())); + assertFalse(HgAssert.isInvalid(new ArrayList() {{ + add(1); + }})); + } + + @Test + public void testIsContains() { + assertTrue(HgAssert.isContains(new Object[]{Integer.valueOf(1), Long.valueOf(2)}, + Long.valueOf(2))); + assertFalse(HgAssert.isContains(new Object[]{Integer.valueOf(1), Long.valueOf(2)}, + Long.valueOf(3))); + } + + @Test + public void testIsContainsT() { + assertTrue(HgAssert.isContains(new ArrayList<>() {{ + add(1); + }}, 1)); + assertFalse(HgAssert.isContains(new ArrayList<>() {{ + add(1); + }}, 2)); + } + + @Test + public void testIsNull() { + assertTrue(HgAssert.isNull(null)); + assertFalse(HgAssert.isNull("abc", "cdf")); + } + +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/KVPairTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/KVPairTest.java new file mode 100644 index 0000000000..b3e1c15adf --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/KVPairTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.common; + +import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class KVPairTest { + + KVPair pair; + + @Before + public void init() { + this.pair = new KVPair<>("key", 1); + } + + @Test + public void testGetKey() { + assertEquals(this.pair.getKey(), "key"); + } + + @Test + public void testSetKey() { + this.pair.setKey("key2"); + assertEquals(this.pair.getKey(), "key2"); + } + + @Test + public void testGetValue() { + assertEquals(1, this.pair.getValue()); + } + + @Test + public void testSetValue() { + this.pair.setValue(2); + assertEquals(2, this.pair.getValue()); + } + + @Test + public void testToString() { + + } + + @Test + public void testHashCode() { + + } + + @Test + public void testEquals() { + var pair2 = new KVPair<>("key", 1); + Assert.assertEquals(pair2, this.pair); + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/MetadataKeyHelperTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/MetadataKeyHelperTest.java new file mode 100644 index 0000000000..ea239ed93c --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/MetadataKeyHelperTest.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.common; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.pd.meta.MetadataKeyHelper; +import org.junit.Test; + +public class MetadataKeyHelperTest { + + @Test + public void testGetStoreInfoKey() { + assertThat(MetadataKeyHelper.getStoreInfoKey(0L)).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetActiveStoreKey() { + assertThat(MetadataKeyHelper.getActiveStoreKey(0L)).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetActiveStorePrefix() { + assertThat(MetadataKeyHelper.getActiveStorePrefix()).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetStorePrefix() { + assertThat(MetadataKeyHelper.getStorePrefix()).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetStoreStatusKey() { + assertThat(MetadataKeyHelper.getStoreStatusKey(0L)).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetShardGroupKey() { + assertThat(MetadataKeyHelper.getShardGroupKey(0L)).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetShardGroupPrefix() { + assertThat(MetadataKeyHelper.getShardGroupPrefix()).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetPartitionKey() { + assertThat(MetadataKeyHelper.getPartitionKey("graphName", 0)).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetPartitionPrefix() { + assertThat(MetadataKeyHelper.getPartitionPrefix("graphName")).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetShardKey() { + assertThat(MetadataKeyHelper.getShardKey(0L, 0)).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetShardPrefix() { + assertThat(MetadataKeyHelper.getShardPrefix(0L)).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetGraphKey() { + assertThat(MetadataKeyHelper.getGraphKey("graphName")).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetGraphPrefix() { + assertThat(MetadataKeyHelper.getGraphPrefix()).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetPartitionStatusKey() { + assertThat(MetadataKeyHelper.getPartitionStatusKey("graphName", + 0)).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetPartitionStatusPrefixKey() { + assertThat(MetadataKeyHelper.getPartitionStatusPrefixKey( + "graphName")).contains(MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetGraphSpaceKey() { + assertThat(MetadataKeyHelper.getGraphSpaceKey("graphSpace")).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetPdConfigKey() { + assertThat(MetadataKeyHelper.getPdConfigKey("configKey")).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetQueueItemPrefix() { + assertThat(MetadataKeyHelper.getQueueItemPrefix()).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetQueueItemKey() { + assertThat(MetadataKeyHelper.getQueueItemKey("itemId")).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetSpitTaskKey() { + assertThat(MetadataKeyHelper.getSplitTaskKey("graphName", 0)).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetSpitTaskPrefix() { + assertThat(MetadataKeyHelper.getSplitTaskPrefix("graph0")).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetLogKey() { + // Setup + final Metapb.LogRecord record = Metapb.LogRecord.newBuilder() + .setAction("value") + .setTimestamp(0L) + .build(); + + // Run the test + final byte[] result = MetadataKeyHelper.getLogKey(record); + + // Verify the results + assertThat(result).contains(MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetLogKeyPrefix() { + assertThat(MetadataKeyHelper.getLogKeyPrefix("action", 0L)).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetKVPrefix() { + assertThat(MetadataKeyHelper.getKVPrefix("prefix", "key")).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetKVTTLPrefix() { + assertThat(MetadataKeyHelper.getKVTTLPrefix("ttlPrefix", "prefix", + "key")).contains( + MetadataKeyHelper.getDelimiter()); + } + + @Test + public void testGetKVWatchKeyPrefix1() { + assertThat( + MetadataKeyHelper.getKVWatchKeyPrefix("key", "watchDelimiter", + 0L)).contains( + String.valueOf(MetadataKeyHelper.getDelimiter())); + } + + @Test + public void testGetKVWatchKeyPrefix2() { + assertThat(MetadataKeyHelper.getKVWatchKeyPrefix("key", + "watchDelimiter")).contains( + String.valueOf(MetadataKeyHelper.getDelimiter())); + } + + @Test + public void testGetDelimiter() { + assertThat(MetadataKeyHelper.getDelimiter()).isEqualTo('/'); + } + + @Test + public void testGetStringBuilderHelper() { + try { + MetadataKeyHelper.getStringBuilderHelper(); + } catch (Exception e) { + + } + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionCacheTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionCacheTest.java new file mode 100644 index 0000000000..d4126967f6 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionCacheTest.java @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.common; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hugegraph.pd.grpc.Metapb; +import org.junit.Before; +import org.junit.Test; + +public class PartitionCacheTest { + + private PartitionCache cache; + + private static Metapb.Partition createPartition(int pid, String graphName, long start, + long end) { + return Metapb.Partition.newBuilder() + .setId(pid) + .setGraphName(graphName) + .setStartKey(start) + .setEndKey(end) + .setState(Metapb.PartitionState.PState_Normal) + .setVersion(1) + .build(); + } + + private static Metapb.ShardGroup creteShardGroup(int pid) { + return Metapb.ShardGroup.newBuilder() + .addShards( + Metapb.Shard.newBuilder().setStoreId(0) + .setRole(Metapb.ShardRole.Leader).build() + ) + .setId(pid) + .setVersion(0) + .setConfVer(0) + .setState(Metapb.PartitionState.PState_Normal) + .build(); + } + + private static Metapb.Shard createShard() { + return Metapb.Shard.newBuilder() + .setStoreId(0) + .setRole(Metapb.ShardRole.Leader) + .build(); + } + + private static Metapb.Store createStore(long storeId) { + return Metapb.Store.newBuilder() + .setId(storeId) + .setAddress("127.0.0.1") + .setCores(4) + .setVersion("1") + .setDataPath("/tmp/junit") + .setDataVersion(1) + .setLastHeartbeat(System.currentTimeMillis()) + .setStartTimestamp(System.currentTimeMillis()) + .setState(Metapb.StoreState.Up) + .setDeployPath("/tmp/junit") + .build(); + } + + private static Metapb.Graph createGraph(String graphName, int partitionCount) { + return Metapb.Graph.newBuilder() + .setGraphName(graphName) + .setPartitionCount(partitionCount) + .setState(Metapb.PartitionState.PState_Normal) + .build(); + } + + private static Metapb.ShardGroup createShardGroup() { + List shards = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + shards.add(Metapb.Shard.newBuilder() + .setStoreId(i) + .setRole(i == 0 ? Metapb.ShardRole.Leader : + Metapb.ShardRole.Follower) + .build() + ); + } + + return Metapb.ShardGroup.newBuilder() + .setId(1) + .setVersion(1) + .setConfVer(1) + .setState(Metapb.PartitionState.PState_Normal) + .addAllShards(shards) + .build(); + } + + @Before + public void setup() { + this.cache = new PartitionCache(); + } + + @Test + public void testGetPartitionById() { + var partition = createPartition(0, "graph0", 0, 65535); + this.cache.updateShardGroup(creteShardGroup(0)); + this.cache.updatePartition(partition); + var ret = this.cache.getPartitionById("graph0", 0); + assertNotNull(ret); + assertEquals(ret.getKey(), partition); + } + + @Test + public void testGetPartitionByKey() throws UnsupportedEncodingException { + var partition = createPartition(0, "graph0", 0, 65535); + this.cache.updateShardGroup(creteShardGroup(0)); + this.cache.updatePartition(partition); + var ret = this.cache.getPartitionByKey("graph0", "0".getBytes(StandardCharsets.UTF_8)); + assertNotNull(ret); + assertEquals(ret.getKey(), partition); + } + + @Test + public void getPartitionByCode() { + var partition = createPartition(0, "graph0", 0, 1024); + this.cache.updateShardGroup(creteShardGroup(0)); + this.cache.updatePartition(partition); + var ret = this.cache.getPartitionByCode("graph0", 10); + assertNotNull(ret); + assertEquals(ret.getKey(), partition); + assertNull(this.cache.getPartitionByCode("graph0", 2000)); + } + + @Test + public void testGetPartitions() { + var partition1 = createPartition(0, "graph0", 0, 1024); + this.cache.updateShardGroup(creteShardGroup(0)); + this.cache.updatePartition(partition1); + assertEquals(this.cache.getPartitions("graph0").size(), 1); + var partition2 = createPartition(1, "graph0", 1024, 2048); + this.cache.updateShardGroup(creteShardGroup(1)); + this.cache.updatePartition(partition2); + assertEquals(this.cache.getPartitions("graph0").size(), 2); + System.out.print(this.cache.debugCacheByGraphName("graph0")); + } + + @Test + public void testAddPartition() { + var partition = createPartition(0, "graph0", 0, 65535); + this.cache.addPartition("graph0", 0, partition); + var ret = this.cache.getPartitionById("graph0", 0); + assertNotNull(ret); + assertEquals(ret.getKey(), partition); + assertNotNull(this.cache.getPartitionByCode("graph0", 2000)); + System.out.print(this.cache.debugCacheByGraphName("graph0")); + var partition2 = createPartition(0, "graph0", 0, 1024); + this.cache.addPartition("graph0", 0, partition2); + ret = this.cache.getPartitionById("graph0", 0); + assertNotNull(ret); + assertEquals(ret.getKey(), partition2); + assertNull(this.cache.getPartitionByCode("graph0", 2000)); + System.out.print(this.cache.debugCacheByGraphName("graph0")); + } + + @Test + public void testUpdatePartition() { + var partition = createPartition(0, "graph0", 0, 65535); + this.cache.updateShardGroup(creteShardGroup(0)); + this.cache.addPartition("graph0", 0, partition); + var partition2 = createPartition(0, "graph0", 0, 1024); + this.cache.updatePartition("graph0", 0, partition2); + var ret = this.cache.getPartitionById("graph0", 0); + assertNotNull(ret); + assertEquals(ret.getKey(), partition2); + assertNull(this.cache.getPartitionByCode("graph0", 2000)); + } + + @Test + public void testUpdatePartition2() { + var partition = createPartition(0, "graph0", 0, 1024); + this.cache.updateShardGroup(creteShardGroup(0)); + assertTrue(this.cache.updatePartition(partition)); + assertFalse(this.cache.updatePartition(partition)); + var ret = this.cache.getPartitionById("graph0", 0); + assertNotNull(ret); + assertEquals(ret.getKey(), partition); + assertNull(this.cache.getPartitionByCode("graph0", 2000)); + } + + @Test + public void testRemovePartition() { + var partition = createPartition(0, "graph0", 0, 1024); + this.cache.updateShardGroup(creteShardGroup(0)); + this.cache.updatePartition(partition); + assertNotNull(this.cache.getPartitionById("graph0", 0)); + this.cache.removePartition("graph0", 0); + assertNull(this.cache.getPartitionById("graph0", 0)); + System.out.print(this.cache.debugCacheByGraphName("graph0")); + } + + @Test + public void testRange() { + var partition1 = createPartition(1, "graph0", 0, 3); + var partition2 = createPartition(2, "graph0", 3, 6); + this.cache.updatePartition(partition1); + this.cache.updatePartition(partition2); + + var partition3 = createPartition(3, "graph0", 1, 2); + var partition4 = createPartition(4, "graph0", 2, 3); + + this.cache.updatePartition(partition3); + this.cache.updatePartition(partition4); + System.out.println(this.cache.debugCacheByGraphName("graph0")); + var partition6 = createPartition(1, "graph0", 0, 1); + this.cache.updatePartition(partition6); + + + System.out.println(this.cache.debugCacheByGraphName("graph0")); + + var partition5 = createPartition(1, "graph0", 0, 3); + this.cache.updatePartition(partition5); + System.out.println(this.cache.debugCacheByGraphName("graph0")); + } + + @Test + public void testRange2() { + var partition1 = createPartition(1, "graph0", 0, 3); + var partition2 = createPartition(2, "graph0", 3, 6); + this.cache.updatePartition(partition1); + this.cache.updatePartition(partition2); + + System.out.println(this.cache.debugCacheByGraphName("graph0")); + + // 中间有缺失 + var partition3 = createPartition(1, "graph0", 2, 3); + this.cache.updatePartition(partition3); + + System.out.println(this.cache.debugCacheByGraphName("graph0")); + + var partition5 = createPartition(1, "graph0", 0, 3); + this.cache.updatePartition(partition5); + System.out.println(this.cache.debugCacheByGraphName("graph0")); + } + + @Test + public void testRemovePartitions() { + var partition1 = createPartition(0, "graph0", 0, 1024); + var partition2 = createPartition(1, "graph0", 1024, 2048); + this.cache.updateShardGroup(creteShardGroup(0)); + this.cache.updatePartition(partition1); + this.cache.updateShardGroup(creteShardGroup(1)); + this.cache.updatePartition(partition2); + assertEquals(this.cache.getPartitions("graph0").size(), 2); + this.cache.removePartitions(); + assertEquals(this.cache.getPartitions("graph0").size(), 0); + } + + @Test + public void testRemoveAll() { + var partition1 = createPartition(0, "graph0", 0, 1024); + var partition2 = createPartition(1, "graph0", 1024, 2048); + var partition3 = createPartition(0, "graph1", 0, 2048); + this.cache.updateShardGroup(creteShardGroup(0)); + this.cache.updateShardGroup(creteShardGroup(1)); + this.cache.updatePartition(partition1); + this.cache.updatePartition(partition2); + this.cache.updatePartition(partition3); + + assertEquals(this.cache.getPartitions("graph0").size(), 2); + assertEquals(this.cache.getPartitions("graph1").size(), 1); + this.cache.removeAll("graph0"); + assertEquals(this.cache.getPartitions("graph0").size(), 0); + assertEquals(this.cache.getPartitions("graph1").size(), 1); + } + + @Test + public void testUpdateShardGroup() { + var shardGroup = createShardGroup(); + this.cache.updateShardGroup(shardGroup); + assertNotNull(this.cache.getShardGroup(shardGroup.getId())); + } + + @Test + public void testGetShardGroup() { + var shardGroup = createShardGroup(); + this.cache.updateShardGroup(shardGroup); + assertEquals(this.cache.getShardGroup(shardGroup.getId()), shardGroup); + } + + @Test + public void testAddStore() { + var store = createStore(1); + this.cache.addStore(1L, store); + assertEquals(this.cache.getStoreById(1L), store); + } + + @Test + public void testGetStoreById() { + var store = createStore(1); + this.cache.addStore(1L, store); + assertEquals(this.cache.getStoreById(1L), store); + } + + @Test + public void testRemoveStore() { + var store = createStore(1); + this.cache.addStore(1L, store); + assertEquals(this.cache.getStoreById(1L), store); + + this.cache.removeStore(1L); + assertNull(this.cache.getStoreById(1L)); + } + + @Test + public void testHasGraph() { + var partition = createPartition(0, "graph0", 0, 65535); + this.cache.updateShardGroup(creteShardGroup(0)); + this.cache.updatePartition(partition); + assertTrue(this.cache.hasGraph("graph0")); + assertFalse(this.cache.hasGraph("graph1")); + } + + @Test + public void testUpdateGraph() { + var graph = createGraph("graph0", 10); + this.cache.updateGraph(graph); + assertEquals(this.cache.getGraph("graph0"), graph); + graph = createGraph("graph0", 12); + this.cache.updateGraph(graph); + assertEquals(this.cache.getGraph("graph0"), graph); + } + + @Test + public void testGetGraph() { + var graph = createGraph("graph0", 12); + this.cache.updateGraph(graph); + assertEquals(this.cache.getGraph("graph0"), graph); + } + + @Test + public void testGetGraphs() { + var graph1 = createGraph("graph0", 12); + var graph2 = createGraph("graph1", 12); + var graph3 = createGraph("graph2", 12); + this.cache.updateGraph(graph1); + this.cache.updateGraph(graph2); + this.cache.updateGraph(graph3); + assertEquals(this.cache.getGraphs().size(), 3); + } + + @Test + public void testReset() { + var graph1 = createGraph("graph0", 12); + var graph2 = createGraph("graph1", 12); + var graph3 = createGraph("graph2", 12); + this.cache.updateGraph(graph1); + this.cache.updateGraph(graph2); + this.cache.updateGraph(graph3); + assertEquals(this.cache.getGraphs().size(), 3); + this.cache.reset(); + assertEquals(this.cache.getGraphs().size(), 0); + } + + @Test + public void testUpdateShardGroupLeader() { + var shardGroup = createShardGroup(); + this.cache.updateShardGroup(shardGroup); + + var leader = + Metapb.Shard.newBuilder().setStoreId(2).setRole(Metapb.ShardRole.Leader).build(); + this.cache.updateShardGroupLeader(shardGroup.getId(), leader); + + assertEquals(this.cache.getLeaderShard(shardGroup.getId()), leader); + } + +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionUtilsTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionUtilsTest.java new file mode 100644 index 0000000000..551bd40ecc --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionUtilsTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.common; + +import java.nio.charset.StandardCharsets; +import lombok.extern.slf4j.Slf4j; +import org.junit.Assert; +import org.junit.Test; + + +@Slf4j +public class PartitionUtilsTest extends BaseCommonTest { + + @Test + public void testCalcHashcode() { + byte[] key = new byte[5]; + long code = PartitionUtils.calcHashcode(key); + Assert.assertEquals(code, 31912L); + } + + // @Test + public void testHashCode() { + int partCount = 10; + int partSize = PartitionUtils.MAX_VALUE / partCount + 1; + int[] counter = new int[partCount]; + for (int i = 0; i < 10000; i++) { + String s = String.format("BATCH-GET-UNIT-%02d", i); + int c = PartitionUtils.calcHashcode(s.getBytes(StandardCharsets.UTF_8)); + + counter[c / partSize]++; + + } + + for (int i = 0; i < counter.length; i++) { + System.out.println(i + " " + counter[i]); + } + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/BaseCoreTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/BaseCoreTest.java new file mode 100644 index 0000000000..dddbffb088 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/BaseCoreTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.core; + +import java.io.File; +import java.io.IOException; + +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.pd.ConfigService; +import org.apache.hugegraph.pd.config.PDConfig; +import org.junit.After; +import org.junit.BeforeClass; + + +public class BaseCoreTest { + + static org.apache.hugegraph.pd.config.PDConfig pdConfig; + + @BeforeClass + public static void init() throws Exception { + String path = "tmp/unitTest"; + deleteDirectory(new File(path)); + pdConfig = new org.apache.hugegraph.pd.config.PDConfig() {{ + this.setClusterId(100); + this.setInitialStoreList("127.0.0.1:8500,127.0.0.1:8501,127.0.0.1:8502," + + "127.0.0.1:8503,127.0.0.1:8504,127.0.0.1:8505"); + }}; + + pdConfig.setStore(new org.apache.hugegraph.pd.config.PDConfig().new Store() {{ + this.setMaxDownTime(3600); + this.setKeepAliveTimeout(3600); + }}); + + pdConfig.setPartition(new org.apache.hugegraph.pd.config.PDConfig().new Partition() {{ + this.setShardCount(3); + this.setMaxShardsPerStore(3); + }}); + pdConfig.setRaft(new org.apache.hugegraph.pd.config.PDConfig().new Raft() {{ + this.setEnable(false); + }}); + pdConfig.setDiscovery(new PDConfig().new Discovery()); + pdConfig.setDataPath(path); + ConfigService configService = new ConfigService(pdConfig); + pdConfig = configService.loadConfig(); + } + + public static void deleteDirectory(File dir) { + try { + FileUtils.deleteDirectory(dir); + } catch (IOException e) { + System.out.printf("Failed to start ....,%s%n", e.getMessage()); + } + } + + @After + public void teardown() throws Exception { + // pass + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreSuiteTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreSuiteTest.java new file mode 100644 index 0000000000..3f616e2123 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreSuiteTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.core; + +import org.apache.hugegraph.pd.core.meta.MetadataKeyHelperTest; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +import lombok.extern.slf4j.Slf4j; + + +@RunWith(Suite.class) +@Suite.SuiteClasses({ + StoreNodeServiceTest.class, + MetadataKeyHelperTest.class +}) + +@Slf4j +public class PDCoreSuiteTest { + + +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/StoreNodeServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/StoreNodeServiceTest.java new file mode 100644 index 0000000000..c8c7bcf157 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/StoreNodeServiceTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.core; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hugegraph.pd.PartitionService; +import org.apache.hugegraph.pd.StoreNodeService; +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.junit.Assert; +import org.junit.Test; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class StoreNodeServiceTest extends BaseCoreTest { + + + @Test + public void testStoreNodeService() throws PDException { + Assert.assertEquals(pdConfig.getPartition().getTotalCount(), + pdConfig.getInitialStoreMap().size() * + pdConfig.getPartition().getMaxShardsPerStore() + / pdConfig.getPartition().getShardCount()); + StoreNodeService storeService = new StoreNodeService(pdConfig); + storeService.init(new PartitionService(pdConfig, storeService)); + int count = 6; + Metapb.Store[] stores = new Metapb.Store[count]; + for (int i = 0; i < count; i++) { + Metapb.Store store = Metapb.Store.newBuilder() + .setId(0) + .setAddress("127.0.0.1:850" + i) + .setDeployPath("/data") + .addLabels(Metapb.StoreLabel.newBuilder() + .setKey("namespace") + .setValue("default") + .build()) + .build(); + stores[i] = storeService.register(store); + System.out.println("新注册store， id = " + stores[i].getId()); + } + Assert.assertEquals(count, storeService.getStores("").size()); + + for (Metapb.Store store : stores) { + Metapb.StoreStats stats = Metapb.StoreStats.newBuilder() + .setStoreId(store.getId()) + .build(); + storeService.heartBeat(stats); + } + + Assert.assertEquals(6, storeService.getActiveStores("").size()); + + Metapb.Graph graph = Metapb.Graph.newBuilder() + .setGraphName("defaultGH") + .setPartitionCount(10) + .build(); + // 分配shard + List shards = storeService.allocShards(graph, 1); + + + Assert.assertEquals(3, shards.size()); + // 设置leader + Assert.assertEquals(pdConfig.getPartition().getTotalCount(), + storeService.getShardGroups().size()); + Metapb.Shard leader = Metapb.Shard.newBuilder(shards.get(0)) + .setRole(Metapb.ShardRole.Leader).build(); + shards = new ArrayList<>(shards); + shards.set(0, leader); + // 增加shard + pdConfig.getPartition().setShardCount(5); + + Metapb.ShardGroup shardGroup = Metapb.ShardGroup.newBuilder() + .setId(1) + .addAllShards(shards).build(); + shards = storeService.reallocShards(shardGroup); + + Assert.assertEquals(5, shards.size()); + // 减少shard + pdConfig.getPartition().setShardCount(3); + shards = storeService.reallocShards(shardGroup); + Assert.assertEquals(3, shards.size()); + // 包含leader，leader不能被删除 + Assert.assertTrue(shards.contains(leader)); + + // 减少shard + pdConfig.getPartition().setShardCount(1); + graph = Metapb.Graph.newBuilder(graph).build(); + shards = storeService.reallocShards(shardGroup); + Assert.assertEquals(1, shards.size()); + // 包含leader，leader不能被删除 + Assert.assertTrue(shards.contains(leader)); + + for (Metapb.Store store : stores) { + storeService.removeStore(store.getId()); + } + Assert.assertEquals(0, storeService.getStores("").size()); + + + } + + +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/meta/MetadataKeyHelperTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/meta/MetadataKeyHelperTest.java new file mode 100644 index 0000000000..7d38b9e132 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/meta/MetadataKeyHelperTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.core.meta; + +import static org.junit.Assert.assertArrayEquals; + +import org.apache.hugegraph.pd.meta.MetadataKeyHelper; +import org.junit.Test; + +public class MetadataKeyHelperTest { + + @Test + public void testMoveTaskKey() { + var key = MetadataKeyHelper.getMoveTaskKey("foo", 0, 1); + assertArrayEquals(key, "TASK_MOVE/foo/0/1".getBytes()); + var key2 = MetadataKeyHelper.getMoveTaskPrefix("foo"); + assertArrayEquals(key2, "TASK_MOVE/foo".getBytes()); + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/grpc/BaseGrpcTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/grpc/BaseGrpcTest.java new file mode 100644 index 0000000000..0e768d6119 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/grpc/BaseGrpcTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.grpc; + +import org.junit.After; +import org.junit.BeforeClass; + + +public class BaseGrpcTest { + + @BeforeClass + public static void init() { + + } + + @After + public void teardown() { + // pass + } + +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/grpc/GrpcSuiteTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/grpc/GrpcSuiteTest.java new file mode 100644 index 0000000000..b994a1fd2e --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/grpc/GrpcSuiteTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.grpc; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +import lombok.extern.slf4j.Slf4j; + + +@RunWith(Suite.class) +@Suite.SuiteClasses({ +}) + +@Slf4j +public class GrpcSuiteTest { + +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/BaseServerTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/BaseServerTest.java new file mode 100644 index 0000000000..e9092a631c --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/BaseServerTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.service; + +import java.io.File; +import java.net.http.HttpClient; + +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.pd.config.PDConfig; +import org.junit.After; +import org.junit.BeforeClass; + + +public class BaseServerTest { + public static HttpClient client; + public static String pdRestAddr; + + @BeforeClass + public static void init() { + client = HttpClient.newHttpClient(); + pdRestAddr = "http://127.0.0.1:8620"; + } + + public static PDConfig getConfig() { + FileUtils.deleteQuietly(new File("tmp/test/")); + PDConfig pdConfig = new PDConfig() {{ + this.setClusterId(100); + this.setPatrolInterval(1); + this.setRaft(new Raft() {{ + setEnable(false); + }}); + this.setDataPath("tmp/test/"); + }}; + return pdConfig; + } + + @After + public void teardown() { + // pass + } + +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/ConfigServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/ConfigServiceTest.java new file mode 100644 index 0000000000..faeacd834e --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/ConfigServiceTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.service; + +import java.util.List; + +import org.apache.hugegraph.pd.ConfigService; +import org.apache.hugegraph.pd.IdService; +import org.apache.hugegraph.pd.config.PDConfig; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class ConfigServiceTest { + + private final PDConfig config = BaseServerTest.getConfig(); + + private ConfigService service; + + @Before + public void setUp() { + this.service = new ConfigService(this.config); + } + + @Test + public void testGetPDConfig() throws Exception { + // Setup + try { + final Metapb.PDConfig config = Metapb.PDConfig.newBuilder() + .setVersion(0L) + .setPartitionCount(0) + .setShardCount(55) + .setMaxShardsPerStore(0) + .setTimestamp(0L).build(); + this.service.setPDConfig(config); + // Run the test + Metapb.PDConfig result = this.service.getPDConfig(0L); + + // Verify the results + Assert.assertTrue(result.getShardCount() == 55); + result = this.service.getPDConfig(); + Assert.assertTrue(result.getShardCount() == 55); + } catch (Exception e) { + + } + + } + + @Test + public void testGetGraphSpace() throws Exception { + // Setup + Metapb.GraphSpace space = Metapb.GraphSpace.newBuilder() + .setName("gs1") + .setTimestamp(0L).build(); + final List expectedResult = List.of(space); + this.service.setGraphSpace(space); + // Run the test + final List result = this.service.getGraphSpace( + "gs1"); + + Assert.assertEquals(1, result.size()); + } + + @Test + public void testUpdatePDConfig() { + try { + final Metapb.PDConfig mConfig = Metapb.PDConfig.newBuilder() + .setVersion(0L) + .setPartitionCount(0) + .setShardCount(0) + .setMaxShardsPerStore(0) + .setTimestamp(0L) + .build(); + final PDConfig expectedResult = new PDConfig(); + expectedResult.setConfigService(new ConfigService(new PDConfig())); + expectedResult.setIdService(new IdService(new PDConfig())); + expectedResult.setClusterId(0L); + expectedResult.setPatrolInterval(0L); + expectedResult.setDataPath("dataPath"); + expectedResult.setMinStoreCount(0); + expectedResult.setInitialStoreList("initialStoreList"); + expectedResult.setHost("host"); + expectedResult.setVerifyPath("verifyPath"); + expectedResult.setLicensePath("licensePath"); + this.service.updatePDConfig(mConfig); + } catch (Exception e) { + + } + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/IdServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/IdServiceTest.java new file mode 100644 index 0000000000..dd0d3feb24 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/IdServiceTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.service; + +import java.io.File; + +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.pd.IdService; +import org.apache.hugegraph.pd.config.PDConfig; +import org.apache.hugegraph.pd.meta.IdMetaStore; +import org.junit.Assert; +import org.junit.Test; + +public class IdServiceTest { + @Test + public void testCid() { + try { + PDConfig pdConfig = BaseServerTest.getConfig(); + int max = 0x2000; + IdService idService = new IdService(pdConfig); + for (int i = 0; i < max; i++) { + idService.getCId("test", max); + } + idService.delCId("test", 1); + idService.delCId("test", 0x10); + idService.delCId("test", 0x100); + idService.delCId("test", 0x1000); + + Assert.assertEquals(1, idService.getCId("test", max)); + Assert.assertEquals(0x10, idService.getCId("test", max)); + Assert.assertEquals(0x100, idService.getCId("test", max)); + Assert.assertEquals(0x1000, idService.getCId("test", max)); + Assert.assertEquals(-1, idService.getCId("test", max)); + + idService.delCId("test", 1); + idService.delCId("test", 0x10); + idService.delCId("test", 0x100); + idService.delCId("test", 0x1000); + + long cid1 = idService.getCId("test", "name", max); + idService.delCIdDelay("test", "name", cid1); + long cid2 = idService.getCId("test", "name", max); + + Assert.assertEquals(cid1, cid2); + idService.delCIdDelay("test", "name", cid2); + Thread.sleep(5000); + long cid3 = idService.getCId("test", "name", max); + } catch (Exception e) { + + } + // MetadataFactory.closeStore(); + } + + @Test + public void testId() { + try { + FileUtils.deleteQuietly(new File("tmp/testId/")); + IdMetaStore.CID_DEL_TIMEOUT = 2000; + PDConfig pdConfig = new PDConfig() {{ + this.setClusterId(100); + this.setPatrolInterval(1); + this.setRaft(new Raft() {{ + setEnable(false); + }}); + this.setDataPath("tmp/testId/"); + }}; + IdService idService = new IdService(pdConfig); + long first = idService.getId("abc", 100); + Assert.assertEquals(first, 0L); + long second = idService.getId("abc", 100); + Assert.assertEquals(second, 100L); + idService.resetId("abc"); + first = idService.getId("abc", 100); + Assert.assertEquals(first, 0L); + } catch (Exception ignored) { + + } + // MetadataFactory.closeStore(); + } + + @Test + public void testMember() { + try { + PDConfig pdConfig = BaseServerTest.getConfig(); + IdService idService = new IdService(pdConfig); + idService.setPdConfig(pdConfig); + PDConfig config = idService.getPdConfig(); + config.getHost(); + } catch (Exception e) { + e.printStackTrace(); + } + // MetadataFactory.closeStore(); + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/KvServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/KvServiceTest.java new file mode 100644 index 0000000000..02870b219e --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/KvServiceTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.service; + +import org.apache.hugegraph.pd.KvService; +import org.apache.hugegraph.pd.config.PDConfig; +import org.junit.Assert; +import org.junit.Test; + +public class KvServiceTest { + + @Test + public void testKv() { + try { + PDConfig pdConfig = BaseServerTest.getConfig(); + KvService service = new KvService(pdConfig); + String key = "kvTest"; + String kvTest = service.get(key); + Assert.assertEquals(kvTest, ""); + service.put(key, "kvTestValue"); + kvTest = service.get(key); + Assert.assertEquals(kvTest, "kvTestValue"); + service.scanWithPrefix(key); + service.delete(key); + service.put(key, "kvTestValue"); + service.deleteWithPrefix(key); + service.put(key, "kvTestValue", 1000L); + service.keepAlive(key); + } catch (Exception e) { + + } + } + + @Test + public void testMember() { + try { + PDConfig pdConfig = BaseServerTest.getConfig(); + KvService service = new KvService(pdConfig); + service.setPdConfig(pdConfig); + PDConfig config = service.getPdConfig(); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/LogServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/LogServiceTest.java new file mode 100644 index 0000000000..266db76c56 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/LogServiceTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.service; + +import java.util.List; + +import org.apache.hugegraph.pd.LogService; +import org.apache.hugegraph.pd.config.PDConfig; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.google.protobuf.Any; + +public class LogServiceTest { + + private final PDConfig mockPdConfig = BaseServerTest.getConfig(); + + private LogService logServiceUnderTest; + + @Before + public void setUp() { + this.logServiceUnderTest = new LogService(this.mockPdConfig); + } + + @Test + public void testGetLog() throws Exception { + this.logServiceUnderTest.insertLog("action", "message", + Any.newBuilder().build()); + + // Run the test + final List result = this.logServiceUnderTest.getLog( + "action", 0L, System.currentTimeMillis()); + + // Verify the results + Assert.assertEquals(result.size(), 1); + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/PartitionServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/PartitionServiceTest.java new file mode 100644 index 0000000000..0a16402d33 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/PartitionServiceTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.service; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; + +import org.apache.hugegraph.pd.PartitionService; +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.grpc.MetaTask; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.pd.grpc.pulse.CleanPartition; +import org.apache.hugegraph.pd.grpc.pulse.CleanType; +import org.junit.Before; +import org.junit.Test; + +public class PartitionServiceTest extends PdTestBase { + + private PartitionService service; + + @Before + public void init() { + this.service = getPartitionService(); + } + + @Test + public void testCombinePartition() throws PDException { + buildEnv(); + // 0, 1, 2-> 0, 3,4,5->1, 6,7,8 ->2, 9,10, 11-> 3 + this.service.combinePartition(4); + + var partition = this.service.getPartitionById("graph0", 0); + assertEquals(0, partition.getStartKey()); + assertEquals(5462, partition.getEndKey()); + + var tasks = getStoreNodeService().getTaskInfoMeta().scanMoveTask("graph0"); + assertEquals(11, tasks.size()); + + for (MetaTask.Task task : tasks) { + var newTask = task.toBuilder().setState(MetaTask.TaskState.Task_Success).build(); + getTaskService().reportTask(newTask); + } + + tasks = getStoreNodeService().getTaskInfoMeta().scanMoveTask("graph0"); + assertEquals(0, tasks.size()); + } + + @Test + public void testCombinePartition2() throws PDException { + buildEnv(); + // 0, 1, 2-> 0, 3,4,5->1, 6,7,8 ->2, 9,10, 11-> 3 + this.service.combinePartition(4); + + var partition = this.service.getPartitionById("graph0", 0); + assertEquals(0, partition.getStartKey()); + assertEquals(5462, partition.getEndKey()); + + var tasks = getStoreNodeService().getTaskInfoMeta().scanMoveTask("graph0"); + assertEquals(11, tasks.size()); + + for (MetaTask.Task task : tasks) { + var newTask = task.toBuilder().setState(MetaTask.TaskState.Task_Failure).build(); + getTaskService().reportTask(newTask); + } + + tasks = getStoreNodeService().getTaskInfoMeta().scanMoveTask("graph0"); + assertEquals(0, tasks.size()); + } + + @Test + public void testHandleCleanTask() { + MetaTask.Task task = MetaTask.Task.newBuilder() + .setType(MetaTask.TaskType.Clean_Partition) + .setPartition( + Metapb.Partition.newBuilder().setGraphName("foo") + .setId(0).build()) + .setCleanPartition(CleanPartition.newBuilder() + .setCleanType( + CleanType.CLEAN_TYPE_KEEP_RANGE) + .setDeletePartition(true) + .setKeyStart(0) + .setKeyEnd(10) + .build()) + .build(); + getTaskService().reportTask(task); + } + + private void buildEnv() throws PDException { + var storeInfoMeta = getStoreNodeService().getStoreInfoMeta(); + storeInfoMeta.updateStore(Metapb.Store.newBuilder() + .setId(99) + .setState(Metapb.StoreState.Up) + .build()); + + long lastId = 0; + for (int i = 0; i < 12; i++) { + Metapb.Shard shard = Metapb.Shard.newBuilder() + .setStoreId(99) + .setRole(Metapb.ShardRole.Leader) + .build(); + + Metapb.ShardGroup shardGroup = Metapb.ShardGroup.newBuilder() + .setId(i) + .setState( + Metapb.PartitionState.PState_Normal) + .addAllShards(List.of(shard)) + .build(); + storeInfoMeta.updateShardGroup(shardGroup); + + var partitionShard = this.service.getPartitionByCode("graph0", lastId); + if (partitionShard != null) { + lastId = partitionShard.getPartition().getEndKey(); + } + } + + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/PdTestBase.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/PdTestBase.java new file mode 100644 index 0000000000..50eadb3a6d --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/PdTestBase.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.service; + +import java.io.File; + +import org.apache.hugegraph.pd.ConfigService; +import org.apache.hugegraph.pd.IdService; +import org.apache.hugegraph.pd.PartitionInstructionListener; +import org.apache.hugegraph.pd.PartitionService; +import org.apache.hugegraph.pd.PartitionStatusListener; +import org.apache.hugegraph.pd.StoreMonitorDataService; +import org.apache.hugegraph.pd.StoreNodeService; +import org.apache.hugegraph.pd.StoreStatusListener; +import org.apache.hugegraph.pd.TaskScheduleService; +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.config.PDConfig; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.pd.grpc.pulse.ChangeShard; +import org.apache.hugegraph.pd.grpc.pulse.CleanPartition; +import org.apache.hugegraph.pd.grpc.pulse.DbCompaction; +import org.apache.hugegraph.pd.grpc.pulse.MovePartition; +import org.apache.hugegraph.pd.grpc.pulse.PartitionKeyRange; +import org.apache.hugegraph.pd.grpc.pulse.SplitPartition; +import org.apache.hugegraph.pd.grpc.pulse.TransferLeader; +import org.apache.hugegraph.pd.raft.RaftEngine; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +public class PdTestBase { + private static final String DATA_PATH = "/tmp/pd_data"; + private static PDConfig pdConfig; + private static StoreNodeService storeNodeService; + private static PartitionService partitionService; + private static TaskScheduleService taskService; + private static StoreMonitorDataService storeMonitorDataService; + + @BeforeClass + public static void initService() throws PDException { + deleteDir(new File(DATA_PATH)); + + PDConfig config = new PDConfig(); + config.setDataPath(DATA_PATH); + config.setMinStoreCount(3); + config.setInitialStoreList("127.0.0.1:8501"); + config.setHost("127.0.0.1"); + config.setVerifyPath(""); + config.setLicensePath(""); + PDConfig.Raft raft = new PDConfig().new Raft(); + raft.setAddress("127.0.0.1:8601"); + raft.setPeersList("127.0.0.1:8601"); + raft.setDataPath(DATA_PATH); + raft.setHost("127.0.0.1"); + raft.setGrpcPort(8688); + raft.setPort(8621); + + config.setRaft(raft); + + config.setStore(new PDConfig().new Store()); + config.setPartition(new PDConfig().new Partition() {{ + setShardCount(1); + setTotalCount(12); + setMaxShardsPerStore(12); + }}); + config.setDiscovery(new PDConfig().new Discovery()); + + pdConfig = config; + + var configService = new ConfigService(pdConfig); + configService.loadConfig(); + + var engine = RaftEngine.getInstance(); + engine.addStateListener(configService); + engine.init(pdConfig.getRaft()); + engine.waitingForLeader(5000); + + storeNodeService = new StoreNodeService(pdConfig); + partitionService = new PartitionService(pdConfig, storeNodeService); + taskService = new TaskScheduleService(pdConfig, storeNodeService, partitionService); + var idService = new IdService(pdConfig); + storeMonitorDataService = new StoreMonitorDataService(pdConfig); + RaftEngine.getInstance().addStateListener(partitionService); + pdConfig.setIdService(idService); + + + storeNodeService.init(partitionService); + partitionService.init(); + partitionService.addInstructionListener(new PartitionInstructionListener() { + @Override + public void changeShard(Metapb.Partition partition, ChangeShard changeShard) throws + PDException { + + } + + @Override + public void transferLeader(Metapb.Partition partition, + TransferLeader transferLeader) throws PDException { + + } + + @Override + public void splitPartition(Metapb.Partition partition, + SplitPartition splitPartition) throws PDException { + + } + + @Override + public void dbCompaction(Metapb.Partition partition, DbCompaction dbCompaction) throws + PDException { + + } + + @Override + public void movePartition(Metapb.Partition partition, + MovePartition movePartition) throws PDException { + + } + + @Override + public void cleanPartition(Metapb.Partition partition, + CleanPartition cleanPartition) throws PDException { + + } + + @Override + public void changePartitionKeyRange(Metapb.Partition partition, + PartitionKeyRange partitionKeyRange) + throws PDException { + + } + }); + + partitionService.addStatusListener(new PartitionStatusListener() { + @Override + public void onPartitionChanged(Metapb.Partition partition, + Metapb.Partition newPartition) { + + } + + @Override + public void onPartitionRemoved(Metapb.Partition partition) { + + } + }); + + storeNodeService.addStatusListener(new StoreStatusListener() { + @Override + public void onStoreStatusChanged(Metapb.Store store, Metapb.StoreState old, + Metapb.StoreState status) { + + } + + @Override + public void onGraphChange(Metapb.Graph graph, Metapb.GraphState stateOld, + Metapb.GraphState stateNew) { + + } + + @Override + public void onStoreRaftChanged(Metapb.Store store) { + + } + }); + + taskService.init(); + } + + @AfterClass + public static void shutdownService() { + var instance = RaftEngine.getInstance(); + if (instance != null) { + instance.shutDown(); + } + } + + private static boolean deleteDir(File dir) { + if (dir.isDirectory()) { + for (File file : dir.listFiles()) { + deleteDir(file); + } + } + return dir.delete(); + } + + public static StoreNodeService getStoreNodeService() { + return storeNodeService; + } + + public static PartitionService getPartitionService() { + return partitionService; + } + + public static PDConfig getPdConfig() { + return pdConfig; + } + + public static TaskScheduleService getTaskService() { + return taskService; + } + + public static StoreMonitorDataService getStoreMonitorDataService() { + return storeMonitorDataService; + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/RestApiTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/RestApiTest.java new file mode 100644 index 0000000000..7e5fec381e --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/RestApiTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.service; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; + +import org.json.JSONException; +import org.json.JSONObject; +import org.junit.Test; + +public class RestApiTest extends BaseServerTest { + @Test + public void testQueryClusterInfo() throws URISyntaxException, IOException, InterruptedException, + JSONException { + String url = pdRestAddr + "/v1/cluster"; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(url)) + .GET() + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + JSONObject obj = new JSONObject(response.body()); + assert obj.getInt("status") == 0; + } + + @Test + public void testQueryClusterMembers() throws URISyntaxException, IOException, + InterruptedException, JSONException { + String url = pdRestAddr + "/v1/members"; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(url)) + .GET() + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + JSONObject obj = new JSONObject(response.body()); + assert obj.getInt("status") == 0; + } + + @Test + public void testQueryStoresInfo() throws URISyntaxException, IOException, InterruptedException, + JSONException { + String url = pdRestAddr + "/v1/stores"; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(url)) + .GET() + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + JSONObject obj = new JSONObject(response.body()); + assert obj.getInt("status") == 0; + } + + @Test + public void testQueryGraphsInfo() throws IOException, InterruptedException, JSONException, + URISyntaxException { + String url = pdRestAddr + "/v1/graphs"; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(url)) + .GET() + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + JSONObject obj = new JSONObject(response.body()); + assert obj.getInt("status") == 0; + } + + @Test + public void testQueryPartitionsInfo() throws IOException, InterruptedException, JSONException, + URISyntaxException { + String url = pdRestAddr + "/v1/highLevelPartitions"; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(url)) + .GET() + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + JSONObject obj = new JSONObject(response.body()); + assert obj.getInt("status") == 0; + } + + @Test + public void testQueryDebugPartitionsInfo() throws URISyntaxException, IOException, + InterruptedException { + String url = pdRestAddr + "/v1/partitions"; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(url)) + .GET() + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + assert response.statusCode() == 200; + } + + @Test + public void testQueryShards() throws URISyntaxException, IOException, InterruptedException, + JSONException { + String url = pdRestAddr + "/v1/shards"; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(url)) + .GET() + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + JSONObject obj = new JSONObject(response.body()); + assert obj.getInt("status") == 0; + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/ServerSuiteTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/ServerSuiteTest.java new file mode 100644 index 0000000000..fe8c0ed044 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/ServerSuiteTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.service; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +import lombok.extern.slf4j.Slf4j; + + +@RunWith(Suite.class) +@Suite.SuiteClasses({ + RestApiTest.class, + ConfigServiceTest.class, + IdServiceTest.class, + KvServiceTest.class, + LogServiceTest.class, + StoreServiceTest.class, + StoreNodeServiceNewTest.class, + StoreMonitorDataServiceTest.class, + TaskScheduleServiceTest.class, + PartitionServiceTest.class +}) + +@Slf4j +public class ServerSuiteTest { +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreMonitorDataServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreMonitorDataServiceTest.java new file mode 100644 index 0000000000..21a2a37dde --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreMonitorDataServiceTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.service; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; + +import org.apache.hugegraph.pd.StoreMonitorDataService; +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.junit.Before; +import org.junit.Test; + +public class StoreMonitorDataServiceTest extends PdTestBase { + + StoreMonitorDataService service; + + @Before + public void init() { + this.service = getStoreMonitorDataService(); + var store = getPdConfig().getStore(); + store.setMonitorDataEnabled(true); + store.setMonitorDataInterval("1s"); + getPdConfig().setStore(store); + } + + @Test + public void test() throws InterruptedException, PDException { + long now = System.currentTimeMillis() / 1000; + for (int i = 0; i < 5; i++) { + this.service.saveMonitorData(genStats()); + now = System.currentTimeMillis() / 1000; + Thread.sleep(1100); + } + assertTrue(this.service.getLatestStoreMonitorDataTimeStamp(1) == 0 || + this.service.getLatestStoreMonitorDataTimeStamp(1) == now); + + var data = this.service.getStoreMonitorData(1); + assertEquals(5, data.size()); + + assertNotNull(this.service.debugMonitorInfo(List.of(Metapb.RecordPair.newBuilder() + .setKey("key1") + .setValue(1) + .build()))); + + assertNotNull(this.service.getStoreMonitorDataText(1)); + + + this.service.removeExpiredMonitorData(1, now + 1); + assertEquals(0, this.service.getStoreMonitorData(1).size()); + } + + + private Metapb.StoreStats genStats() { + return Metapb.StoreStats.newBuilder() + .setStoreId(1) + .addSystemMetrics( + Metapb.RecordPair.newBuilder().setKey("key1").setValue(1) + .build()) + .build(); + } + + +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreNodeServiceNewTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreNodeServiceNewTest.java new file mode 100644 index 0000000000..10d098d1f3 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreNodeServiceNewTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.service; + +import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.apache.hugegraph.pd.StoreNodeService; +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.junit.Before; +import org.junit.Test; + +public class StoreNodeServiceNewTest extends PdTestBase { + private StoreNodeService service; + + @Before + public void init() { + this.service = getStoreNodeService(); + } + + @Test + public void testGetTaskInfoMeta() { + assertNotNull(this.service.getTaskInfoMeta()); + } + + public void testGetStoreInfoMeta() { + assertNotNull(this.service.getStoreInfoMeta()); + } + + @Test + public void testRemoveShardGroup() throws PDException { + for (int i = 0; i < 12; i++) { + Metapb.ShardGroup group = Metapb.ShardGroup.newBuilder() + .setId(i) + .setState( + Metapb.PartitionState.PState_Offline) + .build(); + this.service.getStoreInfoMeta().updateShardGroup(group); + } + + this.service.deleteShardGroup(11); + this.service.deleteShardGroup(10); + + assertEquals(10, getPdConfig().getConfigService().getPDConfig().getPartitionCount()); + // restore + getPdConfig().getConfigService().setPartitionCount(12); + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreServiceTest.java new file mode 100644 index 0000000000..af17db1c72 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreServiceTest.java @@ -0,0 +1,833 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +import org.apache.hugegraph.pd.ConfigService; +import org.apache.hugegraph.pd.IdService; +import org.apache.hugegraph.pd.PartitionService; +import org.apache.hugegraph.pd.StoreNodeService; +import org.apache.hugegraph.pd.StoreStatusListener; +import org.apache.hugegraph.pd.config.PDConfig; +import org.apache.hugegraph.pd.grpc.MetaTask; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.junit.Before; +import org.junit.Test; + +public class StoreServiceTest { + + private PDConfig config; + + private StoreNodeService service; + + @Before + public void setUp() { + this.config = getConfig(); + this.service = new StoreNodeService(this.config); + } + + @Test + public void testInit() { + // Setup + PDConfig pdConfig = getConfig(); + final PDConfig pdConfig1 = getConfig(); + final PartitionService partitionService = new PartitionService(pdConfig, + new StoreNodeService( + pdConfig1)); + + // Run the test + this.service.init(partitionService); + + // Verify the results + } + + private PDConfig getConfig() { + PDConfig pdConfig = new PDConfig(); + pdConfig.setConfigService( + new ConfigService(BaseServerTest.getConfig())); + pdConfig.setIdService(new IdService(BaseServerTest.getConfig())); + pdConfig.setClusterId(0L); + pdConfig.setPatrolInterval(0L); + pdConfig.setDataPath("dataPath"); + pdConfig.setMinStoreCount(0); + pdConfig.setInitialStoreList("initialStoreList"); + pdConfig.setHost("host"); + pdConfig.setVerifyPath("verifyPath"); + pdConfig.setLicensePath("licensePath"); + PDConfig.Raft raft = new PDConfig().new Raft(); + raft.setEnable(false); + pdConfig.setRaft(raft); + final PDConfig.Partition partition = new PDConfig().new Partition(); + partition.setTotalCount(0); + partition.setShardCount(0); + pdConfig.setPartition(partition); + pdConfig.setInitialStoreMap(Map.ofEntries(Map.entry("value", "value"))); + return pdConfig; + } + + @Test + public void testIsOK() { + // Setup + // Run the test + final boolean result = this.service.isOK(); + + // Verify the results + assertThat(result).isTrue(); + } + + @Test + public void testRegister() throws Exception { + // Setup + final Metapb.Store store = Metapb.Store.newBuilder().setId(0L) + .setAddress("address") + .setRaftAddress("raftAddress") + .addLabels(Metapb.StoreLabel + .newBuilder() + .build()) + .setVersion("version").setState( + Metapb.StoreState.Unknown).setStartTimestamp(0L) + .setDeployPath("deployPath") + .setLastHeartbeat(0L).setStats( + Metapb.StoreStats.newBuilder().setStoreId(0L) + .setPartitionCount(0).addGraphStats( + Metapb.GraphStats.newBuilder() + .setGraphName("value") + .setApproximateSize(0L) + .setRole(Metapb.ShardRole.None) + .build()).build()) + .setDataVersion(0).setCores(0) + .setDataPath("dataPath").build(); + final Metapb.Store expectedResult = Metapb.Store.newBuilder().setId(0L) + .setAddress("address") + .setRaftAddress( + "raftAddress") + .addLabels( + Metapb.StoreLabel + .newBuilder() + .build()) + .setVersion("version") + .setState( + Metapb.StoreState.Unknown) + .setStartTimestamp(0L) + .setDeployPath( + "deployPath") + .setLastHeartbeat(0L) + .setStats( + Metapb.StoreStats + .newBuilder() + .setStoreId( + 0L) + .setPartitionCount( + 0) + .addGraphStats( + Metapb.GraphStats + .newBuilder() + .setGraphName( + "value") + .setApproximateSize( + 0L) + .setRole( + Metapb.ShardRole.None) + .build()) + .build()) + .setDataVersion(0) + .setCores(0) + .setDataPath("dataPath") + .build(); + + // Configure PDConfig.getInitialStoreMap(...). + final Map stringStringMap = Map.ofEntries( + Map.entry("value", "value")); + + // Run the test + final Metapb.Store result = this.service.register(store); + } + + @Test + public void testGetStore() throws Exception { + // Setup + try { + Metapb.GraphStats stats = Metapb.GraphStats.newBuilder() + .setGraphName("value") + .setApproximateSize(0L) + .setRole( + Metapb.ShardRole.None) + .build(); + Metapb.StoreStats storeStats = Metapb.StoreStats.newBuilder() + .setStoreId(0L) + .setPartitionCount( + 0) + .addGraphStats( + stats) + .build(); + final Metapb.Store expectedResult = Metapb.Store.newBuilder() + .setId(0L) + .setAddress( + "address") + .setRaftAddress( + "raftAddress") + .addLabels( + Metapb.StoreLabel + .newBuilder() + .build()) + .setVersion( + "version") + .setState( + Metapb.StoreState.Unknown) + .setStartTimestamp( + 0L) + .setDeployPath( + "deployPath") + .setLastHeartbeat( + 0L) + .setStats( + storeStats) + .setDataVersion(0) + .setCores(0) + .setDataPath( + "dataPath") + .build(); + + // Run the test + final Metapb.Store result = this.service.getStore(0L); + } catch (Exception e) { + + } + } + + @Test + public void testUpdateStore() throws Exception { + // Setup + final Metapb.Store store = Metapb.Store.newBuilder().setId(0L) + .setAddress("address") + .setRaftAddress("raftAddress") + .addLabels(Metapb.StoreLabel + .newBuilder() + .build()) + .setVersion("version").setState( + Metapb.StoreState.Unknown).setStartTimestamp(0L) + .setDeployPath("deployPath") + .setLastHeartbeat(0L).setStats( + Metapb.StoreStats.newBuilder().setStoreId(0L) + .setPartitionCount(0).addGraphStats( + Metapb.GraphStats.newBuilder() + .setGraphName("value") + .setApproximateSize(0L) + .setRole(Metapb.ShardRole.None) + .build()).build()) + .setDataVersion(0).setCores(0) + .setDataPath("dataPath").build(); + final Metapb.Store expectedResult = Metapb.Store.newBuilder().setId(0L) + .setAddress("address") + .setRaftAddress( + "raftAddress") + .addLabels( + Metapb.StoreLabel + .newBuilder() + .build()) + .setVersion("version") + .setState( + Metapb.StoreState.Unknown) + .setStartTimestamp(0L) + .setDeployPath( + "deployPath") + .setLastHeartbeat(0L) + .setStats( + Metapb.StoreStats + .newBuilder() + .setStoreId( + 0L) + .setPartitionCount( + 0) + .addGraphStats( + Metapb.GraphStats + .newBuilder() + .setGraphName( + "value") + .setApproximateSize( + 0L) + .setRole( + Metapb.ShardRole.None) + .build()) + .build()) + .setDataVersion(0) + .setCores(0) + .setDataPath("dataPath") + .build(); + + // Configure PDConfig.getPartition(...). + final PDConfig.Partition partition = new PDConfig().new Partition(); + partition.setTotalCount(0); + partition.setMaxShardsPerStore(0); + partition.setShardCount(0); + + // Run the test + final Metapb.Store result = this.service.updateStore(store); + } + + @Test + public void testStoreTurnoff() throws Exception { + // Setup + final Metapb.Store store = Metapb.Store.newBuilder().setId(0L) + .setAddress("address") + .setRaftAddress("raftAddress") + .addLabels(Metapb.StoreLabel + .newBuilder() + .build()) + .setVersion("version").setState( + Metapb.StoreState.Unknown).setStartTimestamp(0L) + .setDeployPath("deployPath") + .setLastHeartbeat(0L).setStats( + Metapb.StoreStats.newBuilder().setStoreId(0L) + .setPartitionCount(0).addGraphStats( + Metapb.GraphStats.newBuilder() + .setGraphName("value") + .setApproximateSize(0L) + .setRole(Metapb.ShardRole.None) + .build()).build()) + .setDataVersion(0).setCores(0) + .setDataPath("dataPath").build(); + + // Configure PDConfig.getPartition(...). + final PDConfig.Partition partition = new PDConfig().new Partition(); + partition.setTotalCount(0); + partition.setMaxShardsPerStore(0); + partition.setShardCount(0); + + // Run the test + this.service.storeTurnoff(store); + + // Verify the results + } + + + @Test + public void testGetStores1() throws Exception { + // Setup + final List expectedResult = List.of( + Metapb.Store.newBuilder().setId(0L).setAddress("address") + .setRaftAddress("raftAddress") + .addLabels(Metapb.StoreLabel.newBuilder().build()) + .setVersion("version") + .setState(Metapb.StoreState.Unknown) + .setStartTimestamp(0L).setDeployPath("deployPath") + .setLastHeartbeat(0L).setStats( + Metapb.StoreStats.newBuilder().setStoreId(0L) + .setPartitionCount(0).addGraphStats( + Metapb.GraphStats.newBuilder() + .setGraphName("value") + .setApproximateSize(0L) + .setRole(Metapb.ShardRole.None) + .build()).build()) + .setDataVersion(0).setCores(0) + .setDataPath("dataPath").build()); + + // Run the test + final List result = this.service.getStores(); + } + + @Test + public void testGetStores2() throws Exception { + // Setup + final List expectedResult = List.of( + Metapb.Store.newBuilder().setId(0L).setAddress("address") + .setRaftAddress("raftAddress") + .addLabels(Metapb.StoreLabel.newBuilder().build()) + .setVersion("version") + .setState(Metapb.StoreState.Unknown) + .setStartTimestamp(0L).setDeployPath("deployPath") + .setLastHeartbeat(0L).setStats( + Metapb.StoreStats.newBuilder().setStoreId(0L) + .setPartitionCount(0).addGraphStats( + Metapb.GraphStats.newBuilder() + .setGraphName("value") + .setApproximateSize(0L) + .setRole(Metapb.ShardRole.None) + .build()).build()) + .setDataVersion(0).setCores(0) + .setDataPath("dataPath").build()); + + // Run the test + final List result = this.service.getStores("graphName"); + } + + + @Test + public void testGetStoreStatus() throws Exception { + // Setup + final List expectedResult = List.of( + Metapb.Store.newBuilder().setId(0L).setAddress("address") + .setRaftAddress("raftAddress") + .addLabels(Metapb.StoreLabel.newBuilder().build()) + .setVersion("version") + .setState(Metapb.StoreState.Unknown) + .setStartTimestamp(0L).setDeployPath("deployPath") + .setLastHeartbeat(0L).setStats( + Metapb.StoreStats.newBuilder().setStoreId(0L) + .setPartitionCount(0).addGraphStats( + Metapb.GraphStats.newBuilder() + .setGraphName("value") + .setApproximateSize(0L) + .setRole(Metapb.ShardRole.None) + .build()).build()) + .setDataVersion(0).setCores(0) + .setDataPath("dataPath").build()); + + // Run the test + final List result = this.service.getStoreStatus(false); + + } + + @Test + public void testGetShardGroups() throws Exception { + // Setup + final List expectedResult = List.of( + Metapb.ShardGroup.newBuilder().setId(0).addShards( + Metapb.Shard.newBuilder().setStoreId(0L) + .setRole(Metapb.ShardRole.None).build()) + .setState(Metapb.PartitionState.PState_None) + .build()); + + // Run the test + final List result = this.service.getShardGroups(); + + } + + @Test + public void testGetShardGroup() throws Exception { + // Setup + final Metapb.ShardGroup expectedResult = Metapb.ShardGroup.newBuilder() + .setId(0) + .addShards( + Metapb.Shard + .newBuilder() + .setStoreId( + 0L) + .setRole( + Metapb.ShardRole.None) + .build()) + .setState( + Metapb.PartitionState.PState_None) + .build(); + + // Run the test + final Metapb.ShardGroup result = this.service.getShardGroup(0); + + // Verify the results + } + + + @Test + public void testGetShardGroupsByStore() throws Exception { + // Setup + final List expectedResult = List.of( + Metapb.ShardGroup.newBuilder().setId(0).addShards( + Metapb.Shard.newBuilder().setStoreId(0L) + .setRole(Metapb.ShardRole.None).build()) + .setState(Metapb.PartitionState.PState_None) + .build()); + + // Run the test + final List result = this.service.getShardGroupsByStore( + 0L); + } + + @Test + public void testGetActiveStores1() throws Exception { + // Setup + final List expectedResult = List.of( + Metapb.Store.newBuilder().setId(0L).setAddress("address") + .setRaftAddress("raftAddress") + .addLabels(Metapb.StoreLabel.newBuilder().build()) + .setVersion("version") + .setState(Metapb.StoreState.Unknown) + .setStartTimestamp(0L).setDeployPath("deployPath") + .setLastHeartbeat(0L).setStats( + Metapb.StoreStats.newBuilder().setStoreId(0L) + .setPartitionCount(0).addGraphStats( + Metapb.GraphStats.newBuilder() + .setGraphName("value") + .setApproximateSize(0L) + .setRole(Metapb.ShardRole.None) + .build()).build()) + .setDataVersion(0).setCores(0) + .setDataPath("dataPath").build()); + + // Run the test + final List result = this.service.getActiveStores("graphName"); + + // Verify the results + } + + @Test + public void testGetActiveStores1ThrowsPDException() { + try { + List stores = this.service.getActiveStores(); + assertThat(stores.size() == 0); + } catch (Exception e) { + + } + } + + @Test + public void testGetTombStores() throws Exception { + // Setup + final List storeList = List.of( + Metapb.Store.newBuilder().setId(0L).setAddress("address") + .setRaftAddress("raftAddress") + .addLabels(Metapb.StoreLabel.newBuilder().build()) + .setVersion("version") + .setState(Metapb.StoreState.Tombstone) + .setStartTimestamp(0L).setDeployPath("deployPath") + .setLastHeartbeat(0L).setStats( + Metapb.StoreStats.newBuilder().setStoreId(0L) + .setPartitionCount(0).addGraphStats( + Metapb.GraphStats.newBuilder() + .setGraphName("value") + .setApproximateSize(0L) + .setRole(Metapb.ShardRole.None) + .build()).build()) + .setDataVersion(0).setCores(0) + .setDataPath("dataPath").build()); + this.service.register(storeList.get(0)); + + // Run the test + final List result = this.service.getTombStores(); + + // Verify the results + assertThat(result.size() == 1); + this.service.removeStore(result.get(0).getId()); + List stores = this.service.getStores(); + assertThat(stores.size() == 0); + } + + + @Test + public void testAllocShards() throws Exception { + // Setup + try { + final Metapb.Graph graph = Metapb.Graph.newBuilder() + .setGraphName("graphName") + .setGraphState( + Metapb.GraphState + .newBuilder() + .setMode( + Metapb.GraphMode.ReadWrite) + .setReason( + Metapb.GraphModeReason.Quota) + .build()) + .build(); + final List expectedResult = List.of( + Metapb.Shard.newBuilder().setStoreId(0L) + .setRole(Metapb.ShardRole.None).build()); + + // Configure PDConfig.getPartition(...). + final PDConfig.Partition partition = new PDConfig().new Partition(); + partition.setTotalCount(0); + partition.setMaxShardsPerStore(0); + partition.setShardCount(0); + + // Run the test + final List result = this.service.allocShards(graph, 0); + } catch (Exception e) { + + } + + } + + @Test + public void testReallocShards() throws Exception { + // Setup + try { + final Metapb.ShardGroup shardGroup = Metapb.ShardGroup.newBuilder() + .setId(0) + .addShards( + Metapb.Shard + .newBuilder() + .setStoreId( + 0L) + .setRole( + Metapb.ShardRole.None) + .build()) + .setState( + Metapb.PartitionState.PState_None) + .build(); + final List expectedResult = List.of( + Metapb.Shard.newBuilder().setStoreId(0L) + .setRole(Metapb.ShardRole.None).build()); + + // Configure PDConfig.getPartition(...). + final PDConfig.Partition partition = new PDConfig().new Partition(); + partition.setTotalCount(0); + partition.setMaxShardsPerStore(0); + partition.setShardCount(0); + when(this.config.getPartition()).thenReturn(partition); + + // Run the test + final List result = this.service.reallocShards(shardGroup); + + // Verify the results + assertThat(result).isEqualTo(expectedResult); + } catch (Exception e) { + + } + + } + + @Test + public void testUpdateShardGroup() { + try { + final List shards = List.of( + Metapb.Shard.newBuilder().setStoreId(0L) + .setRole(Metapb.ShardRole.None).build()); + + // Run the test + this.service.updateShardGroup(0, shards, 0, 0); + } catch (Exception e) { + + } + } + + @Test + public void testUpdateShardGroupState() throws Exception { + try { + this.service.updateShardGroupState(0, Metapb.PartitionState.PState_None); + } catch (Exception e) { + + } + } + + @Test + public void testHeartBeat() throws Exception { + // Setup + try { + final Metapb.StoreStats storeStats = Metapb.StoreStats.newBuilder() + .setStoreId( + 0L) + .setPartitionCount( + 0) + .addGraphStats( + Metapb.GraphStats + .newBuilder() + .setGraphName( + "value") + .setApproximateSize( + 0L) + .setRole( + Metapb.ShardRole.None) + .build()) + .build(); + final Metapb.ClusterStats expectedResult = Metapb.ClusterStats + .newBuilder().setState(Metapb.ClusterState.Cluster_OK) + .setMessage("message").setTimestamp(0L).build(); + when(this.config.getMinStoreCount()).thenReturn(0); + + // Configure PDConfig.getPartition(...). + final PDConfig.Partition partition = new PDConfig().new Partition(); + partition.setTotalCount(0); + partition.setMaxShardsPerStore(0); + partition.setShardCount(0); + when(this.config.getPartition()).thenReturn(partition); + + // Run the test + final Metapb.ClusterStats result = this.service.heartBeat(storeStats); + + // Verify the results + assertThat(result).isEqualTo(expectedResult); + } catch (Exception e) { + + } + } + + + @Test + public void testUpdateClusterStatus1() { + // Setup + final Metapb.ClusterStats expectedResult = Metapb.ClusterStats + .newBuilder().setState(Metapb.ClusterState.Cluster_OK) + .setMessage("message").setTimestamp(0L).build(); + + // Run the test + final Metapb.ClusterStats result = this.service.updateClusterStatus( + Metapb.ClusterState.Cluster_OK); + } + + @Test + public void testUpdateClusterStatus2() { + // Setup + final Metapb.ClusterStats expectedResult = Metapb.ClusterStats + .newBuilder().setState(Metapb.ClusterState.Cluster_OK) + .setMessage("message").setTimestamp(0L).build(); + + // Run the test + final Metapb.ClusterStats result = this.service.updateClusterStatus( + Metapb.PartitionState.PState_None); + } + + @Test + public void testCheckStoreStatus() { + // Setup + // Run the test + this.service.checkStoreStatus(); + + // Verify the results + } + + @Test + public void testAddStatusListener() { + // Setup + final StoreStatusListener mockListener = mock( + StoreStatusListener.class); + + // Run the test + this.service.addStatusListener(mockListener); + + // Verify the results + } + + @Test + public void testOnStoreStatusChanged() { + // Setup + final Metapb.Store store = Metapb.Store.newBuilder().setId(0L) + .setAddress("address") + .setRaftAddress("raftAddress") + .addLabels(Metapb.StoreLabel + .newBuilder() + .build()) + .setVersion("version").setState( + Metapb.StoreState.Unknown).setStartTimestamp(0L) + .setDeployPath("deployPath") + .setLastHeartbeat(0L).setStats( + Metapb.StoreStats.newBuilder().setStoreId(0L) + .setPartitionCount(0).addGraphStats( + Metapb.GraphStats.newBuilder() + .setGraphName("value") + .setApproximateSize(0L) + .setRole(Metapb.ShardRole.None) + .build()).build()) + .setDataVersion(0).setCores(0) + .setDataPath("dataPath").build(); + + // Verify the results + } + + @Test + public void testOnShardGroupSplit() { + // Setup + final Metapb.ShardGroup shardGroup = Metapb.ShardGroup.newBuilder() + .setId(0) + .addShards( + Metapb.Shard + .newBuilder() + .setStoreId( + 0L) + .setRole( + Metapb.ShardRole.None) + .build()) + .setState( + Metapb.PartitionState.PState_None) + .build(); + final List newShardGroups = List.of( + Metapb.ShardGroup.newBuilder().setId(0).addShards( + Metapb.Shard.newBuilder().setStoreId(0L) + .setRole(Metapb.ShardRole.None).build()) + .setState(Metapb.PartitionState.PState_None) + .build()); + final Consumer mockTask = mock(Consumer.class); + + // Verify the results + } + + @Test + public void testCheckStoreCanOffline() { + // Setup + final Metapb.Store currentStore = Metapb.Store.newBuilder().setId(0L) + .setAddress("address") + .setRaftAddress( + "raftAddress") + .addLabels( + Metapb.StoreLabel + .newBuilder() + .build()) + .setVersion("version") + .setState( + Metapb.StoreState.Unknown) + .setStartTimestamp(0L) + .setDeployPath( + "deployPath") + .setLastHeartbeat(0L) + .setStats( + Metapb.StoreStats + .newBuilder() + .setStoreId( + 0L) + .setPartitionCount( + 0) + .addGraphStats( + Metapb.GraphStats + .newBuilder() + .setGraphName( + "value") + .setApproximateSize( + 0L) + .setRole( + Metapb.ShardRole.None) + .build()) + .build()) + .setDataVersion(0) + .setCores(0) + .setDataPath("dataPath") + .build(); + // Run the test + final boolean result = this.service.checkStoreCanOffline(currentStore); + + // Verify the results + assertThat(result).isTrue(); + } + + @Test + public void testShardGroupsDbCompaction() throws Exception { + // Setup + // Run the test + try { + this.service.shardGroupsDbCompaction(0, "tableName"); + } catch (Exception e) { + + } + + // Verify the results + } + + @Test + public void testGetQuota() throws Exception { + // Setup + // Run the test + try { + this.service.getQuota(); + } catch (Exception e) { + + } + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/TaskScheduleServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/TaskScheduleServiceTest.java new file mode 100644 index 0000000000..7966bb7125 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/TaskScheduleServiceTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.service; + +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hugegraph.pd.TaskScheduleService; +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.junit.Before; +import org.junit.Test; + +public class TaskScheduleServiceTest extends PdTestBase { + + TaskScheduleService service; + + @Before + public void init() { + this.service = getTaskService(); + } + + @Test + public void testStoreOffline() { + + } + + public void testPatrolStores() { + + } + + public void testPatrolPartitions() { + + } + + public void testBalancePartitionShard() { + + } + + @Test + public void testBalancePartitionLeader() throws PDException { + + var list = new ArrayList(); + for (int i = 0; i < 6; i++) { + getStoreNodeService().getStoreInfoMeta().updateShardGroup(genShardGroup(i)); + list.add(genPartition(i)); + } + + getPdConfig().getPartition().setShardCount(3); + + getPartitionService().updatePartition(list); + var rst = this.service.balancePartitionLeader(true); + assertTrue(rst.size() > 0); + // recover + getPdConfig().getPartition().setShardCount(1); + getStoreNodeService().getStoreInfoMeta().removeAll(); + } + + public void testSplitPartition() { + + } + + public void testSplitPartition2() { + + } + + public void testCanAllPartitionsMovedOut() { + + } + + private Metapb.ShardGroup genShardGroup(int groupId) { + return Metapb.ShardGroup.newBuilder() + .setId(groupId) + .addAllShards(genShards()) + .build(); + } + + private Metapb.Partition genPartition(int groupId) { + return Metapb.Partition.newBuilder() + .setId(groupId) + .setState(Metapb.PartitionState.PState_Normal) + .setGraphName("graph1") + .setStartKey(groupId * 10) + .setEndKey(groupId * 10 + 10) + .build(); + } + + private List genShards() { + return List.of( + Metapb.Shard.newBuilder().setStoreId(1).setRole(Metapb.ShardRole.Leader).build(), + Metapb.Shard.newBuilder().setStoreId(2).setRole(Metapb.ShardRole.Follower).build(), + Metapb.Shard.newBuilder().setStoreId(3).setRole(Metapb.ShardRole.Follower).build()); + } + +} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImplTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImplTest.java new file mode 100644 index 0000000000..342ac9bc44 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImplTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.store; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; + +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.config.PDConfig; +import org.junit.Assert; +import org.junit.BeforeClass; + +public class HgKVStoreImplTest { + static final String testPath = "tmp/test"; + static PDConfig pdConfig; + + @BeforeClass + public static void init() throws IOException { + File testFile = new File(testPath); + if (testFile.exists()) { + FileUtils.deleteDirectory(testFile); + } + FileUtils.forceMkdir(testFile); + pdConfig = new PDConfig() {{ + setDataPath(testPath); + }}; + } + + // @Test + public void Test() throws PDException { + HgKVStore kvStore = new HgKVStoreImpl(); + kvStore.init(pdConfig); + + { + byte[] key = "hello".getBytes(); + byte[] value = "pd".getBytes(); + kvStore.put(key, value); + } + for (int i = 0; i < 100; i++) { + byte[] key = String.format("k%03d", i).getBytes(); + byte[] value = ("value" + i).getBytes(); + kvStore.put(key, value); + } + + Assert.assertEquals(100, kvStore.scanPrefix("k".getBytes()).size()); + + kvStore.removeByPrefix("k".getBytes()); + Assert.assertEquals(0, kvStore.scanPrefix("k".getBytes()).size()); + } + + // @Test + public void TestSnapshot() throws PDException { + HgKVStore kvStore = new HgKVStoreImpl(); + kvStore.init(pdConfig); + + // put 100 data + for (int i = 0; i < 100; i++) { + byte[] key = String.format("k%03d", i).getBytes(); + byte[] value = ("value" + i).getBytes(); + kvStore.put(key, value); + } + Assert.assertEquals(100, kvStore.scanPrefix("k".getBytes()).size()); + + // save snapshot + String snapshotPath = Paths.get(testPath, "snapshot").toString(); + kvStore.saveSnapshot(snapshotPath); + + // put another 100 data + for (int i = 100; i < 200; i++) { + byte[] key = String.format("k%03d", i).getBytes(); + byte[] value = ("value" + i).getBytes(); + kvStore.put(key, value); + } + Assert.assertEquals(200, kvStore.scanPrefix("k".getBytes()).size()); + + // load snapshot + kvStore.loadSnapshot(snapshotPath); + Assert.assertEquals(100, kvStore.scanPrefix("k".getBytes()).size()); + + // put another 100 data + for (int i = 100; i < 200; i++) { + byte[] key = String.format("k%03d", i).getBytes(); + byte[] value = ("value" + i).getBytes(); + kvStore.put(key, value); + } + Assert.assertEquals(200, kvStore.scanPrefix("k".getBytes()).size()); + } +} diff --git a/hugegraph-pd/hg-pd-test/src/main/resources/log4j2.xml b/hugegraph-pd/hg-pd-test/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..82f1bce8c4 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/resources/log4j2.xml @@ -0,0 +1,139 @@ + + + + + + + + logs + hg-pd-test + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hugegraph-pd/pom.xml b/hugegraph-pd/pom.xml index 72939bbce5..49fd28f384 100644 --- a/hugegraph-pd/pom.xml +++ b/hugegraph-pd/pom.xml @@ -35,12 +35,12 @@ hg-pd-grpc hg-pd-common hg-pd-client + hg-pd-test - From a5fd93fd7b9fa0d341f1ee58b0b0537fa45c76cb Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Mon, 26 Feb 2024 20:03:25 +0800 Subject: [PATCH 05/17] chore: adapt hg-pd-common tests --- .github/workflows/pd-store.yml | 51 ++ hugegraph-pd/hg-pd-test/pom.xml | 122 +-- .../hugegraph/pd/MonitorServiceTest.java | 112 --- .../hugegraph/pd/PartitionCacheTest.java | 102 --- .../hugegraph/pd/PartitionServiceTest.java | 45 - .../hugegraph/pd/StoreNodeServiceTest.java | 485 ---------- .../hugegraph/pd/StoreRegisterTest.java | 135 --- .../org/apache/hugegraph/pd/UnitTestBase.java | 31 - .../hugegraph/pd/client/BaseClientTest.java | 44 - .../pd/client/DiscoveryClientImplTest.java | 147 ---- .../pd/client/DiscoveryClientTest.java | 79 -- .../hugegraph/pd/client/KvClientTest.java | 121 --- .../pd/client/LicenseClientImplTest.java | 127 --- .../pd/client/PDClientSuiteTest.java | 36 - .../hugegraph/pd/client/PDClientTest.java | 418 --------- .../hugegraph/pd/client/PDPulseTest.java | 110 --- .../hugegraph/pd/client/PDWatchTest.java | 86 -- .../pd/client/test/HgPDTestUtil.java | 92 -- .../pd/clitools/BaseCliToolsTest.java | 34 - .../pd/clitools/CliToolsSuiteTest.java | 35 - .../hugegraph/pd/clitools/MainTest.java | 89 -- .../hugegraph/pd/common/CommonSuiteTest.java | 5 - .../pd/common/MetadataKeyHelperTest.java | 217 ----- .../pd/common/PartitionCacheTest.java | 14 +- .../hugegraph/pd/core/BaseCoreTest.java | 74 -- .../hugegraph/pd/core/PDCoreSuiteTest.java | 37 - .../pd/core/StoreNodeServiceTest.java | 119 --- .../pd/core/meta/MetadataKeyHelperTest.java | 34 - .../hugegraph/pd/grpc/BaseGrpcTest.java | 36 - .../hugegraph/pd/grpc/GrpcSuiteTest.java | 33 - .../hugegraph/pd/service/BaseServerTest.java | 57 -- .../pd/service/ConfigServiceTest.java | 106 --- .../hugegraph/pd/service/IdServiceTest.java | 109 --- .../hugegraph/pd/service/KvServiceTest.java | 60 -- .../hugegraph/pd/service/LogServiceTest.java | 54 -- .../pd/service/PartitionServiceTest.java | 133 --- .../hugegraph/pd/service/PdTestBase.java | 219 ----- .../hugegraph/pd/service/RestApiTest.java | 120 --- .../hugegraph/pd/service/ServerSuiteTest.java | 42 - .../service/StoreMonitorDataServiceTest.java | 82 -- .../pd/service/StoreNodeServiceNewTest.java | 64 -- .../pd/service/StoreServiceTest.java | 833 ------------------ .../pd/service/TaskScheduleServiceTest.java | 112 --- .../hugegraph/pd/store/HgKVStoreImplTest.java | 105 --- hugegraph-pd/pom.xml | 98 +-- .../src/assembly/travis/start-pd.sh | 26 + 46 files changed, 94 insertions(+), 5196 deletions(-) create mode 100644 .github/workflows/pd-store.yml delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/MonitorServiceTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/PartitionCacheTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/PartitionServiceTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/StoreNodeServiceTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/StoreRegisterTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/UnitTestBase.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/BaseClientTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImplTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/KvClientTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/LicenseClientImplTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientSuiteTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDPulseTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDWatchTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/test/HgPDTestUtil.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/BaseCliToolsTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/CliToolsSuiteTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/MainTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/MetadataKeyHelperTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/BaseCoreTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreSuiteTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/StoreNodeServiceTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/meta/MetadataKeyHelperTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/grpc/BaseGrpcTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/grpc/GrpcSuiteTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/BaseServerTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/ConfigServiceTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/IdServiceTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/KvServiceTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/LogServiceTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/PartitionServiceTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/PdTestBase.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/RestApiTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/ServerSuiteTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreMonitorDataServiceTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreNodeServiceNewTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreServiceTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/TaskScheduleServiceTest.java delete mode 100644 hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImplTest.java create mode 100644 hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh diff --git a/.github/workflows/pd-store.yml b/.github/workflows/pd-store.yml new file mode 100644 index 0000000000..65fb3ccc9c --- /dev/null +++ b/.github/workflows/pd-store.yml @@ -0,0 +1,51 @@ +name: "pd-store" + +on: + push: + branches: + - master + - 'release-*' + - 'test-*' + pull_request: + +jobs: + pd: + runs-on: ubuntu-latest + env: + USE_STAGE: 'true' # Whether to include the stage repository. + TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis + REPORT_DIR: target/site/jacoco + + steps: + - name: Install JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'zulu' + + - name: Cache Maven packages + uses: actions/cache@v3 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: use staged maven repo settings + if: ${{ env.USE_STAGE == 'true' }} + run: | + cp $HOME/.m2/settings.xml /tmp/settings.xml + mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml + + - name: Run common test + run: | + mvn test -pl hugegraph-pd/hg-pd-test -am -P pd-common-test + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3.0.0 + with: + file: ${{ env.REPORT_DIR }}/*.xml diff --git a/hugegraph-pd/hg-pd-test/pom.xml b/hugegraph-pd/hg-pd-test/pom.xml index 19e1855d8c..31c0fd889d 100644 --- a/hugegraph-pd/hg-pd-test/pom.xml +++ b/hugegraph-pd/hg-pd-test/pom.xml @@ -96,7 +96,7 @@ org.apache.hugegraph - hg-pd-client + hg-pd-common ${revision} @@ -105,11 +105,6 @@ gson 2.8.9 - - org.apache.hugegraph - hg-pd-grpc - ${revision} - commons-io commons-io @@ -131,28 +126,6 @@ 2.13.0 - - - org.springframework.boot spring-boot-starter-test @@ -164,46 +137,6 @@ - - org.apache.hugegraph - hg-pd-clitools - ${revision} - - - org.apache.hugegraph - hg-pd-common - ${revision} - - - - org.apache.hugegraph - hg-pd-core - ${revision} - - - org.apache.hugegraph - hg-pd-dist - ${revision} - compile - - - org.apache.hugegraph - hg-pd-service - ${revision} - - - org.powermock @@ -223,19 +156,24 @@ org.powermock powermock-module-junit4 - 2.0.0-RC.3 + ${powermock.version} compile org.powermock powermock-api-mockito2 - 2.0.0-RC.3 + ${powermock.version} compile + + org.apache.commons + commons-lang3 + 3.14.0 + compile + - @@ -243,42 +181,6 @@ maven-surefire-plugin 2.20 - - pd-client-test - - ${basedir}/src/main/java/ - - ${basedir}/target/classes/ - - - **/PDClientSuiteTest.java - - - - - pd-core-test - - ${basedir}/src/main/java/ - - ${basedir}/target/classes/ - - - **/PDCoreSuiteTest.java - - - - - pd-cli-tools-test - - ${basedir}/src/main/java/ - - ${basedir}/target/classes/ - - - **/CliToolsSuiteTest.java - - - pd-common-test @@ -292,14 +194,14 @@ - pd-service-test + pd-client-test ${basedir}/src/main/java/ ${basedir}/target/classes/ - **/ServerSuiteTest.java + **/PDClientSuiteTest.java @@ -343,7 +245,6 @@ org/apache/hugegraph/pd/grpc/**/*.class org/apache/hugegraph/pd/raft/*.class **/RaftKVStore.class - @@ -355,5 +256,4 @@ - diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/MonitorServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/MonitorServiceTest.java deleted file mode 100644 index 25272127fd..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/MonitorServiceTest.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd; - -import java.util.concurrent.ExecutionException; - -import org.apache.hugegraph.pd.common.PDException; -import org.apache.hugegraph.pd.config.PDConfig; -import org.apache.hugegraph.pd.grpc.Metapb; -import org.junit.Assert; -import org.junit.BeforeClass; - -// import org.junit.Test; - -public class MonitorServiceTest { - static PDConfig pdConfig; - - @BeforeClass - public static void init() throws ExecutionException, InterruptedException { - pdConfig = new PDConfig() {{ - this.setClusterId(100); - this.setPatrolInterval(1); - }}; - - //pdConfig.setEtcd(new PDConfig().new Etcd() {{ - // this.setAddress("http://localhost:2379"); - // - //}}); - pdConfig.setStore(new PDConfig().new Store() {{ - this.setMaxDownTime(1); - this.setKeepAliveTimeout(5); - }}); - - pdConfig.setPartition(new PDConfig().new Partition() {{ - this.setShardCount(3); - this.setTotalCount(10); - }}); - - clearClusterData(); - } - - public static void clearClusterData() throws ExecutionException, InterruptedException { - //Client client = Client.builder().endpoints(pdConfig.getEtcd().getAddress()).build(); - //KV kvClient = client.getKVClient(); - // - //ByteSequence key = ByteSequence.from("HUGEGRAPH/" + pdConfig.getClusterId(), Charset - // .forName("utf-8")); - //CompletableFuture rsp = kvClient.delete(key, DeleteOption.newBuilder() - // .isPrefix(true).build()); - //System.out.println("删除数量 : " + rsp.get().getDeleted()); - //kvClient.close(); - //client.close(); - } - - // @Test - public void testPatrolStores() throws PDException, InterruptedException { - StoreNodeService storeService = new StoreNodeService(pdConfig); - PartitionService partitionService = new PartitionService(pdConfig, storeService); - TaskScheduleService monitorService = - new TaskScheduleService(pdConfig, storeService, partitionService); - storeService.init(partitionService); - partitionService.init(); - monitorService.init(); - - int count = 6; - Metapb.Store[] stores = new Metapb.Store[count]; - for (int i = 0; i < count; i++) { - Metapb.Store store = Metapb.Store.newBuilder() - .setId(0) - .setAddress(String.valueOf(i)) - .setDeployPath("/data") - .addLabels(Metapb.StoreLabel.newBuilder() - .setKey("namespace") - .setValue("default") - .build()) - .build(); - stores[i] = storeService.register(store); - System.out.println("新注册store， id = " + Long.toHexString(stores[i].getId())); - } - Metapb.Graph graph = Metapb.Graph.newBuilder() - .setGraphName("defaultGH") - - .setPartitionCount(10) - .build(); - partitionService.updateGraph(graph); - Thread.sleep(10000); - count = 0; - count += storeService.getStores("").stream() - .filter(store -> store.getState() == Metapb.StoreState.Tombstone) - .count(); - - Assert.assertEquals(6, count); - - } - - -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/PartitionCacheTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/PartitionCacheTest.java deleted file mode 100644 index 71efa70ecd..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/PartitionCacheTest.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.hugegraph.pd.common.KVPair; -import org.apache.hugegraph.pd.common.PartitionCache; -import org.apache.hugegraph.pd.grpc.Metapb; - -import com.google.common.collect.Range; -import com.google.common.collect.RangeMap; -import com.google.common.collect.TreeRangeMap; - -public class PartitionCacheTest { - - // @Test - public void test() { - PartitionCache cache = new PartitionCache(); - for (int i = 0; i < 10; i++) { - KVPair partShards = - new KVPair<>(Metapb.Partition.newBuilder() - .setStartKey(i * 10) - .setEndKey((i + 1) * 10) - .build(), null); - cache.updatePartition("aa", i, partShards.getKey()); - } - - for (int i = 0; i < 100; i++) { - KVPair partShards = cache.getPartitionByCode("aa", i); - System.out.println(" " + i + " " + partShards.getKey().getStartKey()); - } - } - - - // @Test - public void test1() { - Map> keyToPartIdCache = new HashMap<>(); - // graphName + PartitionID组成key - Map> partitionCache = new HashMap<>(); - - // 缓存全部Store，用于全库查询，需要优化 - Map> allStoresCache = new HashMap<>(); - - keyToPartIdCache.put("a", TreeRangeMap.create()); - - keyToPartIdCache.get("a") - .put(Range.closedOpen(1L, 2L), 1); - - allStoresCache.put("a", new ArrayList<>()); - allStoresCache.get("a").add(Metapb.Store.newBuilder().setId(34).build()); - - - Map> keyToPartIdCache2 = - cloneKeyToPartIdCache(keyToPartIdCache); - System.out.println(keyToPartIdCache2.size()); - } - - public Map> cloneKeyToPartIdCache( - Map> cache) { - Map> cacheClone = new HashMap<>(); - cache.forEach((k1, v1) -> { - cacheClone.put(k1, TreeRangeMap.create()); - v1.asMapOfRanges().forEach((k2, v2) -> { - cacheClone.get(k1).put(k2, v2); - }); - }); - return cacheClone; - } - - public Map> - clonePartitionCache(Map> cache) { - Map> cacheClone = new HashMap<>(); - cacheClone.putAll(cache); - return cacheClone; - } - - public Map> - cloneStoreCache(Map> cache) { - Map> cacheClone = new HashMap<>(); - cacheClone.putAll(cache); - return cacheClone; - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/PartitionServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/PartitionServiceTest.java deleted file mode 100644 index 82fd4c9bdb..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/PartitionServiceTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.hugegraph.pd.grpc.Metapb; -import org.junit.Test; - -public class PartitionServiceTest { - @Test - public void testPartitionHeartbeat() { - List shardList = new ArrayList<>(); - shardList.add(Metapb.Shard.newBuilder().setStoreId(1).build()); - shardList.add(Metapb.Shard.newBuilder().setStoreId(2).build()); - shardList.add(Metapb.Shard.newBuilder().setStoreId(3).build()); - shardList = new ArrayList<>(shardList); - Metapb.PartitionStats stats = Metapb.PartitionStats.newBuilder() - .addAllShard(shardList).build(); - List shardList2 = new ArrayList<>(stats.getShardList()); - Collections.shuffle(shardList2); - shardList2.forEach(shard -> { - System.out.println(shard.getStoreId()); - }); - - - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/StoreNodeServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/StoreNodeServiceTest.java deleted file mode 100644 index c8f0ce39e5..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/StoreNodeServiceTest.java +++ /dev/null @@ -1,485 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.commons.io.FileUtils; -import org.apache.hugegraph.pd.common.PDException; -import org.apache.hugegraph.pd.config.PDConfig; -import org.apache.hugegraph.pd.grpc.Metapb; -import org.apache.hugegraph.pd.grpc.pulse.ChangeShard; -import org.apache.hugegraph.pd.grpc.pulse.CleanPartition; -import org.apache.hugegraph.pd.grpc.pulse.DbCompaction; -import org.apache.hugegraph.pd.grpc.pulse.MovePartition; -import org.apache.hugegraph.pd.grpc.pulse.PartitionKeyRange; -import org.apache.hugegraph.pd.grpc.pulse.SplitPartition; -import org.apache.hugegraph.pd.grpc.pulse.TransferLeader; -import org.junit.Assert; -import org.junit.BeforeClass; - -public class StoreNodeServiceTest { - static PDConfig pdConfig; - - @BeforeClass - public static void init() throws Exception { - String path = "tmp/unitTest"; - deleteDirectory(new File(path)); - pdConfig = new PDConfig() {{ - this.setClusterId(100); - this.setInitialStoreList( - "127.0.0.1:8500,127.0.0.1:8501,127.0.0.1:8502,127.0.0.1:8503,127.0.0.1:8504," + - "127.0.0.1:8505"); - }}; - - pdConfig.setStore(new PDConfig().new Store() {{ - this.setMaxDownTime(3600); - this.setKeepAliveTimeout(3600); - }}); - - pdConfig.setPartition(new PDConfig().new Partition() {{ - this.setShardCount(3); - this.setMaxShardsPerStore(3); - }}); - pdConfig.setRaft(new PDConfig().new Raft() {{ - this.setEnable(false); - }}); - pdConfig.setDiscovery(new PDConfig().new Discovery()); - pdConfig.setDataPath(path); - ConfigService configService = new ConfigService(pdConfig); - pdConfig = configService.loadConfig(); - } - - public static byte[] intToByteArray(int i) { - byte[] result = new byte[4]; - result[0] = (byte) ((i >> 24) & 0xFF); - result[1] = (byte) ((i >> 16) & 0xFF); - result[2] = (byte) ((i >> 8) & 0xFF); - result[3] = (byte) (i & 0xFF); - return result; - } - - public static void deleteDirectory(File dir) { - try { - FileUtils.deleteDirectory(dir); - } catch (IOException e) { - System.out.printf("Failed to start ....,%s%n", e.getMessage()); - } - } - - // @Test - public void testStoreNodeService() throws PDException { - Assert.assertEquals(pdConfig.getPartition().getTotalCount(), - (long) pdConfig.getInitialStoreMap().size() * - pdConfig.getPartition().getMaxShardsPerStore() - / pdConfig.getPartition().getShardCount()); - StoreNodeService storeService = new StoreNodeService(pdConfig); - int count = 6; - Metapb.Store[] stores = new Metapb.Store[count]; - for (int i = 0; i < count; i++) { - Metapb.Store store = Metapb.Store.newBuilder() - .setId(0) - .setAddress("127.0.0.1:850" + i) - .setDeployPath("/data") - .addLabels(Metapb.StoreLabel.newBuilder() - .setKey("namespace") - .setValue("default") - .build()) - .build(); - stores[i] = storeService.register(store); - System.out.println("新注册store， id = " + stores[i].getId()); - } - Assert.assertEquals(count, storeService.getStores("").size()); - - for (Metapb.Store store : stores) { - Metapb.StoreStats stats = Metapb.StoreStats.newBuilder() - .setStoreId(store.getId()) - .build(); - storeService.heartBeat(stats); - } - - Assert.assertEquals(6, storeService.getActiveStores("").size()); - - Metapb.Graph graph = Metapb.Graph.newBuilder() - .setGraphName("defaultGH") - .setPartitionCount(10) - .build(); - // 分配shard - List shards = storeService.allocShards(graph, 1); - - - Assert.assertEquals(3, shards.size()); - - Assert.assertEquals(pdConfig.getPartition().getTotalCount(), - storeService.getShardGroups().size()); // 设置leader - Metapb.Shard leader = Metapb.Shard.newBuilder(shards.get(0)) - .setRole(Metapb.ShardRole.Leader).build(); - shards = new ArrayList<>(shards); - shards.set(0, leader); - // 增加shard - pdConfig.getPartition().setShardCount(5); - - Metapb.ShardGroup shardGroup = Metapb.ShardGroup.newBuilder() - .setId(1) - .addAllShards(shards).build(); - shards = storeService.reallocShards(shardGroup); - - Assert.assertEquals(5, shards.size()); - // 减少shard - pdConfig.getPartition().setShardCount(3); - shards = storeService.reallocShards(shardGroup); - Assert.assertEquals(3, shards.size()); - // 包含leader，leader不能被删除 - Assert.assertTrue(shards.contains(leader)); - - // 减少shard - pdConfig.getPartition().setShardCount(1); - graph = Metapb.Graph.newBuilder(graph).build(); - shards = storeService.reallocShards(shardGroup); - Assert.assertEquals(1, shards.size()); - // 包含leader，leader不能被删除 - Assert.assertTrue(shards.contains(leader)); - - for (Metapb.Store store : stores) { - storeService.removeStore(store.getId()); - } - Assert.assertEquals(0, storeService.getStores("").size()); - - - } - - // @Test - public void testSplitPartition() throws PDException { - StoreNodeService storeService = new StoreNodeService(pdConfig); - PartitionService partitionService = new PartitionService(pdConfig, storeService); - - storeService.init(partitionService); - partitionService.addInstructionListener(new PartitionInstructionListener() { - - @Override - public void changeShard(Metapb.Partition partition, ChangeShard changeShard) throws - PDException { - - } - - @Override - public void transferLeader(Metapb.Partition partition, - TransferLeader transferLeader) throws PDException { - - } - - @Override - public void splitPartition(Metapb.Partition partition, - SplitPartition splitPartition) throws PDException { - splitPartition.getNewPartitionList().forEach(p -> { - System.out.println("SplitPartition " + p.getId() + " " + p.getStartKey() + "," + - p.getEndKey()); - }); - } - - @Override - public void dbCompaction(Metapb.Partition partition, DbCompaction dbCompaction) throws - PDException { - - } - - @Override - public void movePartition(Metapb.Partition partition, - MovePartition movePartition) throws PDException { - - } - - @Override - public void cleanPartition(Metapb.Partition partition, - CleanPartition cleanPartition) throws PDException { - - } - - @Override - public void changePartitionKeyRange(Metapb.Partition partition, - PartitionKeyRange partitionKeyRange) throws - PDException { - - } - }); - int count = 6; - Metapb.Store[] stores = new Metapb.Store[count]; - for (int i = 0; i < count; i++) { - Metapb.Store store = Metapb.Store.newBuilder() - .setId(0) - .setAddress("127.0.0.1:850" + i) - .setDeployPath("/data") - .addLabels(Metapb.StoreLabel.newBuilder() - .setKey("namespace") - .setValue("default") - .build()) - .build(); - stores[i] = storeService.register(store); - System.out.println("新注册store， id = " + Long.toHexString(stores[i].getId())); - } - Assert.assertEquals(count, storeService.getStores().size()); - - Metapb.Graph graph = Metapb.Graph.newBuilder() - .setGraphName("defaultGH") - .build(); - Metapb.PartitionShard ptShard = - partitionService.getPartitionByCode(graph.getGraphName(), 0); - System.out.println(ptShard.getPartition().getId()); - { - Metapb.Partition pt = ptShard.getPartition(); - System.out.println(pt.getId() + " " + pt.getStartKey() + "," + pt.getEndKey()); - } - - Assert.assertEquals(6, storeService.getShardGroups().size()); - // storeService.splitShardGroups(ptShard.getPartition().getId(), 4); - Assert.assertEquals(9, storeService.getShardGroups().size()); - storeService.getShardGroups().forEach(shardGroup -> { - System.out.println("shardGroup id = " + shardGroup.getId()); - }); - } - - // @Test - public void testPartitionService() throws PDException, ExecutionException, - InterruptedException { - StoreNodeService storeService = new StoreNodeService(pdConfig); - int count = 6; - Metapb.Store[] stores = new Metapb.Store[count]; - for (int i = 0; i < count; i++) { - Metapb.Store store = Metapb.Store.newBuilder() - .setId(0) - .setAddress(String.valueOf(i)) - .setDeployPath("/data") - .addLabels(Metapb.StoreLabel.newBuilder() - .setKey("namespace") - .setValue("default") - .build()) - .build(); - stores[i] = storeService.register(store); - System.out.println("新注册store， id = " + Long.toHexString(stores[i].getId())); - } - Assert.assertEquals(count, storeService.getStores("").size()); - - - PartitionService partitionService = new PartitionService(pdConfig, storeService); - - Metapb.Graph graph = Metapb.Graph.newBuilder() - .setGraphName("defaultGH") - - .setPartitionCount(10) - .build(); - // 申请分区 - Metapb.PartitionShard[] partitions = new Metapb.PartitionShard[10]; - for (int i = 0; i < partitions.length; i++) { - partitions[i] = - partitionService.getPartitionShard(graph.getGraphName(), intToByteArray(i)); - Assert.assertEquals(3, storeService.getShardGroup(i).getShardsCount()); - } - System.out.println( - "分区数量： " + partitionService.getPartitions(graph.getGraphName()).size()); - - int[] caseNo = {0}; //1 测试增加shard, 2 //测试store下线 - - Metapb.Shard leader = null; - int[] finalCaseNo = caseNo; - - partitionService.addInstructionListener(new PartitionInstructionListener() { - - @Override - public void changeShard(Metapb.Partition partition, ChangeShard changeShard) throws - PDException { - switch (finalCaseNo[0]) { - case 2: - Assert.assertEquals(5, storeService.getShardGroup(partition.getId()) - .getShardsCount()); - break; - case 3: - storeService.getShardGroup(partition.getId()).getShardsList() - .forEach(shard -> { - Assert.assertNotEquals(shard.getStoreId(), - stores[0].getId()); - }); - break; - } - - } - - @Override - public void transferLeader(Metapb.Partition partition, TransferLeader transferLeader) { - - } - - @Override - public void splitPartition(Metapb.Partition partition, SplitPartition splitPartition) { - } - - @Override - public void dbCompaction(Metapb.Partition partition, DbCompaction dbCompaction) throws - PDException { - - } - - @Override - public void movePartition(Metapb.Partition partition, - MovePartition movePartition) throws PDException { - - } - - @Override - public void cleanPartition(Metapb.Partition partition, - CleanPartition cleanPartition) throws PDException { - - } - - @Override - public void changePartitionKeyRange(Metapb.Partition partition, - PartitionKeyRange partitionKeyRange) - throws PDException { - - } - }); - Metapb.Partition partition = partitions[0].getPartition(); - leader = Metapb.Shard.newBuilder( - storeService.getShardGroup(partition.getId()).getShardsList().get(0)).build(); - Metapb.Shard finalLeader = leader; - partitionService.addStatusListener(new PartitionStatusListener() { - @Override - public void onPartitionChanged(Metapb.Partition partition, - Metapb.Partition newPartition) { - - } - - @Override - public void onPartitionRemoved(Metapb.Partition partition) { - - } - }); - // 测试修改图 - caseNo[0] = 1; - partitionService.updateGraph(graph); - for (int i = 0; i < partitions.length; i++) { - partitions[i] = - partitionService.getPartitionShard(graph.getGraphName(), intToByteArray(i)); - Assert.assertEquals(3, storeService.getShardGroup(i).getShardsCount()); - } - - graph = Metapb.Graph.newBuilder(graph) - .setGraphName("defaultGH") - - .setPartitionCount(10) - .build(); - caseNo[0] = 2; - partitionService.updateGraph(graph); - - // 测试store离线 - caseNo[0] = 3; - partitionService.storeOffline(stores[0]); - - - Metapb.PartitionStats stats = Metapb.PartitionStats.newBuilder() - .addGraphName(partition.getGraphName()) - .setId(partition.getId()) - .setLeader( - Metapb.Shard.newBuilder(leader) - .setRole( - Metapb.ShardRole.Leader)) - .build(); - // 测试leader飘移 - caseNo[0] = 4; - partitionService.partitionHeartbeat(stats); - AtomicReference shard = new AtomicReference<>(); - Metapb.PartitionShard ss = - partitionService.getPartitionShardById(partition.getGraphName(), partition.getId()); - storeService.getShardList(partition.getId()).forEach(s -> { - if (s.getRole() == Metapb.ShardRole.Leader) { - Assert.assertNull(shard.get()); - shard.set(s); - } - }); - - Assert.assertEquals(leader.getStoreId(), shard.get().getStoreId()); - - } - - // @Test - public void testMergeGraphParams() throws PDException { - StoreNodeService storeService = new StoreNodeService(pdConfig); - PartitionService partitionService = new PartitionService(pdConfig, storeService); - - Metapb.Graph dfGraph = Metapb.Graph.newBuilder() - - .setPartitionCount( - pdConfig.getPartition().getTotalCount()) - - .build(); - - Metapb.Graph graph1 = Metapb.Graph.newBuilder() - .setGraphName("test") - .setPartitionCount(20) - - .build(); - - Metapb.Graph graph2 = Metapb.Graph.newBuilder() - .setGraphName("test") - .setPartitionCount(7).build(); - Metapb.Graph graph3 = Metapb.Graph.newBuilder() - .setGraphName("test") - .build(); - Metapb.Graph graph4 = Metapb.Graph.newBuilder() - .setGraphName("test") - .build(); - - Metapb.Graph graph = Metapb.Graph.newBuilder(dfGraph).mergeFrom(graph2).build(); - Assert.assertEquals(graph2.getGraphName(), graph.getGraphName()); - - Assert.assertEquals(graph2.getPartitionCount(), graph.getPartitionCount()); - - - graph = Metapb.Graph.newBuilder(dfGraph).mergeFrom(graph3).build(); - Assert.assertEquals(graph3.getGraphName(), graph.getGraphName()); - - Assert.assertEquals(dfGraph.getPartitionCount(), graph.getPartitionCount()); - - - graph = Metapb.Graph.newBuilder(dfGraph).mergeFrom(graph4).build(); - Assert.assertEquals(graph4.getGraphName(), graph.getGraphName()); - - Assert.assertEquals(dfGraph.getPartitionCount(), graph.getPartitionCount()); - - } - - // @Test - public void test() { - int[] n = new int[3]; - - - if (++n[2] > 1) { - System.out.println(n[2]); - } - if (++n[2] > 1) { - System.out.println(n[2]); - } - if (++n[2] > 1) { - System.out.println(n[2]); - } - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/StoreRegisterTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/StoreRegisterTest.java deleted file mode 100644 index 48877273fc..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/StoreRegisterTest.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd; - -import java.nio.charset.StandardCharsets; -import java.util.List; - -import org.apache.hugegraph.pd.client.PDClient; -import org.apache.hugegraph.pd.client.PDConfig; -import org.apache.hugegraph.pd.client.PDPulse; -import org.apache.hugegraph.pd.client.PDPulseImpl; -import org.apache.hugegraph.pd.common.KVPair; -import org.apache.hugegraph.pd.common.PDException; -import org.apache.hugegraph.pd.grpc.Metapb; -import org.apache.hugegraph.pd.grpc.pulse.PartitionHeartbeatRequest; -import org.apache.hugegraph.pd.grpc.pulse.PulseResponse; -import org.apache.hugegraph.pd.pulse.PulseServerNotice; -import org.junit.Assert; -import org.junit.BeforeClass; - -public class StoreRegisterTest { - private static PDClient pdClient; - private final String storeAddr = "localhost"; - private final String graphName = "default/hugegraph/g"; - private long storeId = 0; - - @BeforeClass - public static void beforeClass() throws Exception { - PDConfig config = PDConfig.of("localhost:8686"); - config.setEnableCache(true); - pdClient = PDClient.create(config); - } - - // @Test - public void testRegisterStore() throws PDException { - Metapb.Store store = Metapb.Store.newBuilder().setAddress(storeAddr).build(); - try { - storeId = pdClient.registerStore(store); - } catch (Exception e) { - e.printStackTrace(); - } - Assert.assertTrue("RegisterStore store_id = " + storeId, storeId != 0); - } - - // @Test - public void testGetStore() throws PDException { - testRegisterStore(); - Metapb.Store store = pdClient.getStore(storeId); - Assert.assertEquals(storeAddr, store.getAddress()); - System.out.println(store); - } - - // @Test - public void testGetActiveStores() throws PDException { - testRegisterStore(); - List stores = pdClient.getActiveStores(graphName); - stores.forEach((e) -> { - System.out.println("-------------------------------------"); - System.out.println(e); - }); - } - - - // @Test - public void testStoreHeartbeat() throws PDException { - testRegisterStore(); - Metapb.StoreStats stats = Metapb.StoreStats.newBuilder().setStoreId(storeId).build(); - pdClient.storeHeartbeat(stats); - List stores = pdClient.getActiveStores(graphName); - boolean exist = false; - for (Metapb.Store store : stores) { - if (store.getId() == storeId) { - exist = true; - break; - } - } - Assert.assertTrue(exist); - } - - - // @Test - public void testPartitionHeartbeat() throws InterruptedException, PDException { - testRegisterStore(); - PDPulse pdPulse = new PDPulseImpl(pdClient.getLeaderIp()); - - PDPulse.Notifier notifier = pdPulse.connectPartition( - new PDPulse.Listener() { - - @Override - public void onNext(PulseResponse response) { - - } - - @Override - public void onNotice(PulseServerNotice notice) { - - } - - @Override - public void onError(Throwable throwable) { - - } - - @Override - public void onCompleted() { - - } - }); - KVPair partShard = - pdClient.getPartition("test", "1".getBytes(StandardCharsets.UTF_8)); - notifier.notifyServer(PartitionHeartbeatRequest.newBuilder().setStates( - Metapb.PartitionStats.newBuilder().addGraphName("test") - .setId(partShard.getKey().getId()) - .setLeader(Metapb.Shard.newBuilder().setStoreId(1).build()))); - - - Thread.sleep(10000); - } - -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/UnitTestBase.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/UnitTestBase.java deleted file mode 100644 index 35ada84167..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/UnitTestBase.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd; - -import java.io.File; - -public class UnitTestBase { - public static boolean deleteDir(File dir) { - if (dir.isDirectory()) { - for (File file : dir.listFiles()) { - deleteDir(file); - } - } - return dir.delete(); - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/BaseClientTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/BaseClientTest.java deleted file mode 100644 index 3fca39ade9..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/BaseClientTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.client; - -import org.junit.After; -import org.junit.BeforeClass; -import org.junit.runner.RunWith; -import org.mockito.runners.MockitoJUnitRunner; - -@RunWith(MockitoJUnitRunner.class) -public class BaseClientTest { - public static PDClient pdClient; - public final String storeAddr = "localhost"; - public final String graphName = "default/hugegraph/g"; - public long storeId = 0; - - @BeforeClass - public static void beforeClass() throws Exception { - PDConfig config = PDConfig.of("localhost:8686"); -// PDConfig config = PDConfig.of("10.81.116.77:8986"); - config.setEnableCache(true); - pdClient = PDClient.create(config); - } - - @After - public void teardown() throws Exception { - // pass - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImplTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImplTest.java deleted file mode 100644 index 6d42c5ea73..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImplTest.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.client; - -import java.util.HashMap; -import java.util.Map; -import java.util.Vector; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.hugegraph.pd.grpc.discovery.NodeInfos; -import org.apache.hugegraph.pd.grpc.discovery.Query; -import org.junit.Assert; - -public class DiscoveryClientImplTest { - - private static final AtomicLong label = new AtomicLong(); - String address = "localhost:80"; - int delay = 1000; - int wait = delay * 3 + 500; - - // @Test - public void registerStore() throws InterruptedException { - - HashMap labels = new HashMap<>(); - - labels.put("metrics", "/actuator/prometheus"); - labels.put("target", "10.81.116.77:8520"); - labels.put("scheme", "http"); - labels.put("__relabeling", "http"); - labels.put("no_relabeling", "http"); - getClient("store", "address1", labels); - - labels.put("metrics", "/actuator/prometheus"); - labels.put("target", "10.81.116.78:8520"); - labels.put("scheme", "http"); - getClient("store", "address2", labels); - - labels.put("metrics", "/actuator/prometheus"); - labels.put("target", "10.81.116.79:8520"); - labels.put("scheme", "http"); - getClient("store", "address3", labels); - - labels.put("metrics", "/actuator/prometheus"); - labels.put("target", "10.81.116.78:8620"); - labels.put("scheme", "http"); - getClient("pd", "address1", labels); - - labels.put("metrics", "/graph/metrics"); - labels.put("target", "10.37.1.1:9200"); - labels.put("scheme", "https"); - getClient("hugegraph", "address1", labels); - } - - // @Test - public void testNodes() throws InterruptedException { - String appName = "hugegraph"; - register(appName, address); - } - - // @Test - public void testMultiNode() throws InterruptedException { - for (int i = 0; i < 2; i++) { - register("app" + i, address + i); - } - } - - // @Test - public void testParallelMultiNode() throws InterruptedException { - CountDownLatch latch = new CountDownLatch(30); - Vector exceptions = new Vector<>(); - for (int i = 0; i < 30; i++) { - int finalI = i; - new Thread(() -> { - try { - for (int j = 0; j < 3; j++) { - register("app" + finalI, address + j); - } - } catch (Exception e) { - exceptions.add(e); - } finally { - latch.countDown(); - } - }).start(); - } - latch.await(); - Assert.assertEquals(0, exceptions.size()); - } - - private void register(String appName, String address) throws InterruptedException { - - HashMap labels = new HashMap<>(); - String labelValue = String.valueOf(label.incrementAndGet()); - labels.put("address", labelValue); - labels.put("address1", labelValue); - Query query = Query.newBuilder().setAppName( - appName).setVersion("0.13.0").putAllLabels(labels).build(); - DiscoveryClientImpl discoveryClient = getClient(appName, address, labels); - Thread.sleep(10000); - NodeInfos nodeInfos1 = discoveryClient.getNodeInfos(query); - Assert.assertEquals(1, nodeInfos1.getInfoCount()); - DiscoveryClientImpl discoveryClient1 = getClient(appName, address + 0, labels); - Thread.sleep(10000); - Assert.assertEquals(2, discoveryClient.getNodeInfos(query).getInfoCount()); - Query query1 = Query.newBuilder().setAppName( - appName).setVersion("0.12.0").putAllLabels(labels).build(); - Assert.assertEquals(0, discoveryClient.getNodeInfos(query1).getInfoCount()); - discoveryClient.cancelTask(); - discoveryClient1.cancelTask(); - Thread.sleep(wait); - NodeInfos nodeInfos = discoveryClient.getNodeInfos(query); - System.out.println(nodeInfos); - Assert.assertEquals(0, nodeInfos.getInfoCount()); - discoveryClient.close(); - discoveryClient1.close(); - } - - private DiscoveryClientImpl getClient(String appName, String address, Map labels) { - DiscoveryClientImpl discoveryClient = null; - try { - discoveryClient = DiscoveryClientImpl.newBuilder().setCenterAddress( - "localhost:8687,localhost:8686,localhost:8688").setAddress(address).setAppName( - appName).setDelay(delay).setVersion("0.13.0").setId( - "0").setLabels(labels).build(); - discoveryClient.scheduleTask(); - } catch (Exception e) { - e.printStackTrace(); - } - - return discoveryClient; - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientTest.java deleted file mode 100644 index 928f1dcba5..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientTest.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.client; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Consumer; - -import org.apache.hugegraph.pd.client.DiscoveryClientImpl; -import org.apache.hugegraph.pd.grpc.discovery.NodeInfo; -import org.apache.hugegraph.pd.grpc.discovery.Query; -import org.junit.Before; -import org.junit.Test; - -public class DiscoveryClientTest { - - private DiscoveryClientImpl client; - - @Before - public void setUp() { - this.client = getClient("appName", "localhost:8654", new HashMap()); - } - - @Test - public void testGetRegisterNode() { - // Setup - try { - Consumer result = this.client.getRegisterConsumer(); - final NodeInfo expectedResult = NodeInfo.newBuilder() - .setAppName("appName") - .build(); - - Thread.sleep(3000); - Query query = Query.newBuilder().setAppName("appName") - .setVersion("0.13.0").build(); - - // Run the test - this.client.getNodeInfos(query); - } catch (InterruptedException e) { - e.printStackTrace(); - } finally { - this.client.close(); - } - - } - - private DiscoveryClientImpl getClient(String appName, String address, - Map labels) { - DiscoveryClientImpl discoveryClient = null; - try { - discoveryClient = DiscoveryClientImpl.newBuilder().setCenterAddress( - "localhost:8686").setAddress(address).setAppName(appName) - .setDelay(2000) - .setVersion("0.13.0") - .setId("0").setLabels(labels) - .build(); - discoveryClient.scheduleTask(); - } catch (Exception e) { - e.printStackTrace(); - } - - return discoveryClient; - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/KvClientTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/KvClientTest.java deleted file mode 100644 index c61413b8cd..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/KvClientTest.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.client; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; - -import java.util.function.Consumer; - -import org.apache.commons.lang3.StringUtils; -import org.apache.hugegraph.pd.client.KvClient; -import org.apache.hugegraph.pd.client.PDConfig; -import org.apache.hugegraph.pd.grpc.kv.KResponse; -import org.apache.hugegraph.pd.grpc.kv.ScanPrefixResponse; -import org.apache.hugegraph.pd.grpc.kv.WatchEvent; -import org.apache.hugegraph.pd.grpc.kv.WatchKv; -import org.apache.hugegraph.pd.grpc.kv.WatchResponse; -import org.apache.hugegraph.pd.grpc.kv.WatchState; -import org.apache.hugegraph.pd.grpc.kv.WatchType; -import org.junit.Before; -import org.junit.Test; - -import io.grpc.stub.AbstractBlockingStub; -import io.grpc.stub.AbstractStub; - -public class KvClientTest extends BaseClientTest { - - String key = "key"; - String value = "value"; - private KvClient client; - - @Before - public void setUp() { - this.client = new KvClient<>(PDConfig.of("localhost:8686")); - } - - @Test - public void testCreateStub() { - // Setup - // Run the test - try { - final AbstractStub result = this.client.createStub(); - } catch (Exception e) { - - } - - - // Verify the results - } - - @Test - public void testCreateBlockingStub() { - // Setup - // Run the test - try { - final AbstractBlockingStub result = this.client.createBlockingStub(); - } catch (Exception e) { - - } - } - - @Test - public void testPutAndGet() throws Exception { - // Run the test - try { - this.client.put(this.key, this.value); - // Run the test - KResponse result = this.client.get(this.key); - - // Verify the results - assertThat(result.getValue()).isEqualTo(this.value); - this.client.delete(this.key); - result = this.client.get(this.key); - assertThat(StringUtils.isEmpty(result.getValue())); - this.client.deletePrefix(this.key); - this.client.put(this.key + "1", this.value); - this.client.put(this.key + "2", this.value); - ScanPrefixResponse response = this.client.scanPrefix(this.key); - assertThat(response.getKvsMap().size() == 2); - this.client.putTTL(this.key + "3", this.value, 1000); - this.client.keepTTLAlive(this.key + "3"); - final Consumer mockConsumer = mock(Consumer.class); - - // Run the test - this.client.listen(this.key + "3", mockConsumer); - this.client.listenPrefix(this.key + "4", mockConsumer); - WatchResponse r = WatchResponse.newBuilder().addEvents( - WatchEvent.newBuilder().setCurrent( - WatchKv.newBuilder().setKey(this.key).setValue("value") - .build()).setType(WatchType.Put).build()) - .setClientId(0L) - .setState(WatchState.Starting) - .build(); - this.client.getWatchList(r); - this.client.getWatchMap(r); - this.client.lock(this.key, 3000L); - this.client.isLocked(this.key); - this.client.unlock(this.key); - this.client.lock(this.key, 3000L); - this.client.keepAlive(this.key); - this.client.close(); - } catch (Exception e) { - - } - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/LicenseClientImplTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/LicenseClientImplTest.java deleted file mode 100644 index 4ed11b9b21..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/LicenseClientImplTest.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.client; - -import java.io.File; -import java.util.Iterator; -import java.util.Map; -import java.util.Properties; - -import org.apache.commons.io.FileUtils; -import org.apache.hugegraph.pd.grpc.Pdpb; -import org.apache.hugegraph.pd.grpc.kv.KResponse; -import org.apache.hugegraph.pd.grpc.kv.KvResponse; -import org.yaml.snakeyaml.Yaml; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class LicenseClientImplTest { - - // @Test - public void putLicense() { - PDConfig pdConfig = PDConfig.of("localhost:8686,localhost:8687,localhost:8688"); - //PDConfig pdConfig = PDConfig.of("localhost:8686"); - pdConfig.setEnableCache(true); - try (LicenseClient c = new LicenseClient(pdConfig)) { - File file = new File("../conf/hugegraph.license"); - byte[] bytes = FileUtils.readFileToByteArray(file); - Pdpb.PutLicenseResponse putLicenseResponse = c.putLicense(bytes); - Pdpb.Error error = putLicenseResponse.getHeader().getError(); - log.info(error.getMessage()); - assert error.getType().equals(Pdpb.ErrorType.OK); - } catch (Exception e) { - log.error("put license with error: ", e); - } - } - - // @Test - public void getKv() { - PDConfig pdConfig = PDConfig.of("10.157.12.36:8686"); - pdConfig.setEnableCache(true); - try (KvClient c = new KvClient(pdConfig)) { - KResponse kResponse = c.get("S:FS"); - Pdpb.Error error = kResponse.getHeader().getError(); - log.info(error.getMessage()); - assert error.getType().equals(Pdpb.ErrorType.OK); - Properties ymlConfig = getYmlConfig(kResponse.getValue()); - Object property = ymlConfig.get("rocksdb.write_buffer_size"); - assert property.toString().equals("32000000"); - } catch (Exception e) { - log.error("put license with error: ", e); - } - } - - // @Test - public void putKv() { - PDConfig pdConfig = PDConfig.of("127.0.0.1.70:8688"); - pdConfig.setEnableCache(true); - try (KvClient c = new KvClient(pdConfig)) { - long l = System.currentTimeMillis(); - KvResponse kvResponse = c.put("S:Timestamp", String.valueOf(l)); - Pdpb.Error error = kvResponse.getHeader().getError(); - log.info(error.getMessage()); - assert error.getType().equals(Pdpb.ErrorType.OK); - } catch (Exception e) { - log.error("put license with error: ", e); - } - } - - // @Test - public void putKvLocal() { - PDConfig pdConfig = PDConfig.of("localhost:8686"); - pdConfig.setEnableCache(true); - try (KvClient c = new KvClient(pdConfig)) { - long l = System.currentTimeMillis(); - KvResponse kvResponse = c.put("S:Timestamp", String.valueOf(l)); - Pdpb.Error error = kvResponse.getHeader().getError(); - log.info(error.getMessage()); - assert error.getType().equals(Pdpb.ErrorType.OK); - } catch (Exception e) { - log.error("put license with error: ", e); - } - } - - private Properties getYmlConfig(String yml) { - Yaml yaml = new Yaml(); - Iterable load = yaml.loadAll(yml); - Iterator iterator = load.iterator(); - Properties properties = new Properties(); - while (iterator.hasNext()) { - Map next = (Map) iterator.next(); - map2Properties(next, "", properties); - } - return properties; - } - - private void map2Properties(Map map, String prefix, Properties properties) { - - for (Map.Entry entry : map.entrySet()) { - String key = entry.getKey(); - String newPrefix = prefix == null || prefix.length() == 0 ? key : prefix + "." + key; - Object value = entry.getValue(); - if (!(value instanceof Map)) { - properties.put(newPrefix, value); - } else { - map2Properties((Map) value, newPrefix, properties); - } - - } - } - -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientSuiteTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientSuiteTest.java deleted file mode 100644 index a9d3ae1406..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientSuiteTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.client; - -import org.junit.runner.RunWith; -import org.junit.runners.Suite; - -import lombok.extern.slf4j.Slf4j; - - -@RunWith(Suite.class) -@Suite.SuiteClasses({ - PDClientTest.class, - KvClientTest.class, - DiscoveryClientTest.class -}) - -@Slf4j -public class PDClientSuiteTest { - -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientTest.java deleted file mode 100644 index 5d068c02f8..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDClientTest.java +++ /dev/null @@ -1,418 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.client; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hugegraph.pd.common.PDException; -import org.apache.hugegraph.pd.grpc.MetaTask; -import org.apache.hugegraph.pd.grpc.Metapb; -import org.apache.hugegraph.pd.grpc.Pdpb; -import org.junit.Test; -import org.mockito.Mockito; - -public class PDClientTest extends BaseClientTest { - @Test - public void testDbCompaction() { - System.out.println("testDbCompaction start"); - - try { - pdClient.dbCompaction(""); - pdClient.dbCompaction(); - } catch (PDException e) { - e.printStackTrace(); - } - - System.out.println("pdclienttest testDbCompaction end"); - } - - @Test - public void testRegisterStore() { - Metapb.Store store = Metapb.Store.newBuilder().build(); - try { - pdClient.registerStore(store); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testSetGraph() { - Metapb.Graph graph = Metapb.Graph.newBuilder().setGraphName("test").build(); - try { - pdClient.setGraph(graph); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testGetGraph() { - try { - pdClient.getGraph("test"); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testGetStore() { - try { - pdClient.getStore(0L); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testUpdateStore() { - Metapb.Store store = Metapb.Store.newBuilder().build(); - try { - pdClient.updateStore(store); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testGetActiveStores() { - try { - pdClient.getActiveStores("test"); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testGetAllStores() { - try { - pdClient.getAllStores("test"); - } catch (PDException e) { - e.printStackTrace(); - } - } - -// @Test -// public void testStoreHeartbeat(){ -// Metapb.StoreStats stats = Metapb.StoreStats.newBuilder().build(); -// try { -// pdClient.storeHeartbeat(stats); -// } catch (PDException e) { -// e.printStackTrace(); -// } -// } - - @Test - public void testKeyToCode() { - pdClient.keyToCode("test", "test".getBytes(StandardCharsets.UTF_8)); - } - - @Test - public void testScanPartitions() { - try { - pdClient.scanPartitions("test", "1".getBytes(StandardCharsets.UTF_8), - "9".getBytes(StandardCharsets.UTF_8)); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testGetPartitionsByStore() { - try { - pdClient.getPartitionsByStore(0L); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testQueryPartitions() { - try { - pdClient.queryPartitions(0L, 0); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testGetPartitions() { - try { - pdClient.getPartitions(0L, "test"); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testUpdatePartitionLeader() { - System.out.println("updatePartitionLeader start"); - - pdClient.updatePartitionLeader("aaa", 0, 0L); - } - - @Test - public void testInvalidPartitionCache() { - pdClient.invalidPartitionCache(); - } - - @Test - public void testInvalidStoreCache() { - pdClient.invalidStoreCache(0L); - } - - @Test - public void testUpdatePartitionCache() { - Metapb.Partition partition = Metapb.Partition.newBuilder().build(); - Metapb.Shard leader = Metapb.Shard.newBuilder().build(); - pdClient.updatePartitionCache(partition, leader); - } - - @Test - public void testGetIdByKey() { - try { - pdClient.getIdByKey("test", 1); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testResetIdByKey() { - try { - pdClient.resetIdByKey("test"); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testGetGetLeader() { - try { - pdClient.getLeader(); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testGetMembers() { - try { - pdClient.getMembers(); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testGetClusterStats() { - try { - pdClient.getClusterStats(); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testAddEventListener() { - PDClient.PDEventListener listener = Mockito.mock(PDClient.PDEventListener.class); - pdClient.addEventListener(listener); - } - - @Test - public void testGetWatchClient() { - pdClient.getWatchClient(); - } - - @Test - public void testGetPulseClient() { - // pdClient.getPulseClient(); - } - - @Test - public void testGetStoreStatus() { - try { - pdClient.getStoreStatus(true); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testGetPartition() { - try { - pdClient.getPartition("test", "test".getBytes(StandardCharsets.UTF_8)); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testSetGraphSpace() { - try { - pdClient.setGraphSpace("test", 1L); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testGetGraphSpace() { - try { - pdClient.getGraphSpace("test"); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testSetPDConfig() { - try { - pdClient.setPDConfig(0, "", 0, 0L); - } catch (PDException e) { - e.printStackTrace(); - } - Metapb.PDConfig pdConfig = Metapb.PDConfig.newBuilder().build(); - - try { - pdClient.setPDConfig(pdConfig); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testGetPDConfig() { - try { - pdClient.getPDConfig(0L); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testChangePeerList() { - try { - pdClient.changePeerList(""); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testSplitData() { - try { - Metapb.PDConfig config = pdClient.getPDConfig(); - pdClient.setPDConfig(config.toBuilder().setMaxShardsPerStore(12).build()); - System.out.println(pdClient.getPDConfig()); - - // 开始分区分裂 - pdClient.splitData(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - @Test - public void testBalancePartition() { - try { - pdClient.balancePartition(); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testMovePartition() { - Pdpb.OperationMode mode = Pdpb.OperationMode.Auto; - List params = new ArrayList<>(1); - try { - pdClient.movePartition(mode, params); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testReportTask() { - MetaTask.Task task = MetaTask.Task.newBuilder().build(); - try { - pdClient.reportTask(task); - } catch (PDException e) { - e.printStackTrace(); - } - } - - - @Test - public void testBalanceLeaders() { - try { - pdClient.balanceLeaders(); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testDelStore() { - try { - pdClient.delStore(0L); - } catch (PDException e) { - e.printStackTrace(); - } - } - -// @Test -// public void testgetQuota() { -// try { -// pdClient.getQuota(); -// } catch (PDException e) { -// e.printStackTrace(); -// } -// } - - @Test - public void testUpdatePartition() { - List partitions = new ArrayList<>(1); - try { - pdClient.updatePartition(partitions); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testDelPartition() { - try { - pdClient.delPartition("test", 0); - } catch (PDException e) { - e.printStackTrace(); - } - } - - @Test - public void testdelGraph() { - try { - pdClient.delGraph("test"); - } catch (PDException e) { - e.printStackTrace(); - } - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDPulseTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDPulseTest.java deleted file mode 100644 index d27c4b5e11..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDPulseTest.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.client; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.apache.hugegraph.pd.client.test.HgPDTestUtil; -import org.apache.hugegraph.pd.grpc.pulse.PartitionHeartbeatRequest; -import org.apache.hugegraph.pd.pulse.PulseServerNotice; -import org.junit.BeforeClass; -import org.junit.Test; - -public class PDPulseTest { - private static PDClient pdClient; - - private final long storeId = 0; - private final String storeAddress = "localhost"; - private final String graphName = "graph1"; - - @BeforeClass - public static void beforeClass() throws Exception { - PDConfig pdConfig = PDConfig.of("localhost:8686"); - pdConfig.setEnableCache(true); - pdClient = PDClient.create(pdConfig); - pdClient.getLeader(); - } - - @Test - public void listen() { - - PDPulse pulse = new PDPulseImpl(pdClient.getLeaderIp()); - CountDownLatch latch = new CountDownLatch(60); - - PDPulse.Notifier notifier1 = - pulse.connectPartition(new PulseListener(latch, "listener1")); - PDPulse.Notifier notifier2 = - pulse.connectPartition(new PulseListener(latch, "listener2")); - PDPulse.Notifier notifier3 = - pulse.connectPartition(new PulseListener(latch, "listener3")); - - try { - latch.await(120, TimeUnit.SECONDS); - } catch (InterruptedException e) { - e.printStackTrace(); - } - PartitionHeartbeatRequest.Builder builder = PartitionHeartbeatRequest.newBuilder(); - - notifier1.notifyServer(builder); - - - notifier2.notifyServer(builder); - - notifier3.notifyServer(builder); - - notifier1.close(); - notifier2.close(); - notifier3.close(); - } - - - private class PulseListener implements PDPulse.Listener { - private final String listenerName; - CountDownLatch latch = new CountDownLatch(10); - - private PulseListener(CountDownLatch latch, String listenerName) { - this.latch = latch; - this.listenerName = listenerName; - } - - @Override - public void onNext(T response) { - // println(this.listenerName+" res: "+response); - // this.latch.countDown(); - } - - @Override - public void onNotice(PulseServerNotice notice) { - HgPDTestUtil.println(this.listenerName + " ---> res: " + notice.getContent()); - - notice.ack(); - this.latch.countDown(); - } - - @Override - public void onError(Throwable throwable) { - HgPDTestUtil.println(this.listenerName + " error: " + throwable.toString()); - } - - @Override - public void onCompleted() { - HgPDTestUtil.println(this.listenerName + " is completed"); - } - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDWatchTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDWatchTest.java deleted file mode 100644 index 180b725555..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/PDWatchTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.client; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.apache.hugegraph.pd.client.test.HgPDTestUtil; -import org.junit.BeforeClass; -import org.junit.Test; - -@Deprecated -public class PDWatchTest { - private static PDClient pdClient; - - private final long storeId = 0; - private final String storeAddr = "localhost"; - private final String graphName = "graph1"; - - @BeforeClass - public static void beforeClass() { - pdClient = PDClient.create(PDConfig.of("localhost:9000")); - } - - @Test - public void watch() { - PDWatch watch = pdClient.getWatchClient(); - CountDownLatch latch = new CountDownLatch(10); - - PDWatch.Watcher watcher1 = watch.watchPartition(new WatchListener<>(latch, "watcher1")); - PDWatch.Watcher watcher2 = watch.watchPartition(new WatchListener<>(latch, "watcher2")); - PDWatch.Watcher watcher3 = watch.watchPartition(new WatchListener<>(latch, "watcher3")); - - PDWatch.Watcher nodeWatcher1 = watch.watchNode(new WatchListener<>(latch, "nodeWatcher1")); - - try { - latch.await(15, TimeUnit.SECONDS); - } catch (InterruptedException e) { - e.printStackTrace(); - } - watcher1.close(); - watcher2.close(); - watcher3.close(); - } - - private class WatchListener implements PDWatch.Listener { - private final String watcherName; - CountDownLatch latch; - - private WatchListener(CountDownLatch latch, String watcherName) { - this.latch = latch; - this.watcherName = watcherName; - } - - @Override - public void onNext(T response) { - HgPDTestUtil.println(this.watcherName + " res: " + response); - this.latch.countDown(); - } - - @Override - public void onError(Throwable throwable) { - HgPDTestUtil.println(this.watcherName + " error: " + throwable.toString()); - } - - @Override - public void onCompleted() { - HgPDTestUtil.println(this.watcherName + " is completed"); - } - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/test/HgPDTestUtil.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/test/HgPDTestUtil.java deleted file mode 100644 index 2c581ea6ff..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/test/HgPDTestUtil.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.client.test; - -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.Iterator; -import java.util.List; - -public class HgPDTestUtil { - - public static void println(Object str) { - System.out.println(str); - } - - public static String toStr(byte[] b) { - if (b == null) return ""; - if (b.length == 0) return ""; - return new String(b, StandardCharsets.UTF_8); - } - - public static byte[] toBytes(String str) { - if (str == null) return null; - return str.getBytes(StandardCharsets.UTF_8); - } - - public static byte[] toBytes(long l) { - ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); - buffer.putLong(l); - return buffer.array(); - } - - private static byte[] toBytes(final int i) { - ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES); - buffer.putInt(i); - return buffer.array(); - } - - public static long toLong(byte[] bytes) { - ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); - buffer.put(bytes); - buffer.flip();//need flip - return buffer.getLong(); - } - - public static long toInt(byte[] bytes) { - ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES); - buffer.put(bytes); - buffer.flip();//need flip - return buffer.getInt(); - } - - public static String padLeftZeros(String str, int n) { - return String.format("%1$" + n + "s", str).replace(' ', '0'); - } - - public static String toSuffix(int num, int length) { - return "-" + padLeftZeros(String.valueOf(num), length); - } - - public static int amountOf(List list) { - if (list == null) { - return 0; - } - return list.size(); - } - - public static int amountOf(Iterator iterator) { - if (iterator == null) return 0; - int count = 0; - while (iterator.hasNext()) { - iterator.next(); - count++; - } - return count; - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/BaseCliToolsTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/BaseCliToolsTest.java deleted file mode 100644 index 146cffb139..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/BaseCliToolsTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.clitools; - -import org.junit.After; -import org.junit.BeforeClass; - - -public class BaseCliToolsTest { - @BeforeClass - public static void init() { - - } - - @After - public void teardown() { - // pass - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/CliToolsSuiteTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/CliToolsSuiteTest.java deleted file mode 100644 index abdcbea08f..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/CliToolsSuiteTest.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.clitools; - -import org.junit.runner.RunWith; -import org.junit.runners.Suite; - -import lombok.extern.slf4j.Slf4j; - - -@RunWith(Suite.class) -@Suite.SuiteClasses({ - MainTest.class -}) - -@Slf4j -public class CliToolsSuiteTest { - - -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/MainTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/MainTest.java deleted file mode 100644 index 526558ac95..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/clitools/MainTest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.clitools; - -import java.util.Arrays; -import java.util.List; - -import org.apache.hugegraph.pd.clitools.Main; -import org.apache.hugegraph.pd.common.PDException; -import org.junit.Test; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MainTest extends BaseCliToolsTest { - - - public static boolean test2sup(List arrays, int tail, int res) { - System.out.printf("%d %d%n", tail, res); - if (tail == 0) { - System.out.printf("a = %d %d%n", tail, res); - return false; - } else if (tail == 1) { - System.out.printf("b = %d %d%n", arrays.get(0), res); - return (arrays.get(0) == res); - } else if (tail == 2) { - System.out.printf("c = %d %d %d%n", arrays.get(0), arrays.get(1), res); - return (arrays.get(0) + arrays.get(1) == Math.abs(res)) || - (Math.abs(arrays.get(0) - arrays.get(1)) == Math.abs(res)); - } else { - return test2sup(arrays, tail - 1, res + arrays.get(tail - 1)) || - test2sup(arrays, tail - 1, res - arrays.get(tail - 1)); - } - } - - @Test - public void getConfig() throws PDException { - Main.main(new String[]{"127.0.0.1:8686", "config", "enableBatchLoad"}); - } - - // @Test - public void setBatchTrue() throws PDException { - Main.main(new String[]{"127.0.0.1:8686", "config", "enableBatchLoad= true "}); - } - - // @Test - public void setBatchFalse() throws PDException { - Main.main(new String[]{"127.0.0.1:8686", "config", "enableBatchLoad=false"}); - } - - @Test - public void getConfig2() throws PDException { - Main.main(new String[]{"127.0.0.1:8686", "config", "shardCount"}); - } - - // @Test - public void setShardCount1() throws PDException { - Main.main(new String[]{"127.0.0.1:8686", "config", "shardCount=1"}); - } - - // @Test - public void setShardCount3() throws PDException { - Main.main(new String[]{"127.0.0.1:8686", "config", "shardCount=3"}); - } - - @Test - public void test2() { - Integer[] a = new Integer[]{1, 0, 3, 2}; - List aa = Arrays.asList(a); - System.out.printf(test2sup(aa, aa.size(), 0) ? "TRUE" : "FALSE"); - } - - -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/CommonSuiteTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/CommonSuiteTest.java index 6f676c6068..0395711caa 100644 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/CommonSuiteTest.java +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/CommonSuiteTest.java @@ -17,8 +17,6 @@ package org.apache.hugegraph.pd.common; -import org.apache.hugegraph.pd.service.IdServiceTest; -import org.apache.hugegraph.pd.service.KvServiceTest; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -29,11 +27,8 @@ @Suite.SuiteClasses({ PartitionUtilsTest.class, PartitionCacheTest.class, - MetadataKeyHelperTest.class, - KvServiceTest.class, HgAssertTest.class, KVPairTest.class, - IdServiceTest.class }) @Slf4j diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/MetadataKeyHelperTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/MetadataKeyHelperTest.java deleted file mode 100644 index ea239ed93c..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/MetadataKeyHelperTest.java +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.common; - -import static org.assertj.core.api.Assertions.assertThat; - -import org.apache.hugegraph.pd.grpc.Metapb; -import org.apache.hugegraph.pd.meta.MetadataKeyHelper; -import org.junit.Test; - -public class MetadataKeyHelperTest { - - @Test - public void testGetStoreInfoKey() { - assertThat(MetadataKeyHelper.getStoreInfoKey(0L)).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetActiveStoreKey() { - assertThat(MetadataKeyHelper.getActiveStoreKey(0L)).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetActiveStorePrefix() { - assertThat(MetadataKeyHelper.getActiveStorePrefix()).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetStorePrefix() { - assertThat(MetadataKeyHelper.getStorePrefix()).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetStoreStatusKey() { - assertThat(MetadataKeyHelper.getStoreStatusKey(0L)).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetShardGroupKey() { - assertThat(MetadataKeyHelper.getShardGroupKey(0L)).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetShardGroupPrefix() { - assertThat(MetadataKeyHelper.getShardGroupPrefix()).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetPartitionKey() { - assertThat(MetadataKeyHelper.getPartitionKey("graphName", 0)).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetPartitionPrefix() { - assertThat(MetadataKeyHelper.getPartitionPrefix("graphName")).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetShardKey() { - assertThat(MetadataKeyHelper.getShardKey(0L, 0)).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetShardPrefix() { - assertThat(MetadataKeyHelper.getShardPrefix(0L)).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetGraphKey() { - assertThat(MetadataKeyHelper.getGraphKey("graphName")).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetGraphPrefix() { - assertThat(MetadataKeyHelper.getGraphPrefix()).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetPartitionStatusKey() { - assertThat(MetadataKeyHelper.getPartitionStatusKey("graphName", - 0)).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetPartitionStatusPrefixKey() { - assertThat(MetadataKeyHelper.getPartitionStatusPrefixKey( - "graphName")).contains(MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetGraphSpaceKey() { - assertThat(MetadataKeyHelper.getGraphSpaceKey("graphSpace")).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetPdConfigKey() { - assertThat(MetadataKeyHelper.getPdConfigKey("configKey")).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetQueueItemPrefix() { - assertThat(MetadataKeyHelper.getQueueItemPrefix()).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetQueueItemKey() { - assertThat(MetadataKeyHelper.getQueueItemKey("itemId")).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetSpitTaskKey() { - assertThat(MetadataKeyHelper.getSplitTaskKey("graphName", 0)).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetSpitTaskPrefix() { - assertThat(MetadataKeyHelper.getSplitTaskPrefix("graph0")).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetLogKey() { - // Setup - final Metapb.LogRecord record = Metapb.LogRecord.newBuilder() - .setAction("value") - .setTimestamp(0L) - .build(); - - // Run the test - final byte[] result = MetadataKeyHelper.getLogKey(record); - - // Verify the results - assertThat(result).contains(MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetLogKeyPrefix() { - assertThat(MetadataKeyHelper.getLogKeyPrefix("action", 0L)).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetKVPrefix() { - assertThat(MetadataKeyHelper.getKVPrefix("prefix", "key")).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetKVTTLPrefix() { - assertThat(MetadataKeyHelper.getKVTTLPrefix("ttlPrefix", "prefix", - "key")).contains( - MetadataKeyHelper.getDelimiter()); - } - - @Test - public void testGetKVWatchKeyPrefix1() { - assertThat( - MetadataKeyHelper.getKVWatchKeyPrefix("key", "watchDelimiter", - 0L)).contains( - String.valueOf(MetadataKeyHelper.getDelimiter())); - } - - @Test - public void testGetKVWatchKeyPrefix2() { - assertThat(MetadataKeyHelper.getKVWatchKeyPrefix("key", - "watchDelimiter")).contains( - String.valueOf(MetadataKeyHelper.getDelimiter())); - } - - @Test - public void testGetDelimiter() { - assertThat(MetadataKeyHelper.getDelimiter()).isEqualTo('/'); - } - - @Test - public void testGetStringBuilderHelper() { - try { - MetadataKeyHelper.getStringBuilderHelper(); - } catch (Exception e) { - - } - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionCacheTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionCacheTest.java index d4126967f6..21e757ffa9 100644 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionCacheTest.java +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionCacheTest.java @@ -157,7 +157,7 @@ public void testGetPartitions() { this.cache.updateShardGroup(creteShardGroup(1)); this.cache.updatePartition(partition2); assertEquals(this.cache.getPartitions("graph0").size(), 2); - System.out.print(this.cache.debugCacheByGraphName("graph0")); + System.out.println(this.cache.debugCacheByGraphName("graph0")); } @Test @@ -168,14 +168,14 @@ public void testAddPartition() { assertNotNull(ret); assertEquals(ret.getKey(), partition); assertNotNull(this.cache.getPartitionByCode("graph0", 2000)); - System.out.print(this.cache.debugCacheByGraphName("graph0")); + System.out.println(this.cache.debugCacheByGraphName("graph0")); var partition2 = createPartition(0, "graph0", 0, 1024); this.cache.addPartition("graph0", 0, partition2); ret = this.cache.getPartitionById("graph0", 0); assertNotNull(ret); assertEquals(ret.getKey(), partition2); assertNull(this.cache.getPartitionByCode("graph0", 2000)); - System.out.print(this.cache.debugCacheByGraphName("graph0")); + System.out.println(this.cache.debugCacheByGraphName("graph0")); } @Test @@ -211,7 +211,7 @@ public void testRemovePartition() { assertNotNull(this.cache.getPartitionById("graph0", 0)); this.cache.removePartition("graph0", 0); assertNull(this.cache.getPartitionById("graph0", 0)); - System.out.print(this.cache.debugCacheByGraphName("graph0")); + System.out.println(this.cache.debugCacheByGraphName("graph0")); } @Test @@ -223,14 +223,12 @@ public void testRange() { var partition3 = createPartition(3, "graph0", 1, 2); var partition4 = createPartition(4, "graph0", 2, 3); - this.cache.updatePartition(partition3); this.cache.updatePartition(partition4); System.out.println(this.cache.debugCacheByGraphName("graph0")); + var partition6 = createPartition(1, "graph0", 0, 1); this.cache.updatePartition(partition6); - - System.out.println(this.cache.debugCacheByGraphName("graph0")); var partition5 = createPartition(1, "graph0", 0, 3); @@ -244,13 +242,11 @@ public void testRange2() { var partition2 = createPartition(2, "graph0", 3, 6); this.cache.updatePartition(partition1); this.cache.updatePartition(partition2); - System.out.println(this.cache.debugCacheByGraphName("graph0")); // 中间有缺失 var partition3 = createPartition(1, "graph0", 2, 3); this.cache.updatePartition(partition3); - System.out.println(this.cache.debugCacheByGraphName("graph0")); var partition5 = createPartition(1, "graph0", 0, 3); diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/BaseCoreTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/BaseCoreTest.java deleted file mode 100644 index dddbffb088..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/BaseCoreTest.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.core; - -import java.io.File; -import java.io.IOException; - -import org.apache.commons.io.FileUtils; -import org.apache.hugegraph.pd.ConfigService; -import org.apache.hugegraph.pd.config.PDConfig; -import org.junit.After; -import org.junit.BeforeClass; - - -public class BaseCoreTest { - - static org.apache.hugegraph.pd.config.PDConfig pdConfig; - - @BeforeClass - public static void init() throws Exception { - String path = "tmp/unitTest"; - deleteDirectory(new File(path)); - pdConfig = new org.apache.hugegraph.pd.config.PDConfig() {{ - this.setClusterId(100); - this.setInitialStoreList("127.0.0.1:8500,127.0.0.1:8501,127.0.0.1:8502," + - "127.0.0.1:8503,127.0.0.1:8504,127.0.0.1:8505"); - }}; - - pdConfig.setStore(new org.apache.hugegraph.pd.config.PDConfig().new Store() {{ - this.setMaxDownTime(3600); - this.setKeepAliveTimeout(3600); - }}); - - pdConfig.setPartition(new org.apache.hugegraph.pd.config.PDConfig().new Partition() {{ - this.setShardCount(3); - this.setMaxShardsPerStore(3); - }}); - pdConfig.setRaft(new org.apache.hugegraph.pd.config.PDConfig().new Raft() {{ - this.setEnable(false); - }}); - pdConfig.setDiscovery(new PDConfig().new Discovery()); - pdConfig.setDataPath(path); - ConfigService configService = new ConfigService(pdConfig); - pdConfig = configService.loadConfig(); - } - - public static void deleteDirectory(File dir) { - try { - FileUtils.deleteDirectory(dir); - } catch (IOException e) { - System.out.printf("Failed to start ....,%s%n", e.getMessage()); - } - } - - @After - public void teardown() throws Exception { - // pass - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreSuiteTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreSuiteTest.java deleted file mode 100644 index 3f616e2123..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreSuiteTest.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.core; - -import org.apache.hugegraph.pd.core.meta.MetadataKeyHelperTest; -import org.junit.runner.RunWith; -import org.junit.runners.Suite; - -import lombok.extern.slf4j.Slf4j; - - -@RunWith(Suite.class) -@Suite.SuiteClasses({ - StoreNodeServiceTest.class, - MetadataKeyHelperTest.class -}) - -@Slf4j -public class PDCoreSuiteTest { - - -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/StoreNodeServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/StoreNodeServiceTest.java deleted file mode 100644 index c8c7bcf157..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/StoreNodeServiceTest.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.core; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.hugegraph.pd.PartitionService; -import org.apache.hugegraph.pd.StoreNodeService; -import org.apache.hugegraph.pd.common.PDException; -import org.apache.hugegraph.pd.grpc.Metapb; -import org.junit.Assert; -import org.junit.Test; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class StoreNodeServiceTest extends BaseCoreTest { - - - @Test - public void testStoreNodeService() throws PDException { - Assert.assertEquals(pdConfig.getPartition().getTotalCount(), - pdConfig.getInitialStoreMap().size() * - pdConfig.getPartition().getMaxShardsPerStore() - / pdConfig.getPartition().getShardCount()); - StoreNodeService storeService = new StoreNodeService(pdConfig); - storeService.init(new PartitionService(pdConfig, storeService)); - int count = 6; - Metapb.Store[] stores = new Metapb.Store[count]; - for (int i = 0; i < count; i++) { - Metapb.Store store = Metapb.Store.newBuilder() - .setId(0) - .setAddress("127.0.0.1:850" + i) - .setDeployPath("/data") - .addLabels(Metapb.StoreLabel.newBuilder() - .setKey("namespace") - .setValue("default") - .build()) - .build(); - stores[i] = storeService.register(store); - System.out.println("新注册store， id = " + stores[i].getId()); - } - Assert.assertEquals(count, storeService.getStores("").size()); - - for (Metapb.Store store : stores) { - Metapb.StoreStats stats = Metapb.StoreStats.newBuilder() - .setStoreId(store.getId()) - .build(); - storeService.heartBeat(stats); - } - - Assert.assertEquals(6, storeService.getActiveStores("").size()); - - Metapb.Graph graph = Metapb.Graph.newBuilder() - .setGraphName("defaultGH") - .setPartitionCount(10) - .build(); - // 分配shard - List shards = storeService.allocShards(graph, 1); - - - Assert.assertEquals(3, shards.size()); - // 设置leader - Assert.assertEquals(pdConfig.getPartition().getTotalCount(), - storeService.getShardGroups().size()); - Metapb.Shard leader = Metapb.Shard.newBuilder(shards.get(0)) - .setRole(Metapb.ShardRole.Leader).build(); - shards = new ArrayList<>(shards); - shards.set(0, leader); - // 增加shard - pdConfig.getPartition().setShardCount(5); - - Metapb.ShardGroup shardGroup = Metapb.ShardGroup.newBuilder() - .setId(1) - .addAllShards(shards).build(); - shards = storeService.reallocShards(shardGroup); - - Assert.assertEquals(5, shards.size()); - // 减少shard - pdConfig.getPartition().setShardCount(3); - shards = storeService.reallocShards(shardGroup); - Assert.assertEquals(3, shards.size()); - // 包含leader，leader不能被删除 - Assert.assertTrue(shards.contains(leader)); - - // 减少shard - pdConfig.getPartition().setShardCount(1); - graph = Metapb.Graph.newBuilder(graph).build(); - shards = storeService.reallocShards(shardGroup); - Assert.assertEquals(1, shards.size()); - // 包含leader，leader不能被删除 - Assert.assertTrue(shards.contains(leader)); - - for (Metapb.Store store : stores) { - storeService.removeStore(store.getId()); - } - Assert.assertEquals(0, storeService.getStores("").size()); - - - } - - -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/meta/MetadataKeyHelperTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/meta/MetadataKeyHelperTest.java deleted file mode 100644 index 7d38b9e132..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/meta/MetadataKeyHelperTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.core.meta; - -import static org.junit.Assert.assertArrayEquals; - -import org.apache.hugegraph.pd.meta.MetadataKeyHelper; -import org.junit.Test; - -public class MetadataKeyHelperTest { - - @Test - public void testMoveTaskKey() { - var key = MetadataKeyHelper.getMoveTaskKey("foo", 0, 1); - assertArrayEquals(key, "TASK_MOVE/foo/0/1".getBytes()); - var key2 = MetadataKeyHelper.getMoveTaskPrefix("foo"); - assertArrayEquals(key2, "TASK_MOVE/foo".getBytes()); - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/grpc/BaseGrpcTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/grpc/BaseGrpcTest.java deleted file mode 100644 index 0e768d6119..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/grpc/BaseGrpcTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.grpc; - -import org.junit.After; -import org.junit.BeforeClass; - - -public class BaseGrpcTest { - - @BeforeClass - public static void init() { - - } - - @After - public void teardown() { - // pass - } - -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/grpc/GrpcSuiteTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/grpc/GrpcSuiteTest.java deleted file mode 100644 index b994a1fd2e..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/grpc/GrpcSuiteTest.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.grpc; - -import org.junit.runner.RunWith; -import org.junit.runners.Suite; - -import lombok.extern.slf4j.Slf4j; - - -@RunWith(Suite.class) -@Suite.SuiteClasses({ -}) - -@Slf4j -public class GrpcSuiteTest { - -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/BaseServerTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/BaseServerTest.java deleted file mode 100644 index e9092a631c..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/BaseServerTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.service; - -import java.io.File; -import java.net.http.HttpClient; - -import org.apache.commons.io.FileUtils; -import org.apache.hugegraph.pd.config.PDConfig; -import org.junit.After; -import org.junit.BeforeClass; - - -public class BaseServerTest { - public static HttpClient client; - public static String pdRestAddr; - - @BeforeClass - public static void init() { - client = HttpClient.newHttpClient(); - pdRestAddr = "http://127.0.0.1:8620"; - } - - public static PDConfig getConfig() { - FileUtils.deleteQuietly(new File("tmp/test/")); - PDConfig pdConfig = new PDConfig() {{ - this.setClusterId(100); - this.setPatrolInterval(1); - this.setRaft(new Raft() {{ - setEnable(false); - }}); - this.setDataPath("tmp/test/"); - }}; - return pdConfig; - } - - @After - public void teardown() { - // pass - } - -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/ConfigServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/ConfigServiceTest.java deleted file mode 100644 index faeacd834e..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/ConfigServiceTest.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.service; - -import java.util.List; - -import org.apache.hugegraph.pd.ConfigService; -import org.apache.hugegraph.pd.IdService; -import org.apache.hugegraph.pd.config.PDConfig; -import org.apache.hugegraph.pd.grpc.Metapb; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class ConfigServiceTest { - - private final PDConfig config = BaseServerTest.getConfig(); - - private ConfigService service; - - @Before - public void setUp() { - this.service = new ConfigService(this.config); - } - - @Test - public void testGetPDConfig() throws Exception { - // Setup - try { - final Metapb.PDConfig config = Metapb.PDConfig.newBuilder() - .setVersion(0L) - .setPartitionCount(0) - .setShardCount(55) - .setMaxShardsPerStore(0) - .setTimestamp(0L).build(); - this.service.setPDConfig(config); - // Run the test - Metapb.PDConfig result = this.service.getPDConfig(0L); - - // Verify the results - Assert.assertTrue(result.getShardCount() == 55); - result = this.service.getPDConfig(); - Assert.assertTrue(result.getShardCount() == 55); - } catch (Exception e) { - - } - - } - - @Test - public void testGetGraphSpace() throws Exception { - // Setup - Metapb.GraphSpace space = Metapb.GraphSpace.newBuilder() - .setName("gs1") - .setTimestamp(0L).build(); - final List expectedResult = List.of(space); - this.service.setGraphSpace(space); - // Run the test - final List result = this.service.getGraphSpace( - "gs1"); - - Assert.assertEquals(1, result.size()); - } - - @Test - public void testUpdatePDConfig() { - try { - final Metapb.PDConfig mConfig = Metapb.PDConfig.newBuilder() - .setVersion(0L) - .setPartitionCount(0) - .setShardCount(0) - .setMaxShardsPerStore(0) - .setTimestamp(0L) - .build(); - final PDConfig expectedResult = new PDConfig(); - expectedResult.setConfigService(new ConfigService(new PDConfig())); - expectedResult.setIdService(new IdService(new PDConfig())); - expectedResult.setClusterId(0L); - expectedResult.setPatrolInterval(0L); - expectedResult.setDataPath("dataPath"); - expectedResult.setMinStoreCount(0); - expectedResult.setInitialStoreList("initialStoreList"); - expectedResult.setHost("host"); - expectedResult.setVerifyPath("verifyPath"); - expectedResult.setLicensePath("licensePath"); - this.service.updatePDConfig(mConfig); - } catch (Exception e) { - - } - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/IdServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/IdServiceTest.java deleted file mode 100644 index dd0d3feb24..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/IdServiceTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.service; - -import java.io.File; - -import org.apache.commons.io.FileUtils; -import org.apache.hugegraph.pd.IdService; -import org.apache.hugegraph.pd.config.PDConfig; -import org.apache.hugegraph.pd.meta.IdMetaStore; -import org.junit.Assert; -import org.junit.Test; - -public class IdServiceTest { - @Test - public void testCid() { - try { - PDConfig pdConfig = BaseServerTest.getConfig(); - int max = 0x2000; - IdService idService = new IdService(pdConfig); - for (int i = 0; i < max; i++) { - idService.getCId("test", max); - } - idService.delCId("test", 1); - idService.delCId("test", 0x10); - idService.delCId("test", 0x100); - idService.delCId("test", 0x1000); - - Assert.assertEquals(1, idService.getCId("test", max)); - Assert.assertEquals(0x10, idService.getCId("test", max)); - Assert.assertEquals(0x100, idService.getCId("test", max)); - Assert.assertEquals(0x1000, idService.getCId("test", max)); - Assert.assertEquals(-1, idService.getCId("test", max)); - - idService.delCId("test", 1); - idService.delCId("test", 0x10); - idService.delCId("test", 0x100); - idService.delCId("test", 0x1000); - - long cid1 = idService.getCId("test", "name", max); - idService.delCIdDelay("test", "name", cid1); - long cid2 = idService.getCId("test", "name", max); - - Assert.assertEquals(cid1, cid2); - idService.delCIdDelay("test", "name", cid2); - Thread.sleep(5000); - long cid3 = idService.getCId("test", "name", max); - } catch (Exception e) { - - } - // MetadataFactory.closeStore(); - } - - @Test - public void testId() { - try { - FileUtils.deleteQuietly(new File("tmp/testId/")); - IdMetaStore.CID_DEL_TIMEOUT = 2000; - PDConfig pdConfig = new PDConfig() {{ - this.setClusterId(100); - this.setPatrolInterval(1); - this.setRaft(new Raft() {{ - setEnable(false); - }}); - this.setDataPath("tmp/testId/"); - }}; - IdService idService = new IdService(pdConfig); - long first = idService.getId("abc", 100); - Assert.assertEquals(first, 0L); - long second = idService.getId("abc", 100); - Assert.assertEquals(second, 100L); - idService.resetId("abc"); - first = idService.getId("abc", 100); - Assert.assertEquals(first, 0L); - } catch (Exception ignored) { - - } - // MetadataFactory.closeStore(); - } - - @Test - public void testMember() { - try { - PDConfig pdConfig = BaseServerTest.getConfig(); - IdService idService = new IdService(pdConfig); - idService.setPdConfig(pdConfig); - PDConfig config = idService.getPdConfig(); - config.getHost(); - } catch (Exception e) { - e.printStackTrace(); - } - // MetadataFactory.closeStore(); - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/KvServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/KvServiceTest.java deleted file mode 100644 index 02870b219e..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/KvServiceTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.service; - -import org.apache.hugegraph.pd.KvService; -import org.apache.hugegraph.pd.config.PDConfig; -import org.junit.Assert; -import org.junit.Test; - -public class KvServiceTest { - - @Test - public void testKv() { - try { - PDConfig pdConfig = BaseServerTest.getConfig(); - KvService service = new KvService(pdConfig); - String key = "kvTest"; - String kvTest = service.get(key); - Assert.assertEquals(kvTest, ""); - service.put(key, "kvTestValue"); - kvTest = service.get(key); - Assert.assertEquals(kvTest, "kvTestValue"); - service.scanWithPrefix(key); - service.delete(key); - service.put(key, "kvTestValue"); - service.deleteWithPrefix(key); - service.put(key, "kvTestValue", 1000L); - service.keepAlive(key); - } catch (Exception e) { - - } - } - - @Test - public void testMember() { - try { - PDConfig pdConfig = BaseServerTest.getConfig(); - KvService service = new KvService(pdConfig); - service.setPdConfig(pdConfig); - PDConfig config = service.getPdConfig(); - } catch (Exception e) { - e.printStackTrace(); - } - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/LogServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/LogServiceTest.java deleted file mode 100644 index 266db76c56..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/LogServiceTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.service; - -import java.util.List; - -import org.apache.hugegraph.pd.LogService; -import org.apache.hugegraph.pd.config.PDConfig; -import org.apache.hugegraph.pd.grpc.Metapb; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import com.google.protobuf.Any; - -public class LogServiceTest { - - private final PDConfig mockPdConfig = BaseServerTest.getConfig(); - - private LogService logServiceUnderTest; - - @Before - public void setUp() { - this.logServiceUnderTest = new LogService(this.mockPdConfig); - } - - @Test - public void testGetLog() throws Exception { - this.logServiceUnderTest.insertLog("action", "message", - Any.newBuilder().build()); - - // Run the test - final List result = this.logServiceUnderTest.getLog( - "action", 0L, System.currentTimeMillis()); - - // Verify the results - Assert.assertEquals(result.size(), 1); - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/PartitionServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/PartitionServiceTest.java deleted file mode 100644 index 0a16402d33..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/PartitionServiceTest.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.service; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.util.List; - -import org.apache.hugegraph.pd.PartitionService; -import org.apache.hugegraph.pd.common.PDException; -import org.apache.hugegraph.pd.grpc.MetaTask; -import org.apache.hugegraph.pd.grpc.Metapb; -import org.apache.hugegraph.pd.grpc.pulse.CleanPartition; -import org.apache.hugegraph.pd.grpc.pulse.CleanType; -import org.junit.Before; -import org.junit.Test; - -public class PartitionServiceTest extends PdTestBase { - - private PartitionService service; - - @Before - public void init() { - this.service = getPartitionService(); - } - - @Test - public void testCombinePartition() throws PDException { - buildEnv(); - // 0, 1, 2-> 0, 3,4,5->1, 6,7,8 ->2, 9,10, 11-> 3 - this.service.combinePartition(4); - - var partition = this.service.getPartitionById("graph0", 0); - assertEquals(0, partition.getStartKey()); - assertEquals(5462, partition.getEndKey()); - - var tasks = getStoreNodeService().getTaskInfoMeta().scanMoveTask("graph0"); - assertEquals(11, tasks.size()); - - for (MetaTask.Task task : tasks) { - var newTask = task.toBuilder().setState(MetaTask.TaskState.Task_Success).build(); - getTaskService().reportTask(newTask); - } - - tasks = getStoreNodeService().getTaskInfoMeta().scanMoveTask("graph0"); - assertEquals(0, tasks.size()); - } - - @Test - public void testCombinePartition2() throws PDException { - buildEnv(); - // 0, 1, 2-> 0, 3,4,5->1, 6,7,8 ->2, 9,10, 11-> 3 - this.service.combinePartition(4); - - var partition = this.service.getPartitionById("graph0", 0); - assertEquals(0, partition.getStartKey()); - assertEquals(5462, partition.getEndKey()); - - var tasks = getStoreNodeService().getTaskInfoMeta().scanMoveTask("graph0"); - assertEquals(11, tasks.size()); - - for (MetaTask.Task task : tasks) { - var newTask = task.toBuilder().setState(MetaTask.TaskState.Task_Failure).build(); - getTaskService().reportTask(newTask); - } - - tasks = getStoreNodeService().getTaskInfoMeta().scanMoveTask("graph0"); - assertEquals(0, tasks.size()); - } - - @Test - public void testHandleCleanTask() { - MetaTask.Task task = MetaTask.Task.newBuilder() - .setType(MetaTask.TaskType.Clean_Partition) - .setPartition( - Metapb.Partition.newBuilder().setGraphName("foo") - .setId(0).build()) - .setCleanPartition(CleanPartition.newBuilder() - .setCleanType( - CleanType.CLEAN_TYPE_KEEP_RANGE) - .setDeletePartition(true) - .setKeyStart(0) - .setKeyEnd(10) - .build()) - .build(); - getTaskService().reportTask(task); - } - - private void buildEnv() throws PDException { - var storeInfoMeta = getStoreNodeService().getStoreInfoMeta(); - storeInfoMeta.updateStore(Metapb.Store.newBuilder() - .setId(99) - .setState(Metapb.StoreState.Up) - .build()); - - long lastId = 0; - for (int i = 0; i < 12; i++) { - Metapb.Shard shard = Metapb.Shard.newBuilder() - .setStoreId(99) - .setRole(Metapb.ShardRole.Leader) - .build(); - - Metapb.ShardGroup shardGroup = Metapb.ShardGroup.newBuilder() - .setId(i) - .setState( - Metapb.PartitionState.PState_Normal) - .addAllShards(List.of(shard)) - .build(); - storeInfoMeta.updateShardGroup(shardGroup); - - var partitionShard = this.service.getPartitionByCode("graph0", lastId); - if (partitionShard != null) { - lastId = partitionShard.getPartition().getEndKey(); - } - } - - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/PdTestBase.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/PdTestBase.java deleted file mode 100644 index 50eadb3a6d..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/PdTestBase.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.service; - -import java.io.File; - -import org.apache.hugegraph.pd.ConfigService; -import org.apache.hugegraph.pd.IdService; -import org.apache.hugegraph.pd.PartitionInstructionListener; -import org.apache.hugegraph.pd.PartitionService; -import org.apache.hugegraph.pd.PartitionStatusListener; -import org.apache.hugegraph.pd.StoreMonitorDataService; -import org.apache.hugegraph.pd.StoreNodeService; -import org.apache.hugegraph.pd.StoreStatusListener; -import org.apache.hugegraph.pd.TaskScheduleService; -import org.apache.hugegraph.pd.common.PDException; -import org.apache.hugegraph.pd.config.PDConfig; -import org.apache.hugegraph.pd.grpc.Metapb; -import org.apache.hugegraph.pd.grpc.pulse.ChangeShard; -import org.apache.hugegraph.pd.grpc.pulse.CleanPartition; -import org.apache.hugegraph.pd.grpc.pulse.DbCompaction; -import org.apache.hugegraph.pd.grpc.pulse.MovePartition; -import org.apache.hugegraph.pd.grpc.pulse.PartitionKeyRange; -import org.apache.hugegraph.pd.grpc.pulse.SplitPartition; -import org.apache.hugegraph.pd.grpc.pulse.TransferLeader; -import org.apache.hugegraph.pd.raft.RaftEngine; -import org.junit.AfterClass; -import org.junit.BeforeClass; - -public class PdTestBase { - private static final String DATA_PATH = "/tmp/pd_data"; - private static PDConfig pdConfig; - private static StoreNodeService storeNodeService; - private static PartitionService partitionService; - private static TaskScheduleService taskService; - private static StoreMonitorDataService storeMonitorDataService; - - @BeforeClass - public static void initService() throws PDException { - deleteDir(new File(DATA_PATH)); - - PDConfig config = new PDConfig(); - config.setDataPath(DATA_PATH); - config.setMinStoreCount(3); - config.setInitialStoreList("127.0.0.1:8501"); - config.setHost("127.0.0.1"); - config.setVerifyPath(""); - config.setLicensePath(""); - PDConfig.Raft raft = new PDConfig().new Raft(); - raft.setAddress("127.0.0.1:8601"); - raft.setPeersList("127.0.0.1:8601"); - raft.setDataPath(DATA_PATH); - raft.setHost("127.0.0.1"); - raft.setGrpcPort(8688); - raft.setPort(8621); - - config.setRaft(raft); - - config.setStore(new PDConfig().new Store()); - config.setPartition(new PDConfig().new Partition() {{ - setShardCount(1); - setTotalCount(12); - setMaxShardsPerStore(12); - }}); - config.setDiscovery(new PDConfig().new Discovery()); - - pdConfig = config; - - var configService = new ConfigService(pdConfig); - configService.loadConfig(); - - var engine = RaftEngine.getInstance(); - engine.addStateListener(configService); - engine.init(pdConfig.getRaft()); - engine.waitingForLeader(5000); - - storeNodeService = new StoreNodeService(pdConfig); - partitionService = new PartitionService(pdConfig, storeNodeService); - taskService = new TaskScheduleService(pdConfig, storeNodeService, partitionService); - var idService = new IdService(pdConfig); - storeMonitorDataService = new StoreMonitorDataService(pdConfig); - RaftEngine.getInstance().addStateListener(partitionService); - pdConfig.setIdService(idService); - - - storeNodeService.init(partitionService); - partitionService.init(); - partitionService.addInstructionListener(new PartitionInstructionListener() { - @Override - public void changeShard(Metapb.Partition partition, ChangeShard changeShard) throws - PDException { - - } - - @Override - public void transferLeader(Metapb.Partition partition, - TransferLeader transferLeader) throws PDException { - - } - - @Override - public void splitPartition(Metapb.Partition partition, - SplitPartition splitPartition) throws PDException { - - } - - @Override - public void dbCompaction(Metapb.Partition partition, DbCompaction dbCompaction) throws - PDException { - - } - - @Override - public void movePartition(Metapb.Partition partition, - MovePartition movePartition) throws PDException { - - } - - @Override - public void cleanPartition(Metapb.Partition partition, - CleanPartition cleanPartition) throws PDException { - - } - - @Override - public void changePartitionKeyRange(Metapb.Partition partition, - PartitionKeyRange partitionKeyRange) - throws PDException { - - } - }); - - partitionService.addStatusListener(new PartitionStatusListener() { - @Override - public void onPartitionChanged(Metapb.Partition partition, - Metapb.Partition newPartition) { - - } - - @Override - public void onPartitionRemoved(Metapb.Partition partition) { - - } - }); - - storeNodeService.addStatusListener(new StoreStatusListener() { - @Override - public void onStoreStatusChanged(Metapb.Store store, Metapb.StoreState old, - Metapb.StoreState status) { - - } - - @Override - public void onGraphChange(Metapb.Graph graph, Metapb.GraphState stateOld, - Metapb.GraphState stateNew) { - - } - - @Override - public void onStoreRaftChanged(Metapb.Store store) { - - } - }); - - taskService.init(); - } - - @AfterClass - public static void shutdownService() { - var instance = RaftEngine.getInstance(); - if (instance != null) { - instance.shutDown(); - } - } - - private static boolean deleteDir(File dir) { - if (dir.isDirectory()) { - for (File file : dir.listFiles()) { - deleteDir(file); - } - } - return dir.delete(); - } - - public static StoreNodeService getStoreNodeService() { - return storeNodeService; - } - - public static PartitionService getPartitionService() { - return partitionService; - } - - public static PDConfig getPdConfig() { - return pdConfig; - } - - public static TaskScheduleService getTaskService() { - return taskService; - } - - public static StoreMonitorDataService getStoreMonitorDataService() { - return storeMonitorDataService; - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/RestApiTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/RestApiTest.java deleted file mode 100644 index 7e5fec381e..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/RestApiTest.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.service; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; - -import org.json.JSONException; -import org.json.JSONObject; -import org.junit.Test; - -public class RestApiTest extends BaseServerTest { - @Test - public void testQueryClusterInfo() throws URISyntaxException, IOException, InterruptedException, - JSONException { - String url = pdRestAddr + "/v1/cluster"; - HttpRequest request = HttpRequest.newBuilder() - .uri(new URI(url)) - .GET() - .build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); - JSONObject obj = new JSONObject(response.body()); - assert obj.getInt("status") == 0; - } - - @Test - public void testQueryClusterMembers() throws URISyntaxException, IOException, - InterruptedException, JSONException { - String url = pdRestAddr + "/v1/members"; - HttpRequest request = HttpRequest.newBuilder() - .uri(new URI(url)) - .GET() - .build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); - JSONObject obj = new JSONObject(response.body()); - assert obj.getInt("status") == 0; - } - - @Test - public void testQueryStoresInfo() throws URISyntaxException, IOException, InterruptedException, - JSONException { - String url = pdRestAddr + "/v1/stores"; - HttpRequest request = HttpRequest.newBuilder() - .uri(new URI(url)) - .GET() - .build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); - JSONObject obj = new JSONObject(response.body()); - assert obj.getInt("status") == 0; - } - - @Test - public void testQueryGraphsInfo() throws IOException, InterruptedException, JSONException, - URISyntaxException { - String url = pdRestAddr + "/v1/graphs"; - HttpRequest request = HttpRequest.newBuilder() - .uri(new URI(url)) - .GET() - .build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); - JSONObject obj = new JSONObject(response.body()); - assert obj.getInt("status") == 0; - } - - @Test - public void testQueryPartitionsInfo() throws IOException, InterruptedException, JSONException, - URISyntaxException { - String url = pdRestAddr + "/v1/highLevelPartitions"; - HttpRequest request = HttpRequest.newBuilder() - .uri(new URI(url)) - .GET() - .build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); - JSONObject obj = new JSONObject(response.body()); - assert obj.getInt("status") == 0; - } - - @Test - public void testQueryDebugPartitionsInfo() throws URISyntaxException, IOException, - InterruptedException { - String url = pdRestAddr + "/v1/partitions"; - HttpRequest request = HttpRequest.newBuilder() - .uri(new URI(url)) - .GET() - .build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); - assert response.statusCode() == 200; - } - - @Test - public void testQueryShards() throws URISyntaxException, IOException, InterruptedException, - JSONException { - String url = pdRestAddr + "/v1/shards"; - HttpRequest request = HttpRequest.newBuilder() - .uri(new URI(url)) - .GET() - .build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); - JSONObject obj = new JSONObject(response.body()); - assert obj.getInt("status") == 0; - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/ServerSuiteTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/ServerSuiteTest.java deleted file mode 100644 index fe8c0ed044..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/ServerSuiteTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.service; - -import org.junit.runner.RunWith; -import org.junit.runners.Suite; - -import lombok.extern.slf4j.Slf4j; - - -@RunWith(Suite.class) -@Suite.SuiteClasses({ - RestApiTest.class, - ConfigServiceTest.class, - IdServiceTest.class, - KvServiceTest.class, - LogServiceTest.class, - StoreServiceTest.class, - StoreNodeServiceNewTest.class, - StoreMonitorDataServiceTest.class, - TaskScheduleServiceTest.class, - PartitionServiceTest.class -}) - -@Slf4j -public class ServerSuiteTest { -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreMonitorDataServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreMonitorDataServiceTest.java deleted file mode 100644 index 21a2a37dde..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreMonitorDataServiceTest.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.service; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.util.List; - -import org.apache.hugegraph.pd.StoreMonitorDataService; -import org.apache.hugegraph.pd.common.PDException; -import org.apache.hugegraph.pd.grpc.Metapb; -import org.junit.Before; -import org.junit.Test; - -public class StoreMonitorDataServiceTest extends PdTestBase { - - StoreMonitorDataService service; - - @Before - public void init() { - this.service = getStoreMonitorDataService(); - var store = getPdConfig().getStore(); - store.setMonitorDataEnabled(true); - store.setMonitorDataInterval("1s"); - getPdConfig().setStore(store); - } - - @Test - public void test() throws InterruptedException, PDException { - long now = System.currentTimeMillis() / 1000; - for (int i = 0; i < 5; i++) { - this.service.saveMonitorData(genStats()); - now = System.currentTimeMillis() / 1000; - Thread.sleep(1100); - } - assertTrue(this.service.getLatestStoreMonitorDataTimeStamp(1) == 0 || - this.service.getLatestStoreMonitorDataTimeStamp(1) == now); - - var data = this.service.getStoreMonitorData(1); - assertEquals(5, data.size()); - - assertNotNull(this.service.debugMonitorInfo(List.of(Metapb.RecordPair.newBuilder() - .setKey("key1") - .setValue(1) - .build()))); - - assertNotNull(this.service.getStoreMonitorDataText(1)); - - - this.service.removeExpiredMonitorData(1, now + 1); - assertEquals(0, this.service.getStoreMonitorData(1).size()); - } - - - private Metapb.StoreStats genStats() { - return Metapb.StoreStats.newBuilder() - .setStoreId(1) - .addSystemMetrics( - Metapb.RecordPair.newBuilder().setKey("key1").setValue(1) - .build()) - .build(); - } - - -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreNodeServiceNewTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreNodeServiceNewTest.java deleted file mode 100644 index 10d098d1f3..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreNodeServiceNewTest.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.service; - -import static org.junit.Assert.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.apache.hugegraph.pd.StoreNodeService; -import org.apache.hugegraph.pd.common.PDException; -import org.apache.hugegraph.pd.grpc.Metapb; -import org.junit.Before; -import org.junit.Test; - -public class StoreNodeServiceNewTest extends PdTestBase { - private StoreNodeService service; - - @Before - public void init() { - this.service = getStoreNodeService(); - } - - @Test - public void testGetTaskInfoMeta() { - assertNotNull(this.service.getTaskInfoMeta()); - } - - public void testGetStoreInfoMeta() { - assertNotNull(this.service.getStoreInfoMeta()); - } - - @Test - public void testRemoveShardGroup() throws PDException { - for (int i = 0; i < 12; i++) { - Metapb.ShardGroup group = Metapb.ShardGroup.newBuilder() - .setId(i) - .setState( - Metapb.PartitionState.PState_Offline) - .build(); - this.service.getStoreInfoMeta().updateShardGroup(group); - } - - this.service.deleteShardGroup(11); - this.service.deleteShardGroup(10); - - assertEquals(10, getPdConfig().getConfigService().getPDConfig().getPartitionCount()); - // restore - getPdConfig().getConfigService().setPartitionCount(12); - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreServiceTest.java deleted file mode 100644 index af17db1c72..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/StoreServiceTest.java +++ /dev/null @@ -1,833 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.service; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; - -import org.apache.hugegraph.pd.ConfigService; -import org.apache.hugegraph.pd.IdService; -import org.apache.hugegraph.pd.PartitionService; -import org.apache.hugegraph.pd.StoreNodeService; -import org.apache.hugegraph.pd.StoreStatusListener; -import org.apache.hugegraph.pd.config.PDConfig; -import org.apache.hugegraph.pd.grpc.MetaTask; -import org.apache.hugegraph.pd.grpc.Metapb; -import org.junit.Before; -import org.junit.Test; - -public class StoreServiceTest { - - private PDConfig config; - - private StoreNodeService service; - - @Before - public void setUp() { - this.config = getConfig(); - this.service = new StoreNodeService(this.config); - } - - @Test - public void testInit() { - // Setup - PDConfig pdConfig = getConfig(); - final PDConfig pdConfig1 = getConfig(); - final PartitionService partitionService = new PartitionService(pdConfig, - new StoreNodeService( - pdConfig1)); - - // Run the test - this.service.init(partitionService); - - // Verify the results - } - - private PDConfig getConfig() { - PDConfig pdConfig = new PDConfig(); - pdConfig.setConfigService( - new ConfigService(BaseServerTest.getConfig())); - pdConfig.setIdService(new IdService(BaseServerTest.getConfig())); - pdConfig.setClusterId(0L); - pdConfig.setPatrolInterval(0L); - pdConfig.setDataPath("dataPath"); - pdConfig.setMinStoreCount(0); - pdConfig.setInitialStoreList("initialStoreList"); - pdConfig.setHost("host"); - pdConfig.setVerifyPath("verifyPath"); - pdConfig.setLicensePath("licensePath"); - PDConfig.Raft raft = new PDConfig().new Raft(); - raft.setEnable(false); - pdConfig.setRaft(raft); - final PDConfig.Partition partition = new PDConfig().new Partition(); - partition.setTotalCount(0); - partition.setShardCount(0); - pdConfig.setPartition(partition); - pdConfig.setInitialStoreMap(Map.ofEntries(Map.entry("value", "value"))); - return pdConfig; - } - - @Test - public void testIsOK() { - // Setup - // Run the test - final boolean result = this.service.isOK(); - - // Verify the results - assertThat(result).isTrue(); - } - - @Test - public void testRegister() throws Exception { - // Setup - final Metapb.Store store = Metapb.Store.newBuilder().setId(0L) - .setAddress("address") - .setRaftAddress("raftAddress") - .addLabels(Metapb.StoreLabel - .newBuilder() - .build()) - .setVersion("version").setState( - Metapb.StoreState.Unknown).setStartTimestamp(0L) - .setDeployPath("deployPath") - .setLastHeartbeat(0L).setStats( - Metapb.StoreStats.newBuilder().setStoreId(0L) - .setPartitionCount(0).addGraphStats( - Metapb.GraphStats.newBuilder() - .setGraphName("value") - .setApproximateSize(0L) - .setRole(Metapb.ShardRole.None) - .build()).build()) - .setDataVersion(0).setCores(0) - .setDataPath("dataPath").build(); - final Metapb.Store expectedResult = Metapb.Store.newBuilder().setId(0L) - .setAddress("address") - .setRaftAddress( - "raftAddress") - .addLabels( - Metapb.StoreLabel - .newBuilder() - .build()) - .setVersion("version") - .setState( - Metapb.StoreState.Unknown) - .setStartTimestamp(0L) - .setDeployPath( - "deployPath") - .setLastHeartbeat(0L) - .setStats( - Metapb.StoreStats - .newBuilder() - .setStoreId( - 0L) - .setPartitionCount( - 0) - .addGraphStats( - Metapb.GraphStats - .newBuilder() - .setGraphName( - "value") - .setApproximateSize( - 0L) - .setRole( - Metapb.ShardRole.None) - .build()) - .build()) - .setDataVersion(0) - .setCores(0) - .setDataPath("dataPath") - .build(); - - // Configure PDConfig.getInitialStoreMap(...). - final Map stringStringMap = Map.ofEntries( - Map.entry("value", "value")); - - // Run the test - final Metapb.Store result = this.service.register(store); - } - - @Test - public void testGetStore() throws Exception { - // Setup - try { - Metapb.GraphStats stats = Metapb.GraphStats.newBuilder() - .setGraphName("value") - .setApproximateSize(0L) - .setRole( - Metapb.ShardRole.None) - .build(); - Metapb.StoreStats storeStats = Metapb.StoreStats.newBuilder() - .setStoreId(0L) - .setPartitionCount( - 0) - .addGraphStats( - stats) - .build(); - final Metapb.Store expectedResult = Metapb.Store.newBuilder() - .setId(0L) - .setAddress( - "address") - .setRaftAddress( - "raftAddress") - .addLabels( - Metapb.StoreLabel - .newBuilder() - .build()) - .setVersion( - "version") - .setState( - Metapb.StoreState.Unknown) - .setStartTimestamp( - 0L) - .setDeployPath( - "deployPath") - .setLastHeartbeat( - 0L) - .setStats( - storeStats) - .setDataVersion(0) - .setCores(0) - .setDataPath( - "dataPath") - .build(); - - // Run the test - final Metapb.Store result = this.service.getStore(0L); - } catch (Exception e) { - - } - } - - @Test - public void testUpdateStore() throws Exception { - // Setup - final Metapb.Store store = Metapb.Store.newBuilder().setId(0L) - .setAddress("address") - .setRaftAddress("raftAddress") - .addLabels(Metapb.StoreLabel - .newBuilder() - .build()) - .setVersion("version").setState( - Metapb.StoreState.Unknown).setStartTimestamp(0L) - .setDeployPath("deployPath") - .setLastHeartbeat(0L).setStats( - Metapb.StoreStats.newBuilder().setStoreId(0L) - .setPartitionCount(0).addGraphStats( - Metapb.GraphStats.newBuilder() - .setGraphName("value") - .setApproximateSize(0L) - .setRole(Metapb.ShardRole.None) - .build()).build()) - .setDataVersion(0).setCores(0) - .setDataPath("dataPath").build(); - final Metapb.Store expectedResult = Metapb.Store.newBuilder().setId(0L) - .setAddress("address") - .setRaftAddress( - "raftAddress") - .addLabels( - Metapb.StoreLabel - .newBuilder() - .build()) - .setVersion("version") - .setState( - Metapb.StoreState.Unknown) - .setStartTimestamp(0L) - .setDeployPath( - "deployPath") - .setLastHeartbeat(0L) - .setStats( - Metapb.StoreStats - .newBuilder() - .setStoreId( - 0L) - .setPartitionCount( - 0) - .addGraphStats( - Metapb.GraphStats - .newBuilder() - .setGraphName( - "value") - .setApproximateSize( - 0L) - .setRole( - Metapb.ShardRole.None) - .build()) - .build()) - .setDataVersion(0) - .setCores(0) - .setDataPath("dataPath") - .build(); - - // Configure PDConfig.getPartition(...). - final PDConfig.Partition partition = new PDConfig().new Partition(); - partition.setTotalCount(0); - partition.setMaxShardsPerStore(0); - partition.setShardCount(0); - - // Run the test - final Metapb.Store result = this.service.updateStore(store); - } - - @Test - public void testStoreTurnoff() throws Exception { - // Setup - final Metapb.Store store = Metapb.Store.newBuilder().setId(0L) - .setAddress("address") - .setRaftAddress("raftAddress") - .addLabels(Metapb.StoreLabel - .newBuilder() - .build()) - .setVersion("version").setState( - Metapb.StoreState.Unknown).setStartTimestamp(0L) - .setDeployPath("deployPath") - .setLastHeartbeat(0L).setStats( - Metapb.StoreStats.newBuilder().setStoreId(0L) - .setPartitionCount(0).addGraphStats( - Metapb.GraphStats.newBuilder() - .setGraphName("value") - .setApproximateSize(0L) - .setRole(Metapb.ShardRole.None) - .build()).build()) - .setDataVersion(0).setCores(0) - .setDataPath("dataPath").build(); - - // Configure PDConfig.getPartition(...). - final PDConfig.Partition partition = new PDConfig().new Partition(); - partition.setTotalCount(0); - partition.setMaxShardsPerStore(0); - partition.setShardCount(0); - - // Run the test - this.service.storeTurnoff(store); - - // Verify the results - } - - - @Test - public void testGetStores1() throws Exception { - // Setup - final List expectedResult = List.of( - Metapb.Store.newBuilder().setId(0L).setAddress("address") - .setRaftAddress("raftAddress") - .addLabels(Metapb.StoreLabel.newBuilder().build()) - .setVersion("version") - .setState(Metapb.StoreState.Unknown) - .setStartTimestamp(0L).setDeployPath("deployPath") - .setLastHeartbeat(0L).setStats( - Metapb.StoreStats.newBuilder().setStoreId(0L) - .setPartitionCount(0).addGraphStats( - Metapb.GraphStats.newBuilder() - .setGraphName("value") - .setApproximateSize(0L) - .setRole(Metapb.ShardRole.None) - .build()).build()) - .setDataVersion(0).setCores(0) - .setDataPath("dataPath").build()); - - // Run the test - final List result = this.service.getStores(); - } - - @Test - public void testGetStores2() throws Exception { - // Setup - final List expectedResult = List.of( - Metapb.Store.newBuilder().setId(0L).setAddress("address") - .setRaftAddress("raftAddress") - .addLabels(Metapb.StoreLabel.newBuilder().build()) - .setVersion("version") - .setState(Metapb.StoreState.Unknown) - .setStartTimestamp(0L).setDeployPath("deployPath") - .setLastHeartbeat(0L).setStats( - Metapb.StoreStats.newBuilder().setStoreId(0L) - .setPartitionCount(0).addGraphStats( - Metapb.GraphStats.newBuilder() - .setGraphName("value") - .setApproximateSize(0L) - .setRole(Metapb.ShardRole.None) - .build()).build()) - .setDataVersion(0).setCores(0) - .setDataPath("dataPath").build()); - - // Run the test - final List result = this.service.getStores("graphName"); - } - - - @Test - public void testGetStoreStatus() throws Exception { - // Setup - final List expectedResult = List.of( - Metapb.Store.newBuilder().setId(0L).setAddress("address") - .setRaftAddress("raftAddress") - .addLabels(Metapb.StoreLabel.newBuilder().build()) - .setVersion("version") - .setState(Metapb.StoreState.Unknown) - .setStartTimestamp(0L).setDeployPath("deployPath") - .setLastHeartbeat(0L).setStats( - Metapb.StoreStats.newBuilder().setStoreId(0L) - .setPartitionCount(0).addGraphStats( - Metapb.GraphStats.newBuilder() - .setGraphName("value") - .setApproximateSize(0L) - .setRole(Metapb.ShardRole.None) - .build()).build()) - .setDataVersion(0).setCores(0) - .setDataPath("dataPath").build()); - - // Run the test - final List result = this.service.getStoreStatus(false); - - } - - @Test - public void testGetShardGroups() throws Exception { - // Setup - final List expectedResult = List.of( - Metapb.ShardGroup.newBuilder().setId(0).addShards( - Metapb.Shard.newBuilder().setStoreId(0L) - .setRole(Metapb.ShardRole.None).build()) - .setState(Metapb.PartitionState.PState_None) - .build()); - - // Run the test - final List result = this.service.getShardGroups(); - - } - - @Test - public void testGetShardGroup() throws Exception { - // Setup - final Metapb.ShardGroup expectedResult = Metapb.ShardGroup.newBuilder() - .setId(0) - .addShards( - Metapb.Shard - .newBuilder() - .setStoreId( - 0L) - .setRole( - Metapb.ShardRole.None) - .build()) - .setState( - Metapb.PartitionState.PState_None) - .build(); - - // Run the test - final Metapb.ShardGroup result = this.service.getShardGroup(0); - - // Verify the results - } - - - @Test - public void testGetShardGroupsByStore() throws Exception { - // Setup - final List expectedResult = List.of( - Metapb.ShardGroup.newBuilder().setId(0).addShards( - Metapb.Shard.newBuilder().setStoreId(0L) - .setRole(Metapb.ShardRole.None).build()) - .setState(Metapb.PartitionState.PState_None) - .build()); - - // Run the test - final List result = this.service.getShardGroupsByStore( - 0L); - } - - @Test - public void testGetActiveStores1() throws Exception { - // Setup - final List expectedResult = List.of( - Metapb.Store.newBuilder().setId(0L).setAddress("address") - .setRaftAddress("raftAddress") - .addLabels(Metapb.StoreLabel.newBuilder().build()) - .setVersion("version") - .setState(Metapb.StoreState.Unknown) - .setStartTimestamp(0L).setDeployPath("deployPath") - .setLastHeartbeat(0L).setStats( - Metapb.StoreStats.newBuilder().setStoreId(0L) - .setPartitionCount(0).addGraphStats( - Metapb.GraphStats.newBuilder() - .setGraphName("value") - .setApproximateSize(0L) - .setRole(Metapb.ShardRole.None) - .build()).build()) - .setDataVersion(0).setCores(0) - .setDataPath("dataPath").build()); - - // Run the test - final List result = this.service.getActiveStores("graphName"); - - // Verify the results - } - - @Test - public void testGetActiveStores1ThrowsPDException() { - try { - List stores = this.service.getActiveStores(); - assertThat(stores.size() == 0); - } catch (Exception e) { - - } - } - - @Test - public void testGetTombStores() throws Exception { - // Setup - final List storeList = List.of( - Metapb.Store.newBuilder().setId(0L).setAddress("address") - .setRaftAddress("raftAddress") - .addLabels(Metapb.StoreLabel.newBuilder().build()) - .setVersion("version") - .setState(Metapb.StoreState.Tombstone) - .setStartTimestamp(0L).setDeployPath("deployPath") - .setLastHeartbeat(0L).setStats( - Metapb.StoreStats.newBuilder().setStoreId(0L) - .setPartitionCount(0).addGraphStats( - Metapb.GraphStats.newBuilder() - .setGraphName("value") - .setApproximateSize(0L) - .setRole(Metapb.ShardRole.None) - .build()).build()) - .setDataVersion(0).setCores(0) - .setDataPath("dataPath").build()); - this.service.register(storeList.get(0)); - - // Run the test - final List result = this.service.getTombStores(); - - // Verify the results - assertThat(result.size() == 1); - this.service.removeStore(result.get(0).getId()); - List stores = this.service.getStores(); - assertThat(stores.size() == 0); - } - - - @Test - public void testAllocShards() throws Exception { - // Setup - try { - final Metapb.Graph graph = Metapb.Graph.newBuilder() - .setGraphName("graphName") - .setGraphState( - Metapb.GraphState - .newBuilder() - .setMode( - Metapb.GraphMode.ReadWrite) - .setReason( - Metapb.GraphModeReason.Quota) - .build()) - .build(); - final List expectedResult = List.of( - Metapb.Shard.newBuilder().setStoreId(0L) - .setRole(Metapb.ShardRole.None).build()); - - // Configure PDConfig.getPartition(...). - final PDConfig.Partition partition = new PDConfig().new Partition(); - partition.setTotalCount(0); - partition.setMaxShardsPerStore(0); - partition.setShardCount(0); - - // Run the test - final List result = this.service.allocShards(graph, 0); - } catch (Exception e) { - - } - - } - - @Test - public void testReallocShards() throws Exception { - // Setup - try { - final Metapb.ShardGroup shardGroup = Metapb.ShardGroup.newBuilder() - .setId(0) - .addShards( - Metapb.Shard - .newBuilder() - .setStoreId( - 0L) - .setRole( - Metapb.ShardRole.None) - .build()) - .setState( - Metapb.PartitionState.PState_None) - .build(); - final List expectedResult = List.of( - Metapb.Shard.newBuilder().setStoreId(0L) - .setRole(Metapb.ShardRole.None).build()); - - // Configure PDConfig.getPartition(...). - final PDConfig.Partition partition = new PDConfig().new Partition(); - partition.setTotalCount(0); - partition.setMaxShardsPerStore(0); - partition.setShardCount(0); - when(this.config.getPartition()).thenReturn(partition); - - // Run the test - final List result = this.service.reallocShards(shardGroup); - - // Verify the results - assertThat(result).isEqualTo(expectedResult); - } catch (Exception e) { - - } - - } - - @Test - public void testUpdateShardGroup() { - try { - final List shards = List.of( - Metapb.Shard.newBuilder().setStoreId(0L) - .setRole(Metapb.ShardRole.None).build()); - - // Run the test - this.service.updateShardGroup(0, shards, 0, 0); - } catch (Exception e) { - - } - } - - @Test - public void testUpdateShardGroupState() throws Exception { - try { - this.service.updateShardGroupState(0, Metapb.PartitionState.PState_None); - } catch (Exception e) { - - } - } - - @Test - public void testHeartBeat() throws Exception { - // Setup - try { - final Metapb.StoreStats storeStats = Metapb.StoreStats.newBuilder() - .setStoreId( - 0L) - .setPartitionCount( - 0) - .addGraphStats( - Metapb.GraphStats - .newBuilder() - .setGraphName( - "value") - .setApproximateSize( - 0L) - .setRole( - Metapb.ShardRole.None) - .build()) - .build(); - final Metapb.ClusterStats expectedResult = Metapb.ClusterStats - .newBuilder().setState(Metapb.ClusterState.Cluster_OK) - .setMessage("message").setTimestamp(0L).build(); - when(this.config.getMinStoreCount()).thenReturn(0); - - // Configure PDConfig.getPartition(...). - final PDConfig.Partition partition = new PDConfig().new Partition(); - partition.setTotalCount(0); - partition.setMaxShardsPerStore(0); - partition.setShardCount(0); - when(this.config.getPartition()).thenReturn(partition); - - // Run the test - final Metapb.ClusterStats result = this.service.heartBeat(storeStats); - - // Verify the results - assertThat(result).isEqualTo(expectedResult); - } catch (Exception e) { - - } - } - - - @Test - public void testUpdateClusterStatus1() { - // Setup - final Metapb.ClusterStats expectedResult = Metapb.ClusterStats - .newBuilder().setState(Metapb.ClusterState.Cluster_OK) - .setMessage("message").setTimestamp(0L).build(); - - // Run the test - final Metapb.ClusterStats result = this.service.updateClusterStatus( - Metapb.ClusterState.Cluster_OK); - } - - @Test - public void testUpdateClusterStatus2() { - // Setup - final Metapb.ClusterStats expectedResult = Metapb.ClusterStats - .newBuilder().setState(Metapb.ClusterState.Cluster_OK) - .setMessage("message").setTimestamp(0L).build(); - - // Run the test - final Metapb.ClusterStats result = this.service.updateClusterStatus( - Metapb.PartitionState.PState_None); - } - - @Test - public void testCheckStoreStatus() { - // Setup - // Run the test - this.service.checkStoreStatus(); - - // Verify the results - } - - @Test - public void testAddStatusListener() { - // Setup - final StoreStatusListener mockListener = mock( - StoreStatusListener.class); - - // Run the test - this.service.addStatusListener(mockListener); - - // Verify the results - } - - @Test - public void testOnStoreStatusChanged() { - // Setup - final Metapb.Store store = Metapb.Store.newBuilder().setId(0L) - .setAddress("address") - .setRaftAddress("raftAddress") - .addLabels(Metapb.StoreLabel - .newBuilder() - .build()) - .setVersion("version").setState( - Metapb.StoreState.Unknown).setStartTimestamp(0L) - .setDeployPath("deployPath") - .setLastHeartbeat(0L).setStats( - Metapb.StoreStats.newBuilder().setStoreId(0L) - .setPartitionCount(0).addGraphStats( - Metapb.GraphStats.newBuilder() - .setGraphName("value") - .setApproximateSize(0L) - .setRole(Metapb.ShardRole.None) - .build()).build()) - .setDataVersion(0).setCores(0) - .setDataPath("dataPath").build(); - - // Verify the results - } - - @Test - public void testOnShardGroupSplit() { - // Setup - final Metapb.ShardGroup shardGroup = Metapb.ShardGroup.newBuilder() - .setId(0) - .addShards( - Metapb.Shard - .newBuilder() - .setStoreId( - 0L) - .setRole( - Metapb.ShardRole.None) - .build()) - .setState( - Metapb.PartitionState.PState_None) - .build(); - final List newShardGroups = List.of( - Metapb.ShardGroup.newBuilder().setId(0).addShards( - Metapb.Shard.newBuilder().setStoreId(0L) - .setRole(Metapb.ShardRole.None).build()) - .setState(Metapb.PartitionState.PState_None) - .build()); - final Consumer mockTask = mock(Consumer.class); - - // Verify the results - } - - @Test - public void testCheckStoreCanOffline() { - // Setup - final Metapb.Store currentStore = Metapb.Store.newBuilder().setId(0L) - .setAddress("address") - .setRaftAddress( - "raftAddress") - .addLabels( - Metapb.StoreLabel - .newBuilder() - .build()) - .setVersion("version") - .setState( - Metapb.StoreState.Unknown) - .setStartTimestamp(0L) - .setDeployPath( - "deployPath") - .setLastHeartbeat(0L) - .setStats( - Metapb.StoreStats - .newBuilder() - .setStoreId( - 0L) - .setPartitionCount( - 0) - .addGraphStats( - Metapb.GraphStats - .newBuilder() - .setGraphName( - "value") - .setApproximateSize( - 0L) - .setRole( - Metapb.ShardRole.None) - .build()) - .build()) - .setDataVersion(0) - .setCores(0) - .setDataPath("dataPath") - .build(); - // Run the test - final boolean result = this.service.checkStoreCanOffline(currentStore); - - // Verify the results - assertThat(result).isTrue(); - } - - @Test - public void testShardGroupsDbCompaction() throws Exception { - // Setup - // Run the test - try { - this.service.shardGroupsDbCompaction(0, "tableName"); - } catch (Exception e) { - - } - - // Verify the results - } - - @Test - public void testGetQuota() throws Exception { - // Setup - // Run the test - try { - this.service.getQuota(); - } catch (Exception e) { - - } - } -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/TaskScheduleServiceTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/TaskScheduleServiceTest.java deleted file mode 100644 index 7966bb7125..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/service/TaskScheduleServiceTest.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.service; - -import static org.junit.Assert.assertTrue; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.hugegraph.pd.TaskScheduleService; -import org.apache.hugegraph.pd.common.PDException; -import org.apache.hugegraph.pd.grpc.Metapb; -import org.junit.Before; -import org.junit.Test; - -public class TaskScheduleServiceTest extends PdTestBase { - - TaskScheduleService service; - - @Before - public void init() { - this.service = getTaskService(); - } - - @Test - public void testStoreOffline() { - - } - - public void testPatrolStores() { - - } - - public void testPatrolPartitions() { - - } - - public void testBalancePartitionShard() { - - } - - @Test - public void testBalancePartitionLeader() throws PDException { - - var list = new ArrayList(); - for (int i = 0; i < 6; i++) { - getStoreNodeService().getStoreInfoMeta().updateShardGroup(genShardGroup(i)); - list.add(genPartition(i)); - } - - getPdConfig().getPartition().setShardCount(3); - - getPartitionService().updatePartition(list); - var rst = this.service.balancePartitionLeader(true); - assertTrue(rst.size() > 0); - // recover - getPdConfig().getPartition().setShardCount(1); - getStoreNodeService().getStoreInfoMeta().removeAll(); - } - - public void testSplitPartition() { - - } - - public void testSplitPartition2() { - - } - - public void testCanAllPartitionsMovedOut() { - - } - - private Metapb.ShardGroup genShardGroup(int groupId) { - return Metapb.ShardGroup.newBuilder() - .setId(groupId) - .addAllShards(genShards()) - .build(); - } - - private Metapb.Partition genPartition(int groupId) { - return Metapb.Partition.newBuilder() - .setId(groupId) - .setState(Metapb.PartitionState.PState_Normal) - .setGraphName("graph1") - .setStartKey(groupId * 10) - .setEndKey(groupId * 10 + 10) - .build(); - } - - private List genShards() { - return List.of( - Metapb.Shard.newBuilder().setStoreId(1).setRole(Metapb.ShardRole.Leader).build(), - Metapb.Shard.newBuilder().setStoreId(2).setRole(Metapb.ShardRole.Follower).build(), - Metapb.Shard.newBuilder().setStoreId(3).setRole(Metapb.ShardRole.Follower).build()); - } - -} diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImplTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImplTest.java deleted file mode 100644 index 342ac9bc44..0000000000 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImplTest.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hugegraph.pd.store; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Paths; - -import org.apache.commons.io.FileUtils; -import org.apache.hugegraph.pd.common.PDException; -import org.apache.hugegraph.pd.config.PDConfig; -import org.junit.Assert; -import org.junit.BeforeClass; - -public class HgKVStoreImplTest { - static final String testPath = "tmp/test"; - static PDConfig pdConfig; - - @BeforeClass - public static void init() throws IOException { - File testFile = new File(testPath); - if (testFile.exists()) { - FileUtils.deleteDirectory(testFile); - } - FileUtils.forceMkdir(testFile); - pdConfig = new PDConfig() {{ - setDataPath(testPath); - }}; - } - - // @Test - public void Test() throws PDException { - HgKVStore kvStore = new HgKVStoreImpl(); - kvStore.init(pdConfig); - - { - byte[] key = "hello".getBytes(); - byte[] value = "pd".getBytes(); - kvStore.put(key, value); - } - for (int i = 0; i < 100; i++) { - byte[] key = String.format("k%03d", i).getBytes(); - byte[] value = ("value" + i).getBytes(); - kvStore.put(key, value); - } - - Assert.assertEquals(100, kvStore.scanPrefix("k".getBytes()).size()); - - kvStore.removeByPrefix("k".getBytes()); - Assert.assertEquals(0, kvStore.scanPrefix("k".getBytes()).size()); - } - - // @Test - public void TestSnapshot() throws PDException { - HgKVStore kvStore = new HgKVStoreImpl(); - kvStore.init(pdConfig); - - // put 100 data - for (int i = 0; i < 100; i++) { - byte[] key = String.format("k%03d", i).getBytes(); - byte[] value = ("value" + i).getBytes(); - kvStore.put(key, value); - } - Assert.assertEquals(100, kvStore.scanPrefix("k".getBytes()).size()); - - // save snapshot - String snapshotPath = Paths.get(testPath, "snapshot").toString(); - kvStore.saveSnapshot(snapshotPath); - - // put another 100 data - for (int i = 100; i < 200; i++) { - byte[] key = String.format("k%03d", i).getBytes(); - byte[] value = ("value" + i).getBytes(); - kvStore.put(key, value); - } - Assert.assertEquals(200, kvStore.scanPrefix("k".getBytes()).size()); - - // load snapshot - kvStore.loadSnapshot(snapshotPath); - Assert.assertEquals(100, kvStore.scanPrefix("k".getBytes()).size()); - - // put another 100 data - for (int i = 100; i < 200; i++) { - byte[] key = String.format("k%03d", i).getBytes(); - byte[] value = ("value" + i).getBytes(); - kvStore.put(key, value); - } - Assert.assertEquals(200, kvStore.scanPrefix("k".getBytes()).size()); - } -} diff --git a/hugegraph-pd/pom.xml b/hugegraph-pd/pom.xml index 49fd28f384..6dde68c402 100644 --- a/hugegraph-pd/pom.xml +++ b/hugegraph-pd/pom.xml @@ -147,79 +147,7 @@ - - - pd-client-test - - true - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.20 - - - pd-client-test - - test - - test - - - - - - - - pd-core-test - - true - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.20 - - - pd-core-test - - test - - test - - - - - - - - pd-cli-tools-test - - true - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.20 - - - pd-cli-tools-test - - test - - test - - - - - - + pd-common-test @@ -244,29 +172,5 @@ - - pd-service-test - - true - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.20 - - - pd-service-test - - test - - test - - - - - - diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh new file mode 100644 index 0000000000..6da114a707 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -ev + +HOME_DIR=$(pwd) +PD_DIR=$HOME_DIR/hugegraph-pd/dist/hugegraph-pd-1.5.0.1 + +pushd $PD_DIR +. bin/start-hugegraph-pd.sh +sleep 10 +popd From 4af39d03dc07f7d4fc1b1035f200e542d3719922 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Mon, 26 Feb 2024 20:36:14 +0800 Subject: [PATCH 06/17] fix: update DEFAULT_VERSION --- .../src/main/java/org/apache/hugegraph/version/CoreVersion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/version/CoreVersion.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/version/CoreVersion.java index 84dc28dedc..d878ab6ea3 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/version/CoreVersion.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/version/CoreVersion.java @@ -23,7 +23,7 @@ public class CoreVersion { public static final String NAME = "hugegraph-core"; - public static final String DEFAULT_VERSION = "1.2.0"; + public static final String DEFAULT_VERSION = "1.5.0.1"; /** * The second parameter of Version.of() is for IDE running without JAR */ From 9ce9fc254667f2a6f779fd169ac66bb16288c4e7 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Mon, 26 Feb 2024 20:43:42 +0800 Subject: [PATCH 07/17] chore: remove start-pd.sh --- .../src/assembly/travis/start-pd.sh | 26 ------------------- 1 file changed, 26 deletions(-) delete mode 100644 hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh deleted file mode 100644 index 6da114a707..0000000000 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -ev - -HOME_DIR=$(pwd) -PD_DIR=$HOME_DIR/hugegraph-pd/dist/hugegraph-pd-1.5.0.1 - -pushd $PD_DIR -. bin/start-hugegraph-pd.sh -sleep 10 -popd From f60af68e0a58973387f7352a0526c8769bca010b Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 28 Feb 2024 12:12:53 +0800 Subject: [PATCH 08/17] chore: format hg-pd-test --- .../hugegraph/pd/common/BaseCommonTest.java | 1 + .../hugegraph/pd/common/CommonSuiteTest.java | 2 - .../hugegraph/pd/common/HgAssertTest.java | 2 - .../hugegraph/pd/common/KVPairTest.java | 1 - .../pd/common/PartitionUtilsTest.java | 3 +- .../hg-pd-test/src/main/resources/log4j2.xml | 68 +++++++++---------- 6 files changed, 37 insertions(+), 40 deletions(-) diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/BaseCommonTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/BaseCommonTest.java index 591779c0d5..fb4478e3d6 100644 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/BaseCommonTest.java +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/BaseCommonTest.java @@ -21,6 +21,7 @@ import org.junit.BeforeClass; public class BaseCommonTest { + @BeforeClass public static void init() { diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/CommonSuiteTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/CommonSuiteTest.java index 0395711caa..02a5dfca64 100644 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/CommonSuiteTest.java +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/CommonSuiteTest.java @@ -22,7 +22,6 @@ import lombok.extern.slf4j.Slf4j; - @RunWith(Suite.class) @Suite.SuiteClasses({ PartitionUtilsTest.class, @@ -34,5 +33,4 @@ @Slf4j public class CommonSuiteTest { - } diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/HgAssertTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/HgAssertTest.java index 253a9c67a0..3e61dd0a94 100644 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/HgAssertTest.java +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/HgAssertTest.java @@ -23,7 +23,6 @@ import java.util.ArrayList; import java.util.HashMap; -import org.apache.hugegraph.pd.common.HgAssert; import org.junit.Test; public class HgAssertTest { @@ -78,7 +77,6 @@ public void testIsNotNull() { HgAssert.isNotNull(null, ""); } - @Test public void testIsInvalid() { assertFalse(HgAssert.isInvalid("abc", "test")); diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/KVPairTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/KVPairTest.java index b3e1c15adf..9fb676d392 100644 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/KVPairTest.java +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/KVPairTest.java @@ -17,7 +17,6 @@ package org.apache.hugegraph.pd.common; -import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; import org.junit.Assert; diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionUtilsTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionUtilsTest.java index 551bd40ecc..e0742a4838 100644 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionUtilsTest.java +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/common/PartitionUtilsTest.java @@ -18,10 +18,11 @@ package org.apache.hugegraph.pd.common; import java.nio.charset.StandardCharsets; -import lombok.extern.slf4j.Slf4j; + import org.junit.Assert; import org.junit.Test; +import lombok.extern.slf4j.Slf4j; @Slf4j public class PartitionUtilsTest extends BaseCommonTest { diff --git a/hugegraph-pd/hg-pd-test/src/main/resources/log4j2.xml b/hugegraph-pd/hg-pd-test/src/main/resources/log4j2.xml index 82f1bce8c4..e462bf16e9 100644 --- a/hugegraph-pd/hg-pd-test/src/main/resources/log4j2.xml +++ b/hugegraph-pd/hg-pd-test/src/main/resources/log4j2.xml @@ -26,32 +26,32 @@ - - + + - - + + - - + + - + - - + + @@ -61,24 +61,24 @@ - - + + - - + + - + - - + + @@ -88,25 +88,25 @@ - + - + - - + + - - + + - - + + @@ -115,25 +115,25 @@ - - + + - - + + - - + + - - + + - - + + From 8a671dea48ec16ac506b5543de750c2e14d4d99d Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 28 Feb 2024 13:24:32 +0800 Subject: [PATCH 09/17] chore: format hg-pd-client --- .../hugegraph/pd/client/DiscoveryClient.java | 12 +++---- .../pd/client/DiscoveryClientImpl.java | 3 -- .../apache/hugegraph/pd/client/KvClient.java | 33 +++++++------------ .../apache/hugegraph/pd/client/PDClient.java | 2 ++ .../apache/hugegraph/pd/client/PDConfig.java | 1 + .../apache/hugegraph/pd/client/PDPulse.java | 2 ++ .../hugegraph/pd/client/PDPulseImpl.java | 3 +- .../apache/hugegraph/pd/client/PDWatch.java | 33 ++++++++++--------- .../hugegraph/pd/client/PDWatchImpl.java | 30 +++++++++-------- .../hugegraph/pd/pulse/PartitionNotice.java | 1 + .../hugegraph/pd/pulse/PulseServerNotice.java | 1 + .../apache/hugegraph/pd/watch/NodeEvent.java | 1 + .../hugegraph/pd/watch/PartitionEvent.java | 1 + 13 files changed, 63 insertions(+), 60 deletions(-) diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClient.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClient.java index 890e9e5864..7a9f28c013 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClient.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClient.java @@ -124,20 +124,20 @@ private void resetChannel(String singleAddress) throws PDException { try { if (requireResetStub.get()) { while (channel != null && !channel.shutdownNow().awaitTermination( - 100, TimeUnit.MILLISECONDS)) { + 100, TimeUnit.MILLISECONDS)) { continue; } channel = ManagedChannelBuilder.forTarget( - singleAddress).usePlaintext().build(); + singleAddress).usePlaintext().build(); this.registerStub = DiscoveryServiceGrpc.newBlockingStub( - channel); + channel); this.blockingStub = DiscoveryServiceGrpc.newBlockingStub( - channel); + channel); requireResetStub.set(false); } } catch (Exception e) { throw new PDException(-1, String.format( - "Reset channel with error : %s.", e.getMessage())); + "Reset channel with error : %s.", e.getMessage())); } finally { readWriteLock.writeLock().unlock(); } @@ -209,7 +209,7 @@ public void close() { readWriteLock.writeLock().lock(); try { while (channel != null && !channel.shutdownNow().awaitTermination( - 100, TimeUnit.MILLISECONDS)) { + 100, TimeUnit.MILLISECONDS)) { continue; } } catch (Exception e) { diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImpl.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImpl.java index 1208370d3f..0ded328c17 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImpl.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/DiscoveryClientImpl.java @@ -34,7 +34,6 @@ public class DiscoveryClientImpl extends DiscoveryClient { private final Map labels; private final Consumer registerConsumer; - private DiscoveryClientImpl(Builder builder) { super(builder.centerAddress, builder.delay); period = builder.delay; @@ -52,7 +51,6 @@ public static Builder newBuilder() { return new Builder(); } - @Override NodeInfo getRegisterNode() { return NodeInfo.newBuilder().setAddress(this.address) @@ -66,7 +64,6 @@ Consumer getRegisterConsumer() { return registerConsumer; } - public static final class Builder { private int delay; diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/KvClient.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/KvClient.java index 3e5c2359ed..7e0795b2e4 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/KvClient.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/KvClient.java @@ -28,8 +28,6 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; -import org.apache.hugegraph.pd.client.AbstractClient; -import org.apache.hugegraph.pd.client.PDConfig; import org.apache.hugegraph.pd.common.PDException; import org.apache.hugegraph.pd.grpc.kv.K; import org.apache.hugegraph.pd.grpc.kv.KResponse; @@ -80,7 +78,6 @@ public KvResponse put(String key, String value) throws PDException { return response; } - public KResponse get(String key) throws PDException { K k = K.newBuilder().setKey(key).build(); KResponse response = blockingUnaryCall(KvServiceGrpc.getGetMethod(), k); @@ -88,7 +85,6 @@ public KResponse get(String key) throws PDException { return response; } - public KvResponse delete(String key) throws PDException { K k = K.newBuilder().setKey(key).build(); KvResponse response = blockingUnaryCall(KvServiceGrpc.getDeleteMethod(), k); @@ -96,7 +92,6 @@ public KvResponse delete(String key) throws PDException { return response; } - public KvResponse deletePrefix(String prefix) throws PDException { K k = K.newBuilder().setKey(prefix).build(); KvResponse response = blockingUnaryCall(KvServiceGrpc.getDeletePrefixMethod(), k); @@ -104,7 +99,6 @@ public KvResponse deletePrefix(String prefix) throws PDException { return response; } - public ScanPrefixResponse scanPrefix(String prefix) throws PDException { K k = K.newBuilder().setKey(prefix).build(); ScanPrefixResponse response = blockingUnaryCall(KvServiceGrpc.getScanPrefixMethod(), k); @@ -183,7 +177,6 @@ public void onError(Throwable t) { listenWrapper.accept(key, consumer); } - @Override public void onCompleted() { @@ -269,17 +262,7 @@ public LockResponse lock(String key, long ttl) throws PDException { throw e; } return response; - } BiConsumer listenWrapper = (key, consumer) -> { - try { - listen(key, consumer); - } catch (PDException e) { - try { - log.warn("start listen with warning:", e); - Thread.sleep(1000); - } catch (InterruptedException ex) { - } - } - }; + } public LockResponse lockWithoutReentrant(String key, long ttl) throws PDException { acquire(); @@ -334,7 +317,17 @@ public void close() { super.close(); } - + BiConsumer listenWrapper = (key, consumer) -> { + try { + listen(key, consumer); + } catch (PDException e) { + try { + log.warn("start listen with warning:", e); + Thread.sleep(1000); + } catch (InterruptedException ex) { + } + } + }; BiConsumer prefixListenWrapper = (key, consumer) -> { try { @@ -347,6 +340,4 @@ public void close() { } } }; - - } diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDClient.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDClient.java index fa1e595264..6c3eae4251 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDClient.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDClient.java @@ -57,6 +57,7 @@ */ @Slf4j public class PDClient { + private final PDConfig config; private final Pdpb.RequestHeader header; private final ClientCache cache; @@ -1287,6 +1288,7 @@ public void updatePdRaft(String raftConfig) throws PDException { } public interface PDEventListener { + void onStoreChanged(NodeEvent event); void onPartitionChanged(PartitionEvent event); diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDConfig.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDConfig.java index 16686a265b..a1c72a2bcf 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDConfig.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDConfig.java @@ -18,6 +18,7 @@ package org.apache.hugegraph.pd.client; public final class PDConfig { + //TODO multi-server private String serverHost = "localhost:9000"; private long grpcTimeOut = 60000; // grpc调用超时时间 10秒 diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulse.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulse.java index 4b9bf32c4e..485417b917 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulse.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulse.java @@ -90,6 +90,7 @@ public void onCompleted() { * Interface of pulse. */ interface Listener { + /** * Invoked on new events. * @@ -128,6 +129,7 @@ default void onNotice(PulseServerNotice notice) { * @param */ interface Notifier extends Closeable { + /** * closes this watcher and all its resources. */ diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulseImpl.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulseImpl.java index d01cd42d70..0afc10c831 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulseImpl.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDPulseImpl.java @@ -55,7 +55,6 @@ public PDPulseImpl(String pdServerAddress) { threadPool = Executors.newSingleThreadExecutor(namedThreadFactory); } - private String getCurrentHost() { return this.pdServerAddress; } @@ -93,6 +92,7 @@ public boolean resetStub(String host, Notifier notifier) { private class PartitionHeartbeat extends AbstractConnector { + private long observerId = -1; PartitionHeartbeat(Listener listener) { @@ -131,6 +131,7 @@ public void onNext(PulseResponse pulseResponse) { private abstract class AbstractConnector implements Notifier, StreamObserver { + Listener listener; StreamObserver reqStream; PulseType pulseType; diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatch.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatch.java index 1428fe762d..c6c46d03d1 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatch.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatch.java @@ -34,19 +34,6 @@ public interface PDWatch { */ //PDWatcher watchNode(Listener listener); - /** - * Watch the events of the store-nodes assigned to a specified graph. - * - * @param graph the graph name which you want to watch - * @param listener - * @return - */ - //PDWatcher watchNode(String graph, Listener listener); - - String getCurrentHost(); - - boolean checkChannel(); - /*** inner static methods ***/ static Listener listener(Consumer onNext) { return listener(onNext, t -> { @@ -84,6 +71,19 @@ public void onCompleted() { }; } + /** + * Watch the events of the store-nodes assigned to a specified graph. + * + * @param graph the graph name which you want to watch + * @param listener + * @return + */ + //PDWatcher watchNode(String graph, Listener listener); + + String getCurrentHost(); + + boolean checkChannel(); + /** * @param listener * @return @@ -96,11 +96,11 @@ public void onCompleted() { Watcher watchShardGroup(Listener listener); - /** * Interface of Watcher. */ interface Listener { + /** * Invoked on new events. * @@ -118,10 +118,13 @@ interface Listener { /** * Invoked on completion. */ - default void onCompleted() {}; + default void onCompleted() { + } + } interface Watcher extends Closeable { + /** * closes this watcher and all its resources. */ diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatchImpl.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatchImpl.java index a35893a33a..9b136bb26a 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatchImpl.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/PDWatchImpl.java @@ -91,9 +91,9 @@ private class GraphWatcher extends AbstractWatcher { private GraphWatcher(Listener listener) { super(listener, () -> WatchCreateRequest - .newBuilder() - .setWatchType(WatchType.WATCH_TYPE_GRAPH_CHANGE) - .build() + .newBuilder() + .setWatchType(WatchType.WATCH_TYPE_GRAPH_CHANGE) + .build() ); } @@ -108,9 +108,9 @@ private class ShardGroupWatcher extends AbstractWatcher { private ShardGroupWatcher(Listener listener) { super(listener, () -> WatchCreateRequest - .newBuilder() - .setWatchType(WatchType.WATCH_TYPE_SHARD_GROUP_CHANGE) - .build() + .newBuilder() + .setWatchType(WatchType.WATCH_TYPE_SHARD_GROUP_CHANGE) + .build() ); } @@ -125,9 +125,9 @@ private class PartitionWatcher extends AbstractWatcher { private PartitionWatcher(Listener listener) { super(listener, () -> WatchCreateRequest - .newBuilder() - .setWatchType(WatchType.WATCH_TYPE_PARTITION_CHANGE) - .build() + .newBuilder() + .setWatchType(WatchType.WATCH_TYPE_PARTITION_CHANGE) + .build() ); } @@ -136,18 +136,19 @@ public void onNext(WatchResponse watchResponse) { WatchPartitionResponse res = watchResponse.getPartitionResponse(); PartitionEvent event = new PartitionEvent(res.getGraph(), res.getPartitionId(), PartitionEvent.ChangeType.grpcTypeOf( - res.getChangeType())); + res.getChangeType())); this.listener.onNext(event); } } private class NodeWatcher extends AbstractWatcher { + private NodeWatcher(Listener listener) { super(listener, () -> WatchCreateRequest - .newBuilder() - .setWatchType(WatchType.WATCH_TYPE_STORE_NODE_CHANGE) - .build() + .newBuilder() + .setWatchType(WatchType.WATCH_TYPE_STORE_NODE_CHANGE) + .build() ); } @@ -161,6 +162,7 @@ public void onNext(WatchResponse watchResponse) { } private abstract class AbstractWatcher implements Watcher, StreamObserver { + Listener listener; StreamObserver reqStream; Supplier requestSupplier; @@ -175,7 +177,7 @@ private AbstractWatcher(Listener listener, void init() { this.reqStream = PDWatchImpl.this.stub.watch(this); this.reqStream.onNext(WatchRequest.newBuilder().setCreateRequest( - this.requestSupplier.get() + this.requestSupplier.get() ).build()); } diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PartitionNotice.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PartitionNotice.java index f8ee9b994c..80aa8951b7 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PartitionNotice.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PartitionNotice.java @@ -22,6 +22,7 @@ import org.apache.hugegraph.pd.grpc.pulse.PulseResponse; public class PartitionNotice implements PulseServerNotice { + private final long noticeId; private final Consumer ackConsumer; private final PulseResponse content; diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PulseServerNotice.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PulseServerNotice.java index 0c83f2b06e..9a30e2679a 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PulseServerNotice.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/pulse/PulseServerNotice.java @@ -18,6 +18,7 @@ package org.apache.hugegraph.pd.pulse; public interface PulseServerNotice { + /** * @throws RuntimeException when failed to send ack-message to pd-server */ diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/NodeEvent.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/NodeEvent.java index 484d9a08c7..bb68383b83 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/NodeEvent.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/NodeEvent.java @@ -22,6 +22,7 @@ import org.apache.hugegraph.pd.grpc.watch.NodeEventType; public class NodeEvent { + private final String graph; private final long nodeId; private final EventType eventType; diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/PartitionEvent.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/PartitionEvent.java index 0211cf8ec9..e5be1b3484 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/PartitionEvent.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/watch/PartitionEvent.java @@ -22,6 +22,7 @@ import org.apache.hugegraph.pd.grpc.watch.WatchChangeType; public class PartitionEvent { + private final String graph; private final int partitionId; private final ChangeType changeType; From c1cf4a2c9de1e963255175b357ae571cccbb4d54 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 28 Feb 2024 13:26:08 +0800 Subject: [PATCH 10/17] chore: format hg-pd-common --- .../java/org/apache/hugegraph/pd/common/PartitionCache.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionCache.java b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionCache.java index 8470faae0d..9bd233fd21 100644 --- a/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionCache.java +++ b/hugegraph-pd/hg-pd-common/src/main/java/org/apache/hugegraph/pd/common/PartitionCache.java @@ -42,18 +42,16 @@ public class PartitionCache { // 读写锁对象 private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); + private final Map locks = new HashMap<>(); Lock writeLock = readWriteLock.writeLock(); // 每张图一个缓存 private volatile Map> keyToPartIdCache; // graphName + PartitionID 组成 key private volatile Map> partitionCache; - private volatile Map shardGroupCache; private volatile Map storeCache; private volatile Map graphCache; - private final Map locks = new HashMap<>(); - public PartitionCache() { keyToPartIdCache = new HashMap<>(); partitionCache = new HashMap<>(); From f086c44479966589273e0314732ad77f54a7ffae Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 28 Feb 2024 13:26:47 +0800 Subject: [PATCH 11/17] chore: format hg-pd-grpc --- .../hg-pd-grpc/src/main/proto/pd_pulse.proto | 32 +++++++++---------- .../hg-pd-grpc/src/main/proto/pd_watch.proto | 12 +++---- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_pulse.proto b/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_pulse.proto index f6c5224536..fb8940df6c 100644 --- a/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_pulse.proto +++ b/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_pulse.proto @@ -66,14 +66,14 @@ message PartitionHeartbeatRequest { /* responses */ message PulseResponse { - PulseType pulse_type = 1; - int64 observer_id = 2; - int32 status = 3; //0=ok,1=fail - int64 notice_id=4; - oneof response_union { - PartitionHeartbeatResponse partition_heartbeat_response = 10; - PdInstructionResponse instruction_response = 11; - } + PulseType pulse_type = 1; + int64 observer_id = 2; + int32 status = 3; //0=ok,1=fail + int64 notice_id = 4; + oneof response_union { + PartitionHeartbeatResponse partition_heartbeat_response = 10; + PdInstructionResponse instruction_response = 11; + } } message PartitionHeartbeatResponse { @@ -136,15 +136,15 @@ message PartitionKeyRange{ } message PdInstructionResponse { - PdInstructionType instruction_type = 1; - string leader_ip = 2; + PdInstructionType instruction_type = 1; + string leader_ip = 2; } /* enums */ enum PulseType { - PULSE_TYPE_UNKNOWN = 0; - PULSE_TYPE_PARTITION_HEARTBEAT = 1; - PULSE_TYPE_PD_INSTRUCTION = 2; + PULSE_TYPE_UNKNOWN = 0; + PULSE_TYPE_PARTITION_HEARTBEAT = 1; + PULSE_TYPE_PD_INSTRUCTION = 2; } enum PulseChangeType { @@ -163,10 +163,10 @@ enum ConfChangeType { } enum CleanType { - CLEAN_TYPE_KEEP_RANGE = 0; // 仅保留这个range - CLEAN_TYPE_EXCLUDE_RANGE = 1; // 删除这个range + CLEAN_TYPE_KEEP_RANGE = 0; // 仅保留这个range + CLEAN_TYPE_EXCLUDE_RANGE = 1; // 删除这个range } enum PdInstructionType { - CHANGE_TO_FOLLOWER = 0; + CHANGE_TO_FOLLOWER = 0; } diff --git a/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_watch.proto b/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_watch.proto index 4911307260..febc41f522 100644 --- a/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_watch.proto +++ b/hugegraph-pd/hg-pd-grpc/src/main/proto/pd_watch.proto @@ -94,10 +94,10 @@ enum WatchChangeType { } enum NodeEventType { - NODE_EVENT_TYPE_UNKNOWN = 0; - NODE_EVENT_TYPE_NODE_ONLINE = 1; - NODE_EVENT_TYPE_NODE_OFFLINE = 2; - NODE_EVENT_TYPE_NODE_RAFT_CHANGE = 3; - // pd leader 变更 - NODE_EVENT_TYPE_PD_LEADER_CHANGE = 4; + NODE_EVENT_TYPE_UNKNOWN = 0; + NODE_EVENT_TYPE_NODE_ONLINE = 1; + NODE_EVENT_TYPE_NODE_OFFLINE = 2; + NODE_EVENT_TYPE_NODE_RAFT_CHANGE = 3; + // pd leader 变更 + NODE_EVENT_TYPE_PD_LEADER_CHANGE = 4; } From e4cd901868818cb70cc260d3a143638431d138cd Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 1 Mar 2024 20:02:43 +0800 Subject: [PATCH 12/17] chore: bump jackson.version from 2.13.0 to 2.14.2, grpc.version from 1.39.0 to 1.53.0 --- hugegraph-pd/hg-pd-grpc/pom.xml | 2 +- hugegraph-pd/hg-pd-test/pom.xml | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/hugegraph-pd/hg-pd-grpc/pom.xml b/hugegraph-pd/hg-pd-grpc/pom.xml index cef49e957d..8d2d75cc76 100644 --- a/hugegraph-pd/hg-pd-grpc/pom.xml +++ b/hugegraph-pd/hg-pd-grpc/pom.xml @@ -33,7 +33,7 @@ 1.6.0 - 1.39.0 + 1.53.0 3.17.2 0.6.1 diff --git a/hugegraph-pd/hg-pd-test/pom.xml b/hugegraph-pd/hg-pd-test/pom.xml index 31c0fd889d..1d8211f275 100644 --- a/hugegraph-pd/hg-pd-test/pom.xml +++ b/hugegraph-pd/hg-pd-test/pom.xml @@ -33,6 +33,7 @@ true 2.0.0-RC.3 + 2.14.2 @@ -113,17 +114,17 @@ com.fasterxml.jackson.core jackson-databind - 2.13.0 + ${jackson.version} com.fasterxml.jackson.core jackson-core - 2.13.0 + ${jackson.version} com.fasterxml.jackson.core jackson-annotations - 2.13.0 + ${jackson.version} From 37982e57d58d5864da16e30c18de9b8d657d73e9 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 1 Mar 2024 20:19:35 +0800 Subject: [PATCH 13/17] Revert "chore: bump jackson.version from 2.13.0 to 2.14.2, grpc.version from 1.39.0 to 1.53.0" This reverts commit e4cd901868818cb70cc260d3a143638431d138cd. --- hugegraph-pd/hg-pd-grpc/pom.xml | 2 +- hugegraph-pd/hg-pd-test/pom.xml | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/hugegraph-pd/hg-pd-grpc/pom.xml b/hugegraph-pd/hg-pd-grpc/pom.xml index 8d2d75cc76..cef49e957d 100644 --- a/hugegraph-pd/hg-pd-grpc/pom.xml +++ b/hugegraph-pd/hg-pd-grpc/pom.xml @@ -33,7 +33,7 @@ 1.6.0 - 1.53.0 + 1.39.0 3.17.2 0.6.1 diff --git a/hugegraph-pd/hg-pd-test/pom.xml b/hugegraph-pd/hg-pd-test/pom.xml index 1d8211f275..31c0fd889d 100644 --- a/hugegraph-pd/hg-pd-test/pom.xml +++ b/hugegraph-pd/hg-pd-test/pom.xml @@ -33,7 +33,6 @@ true 2.0.0-RC.3 - 2.14.2 @@ -114,17 +113,17 @@ com.fasterxml.jackson.core jackson-databind - ${jackson.version} + 2.13.0 com.fasterxml.jackson.core jackson-core - ${jackson.version} + 2.13.0 com.fasterxml.jackson.core jackson-annotations - ${jackson.version} + 2.13.0 From 8214ac03e49dfd592f240507380ac6bdcc936a5f Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Fri, 1 Mar 2024 20:21:33 +0800 Subject: [PATCH 14/17] chore: set dependency-review fail-on-severity from low to critical --- .github/workflows/check-dependencies.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check-dependencies.yml b/.github/workflows/check-dependencies.yml index e3632f5d38..311fc3519e 100644 --- a/.github/workflows/check-dependencies.yml +++ b/.github/workflows/check-dependencies.yml @@ -49,7 +49,8 @@ jobs: uses: actions/dependency-review-action@v3 # Refer: https://github.com/actions/dependency-review-action with: - fail-on-severity: low + # TODO: reset critical to low before releasing + fail-on-severity: critical # Action will fail if dependencies don't match the list #allow-licenses: Apache-2.0, MIT #deny-licenses: GPL-3.0, AGPL-1.0, AGPL-3.0, LGPL-2.0, CC-BY-3.0 From 07ed6ffbce3eb68fe0e967c03146f7edac3fa793 Mon Sep 17 00:00:00 2001 From: imbajin Date: Mon, 11 Mar 2024 15:50:12 +0800 Subject: [PATCH 15/17] chore: enhance the compilation in ARM-Mac --- hugegraph-pd/.gitignore | 1 + hugegraph-pd/README.md | 9 ++++++--- pom.xml | 17 ++++++++++++++--- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/hugegraph-pd/.gitignore b/hugegraph-pd/.gitignore index 08bc7dd8af..c2bd7537ea 100644 --- a/hugegraph-pd/.gitignore +++ b/hugegraph-pd/.gitignore @@ -1 +1,2 @@ +# Exclude the generated PB files hg-pd-grpc/src/main/java/ diff --git a/hugegraph-pd/README.md b/hugegraph-pd/README.md index 49548c216d..1aea07d7dd 100644 --- a/hugegraph-pd/README.md +++ b/hugegraph-pd/README.md @@ -1,5 +1,8 @@ -# HugeGraph PD +> Note: From revision 1.5.0, the code of HugeGraph-PD will be adapted to this location (WIP). -HugeGraph PD is a meta server responsible for service discovery, partition information storage, and node scheduling. +# HugeGraph PD (BETA) -> Note: Currently, the contents of this folder are empty. Starting from revision 1.5.0, the code of HugeGraph PD will be adapted to this location (WIP). +HugeGraph PD is a meta server responsible for service discovery, partition information storage, and +node scheduling. + +> BTW, if you meet any problem when using HugeGraph PD, please feel free to contact us for help diff --git a/pom.xml b/pom.xml index 452659e517..a691ac3f0e 100644 --- a/pom.xml +++ b/pom.xml @@ -89,8 +89,6 @@ - - 1.5.0.1 @@ -294,7 +292,7 @@ - + stage @@ -304,5 +302,18 @@ + + + arm-mac + + + mac + aarch64 + + + + osx-x86_64 + + From 16d9e1620a01653cbf0546464f482321bc758fe6 Mon Sep 17 00:00:00 2001 From: imbajin Date: Mon, 11 Mar 2024 16:20:37 +0800 Subject: [PATCH 16/17] chore: clean .flattened-pom.xml TODO: also clean it in other repos --- hugegraph-pd/pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hugegraph-pd/pom.xml b/hugegraph-pd/pom.xml index 6dde68c402..6253cfd443 100644 --- a/hugegraph-pd/pom.xml +++ b/hugegraph-pd/pom.xml @@ -121,6 +121,14 @@ clean + + + remove-flattened-pom + install + + clean + + From 19a71840e088288b34d6cd9156f0bce66d116c1c Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Mon, 11 Mar 2024 16:34:33 +0800 Subject: [PATCH 17/17] chore: remove maven prerequisites --- pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pom.xml b/pom.xml index a691ac3f0e..1706c79ef3 100644 --- a/pom.xml +++ b/pom.xml @@ -84,10 +84,6 @@ scm:git:https://github.com/apache/hugegraph.git - - 3.5.0 - - 1.5.0.1