diff --git a/README.md b/README.md index 2e6b2e78d..11cad8faf 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![Build Status](https://github.com/apache/hugegraph-toolchain/actions/workflows/hubble-ci.yml/badge.svg)](https://github.com/apache/hugegraph-toolchain/actions/workflows/hubble-ci.yml) [![Build Status](https://github.com/apache/hugegraph-toolchain/actions/workflows/tools-ci.yml/badge.svg)](https://github.com/apache/hugegraph-toolchain/actions/workflows/tools-ci.yml) [![codecov](https://codecov.io/gh/apache/hugegraph-toolchain/branch/master/graph/badge.svg)](https://codecov.io/gh/apache/hugegraph-toolchain) -[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.apache.hugegraph/hugegraph-loader/badge.svg)](https://mvnrepository.com/artifact/org.apache.hugegraph/hugegraph-loader) +[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.apache.hugegraph/hugegraph-client/badge.svg)](https://mvnrepository.com/artifact/org.apache.hugegraph/hugegraph-client) `hugegraph-toolchain` is the integration project of a series of utilities for [HugeGraph](https://github.com/apache/hugegraph), it includes 4 main modules. diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/util/CommonUtil.java b/hugegraph-client/src/main/java/org/apache/hugegraph/util/CommonUtil.java index b67d89aad..f430fd07c 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/util/CommonUtil.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/util/CommonUtil.java @@ -17,10 +17,18 @@ package org.apache.hugegraph.util; +import java.io.File; +import java.io.IOException; +import java.net.URL; import java.util.Map; +import org.apache.commons.io.FileUtils; + public final class CommonUtil { + public static final String PREFIX = "https://github.com/apache/incubator-hugegraph-doc/" + + "raw/binary/dist/toolchain/"; + public static void checkMapClass(Object object, Class kClass, Class vClass) { E.checkArgumentNotNull(object, "The object can't be null"); @@ -41,4 +49,14 @@ public static void checkMapClass(Object object, Class kClass, "but got '%s'(%s)", vClass, value, value.getClass()); } } + + public static void downloadFileByUrl(String url, String destPath) { + int connectTimeout = 5000; + int readTimeout = 5000; + try { + FileUtils.copyURLToFile(new URL(url), new File(destPath), connectTimeout, readTimeout); + } catch (IOException e) { + throw new RuntimeException("Failed to download file, please check the network", e); + } + } } diff --git a/hugegraph-client/src/test/java/org/apache/hugegraph/functional/HugeClientHttpsTest.java b/hugegraph-client/src/test/java/org/apache/hugegraph/functional/HugeClientHttpsTest.java index c8c733338..3a1d846b7 100644 --- a/hugegraph-client/src/test/java/org/apache/hugegraph/functional/HugeClientHttpsTest.java +++ b/hugegraph-client/src/test/java/org/apache/hugegraph/functional/HugeClientHttpsTest.java @@ -25,7 +25,9 @@ import org.apache.hugegraph.structure.constant.T; import org.apache.hugegraph.structure.graph.Vertex; import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.util.CommonUtil; import org.junit.After; +import org.junit.Before; import org.junit.Test; import com.google.common.collect.ImmutableMap; @@ -40,11 +42,16 @@ public class HugeClientHttpsTest { private static final int MAX_CONNS_PER_ROUTE = 10; private static final int MAX_CONNS = 10; private static final int IDLE_TIME = 30; - private static final String TRUST_STORE_FILE = "src/test/resources/hugegraph.truststore"; + private static final String TRUST_STORE_PATH = "src/test/resources/hugegraph.truststore"; private static final String TRUST_STORE_PASSWORD = "hugegraph"; private static HugeClient client; + @Before + public void init() { + CommonUtil.downloadFileByUrl(CommonUtil.PREFIX + "hugegraph.truststore", TRUST_STORE_PATH); + } + @After public void teardown() throws Exception { Assert.assertNotNull("Client is not opened", client); @@ -55,7 +62,7 @@ public void teardown() throws Exception { public void testHttpsClientBuilderWithConnection() { client = HugeClient.builder(BASE_URL, GRAPH) .configUser(USERNAME, PASSWORD) - .configSSL(TRUST_STORE_FILE, TRUST_STORE_PASSWORD) + .configSSL(TRUST_STORE_PATH, TRUST_STORE_PASSWORD) .build(); Assert.assertTrue(client.graphs().listGraph().contains("hugegraph")); this.addVertexAndCheckPropertyValue(); @@ -66,7 +73,7 @@ public void testHttpsClientWithConnectionPoolNoUserParam() { client = HugeClient.builder(BASE_URL, GRAPH) .configTimeout(TIMEOUT) .configPool(MAX_CONNS, MAX_CONNS_PER_ROUTE) - .configSSL(TRUST_STORE_FILE, TRUST_STORE_PASSWORD) + .configSSL(TRUST_STORE_PATH, TRUST_STORE_PASSWORD) .build(); Assert.assertTrue(client.graphs().listGraph().contains("hugegraph")); this.addVertexAndCheckPropertyValue(); @@ -77,7 +84,7 @@ public void testHttpsClientWithConnectionPoolNoTimeOutParam() { client = HugeClient.builder(BASE_URL, GRAPH) .configUser(USERNAME, PASSWORD) .configPool(MAX_CONNS, MAX_CONNS_PER_ROUTE) - .configSSL(TRUST_STORE_FILE, TRUST_STORE_PASSWORD) + .configSSL(TRUST_STORE_PATH, TRUST_STORE_PASSWORD) .build(); Assert.assertTrue(client.graphs().listGraph().contains("hugegraph")); this.addVertexAndCheckPropertyValue(); @@ -88,7 +95,7 @@ public void testHttpsClientNewBuilderWithConnectionNoPoolParam() { client = HugeClient.builder(BASE_URL, GRAPH) .configUser(USERNAME, PASSWORD) .configTimeout(TIMEOUT) - .configSSL(TRUST_STORE_FILE, TRUST_STORE_PASSWORD) + .configSSL(TRUST_STORE_PATH, TRUST_STORE_PASSWORD) .build(); Assert.assertTrue(client.graphs().listGraph().contains("hugegraph")); this.addVertexAndCheckPropertyValue(); @@ -100,7 +107,7 @@ public void testHttpsClientNewBuilderWithConnectionPool() { .configUser(USERNAME, PASSWORD) .configTimeout(TIMEOUT) .configPool(MAX_CONNS, MAX_CONNS_PER_ROUTE) - .configSSL(TRUST_STORE_FILE, TRUST_STORE_PASSWORD) + .configSSL(TRUST_STORE_PATH, TRUST_STORE_PASSWORD) .configIdleTime(IDLE_TIME) .build(); Assert.assertTrue(client.graphs().listGraph().contains("hugegraph")); @@ -113,7 +120,7 @@ public void testHttpsClientNewBuilderZeroPoolParam() { .configUser(USERNAME, PASSWORD) .configTimeout(TIMEOUT) .configPool(0, 0) - .configSSL(TRUST_STORE_FILE, TRUST_STORE_PASSWORD) + .configSSL(TRUST_STORE_PATH, TRUST_STORE_PASSWORD) .build(); Assert.assertTrue(client.graphs().listGraph().contains("hugegraph")); this.addVertexAndCheckPropertyValue(); diff --git a/hugegraph-client/src/test/java/org/apache/hugegraph/testutil/Utils.java b/hugegraph-client/src/test/java/org/apache/hugegraph/testutil/Utils.java index 490a28269..1e4dd63fd 100644 --- a/hugegraph-client/src/test/java/org/apache/hugegraph/testutil/Utils.java +++ b/hugegraph-client/src/test/java/org/apache/hugegraph/testutil/Utils.java @@ -135,10 +135,7 @@ public static boolean equalPropertyKey(PropertyKey left, if (left.dataType() != right.dataType()) { return false; } - if (left.cardinality() != right.cardinality()) { - return false; - } - return true; + return left.cardinality() == right.cardinality(); } public static boolean equalVertexLabel(VertexLabel left, diff --git a/hugegraph-client/src/test/resources/hugegraph.truststore b/hugegraph-client/src/test/resources/hugegraph.truststore deleted file mode 100644 index 5b0828dc3..000000000 Binary files a/hugegraph-client/src/test/resources/hugegraph.truststore and /dev/null differ diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/SettingSSLService.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/SettingSSLService.java index 25eef7c6c..74dc1f85f 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/SettingSSLService.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/SettingSSLService.java @@ -18,11 +18,10 @@ package org.apache.hugegraph.service; -import org.springframework.stereotype.Service; - import org.apache.hugegraph.config.HugeConfig; import org.apache.hugegraph.entity.GraphConnection; import org.apache.hugegraph.options.HubbleOptions; +import org.springframework.stereotype.Service; import lombok.extern.log4j.Log4j2; @@ -32,7 +31,7 @@ public class SettingSSLService { public void configSSL(HugeConfig config, GraphConnection connection) { String protocol = config.get(HubbleOptions.SERVER_PROTOCOL); - if (protocol != null && protocol.equals("https")) { + if ("https".equals(protocol)) { connection.setProtocol(protocol); String trustStoreFile = config.get( HubbleOptions.CLIENT_TRUSTSTORE_FILE); diff --git a/hugegraph-hubble/hubble-be/src/main/resources/public-certs.store b/hugegraph-hubble/hubble-be/src/main/resources/public-certs.store deleted file mode 100644 index 0da6d2269..000000000 Binary files a/hugegraph-hubble/hubble-be/src/main/resources/public-certs.store and /dev/null differ diff --git a/hugegraph-hubble/hubble-be/src/main/resources/verify-license.json b/hugegraph-hubble/hubble-be/src/main/resources/verify-license.json deleted file mode 100644 index fed0968a8..000000000 --- a/hugegraph-hubble/hubble-be/src/main/resources/verify-license.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "subject": "hugegraph-license", - "public_alias": "publiccert", - "store_ticket": "803b6cc3-d144-47e8-948f-ec8b39c8881e", - "publickey_path": "/public-certs.store", - "license_path": "conf/hugegraph-community.license" -} diff --git a/hugegraph-hubble/yarn.lock b/hugegraph-hubble/yarn.lock deleted file mode 100644 index e69de29bb..000000000 diff --git a/hugegraph-loader/assembly/static/conf/hugegraph.truststore b/hugegraph-loader/assembly/static/conf/hugegraph.truststore deleted file mode 100644 index 5b0828dc3..000000000 Binary files a/hugegraph-loader/assembly/static/conf/hugegraph.truststore and /dev/null differ diff --git a/hugegraph-loader/assembly/travis/conf/hugegraph.truststore b/hugegraph-loader/assembly/travis/conf/hugegraph.truststore deleted file mode 100644 index 5b0828dc3..000000000 Binary files a/hugegraph-loader/assembly/travis/conf/hugegraph.truststore and /dev/null differ diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/constant/Constants.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/constant/Constants.java index d2abeab99..563cd401c 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/constant/Constants.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/constant/Constants.java @@ -31,7 +31,7 @@ public final class Constants { public static final String HTTPS_PREFIX = "https://"; public static final String JSON_SUFFIX = ".json"; public static final String GROOVY_SCHEMA = "schema"; - public static final String TRUST_STORE_FILE = "conf/hugegraph.truststore"; + public static final String TRUST_STORE_PATH = "conf/hugegraph.truststore"; public static final String FIELD_VERSION = "version"; public static final String V1_STRUCT_VERSION = "1.0"; diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/HugeClientHolder.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/HugeClientHolder.java index 31d21ddb1..124b3bd9c 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/HugeClientHolder.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/HugeClientHolder.java @@ -20,7 +20,6 @@ import java.nio.file.Paths; import org.apache.commons.lang3.StringUtils; - import org.apache.hugegraph.driver.HugeClient; import org.apache.hugegraph.driver.HugeClientBuilder; import org.apache.hugegraph.exception.ServerException; @@ -61,8 +60,7 @@ public static HugeClient create(LoadOptions options) { "The system property 'loader.home.path' " + "can't be null or empty when enable " + "https protocol"); - trustFile = Paths.get(homePath, Constants.TRUST_STORE_FILE) - .toString(); + trustFile = Paths.get(homePath, Constants.TRUST_STORE_PATH).toString(); } else { trustFile = options.trustStoreFile; } diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/functional/FileLoadTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/functional/FileLoadTest.java index 8a4eac1e1..06cbe0e54 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/functional/FileLoadTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/functional/FileLoadTest.java @@ -54,6 +54,7 @@ import org.apache.hugegraph.structure.schema.PropertyKey; import org.apache.hugegraph.testutil.Assert; import org.apache.hugegraph.testutil.Whitebox; +import org.apache.hugegraph.util.CommonUtil; import org.apache.hugegraph.util.LongEncoding; import org.junit.After; import org.junit.Before; @@ -2424,11 +2425,13 @@ public void testParquetCompressFile() { "--batch-insert-threads", "2", "--test-mode", "true" }; + String path = configPath("parquet_compress_file/vertex_person.parquet"); + CommonUtil.downloadFileByUrl(CommonUtil.PREFIX + "vertex_person.parquet", path); if (this.ioUtil instanceof HDFSUtil) { HDFSUtil hdfsUtil = (HDFSUtil) this.ioUtil; - hdfsUtil.copy(configPath( - "parquet_compress_file/vertex_person.parquet"), - "hdfs://localhost:8020/files/vertex_person.parquet"); + CommonUtil.downloadFileByUrl(CommonUtil.PREFIX + "vertex_person.parquet", + "src/test/resources/parquet_compress_file/vertex_person.parquet"); + hdfsUtil.copy(path, "hdfs://localhost:8020/files/vertex_person.parquet"); } HugeGraphLoader.main(args); @@ -2437,8 +2440,7 @@ public void testParquetCompressFile() { } @Test - public void testNumberAndDatePrimaryKeysEncoded() - throws java.text.ParseException { + public void testNumberAndDatePrimaryKeysEncoded() { ioUtil.write("vertex_person.csv", "id,name,age,city", "100,marko,29,Beijing"); @@ -3043,6 +3045,8 @@ public void testHttpsClientValueMapping() { ioUtil.write("vertex_person.csv", "tiny,1,1,1", "mary,2,2,2"); + + CommonUtil.downloadFileByUrl(FILE_URL, TRUST_STORE_PATH); String[] args = new String[]{ "-f", structPath("value_mapping/struct.json"), "-s", configPath("value_mapping/schema.groovy"), @@ -3050,7 +3054,7 @@ public void testHttpsClientValueMapping() { "-h", SERVER, "-p", String.valueOf(HTTPS_PORT), "--protocol", HTTPS_PROTOCOL, - "--trust-store-file", TRUST_STORE_FILE, + "--trust-store-file", TRUST_STORE_PATH, "--trust-store-password", "hugegraph", "--batch-insert-threads", "2", "--test-mode", "true" @@ -3060,7 +3064,7 @@ public void testHttpsClientValueMapping() { HugeClient httpsClient = null; try { httpsClient = HugeClient.builder(HTTPS_URL, GRAPH) - .configSSL(TRUST_STORE_FILE, "hugegraph") + .configSSL(TRUST_STORE_PATH, "hugegraph") .build(); List vertices = httpsClient.graph().listVertices(); Assert.assertEquals(2, vertices.size()); @@ -3074,6 +3078,8 @@ public void testHttpsHolderClientValueMapping() { ioUtil.write("vertex_person.csv", "marko,1,1,1", "vadas,2,2,2"); + + CommonUtil.downloadFileByUrl(FILE_URL, TRUST_STORE_PATH); String[] args = new String[]{ "-f", structPath("value_mapping/struct.json"), "-s", configPath("value_mapping/schema.groovy"), @@ -3081,7 +3087,7 @@ public void testHttpsHolderClientValueMapping() { "-h", SERVER, "-p", String.valueOf(HTTPS_PORT), "--protocol", HTTPS_PROTOCOL, - "--trust-store-file", TRUST_STORE_FILE, + "--trust-store-file", TRUST_STORE_PATH, "--trust-store-password", "hugegraph", "--batch-insert-threads", "2", "--test-mode", "true" @@ -3093,7 +3099,7 @@ public void testHttpsHolderClientValueMapping() { options.port = HTTPS_PORT; options.graph = GRAPH; options.protocol = HTTPS_PROTOCOL; - options.trustStoreFile = TRUST_STORE_FILE; + options.trustStoreFile = TRUST_STORE_PATH; options.trustStoreToken = "hugegraph"; HugeClient httpsClient = null; diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/functional/LoadTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/functional/LoadTest.java index 1f597f799..e518dab49 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/functional/LoadTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/functional/LoadTest.java @@ -29,8 +29,8 @@ import org.apache.hugegraph.structure.constant.T; import org.apache.hugegraph.structure.graph.Edge; import org.apache.hugegraph.structure.graph.Vertex; - import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.util.CommonUtil; public class LoadTest { @@ -43,7 +43,8 @@ public class LoadTest { protected static final String URL = String.format("http://%s:%s", SERVER, PORT); protected static final String HTTPS_URL = String.format("https://%s:%s", SERVER, HTTPS_PORT); protected static final String HTTPS_PROTOCOL = "https"; - protected static final String TRUST_STORE_FILE = "assembly/travis/conf/hugegraph.truststore"; + protected static final String TRUST_STORE_PATH = "assembly/travis/conf/hugegraph.truststore"; + protected static final String FILE_URL = CommonUtil.PREFIX + "hugegraph.truststore"; protected static final HugeClient CLIENT = HugeClient.builder(URL, GRAPH).build(); public static String configPath(String fileName) { diff --git a/hugegraph-loader/src/test/resources/parquet_compress_file/vertex_person.parquet b/hugegraph-loader/src/test/resources/parquet_compress_file/vertex_person.parquet deleted file mode 100644 index 604ad3c48..000000000 Binary files a/hugegraph-loader/src/test/resources/parquet_compress_file/vertex_person.parquet and /dev/null differ diff --git a/hugegraph-tools/assembly/bin/keystore b/hugegraph-tools/assembly/bin/keystore deleted file mode 100644 index 606b04840..000000000 Binary files a/hugegraph-tools/assembly/bin/keystore and /dev/null differ diff --git a/hugegraph-tools/assembly/conf/hugegraph.truststore b/hugegraph-tools/assembly/conf/hugegraph.truststore deleted file mode 100644 index 5b0828dc3..000000000 Binary files a/hugegraph-tools/assembly/conf/hugegraph.truststore and /dev/null differ diff --git a/hugegraph-tools/src/main/java/org/apache/hugegraph/base/ToolClient.java b/hugegraph-tools/src/main/java/org/apache/hugegraph/base/ToolClient.java index a24a940f8..86cea21fd 100644 --- a/hugegraph-tools/src/main/java/org/apache/hugegraph/base/ToolClient.java +++ b/hugegraph-tools/src/main/java/org/apache/hugegraph/base/ToolClient.java @@ -34,12 +34,11 @@ public class ToolClient { - private static final String DEFAULT_TRUST_STORE_FILE = - "conf/hugegraph.truststore"; + private static final String DEFAULT_TRUST_STORE_FILE = "conf/hugegraph.truststore"; private static final String DEFAULT_TRUST_STORE_PASSWORD = "hugegraph"; - private HugeClient client; - private ObjectMapper mapper; + private final HugeClient client; + private final ObjectMapper mapper; public ToolClient(ConnectionInfo info) { if (info.username == null) { @@ -125,13 +124,13 @@ public void close() { public static class ConnectionInfo { - private String url; - private String graph; + private final String url; + private final String graph; private String username; private String password; - private Integer timeout; - private String trustStoreFile; - private String trustStorePassword; + private final Integer timeout; + private final String trustStoreFile; + private final String trustStorePassword; public ConnectionInfo(String url, String graph, String username, String password,