diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..27f7932 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,27 @@ +language: java + +jdk: + - openjdk8 + +sudo: required + +branches: + only: + - master + - /^release-.*$/ + - /^test-.*$/ + +install: mvn compile -Dmaven.javadoc.skip=true | grep -v "Downloading\|Downloaded" + +before_script: + - $TRAVIS_DIR/install-hugegraph.sh $TRAVIS_BRANCH | grep -v "Downloading\|Downloaded" + +script: + - mvn test -Dtest=FuncTestSuite + +after_success: + - bash <(curl -s https://codecov.io/bash) + +env: + global: + - TRAVIS_DIR=assembly/travis diff --git a/assembly/travis/conf/gremlin-server.yaml b/assembly/travis/conf/gremlin-server.yaml new file mode 100644 index 0000000..484ee12 --- /dev/null +++ b/assembly/travis/conf/gremlin-server.yaml @@ -0,0 +1,110 @@ +# host and port of gremlin server, need to be consistent with host and port in rest-server.properties +#host: 127.0.0.1 +#port: 8182 + +# timeout in ms of gremlin query +scriptEvaluationTimeout: 30000 + +channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer +graphs: { + hugegraph: conf/hugegraph.properties +} +scriptEngines: { + gremlin-groovy: { + plugins: { + com.baidu.hugegraph.plugin.HugeGraphGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: { + classImports: [ + java.lang.Math, + com.baidu.hugegraph.backend.id.IdGenerator, + com.baidu.hugegraph.type.define.Directions, + com.baidu.hugegraph.type.define.NodeRole, + com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser, + com.baidu.hugegraph.traversal.algorithm.CountTraverser, + com.baidu.hugegraph.traversal.algorithm.CustomizedCrosspointsTraverser, + com.baidu.hugegraph.traversal.algorithm.CustomizePathsTraverser, + com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser, + com.baidu.hugegraph.traversal.algorithm.HugeTraverser, + com.baidu.hugegraph.traversal.algorithm.JaccardSimilarTraverser, + com.baidu.hugegraph.traversal.algorithm.KneighborTraverser, + com.baidu.hugegraph.traversal.algorithm.KoutTraverser, + com.baidu.hugegraph.traversal.algorithm.MultiNodeShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser, + com.baidu.hugegraph.traversal.algorithm.PathsTraverser, + com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser, + com.baidu.hugegraph.traversal.algorithm.SameNeighborTraverser, + com.baidu.hugegraph.traversal.algorithm.ShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.SingleSourceShortestPathTraverser, + com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser, + com.baidu.hugegraph.traversal.algorithm.TemplatePathsTraverser, + com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep, + com.baidu.hugegraph.traversal.algorithm.steps.RepeatEdgeStep, + com.baidu.hugegraph.traversal.algorithm.steps.WeightedEdgeStep, + com.baidu.hugegraph.traversal.optimize.Text, + com.baidu.hugegraph.traversal.optimize.TraversalUtil, + com.baidu.hugegraph.util.DateUtil + ], + methodImports: [java.lang.Math#*] + }, + org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: { + files: [scripts/empty-sample.groovy] + } + } + } +} +serializers: + - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoLiteMessageSerializerV1d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0, + config: { + serializeResultToString: false, + ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry] + } + } +metrics: { + consoleReporter: {enabled: false, interval: 180000}, + csvReporter: {enabled: true, interval: 180000, fileName: /tmp/gremlin-server-metrics.csv}, + jmxReporter: {enabled: false}, + slf4jReporter: {enabled: false, interval: 180000}, + gangliaReporter: {enabled: false, interval: 180000, addressingMode: MULTICAST}, + graphiteReporter: {enabled: false, interval: 180000} +} +maxInitialLineLength: 4096 +maxHeaderSize: 8192 +maxChunkSize: 8192 +maxContentLength: 65536 +maxAccumulationBufferComponents: 1024 +resultIterationBatchSize: 64 +writeBufferLowWaterMark: 32768 +writeBufferHighWaterMark: 65536 +ssl: { + enabled: false +} +authentication: { + authenticator: com.baidu.hugegraph.auth.StandardAuthenticator, + authenticationHandler: com.baidu.hugegraph.auth.WsAndHttpBasicAuthHandler, + config: {tokens: conf/rest-server.properties} +} diff --git a/assembly/travis/conf/hugegraph.properties b/assembly/travis/conf/hugegraph.properties new file mode 100644 index 0000000..c66d0a7 --- /dev/null +++ b/assembly/travis/conf/hugegraph.properties @@ -0,0 +1,68 @@ +# gremlin entrence to create graph +gremlin.graph=com.baidu.hugegraph.auth.HugeFactoryAuthProxy + +# cache config +#schema.cache_capacity=100000 +# vertex-cache default is 1000w, 10min expired +#vertex.cache_capacity=10000000 +#vertex.cache_expire=600 +# edge-cache default is 100w, 10min expired +#edge.cache_capacity=1000000 +#edge.cache_expire=600 + + +# schema illegal name template +#schema.illegal_name_regex=\s+|~.* + +#vertex.default_label=vertex + +backend=rocksdb +serializer=binary + +store=hugegraph + +search.text_analyzer=jieba +search.text_analyzer_mode=INDEX + +# rocksdb backend config +#rocksdb.data_path=/path/to/disk +#rocksdb.wal_path=/path/to/disk + + +# cassandra backend config +cassandra.host=localhost +cassandra.port=9042 +cassandra.username= +cassandra.password= +#cassandra.connect_timeout=5 +#cassandra.read_timeout=20 +#cassandra.keyspace.strategy=SimpleStrategy +#cassandra.keyspace.replication=3 + +# hbase backend config +#hbase.hosts=localhost +#hbase.port=2181 +#hbase.znode_parent=/hbase +#hbase.threads_max=64 + +# mysql backend config +#jdbc.driver=com.mysql.jdbc.Driver +#jdbc.url=jdbc:mysql://127.0.0.1:3306 +#jdbc.username=root +#jdbc.password= +#jdbc.reconnect_max_times=3 +#jdbc.reconnect_interval=3 +#jdbc.sslmode=false + +# palo backend config +#palo.host=127.0.0.1 +#palo.poll_interval=10 +#palo.temp_dir=./palo-data +#palo.file_limit_size=32 + +# postgresql & cockroachdb backend config +#jdbc.driver=org.postgresql.Driver +#jdbc.url=jdbc:postgresql://localhost:5432/ +#jdbc.username=postgres +#jdbc.password= +#jdbc.postgresql.connect_database=template1 diff --git a/assembly/travis/conf/rest-server.properties b/assembly/travis/conf/rest-server.properties new file mode 100644 index 0000000..a9ea865 --- /dev/null +++ b/assembly/travis/conf/rest-server.properties @@ -0,0 +1,12 @@ +# bind url +restserver.url=http://127.0.0.1:8080 +# gremlin server url, need to be consistent with host and port in gremlin-server.yaml +#gremlinserver.url=http://127.0.0.1:8182 + +# graphs list with pair NAME:CONF_PATH +graphs=[hugegraph:conf/hugegraph.properties] + +# authentication +auth.authenticator=com.baidu.hugegraph.auth.StandardAuthenticator +#auth.admin_token= +#auth.user_tokens=[] diff --git a/assembly/travis/install-hugegraph.sh b/assembly/travis/install-hugegraph.sh new file mode 100755 index 0000000..3c4eedf --- /dev/null +++ b/assembly/travis/install-hugegraph.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -ev + +TRAVIS_DIR=`dirname $0` + +if [ $# -ne 1 ]; then + echo "Must pass base branch name of pull request" + exit 1 +fi + +CLIENT_BRANCH=$1 +HUGEGRAPH_BRANCH=$CLIENT_BRANCH + +HUGEGRAPH_GIT_URL="https://github.com/hugegraph/hugegraph.git" + +git clone $HUGEGRAPH_GIT_URL + +cd hugegraph + +git checkout $HUGEGRAPH_BRANCH + +mvn package -DskipTests + +mv hugegraph-*.tar.gz ../ + +cd ../ + +rm -rf hugegraph + +tar -zxvf hugegraph-*.tar.gz + +HTTPS_SERVER_DIR="hugegraph_https" + +mkdir $HTTPS_SERVER_DIR + +cp -r hugegraph-*/. $HTTPS_SERVER_DIR + +cd hugegraph-* + +cp ../$TRAVIS_DIR/conf/* conf + +echo -e "123456" | bin/init-store.sh + +bin/start-hugegraph.sh diff --git a/checkstyle.xml b/checkstyle.xml new file mode 100644 index 0000000..80c26bf --- /dev/null +++ b/checkstyle.xml @@ -0,0 +1,87 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 3d606b6..80a7a97 100644 --- a/pom.xml +++ b/pom.xml @@ -238,6 +238,28 @@ release + + org.apache.maven.plugins + maven-checkstyle-plugin + 3.1.0 + + checkstyle.xml + UTF-8 + true + true + false + false + + + + validate + validate + + check + + + + maven-source-plugin diff --git a/src/main/java/com/baidu/hugegraph/base/ToolClient.java b/src/main/java/com/baidu/hugegraph/base/ToolClient.java index d054e08..8516958 100644 --- a/src/main/java/com/baidu/hugegraph/base/ToolClient.java +++ b/src/main/java/com/baidu/hugegraph/base/ToolClient.java @@ -23,6 +23,7 @@ import org.apache.commons.lang3.StringUtils; +import com.baidu.hugegraph.driver.AuthManager; import com.baidu.hugegraph.driver.GraphManager; import com.baidu.hugegraph.driver.GremlinManager; import com.baidu.hugegraph.driver.HugeClient; @@ -113,6 +114,16 @@ public static String homePath() { return homePath; } + public AuthManager authManager() { + return this.client.auth(); + } + + public void close() { + if (this.client != null) { + this.client.close(); + } + } + public static class ConnectionInfo { private String url; diff --git a/src/main/java/com/baidu/hugegraph/base/ToolManager.java b/src/main/java/com/baidu/hugegraph/base/ToolManager.java index f5d00f1..1828d2b 100644 --- a/src/main/java/com/baidu/hugegraph/base/ToolManager.java +++ b/src/main/java/com/baidu/hugegraph/base/ToolManager.java @@ -65,4 +65,8 @@ protected List readList(String key, Class clazz, "Failed to deserialize %s", e, content); } } + + public void close () { + this.client.close(); + } } diff --git a/src/main/java/com/baidu/hugegraph/cmd/HugeGraphCommand.java b/src/main/java/com/baidu/hugegraph/cmd/HugeGraphCommand.java index dab5361..2eb5c53 100644 --- a/src/main/java/com/baidu/hugegraph/cmd/HugeGraphCommand.java +++ b/src/main/java/com/baidu/hugegraph/cmd/HugeGraphCommand.java @@ -19,15 +19,23 @@ package com.baidu.hugegraph.cmd; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.util.Strings; + import com.baidu.hugegraph.base.Printer; import com.baidu.hugegraph.base.ToolClient; import com.baidu.hugegraph.base.ToolClient.ConnectionInfo; import com.baidu.hugegraph.base.ToolManager; +import com.baidu.hugegraph.constant.Constants; +import com.baidu.hugegraph.exception.ExitException; +import com.baidu.hugegraph.manager.AuthBackupRestoreManager; import com.baidu.hugegraph.manager.BackupManager; import com.baidu.hugegraph.manager.DumpGraphManager; import com.baidu.hugegraph.manager.GraphsManager; @@ -39,9 +47,11 @@ import com.baidu.hugegraph.structure.gremlin.Result; import com.baidu.hugegraph.structure.gremlin.ResultSet; import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.util.ToolUtil; import com.beust.jcommander.JCommander; import com.beust.jcommander.ParameterException; import com.beust.jcommander.ParametersDelegate; +import com.google.common.collect.Lists; import static com.baidu.hugegraph.manager.BackupManager.BACKUP_DEFAULT_TIMEOUT; @@ -51,6 +61,8 @@ public class HugeGraphCommand { private SubCommands subCommands; + private List taskManagers; + @ParametersDelegate private SubCommands.Url url = new SubCommands.Url(); @@ -74,8 +86,12 @@ public class HugeGraphCommand { private SubCommands.TrustStorePassword trustStorePassword = new SubCommands.TrustStorePassword(); + @ParametersDelegate + private SubCommands.ThrowMode throwMode = new SubCommands.ThrowMode(); + public HugeGraphCommand() { this.subCommands = new SubCommands(); + this.taskManagers = Lists.newArrayList(); } public Map subCommands() { @@ -143,6 +159,14 @@ public void trustStorePassword(String trustStorePassword) { this.trustStorePassword.trustStorePassword = trustStorePassword; } + public boolean throwMode() { + return this.throwMode.throwMode; + } + + private void throwMode(boolean throwMode) { + this.throwMode.throwMode = throwMode; + } + public JCommander jCommander() { JCommander.Builder builder = JCommander.newBuilder(); @@ -328,8 +352,21 @@ private void execute(String subCmd, JCommander jCommander) { Printer.print("Tasks are cleared[force=%s]", taskClear.force()); break; - case "help": - jCommander.usage(); + case "auth-backup": + Printer.print("Auth backup start..."); + SubCommands.AuthBackup authBackup = this.subCommand(subCmd); + AuthBackupRestoreManager authBackupManager = manager(AuthBackupRestoreManager.class); + + authBackupManager.init(authBackup); + authBackupManager.backup(authBackup.types()); + break; + case "auth-restore": + Printer.print("Auth restore start..."); + SubCommands.AuthRestore authRestore = this.subCommand(subCmd); + AuthBackupRestoreManager authRestoreManager = manager(AuthBackupRestoreManager.class); + + authRestoreManager.init(authRestore); + authRestoreManager.restore(authRestore.types()); break; default: throw new ParameterException(String.format( @@ -337,6 +374,11 @@ private void execute(String subCmd, JCommander jCommander) { } } + private void execute(String[] args) { + JCommander jCommander = this.parseCommand(args); + this.execute(jCommander.getParsedCommand(), jCommander); + } + private void checkMainParams() { E.checkArgument(this.url() != null, "Url can't be null"); E.checkArgument(this.username() == null && this.password() == null || @@ -353,8 +395,10 @@ private T manager(Class clz) { this.timeout(), this.trustStoreFile(), this.trustStorePassword()); - return clz.getConstructor(ToolClient.ConnectionInfo.class) - .newInstance(info); + T toolManager = clz.getConstructor(ToolClient.ConnectionInfo.class) + .newInstance(info); + this.taskManagers.add(toolManager); + return toolManager; } catch (Exception e) { throw new RuntimeException(String.format( "Construct manager failed for class '%s', please make " + @@ -402,29 +446,87 @@ private GraphMode mode() { return mode; } - public static void main(String[] args) { - HugeGraphCommand cmd = new HugeGraphCommand(); - JCommander jCommander = cmd.jCommander(); - + public JCommander parseCommand(String[] args) { + JCommander jCommander = this.jCommander(); if (args.length == 0) { - jCommander.usage(); - System.exit(-1); + throw ExitException.exception(ToolUtil.commandUsage(jCommander), + "No command found, please input" + + " command"); } - try { + if (this.parseHelp(args, jCommander)) { + assert false; + } else { jCommander.parse(args); - } catch (ParameterException e) { - Printer.print(e.getMessage()); - System.exit(-1); } - String subCommand = jCommander.getParsedCommand(); if (subCommand == null) { - Printer.print("Must provide one sub-command"); - jCommander.usage(); - System.exit(-1); + throw ExitException.normal(ToolUtil.commandsCategory( + jCommander), + "No sub-command found"); + } + return jCommander; + } + + public boolean parseHelp(String[] args, JCommander jCommander) { + String subCommand = Strings.EMPTY; + List list = Arrays.asList(args); + if (!list.contains(Constants.COMMAND_HELP)) { + return false; + } + // Parse the '--throw-mode' command + if (list.contains(Constants.COMMAND_THROW_MODE)) { + int index = list.indexOf(Constants.COMMAND_THROW_MODE) + 1; + jCommander.parse(Constants.COMMAND_THROW_MODE, + list.get(index)); + } + int index = list.indexOf(Constants.COMMAND_HELP); + if (list.size() > index + 1) { + subCommand = list.get(index + 1); + } + if (StringUtils.isEmpty(subCommand)) { + throw ExitException.normal(ToolUtil.commandUsage(jCommander), + "Command : hugegragh help"); } - cmd.execute(subCommand, jCommander); - System.exit(0); + Map commands = jCommander.getCommands(); + if (commands.containsKey(subCommand)) { + throw ExitException.normal(ToolUtil.commandUsage( + commands.get(subCommand)), + "Command : hugegragh help %s", + subCommand); + } else { + throw ExitException.exception(ToolUtil.commandsCategory(jCommander), + "Unexpected help sub-command " + + "%s", subCommand); + } + } + + public void shutdown() { + if (CollectionUtils.isEmpty(this.taskManagers)) { + return; + } + for (ToolManager toolManager : this.taskManagers) { + toolManager.close(); + } + } + + public static void main(String[] args) { + HugeGraphCommand cmd = new HugeGraphCommand(); + int exitCode = Constants.EXIT_CODE_NORMAL; + try { + cmd.execute(args); + } catch (ExitException e) { + exitCode = e.exitCode(); + ToolUtil.exitOrThrow(e, cmd.throwMode()); + } catch (Throwable e) { + exitCode = Constants.EXIT_CODE_ERROR; + ToolUtil.printOrThrow(e, cmd.throwMode()); + } finally { + cmd.shutdown(); + } + + if (exitCode != Constants.EXIT_CODE_NORMAL) { + System.exit(exitCode); + } } } diff --git a/src/main/java/com/baidu/hugegraph/cmd/SubCommands.java b/src/main/java/com/baidu/hugegraph/cmd/SubCommands.java index 5a71e38..743a88f 100644 --- a/src/main/java/com/baidu/hugegraph/cmd/SubCommands.java +++ b/src/main/java/com/baidu/hugegraph/cmd/SubCommands.java @@ -22,14 +22,17 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import com.baidu.hugegraph.api.API; +import com.baidu.hugegraph.constant.AuthRestoreConflictStrategy; import com.baidu.hugegraph.manager.TasksManager; import com.baidu.hugegraph.structure.constant.GraphMode; import com.baidu.hugegraph.structure.constant.HugeType; @@ -82,6 +85,9 @@ private void initSubCommands() { this.commands.put("clear", new Clear()); this.commands.put("stop-all", new StopAll()); + this.commands.put("auth-backup", new AuthBackup()); + this.commands.put("auth-restore", new AuthRestore()); + this.commands.put("help", new Help()); } @@ -135,7 +141,7 @@ public static class Backup extends BackupRestore { public boolean compress = true; @Parameter(names = {"--label"}, arity = 1, - description = "Vertex or edge label, only valid when type " + + description = "Vertex label or edge label, only valid when type " + "is vertex or edge") public String label; @@ -668,8 +674,8 @@ public static class Protocol { public static class TrustStoreFile { @Parameter(names = {"--trust-store-file"}, arity = 1, - description = "The path of client truststore file used when https " + - "protocol is enabled") + description = "The path of client truststore file used when " + + "https protocol is enabled") public String trustStoreFile; } @@ -681,16 +687,26 @@ public static class TrustStorePassword { public String trustStorePassword; } + public static class ThrowMode { + + @Parameter(names = {"--throw-mode"}, arity = 1, + description = "Whether the hugegraph-tools work " + + "to throw an exception") + public boolean throwMode = false; + } + public static class HugeTypes { @Parameter(names = {"--huge-types", "-t"}, listConverter = HugeTypeListConverter.class, - description = "Type of schema/data. " + - "Concat with ',' if more than one. " + - "'all' means all vertices, edges and schema," + - " in other words, 'all' equals with " + - "'vertex,edge,vertex_label," + - "edge_label,property_key,index_label'") + description = "Type of schema/data. Concat with ',' if more " + + "than one. Other types include 'all' and " + + "'schema'. 'all' means all vertices, edges and " + + "schema. In other words, 'all' equals with " + + "'vertex, edge, vertex_label, edge_label, " + + "property_key, index_label'. 'schema' equals " + + "with 'vertex_label, edge_label, property_key, " + + "index_label'.") public List types = HugeTypeListConverter.ALL_TYPES; } @@ -812,6 +828,113 @@ public static class TaskId { private long taskId; } + public static class AuthBackupRestore { + + @ParametersDelegate + private AuthTypes types = new AuthTypes(); + + @Parameter(names = {"--directory"}, arity = 1, + description = "Directory of auth information, default " + + "is './{auth-backup-restore}' in local " + + "file system or '{fs.default.name}/" + + "{auth-backup-restore}' in HDFS") + public String directory; + + @DynamicParameter(names = "-D", + description = "HDFS config parameters") + private Map hdfsConf = new HashMap<>(); + + @ParametersDelegate + private Retry retry = new Retry(); + + public List types() { + return this.types.types; + } + + public void types(List types) { + this.types.types = types; + } + + public int retry() { + return this.retry.retry; + } + + public void retry(int retry) { + this.retry.retry = retry; + } + + public String directory() { + return this.directory; + } + + public void directory(String directory) { + this.directory = directory; + } + + public Map hdfsConf() { + return this.hdfsConf; + } + + public void hdfsConf(Map hdfsConf) { + this.hdfsConf = hdfsConf; + } + } + + public static class AuthBackup extends AuthBackupRestore { + } + + public static class AuthRestore extends AuthBackupRestore { + + @Parameter(names = {"--strategy"}, + converter = AuthStrategyConverter.class, + description = "The strategy needs to be chosen in the event " + + "of a conflict when restoring. Valid " + + "strategies include 'stop' and 'ignore', " + + "default is 'stop'. 'stop' means if there " + + "a conflict, stop restore. 'ignore' means if " + + "there a conflict, ignore and continue to " + + "restore.") + public AuthRestoreConflictStrategy strategy = AuthStrategyConverter.strategy; + + @Parameter(names = {"--init-password"}, arity = 1, + description = "Init user password, if restore type include " + + "'user', please specify the init-password of " + + "users.") + public String initPassword = StringUtils.EMPTY; + + public AuthRestoreConflictStrategy strategy() { + return this.strategy; + } + + public void strategy(AuthRestoreConflictStrategy strategy) { + this.strategy = strategy; + } + + public String initPassword() { + return this.initPassword; + } + + public void initPassword(String initPassword) { + this.initPassword = initPassword; + } + } + + public static class AuthTypes { + + @Parameter(names = {"--types", "-t"}, + listConverter = AuthHugeTypeConverter.class, + description = "Type of auth data to restore and backup, " + + "concat with ',' if more than one. 'all' " + + "means all auth information. In other words, " + + "'all' equals with 'user, group, target, " + + "belong, access'. In addition, 'belong' or " + + "'access' can not backup or restore alone, if " + + "type contains 'belong' then should contains " + + "'user' and 'group'. If type contains 'access' " + + "then should contains 'group' and 'target'.") + public List types = AuthHugeTypeConverter.AUTH_ALL_TYPES; + } + public static class GraphModeConverter implements IStringConverter { @@ -832,6 +955,11 @@ public static class HugeTypeListConverter HugeType.VERTEX, HugeType.EDGE ); + public static final List SCHEMA_TYPES = ImmutableList.of( + HugeType.PROPERTY_KEY, HugeType.VERTEX_LABEL, + HugeType.EDGE_LABEL, HugeType.INDEX_LABEL + ); + @Override public List convert(String value) { E.checkArgument(value != null && !value.isEmpty(), @@ -840,6 +968,9 @@ public List convert(String value) { if (types.length == 1 && types[0].equalsIgnoreCase("all")) { return ALL_TYPES; } + if (types.length == 1 && types[0].equalsIgnoreCase("schema")) { + return SCHEMA_TYPES; + } List hugeTypes = new ArrayList<>(); for (String type : types) { try { @@ -855,6 +986,66 @@ public List convert(String value) { } } + public static class AuthHugeTypeConverter + implements IStringConverter> { + + public static final List AUTH_ALL_TYPES = ImmutableList.of( + HugeType.TARGET, HugeType.GROUP, + HugeType.USER, HugeType.ACCESS, + HugeType.BELONG + ); + + @Override + public List convert(String value) { + E.checkArgument(value != null && !value.isEmpty(), + "HugeType can't be null or empty"); + String[] types = value.split(","); + if (types.length == 1 && types[0].equalsIgnoreCase("all")) { + return AUTH_ALL_TYPES; + } + List typeList = Arrays.asList(types); + E.checkArgument(!typeList.contains(HugeType.BELONG.toString().toLowerCase()) || + (typeList.contains(HugeType.USER.toString().toLowerCase()) && + typeList.contains(HugeType.GROUP.toString().toLowerCase())), + "Invalid --type '%s', if type contains 'belong'" + + " then 'user' and 'group' are required.", value); + E.checkArgument(!typeList.contains(HugeType.ACCESS.toString().toLowerCase()) || + (typeList.contains(HugeType.GROUP.toString().toLowerCase()) && + typeList.contains(HugeType.TARGET.toString().toLowerCase())), + "Invalid --type '%s', if type contains 'access'" + + " then 'group' and 'target' are required.", value); + List hugeTypes = new ArrayList<>(); + for (String type : types) { + try { + hugeTypes.add(HugeType.valueOf(type.toUpperCase())); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(String.format( + "Invalid --type '%s', valid value is 'all' or " + + "combination of [user,group,target," + + "belong,access]", type)); + } + } + return hugeTypes; + } + } + + public static class AuthStrategyConverter + implements IStringConverter { + + public static final AuthRestoreConflictStrategy strategy = + AuthRestoreConflictStrategy.STOP; + + @Override + public AuthRestoreConflictStrategy convert(String value) { + E.checkArgument(value != null && !value.isEmpty(), + "Strategy can't be null or empty"); + E.checkArgument(AuthRestoreConflictStrategy.matchStrategy(value), + "Invalid --strategy '%s', valid value is " + + "'stop' or 'ignore", value); + return AuthRestoreConflictStrategy.fromName(value); + } + } + public static class MapConverter implements IStringConverter> { @@ -876,7 +1067,7 @@ public Map convert(String value) { } public static class FileNameToContentConverter - implements IStringConverter { + implements IStringConverter { @Override public String convert(String value) { diff --git a/src/main/java/com/baidu/hugegraph/constant/AuthRestoreConflictStrategy.java b/src/main/java/com/baidu/hugegraph/constant/AuthRestoreConflictStrategy.java new file mode 100644 index 0000000..ff86506 --- /dev/null +++ b/src/main/java/com/baidu/hugegraph/constant/AuthRestoreConflictStrategy.java @@ -0,0 +1,69 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.constant; + +public enum AuthRestoreConflictStrategy { + + STOP(1, "stop"), + IGNORE(2, "ignore"); + + private int code; + private String name = null; + + AuthRestoreConflictStrategy(int code, String name) { + assert code < 256; + this.code = code; + this.name = name; + } + + public int code() { + return this.code; + } + + public String string() { + return this.name; + } + + public boolean isStopStrategy() { + return this == AuthRestoreConflictStrategy.STOP; + } + + public boolean isIgnoreStrategy() { + return this == AuthRestoreConflictStrategy.IGNORE; + } + + public static boolean matchStrategy(String strategy) { + if (AuthRestoreConflictStrategy.STOP.string().equals(strategy) || + AuthRestoreConflictStrategy.IGNORE.string().equals(strategy)) { + return true; + } + return false; + } + + public static AuthRestoreConflictStrategy fromName(String name) { + AuthRestoreConflictStrategy[] restoreStrategys = AuthRestoreConflictStrategy.values(); + for (AuthRestoreConflictStrategy strategy : restoreStrategys) { + if (strategy.string().equals(name)) { + return strategy; + } + } + return null; + } +} diff --git a/src/main/java/com/baidu/hugegraph/constant/Constants.java b/src/main/java/com/baidu/hugegraph/constant/Constants.java new file mode 100644 index 0000000..e365e01 --- /dev/null +++ b/src/main/java/com/baidu/hugegraph/constant/Constants.java @@ -0,0 +1,31 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.constant; + +public final class Constants { + + public static final int EXIT_CODE_ERROR = -1; + public static final int EXIT_CODE_NORMAL = 0; + + public static final String INPUT_YES = "yes"; + public static final String INPUT_Y= "y"; + public static final String COMMAND_HELP = "help"; + public static final String COMMAND_THROW_MODE = "--throw-mode"; +} diff --git a/src/main/java/com/baidu/hugegraph/exception/ExitException.java b/src/main/java/com/baidu/hugegraph/exception/ExitException.java new file mode 100644 index 0000000..552466d --- /dev/null +++ b/src/main/java/com/baidu/hugegraph/exception/ExitException.java @@ -0,0 +1,90 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.exception; + +import com.baidu.hugegraph.constant.Constants; + +public class ExitException extends RuntimeException { + + private final String details; + private final Integer exitCode; + + public ExitException(String details, String reason) { + super(reason); + this.details = details; + this.exitCode = Constants.EXIT_CODE_NORMAL; + } + + public ExitException(String details, String reason, + Object... args) { + super(String.format(reason, args)); + this.details = details; + this.exitCode = Constants.EXIT_CODE_NORMAL; + } + + public ExitException(Integer exitCode, String details, + String reason) { + super(reason); + this.details = details; + this.exitCode = exitCode; + } + + public ExitException(Integer exitCode, String details, + String reason, Throwable cause) { + super(reason, cause); + this.details = details; + this.exitCode = exitCode; + } + + public ExitException(Integer exitCode, String details, + String reason, Object... args) { + super(String.format(reason, args)); + this.details = details; + this.exitCode = exitCode; + } + + public ExitException(Integer exitCode, String details, + String reason, Throwable cause, + Object... args) { + super(String.format(reason, args), cause); + this.details = details; + this.exitCode = exitCode; + } + + public String details() { + return this.details; + } + + public Integer exitCode() { + return this.exitCode; + } + + public static ExitException exception(String details, String reason, + Object... args) { + return new ExitException(Constants.EXIT_CODE_ERROR, + details, reason, args); + } + + public static ExitException normal(String details, String reason, + Object... args) { + return new ExitException(Constants.EXIT_CODE_NORMAL, + details, reason, args); + } +} diff --git a/src/main/java/com/baidu/hugegraph/manager/AuthBackupRestoreManager.java b/src/main/java/com/baidu/hugegraph/manager/AuthBackupRestoreManager.java new file mode 100644 index 0000000..a1a49f2 --- /dev/null +++ b/src/main/java/com/baidu/hugegraph/manager/AuthBackupRestoreManager.java @@ -0,0 +1,595 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.manager; + +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.util.Strings; + +import com.baidu.hugegraph.api.API; +import com.baidu.hugegraph.base.HdfsDirectory; +import com.baidu.hugegraph.base.LocalDirectory; +import com.baidu.hugegraph.base.Printer; +import com.baidu.hugegraph.base.ToolClient; +import com.baidu.hugegraph.cmd.SubCommands; +import com.baidu.hugegraph.constant.AuthRestoreConflictStrategy; +import com.baidu.hugegraph.exception.ToolsException; +import com.baidu.hugegraph.structure.auth.Access; +import com.baidu.hugegraph.structure.auth.Belong; +import com.baidu.hugegraph.structure.auth.Group; +import com.baidu.hugegraph.structure.auth.Target; +import com.baidu.hugegraph.structure.auth.User; +import com.baidu.hugegraph.structure.constant.HugeType; +import com.baidu.hugegraph.util.E; +import com.baidu.hugegraph.util.JsonUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +public class AuthBackupRestoreManager extends BackupRestoreBaseManager { + + private static final String AUTH_BACKUP_RESTORE = "auth-backup-restore"; + private static final int NO_CONFLICT = 0; + + private AuthRestoreConflictStrategy strategy; + private String initPassword; + /* + * The collection of id relationships of users, groups and targets + * is the basic data of belong and accesses。 + */ + private Map idsMap; + private Map usersByName; + private Map groupsByName; + private Map targetsByName; + private Map belongsByName; + private Map accessesByName; + + public AuthBackupRestoreManager(ToolClient.ConnectionInfo info) { + super(info, AUTH_BACKUP_RESTORE); + } + + public void init(SubCommands.AuthBackup authBackup) { + this.retry(authBackup.retry()); + this.directory(authBackup.directory(), authBackup.hdfsConf()); + this.ensureDirectoryExist(true); + } + + public void init(SubCommands.AuthRestore authRestore) { + this.retry(authRestore.retry()); + this.directory(authRestore.directory(), authRestore.hdfsConf()); + this.ensureDirectoryExist(false); + this.strategy = authRestore.strategy(); + this.initPassword(authRestore.types(), authRestore.initPassword()); + this.idsMap = Maps.newHashMap(); + this.usersByName = Maps.newHashMap(); + this.groupsByName = Maps.newHashMap(); + this.targetsByName = Maps.newHashMap(); + this.belongsByName = Maps.newHashMap(); + this.accessesByName = Maps.newHashMap(); + } + + public void backup(List types) { + try { + this.doBackup(this.addAuthManagers(types)); + } catch (Throwable e) { + throw e; + } finally { + this.shutdown(this.type()); + } + } + + private void doBackup(List authManagers) { + E.checkState(CollectionUtils.isNotEmpty(authManagers), + "Backup data is empty, please check the type"); + for (AuthManager authManager : authManagers) { + authManager.backup(); + } + } + + public void restore(List types) { + List sortedHugeTypes = this.sortListByCode(types); + try { + this.doRestore(this.addAuthManagers(sortedHugeTypes)); + } catch (Throwable e) { + throw e; + } finally { + this.shutdown(this.type()); + } + } + + private void doRestore(List authManagers) { + E.checkState(CollectionUtils.isNotEmpty(authManagers), + "Restore data is empty, please check the type"); + List allConflicts = Lists.newArrayList(); + for (AuthManager authManager : authManagers) { + allConflicts.addAll(authManager.checkConflict()); + } + E.checkState(CollectionUtils.isEmpty(allConflicts), + "Restore conflict with STOP strategy, conflicting " + + "data is s%", JsonUtil.toJson(allConflicts)); + for (AuthManager authManager : authManagers) { + authManager.restore(); + } + } + + private List addAuthManagers(List types) { + List authManagers = Lists.newArrayList(); + for (HugeType type : types) { + switch (type) { + case USER: + authManagers.add(new UserManager()); + break; + case GROUP: + authManagers.add(new GroupManager()); + break; + case TARGET: + authManagers.add(new TargetManager()); + break; + case BELONG: + authManagers.add(new BelongManager()); + break; + case ACCESS: + authManagers.add(new AccessManager()); + break; + default: + throw new AssertionError(String.format( + "Bad auth restore type: %s", type)); + } + } + return authManagers; + } + + private boolean checkAllExistInIdMaps(String oneId, String otherId) { + if (this.idsMap.containsKey(oneId) && + this.idsMap.containsKey(otherId)) { + return true; + } + return false; + } + + private List readRestoreData(HugeType type) { + List resultList = Lists.newArrayList(); + InputStream is = this.inputStream(type.string()); + try (InputStreamReader isr = new InputStreamReader(is, API.CHARSET); + BufferedReader reader = new BufferedReader(isr)) { + String line; + while ((line = reader.readLine()) != null) { + resultList.add(line); + } + } catch (IOException e) { + throw new ToolsException("Failed to deserialize %s from %s", + e, type.string(), resultList); + } + return resultList; + } + + private long writeBackupData(HugeType type, List list) { + long count = 0L; + try { + OutputStream os = this.outputStream(type.string(), false); + ByteArrayOutputStream baos = new ByteArrayOutputStream(LBUF_SIZE); + StringBuilder builder = new StringBuilder(LBUF_SIZE); + + for (Object e : list) { + count++; + builder.append(JsonUtil.toJson(e)).append("\n"); + } + baos.write(builder.toString().getBytes(API.CHARSET)); + os.write(baos.toByteArray()); + } catch (Throwable e) { + throw new ToolsException("Failed to serialize %s to %s", + e, list, type.string()); + } + return count; + } + + protected void directory(String dir, Map hdfsConf) { + if (hdfsConf == null || hdfsConf.isEmpty()) { + // Local FS directory + super.directory = LocalDirectory.constructDir(dir, AUTH_BACKUP_RESTORE); + } else { + // HDFS directory + super.directory = HdfsDirectory.constructDir(dir, AUTH_BACKUP_RESTORE, + hdfsConf); + } + } + + private List sortListByCode(List hugeTypes) { + return hugeTypes.stream() + .sorted(Comparator.comparing(HugeType::code)) + .collect(Collectors.toList()); + } + + private void initPassword(List types, String password) { + E.checkState(!types.contains(HugeType.USER) || + Strings.isNotEmpty(password), + "The following option is " + + "required: [--init-password]"); + this.initPassword = password; + } + + private interface AuthManager { + + void backup(); + + List checkConflict(); + + void restore(); + } + + private class UserManager implements AuthManager { + + @Override + public void backup() { + Printer.print("Users backup started..."); + List users = retry(client.authManager()::listUsers, + "querying users of authority"); + long writeLines = writeBackupData(HugeType.USER, users); + Printer.print("Users backup finished, write lines: %d", + writeLines); + } + + @Override + public List checkConflict() { + List users = retry(client.authManager()::listUsers, + "querying users of authority"); + Map userMap = Maps.newHashMap(); + for (User user : users) { + userMap.put(user.name(), user); + } + List userJsons = readRestoreData(HugeType.USER); + List conflicts = Lists.newArrayList(); + for (String user : userJsons) { + int conflict = NO_CONFLICT; + User restoreUser = JsonUtil.fromJson(user, User.class); + if (!userMap.containsKey(restoreUser.name())) { + this.prepareUserForRestore(restoreUser); + continue; + } + User existUser = userMap.get(restoreUser.name()); + if (!StringUtils.equals(existUser.phone(), + restoreUser.phone())) { + conflict++; + } + if (!StringUtils.equals(existUser.email(), + restoreUser.email())) { + conflict++; + } + if (!StringUtils.equals(existUser.avatar(), + restoreUser.avatar())) { + conflict++; + } + if (conflict > NO_CONFLICT) { + E.checkState(strategy.isStopStrategy() || + strategy.isIgnoreStrategy(), + "Restore users strategy is not found"); + if (strategy.isStopStrategy()) { + conflicts.add(restoreUser.toString()); + } + } else { + idsMap.put(restoreUser.id().toString(), + existUser.id().toString()); + } + } + return conflicts; + } + + @Override + public void restore() { + int count = 0; + for (Map.Entry entry : usersByName.entrySet()) { + User restoreUser = entry.getValue(); + restoreUser.password(initPassword); + User user = retry(() -> { + return client.authManager().createUser(restoreUser); + }, "restore users of authority"); + idsMap.put(restoreUser.id().toString(), user.id().toString()); + count++; + } + Printer.print("Restore users finished, total count is %d", + count); + } + + private void prepareUserForRestore(User restoreUser) { + idsMap.put(restoreUser.id().toString(), restoreUser.id().toString()); + usersByName.put(restoreUser.name(), restoreUser); + } + } + + private class GroupManager implements AuthManager { + + @Override + public void backup() { + Printer.print("Groups backup started..."); + List groups = retry(client.authManager()::listGroups, + "querying groups of authority"); + long writeLines = writeBackupData(HugeType.GROUP, groups); + Printer.print("Groups backup finished, write lines: %d", + writeLines); + } + + @Override + public List checkConflict() { + List groups = retry(client.authManager()::listGroups, + "querying groups of authority"); + Map groupMap = Maps.newHashMap(); + for (Group group : groups) { + groupMap.put(group.name(), group); + } + List groupJsons = readRestoreData(HugeType.GROUP); + List conflicts = Lists.newArrayList(); + for (String group : groupJsons) { + int conflict = NO_CONFLICT; + Group restoreGroup = JsonUtil.fromJson(group, Group.class); + if (!groupMap.containsKey(restoreGroup.name())) { + this.prepareGroupForRestore(restoreGroup); + continue; + } + Group existGroup = groupMap.get(restoreGroup.name()); + if (!StringUtils.equals(existGroup.description(), + restoreGroup.description())) { + conflict++; + } + if (conflict > NO_CONFLICT) { + E.checkState(strategy.isStopStrategy() || + strategy.isIgnoreStrategy(), + "Restore groups strategy is not found"); + if (strategy.isStopStrategy()) { + conflicts.add(restoreGroup.toString()); + } + } else { + idsMap.put(restoreGroup.id().toString(), + existGroup.id().toString()); + } + } + return conflicts; + } + + @Override + public void restore() { + int count = 0; + for (Map.Entry entry : groupsByName.entrySet()) { + Group restoreGroup = entry.getValue(); + Group group = retry(() -> { + return client.authManager().createGroup(restoreGroup); + }, "restore groups of authority"); + idsMap.put(restoreGroup.id().toString(), group.id().toString()); + count++; + } + Printer.print("Restore groups finished, total count is %d", + count); + } + + private void prepareGroupForRestore(Group restoreGroup) { + idsMap.put(restoreGroup.id().toString(), restoreGroup.id().toString()); + groupsByName.put(restoreGroup.name(), restoreGroup); + } + } + + private class TargetManager implements AuthManager { + + @Override + public void backup() { + Printer.print("Targets backup started..."); + List targets = retry(client.authManager()::listTargets, + "querying targets of authority"); + long writeLines = writeBackupData(HugeType.TARGET, targets); + Printer.print("Targets backup finished, write lines: %d", + writeLines); + } + + @Override + public List checkConflict() { + List targets = retry(client.authManager()::listTargets, + "querying targets of authority"); + Map targetMap = Maps.newHashMap(); + for (Target target : targets) { + targetMap.put(target.name(), target); + } + List targetJsons = readRestoreData(HugeType.TARGET); + List conflicts = Lists.newArrayList(); + for (String target : targetJsons) { + int conflict = NO_CONFLICT; + Target restoreTarget = JsonUtil.fromJson(target, Target.class); + if (!targetMap.containsKey(restoreTarget.name())) { + this.prepareTargetForRestore(restoreTarget); + continue; + } + Target existTarget = targetMap.get(restoreTarget.name()); + if (!StringUtils.equals(existTarget.graph(), + restoreTarget.graph())) { + conflict++; + } + if (!StringUtils.equals(existTarget.url(), + restoreTarget.url())) { + conflict++; + } + if (conflict > NO_CONFLICT) { + E.checkState(strategy.isStopStrategy() || + strategy.isIgnoreStrategy(), + "Restore targets strategy is not found"); + if (strategy.isStopStrategy()) { + conflicts.add(restoreTarget.toString()); + } + } else { + idsMap.put(restoreTarget.id().toString(), + existTarget.id().toString()); + } + } + return conflicts; + } + + @Override + public void restore() { + int count = 0; + for (Map.Entry entry : targetsByName.entrySet()) { + Target restoreTarget = entry.getValue(); + Target target = retry(() -> { + return client.authManager().createTarget(restoreTarget); + }, "restore targets of authority"); + idsMap.put(restoreTarget.id().toString(), + target.id().toString()); + count++; + } + Printer.print("Restore targets finished, total count is %d", + count); + } + + private void prepareTargetForRestore(Target restoreTarget) { + idsMap.put(restoreTarget.id().toString(), + restoreTarget.id().toString()); + targetsByName.put(restoreTarget.name(), restoreTarget); + } + } + + private class BelongManager implements AuthManager { + + @Override + public void backup() { + Printer.print("Belongs backup started..."); + List belongs = retry(client.authManager()::listBelongs, + "querying belongs of authority"); + long writeLines = writeBackupData(HugeType.BELONG, belongs); + Printer.print("Belongs backup finished, write lines: %d", + writeLines); + } + + @Override + public List checkConflict() { + List belongs = retry(client.authManager()::listBelongs, + "querying belongs of authority"); + Map belongMap = Maps.newHashMap(); + for (Belong belong : belongs) { + String belongKey = belong.user() + ":" + belong.group(); + belongMap.put(belongKey, belong); + } + List belongJsons = readRestoreData(HugeType.BELONG); + List conflicts = Lists.newArrayList(); + for (String belong : belongJsons) { + Belong restoreBelong = JsonUtil.fromJson(belong, Belong.class); + if (!checkAllExistInIdMaps(restoreBelong.user().toString(), + restoreBelong.group().toString())) { + continue; + } + String ids = idsMap.get(restoreBelong.user()) + ":" + + idsMap.get(restoreBelong.group()); + if (belongMap.containsKey(ids)) { + E.checkState(strategy.isStopStrategy() || + strategy.isIgnoreStrategy(), + "Restore belongs strategy is not found"); + if (strategy.isStopStrategy()) { + conflicts.add(restoreBelong.toString()); + } + continue; + } + belongsByName.put(restoreBelong.id().toString(), restoreBelong); + } + return conflicts; + } + + @Override + public void restore() { + int count = 0; + for (Map.Entry entry : belongsByName.entrySet()) { + Belong restoreBelong = entry.getValue(); + restoreBelong.user(idsMap.get(restoreBelong.user().toString())); + restoreBelong.group(idsMap.get(restoreBelong.group().toString())); + retry(() -> { + return client.authManager().createBelong(restoreBelong); + }, "restore belongs of authority"); + count++; + } + Printer.print("Restore belongs finished, total count is %d", + count); + } + } + + private class AccessManager implements AuthManager { + + @Override + public void backup() { + Printer.print("Accesses backup started..."); + List accesses = retry(client.authManager()::listAccesses, + "querying accesses of authority"); + long writeLines = writeBackupData(HugeType.ACCESS, accesses); + Printer.print("Accesses backup finished, write lines: %d", + writeLines); + } + + @Override + public List checkConflict() { + List accesses = retry(client.authManager()::listAccesses, + "querying accesses of authority"); + Map accessMap = Maps.newHashMap(); + for (Access access : accesses) { + String accessKey = access.group() + ":" + access.target(); + accessMap.put(accessKey, access); + } + List accessJsons = readRestoreData(HugeType.ACCESS); + List conflicts = Lists.newArrayList(); + for (String access : accessJsons) { + Access restoreAccess = JsonUtil.fromJson(access, Access.class); + if (!checkAllExistInIdMaps(restoreAccess.group().toString(), + restoreAccess.target().toString())) { + continue; + } + String ids = idsMap.get(restoreAccess.group()) + ":" + + idsMap.get(restoreAccess.target()); + if (accessMap.containsKey(ids)) { + E.checkState(strategy.isStopStrategy() || + strategy.isIgnoreStrategy(), + "Restore accesses strategy is not found"); + if (strategy.isStopStrategy()) { + conflicts.add(restoreAccess.toString()); + } + continue; + } + accessesByName.put(restoreAccess.id().toString(), restoreAccess); + } + return conflicts; + } + + @Override + public void restore() { + int count = 0; + for (Map.Entry entry : accessesByName.entrySet()) { + Access restoreAccess = entry.getValue(); + restoreAccess.target(idsMap.get(restoreAccess.target().toString())); + restoreAccess.group(idsMap.get(restoreAccess.group().toString())); + retry(() -> { + return client.authManager().createAccess(restoreAccess); + }, "restore accesses of authority"); + count++; + } + Printer.print("Restore accesses finished, total count is %d", + count); + } + } +} diff --git a/src/main/java/com/baidu/hugegraph/manager/BackupManager.java b/src/main/java/com/baidu/hugegraph/manager/BackupManager.java index 7e9e237..c08ed93 100644 --- a/src/main/java/com/baidu/hugegraph/manager/BackupManager.java +++ b/src/main/java/com/baidu/hugegraph/manager/BackupManager.java @@ -120,6 +120,16 @@ public long splitSize() { } public void backup(List types) { + try { + this.doBackup(types); + } catch (Throwable e) { + throw e; + } finally { + this.shutdown(this.type()); + } + } + + public void doBackup(List types) { this.startTimer(); for (HugeType type : types) { switch (type) { @@ -146,7 +156,6 @@ public void backup(List types) { "Bad backup type: %s", type)); } } - this.shutdown(this.type()); this.printSummary(); } diff --git a/src/main/java/com/baidu/hugegraph/manager/BackupRestoreBaseManager.java b/src/main/java/com/baidu/hugegraph/manager/BackupRestoreBaseManager.java index d8363d3..2d5d0d4 100644 --- a/src/main/java/com/baidu/hugegraph/manager/BackupRestoreBaseManager.java +++ b/src/main/java/com/baidu/hugegraph/manager/BackupRestoreBaseManager.java @@ -48,8 +48,6 @@ import com.baidu.hugegraph.util.E; import com.google.common.collect.ImmutableMap; -import static com.baidu.hugegraph.base.Directory.closeAndIgnoreException; - public class BackupRestoreBaseManager extends RetryManager { public static final int BATCH = 500; @@ -199,13 +197,13 @@ protected OutputStream outputStream(String file, boolean compress) { os = this.directory.outputStream(file, compress, true); OutputStream prev = this.outputStreams.putIfAbsent(file, os); if (prev != null) { - closeAndIgnoreException(os); + Directory.closeAndIgnoreException(os); os = prev; } return os; } - private InputStream inputStream(String file) { + protected InputStream inputStream(String file) { InputStream is = this.inputStreams.get(file); if (is != null) { return is; @@ -213,7 +211,7 @@ private InputStream inputStream(String file) { is = this.directory.inputStream(file); InputStream prev = this.inputStreams.putIfAbsent(file, is); if (prev != null) { - closeAndIgnoreException(is); + Directory.closeAndIgnoreException(is); is = prev; } return is; diff --git a/src/main/java/com/baidu/hugegraph/manager/DumpGraphManager.java b/src/main/java/com/baidu/hugegraph/manager/DumpGraphManager.java index 7a9655a..fe65d60 100644 --- a/src/main/java/com/baidu/hugegraph/manager/DumpGraphManager.java +++ b/src/main/java/com/baidu/hugegraph/manager/DumpGraphManager.java @@ -20,10 +20,7 @@ package com.baidu.hugegraph.manager; import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileOutputStream; import java.io.OutputStream; -import java.nio.file.Paths; import java.util.Collection; import java.util.List; @@ -74,17 +71,21 @@ public void init(SubCommands.DumpGraph dump) { public void dump() { this.startTimer(); - - // Fetch data to JsonGraph - this.backupVertices(); - this.backupEdges(); - - // Dump to file - for (String table : this.graph.tables()) { - this.submit(() -> dump(table, this.graph.table(table).values())); + try { + // Fetch data to JsonGraph + this.backupVertices(); + this.backupEdges(); + + // Dump to file + for (String table : this.graph.tables()) { + this.submit(() -> dump(table, this.graph.table(table).values())); + } + } catch (Throwable e) { + throw e; + } finally { + this.shutdown(this.type()); } - this.shutdown(this.type()); this.printSummary("dump graph"); } diff --git a/src/main/java/com/baidu/hugegraph/manager/RestoreManager.java b/src/main/java/com/baidu/hugegraph/manager/RestoreManager.java index 713848b..91ac811 100644 --- a/src/main/java/com/baidu/hugegraph/manager/RestoreManager.java +++ b/src/main/java/com/baidu/hugegraph/manager/RestoreManager.java @@ -60,6 +60,16 @@ public void mode(GraphMode mode) { } public void restore(List types) { + try { + this.doRestore(types); + } catch (Throwable e) { + throw e; + } finally { + this.shutdown(this.type()); + } + } + + public void doRestore(List types) { E.checkNotNull(this.mode, "mode"); this.startTimer(); for (HugeType type : types) { @@ -87,7 +97,6 @@ public void restore(List types) { "Bad restore type: %s", type)); } } - this.shutdown(this.type()); this.printSummary(); if (this.clean) { this.removeDirectory(); diff --git a/src/main/java/com/baidu/hugegraph/util/ToolUtil.java b/src/main/java/com/baidu/hugegraph/util/ToolUtil.java new file mode 100644 index 0000000..11ecbf9 --- /dev/null +++ b/src/main/java/com/baidu/hugegraph/util/ToolUtil.java @@ -0,0 +1,98 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.util; + +import java.util.Map; +import java.util.Scanner; + +import com.baidu.hugegraph.base.Printer; +import com.baidu.hugegraph.constant.Constants; +import com.baidu.hugegraph.exception.ExitException; +import com.beust.jcommander.JCommander; + +public final class ToolUtil { + + public static void printOrThrow(Throwable e, boolean throwMode) { + Printer.print("Failed to execute %s", e.getMessage()); + if (throwMode) { + if (e instanceof RuntimeException) { + throw (RuntimeException) e; + } + throw new RuntimeException(e); + } + printExceptionStackIfNeeded(e); + } + + public static void printExceptionStackIfNeeded(Throwable e) { + System.out.println("Type y(yes) to print exception stack[default n]?"); + Scanner scan = new Scanner(System.in); + String inputInfomation = scan.nextLine(); + + if (inputInfomation.equalsIgnoreCase(Constants.INPUT_YES) || + inputInfomation.equalsIgnoreCase(Constants.INPUT_Y)) { + e.printStackTrace(); + } + } + + public static void exitOrThrow(ExitException e, boolean throwMode) { + if (throwMode) { + throw e; + } + + if(e.exitCode() != Constants.EXIT_CODE_NORMAL) { + Printer.print(e.getMessage()); + } + Printer.print(e.details()); + } + + public static String commandsCategory(JCommander jCommander) { + StringBuffer sb = new StringBuffer(); + sb.append("================================================"); + sb.append("\n"); + sb.append("Warning : must provide one sub-command"); + sb.append("\n"); + sb.append("================================================"); + sb.append("\n"); + sb.append("Here are some sub-command :"); + sb.append("\n"); + Map subCommandes = jCommander.getCommands(); + for (String subCommand : subCommandes.keySet()) { + sb.append("|"); + sb.append(subCommand); + sb.append("\n"); + } + sb.append("================================================"); + sb.append("\n"); + sb.append("Please use 'hugegraph help' to get detail help info " + + "of all sub-commands or 'hugegraph help {sub-command}' " + + "to get detail help info of one sub-command"); + sb.append("\n"); + sb.append("================================================"); + + return sb.toString(); + } + + public static String commandUsage(JCommander jCommander) { + StringBuilder sb = new StringBuilder(); + jCommander.usage(sb); + + return sb.toString(); + } +} diff --git a/src/test/java/com/baidu/hugegraph/test/functional/AuthBackupTest.java b/src/test/java/com/baidu/hugegraph/test/functional/AuthBackupTest.java new file mode 100644 index 0000000..3a63a71 --- /dev/null +++ b/src/test/java/com/baidu/hugegraph/test/functional/AuthBackupTest.java @@ -0,0 +1,107 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.test.functional; + +import java.util.List; + +import org.junit.Before; +import org.junit.Test; + +import com.baidu.hugegraph.cmd.HugeGraphCommand; +import com.baidu.hugegraph.test.util.FileUtil; +import com.baidu.hugegraph.testutil.Assert; + +public class AuthBackupTest extends AuthTest { + + @Before + public void init() { + FileUtil.clearDirectories(DEFAULT_URL); + } + + @Test + public void testAuthBackup() { + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD, + "auth-backup" + }; + + HugeGraphCommand.main(args); + + Assert.assertTrue(FileUtil.checkFileExists(DEFAULT_URL)); + List fileNames = FileUtil.subdirectories(DEFAULT_URL); + Assert.assertTrue(fileNames.size() == 5); + } + + @Test + public void testAuthBackupByTypes() { + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD, + "auth-backup", + "--types", "user,group" + }; + + HugeGraphCommand.main(args); + + Assert.assertTrue(FileUtil.checkFileExists(DEFAULT_URL)); + List fileNames = FileUtil.subdirectories(DEFAULT_URL); + Assert.assertTrue(fileNames.size() == 2); + } + + @Test + public void testAuthBackupWithWrongType() { + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD, + "auth-backup", + "--types", "user,group,test" + }; + + Assert.assertThrows(IllegalArgumentException.class, () -> { + HugeGraphCommand.main(args); + }, e -> { + Assert.assertContains("valid value is 'all' or combination of " + + "[user,group,target,belong,access]", + e.getMessage()); + }); + } + + @Test + public void testAuthBackupByDirectory() { + String directory = "./backup"; + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD, + "auth-backup", + "--directory", directory + }; + + HugeGraphCommand.main(args); + + Assert.assertTrue(FileUtil.checkFileExists(directory)); + List fileNames = FileUtil.subdirectories(directory); + Assert.assertTrue(fileNames.size() == 5); + } +} diff --git a/src/test/java/com/baidu/hugegraph/test/functional/AuthRestoreTest.java b/src/test/java/com/baidu/hugegraph/test/functional/AuthRestoreTest.java new file mode 100644 index 0000000..22e6a9e --- /dev/null +++ b/src/test/java/com/baidu/hugegraph/test/functional/AuthRestoreTest.java @@ -0,0 +1,345 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.test.functional; + +import java.util.List; +import java.util.Map; + +import org.apache.commons.collections.CollectionUtils; +import org.junit.Before; +import org.junit.Test; + +import com.baidu.hugegraph.cmd.HugeGraphCommand; +import com.baidu.hugegraph.driver.HugeClient; +import com.baidu.hugegraph.structure.auth.Access; +import com.baidu.hugegraph.structure.auth.Belong; +import com.baidu.hugegraph.structure.auth.Group; +import com.baidu.hugegraph.structure.auth.Target; +import com.baidu.hugegraph.structure.auth.User; +import com.baidu.hugegraph.structure.constant.HugeType; +import com.baidu.hugegraph.test.util.FileUtil; +import com.baidu.hugegraph.testutil.Assert; +import com.beust.jcommander.ParameterException; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +public class AuthRestoreTest extends AuthTest { + + private HugeClient client; + + @Before + public void init() { + client = HugeClient.builder(URL, GRAPH) + .configUser(USER_NAME, USER_PASSWORD) + .configTimeout(TIME_OUT) + .configSSL(TRUST_STORE_FILE, TRUST_STORE_PASSWORD) + .build(); + } + + @Test + public void testAuthRestoreForAllType() { + this.loadData(HugeType.USER, "auth_users.txt"); + this.loadData(HugeType.TARGET, "auth_targets.txt"); + this.loadData(HugeType.GROUP, "auth_groups.txt"); + this.loadData(HugeType.BELONG, "auth_belongs.txt"); + this.loadData(HugeType.ACCESS, "auth_accesses.txt"); + + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD, + "auth-restore", + "--directory", DEFAULT_URL, + "--init-password", "123456", + "--strategy", "ignore" + }; + + HugeGraphCommand.main(args); + + List idList = Lists.newArrayList(); + List userList = this.client.auth().listUsers(); + Map userMap = Maps.newHashMap(); + for (User user1 : userList) { + userMap.put(user1.name(), user1); + } + Assert.assertTrue(userMap.containsKey("test_user1")); + idList.add(userMap.get("test_user1").id().toString()); + + List groups = this.client.auth().listGroups(); + Map groupMap = Maps.newHashMap(); + for (Group group : groups) { + groupMap.put(group.name(), group); + } + Assert.assertTrue(groupMap.containsKey("test_group6")); + idList.add(groupMap.get("test_group6").id().toString()); + + List targets = this.client.auth().listTargets(); + Map targetMap = Maps.newHashMap(); + for (Target target : targets) { + targetMap.put(target.name(), target); + } + Assert.assertTrue(targetMap.containsKey("test_target1")); + idList.add(targetMap.get("test_target1").id().toString()); + + List belongs = this.client.auth().listBelongs(); + Assert.assertTrue(CollectionUtils.isNotEmpty(belongs)); + boolean checkUserAndGroup = false; + for (Belong belong : belongs) { + if (idList.contains(belong.user().toString()) && + idList.contains(belong.group().toString())) { + checkUserAndGroup = true; + break; + } + } + Assert.assertTrue(checkUserAndGroup); + + List accesses = this.client.auth().listAccesses(); + Assert.assertTrue(CollectionUtils.isNotEmpty(accesses)); + boolean checkGroupAndTarget = false; + for (Access access : accesses) { + if (idList.contains(access.group().toString()) && + idList.contains(access.target().toString())) { + checkGroupAndTarget = true; + break; + } + } + Assert.assertTrue(checkGroupAndTarget); + } + + @Test + public void testAuthRestoreForUser() { + this.loadData(HugeType.USER, "auth_users.txt"); + + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD, + "auth-restore", + "--types", "user", + "--directory", DEFAULT_URL, + "--init-password", "123456" + }; + + HugeGraphCommand.main(args); + + List userList = this.client.auth().listUsers(); + Map userMap = Maps.newHashMap(); + for (User user1 : userList) { + userMap.put(user1.name(), user1); + } + + Assert.assertTrue(userMap.containsKey("test_user1")); + } + + @Test + public void testRestoreWithoutInitPassword() { + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD, + "auth-restore", + "--types", "user", + "--directory", DEFAULT_URL + }; + + Assert.assertThrows(IllegalStateException.class, () -> { + HugeGraphCommand.main(args); + }, e -> { + String msg = e.getMessage(); + Assert.assertTrue(msg.endsWith("The following option is " + + "required: [--init-password]")); + }); + } + + @Test + public void testAuthRestoreWithConflictAndStopStrategy() { + this.loadData(HugeType.USER, "auth_users_conflict.txt"); + + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD, + "auth-restore", + "--types", "user", + "--strategy", "stop", + "--init-password", "123456" + }; + + Assert.assertThrows(IllegalStateException.class, () -> { + HugeGraphCommand.main(args); + }, e -> { + Assert.assertContains("Restore conflict with STOP strategy", + e.getMessage()); + }); + } + + @Test + public void testAuthRestoreWithIgnoreStrategy() { + this.loadData(HugeType.USER, "auth_users_conflict.txt"); + + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD, + "auth-restore", + "--types", "user", + "--strategy", "ignore", + "--init-password", "123456" + }; + + HugeGraphCommand.main(args); + + List userList = this.client.auth().listUsers(); + Map userMap = Maps.newHashMap(); + for (User user1 : userList) { + userMap.put(user1.name(), user1); + } + + Assert.assertTrue(userMap.containsKey("admin")); + } + + @Test + public void testAuthRestoreWithWrongDirectory() { + String filePath = "./auth-test-test"; + + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD, + "auth-restore", + "--types", "user", + "--strategy", "stop", + "--init-password", "123456", + "--directory", filePath + }; + + Assert.assertThrows(IllegalStateException.class, () -> { + HugeGraphCommand.main(args); + }, e -> { + Assert.assertContains("The directory does not exist", + e.getMessage()); + }); + } + + @Test + public void testAuthRestoreWithWrongType() { + String filePath = "./auth-test-test"; + + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD, + "auth-restore", + "--types", "user，test", + "--strategy", "stop", + "--init-password", "123456", + "--directory", filePath + }; + + Assert.assertThrows(IllegalArgumentException.class, () -> { + HugeGraphCommand.main(args); + }, e -> { + Assert.assertContains("valid value is 'all' or combination of " + + "[user,group,target,belong,access]", + e.getMessage()); + }); + } + + @Test + public void testAuthRestoreByBelongWithoutDependency() { + String filePath = "./auth-test-test"; + + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD, + "auth-restore", + "--types", "belong", + "--strategy", "stop", + "--init-password", "123456", + "--directory", filePath + }; + + Assert.assertThrows(IllegalArgumentException.class, () -> { + HugeGraphCommand.main(args); + }, e -> { + Assert.assertContains("if type contains 'belong' then " + + "'user' and 'group' are required.", + e.getMessage()); + }); + } + + @Test + public void testAuthRestoreByAccessWithoutDependency() { + String filePath = "./auth-test-test"; + + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD, + "auth-restore", + "--types", "access", + "--strategy", "stop", + "--init-password", "123456", + "--directory", filePath + }; + + Assert.assertThrows(IllegalArgumentException.class, () -> { + HugeGraphCommand.main(args); + }, e -> { + Assert.assertContains("if type contains 'access' then " + + "'group' and 'target' are required.", + e.getMessage()); + }); + } + + @Test + public void testAuthRestoreWithWrongStrategy() { + String filePath = "./auth-test-test"; + + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD, + "auth-restore", + "--types", "user", + "--strategy", "test", + "--init-password", "123456", + "--directory", filePath + }; + + Assert.assertThrows(IllegalArgumentException.class, () -> { + HugeGraphCommand.main(args); + }, e -> { + Assert.assertContains("Invalid --strategy 'test', valid " + + "value is 'stop' or 'ignore", + e.getMessage()); + }); + } + + private void loadData(HugeType hugeType, String dataFilePath) { + String restoreDataPath = DEFAULT_URL + hugeType.string(); + String testRestoreDataPath = DEFAULT_TEST_URL + dataFilePath; + + List list = FileUtil.readTestRestoreData(FileUtil.configPath( + testRestoreDataPath)); + FileUtil.writeTestRestoreData(restoreDataPath, list); + } +} diff --git a/src/test/java/com/baidu/hugegraph/test/functional/AuthTest.java b/src/test/java/com/baidu/hugegraph/test/functional/AuthTest.java new file mode 100644 index 0000000..5d3ad3a --- /dev/null +++ b/src/test/java/com/baidu/hugegraph/test/functional/AuthTest.java @@ -0,0 +1,33 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.test.functional; + +public class AuthTest { + + public static final String DEFAULT_URL = "./auth-backup-restore/"; + public static final String DEFAULT_TEST_URL = "/auth/"; + public static final String USER_NAME = "admin"; + public static final String USER_PASSWORD = "123456"; + public static final String URL = "http://127.0.0.1:8080"; + public static final String GRAPH = "hugegraph"; + public static final Integer TIME_OUT = 30; + public static final String TRUST_STORE_FILE = ""; + public static final String TRUST_STORE_PASSWORD = ""; +} diff --git a/src/test/java/com/baidu/hugegraph/test/functional/CommandTest.java b/src/test/java/com/baidu/hugegraph/test/functional/CommandTest.java new file mode 100644 index 0000000..5676d16 --- /dev/null +++ b/src/test/java/com/baidu/hugegraph/test/functional/CommandTest.java @@ -0,0 +1,110 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.test.functional; + +import org.junit.Test; + +import com.baidu.hugegraph.cmd.HugeGraphCommand; +import com.baidu.hugegraph.exception.ExitException; +import com.baidu.hugegraph.testutil.Assert; + +public class CommandTest extends AuthTest { + + @Test + public void testHelpCommand() { + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD, + "help" + }; + + Assert.assertThrows(ExitException.class, () -> { + HugeGraphCommand.main(args); + }, e -> { + ExitException exception = (ExitException) e; + Assert.assertContains("Command : hugegragh help", + exception.getMessage()); + Assert.assertContains("Usage: hugegraph [options] [command]", + exception.details()); + }); + } + + @Test + public void testHelpSubCommand() { + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD, + "help", "auth-backup" + }; + + Assert.assertThrows(ExitException.class, () -> { + HugeGraphCommand.main(args); + }, e -> { + ExitException exception = (ExitException) e; + Assert.assertContains("Command : hugegragh help auth-backup", + exception.getMessage()); + Assert.assertContains("Usage: auth-backup [options]", + exception.details()); + }); + } + + @Test + public void testBadHelpSubCommandException() { + String badCommand = "asd"; + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD, + "help", badCommand + }; + + Assert.assertThrows(ExitException.class, () -> { + HugeGraphCommand.main(args); + }, e -> { + ExitException exception = (ExitException) e; + Assert.assertContains(String.format( + "Unexpected help sub-command %s", + badCommand), exception.getMessage()); + Assert.assertContains("Here are some sub-command ", + exception.details()); + }); + } + + @Test + public void testEmptyCommandException() { + String[] args = new String[]{ + "--throw-mode", "true", + "--user", USER_NAME, + "--password", USER_PASSWORD + }; + + Assert.assertThrows(ExitException.class, () -> { + HugeGraphCommand.main(args); + }, e -> { + ExitException exception = (ExitException) e; + Assert.assertContains("No sub-command found", + exception.getMessage()); + Assert.assertContains("Warning : must provide one sub-command", + exception.details()); + }); + } +} diff --git a/src/test/java/com/baidu/hugegraph/test/functional/FuncTestSuite.java b/src/test/java/com/baidu/hugegraph/test/functional/FuncTestSuite.java new file mode 100644 index 0000000..6e770f6 --- /dev/null +++ b/src/test/java/com/baidu/hugegraph/test/functional/FuncTestSuite.java @@ -0,0 +1,32 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.test.functional; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +@RunWith(Suite.class) +@Suite.SuiteClasses({ + AuthBackupTest.class, + AuthRestoreTest.class, + CommandTest.class +}) +public class FuncTestSuite { +} diff --git a/src/test/java/com/baidu/hugegraph/test/util/FileUtil.java b/src/test/java/com/baidu/hugegraph/test/util/FileUtil.java new file mode 100644 index 0000000..13858f9 --- /dev/null +++ b/src/test/java/com/baidu/hugegraph/test/util/FileUtil.java @@ -0,0 +1,117 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.test.util; + +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.file.Paths; +import java.util.List; + +import org.apache.commons.collections.ListUtils; + +import com.baidu.hugegraph.api.API; +import com.baidu.hugegraph.exception.ToolsException; +import com.google.common.collect.Lists; + +public class FileUtil { + + protected static final int LBUF_SIZE = 1024; + protected static final String CONFIG_PATH = "src/test/resources"; + + public static String configPath(String fileName) { + return Paths.get(CONFIG_PATH, fileName).toString(); + } + + public static boolean checkFileExists(String filePath) { + File file = new File(filePath); + if (file.exists()) { + return true; + } + return false; + } + + public static List subdirectories(String filePath) { + File file = new File(filePath); + if (!file.exists()) { + return ListUtils.EMPTY_LIST; + } + String[] files = file.list(); + List list = Lists.newArrayList(); + for (int i = 0; i < files.length; i++) { + File fileDir = new File(file, files[i]); + list.add(fileDir.getName()); + } + + return list; + } + + public static void clearDirectories(String filePath) { + File file = new File(filePath); + if (file.exists()) { + String[] files = file.list(); + for (int i = 0; i < files.length; i++) { + File fileDir = new File(file, files[i]); + fileDir.delete(); + } + } + } + + public static long writeTestRestoreData(String filePath, List list) { + long count = 0L; + try (FileOutputStream os = new FileOutputStream(filePath); + ByteArrayOutputStream baos = new ByteArrayOutputStream(LBUF_SIZE)) { + StringBuilder builder = new StringBuilder(LBUF_SIZE); + for (Object e : list) { + count++; + builder.append(e).append("\n"); + } + baos.write(builder.toString().getBytes(API.CHARSET)); + os.write(baos.toByteArray()); + } catch (IOException e) { + throw new ToolsException("Failed write file path is %s", + e, filePath); + } + + return count; + } + + public static List readTestRestoreData(String filePath) { + List results = Lists.newArrayList(); + try (InputStream is = new FileInputStream(filePath); + InputStreamReader isr = new InputStreamReader(is, API.CHARSET)) { + BufferedReader reader = new BufferedReader(isr); + String line; + while ((line = reader.readLine()) != null) { + results.add(line); + } + } catch (IOException e) { + throw new ToolsException("Failed read file path is %s", + e, filePath); + } + + return results; + } +} diff --git a/src/test/resources/auth/auth_accesses.txt b/src/test/resources/auth/auth_accesses.txt new file mode 100644 index 0000000..ffae858 --- /dev/null +++ b/src/test/resources/auth/auth_accesses.txt @@ -0,0 +1 @@ +{"id":"S-66:test_group6>-88>11>S-66:test_target1","group":"-66:test_group6","target":"-66:test_target1","access_permission":"READ","access_description":"test","access_create":"2020-11-11 15:54:54.008","access_update":"2020-11-18 15:01:13.518","access_creator":"admin"} \ No newline at end of file diff --git a/src/test/resources/auth/auth_belongs.txt b/src/test/resources/auth/auth_belongs.txt new file mode 100644 index 0000000..66e8233 --- /dev/null +++ b/src/test/resources/auth/auth_belongs.txt @@ -0,0 +1 @@ +{"id":"S-66:test_user1>-82>>S-66:test_group6","user":"-66:test_user1","group":"-66:test_group6","belong_description":"restore test","belong_create":"2020-12-01 09:44:40.117","belong_update":"2020-12-01 09:44:40.117","belong_creator":"admin"} \ No newline at end of file diff --git a/src/test/resources/auth/auth_groups.txt b/src/test/resources/auth/auth_groups.txt new file mode 100644 index 0000000..3dedb4b --- /dev/null +++ b/src/test/resources/auth/auth_groups.txt @@ -0,0 +1 @@ +{"id":"-66:test_group6","group_name":"test_group6","group_description":"user is conflict check user restore test test","group_create":"2020-11-27 20:08:21.270","group_update":"2020-11-27 20:08:21.270","group_creator":"admin"} \ No newline at end of file diff --git a/src/test/resources/auth/auth_targets.txt b/src/test/resources/auth/auth_targets.txt new file mode 100644 index 0000000..96d411f --- /dev/null +++ b/src/test/resources/auth/auth_targets.txt @@ -0,0 +1 @@ +{"id":"-66:test_target1","target_name":"test_target1","target_graph":"hugegraph","target_url":"127.0.0.1:8080","target_resources":[{"type":"ALL","label":"*","properties":null}],"target_create":"2020-11-11 15:32:01.192","target_update":"2020-11-11 15:32:01.192","target_creator":"admin"} \ No newline at end of file diff --git a/src/test/resources/auth/auth_users.txt b/src/test/resources/auth/auth_users.txt new file mode 100644 index 0000000..233cbb1 --- /dev/null +++ b/src/test/resources/auth/auth_users.txt @@ -0,0 +1 @@ +{"id":"-66:test_user1","user_name":"test_user1","user_password":"$2a$04$vXkz8UYV7Gwagj6zA1ifNuSQfAmzuYb2tXdqDoWKEG.nYVc186JXO","user_create":"2020-11-30 22:26:42.225","user_update":"2020-11-30 22:26:42.225","user_creator":"admin"} \ No newline at end of file diff --git a/src/test/resources/auth/auth_users_conflict.txt b/src/test/resources/auth/auth_users_conflict.txt new file mode 100644 index 0000000..7dc07c5 --- /dev/null +++ b/src/test/resources/auth/auth_users_conflict.txt @@ -0,0 +1 @@ +{"id":"-63:admin","user_name":"admin","user_phone":"13255447788","user_password":"$2a$04$1tl1IKTncjcmMojLdt2qO.EAJ1w0TGunAZ5IJXWwBgPLvTPk366Ly","user_create":"2020-11-11 11:41:12.254","user_update":"2020-11-11 11:41:12.254","user_creator":"system"} \ No newline at end of file