From de166345045a036f4c25283d5cd0970a3a1a0f4f Mon Sep 17 00:00:00 2001 From: Zhangmei Li Date: Sun, 19 Aug 2018 10:40:29 +0800 Subject: [PATCH] speed up tinkerpop test how to: 1.speed up rocksdb backend by truncating tables 2.don't need to clear if the database is empty 3.don't need to clear variables if it's empty 4.don't need to commit hbase each len-prefix for delete index-label potential problems: 1.mysql may block if create task vertex-label when truncating table 2.rocksdb may miss CF if create task vertex-label when truncating table 3.hbase may block if truncate hbase with version<2.0 improve #14 Change-Id: I4b2393aea8b0fc63c1886e984a576a1f5808b25c --- .../com/baidu/hugegraph/api/GraphsAPI.java | 25 +----- .../com/baidu/hugegraph/api/job/TaskAPI.java | 2 +- .../hugegraph/auth/HugeGraphAuthProxy.java | 5 ++ .../store/cassandra/CassandraSessionPool.java | 4 + .../store/cassandra/CassandraStore.java | 66 +++++++++----- .../store/cassandra/CassandraTable.java | 5 ++ hugegraph-core/pom.xml | 2 +- .../java/com/baidu/hugegraph/HugeGraph.java | 5 ++ .../baidu/hugegraph/backend/LocalCounter.java | 4 + .../backend/cache/CachedBackendStore.java | 5 ++ .../cache/CachedSchemaTransaction.java | 85 ++++++++++++------- .../backend/serializer/BinarySerializer.java | 1 + .../store/AbstractBackendStoreProvider.java | 38 ++++++++- .../hugegraph/backend/store/BackendStore.java | 3 + .../backend/store/BackendStoreProvider.java | 4 + .../backend/store/memory/InMemoryDBStore.java | 30 +++++-- .../store/memory/InMemoryDBTables.java | 9 +- .../backend/tx/GraphTransaction.java | 3 +- .../backend/tx/SchemaTransaction.java | 1 + .../hugegraph/task/HugeTaskScheduler.java | 8 +- .../java/com/baidu/hugegraph/util/Events.java | 1 + .../baidu/hugegraph/version/CoreVersion.java | 2 +- .../backend/store/hbase/HbaseSessions.java | 14 +++ .../backend/store/hbase/HbaseStore.java | 22 +++++ .../backend/store/hbase/HbaseTables.java | 9 +- .../backend/store/mysql/MysqlSessions.java | 13 +++ .../backend/store/mysql/MysqlStore.java | 51 ++++++----- .../store/mysql/MysqlStoreProvider.java | 2 +- .../backend/store/mysql/MysqlTable.java | 35 +++++--- .../backend/store/palo/PaloStore.java | 2 +- .../backend/store/rocksdb/RocksDBStore.java | 14 ++- .../backend/store/rocksdb/RocksDBTables.java | 38 ++++----- .../baidu/hugegraph/tinkerpop/TestGraph.java | 31 +++++++ .../tinkerpop/TestGraphProvider.java | 45 +++++----- .../src/main/resources/hugegraph.properties | 15 +++- 35 files changed, 422 insertions(+), 177 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/GraphsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/GraphsAPI.java index 7ee91fd6d6..f64e8a03f0 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/GraphsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/GraphsAPI.java @@ -36,13 +36,10 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.SecurityContext; -import org.apache.tinkerpop.gremlin.structure.Edge; -import org.apache.tinkerpop.gremlin.structure.Vertex; import org.slf4j.Logger; import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.core.GraphManager; -import com.baidu.hugegraph.schema.SchemaManager; import com.baidu.hugegraph.server.RestServer; import com.baidu.hugegraph.util.Log; import com.codahale.metrics.annotation.Timed; @@ -127,27 +124,7 @@ public void clear(@Context GraphManager manager, throw new IllegalArgumentException(String.format( "Please take the message: %s", CONFIRM_CLEAR)); } - - // Clear vertex and edge - commit(g, () -> { - g.traversal().E().toStream().forEach(Edge::remove); - g.traversal().V().toStream().forEach(Vertex::remove); - }); - - // Schema operation will auto commit - SchemaManager schema = g.schema(); - schema.getIndexLabels().forEach(elem -> { - schema.indexLabel(elem.name()).remove(); - }); - schema.getEdgeLabels().forEach(elem -> { - schema.edgeLabel(elem.name()).remove(); - }); - schema.getVertexLabels().forEach(elem -> { - schema.vertexLabel(elem.name()).remove(); - }); - schema.getPropertyKeys().forEach(elem -> { - schema.propertyKey(elem.name()).remove(); - }); + g.truncateBackend(); } @PUT diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/TaskAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/TaskAPI.java index 0d99f0fb2e..34d7387e31 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/TaskAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/TaskAPI.java @@ -108,7 +108,7 @@ public void delete(@Context GraphManager manager, HugeGraph g = graph(manager, graph); HugeTaskScheduler scheduler = g.taskScheduler(); - HugeTask task = scheduler.task(IdGenerator.of(id)); + HugeTask task = scheduler.task(IdGenerator.of(id)); if (task.completed()) { scheduler.deleteTask(IdGenerator.of(id)); } else { diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java index 9000ed1eb6..8264a94834 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java @@ -168,6 +168,11 @@ public void clearBackend() { this.hugegraph.clearBackend(); } + public void truncateBackend() { + this.verifyPermission(ROLE_ADMIN); + this.hugegraph.truncateBackend(); + } + public void restoring(boolean restoring) { this.verifyPermission(ROLE_ADMIN); this.hugegraph.restoring(restoring); diff --git a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraSessionPool.java b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraSessionPool.java index b80c1481a4..5a2d8457de 100644 --- a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraSessionPool.java +++ b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraSessionPool.java @@ -222,6 +222,10 @@ public void open() { this.session = cluster().connect(keyspace()); } + public boolean opened() { + return this.session != null; + } + @Override public boolean closed() { if (this.session == null) { diff --git a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraStore.java b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraStore.java index b1db12ba55..401dde7f53 100644 --- a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraStore.java +++ b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraStore.java @@ -19,8 +19,11 @@ package com.baidu.hugegraph.backend.store.cassandra; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -242,12 +245,25 @@ public BackendFeatures features() { @Override public void init() { this.checkClusterConnected(); - this.initKeyspace(); + if (this.sessions.session().opened()) { + // Session has ever been opened. + LOG.warn("Session has ever been opened(exist keyspace '{}' before)", + this.keyspace); + } else { + // Create keyspace if need + if (!this.existsKeyspace()) { + this.initKeyspace(); + } + // Open session explicitly to get the exception when it fails + this.sessions.session().open(); + } + + // Create tables this.checkSessionConnected(); this.initTables(); - LOG.info("Store initialized: {}", this.store); + LOG.debug("Store initialized: {}", this.store); } @Override @@ -260,7 +276,15 @@ public void clear() { this.clearKeyspace(); } - LOG.info("Store cleared: {}", this.store); + LOG.debug("Store cleared: {}", this.store); + } + + @Override + public void truncate() { + this.checkSessionConnected(); + + this.truncateTables(); + LOG.debug("Store truncated: {}", this.store); } @Override @@ -358,7 +382,8 @@ protected void initKeyspace() { replication.putIfAbsent("replication_factor", factor); Statement stmt = SchemaBuilder.createKeyspace(this.keyspace) - .ifNotExists().with().replication(replication); + .ifNotExists().with() + .replication(replication); // Create keyspace with non-keyspace-session LOG.debug("Create keyspace: {}", stmt); @@ -370,7 +395,6 @@ protected void initKeyspace() { session.close(); } } - this.sessions.session().open(); } protected void clearKeyspace() { @@ -394,18 +418,29 @@ protected boolean existsKeyspace() { protected void initTables() { CassandraSessionPool.Session session = this.sessions.session(); - for (CassandraTable table : this.tables.values()) { + for (CassandraTable table : this.tables()) { table.init(session); } } protected void clearTables() { CassandraSessionPool.Session session = this.sessions.session(); - for (CassandraTable table : this.tables.values()) { + for (CassandraTable table : this.tables()) { table.clear(session); } } + protected void truncateTables() { + CassandraSessionPool.Session session = this.sessions.session(); + for (CassandraTable table : this.tables()) { + table.truncate(session); + } + } + + protected Collection tables() { + return this.tables.values(); + } + protected final CassandraTable table(HugeType type) { assert type != null; CassandraTable table = this.tables.get(type); @@ -464,19 +499,10 @@ public CassandraSchemaStore(BackendStoreProvider provider, } @Override - protected void initTables() { - super.initTables(); - - CassandraSessionPool.Session session = super.sessions.session(); - this.counters.init(session); - } - - @Override - protected void clearTables() { - super.clearTables(); - - CassandraSessionPool.Session session = super.sessions.session(); - this.counters.clear(session); + protected Collection tables() { + List tables = new ArrayList<>(super.tables()); + tables.add(this.counters); + return tables; } @Override diff --git a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java index fe9f010849..30f33b4fc9 100644 --- a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java +++ b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java @@ -570,4 +570,9 @@ protected void createIndex(CassandraSessionPool.Session session, public void clear(CassandraSessionPool.Session session) { this.dropTable(session); } + + public void truncate(CassandraSessionPool.Session session) { + LOG.debug("Truncate table: {}", this.table()); + session.execute(QueryBuilder.truncate(this.table())); + } } diff --git a/hugegraph-core/pom.xml b/hugegraph-core/pom.xml index 975bfee81f..120b93af2e 100644 --- a/hugegraph-core/pom.xml +++ b/hugegraph-core/pom.xml @@ -19,7 +19,7 @@ com.baidu.hugegraph hugegraph-common - 1.4.9 + 1.5.1 diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java index cefe6ca613..b88e9daf4c 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java @@ -208,6 +208,11 @@ public void clearBackend() { } } + public void truncateBackend() { + this.storeProvider.truncate(); + new BackendStoreInfo(this).init(); + } + private SchemaTransaction openSchemaTransaction() throws HugeException { try { return new CachedSchemaTransaction(this, this.loadSchemaStore()); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/LocalCounter.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/LocalCounter.java index 074a1c61e6..49b1f2dc71 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/LocalCounter.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/LocalCounter.java @@ -46,4 +46,8 @@ public Id nextId(HugeType type) { } return IdGenerator.of(counter.incrementAndGet()); } + + public void reset() { + this.counters.clear(); + } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedBackendStore.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedBackendStore.java index b875027aa2..a56b687706 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedBackendStore.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedBackendStore.java @@ -81,6 +81,11 @@ public void clear() { this.store.clear(); } + @Override + public void truncate() { + this.store.truncate(); + } + @Override public void beginTx() { this.store.beginTx(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java index 00289d6f96..5a4431f3da 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; @@ -33,16 +34,20 @@ import com.baidu.hugegraph.config.CoreOptions; import com.baidu.hugegraph.config.HugeConfig; import com.baidu.hugegraph.event.EventHub; +import com.baidu.hugegraph.event.EventListener; import com.baidu.hugegraph.schema.SchemaElement; import com.baidu.hugegraph.type.HugeType; import com.baidu.hugegraph.util.Events; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; public class CachedSchemaTransaction extends SchemaTransaction { private final Cache idCache; private final Cache nameCache; + private EventListener storeEventListener; + private EventListener cacheEventListener; + private final Map cachedTypes; public CachedSchemaTransaction(HugeGraph graph, BackendStore store) { @@ -56,6 +61,12 @@ public CachedSchemaTransaction(HugeGraph graph, BackendStore store) { this.listenChanges(); } + @Override + public void close() { + super.close(); + this.unlistenChanges(); + } + private Cache cache(String prefix) { HugeConfig conf = super.graph().configuration(); @@ -66,47 +77,61 @@ private Cache cache(String prefix) { } private void listenChanges() { - // Listen store event: "store.init", "store.clear" - List events = ImmutableList.of(Events.STORE_INIT, - Events.STORE_CLEAR); - super.store().provider().listen(event -> { - if (events.contains(event.name())) { - LOG.info("Clear cache on event '{}'", event.name()); + // Listen store event: "store.init", "store.clear", ... + Set storeEvents = ImmutableSet.of(Events.STORE_INIT, + Events.STORE_CLEAR, + Events.STORE_TRUNCATE); + this.storeEventListener = event -> { + if (storeEvents.contains(event.name())) { + LOG.debug("Graph {} clear cache on event '{}'", + this.graph(), event.name()); this.idCache.clear(); this.nameCache.clear(); this.cachedTypes.clear(); return true; } return false; - }); + }; + this.store().provider().listen(this.storeEventListener); // Listen cache event: "cache"(invalid cache item) - EventHub schemaEventHub = super.graph().schemaEventHub(); - if (!schemaEventHub.containsListener(Events.CACHE)) { - schemaEventHub.listen(Events.CACHE, event -> { - LOG.debug("Received event: {}", event); - event.checkArgs(String.class, Id.class); - Object[] args = event.args(); - if (args[0].equals("invalid")) { - Id id = (Id) args[1]; - Object value = this.idCache.get(id); - if (value != null) { - // Invalidate id cache - this.idCache.invalidate(id); - - // Invalidate name cache - SchemaElement schema = (SchemaElement) value; - Id prefixedName = generateId(schema.type(), - schema.name()); - this.nameCache.invalidate(prefixedName); - } - return true; + this.cacheEventListener = event -> { + LOG.debug("Graph {} received cache event: {}", + this.graph(), event); + event.checkArgs(String.class, Id.class); + Object[] args = event.args(); + if (args[0].equals("invalid")) { + Id id = (Id) args[1]; + Object value = this.idCache.get(id); + if (value != null) { + // Invalidate id cache + this.idCache.invalidate(id); + + // Invalidate name cache + SchemaElement schema = (SchemaElement) value; + Id prefixedName = generateId(schema.type(), + schema.name()); + this.nameCache.invalidate(prefixedName); } - return false; - }); + return true; + } + return false; + }; + EventHub schemaEventHub = this.graph().schemaEventHub(); + if (!schemaEventHub.containsListener(Events.CACHE)) { + schemaEventHub.listen(Events.CACHE, this.cacheEventListener); } } + private void unlistenChanges() { + // Unlisten store event + this.store().provider().unlisten(this.storeEventListener); + + // Unlisten cache event + EventHub schemaEventHub = this.graph().schemaEventHub(); + schemaEventHub.unlisten(Events.CACHE, this.cacheEventListener); + } + private void resetCachedAllIfReachedCapacity() { if (this.idCache.size() >= this.idCache.capacity()) { LOG.warn("Schema cache reached capacity({}): {}", diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java index 1ae95cc02b..fb828d6929 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/BinarySerializer.java @@ -536,6 +536,7 @@ private BinaryBackendEntry formatILDeletion(HugeIndex index) { case SEARCH_INDEX: String idString = id.asString(); int idLength = idString.length(); + // TODO: to improve, use BytesBuffer to generate a mask for (int i = idLength - 1; i < 128; i++) { BytesBuffer buffer = BytesBuffer.allocate(idLength + 1); /* diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java index e717daadd8..ab4dc82ea9 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java @@ -21,6 +21,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; import org.slf4j.Logger; @@ -38,11 +39,20 @@ public abstract class AbstractBackendStoreProvider private String graph = null; + private EventHub storeEventHub = new EventHub("store"); + protected Map stores = null; - protected EventHub storeEventHub = new EventHub("store"); + protected final void notifyAndWaitEvent(String event) { + Future future = this.storeEventHub.notify(event, this); + try { + future.get(); + } catch (Throwable e) { + LOG.warn("Error when waiting for event execution: {}", event, e); + } + } - protected void checkOpened() { + protected final void checkOpened() { E.checkState(this.graph != null && this.stores != null, "The BackendStoreProvider has not been opened"); } @@ -56,6 +66,11 @@ public void listen(EventListener listener) { this.storeEventHub.listen(EventHub.ANY_EVENT, listener); } + @Override + public void unlisten(EventListener listener) { + this.storeEventHub.unlisten(EventHub.ANY_EVENT, listener); + } + @Override public String graph() { this.checkOpened(); @@ -85,7 +100,9 @@ public void init() { for (BackendStore store : this.stores.values()) { store.init(); } - this.storeEventHub.notify(Events.STORE_INIT, this); + this.notifyAndWaitEvent(Events.STORE_INIT); + + LOG.info("Graph '{}' has been initialized", this.graph); } @Override @@ -94,7 +111,20 @@ public void clear() throws BackendException { for (BackendStore store : this.stores.values()) { store.clear(); } - this.storeEventHub.notify(Events.STORE_CLEAR, this); + this.notifyAndWaitEvent(Events.STORE_CLEAR); + + LOG.info("Graph '{}' has been cleared", this.graph); + } + + @Override + public void truncate() { + this.checkOpened(); + for (BackendStore store : this.stores.values()) { + store.truncate(); + } + this.notifyAndWaitEvent(Events.STORE_TRUNCATE); + + LOG.info("Graph '{}' has been truncated", this.graph); } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStore.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStore.java index e5e47369af..f9c9d53c99 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStore.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStore.java @@ -45,6 +45,9 @@ public interface BackendStore { public void init(); public void clear(); + // Delete all data of database (keep table structure) + public void truncate(); + // Add/delete data public void mutate(BackendMutation mutation); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java index 52aba0bfbc..c1b6cb1a1a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java @@ -46,5 +46,9 @@ public interface BackendStoreProvider { public void clear(); + public void truncate(); + public void listen(EventListener listener); + + public void unlisten(EventListener listener); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/memory/InMemoryDBStore.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/memory/InMemoryDBStore.java index 87efa43033..ae235cf7c3 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/memory/InMemoryDBStore.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/memory/InMemoryDBStore.java @@ -19,6 +19,7 @@ package com.baidu.hugegraph.backend.store.memory; +import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -83,6 +84,10 @@ protected void registerTableManager(HugeType type, InMemoryDBTable table) { this.tables.put(type, table); } + protected Collection tables() { + return this.tables.values(); + } + protected final InMemoryDBTable table(HugeType type) { assert type != null; InMemoryDBTable table = this.tables.get(type); @@ -153,28 +158,41 @@ public BackendStoreProvider provider() { @Override public void open(HugeConfig config) { - LOG.debug("open()"); + LOG.debug("Store opened: {}", this.store); } @Override public void close() throws BackendException { - LOG.debug("close()"); + LOG.debug("Store closed: {}", this.store); } @Override public void init() { - LOG.info("init()"); - for (InMemoryDBTable table : this.tables.values()) { + for (InMemoryDBTable table : this.tables()) { table.init(null); } + + LOG.debug("Store initialized: {}", this.store); } @Override public void clear() { - LOG.info("clear()"); - for (InMemoryDBTable table : this.tables.values()) { + for (InMemoryDBTable table : this.tables()) { + table.clear(null); + } + this.counter.reset(); + + LOG.debug("Store cleared: {}", this.store); + } + + @Override + public void truncate() { + for (InMemoryDBTable table : this.tables()) { table.clear(null); } + this.counter.reset(); + + LOG.debug("Store truncated: {}", this.store); } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/memory/InMemoryDBTables.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/memory/InMemoryDBTables.java index 37df581e93..5ebb9ca02e 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/memory/InMemoryDBTables.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/memory/InMemoryDBTables.java @@ -345,16 +345,21 @@ private Iterator betweenQuery(Id indexLabelId, Object keyMin, boolean keyMinEq) { NavigableMap rs = this.store(); - Map results = new HashMap<>(); Id min = HugeIndex.formatIndexId(HugeType.RANGE_INDEX, indexLabelId, keyMin); Id max = HugeIndex.formatIndexId(HugeType.RANGE_INDEX, indexLabelId, keyMax); + + max = keyMaxEq ? rs.floorKey(max) : rs.lowerKey(max); + if (max == null) { + return Collections.emptyIterator(); + } + + Map results = new HashMap<>(); Map.Entry entry = keyMinEq ? rs.ceilingEntry(min) : rs.higherEntry(min); - max = keyMaxEq ? rs.floorKey(max) : rs.lowerKey(max); while (entry != null) { if (entry.getKey().compareTo(max) > 0) { break; diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java index 71987baefe..d4767b6941 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphTransaction.java @@ -676,7 +676,8 @@ public void removeVertexProperty(HugeVertexProperty prop) { return; } // Check is updating property of added/removed vertex - E.checkArgument(!this.addedVertexes.containsKey(vertex.id()), + E.checkArgument(!this.addedVertexes.containsKey(vertex.id()) || + this.updatedVertexes.containsKey(vertex.id()), "Can't remove property '%s' for adding-state vertex", prop.key()); E.checkArgument(!this.removedVertexes.containsKey(vertex.id()), diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/SchemaTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/SchemaTransaction.java index 1245654ac5..5436eba40f 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/SchemaTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/SchemaTransaction.java @@ -446,6 +446,7 @@ protected void removeIndexLabelFromBaseLabel(IndexLabel label) { assert baseType == HugeType.EDGE_LABEL; schemaLabel = this.getEdgeLabel(baseValue); } + assert schemaLabel != null; schemaLabel.removeIndexLabel(label.id()); this.updateSchema(schemaLabel); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/HugeTaskScheduler.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/HugeTaskScheduler.java index 7e74316e16..0df9ca928a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/task/HugeTaskScheduler.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/task/HugeTaskScheduler.java @@ -23,6 +23,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -55,6 +56,7 @@ import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.Events; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; public class HugeTaskScheduler { @@ -105,9 +107,11 @@ private TaskTransaction tx() { } private void listenChanges() { - // Listen store event: "store.init" + // Listen store event: "store.init", "store.truncate" + Set storeEvents = ImmutableSet.of(Events.STORE_INIT, + Events.STORE_TRUNCATE); this.graph.loadSystemStore().provider().listen(event -> { - if (Events.STORE_INIT.equals(event.name())) { + if (storeEvents.contains(event.name())) { this.submit(() -> this.tx().initSchema()); return true; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/Events.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/Events.java index 4609da83d3..0e594e5fef 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/Events.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/Events.java @@ -27,4 +27,5 @@ public final class Events { public static final String STORE_CLOSE = "store.close"; public static final String STORE_INIT = "store.init"; public static final String STORE_CLEAR = "store.clear"; + public static final String STORE_TRUNCATE = "store.truncate"; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/version/CoreVersion.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/version/CoreVersion.java index 6a1972b322..11105d183a 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/version/CoreVersion.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/version/CoreVersion.java @@ -38,7 +38,7 @@ public class CoreVersion { public static void check() { // Check version of hugegraph-common - VersionUtil.check(CommonVersion.VERSION, "1.4.0", "1.5", + VersionUtil.check(CommonVersion.VERSION, "1.5", "1.6", CommonVersion.NAME); } } diff --git a/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseSessions.java b/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseSessions.java index 1cbd884cf0..3f2ff14962 100644 --- a/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseSessions.java +++ b/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseSessions.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -172,6 +173,19 @@ public void dropTable(String table) throws IOException { } } + public Future truncateTable(String table) throws IOException { + assert this.existsTable(table); + TableName tableName = TableName.valueOf(this.namespace, table); + try(Admin admin = this.hbase.getAdmin()) { + try { + admin.disableTable(tableName); + } catch (TableNotEnabledException ignored) { + // pass + } + return admin.truncateTableAsync(tableName, false); + } + } + public long storeSize(String table) throws IOException { long total = 0; try(Admin admin = this.hbase.getAdmin()) { diff --git a/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseStore.java b/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseStore.java index 58c3b53b6c..45ed3b9839 100644 --- a/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseStore.java +++ b/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseStore.java @@ -216,6 +216,8 @@ public void init() { e, table, this.store); } } + + LOG.debug("Store initialized: {}", this.store); } @Override @@ -258,6 +260,26 @@ public void clear() { e, this.namespace, this.store); } } + + LOG.debug("Store cleared: {}", this.store); + } + + @Override + public void truncate() { + this.checkOpened(); + + // Truncate tables + for (String table : this.tableNames()) { + try { + this.sessions.truncateTable(table); + } catch (IOException e) { + throw new BackendException( + "Failed to truncate table '%s' for '%s'", + e, table, this.store); + } + } + + LOG.debug("Store truncated: {}", this.store); } @Override diff --git a/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseTables.java b/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseTables.java index 022e027377..a2ce9dfb9b 100644 --- a/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseTables.java +++ b/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseTables.java @@ -156,6 +156,12 @@ public IndexTable(String table) { super(table); } + @Override + public void eliminate(Session session, BackendEntry entry) { + assert entry.columns().size() == 1; + super.delete(session, entry); + } + @Override public void delete(Session session, BackendEntry entry) { /* @@ -164,6 +170,7 @@ public void delete(Session session, BackendEntry entry) { */ int count = 0; for (BackendColumn column : entry.columns()) { + session.commit(); // Prefix query index label related indexes RowIterator iter = session.scan(this.table(), column.name); while (iter.hasNext()) { @@ -190,7 +197,7 @@ public SecondaryIndex(String store) { } } - public static class SearchIndex extends SecondaryIndex { + public static class SearchIndex extends IndexTable { public static final String TABLE = "fi"; diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java index 8ea6d6250d..94a520c86c 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java @@ -293,6 +293,10 @@ public void begin() throws SQLException { this.conn.setAutoCommit(false); } + public void end() throws SQLException { + this.conn.setAutoCommit(true); + } + @Override public Integer commit() { int updated = 0; @@ -304,6 +308,10 @@ public Integer commit() { this.clear(); } catch (SQLException e) { throw new BackendException("Failed to commit", e); + } finally { + try { + this.end(); + } catch (SQLException ignored) {} } return updated; } @@ -314,6 +322,10 @@ public void rollback() { this.conn.rollback(); } catch (SQLException e) { throw new BackendException("Failed to rollback", e); + } finally { + try { + this.end(); + } catch (SQLException ignored) {} } } @@ -323,6 +335,7 @@ public boolean hasChanges() { } public ResultSet select(String sql) throws SQLException { + assert this.conn.getAutoCommit(); return this.conn.createStatement().executeQuery(sql); } diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStore.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStore.java index 4970cd6470..0a6b1dbc39 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStore.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStore.java @@ -20,7 +20,10 @@ package com.baidu.hugegraph.backend.store.mysql; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -77,10 +80,6 @@ protected MysqlSessions openSessionPool(HugeConfig config) { return new MysqlSessions(config, this.database, this.store); } - public Map tables() { - return this.tables; - } - @Override public String store() { return this.store; @@ -157,7 +156,7 @@ public void init() { this.checkSessionConnected(); this.initTables(); - LOG.info("Store initialized: {}", this.store); + LOG.debug("Store initialized: {}", this.store); } @Override @@ -171,7 +170,15 @@ public void clear() { this.sessions.dropDatabase(); } - LOG.info("Store cleared: {}", this.store); + LOG.debug("Store cleared: {}", this.store); + } + + @Override + public void truncate() { + this.checkSessionConnected(); + + this.truncateTables(); + LOG.debug("Store truncated: {}", this.store); } @Override @@ -266,18 +273,29 @@ public String toString() { protected void initTables() { MysqlSessions.Session session = this.sessions.session(); - for (MysqlTable table : this.tables.values()) { + for (MysqlTable table : this.tables()) { table.init(session); } } protected void clearTables() { MysqlSessions.Session session = this.sessions.session(); - for (MysqlTable table : this.tables.values()) { + for (MysqlTable table : this.tables()) { table.clear(session); } } + protected void truncateTables() { + MysqlSessions.Session session = this.sessions.session(); + for (MysqlTable table : this.tables()) { + table.truncate(session); + } + } + + protected Collection tables() { + return this.tables.values(); + } + protected final MysqlTable table(HugeType type) { assert type != null; MysqlTable table = this.tables.get(type); @@ -326,19 +344,10 @@ public MysqlSchemaStore(BackendStoreProvider provider, } @Override - protected void initTables() { - super.initTables(); - - MysqlSessions.Session session = super.sessions.session(); - this.counters.init(session); - } - - @Override - protected void clearTables() { - super.clearTables(); - - MysqlSessions.Session session = super.sessions.session(); - this.counters.clear(session); + protected Collection tables() { + List tables = new ArrayList<>(super.tables()); + tables.add(this.counters); + return tables; } @Override diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStoreProvider.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStoreProvider.java index 9a54e156db..ad1c4f9026 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStoreProvider.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStoreProvider.java @@ -55,7 +55,7 @@ public void clear() throws BackendException { if (itor.hasNext()) { itor.next().clear(); } - this.storeEventHub.notify(Events.STORE_CLEAR, this); + this.notifyAndWaitEvent(Events.STORE_CLEAR); } @Override diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java index 723e5ba30c..b571db93e4 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java @@ -66,8 +66,17 @@ public void init(MysqlSessions.Session session) { this.createTable(session, this.tableDefine()); } - public void createTable(MysqlSessions.Session session, - TableDefine tableDefine) { + @Override + public void clear(MysqlSessions.Session session) { + this.dropTable(session); + } + + public void truncate(MysqlSessions.Session session) { + this.truncateTable(session); + } + + protected void createTable(MysqlSessions.Session session, + TableDefine tableDefine) { StringBuilder sql = new StringBuilder(); sql.append("CREATE TABLE IF NOT EXISTS "); sql.append(this.table()).append(" ("); @@ -100,18 +109,28 @@ public void createTable(MysqlSessions.Session session, } } - public void dropTable(MysqlSessions.Session session) { + protected void dropTable(MysqlSessions.Session session) { LOG.debug("Drop table: {}", this.table()); - StringBuilder sql = new StringBuilder(); - sql.append("DROP TABLE IF EXISTS ").append(this.table()).append(";"); + String sql = String.format("DROP TABLE IF EXISTS %s;", this.table()); try { - session.execute(sql.toString()); + session.execute(sql); } catch (SQLException e) { throw new BackendException("Failed to drop table with '%s'", e, sql); } } + protected void truncateTable(MysqlSessions.Session session) { + LOG.debug("Truncate table: {}", this.table()); + String sql = String.format("TRUNCATE TABLE %s;", this.table()); + try { + session.execute(sql); + } catch (SQLException e) { + throw new BackendException("Failed to truncate table with '%s'", + e, sql); + } + } + protected List idColumnName() { return this.tableDefine().keys(); } @@ -528,10 +547,6 @@ protected void appendPartition(StringBuilder delete) { // pass } - public void clear(MysqlSessions.Session session) { - this.dropTable(session); - } - public static String formatKey(HugeKeys key) { return key.name(); } diff --git a/hugegraph-palo/src/main/java/com/baidu/hugegraph/backend/store/palo/PaloStore.java b/hugegraph-palo/src/main/java/com/baidu/hugegraph/backend/store/palo/PaloStore.java index 1bee5ec503..04bde0acb3 100644 --- a/hugegraph-palo/src/main/java/com/baidu/hugegraph/backend/store/palo/PaloStore.java +++ b/hugegraph-palo/src/main/java/com/baidu/hugegraph/backend/store/palo/PaloStore.java @@ -47,7 +47,7 @@ protected PaloSessions openSessionPool(HugeConfig config) { } private List tableNames() { - return this.tables().values().stream().map(BackendTable::table) + return this.tables().stream().map(BackendTable::table) .collect(Collectors.toList()); } } diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStore.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStore.java index 6056831a94..63178bb479 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStore.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStore.java @@ -276,7 +276,7 @@ public void init() { this.createTable(db, table); } - LOG.info("Store initialized: {}", this.store); + LOG.debug("Store initialized: {}", this.store); } private void createTable(RocksDBSessions db, String table) { @@ -303,7 +303,7 @@ public void clear() { this.dropTable(db, table); } - LOG.info("Store cleared: {}", this.store); + LOG.debug("Store cleared: {}", this.store); } private void dropTable(RocksDBSessions db, String table) { @@ -320,6 +320,16 @@ private void dropTable(RocksDBSessions db, String table) { } } + @Override + public void truncate() { + this.checkOpened(); + + this.clear(); + this.init(); + + LOG.debug("Store truncated: {}", this.store); + } + @Override public void beginTx() { // pass diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTables.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTables.java index 610dd61336..17de8539e9 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTables.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTables.java @@ -152,16 +152,16 @@ public static Edge in(String database) { } } - public static class SecondaryIndex extends RocksDBTable { + public static class IndexTable extends RocksDBTable { - public static final String TABLE = "si"; - - public SecondaryIndex(String database) { - this(database, TABLE); + public IndexTable(String database, String table) { + super(database, table); } - protected SecondaryIndex(String database, String table) { - super(database, table); + @Override + public void eliminate(Session session, BackendEntry entry) { + assert entry.columns().size() == 1; + super.delete(session, entry); } @Override @@ -177,7 +177,16 @@ public void delete(Session session, BackendEntry entry) { } } - public static class SearchIndex extends SecondaryIndex { + public static class SecondaryIndex extends IndexTable { + + public static final String TABLE = "si"; + + public SecondaryIndex(String database) { + super(database, TABLE); + } + } + + public static class SearchIndex extends IndexTable { public static final String TABLE = "fi"; @@ -186,7 +195,7 @@ public SearchIndex(String database) { } } - public static class RangeIndex extends RocksDBTable { + public static class RangeIndex extends IndexTable { public static final String TABLE = "ri"; @@ -247,16 +256,5 @@ protected BackendColumnIterator queryByCond(Session session, return session.scan(this.table(), begin, end, type); } } - - @Override - public void delete(Session session, BackendEntry entry) { - /* - * Only delete index by label will come here - * Regular index delete will call eliminate() - */ - for (BackendEntry.BackendColumn column : entry.columns()) { - session.delete(this.table(), column.name); - } - } } } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/tinkerpop/TestGraph.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/tinkerpop/TestGraph.java index 7bfdfe467a..9972e3b4f9 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/tinkerpop/TestGraph.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/tinkerpop/TestGraph.java @@ -35,6 +35,7 @@ import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.io.HugeGraphIoRegistry; +import com.baidu.hugegraph.schema.PropertyKey; import com.baidu.hugegraph.schema.SchemaManager; import com.baidu.hugegraph.structure.HugeFeatures; import com.baidu.hugegraph.type.define.IdStrategy; @@ -47,6 +48,9 @@ public class TestGraph implements Graph { public static final String DEFAULT_VL = "vertex"; + public static final List TRUNCATE_BACKENDS = + Arrays.asList("rocksdb", "mysql"); + private static volatile int id = 666; private HugeGraph graph; @@ -74,6 +78,33 @@ protected void clearBackend() { this.initedBackend = false; } + protected void clearAll(String testClass) { + List pks = this.graph.schema().getPropertyKeys(); + if (pks.isEmpty()) { + // No need to clear if there is no PKs(that's no schema and data) + return; + } + + if (TRUNCATE_BACKENDS.contains(this.graph.backend())) { + // Delete all data by truncating tables + this.truncateBackend(); + } else { + // Clear schema (also include data) + this.clearSchema(); + + // Clear variables if needed (would not clear when clearing schema) + if (testClass.endsWith("VariableAsMapTest")) { + this.clearVariables(); + } + + this.tx().commit(); + } + } + + protected void truncateBackend() { + this.graph.truncateBackend(); + } + protected void clearSchema() { // Clear schema and graph data will be cleared at same time SchemaManager schema = this.graph.schema(); diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/tinkerpop/TestGraphProvider.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/tinkerpop/TestGraphProvider.java index 99595e3e14..7045519019 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/tinkerpop/TestGraphProvider.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/tinkerpop/TestGraphProvider.java @@ -63,15 +63,18 @@ public class TestGraphProvider extends AbstractGraphProvider { private static final String CONF_PATH = "hugegraph.properties"; private static final String FILTER = "test.tinkerpop.filter"; private static final String DEFAULT_FILTER = "methods.filter"; + private static final String TEST_CLASS = "testClass"; private static final String TEST_METHOD = "testMethod"; + private static final String LOAD_GRAPH = "loadGraph"; - private static final String AKEY = "aKey"; private static final String STANDARD = "standard"; private static final String REGULAR_LOAD = "regularLoad"; + private static final String GREMLIN_GRAPH_KEY = "gremlin.graph"; private static final String GREMLIN_GRAPH_VALUE = "com.baidu.hugegraph.tinkerpop.TestGraphFactory"; + private static final String AKEY_CLASS_PREFIX = "org.apache.tinkerpop.gremlin.structure." + "PropertyTest.PropertyFeatureSupportTest"; @@ -160,10 +163,10 @@ public PropertiesConfiguration getConf() { @Override public Map getBaseConfiguration( String graphName, - Class test, String testMethod, + Class testClass, String testMethod, LoadGraphWith.GraphData graphData) { // Check if test in blackList - String testFullName = test.getCanonicalName() + "." + testMethod; + String testFullName = testClass.getCanonicalName() + "." + testMethod; int index = testFullName.indexOf('@') == -1 ? testFullName.length() : testFullName.indexOf('@'); @@ -187,7 +190,7 @@ public Map getBaseConfiguration( confMap.put(CoreOptions.STORE.name(), storePrefix + "_" + this.suite + "_" + graphName); confMap.put(GREMLIN_GRAPH_KEY, GREMLIN_GRAPH_VALUE); - confMap.put(TEST_CLASS, test); + confMap.put(TEST_CLASS, testClass); confMap.put(TEST_METHOD, testMethod); confMap.put(LOAD_GRAPH, graphData); @@ -223,34 +226,30 @@ public Graph openTestGraph(final Configuration config) { } TestGraph testGraph = this.graphs.get(graphName); + // Ensure tx clean testGraph.tx().rollback(); // Define property key 'aKey' based on specified type in test name - String aKeyType = getAKeyType( - (Class) config.getProperty(TEST_CLASS), - config.getString(TEST_METHOD)); + Class testClass = (Class) config.getProperty(TEST_CLASS); + String testMethod = config.getString(TEST_METHOD); + String aKeyType = getAKeyType(testClass, testMethod); if (aKeyType != null) { - testGraph.initPropertyKey(AKEY, aKeyType); + testGraph.initPropertyKey("aKey", aKeyType); } - String ioType = getIoType((Class) config.getProperty(TEST_CLASS), - config.getString(TEST_METHOD)); - // Basic schema is initiated by default once a graph is open - testGraph.initBasicSchema(IdStrategy.AUTOMATIC, - TestGraph.DEFAULT_VL); + testGraph.initBasicSchema(IdStrategy.AUTOMATIC, TestGraph.DEFAULT_VL); testGraph.tx().commit(); testGraph.isLastIdCustomized(false); - testGraph.loadedGraph(ioType); + testGraph.loadedGraph(getIoType(testClass, testMethod)); testGraph.autoPerson(false); - testGraph.ioTest(ioTest((Class) config.getProperty(TEST_CLASS))); + testGraph.ioTest(ioTest(testClass)); Object loadGraph = config.getProperty(LOAD_GRAPH); if (loadGraph != null && !graphName.endsWith(STANDARD)) { - this.loadGraphData(testGraph, - (LoadGraphWith.GraphData) loadGraph); + this.loadGraphData(testGraph, (LoadGraphWith.GraphData) loadGraph); } return testGraph; @@ -273,12 +272,9 @@ public void clear(Graph graph, Configuration config) throws Exception { graph.tx().onReadWrite(Transaction.READ_WRITE_BEHAVIOR.AUTO); graph.tx().onClose(Transaction.CLOSE_BEHAVIOR.ROLLBACK); - // Clear schema (also include data) - testGraph.clearSchema(); - - // Clear variables (would not clear when clearing schema) - testGraph.clearVariables(); - graph.tx().commit(); + // Clear all data + Class testClass = (Class) config.getProperty(TEST_CLASS); + testGraph.clearAll(testClass.getCanonicalName()); } public void clearBackends() { @@ -308,8 +304,7 @@ public void loadGraphData(final Graph graph, TestGraph testGraph = (TestGraph) graph; // Clear basic schema initiated in openTestGraph - testGraph.clearSchema(); - testGraph.tx().commit(); + testGraph.clearAll(""); if (testGraph.loadedGraph() == null) { testGraph.loadedGraph(REGULAR_LOAD); diff --git a/hugegraph-test/src/main/resources/hugegraph.properties b/hugegraph-test/src/main/resources/hugegraph.properties index 2e899ba208..e84d6c0191 100644 --- a/hugegraph-test/src/main/resources/hugegraph.properties +++ b/hugegraph-test/src/main/resources/hugegraph.properties @@ -1,7 +1,10 @@ gremlin.graph=com.baidu.hugegraph.HugeFactory -backend=${backend} -serializer=${serializer} +backend=rocksdb +serializer=binary + +#backend=cassandra +#serializer=cassandra store=hugegraph @@ -21,9 +24,13 @@ cassandra.read_timeout=120 #rocksdb.wal_path= # hbase backend config -hbase.hosts=localhost -hbase.port=8218 +#backend=hbase +#serializer=hbase +hbase.hosts=127.0.0.1 +hbase.port=2181 +#backend=mysql +#serializer=mysql # mysql backend config jdbc.driver=com.mysql.jdbc.Driver jdbc.url=jdbc:mysql://127.0.0.1:3306