diff --git a/hugegraph-api/pom.xml b/hugegraph-api/pom.xml index 7082c4ad44..98dc09a54e 100644 --- a/hugegraph-api/pom.xml +++ b/hugegraph-api/pom.xml @@ -86,7 +86,7 @@ - 0.35.0.0 + 0.36.0.0 diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java index b786f8268b..1e7c01898c 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java @@ -38,6 +38,7 @@ import com.baidu.hugegraph.api.filter.CompressInterceptor.Compress; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.query.ConditionQuery; +import com.baidu.hugegraph.backend.query.Query; import com.baidu.hugegraph.backend.store.Shard; import com.baidu.hugegraph.core.GraphManager; import com.baidu.hugegraph.server.RestServer; @@ -101,16 +102,19 @@ public String shards(@Context GraphManager manager, public String scan(@Context GraphManager manager, @PathParam("graph") String graph, @QueryParam("start") String start, - @QueryParam("end") String end) { - LOG.debug("Graph [{}] query edges by shard(start: {}, end: {}) ", - graph, start, end); + @QueryParam("end") String end, + @QueryParam("page") String page) { + LOG.debug("Graph [{}] query edges by shard(start: {}, end: {}, " + + "page: {}) ", graph, start, end, page); HugeGraph g = graph(manager, graph); ConditionQuery query = new ConditionQuery(HugeType.EDGE_OUT); query.scan(start, end); + query.limit(Query.DEFAULT_CAPACITY); + query.page(page); Iterator edges = g.edges(query); - return manager.serializer(g).writeEdges(edges, false); + return manager.serializer(g).writeEdges(edges, true); } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java index d002d14327..96952898d2 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java @@ -39,6 +39,7 @@ import com.baidu.hugegraph.api.graph.VertexAPI; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.query.ConditionQuery; +import com.baidu.hugegraph.backend.query.Query; import com.baidu.hugegraph.backend.store.Shard; import com.baidu.hugegraph.core.GraphManager; import com.baidu.hugegraph.server.RestServer; @@ -101,16 +102,19 @@ public String shards(@Context GraphManager manager, public String scan(@Context GraphManager manager, @PathParam("graph") String graph, @QueryParam("start") String start, - @QueryParam("end") String end) { - LOG.debug("Graph [{}] query vertices by shard(start: {}, end: {}) ", - graph, start, end); + @QueryParam("end") String end, + @QueryParam("page") String page) { + LOG.debug("Graph [{}] query vertices by shard(start: {}, end: {}, " + + "page: {}) ", graph, start, end, page); HugeGraph g = graph(manager, graph); ConditionQuery query = new ConditionQuery(HugeType.VERTEX); query.scan(start, end); + query.limit(Query.DEFAULT_CAPACITY); + query.page(page); Iterator vertices = g.vertices(query); - return manager.serializer(g).writeVertices(vertices, false); + return manager.serializer(g).writeVertices(vertices, true); } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/serializer/JsonSerializer.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/serializer/JsonSerializer.java index fa77febad3..4275500697 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/serializer/JsonSerializer.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/serializer/JsonSerializer.java @@ -36,6 +36,7 @@ import com.baidu.hugegraph.api.API; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.store.Shard; +import com.baidu.hugegraph.iterator.Metadatable; import com.baidu.hugegraph.schema.EdgeLabel; import com.baidu.hugegraph.schema.IndexLabel; import com.baidu.hugegraph.schema.PropertyKey; @@ -100,7 +101,16 @@ private String writeIterator(String label, Iterator itor, // Write page if (paging) { - String page = TraversalUtil.page((GraphTraversal) itor); + String page; + if (itor instanceof GraphTraversal) { + page = TraversalUtil.page((GraphTraversal) itor); + } else if (itor instanceof Metadatable) { + page = (String) ((Metadatable) itor).metadata("page"); + } else { + throw new HugeException( + "Error type '%s' of paging iterator '%s'", + itor.getClass(), itor); + } if (page != null) { page = String.format(",\"page\": \"%s\"", page); } else { diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java index 9a627212a3..a0dd5e71c6 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/version/ApiVersion.java @@ -75,14 +75,17 @@ public final class ApiVersion { * [0.29] Issue-39: Add rays and rings RESTful API * [0.30] Issue-32: Change index create API to return indexLabel and task id * [0.31] Issue-182: Support restore graph in restoring and merging mode + * + * version 0.9: * [0.32] Issue-250: Keep depth and degree consistent for traverser api * [0.33] Issue-305: Implement customized paths and crosspoints RESTful API * [0.34] Issue-307: Let VertexAPI use simplified property serializer * [0.35] Issue-287: Support pagination when do index query + * [0.36] Issue-360: Support paging for scan api */ // The second parameter of Version.of() is for IDE running without JAR - public static final Version VERSION = Version.of(ApiVersion.class, "0.35"); + public static final Version VERSION = Version.of(ApiVersion.class, "0.36"); public static final void check() { // Check version of hugegraph-core. Firstly do check from version 0.3 diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java index e8e21942a3..695675a9a6 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedGraphTransaction.java @@ -105,8 +105,8 @@ private Iterator queryVerticesByIds(IdQuery query) { @Override protected Iterator queryEdgesFromBackend(Query query) { - if (query.empty()) { - // Query all edges, don't cache it + if (query.empty() || query.paging()) { + // Query all edges or query edges in paging, don't cache it return super.queryEdgesFromBackend(query); } diff --git a/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseTable.java b/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseTable.java index d8201b82ef..d7adf750b5 100644 --- a/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseTable.java +++ b/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseTable.java @@ -203,14 +203,21 @@ protected RowIterator queryByCond(Session session, ConditionQuery query) { "Invalid scan with multi conditions: %s", query); Relation scan = query.relations().iterator().next(); Shard shard = (Shard) scan.value(); - return this.queryByRange(session, shard); + return this.queryByRange(session, shard, query.page()); } throw new NotSupportException("query: %s", query); } - protected RowIterator queryByRange(Session session, Shard shard) { + protected RowIterator queryByRange(Session session, Shard shard, + String page) { byte[] start = this.shardSpliter.position(shard.start()); byte[] end = this.shardSpliter.position(shard.end()); + if (page != null && !page.isEmpty()) { + byte[] position = PageState.fromString(page).position(); + E.checkArgument(Bytes.compare(position, start) >= 0, + "Invalid page out of lower bound"); + start = position; + } return session.scan(this.table(), start, end); } diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java index 10c8394fe8..5ce44e8c51 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java @@ -42,6 +42,7 @@ import com.baidu.hugegraph.exception.NotSupportException; import com.baidu.hugegraph.iterator.ExtendableIterator; import com.baidu.hugegraph.type.HugeType; +import com.baidu.hugegraph.util.Bytes; import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.Log; import com.google.common.collect.ImmutableList; @@ -193,14 +194,21 @@ protected BackendColumnIterator queryByCond(Session session, "Invalid scan with multi conditions: %s", query); Relation scan = query.relations().iterator().next(); Shard shard = (Shard) scan.value(); - return this.queryByRange(session, shard); + return this.queryByRange(session, shard, query.page()); } throw new NotSupportException("query: %s", query); } - protected BackendColumnIterator queryByRange(Session session, Shard shard) { + protected BackendColumnIterator queryByRange(Session session, Shard shard, + String page) { byte[] start = this.shardSpliter.position(shard.start()); byte[] end = this.shardSpliter.position(shard.end()); + if (page != null && !page.isEmpty()) { + byte[] position = PageState.fromString(page).position(); + E.checkArgument(Bytes.compare(position, start) >= 0, + "Invalid page out of lower bound"); + start = position; + } return session.scan(this.table(), start, end); } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java index 4ba026887f..c2499f4c25 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java @@ -22,6 +22,7 @@ import java.util.Date; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.NoSuchElementException; import java.util.Set; @@ -51,6 +52,7 @@ import com.baidu.hugegraph.config.CoreOptions; import com.baidu.hugegraph.exception.LimitExceedException; import com.baidu.hugegraph.exception.NotFoundException; +import com.baidu.hugegraph.iterator.Metadatable; import com.baidu.hugegraph.schema.SchemaManager; import com.baidu.hugegraph.testutil.Assert; import com.baidu.hugegraph.testutil.FakeObjects.FakeEdge; @@ -1745,6 +1747,32 @@ public void testScanEdge() { Assert.assertEquals(18, edges.size()); } + @Test + public void testScanEdgeInPaging() { + HugeGraph graph = graph(); + Assume.assumeTrue("Not support scan", + storeFeatures().supportsScanToken() || + storeFeatures().supportsScanKeyRange()); + init18Edges(); + + List edges = new LinkedList<>(); + + ConditionQuery query = new ConditionQuery(HugeType.EDGE); + query.scan(String.valueOf(Long.MIN_VALUE), + String.valueOf(Long.MAX_VALUE)); + query.limit(1); + String page = ""; + while (page != null) { + query.page(page); + Iterator iterator = graph.edges(query); + while (iterator.hasNext()) { + edges.add(iterator.next()); + } + page = (String) ((Metadatable) iterator).metadata("page"); + } + Assert.assertEquals(18, edges.size()); + } + @Test public void testRemoveEdge() { HugeGraph graph = graph(); diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java index f25a6a7a14..7712f727fd 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java @@ -52,6 +52,7 @@ import com.baidu.hugegraph.backend.store.Shard; import com.baidu.hugegraph.backend.tx.GraphTransaction; import com.baidu.hugegraph.exception.NoIndexException; +import com.baidu.hugegraph.iterator.Metadatable; import com.baidu.hugegraph.schema.PropertyKey; import com.baidu.hugegraph.schema.SchemaManager; import com.baidu.hugegraph.schema.VertexLabel; @@ -3008,6 +3009,32 @@ public void testScanVertex() { Assert.assertEquals(10, vertexes.size()); } + @Test + public void testScanVertexInPaging() { + HugeGraph graph = graph(); + Assume.assumeTrue("Not support scan", + storeFeatures().supportsScanToken() || + storeFeatures().supportsScanKeyRange()); + init10Vertices(); + + List vertexes = new LinkedList<>(); + + ConditionQuery query = new ConditionQuery(HugeType.VERTEX); + query.scan(String.valueOf(Long.MIN_VALUE), + String.valueOf(Long.MAX_VALUE)); + query.limit(1); + String page = ""; + while (page != null) { + query.page(page); + Iterator iterator = graph.vertices(query); + while (iterator.hasNext()) { + vertexes.add(iterator.next()); + } + page = (String) ((Metadatable) iterator).metadata("page"); + } + Assert.assertEquals(10, vertexes.size()); + } + @Test public void testScanVertexWithSplitSizeLt1MB() { HugeGraph graph = graph();