From e2db3e559d19ed9bc621ebf10dc3c4e3819a7c89 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Thu, 28 Mar 2019 15:35:55 +0800 Subject: [PATCH 1/4] Throw exception if exceeds 80w without paging resolved: #360 Change-Id: I46c7a61df21d6902aba6c0c965e8f63aa3b76fe2 --- .../java/com/baidu/hugegraph/api/traversers/EdgesAPI.java | 7 +++++-- .../com/baidu/hugegraph/api/traversers/VerticesAPI.java | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java index 1e7c01898c..9f1d191d3b 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java @@ -111,10 +111,13 @@ public String scan(@Context GraphManager manager, ConditionQuery query = new ConditionQuery(HugeType.EDGE_OUT); query.scan(start, end); - query.limit(Query.DEFAULT_CAPACITY); query.page(page); + boolean paging = page != null; + if (paging) { + query.limit(Query.DEFAULT_CAPACITY); + } Iterator edges = g.edges(query); - return manager.serializer(g).writeEdges(edges, true); + return manager.serializer(g).writeEdges(edges, paging); } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java index 96952898d2..c57b7a3504 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java @@ -111,10 +111,13 @@ public String scan(@Context GraphManager manager, ConditionQuery query = new ConditionQuery(HugeType.VERTEX); query.scan(start, end); - query.limit(Query.DEFAULT_CAPACITY); query.page(page); + boolean paging = page != null; + if (paging) { + query.limit(Query.DEFAULT_CAPACITY); + } Iterator vertices = g.vertices(query); - return manager.serializer(g).writeVertices(vertices, true); + return manager.serializer(g).writeVertices(vertices, paging); } } From ec462a26c7e3ebb4e15aa8b8eaab6b5fc577da06 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Thu, 28 Mar 2019 17:27:51 +0800 Subject: [PATCH 2/4] tiny improve Change-Id: Ia89e0578306091fc0cd54fd7c64a91b2b0881b9f --- .../java/com/baidu/hugegraph/api/traversers/EdgesAPI.java | 5 ++--- .../java/com/baidu/hugegraph/api/traversers/VerticesAPI.java | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java index 9f1d191d3b..0542d328f0 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java @@ -112,12 +112,11 @@ public String scan(@Context GraphManager manager, ConditionQuery query = new ConditionQuery(HugeType.EDGE_OUT); query.scan(start, end); query.page(page); - boolean paging = page != null; - if (paging) { + if (query.paging()) { query.limit(Query.DEFAULT_CAPACITY); } Iterator edges = g.edges(query); - return manager.serializer(g).writeEdges(edges, paging); + return manager.serializer(g).writeEdges(edges, query.paging()); } } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java index c57b7a3504..b893233c7f 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java @@ -112,12 +112,11 @@ public String scan(@Context GraphManager manager, ConditionQuery query = new ConditionQuery(HugeType.VERTEX); query.scan(start, end); query.page(page); - boolean paging = page != null; - if (paging) { + if (query.paging()) { query.limit(Query.DEFAULT_CAPACITY); } Iterator vertices = g.vertices(query); - return manager.serializer(g).writeVertices(vertices, paging); + return manager.serializer(g).writeVertices(vertices, query.paging()); } } From a0641bd37d3602245c97a7d1718def6cdb3d7436 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Thu, 28 Mar 2019 19:40:20 +0800 Subject: [PATCH 3/4] set scan limit to empirical value 10w to ensure return in 3s Change-Id: I6acd4fc29acfd2aa03567fc612adcaab2bbee593 --- hugegraph-api/src/main/java/com/baidu/hugegraph/api/API.java | 3 +++ .../main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java | 3 +-- .../java/com/baidu/hugegraph/api/traversers/VerticesAPI.java | 3 +-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/API.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/API.java index 3f477f2409..b3293f5760 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/API.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/API.java @@ -48,6 +48,9 @@ public class API { private static final Logger LOG = Log.logger(RestServer.class); + // Empirical value of scan limit, with which results can be returned in 3s + protected static final long SCAN_LIMIT = 100_000L; + public static final String CHARSET = "UTF-8"; public static final String APPLICATION_JSON = MediaType.APPLICATION_JSON; diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java index 0542d328f0..f3f2a560b7 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java @@ -38,7 +38,6 @@ import com.baidu.hugegraph.api.filter.CompressInterceptor.Compress; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.query.ConditionQuery; -import com.baidu.hugegraph.backend.query.Query; import com.baidu.hugegraph.backend.store.Shard; import com.baidu.hugegraph.core.GraphManager; import com.baidu.hugegraph.server.RestServer; @@ -113,7 +112,7 @@ public String scan(@Context GraphManager manager, query.scan(start, end); query.page(page); if (query.paging()) { - query.limit(Query.DEFAULT_CAPACITY); + query.limit(SCAN_LIMIT); } Iterator edges = g.edges(query); diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java index b893233c7f..b986e27c6a 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java @@ -39,7 +39,6 @@ import com.baidu.hugegraph.api.graph.VertexAPI; import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.query.ConditionQuery; -import com.baidu.hugegraph.backend.query.Query; import com.baidu.hugegraph.backend.store.Shard; import com.baidu.hugegraph.core.GraphManager; import com.baidu.hugegraph.server.RestServer; @@ -113,7 +112,7 @@ public String scan(@Context GraphManager manager, query.scan(start, end); query.page(page); if (query.paging()) { - query.limit(Query.DEFAULT_CAPACITY); + query.limit(SCAN_LIMIT); } Iterator vertices = g.vertices(query); From 45f9a99b57e49e74a2d2d142d4ed4cc4b3135b3f Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Thu, 28 Mar 2019 21:26:56 +0800 Subject: [PATCH 4/4] tiny improve Change-Id: I99d5ac692d54266dbff863365245ca8491a00008 --- hugegraph-api/src/main/java/com/baidu/hugegraph/api/API.java | 2 +- .../main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java | 2 +- .../java/com/baidu/hugegraph/api/traversers/VerticesAPI.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/API.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/API.java index b3293f5760..3f9e3c5e62 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/API.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/API.java @@ -49,7 +49,7 @@ public class API { private static final Logger LOG = Log.logger(RestServer.class); // Empirical value of scan limit, with which results can be returned in 3s - protected static final long SCAN_LIMIT = 100_000L; + protected static final long SCAN_PAGE_LIMIT = 100_000L; public static final String CHARSET = "UTF-8"; diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java index f3f2a560b7..fde5aded58 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/EdgesAPI.java @@ -112,7 +112,7 @@ public String scan(@Context GraphManager manager, query.scan(start, end); query.page(page); if (query.paging()) { - query.limit(SCAN_LIMIT); + query.limit(SCAN_PAGE_LIMIT); } Iterator edges = g.edges(query); diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java index b986e27c6a..d419a2b0e1 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/traversers/VerticesAPI.java @@ -112,7 +112,7 @@ public String scan(@Context GraphManager manager, query.scan(start, end); query.page(page); if (query.paging()) { - query.limit(SCAN_LIMIT); + query.limit(SCAN_PAGE_LIMIT); } Iterator vertices = g.vertices(query);