Improve schema deletion by paging for large data

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/IdHolderList.java

@@ -26,6 +26,8 @@
 
 public final class IdHolderList extends ArrayList<IdHolder> {
 
+    private static final long serialVersionUID = -738694176552424990L;
+
     private final boolean paging;
 
     public IdHolderList(boolean paging) {

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/PageEntryIterator.java

@@ -22,6 +22,7 @@
 import java.util.Iterator;
 import java.util.NoSuchElementException;
 
+import com.baidu.hugegraph.backend.query.Query;
 import com.baidu.hugegraph.backend.store.BackendEntry;
 import com.baidu.hugegraph.exception.NotSupportException;
 import com.baidu.hugegraph.iterator.Metadatable;
@@ -44,7 +45,7 @@ public PageEntryIterator(QueryList queries, long pageSize) {
     }
 
     private PageState parsePageState() {
-        String page = this.queries.parent().page();
+        String page = this.queries.parent().pageWithoutCheck();
         PageState pageState = PageState.fromString(page);
         E.checkState(pageState.offset() < this.queries.total(),
                      "Invalid page '%s' with an offset '%s' exceeds " +
@@ -61,13 +62,15 @@ public boolean hasNext() {
     }
 
     private boolean fetch() {
-        if (this.remaining <= 0 ||
+        if ((this.remaining != Query.NO_LIMIT && this.remaining <= 0L) ||
             this.pageState.offset() >= this.queries.total()) {
             return false;
         }
 
-        long pageSize = this.remaining < this.pageSize ?
-                        this.remaining : this.pageSize;
+        long pageSize = this.pageSize;
+        if (this.remaining != Query.NO_LIMIT && this.remaining < pageSize) {
+            pageSize = this.remaining;
+        }
         this.results = this.queries.fetchNext(this.pageState, pageSize);
         assert this.results != null;
 
@@ -90,7 +93,10 @@ public BackendEntry next() {
             throw new NoSuchElementException();
         }
         BackendEntry entry = this.results.iterator().next();
-        this.remaining--;
+        if (this.remaining != Query.NO_LIMIT) {
+            // Assume one result in each entry (just for index query)
+            this.remaining--;
+        }
         return entry;
     }
 

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/QueryList.java

@@ -90,7 +90,7 @@ public boolean empty() {
     }
 
     public Iterator<BackendEntry> fetch() {
-        assert !queries.isEmpty();
+        assert !this.queries.isEmpty();
         if (this.parent.paging()) {
             int pageSize = this.graph.configuration()
                                      .get(CoreOptions.QUERY_PAGE_SIZE);
@@ -122,6 +122,7 @@ protected PageIterator fetchNext(PageState pageState, long pageSize) {
         return query.iterator(offset - current, pageState.page(), pageSize);
     }
 
+    @SuppressWarnings("unused")
     private static Set<Id> limit(Set<Id> ids, Query query) {
         long fromIndex = query.offset();
         E.checkArgument(fromIndex <= Integer.MAX_VALUE,
@@ -167,7 +168,7 @@ private interface QueryHolder {
     }
 
     /**
-     * Generate queries from index.optimizeQuery()
+     * Generate queries from tx.optimizeQuery()
      */
     private class OptimizedQuery implements QueryHolder {
 
@@ -183,9 +184,14 @@ public Iterator<BackendEntry> iterator() {
 
         public PageIterator iterator(int index, String page, long pageSize) {
             assert index == 0;
-            this.query.page(page);
-            Iterator<BackendEntry> iterator = fetcher.apply(this.query);
-            // Must iterate all entries before getting the next page info
+            Query query = this.query.copy();
+            query.page(page);
+            // Not set limit to pageSize due to PageEntryIterator.remaining
+            if (this.query.limit() == Query.NO_LIMIT) {
+                query.limit(pageSize);
+            }
+            Iterator<BackendEntry> iterator = fetcher.apply(query);
+            // Must iterate all entries before get the next page
             List<BackendEntry> results = IteratorUtils.list(iterator);
             return new PageIterator(results.iterator(),
                                     PageState.page(iterator));
@@ -197,7 +203,7 @@ public int total() {
     }
 
     /**
-     * Generate queries from index.indexQuery()
+     * Generate queries from tx.indexQuery()
      */
     private class IndexQuery implements QueryHolder {
 
@@ -214,10 +220,15 @@ public Iterator<BackendEntry> iterator() {
                     return null;
                 }
                 Set<Id> ids = holder.ids();
-                if (ids.size() > parent.limit()) {
-                    ids = limit(ids, parent);
+                if (parent.limit() != Query.NO_LIMIT &&
+                    ids.size() > parent.limit()) {
+                    /*
+                     * Avoid too many ids in one time query,
+                     * Assume it will get one result by each id
+                     */
+                    ids = CollectionUtil.subSet(ids, 0, (int) parent.limit());
                 }
-                IdQuery query = new IdQuery(parent.resultType(), ids);
+                IdQuery query = new IdQuery(parent, ids);
                 return fetcher.apply(query);
             });
         }
@@ -228,7 +239,7 @@ public PageIterator iterator(int index, String page, long pageSize) {
             if (pageIds.empty()) {
                 return PageIterator.EMPTY;
             }
-            IdQuery query = new IdQuery(parent.resultType(), pageIds.ids());
+            IdQuery query = new IdQuery(parent, pageIds.ids());
             return new PageIterator(fetcher.apply(query), pageIds.page());
         }
 

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Query.java

@@ -22,6 +22,7 @@
 import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 import com.baidu.hugegraph.backend.BackendException;
@@ -31,7 +32,6 @@
 import com.baidu.hugegraph.type.HugeType;
 import com.baidu.hugegraph.type.define.HugeKeys;
 import com.baidu.hugegraph.util.E;
-import com.google.common.base.Objects;
 import com.google.common.collect.ImmutableSet;
 
 public class Query implements Cloneable {
@@ -48,6 +48,7 @@ public class Query implements Cloneable {
     private String page;
     private long capacity;
     private boolean showHidden;
+    private boolean showDeleting;
 
     private Query originQuery;
 
@@ -70,6 +71,7 @@ public Query(HugeType resultType, Query originQuery) {
         this.capacity = defaultCapacity();
 
         this.showHidden = false;
+        this.showDeleting = false;
     }
 
     public HugeType resultType() {
@@ -174,6 +176,10 @@ public String page() {
         return this.page;
     }
 
+    public String pageWithoutCheck() {
+        return this.page;
+    }
+
     public void page(String page) {
         this.page = page;
     }
@@ -212,6 +218,14 @@ public void showHidden(boolean showHidden) {
         this.showHidden = showHidden;
     }
 
+    public boolean showDeleting() {
+        return this.showDeleting;
+    }
+
+    public void showDeleting(boolean showDeleting) {
+        this.showDeleting = showDeleting;
+    }
+
     public Set<Id> ids() {
         return ImmutableSet.of();
     }
@@ -246,8 +260,7 @@ public boolean equals(Object object) {
                this.orders.equals(other.orders) &&
                this.offset == other.offset &&
                this.limit == other.limit &&
-               ((this.page == null && other.page == null) ||
-                this.page.equals(other.page)) &&
+               Objects.equals(this.page, other.page) &&
                this.ids().equals(other.ids()) &&
                this.conditions().equals(other.conditions());
     }

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendEntryIterator.java

@@ -27,6 +27,7 @@
 import com.baidu.hugegraph.exception.LimitExceedException;
 import com.baidu.hugegraph.exception.NotSupportException;
 import com.baidu.hugegraph.iterator.Metadatable;
+import com.baidu.hugegraph.util.E;
 
 public abstract class BackendEntryIterator
                 implements Iterator<BackendEntry>, AutoCloseable, Metadatable {
@@ -38,6 +39,7 @@ public abstract class BackendEntryIterator
     private long count;
 
     public BackendEntryIterator(Query query) {
+        E.checkNotNull(query, "query");
         this.query = query;
         this.count = 0L;
         this.current = null;
@@ -150,4 +152,26 @@ protected long skip(BackendEntry entry, long skip) {
     protected abstract boolean fetch();
 
     protected abstract String pageState();
+
+    public static final class EmptyIterator extends BackendEntryIterator {
+
+        public EmptyIterator(Query query) {
+            super(query);
+        }
+
+        @Override
+        protected boolean fetch() {
+            return false;
+        }
+
+        @Override
+        protected String pageState() {
+            return null;
+        }
+
+        @Override
+        public void close() throws Exception {
+            return;
+        }
+    }
 }

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java

@@ -285,7 +285,7 @@ private List<IdHolder> queryByLabel(ConditionQuery query) {
         indexQuery.eq(HugeKeys.INDEX_LABEL_ID, il.id());
         indexQuery.eq(HugeKeys.FIELD_VALUES, label);
         // Set offset and limit to avoid redundant element ids
-        indexQuery.page(query.page());
+        indexQuery.page(query.pageWithoutCheck());
         indexQuery.limit(query.limit());
         indexQuery.offset(query.offset());
         indexQuery.capacity(query.capacity());