[Optimize](search) Optimize implement the new query interface

src/core/CLucene/index/DocRange.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <stdint.h>
+
+#include <vector>
+
+#include "CLucene/CLConfig.h"
+
+enum class DocRangeType {
+  kMany = 0,
+  kRange = 1,
+
+  kNone
+};
+
+class DocRange {
+ public:
+  DocRange() : doc_many(PFOR_BLOCK_SIZE + 3), freq_many(PFOR_BLOCK_SIZE + 3) {}
+
+ public:
+  DocRangeType type_ = DocRangeType::kNone;
+
+  uint32_t doc_many_size_ = 0;
+  uint32_t freq_many_size_ = 0;
+  std::vector<uint32_t> doc_many;
+  std::vector<uint32_t> freq_many;
+
+  std::pair<uint32_t, uint32_t> doc_range;
+};

src/core/CLucene/index/MultiSegmentReader.cpp

@@ -18,6 +18,8 @@
 #include "MultiReader.h"
 #include "_MultiSegmentReader.h"
 
+#include <algorithm>
+
 CL_NS_USE(document)
 CL_NS_USE(store)
 CL_NS_USE(util)
@@ -612,6 +614,32 @@ int32_t MultiTermDocs::read(int32_t* docs, int32_t* freqs, int32_t length) {
 	}
 }
 
+bool MultiTermDocs::readRange(DocRange* docRange) {
+	while (true) {
+		while (current == NULL) {
+			if (pointer < subReaders->length) {
+				base = starts[pointer];
+				current = termDocs(pointer++);
+			} else {
+				return false;
+			}
+		}
+		if (!current->readRange(docRange)) {
+			current = nullptr;
+		} else {
+			if (docRange->type_ == DocRangeType::kMany) {
+				auto begin = docRange->doc_many.begin();
+				auto end = docRange->doc_many.begin() + docRange->doc_many_size_;
+				std::transform(begin, end, begin, [this](int32_t val) { return val + base; });
+			} else if (docRange->type_ == DocRangeType::kRange) {
+				docRange->doc_range.first += base;
+				docRange->doc_range.second += base;
+			}
+			return true;
+		}
+	}
+}
+
 bool MultiTermDocs::skipTo(const int32_t target) {
 //	do {
 //	  if (!next())

src/core/CLucene/index/MultipleTermPositions.cpp

@@ -26,6 +26,10 @@ int32_t MultipleTermPositions::read(int32_t*, int32_t*,int32_t) {
 	_CLTHROWA(CL_ERR_UnsupportedOperation, "Unsupported operation: MultipleTermPositions::read");
 }
 
+bool MultipleTermPositions::readRange(DocRange* docRange) {
+	_CLTHROWA(CL_ERR_UnsupportedOperation, "Unsupported operation: MultipleTermPositions::readRange");
+}
+
 int32_t MultipleTermPositions::getPayloadLength() const {
 	_CLTHROWA(CL_ERR_UnsupportedOperation, "Unsupported operation: MultipleTermPositions::getPayloadLength");
 }

src/core/CLucene/index/MultipleTermPositions.h

@@ -63,6 +63,7 @@ class CLUCENE_EXPORT MultipleTermPositions : public TermPositions {
 	* @throws UnsupportedOperationException
 	*/
 	int32_t read(int32_t*, int32_t*,int32_t);
+	bool readRange(DocRange* docRange) override;
 
 	/**
 	* Not implemented.

src/core/CLucene/index/SegmentTermDocs.cpp

@@ -238,6 +238,86 @@ int32_t SegmentTermDocs::read(int32_t *docs, int32_t *freqs, int32_t length) {
     return i;
 }
 
+bool SegmentTermDocs::readRange(DocRange* docRange) {
+    if (count >= df) {
+        return false;
+    }
+
+    if (hasProx) {
+        char mode = freqStream->readByte();
+        uint32_t arraySize = freqStream->readVInt();
+        if (mode == (char)CodeMode::kDefault) {
+            uint32_t docDelta = 0;
+            for (uint32_t i = 0; i < arraySize; i++) {
+                uint32_t docCode = freqStream->readVInt();
+                docDelta += (docCode >> 1);
+                docRange->doc_many[i] = docDelta;
+                if ((docCode & 1) != 0) {
+                    docRange->freq_many[i] = 1;
+                } else {
+                    docRange->freq_many[i] = freqStream->readVInt();
+                }
+            }
+            docRange->type_ = DocRangeType::kMany;
+            docRange->doc_many_size_ = arraySize;
+            docRange->freq_many_size_ = arraySize;
+        } else {
+            {
+                uint32_t SerializedSize = freqStream->readVInt();
+                std::vector<uint8_t> buf(SerializedSize + PFOR_BLOCK_SIZE);
+                freqStream->readBytes(buf.data(), SerializedSize);
+                P4DEC(buf.data(), arraySize, docRange->doc_many.data());
+            }
+            {
+                uint32_t SerializedSize = freqStream->readVInt();
+                std::vector<uint8_t> buf(SerializedSize + PFOR_BLOCK_SIZE);
+                freqStream->readBytes(buf.data(), SerializedSize);
+                P4NZDEC(buf.data(), arraySize, docRange->freq_many.data());
+            }
+            docRange->type_ = DocRangeType::kMany;
+            docRange->doc_many_size_ = arraySize;
+            docRange->freq_many_size_ = arraySize;
+        }
+        count += arraySize;
+    } else {
+        uint32_t arraySize = freqStream->readVInt();
+        if (arraySize < PFOR_BLOCK_SIZE) {
+            uint32_t docDelta = 0;
+            for (uint32_t i = 0; i < arraySize; i++) {
+                uint32_t docCode = freqStream->readVInt();
+                docDelta += docCode;
+                docRange->doc_many[i] = docDelta;
+            }
+            docRange->type_ = DocRangeType::kMany;
+            docRange->doc_many_size_ = arraySize;
+        } else {
+            {
+                uint32_t serializedSize = freqStream->readVInt();
+                std::vector<uint8_t> buf(serializedSize + PFOR_BLOCK_SIZE);
+                freqStream->readBytes(buf.data(), serializedSize);
+                P4DEC(buf.data(), arraySize, docRange->doc_many.data());
+            }
+            docRange->type_ = DocRangeType::kMany;
+            docRange->doc_many_size_ = arraySize;
+        }
+        count += arraySize;
+    }
+
+    if (docRange->type_ == DocRangeType::kMany) {
+        if (docRange->doc_many_size_ > 0) {
+            uint32_t start = docRange->doc_many[0];
+            uint32_t end = docRange->doc_many[docRange->doc_many_size_ - 1];
+            if ((end - start) == docRange->doc_many_size_ - 1) {
+                docRange->doc_range.first = start;
+                docRange->doc_range.second = start + docRange->doc_many_size_;
+                docRange->type_ = DocRangeType::kRange;
+            }
+        }
+    }
+
+    return true;
+}
+
 bool SegmentTermDocs::skipTo(const int32_t target) {
     assert(count <= df);
 

src/core/CLucene/index/SegmentTermPositions.cpp

@@ -100,6 +100,10 @@ int32_t SegmentTermPositions::read(int32_t* /*docs*/, int32_t* /*freqs*/, int32_
     _CLTHROWA(CL_ERR_UnsupportedOperation,"TermPositions does not support processing multiple documents in one call. Use TermDocs instead.");
 }
 
+bool SegmentTermPositions::readRange(DocRange* docRange) {
+    _CLTHROWA(CL_ERR_UnsupportedOperation, "Unsupported operation: SegmentTermPositions::readDocRange");
+}
+
 void SegmentTermPositions::skipProx(const int64_t proxPointer, const int32_t _payloadLength){
     // we save the pointer, we might have to skip there lazily
     lazySkipPointer = proxPointer;

src/core/CLucene/index/Terms.h

@@ -8,6 +8,8 @@
 #define _lucene_index_Terms_
 
 #include "CLucene/util/Equators.h"
+#include "CLucene/index/DocRange.h"
+
 CL_NS_DEF(index)
 
 //predefine
@@ -56,6 +58,7 @@ class CLUCENE_EXPORT TermDocs {
 	// <p>Returns the number of entries read.  Zero is only returned when the
 	// stream has been exhausted.
 	virtual int32_t read(int32_t* docs, int32_t* freqs, int32_t length)=0;
+	virtual bool readRange(DocRange* docRange) = 0;
 
 	// Skips entries to the first beyond the current whose document number is
 	// greater than or equal to <i>target</i>. <p>Returns true iff there is such

src/core/CLucene/index/_MultiSegmentReader.h

@@ -151,6 +151,7 @@ class MultiTermDocs:public virtual TermDocs {
 
   /** Optimized implementation. */
   int32_t read(int32_t* docs, int32_t* freqs, int32_t length);
+  bool readRange(DocRange* docRange) override;
 
    /* A Possible future optimization could skip entire segments */
   bool skipTo(const int32_t target);

src/core/CLucene/index/_SegmentHeader.h

@@ -75,6 +75,7 @@ class SegmentTermDocs:public virtual TermDocs {
 
   /** Optimized implementation. */
   virtual int32_t read(int32_t* docs, int32_t* freqs, int32_t length);
+  bool readRange(DocRange* docRange) override;
 
   /** Optimized implementation. */
   virtual bool skipTo(const int32_t target);
@@ -125,6 +126,7 @@ class SegmentTermPositions: public SegmentTermDocs, public TermPositions {
 public:
   bool next();
   int32_t read(int32_t* docs, int32_t* freqs, int32_t length);
+  bool readRange(DocRange* docRange) override;
 
 protected:
   /** Called by super.skipTo(). */

src/core/CLucene/search/IndexSearcher.cpp

@@ -59,15 +59,22 @@ CL_NS_DEF(search)
     	}
 	};
 
-    class AllTopDocsCollector:public HitCollector {
-        public:
-            AllTopDocsCollector(const Functor& cb) : cb_(cb) {}
-            void collect(const int32_t doc, const float_t score) {
-                cb_(doc, score);
-            }
+    class AllTopDocsCollector : public HitCollector {
+      public:
+        AllTopDocsCollector(const Functor& cb) : cb_(cb) {}
+        AllTopDocsCollector(const DocRangeFunctor& range_cb) : range_cb_(range_cb) {} 
+
+        void collect(const int32_t doc, const float_t score) override {
+          cb_(doc, score);
+        }
 
-        private:
-            Functor cb_;
+        void collectRange(DocRange* docRange) override {
+          range_cb_(docRange);
+        }
+
+      private:
+        Functor cb_;
+        DocRangeFunctor range_cb_;
     };
 
 	class SortedTopDocsCollector:public HitCollector{ 
@@ -402,6 +409,38 @@ CL_NS_DEF(search)
       });
   }
 
+  void IndexSearcher::_search(Query* query, const DocRangeFunctor& cb) {
+      CND_PRECONDITION(reader != NULL, "reader is NULL");
+      CND_PRECONDITION(query != NULL, "query is NULL");
+
+      Weight* weight = NULL;
+      Scorer* scorer = NULL;
+      try
+      {
+        weight = query->weight(this);
+        scorer = weight->scorer(reader);
+        if (scorer == NULL) {
+          Query* wq = weight->getQuery();
+          if (wq != query) // query was rewritten
+            _CLLDELETE(wq);
+          _CLLDELETE(weight);
+          return;
+        }
+
+        AllTopDocsCollector hitCol(cb);
+        scorer->scoreRange( &hitCol );
+      } _CLFINALLY({
+        _CLDELETE(scorer);
+       if (weight != NULL)
+       {
+          Query* wq = weight->getQuery();
+          if (wq != query) // query was rewritten
+            _CLLDELETE(wq);
+       }
+        _CLLDELETE(weight);
+      });
+  }
+
   Query* IndexSearcher::rewrite(Query* original) {
         Query* query = original;
 		Query* last = original;

src/core/CLucene/search/IndexSearcher.h

@@ -7,6 +7,7 @@
 #ifndef _lucene_search_IndexSearcher_
 #define _lucene_search_IndexSearcher_
 
+#include "CLucene/index/DocRange.h"
 
 #include <functional>
 
@@ -36,6 +37,7 @@ CL_NS_DEF(search)
 */
 
 typedef std::function<void(const int32_t doc, const float_t score)> Functor;
+typedef std::function<void(DocRange* docRange)> DocRangeFunctor;
 
 class CLUCENE_EXPORT IndexSearcher:public Searcher{
 	CL_NS(index)::IndexReader* reader;
@@ -87,6 +89,7 @@ class CLUCENE_EXPORT IndexSearcher:public Searcher{
 	void _search(Query* query, Filter* filter, HitCollector* results);
 
 	void _search(Query* query, const Functor& cb);
+	void _search(Query* query, const DocRangeFunctor& cb);
 
 	CL_NS(index)::IndexReader* getReader();
 

src/core/CLucene/search/Scorer.cpp

@@ -26,6 +26,13 @@ void Scorer::score(HitCollector* hc) {
 	}
 }
 
+void Scorer::scoreRange(HitCollector* hc) {
+	DocRange docRange;
+	while (nextRange(&docRange)) {
+		hc->collectRange(&docRange);
+	}
+}
+
 bool Scorer::score( HitCollector* results, const int32_t maxDoc ) {
 	while( doc() < maxDoc ) {
 		results->collect( doc(), score() );

src/core/CLucene/search/Scorer.h

@@ -7,6 +7,8 @@
 #ifndef _lucene_search_Scorer_
 #define _lucene_search_Scorer_
 
+#include "CLucene/index/DocRange.h"
+
 CL_CLASS_DEF(search,Similarity)
 CL_CLASS_DEF(search,HitCollector)
 CL_CLASS_DEF(search,Explanation)
@@ -47,7 +49,8 @@ class CLUCENE_EXPORT Scorer {
 	* {@link HitCollector#collect(int, float)}.
 	* <br>When this method is used the {@link #explain(int)} method should not be used.
 	*/
-	virtual void score(HitCollector* hc) ;
+	virtual void score(HitCollector* hc);
+	virtual void scoreRange(HitCollector* hc);
 
 	/** Expert: Collects matching documents in a range.  Hook for optimization.
 	* Note that {@link #next()} must be called once before this method is called
@@ -74,6 +77,7 @@ class CLUCENE_EXPORT Scorer {
 	* @see BooleanQuery#setAllowDocsOutOfOrder
 	*/
 	virtual bool next() = 0;
+	virtual bool nextRange(DocRange* docRange) { _CLTHROWA(CL_ERR_UnsupportedOperation, "Unsupported operation: Scorer::nextRange"); }
 
 	/** Returns the current document number matching the query.
 	* Initially invalid, until {@link #next()} is called the first time.

src/core/CLucene/search/SearchHeader.h

@@ -7,6 +7,7 @@
 #ifndef _lucene_search_SearchHeader_
 #define _lucene_search_SearchHeader_
 
+#include "CLucene/index/DocRange.h"
 
 //#include "CLucene/index/IndexReader.h"
 CL_CLASS_DEF(index,Term)
@@ -95,6 +96,8 @@ CL_NS_DEF(search)
       * between 0 and 1.
       */
       virtual void collect(const int32_t doc, const float_t score) = 0;
+      virtual void collectRange(DocRange* docRange) { _CLTHROWA(CL_ERR_UnsupportedOperation, "Unsupported operation: HitCollector::collectRange"); }
+
       virtual ~HitCollector(){}
     };
 

src/core/CLucene/search/TermScorer.cpp

@@ -54,6 +54,10 @@ CL_NS_DEF(search)
     return true;
   }
 
+  bool TermScorer::nextRange(DocRange* docRange) {
+    return termDocs->readRange(docRange);
+  }
+
   bool TermScorer::skipTo(int32_t target) {
     // first scan in cache
     for (pointer++; pointer < pointerMax; pointer++) {