stopwords) {
+ try {
+ final Class classAnalyzer = Class.forName(analyzerFQN);
+ final Constructor constructor = classAnalyzer.getDeclaredConstructor(CharArraySet.class);
+ return (Analyzer) constructor.newInstance(new CharArraySet(stopwords, true));
+ } catch (final ClassNotFoundException e) {
+ throw OException.wrapException(
+ new OIndexException("Analyzer: " + analyzerFQN + " not found"), e);
+ } catch (final NoSuchMethodException e) {
+ throw OException.wrapException(
+ new OIndexException("Couldn't instantiate analyzer: public constructor not found"), e);
+ } catch (final Exception e) {
+ logger.error(
+ "Error on getting analyzer for Lucene index (continuing with StandardAnalyzer)", e);
+ return new StandardAnalyzer();
+ }
+ }
+
+ public enum AnalyzerKind {
+ INDEX,
+ QUERY;
+
+ @Override
+ public String toString() {
+ return name().toLowerCase(Locale.ENGLISH);
+ }
+ }
+}
diff --git a/lucene/src/main/java/com/arcadedb/lucene/analyzer/OLucenePerFieldAnalyzerWrapper.java b/lucene/src/main/java/com/arcadedb/lucene/analyzer/OLucenePerFieldAnalyzerWrapper.java
new file mode 100644
index 0000000000..53237815ff
--- /dev/null
+++ b/lucene/src/main/java/com/arcadedb/lucene/analyzer/OLucenePerFieldAnalyzerWrapper.java
@@ -0,0 +1,89 @@
+package com.arcadedb.lucene.analyzer;
+
+import static com.arcadedb.lucene.engine.OLuceneIndexEngineAbstract.RID;
+
+import com.arcadedb.lucene.builder.OLuceneIndexType;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+
+/**
+ * Created by frank on 10/12/15.
+ *
+ * Doesn't allow to wrap components or readers. Thread local resources can be
+ delegated to the
+ * delegate analyzer, but not allocated on this analyzer (limit memory consumption). Uses a per
+ * field reuse strategy.
+ */
+public class OLucenePerFieldAnalyzerWrapper extends DelegatingAnalyzerWrapper {
+ private final Analyzer defaultDelegateAnalyzer;
+ private final Map fieldAnalyzers;
+
+ /**
+ * Constructs with default analyzer.
+ *
+ * @param defaultAnalyzer Any fields not specifically defined to use a different analyzer will use
+ * the one provided here.
+ */
+ public OLucenePerFieldAnalyzerWrapper(final Analyzer defaultAnalyzer) {
+ this(defaultAnalyzer, new HashMap<>());
+ }
+
+ /**
+ * Constructs with default analyzer and a map of analyzers to use for specific fields.
+ *
+ * @param defaultAnalyzer Any fields not specifically defined to use a different analyzer will use
+ * the one provided here.
+ * @param fieldAnalyzers a Map (String field name to the Analyzer) to be used for those fields
+ */
+ public OLucenePerFieldAnalyzerWrapper(
+ final Analyzer defaultAnalyzer, final Map fieldAnalyzers) {
+ super(PER_FIELD_REUSE_STRATEGY);
+ this.defaultDelegateAnalyzer = defaultAnalyzer;
+ this.fieldAnalyzers = new HashMap<>();
+
+ this.fieldAnalyzers.putAll(fieldAnalyzers);
+
+ this.fieldAnalyzers.put(RID, new KeywordAnalyzer());
+ this.fieldAnalyzers.put(OLuceneIndexType.RID_HASH, new KeywordAnalyzer());
+ this.fieldAnalyzers.put("_CLASS", new KeywordAnalyzer());
+ this.fieldAnalyzers.put("_CLUSTER", new KeywordAnalyzer());
+ this.fieldAnalyzers.put("_JSON", new KeywordAnalyzer());
+ }
+
+ @Override
+ protected Analyzer getWrappedAnalyzer(final String fieldName) {
+ final Analyzer analyzer = fieldAnalyzers.get(fieldName);
+ return (analyzer != null) ? analyzer : defaultDelegateAnalyzer;
+ }
+
+ @Override
+ public String toString() {
+ return "PerFieldAnalyzerWrapper("
+ + fieldAnalyzers
+ + ", default="
+ + defaultDelegateAnalyzer
+ + ")";
+ }
+
+ public OLucenePerFieldAnalyzerWrapper add(final String field, final Analyzer analyzer) {
+ fieldAnalyzers.put(field, analyzer);
+ return this;
+ }
+
+ public OLucenePerFieldAnalyzerWrapper add(final OLucenePerFieldAnalyzerWrapper analyzer) {
+ fieldAnalyzers.putAll(analyzer.getAnalyzers());
+ return this;
+ }
+
+ public OLucenePerFieldAnalyzerWrapper remove(final String field) {
+ fieldAnalyzers.remove(field);
+ return this;
+ }
+
+ protected Map getAnalyzers() {
+ return fieldAnalyzers;
+ }
+}
diff --git a/lucene/src/main/java/com/arcadedb/lucene/builder/OLuceneIndexType.java b/lucene/src/main/java/com/arcadedb/lucene/builder/OLuceneIndexType.java
new file mode 100644
index 0000000000..8459173f22
--- /dev/null
+++ b/lucene/src/main/java/com/arcadedb/lucene/builder/OLuceneIndexType.java
@@ -0,0 +1,207 @@
+/*
+ * Copyright 2010-2016 OrientDB LTD (http://orientdb.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.arcadedb.lucene.builder;
+
+import com.arcadedb.common.exception.OException;
+import com.arcadedb.lucene.engine.OLuceneIndexEngineAbstract;
+import com.arcadedb.lucene.exception.OLuceneIndexException;
+import com.arcadedb.database.OIdentifiable;
+import com.arcadedb.database.index.OCompositeKey;
+import com.arcadedb.database.index.OIndexDefinition;
+import com.arcadedb.database.record.impl.ODocument;
+import java.io.UnsupportedEncodingException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.Date;
+import java.util.List;
+import java.util.Locale;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoubleDocValuesField;
+import org.apache.lucene.document.DoublePoint;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FloatDocValuesField;
+import org.apache.lucene.document.FloatPoint;
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.BytesRef;
+
+/** Created by enricorisa on 21/03/14. */
+public class OLuceneIndexType {
+ public static final String RID_HASH = "_RID_HASH";
+
+ public static Field createField(
+ final String fieldName, final Object value, final Field.Store store /*,Field.Index index*/) {
+ // metadata fields: _CLASS, _CLUSTER
+ if (fieldName.startsWith("_CLASS") || fieldName.startsWith("_CLUSTER")) {
+ return new StringField(fieldName, value.toString(), store);
+ }
+ return new TextField(fieldName, value.toString(), Field.Store.YES);
+ }
+
+ public static String extractId(Document doc) {
+ String value = doc.get(RID_HASH);
+ if (value != null) {
+ int pos = value.indexOf("|");
+ if (pos > 0) {
+ return value.substring(0, pos);
+ } else {
+ return value;
+ }
+ } else {
+ return null;
+ }
+ }
+
+ public static Field createIdField(final OIdentifiable id, final Object key) {
+ return new StringField(RID_HASH, genValueId(id, key), Field.Store.YES);
+ }
+
+ public static Field createOldIdField(final OIdentifiable id) {
+ return new StringField(
+ OLuceneIndexEngineAbstract.RID, id.getIdentity().toString(), Field.Store.YES);
+ }
+
+ public static String genValueId(final OIdentifiable id, final Object key) {
+ String value = id.getIdentity().toString() + "|";
+ value += hashKey(key);
+ return value;
+ }
+
+ public static List createFields(
+ String fieldName, Object value, Field.Store store, Boolean sort) {
+ List fields = new ArrayList<>();
+ if (value instanceof Number) {
+ Number number = (Number) value;
+ if (value instanceof Long) {
+ fields.add(new NumericDocValuesField(fieldName, number.longValue()));
+ fields.add(new LongPoint(fieldName, number.longValue()));
+ return fields;
+ } else if (value instanceof Float) {
+ fields.add(new FloatDocValuesField(fieldName, number.floatValue()));
+ fields.add(new FloatPoint(fieldName, number.floatValue()));
+ return fields;
+ } else if (value instanceof Double) {
+ fields.add(new DoubleDocValuesField(fieldName, number.doubleValue()));
+ fields.add(new DoublePoint(fieldName, number.doubleValue()));
+ return fields;
+ }
+ fields.add(new NumericDocValuesField(fieldName, number.longValue()));
+ fields.add(new IntPoint(fieldName, number.intValue()));
+ return fields;
+ } else if (value instanceof Date) {
+ Date date = (Date) value;
+ fields.add(new NumericDocValuesField(fieldName, date.getTime()));
+ fields.add(new LongPoint(fieldName, date.getTime()));
+ return fields;
+ }
+ if (Boolean.TRUE.equals(sort)) {
+ fields.add(new SortedDocValuesField(fieldName, new BytesRef(value.toString())));
+ }
+ fields.add(new TextField(fieldName, value.toString(), Field.Store.YES));
+ return fields;
+ }
+
+ public static Query createExactQuery(OIndexDefinition index, Object key) {
+ Query query = null;
+ if (key instanceof String) {
+ final BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
+ if (index.getFields().size() > 0) {
+ for (String idx : index.getFields()) {
+ queryBuilder.add(
+ new TermQuery(new Term(idx, key.toString())), BooleanClause.Occur.SHOULD);
+ }
+ } else {
+ queryBuilder.add(
+ new TermQuery(new Term(OLuceneIndexEngineAbstract.KEY, key.toString())),
+ BooleanClause.Occur.SHOULD);
+ }
+ query = queryBuilder.build();
+ } else if (key instanceof OCompositeKey) {
+ final BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
+ int i = 0;
+ OCompositeKey keys = (OCompositeKey) key;
+ for (String idx : index.getFields()) {
+ String val = (String) keys.getKeys().get(i);
+ queryBuilder.add(new TermQuery(new Term(idx, val)), BooleanClause.Occur.MUST);
+ i++;
+ }
+ query = queryBuilder.build();
+ }
+ return query;
+ }
+
+ public static Query createQueryId(OIdentifiable value) {
+ return new TermQuery(new Term(OLuceneIndexEngineAbstract.RID, value.getIdentity().toString()));
+ }
+
+ public static Query createQueryId(OIdentifiable value, Object key) {
+ return new TermQuery(new Term(RID_HASH, genValueId(value, key)));
+ }
+
+ public static String hashKey(Object key) {
+ try {
+ String keyString;
+ if (key instanceof ODocument) {
+ keyString = ((ODocument) key).toJSON();
+ } else {
+ keyString = key.toString();
+ }
+ MessageDigest sha256 = MessageDigest.getInstance("SHA-256");
+ byte[] bytes = sha256.digest(keyString.getBytes("UTF-8"));
+ return Base64.getEncoder().encodeToString(bytes);
+ } catch (NoSuchAlgorithmException e) {
+ throw OException.wrapException(new OLuceneIndexException("fail to find sha algorithm"), e);
+
+ } catch (UnsupportedEncodingException e) {
+ throw OException.wrapException(new OLuceneIndexException("fail to find utf-8 encoding"), e);
+ }
+ }
+
+ public static Query createDeleteQuery(
+ OIdentifiable value, List fields, Object key, ODocument metadata) {
+
+ // TODO Implementation of Composite keys with Collection
+ final BooleanQuery.Builder filter = new BooleanQuery.Builder();
+ final BooleanQuery.Builder builder = new BooleanQuery.Builder();
+ // TODO: Condition on Id and field key only for backward compatibility
+ if (value != null) {
+ builder.add(createQueryId(value), BooleanClause.Occur.MUST);
+ }
+ String field = fields.iterator().next();
+ builder.add(
+ new TermQuery(new Term(field, key.toString().toLowerCase(Locale.ENGLISH))),
+ BooleanClause.Occur.MUST);
+
+ filter.add(builder.build(), BooleanClause.Occur.SHOULD);
+ if (value != null) {
+ filter.add(createQueryId(value, key), BooleanClause.Occur.SHOULD);
+ }
+
+ return filter.build();
+ }
+}
diff --git a/lucene/src/main/java/com/arcadedb/lucene/engine/OLuceneCrossClassIndexEngine.java b/lucene/src/main/java/com/arcadedb/lucene/engine/OLuceneCrossClassIndexEngine.java
new file mode 100644
index 0000000000..31a8811ddd
--- /dev/null
+++ b/lucene/src/main/java/com/arcadedb/lucene/engine/OLuceneCrossClassIndexEngine.java
@@ -0,0 +1,399 @@
+package com.arcadedb.lucene.engine;
+
+import static com.arcadedb.lucene.OLuceneIndexFactory.LUCENE_ALGORITHM;
+
+import com.arcadedb.common.log.OLogManager;
+import com.arcadedb.common.log.OLogger;
+import com.arcadedb.common.util.ORawPair;
+import com.arcadedb.lucene.analyzer.OLucenePerFieldAnalyzerWrapper;
+import com.arcadedb.lucene.collections.OLuceneResultSet;
+import com.arcadedb.lucene.index.OLuceneFullTextIndex;
+import com.arcadedb.lucene.parser.OLuceneMultiFieldQueryParser;
+import com.arcadedb.lucene.query.OLuceneKeyAndMetadata;
+import com.arcadedb.lucene.query.OLuceneQueryContext;
+import com.arcadedb.lucene.tx.OLuceneTxChanges;
+import com.arcadedb.database.config.IndexEngineData;
+import com.arcadedb.database.ODatabaseRecordThreadLocal;
+import com.arcadedb.database.OIdentifiable;
+import com.arcadedb.database.id.OContextualRecordId;
+import com.arcadedb.database.id.ORID;
+import com.arcadedb.database.index.OIndex;
+import com.arcadedb.database.index.OIndexDefinition;
+import com.arcadedb.database.index.OIndexKeyUpdater;
+import com.arcadedb.database.index.OIndexMetadata;
+import com.arcadedb.database.index.engine.IndexEngineValidator;
+import com.arcadedb.database.index.engine.IndexEngineValuesTransformer;
+import com.arcadedb.database.metadata.schema.OClass;
+import com.arcadedb.database.metadata.schema.OType;
+import com.arcadedb.database.record.impl.ODocument;
+import com.arcadedb.database.storage.OStorage;
+import com.arcadedb.database.storage.impl.local.paginated.atomicoperations.OAtomicOperation;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiReader;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.highlight.TextFragment;
+
+/**
+ * Created by frank on 03/11/2016.
+ */
+public class OLuceneCrossClassIndexEngine implements OLuceneIndexEngine {
+ private static final OLogger logger =
+ OLogManager.instance().logger(OLuceneCrossClassIndexEngine.class);
+ private final OStorage storage;
+ private final String indexName;
+ private final int indexId;
+
+ public OLuceneCrossClassIndexEngine(int indexId, OStorage storage, String indexName) {
+ this.indexId = indexId;
+
+ this.storage = storage;
+ this.indexName = indexName;
+ }
+
+ @Override
+ public void init(OIndexMetadata metadata) {}
+
+ @Override
+ public void flush() {}
+
+ @Override
+ public int getId() {
+ return indexId;
+ }
+
+ @Override
+ public void create(OAtomicOperation atomicOperation, IndexEngineData data) throws IOException {}
+
+ @Override
+ public void delete(OAtomicOperation atomicOperation) {}
+
+ @Override
+ public void load(IndexEngineData data) {}
+
+ @Override
+ public boolean remove(OAtomicOperation atomicOperation, Object key) {
+ return false;
+ }
+
+ @Override
+ public void clear(OAtomicOperation atomicOperation) {}
+
+ @Override
+ public void close() {}
+
+ @Override
+ public Object get(Object key) {
+
+ final OLuceneKeyAndMetadata keyAndMeta = (OLuceneKeyAndMetadata) key;
+ final ODocument metadata = keyAndMeta.metadata;
+ final List excludes =
+ Optional.ofNullable(metadata.>getProperty("excludes"))
+ .orElse(Collections.emptyList());
+ final List includes =
+ Optional.ofNullable(metadata.>getProperty("includes"))
+ .orElse(Collections.emptyList());
+
+ final Collection indexes =
+ ODatabaseRecordThreadLocal.instance()
+ .get()
+ .getMetadata()
+ .getIndexManager()
+ .getIndexes()
+ .stream()
+ .filter(i -> !excludes.contains(i.getName()))
+ .filter(i -> includes.isEmpty() || includes.contains(i.getName()))
+ .collect(Collectors.toList());
+
+ final OLucenePerFieldAnalyzerWrapper globalAnalyzer =
+ new OLucenePerFieldAnalyzerWrapper(new StandardAnalyzer());
+
+ final List globalFields = new ArrayList();
+
+ final List globalReaders = new ArrayList();
+ final Map types = new HashMap<>();
+
+ try {
+ for (OIndex index : indexes) {
+
+ if (index.getAlgorithm().equalsIgnoreCase(LUCENE_ALGORITHM)
+ && index.getType().equalsIgnoreCase(OClass.INDEX_TYPE.FULLTEXT.toString())) {
+
+ final OIndexDefinition definition = index.getDefinition();
+ final String className = definition.getClassName();
+
+ String[] indexFields =
+ definition.getFields().toArray(new String[definition.getFields().size()]);
+
+ for (int i = 0; i < indexFields.length; i++) {
+ String field = indexFields[i];
+
+ types.put(className + "." + field, definition.getTypes()[i]);
+ globalFields.add(className + "." + field);
+ }
+
+ OLuceneFullTextIndex fullTextIndex = (OLuceneFullTextIndex) index.getInternal();
+
+ globalAnalyzer.add((OLucenePerFieldAnalyzerWrapper) fullTextIndex.queryAnalyzer());
+
+ globalReaders.add(fullTextIndex.searcher().getIndexReader());
+ }
+ }
+
+ IndexReader indexReader = new MultiReader(globalReaders.toArray(new IndexReader[] {}));
+
+ IndexSearcher searcher = new IndexSearcher(indexReader);
+
+ Map boost =
+ Optional.ofNullable(metadata.>getProperty("boost"))
+ .orElse(new HashMap<>());
+
+ OLuceneMultiFieldQueryParser p =
+ new OLuceneMultiFieldQueryParser(
+ types, globalFields.toArray(new String[] {}), globalAnalyzer, boost);
+
+ p.setAllowLeadingWildcard(
+ Optional.ofNullable(metadata.getProperty("allowLeadingWildcard")).orElse(false));
+
+ p.setSplitOnWhitespace(
+ Optional.ofNullable(metadata.getProperty("splitOnWhitespace")).orElse(true));
+
+ Object params = keyAndMeta.key.getKeys().get(0);
+
+ Query query = p.parse(params.toString());
+
+ final List fields = OLuceneIndexEngineUtils.buildSortFields(metadata);
+
+ OLuceneQueryContext ctx = new OLuceneQueryContext(null, searcher, query, fields);
+ return new OLuceneResultSet(this, ctx, metadata);
+ } catch (IOException e) {
+ logger.error("unable to create multi-reader", e);
+ } catch (ParseException e) {
+ logger.error("unable to parse query", e);
+ }
+
+ return null;
+ }
+
+ @Override
+ public void put(OAtomicOperation atomicOperation, Object key, Object value) {}
+
+ @Override
+ public void put(OAtomicOperation atomicOperation, Object key, ORID value) {}
+
+ @Override
+ public boolean remove(OAtomicOperation atomicOperation, Object key, ORID value) {
+ return false;
+ }
+
+ @Override
+ public void update(
+ OAtomicOperation atomicOperation, Object key, OIndexKeyUpdater