diff --git a/.github/workflows/ci-mapped.yml b/.github/workflows/ci-mapped.yml new file mode 100644 index 0000000000..891ad8fe61 --- /dev/null +++ b/.github/workflows/ci-mapped.yml @@ -0,0 +1,43 @@ +# Copyright 2021 JanusGraph Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Publish package to GitHub Packages +on: + release: + types: [created] +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + java-version: '8' + distribution: 'adopt' + + - name: Setup unique version + run: mvn versions:set -DnewVersion=$(date +%s) -DgenerateBackupPoms=false + + - name: Compile and prepare release + run: mvn clean install -Pjanusgraph-release -Dgpg.skip=true -DskipTests=true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Deploy release + run: mvn deploy -Pjanusgraph-release -Dgpg.skip=true -DskipTests=true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java index 5599e187bd..f1a29a875f 100644 --- a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java +++ b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java @@ -8242,6 +8242,68 @@ public void testIndexStoreForceInvalidationFromDBCache() throws InterruptedExcep graph2.close(); } + @Test + public void testIndexWithIndexOnlyConstraintForceInvalidationFromDBCache() throws InterruptedException, ExecutionException { + if (features.hasLocking() || !features.isDistributed()) { + return; + } + + String indexPropName = "indexedProp"; + String vertexLabelName = "vertexLabelForIndexOnlyConstraint"; + String indexName = "indexWithIndexOnlyConstraint"; + PropertyKey indexedProp = mgmt.makePropertyKey(indexPropName).dataType(Integer.class).cardinality(Cardinality.SINGLE).make(); + VertexLabel vertexLabel = mgmt.makeVertexLabel(vertexLabelName).make(); + mgmt.buildIndex(indexName, Vertex.class).addKey(indexedProp).indexOnly(vertexLabel).buildCompositeIndex(); + finishSchema(); + ManagementSystem.awaitGraphIndexStatus(graph, indexName).call(); + mgmt.updateIndex(mgmt.getGraphIndex(indexName), SchemaAction.REINDEX).get(); + finishSchema(); + + StandardJanusGraph graph1 = openInstanceWithDBCacheEnabled("testIndexWithIndexOnlyConstraintForceInvalidationFromDBCache1"); + StandardJanusGraph graph2 = openInstanceWithDBCacheEnabled("testIndexWithIndexOnlyConstraintForceInvalidationFromDBCache2"); + + JanusGraphVertex v1 = graph1.addVertex(vertexLabelName); + v1.property(indexPropName, 1); + + graph1.tx().commit(); + + // Cache data + JanusGraphTransaction tx2 = graph2.newTransaction(); + assertEquals(1L, tx2.traversal().V().hasLabel(vertexLabelName).has(indexPropName, 1).count().next()); + tx2.commit(); + + // Remove vertex + JanusGraphTransaction tx1 = graph1.newTransaction(); + tx1.traversal().V(v1.id()).drop().iterate(); + tx1.commit(); + + // Check that cached indexed vertex in graph2 was not refreshed + tx2 = graph2.newTransaction(); + assertEquals(1L, tx2.traversal().V().hasLabel(vertexLabelName).has(indexPropName, 1).count().next()); + + // Try to invalidate data without vertex label with index constraints filter enabled + invalidateUpdatedVertexProperty(graph2, v1.longId(), indexPropName, 1, -1, true); + tx2.rollback(); + + tx2 = graph2.newTransaction(); + // Check that invalidation didn't work + assertEquals(1L, tx2.traversal().V().hasLabel(vertexLabelName).has(indexPropName, 1).count().next()); + tx2.rollback(); + + tx2 = graph2.newTransaction(); + // Invalidate data without vertex label with index constraints filter disabled + invalidateUpdatedVertexProperty(graph2, v1.longId(), indexPropName, 1, -1, false); + tx2.commit(); + + tx2 = graph2.newTransaction(); + // Check that invalidation worked + assertEquals(0L, tx2.traversal().V().hasLabel(vertexLabelName).has(indexPropName, 1).count().next()); + tx2.rollback(); + + graph1.close(); + graph2.close(); + } + @Test public void testFullDBCacheInvalidation() throws InterruptedException, ExecutionException { if (features.hasLocking() || !features.isDistributed()) { @@ -8339,6 +8401,10 @@ public void testFullDBCacheInvalidation() throws InterruptedException, Execution } private void invalidateUpdatedVertexProperty(StandardJanusGraph graph, long vertexIdUpdated, String propertyNameUpdated, Object previousPropertyValue, Object newPropertyValue){ + invalidateUpdatedVertexProperty(graph, vertexIdUpdated, propertyNameUpdated, previousPropertyValue, newPropertyValue, true); + } + + private void invalidateUpdatedVertexProperty(StandardJanusGraph graph, long vertexIdUpdated, String propertyNameUpdated, Object previousPropertyValue, Object newPropertyValue, boolean withIndexConstraintsFilter){ JanusGraphTransaction tx = graph.newTransaction(); JanusGraphManagement graphMgmt = graph.openManagement(); PropertyKey propertyKey = graphMgmt.getPropertyKey(propertyNameUpdated); @@ -8347,7 +8413,13 @@ private void invalidateUpdatedVertexProperty(StandardJanusGraph graph, long vert StandardVertexProperty propertyNewVal = new StandardVertexProperty(propertyKey.longId(), propertyKey, cacheVertex, newPropertyValue, ElementLifeCycle.New); IndexSerializer indexSerializer = graph.getIndexSerializer(); - Collection indexUpdates = indexSerializer.getIndexUpdates(cacheVertex, Arrays.asList(propertyPreviousVal, propertyNewVal)); + Collection indexUpdates; + if(withIndexConstraintsFilter){ + indexUpdates = indexSerializer.getIndexUpdates(cacheVertex, Arrays.asList(propertyPreviousVal, propertyNewVal)); + } else { + indexUpdates = indexSerializer.getIndexUpdatesNoConstraints(cacheVertex, Arrays.asList(propertyPreviousVal, propertyNewVal)); + } + CacheInvalidationService invalidationService = graph.getDBCacheInvalidationService(); for(IndexUpdate indexUpdate : indexUpdates){ diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/IndexSerializer.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/IndexSerializer.java index 64e2d6db0f..90d1072b75 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/IndexSerializer.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/IndexSerializer.java @@ -51,6 +51,7 @@ import org.janusgraph.graphdb.database.index.IndexRecords; import org.janusgraph.graphdb.database.index.IndexUpdate; import org.janusgraph.graphdb.database.serialize.Serializer; +import org.janusgraph.graphdb.database.util.IndexAppliesToFunction; import org.janusgraph.graphdb.database.util.IndexRecordUtil; import org.janusgraph.graphdb.internal.ElementCategory; import org.janusgraph.graphdb.internal.InternalRelation; @@ -88,6 +89,8 @@ import javax.annotation.Nullable; import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.INDEX_NAME_MAPPING; +import static org.janusgraph.graphdb.database.util.IndexRecordUtil.FULL_INDEX_APPLIES_TO_FILTER; +import static org.janusgraph.graphdb.database.util.IndexRecordUtil.INDEX_APPLIES_TO_NO_CONSTRAINTS_FILTER; import static org.janusgraph.graphdb.database.util.IndexRecordUtil.bytebuffer2RelationId; import static org.janusgraph.graphdb.database.util.IndexRecordUtil.getCompositeIndexUpdate; import static org.janusgraph.graphdb.database.util.IndexRecordUtil.element2String; @@ -180,6 +183,22 @@ public IndexInfoRetriever getIndexInfoRetriever(StandardJanusGraphTx tx) { ################################################### */ public Collection getIndexUpdates(InternalRelation relation) { + return getIndexUpdates(relation, FULL_INDEX_APPLIES_TO_FILTER); + } + + public Collection getIndexUpdates(InternalVertex vertex, Collection updatedProperties) { + return getIndexUpdates(vertex, updatedProperties, FULL_INDEX_APPLIES_TO_FILTER); + } + + public Collection getIndexUpdatesNoConstraints(InternalRelation relation) { + return getIndexUpdates(relation, INDEX_APPLIES_TO_NO_CONSTRAINTS_FILTER); + } + + public Collection getIndexUpdatesNoConstraints(InternalVertex vertex, Collection updatedProperties) { + return getIndexUpdates(vertex, updatedProperties, INDEX_APPLIES_TO_NO_CONSTRAINTS_FILTER); + } + + public Collection getIndexUpdates(InternalRelation relation, IndexAppliesToFunction indexFilter) { assert relation.isNew() || relation.isRemoved(); final Set updates = new HashSet<>(); final IndexMutationType updateType = getUpdateType(relation); @@ -187,7 +206,7 @@ public Collection getIndexUpdates(InternalRelation relation) { for (final PropertyKey type : relation.getPropertyKeysDirect()) { if (type == null) continue; for (final IndexType index : ((InternalRelationType) type).getKeyIndexes()) { - if (!indexAppliesTo(index,relation)) continue; + if (!indexFilter.indexAppliesTo(index,relation)) continue; IndexUpdate update; if (index instanceof CompositeIndexType) { final CompositeIndexType iIndex= (CompositeIndexType) index; @@ -206,7 +225,7 @@ public Collection getIndexUpdates(InternalRelation relation) { return updates; } - public Collection getIndexUpdates(InternalVertex vertex, Collection updatedProperties) { + public Collection getIndexUpdates(InternalVertex vertex, Collection updatedProperties, IndexAppliesToFunction indexFilter) { if (updatedProperties.isEmpty()) return Collections.emptyList(); final Set updates = new HashSet<>(); @@ -216,7 +235,7 @@ public Collection getIndexUpdates(InternalVertex vertex, Collection assert rel.isNew() || rel.isRemoved(); assert rel.getVertex(0).equals(vertex); final IndexMutationType updateType = getUpdateType(rel); for (final IndexType index : ((InternalRelationType)p.propertyKey()).getKeyIndexes()) { - if (!indexAppliesTo(index,vertex)) continue; + if (!indexFilter.indexAppliesTo(index,vertex)) continue; if (index.isCompositeIndex()) { //Gather composite indexes final CompositeIndexType cIndex = (CompositeIndexType)index; final IndexRecords updateRecords = indexMatches(vertex,cIndex,updateType==IndexMutationType.DELETE,p.propertyKey(),new IndexRecordEntry(p)); diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/util/IndexAppliesToFunction.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/util/IndexAppliesToFunction.java new file mode 100644 index 0000000000..8fb5d6eaed --- /dev/null +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/util/IndexAppliesToFunction.java @@ -0,0 +1,22 @@ +// Copyright 2022 JanusGraph Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.janusgraph.graphdb.database.util; + +import org.janusgraph.core.JanusGraphElement; +import org.janusgraph.graphdb.types.IndexType; + +public interface IndexAppliesToFunction { + boolean indexAppliesTo(IndexType index, JanusGraphElement element); +} diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/util/IndexRecordUtil.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/util/IndexRecordUtil.java index 9f48ded1b6..d00ec6543f 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/util/IndexRecordUtil.java +++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/util/IndexRecordUtil.java @@ -61,6 +61,9 @@ public class IndexRecordUtil { + public static final IndexAppliesToFunction FULL_INDEX_APPLIES_TO_FILTER = IndexRecordUtil::indexAppliesTo; + public static final IndexAppliesToFunction INDEX_APPLIES_TO_NO_CONSTRAINTS_FILTER = IndexRecordUtil::indexAppliesToWithoutConstraints; + private static final int DEFAULT_OBJECT_BYTELEN = 30; private static final byte FIRST_INDEX_COLUMN_BYTE = 0; @@ -134,10 +137,17 @@ public static IndexMutationType getUpdateType(InternalRelation relation) { } public static boolean indexAppliesTo(IndexType index, JanusGraphElement element) { + return indexAppliesToWithoutConstraints(index, element) && indexMatchesConstraints(index, element); + } + + public static boolean indexAppliesToWithoutConstraints(IndexType index, JanusGraphElement element) { return index.getElement().isInstance(element) && - (!(index instanceof CompositeIndexType) || ((CompositeIndexType)index).getStatus()!= SchemaStatus.DISABLED) && - (!index.hasSchemaTypeConstraint() || - index.getElement().matchesConstraint(index.getSchemaTypeConstraint(),element)); + (!(index instanceof CompositeIndexType) || ((CompositeIndexType)index).getStatus()!= SchemaStatus.DISABLED); + } + + public static boolean indexMatchesConstraints(IndexType index, JanusGraphElement element) { + return !index.hasSchemaTypeConstraint() || + index.getElement().matchesConstraint(index.getSchemaTypeConstraint(),element); } public static PropertyKey[] getKeysOfRecords(IndexRecordEntry[] record) { diff --git a/pom.xml b/pom.xml index 49070c16ea..33d145329c 100644 --- a/pom.xml +++ b/pom.xml @@ -46,12 +46,14 @@ - ossrh - https://oss.sonatype.org/content/repositories/snapshots + github + GitHub Packages + https://maven.pkg.github.com/mapped/janusgraph - ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2/ + github + GitHub Packages + https://maven.pkg.github.com/mapped/janusgraph @@ -115,7 +117,7 @@ 2.9.1 1.33 - 9.4.44.v20210927 + 10.0.14 2.18.0 22.2.0 2.9.3