From 536c2ef130ac0f727ed036bb7397267bd83fcdc9 Mon Sep 17 00:00:00 2001 From: brandon Date: Mon, 16 Apr 2018 16:31:57 -0700 Subject: [PATCH 01/14] Started implementation for intersect operator --- .../vxquery/functions/builtin-operators.xml | 1 + .../OpIntersectScalarEvaluatorFactory.java | 74 +++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java diff --git a/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-operators.xml b/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-operators.xml index ec2786414..46e06b6a1 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-operators.xml +++ b/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-operators.xml @@ -252,6 +252,7 @@ + diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java new file mode 100644 index 000000000..93e9944fd --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.vxquery.runtime.functions.sequence; + +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; +import org.apache.hyracks.api.context.IHyracksTaskContext; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.data.std.api.IPointable; +import org.apache.hyracks.data.std.primitive.LongPointable; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; +import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; +import org.apache.vxquery.datamodel.values.ValueTag; +import org.apache.vxquery.datamodel.values.XDMConstants; +import org.apache.vxquery.exceptions.ErrorCode; +import org.apache.vxquery.exceptions.SystemException; +import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator; +import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory; + +public class OpIntersectScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory { + private static final long serialVersionUID = 1L; + + public OpIntersectScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) { + super(args); + } + + @Override + protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) + throws HyracksDataException { + final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); + final ArrayBackedValueStorage abvsInner = new ArrayBackedValueStorage(); + final DataOutput dOutInner = abvsInner.getDataOutput(); + final SequenceBuilder sb = new SequenceBuilder(); + + return new AbstractTaggedValueArgumentScalarEvaluator(args) { + @Override + protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { + try { + TaggedValuePointable tvp1 = args[0]; + TaggedValuePointable tvp2 = args[1]; + if (!ValueTag.isNode(tvp1.getTag()) || !ValueTag.isNode(tvp2.getTag())) { + throw new SystemException(ErrorCode.XPTY0004); + } + + abvs.reset(); + sb.reset(abvs); + //TODO + sb.finish(); + result.set(abvs); + } catch (IOException e) { + throw new SystemException(ErrorCode.SYSE0001); + } + } + }; + } +} From ddd159bacfa7915efdfa4a99cb804eb7f6f2bd4b Mon Sep 17 00:00:00 2001 From: brandon Date: Fri, 20 Apr 2018 15:58:09 -0700 Subject: [PATCH 02/14] Trying to get an intersect query to compile --- .../sequence/OpIntersectScalarEvaluatorFactory.java | 8 ++------ .../vxquery/xmlquery/translator/XMLQueryTranslator.java | 2 ++ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java index 93e9944fd..61b85ee7a 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java @@ -56,15 +56,11 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S try { TaggedValuePointable tvp1 = args[0]; TaggedValuePointable tvp2 = args[1]; - if (!ValueTag.isNode(tvp1.getTag()) || !ValueTag.isNode(tvp2.getTag())) { + if (tvp1.getTag() != ValueTag.SEQUENCE_TAG) { throw new SystemException(ErrorCode.XPTY0004); } - abvs.reset(); - sb.reset(abvs); - //TODO - sb.finish(); - result.set(abvs); + result.set(tvp1); } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001); } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/translator/XMLQueryTranslator.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/translator/XMLQueryTranslator.java index 6d0b35a67..85ade6464 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/translator/XMLQueryTranslator.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/translator/XMLQueryTranslator.java @@ -1538,10 +1538,12 @@ private LogicalVariable translateInfixExprNode(TranslationContext tCtx, InfixExp LogicalVariable varRight = translateExpression(ie.getRightExpr(), tCtx); ILogicalExpression arg1 = normalize(vre(varLeft), sign.getParameterType(0)); ILogicalExpression arg2 = normalize(vre(varRight), sign.getParameterType(1)); + /* if (BuiltinOperators.EXCEPT.equals(operator) || BuiltinOperators.INTERSECT.equals(operator)) { arg1 = sfce(BuiltinOperators.SORT_DISTINCT_NODES_ASC, arg1); arg2 = sfce(BuiltinOperators.SORT_DISTINCT_NODES_ASC, arg2); } + */ ILogicalExpression result = sfce(operator, arg1, arg2); if (BuiltinOperators.UNION.equals(operator)) { result = sfce(BuiltinOperators.SORT_DISTINCT_NODES_ASC, result); From 84bd445402b33e8b3ee5ea40d9843208127e61da Mon Sep 17 00:00:00 2001 From: brandon Date: Thu, 3 May 2018 10:01:33 -0700 Subject: [PATCH 03/14] Naive Implementation of Intersect --- .../OpIntersectScalarEvaluatorFactory.java | 83 +++++++++++++++++-- .../Queries/XQuery/Intersect/intersect.xq | 3 + .../resources/TestSources/xml/catalog.xml | 57 +++++++++++++ .../src/test/resources/VXQueryCatalog.xml | 17 ++++ .../test/resources/cat/SetOperatorQueries.xml | 26 ++++++ 5 files changed, 181 insertions(+), 5 deletions(-) create mode 100644 vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect.xq create mode 100644 vxquery-xtest/src/test/resources/TestSources/xml/catalog.xml create mode 100644 vxquery-xtest/src/test/resources/cat/SetOperatorQueries.xml diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java index 61b85ee7a..70dfcbb3a 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java @@ -18,6 +18,8 @@ import java.io.DataOutput; import java.io.IOException; +import java.util.HashSet; +import java.util.Set; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; @@ -25,8 +27,15 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.api.IPointable; import org.apache.hyracks.data.std.primitive.LongPointable; +import org.apache.hyracks.data.std.primitive.UTF8StringPointable; +import org.apache.hyracks.data.std.primitive.VoidPointable; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.vxquery.context.DynamicContext; +import org.apache.vxquery.datamodel.accessors.SequencePointable; import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; +import org.apache.vxquery.datamodel.accessors.TypedPointables; +import org.apache.vxquery.datamodel.accessors.nodes.ElementNodePointable; +import org.apache.vxquery.datamodel.accessors.nodes.NodeTreePointable; import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; import org.apache.vxquery.datamodel.values.ValueTag; import org.apache.vxquery.datamodel.values.XDMConstants; @@ -34,6 +43,11 @@ import org.apache.vxquery.exceptions.SystemException; import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator; import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory; +import org.apache.vxquery.runtime.functions.comparison.AbstractValueComparisonOperation; +import org.apache.vxquery.runtime.functions.comparison.ValueEqComparisonOperation; +import org.apache.vxquery.runtime.functions.comparison.general.AbstractGeneralComparisonScalarEvaluatorFactory; +import org.apache.vxquery.runtime.functions.comparison.general.GeneralEqComparisonScalarEvaluatorFactory; +import org.apache.vxquery.runtime.functions.util.FunctionHelper; public class OpIntersectScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory { private static final long serialVersionUID = 1L; @@ -46,21 +60,80 @@ public OpIntersectScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) { protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws HyracksDataException { final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); - final ArrayBackedValueStorage abvsInner = new ArrayBackedValueStorage(); - final DataOutput dOutInner = abvsInner.getDataOutput(); + final DataOutput dOut = abvs.getDataOutput(); + final AbstractValueComparisonOperation aOp = new ValueEqComparisonOperation(); + final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); + + final NodeTreePointable ntp1 = (NodeTreePointable) NodeTreePointable.FACTORY.createPointable(); + final NodeTreePointable ntp2 = (NodeTreePointable) NodeTreePointable.FACTORY.createPointable(); + + + final SequencePointable seqleft = (SequencePointable) SequencePointable.FACTORY.createPointable(); + final SequencePointable seqright = (SequencePointable) SequencePointable.FACTORY.createPointable(); + final TaggedValuePointable tvpleft = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); + final TaggedValuePointable tvpright = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); + final TypedPointables tp1 = new TypedPointables(); + final TypedPointables tp2 = new TypedPointables(); + final SequenceBuilder sb = new SequenceBuilder(); - + + Set nodeIDs = new HashSet(); + return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { try { + abvs.reset(); + sb.reset(abvs); TaggedValuePointable tvp1 = args[0]; TaggedValuePointable tvp2 = args[1]; + + // If either operand only has one item in it then the tag is a node_tree_tag + // TODO: handle if either operand is empty or only has one item if (tvp1.getTag() != ValueTag.SEQUENCE_TAG) { - throw new SystemException(ErrorCode.XPTY0004); + System.out.println("Not a Sequence: " + tvp1.getTag()); + throw new SystemException(ErrorCode.FORG0006); } + if (tvp2.getTag() != ValueTag.SEQUENCE_TAG) { + System.out.println("Not a Sequence: " + tvp2.getTag()); + throw new SystemException(ErrorCode.FORG0006); + } + tvp1.getValue(seqleft); + tvp2.getValue(seqright); - result.set(tvp1); + // naive implementation + int seqllen = seqleft.getEntryCount(); + int seqrlen = seqright.getEntryCount(); + for (int i = 0 ; i < seqllen; ++i) { + seqleft.getEntry(i, tvpleft); + if (tvpleft.getTag() != ValueTag.NODE_TREE_TAG) { + throw new SystemException(ErrorCode.XPTY0004); + } + int lNodeId = FunctionHelper.getLocalNodeId(tvpleft, tp1); + if (lNodeId == -1) { + //TODO + + } + + for (int j = 0; j < seqrlen; ++j) { + seqright.getEntry(j, tvpright); + if (tvpright.getTag() != ValueTag.NODE_TREE_TAG) { + throw new SystemException(ErrorCode.XPTY0004); + } + int rNodeId = FunctionHelper.getLocalNodeId(tvpright, tp1); + if (rNodeId == -1) { + //TODO + + } + + if (lNodeId == rNodeId) { + sb.addItem(tvpleft); + break; + } + } + } + sb.finish(); + result.set(abvs); } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001); } diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect.xq new file mode 100644 index 000000000..e42bf9bc6 --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect.xq @@ -0,0 +1,3 @@ +let $a := doc("src/test/resources/TestSources/xml/catalog.xml")/catalog/book[price<50] +let $b := doc("src/test/resources/TestSources/xml/catalog.xml")/catalog/book[price<40] +return $a intersect $b diff --git a/vxquery-xtest/src/test/resources/TestSources/xml/catalog.xml b/vxquery-xtest/src/test/resources/TestSources/xml/catalog.xml new file mode 100644 index 000000000..45d8c7bc3 --- /dev/null +++ b/vxquery-xtest/src/test/resources/TestSources/xml/catalog.xml @@ -0,0 +1,57 @@ + + + + + Hightower, Kim + The First Book + Fiction + 44.95 + 2000-10-01 + An amazing story of nothing. + + + + Oberg, Bruce + The Poet's First Poem + Poem + 24.95 + The least poetic poems of the decade. + + + + Nagata, Suanne + Becoming Somebody + Biography + 39 + A masterpiece of the fine art of gossiping. + + + happy + + Ted + + + + Nye, Bill + Science + + + + + + diff --git a/vxquery-xtest/src/test/resources/VXQueryCatalog.xml b/vxquery-xtest/src/test/resources/VXQueryCatalog.xml index 7a4438e5e..d0e0672a1 100644 --- a/vxquery-xtest/src/test/resources/VXQueryCatalog.xml +++ b/vxquery-xtest/src/test/resources/VXQueryCatalog.xml @@ -58,6 +58,8 @@ + + ]> @@ -145,6 +147,9 @@ File + + File + @@ -381,4 +386,16 @@ &XMLInJSONQueries; + + + Set Operators + Set Operators: Intersect and Union + + + + Tests for Set Operators + + &SetOperatorQueries; + + diff --git a/vxquery-xtest/src/test/resources/cat/SetOperatorQueries.xml b/vxquery-xtest/src/test/resources/cat/SetOperatorQueries.xml new file mode 100644 index 000000000..231f749c3 --- /dev/null +++ b/vxquery-xtest/src/test/resources/cat/SetOperatorQueries.xml @@ -0,0 +1,26 @@ + + + + + Set Operator Test + + + + Intersects two sets both with more than 1 item + + intersect.txt + + From 1c0cb0cf60e8cd413aa8fcf3be7b50b9249c431f Mon Sep 17 00:00:00 2001 From: brandon Date: Thu, 3 May 2018 10:32:59 -0700 Subject: [PATCH 04/14] Hash map implementation of intersect --- .../OpIntersectScalarEvaluatorFactory.java | 118 ++++++++++-------- 1 file changed, 69 insertions(+), 49 deletions(-) diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java index 70dfcbb3a..145067d55 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java @@ -18,8 +18,8 @@ import java.io.DataOutput; import java.io.IOException; -import java.util.HashSet; -import java.util.Set; +import java.util.HashMap; +import java.util.Map; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; @@ -60,24 +60,16 @@ public OpIntersectScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) { protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) throws HyracksDataException { final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); - final DataOutput dOut = abvs.getDataOutput(); - final AbstractValueComparisonOperation aOp = new ValueEqComparisonOperation(); - final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); - - final NodeTreePointable ntp1 = (NodeTreePointable) NodeTreePointable.FACTORY.createPointable(); - final NodeTreePointable ntp2 = (NodeTreePointable) NodeTreePointable.FACTORY.createPointable(); - - + final SequenceBuilder sb = new SequenceBuilder(); + final SequencePointable seqleft = (SequencePointable) SequencePointable.FACTORY.createPointable(); final SequencePointable seqright = (SequencePointable) SequencePointable.FACTORY.createPointable(); final TaggedValuePointable tvpleft = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); final TaggedValuePointable tvpright = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); - final TypedPointables tp1 = new TypedPointables(); - final TypedPointables tp2 = new TypedPointables(); - - final SequenceBuilder sb = new SequenceBuilder(); - - Set nodeIDs = new HashSet(); + final TypedPointables tpleft = new TypedPointables(); + final TypedPointables tpright = new TypedPointables(); + + Map nodes = new HashMap(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override @@ -88,50 +80,78 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S TaggedValuePointable tvp1 = args[0]; TaggedValuePointable tvp2 = args[1]; - // If either operand only has one item in it then the tag is a node_tree_tag - // TODO: handle if either operand is empty or only has one item + // If an operand has one item then it is a node tree + // If an operand has more than one item then it is a sequence + + + // Add items from the left operand into the hash map if (tvp1.getTag() != ValueTag.SEQUENCE_TAG) { - System.out.println("Not a Sequence: " + tvp1.getTag()); - throw new SystemException(ErrorCode.FORG0006); + if (tvp1.getTag() != ValueTag.NODE_TREE_TAG) { + System.out.println("Not a Sequence: " + tvp1.getTag()); + throw new SystemException(ErrorCode.FORG0006); + } + else { + // Is only one item + int nodeId = FunctionHelper.getLocalNodeId(tvp1, tpleft); + if (nodeId == -1) { + //TODO + } + nodes.put(nodeId, tvp1); + } + } + else { + // Is a sequence, so add all the items + tvp1.getValue(seqleft); + int seqleftlen = seqleft.getEntryCount(); + for (int i = 0; i < seqleftlen; ++i) { + seqleft.getEntry(i, tvpleft); + if (tvpleft.getTag() != ValueTag.NODE_TREE_TAG) { + throw new SystemException(ErrorCode.XPTY0004); + } + int nodeId = FunctionHelper.getLocalNodeId(tvpleft, tpleft); + if (nodeId == -1) { + //TODO + } + nodes.put(nodeId, tvpleft); + } } - if (tvp2.getTag() != ValueTag.SEQUENCE_TAG) { - System.out.println("Not a Sequence: " + tvp2.getTag()); - throw new SystemException(ErrorCode.FORG0006); - } - tvp1.getValue(seqleft); - tvp2.getValue(seqright); - // naive implementation - int seqllen = seqleft.getEntryCount(); - int seqrlen = seqright.getEntryCount(); - for (int i = 0 ; i < seqllen; ++i) { - seqleft.getEntry(i, tvpleft); - if (tvpleft.getTag() != ValueTag.NODE_TREE_TAG) { - throw new SystemException(ErrorCode.XPTY0004); - } - int lNodeId = FunctionHelper.getLocalNodeId(tvpleft, tp1); - if (lNodeId == -1) { - //TODO - + // Check if node IDs from right operand is in the hash map + if (tvp2.getTag() != ValueTag.SEQUENCE_TAG) { + if (tvp2.getTag() != ValueTag.NODE_TREE_TAG) { + System.out.println("Not a Sequence: " + tvp2.getTag()); + throw new SystemException(ErrorCode.FORG0006); + } + else { + // Is only one item + int nodeId = FunctionHelper.getLocalNodeId(tvp2, tpright); + if (nodeId == -1) { + //TODO + } + if (nodes.containsKey(nodeId)) { + sb.addItem(tvp2); + } } - - for (int j = 0; j < seqrlen; ++j) { - seqright.getEntry(j, tvpright); + } + else { + // Is a sequence, so check all the items + tvp2.getValue(seqright); + int seqrightlen = seqright.getEntryCount(); + for (int i = 0; i < seqrightlen; ++i) { + seqright.getEntry(i, tvpright); if (tvpright.getTag() != ValueTag.NODE_TREE_TAG) { throw new SystemException(ErrorCode.XPTY0004); } - int rNodeId = FunctionHelper.getLocalNodeId(tvpright, tp1); - if (rNodeId == -1) { + int nodeId = FunctionHelper.getLocalNodeId(tvpright, tpright); + if (nodeId == -1) { //TODO - } - - if (lNodeId == rNodeId) { - sb.addItem(tvpleft); - break; + if (nodes.containsKey(nodeId)) { + sb.addItem(tvpright); } - } + } } + sb.finish(); result.set(abvs); } catch (IOException e) { From e257965673d96f3ec6fbf7561466cf556890dd96 Mon Sep 17 00:00:00 2001 From: brandon Date: Wed, 16 May 2018 15:26:09 -0700 Subject: [PATCH 05/14] Clean up intersect. Added tests --- .../OpIntersectScalarEvaluatorFactory.java | 174 +++++++++--------- .../Intersect/intersect01.txt | 14 ++ .../Intersect/intersect02.txt | 7 + .../Intersect/intersect03.txt | 7 + .../Intersect/intersect04.txt | 0 .../Queries/XQuery/Intersect/intersect.xq | 3 - .../Queries/XQuery/Intersect/intersect01.xq | 21 +++ .../Queries/XQuery/Intersect/intersect02.xq | 21 +++ .../Queries/XQuery/Intersect/intersect03.xq | 21 +++ .../Queries/XQuery/Intersect/intersect04.xq | 21 +++ .../resources/TestSources/xml/catalog.xml | 14 +- .../test/resources/cat/SetOperatorQueries.xml | 21 ++- 12 files changed, 220 insertions(+), 104 deletions(-) create mode 100644 vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect01.txt create mode 100644 vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect02.txt create mode 100644 vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect03.txt create mode 100644 vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect04.txt delete mode 100644 vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect.xq create mode 100644 vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect01.xq create mode 100644 vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect02.xq create mode 100644 vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect03.xq create mode 100644 vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect04.xq diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java index 145067d55..31adb86de 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java @@ -16,8 +16,8 @@ */ package org.apache.vxquery.runtime.functions.sequence; -import java.io.DataOutput; import java.io.IOException; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -26,34 +26,24 @@ import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.api.IPointable; -import org.apache.hyracks.data.std.primitive.LongPointable; -import org.apache.hyracks.data.std.primitive.UTF8StringPointable; -import org.apache.hyracks.data.std.primitive.VoidPointable; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; -import org.apache.vxquery.context.DynamicContext; import org.apache.vxquery.datamodel.accessors.SequencePointable; import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; import org.apache.vxquery.datamodel.accessors.TypedPointables; -import org.apache.vxquery.datamodel.accessors.nodes.ElementNodePointable; import org.apache.vxquery.datamodel.accessors.nodes.NodeTreePointable; import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; import org.apache.vxquery.datamodel.values.ValueTag; -import org.apache.vxquery.datamodel.values.XDMConstants; import org.apache.vxquery.exceptions.ErrorCode; import org.apache.vxquery.exceptions.SystemException; import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator; import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory; -import org.apache.vxquery.runtime.functions.comparison.AbstractValueComparisonOperation; -import org.apache.vxquery.runtime.functions.comparison.ValueEqComparisonOperation; -import org.apache.vxquery.runtime.functions.comparison.general.AbstractGeneralComparisonScalarEvaluatorFactory; -import org.apache.vxquery.runtime.functions.comparison.general.GeneralEqComparisonScalarEvaluatorFactory; import org.apache.vxquery.runtime.functions.util.FunctionHelper; public class OpIntersectScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory { private static final long serialVersionUID = 1L; public OpIntersectScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) { - super(args); + super(args); } @Override @@ -61,103 +51,117 @@ protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvalu throws HyracksDataException { final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final SequenceBuilder sb = new SequenceBuilder(); - + + final NodeTreePointable temp = (NodeTreePointable) NodeTreePointable.FACTORY.createPointable(); final SequencePointable seqleft = (SequencePointable) SequencePointable.FACTORY.createPointable(); final SequencePointable seqright = (SequencePointable) SequencePointable.FACTORY.createPointable(); final TaggedValuePointable tvpleft = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); final TaggedValuePointable tvpright = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); final TypedPointables tpleft = new TypedPointables(); final TypedPointables tpright = new TypedPointables(); - + Map nodes = new HashMap(); - + return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { - try { - abvs.reset(); - sb.reset(abvs); - TaggedValuePointable tvp1 = args[0]; + try { + abvs.reset(); + sb.reset(abvs); + TaggedValuePointable tvp1 = args[0]; TaggedValuePointable tvp2 = args[1]; - + + if ((tvp1.getTag() != ValueTag.SEQUENCE_TAG && tvp1.getTag() != ValueTag.NODE_TREE_TAG) + || (tvp2.getTag() != ValueTag.SEQUENCE_TAG && tvp2.getTag() != ValueTag.NODE_TREE_TAG)) { + throw new SystemException(ErrorCode.FORG0006); + } + // If an operand has one item then it is a node tree - // If an operand has more than one item then it is a sequence - + // If an operand has more than one item then it is a sequence // Add items from the left operand into the hash map - if (tvp1.getTag() != ValueTag.SEQUENCE_TAG) { - if (tvp1.getTag() != ValueTag.NODE_TREE_TAG) { - System.out.println("Not a Sequence: " + tvp1.getTag()); - throw new SystemException(ErrorCode.FORG0006); - } - else { - // Is only one item - int nodeId = FunctionHelper.getLocalNodeId(tvp1, tpleft); - if (nodeId == -1) { - //TODO - } - nodes.put(nodeId, tvp1); - } - } - else { - // Is a sequence, so add all the items + if (tvp1.getTag() == ValueTag.SEQUENCE_TAG) { tvp1.getValue(seqleft); - int seqleftlen = seqleft.getEntryCount(); - for (int i = 0; i < seqleftlen; ++i) { - seqleft.getEntry(i, tvpleft); - if (tvpleft.getTag() != ValueTag.NODE_TREE_TAG) { - throw new SystemException(ErrorCode.XPTY0004); - } - int nodeId = FunctionHelper.getLocalNodeId(tvpleft, tpleft); - if (nodeId == -1) { - //TODO - } - nodes.put(nodeId, tvpleft); + for (int i = 0; i < seqleft.getEntryCount(); ++i) { + seqleft.getEntry(i, tvpleft); + if (tvpleft.getTag() != ValueTag.NODE_TREE_TAG) { + throw new SystemException(ErrorCode.XPTY0004); + } + if (addItem(tvpleft, tpleft, nodes)) { + byte data[] = tvpleft.getByteArray(); + tvpleft.getValue(temp); + byte data2[] = temp.getByteArray(); + if (Arrays.equals(data, data2)) { + System.out.println("true"); + } + else { + System.out.println("false"); + } + } + } + } else { + if (!addItem(tvp1, tpleft, nodes)) { + } } - - // Check if node IDs from right operand is in the hash map - if (tvp2.getTag() != ValueTag.SEQUENCE_TAG) { - if (tvp2.getTag() != ValueTag.NODE_TREE_TAG) { - System.out.println("Not a Sequence: " + tvp2.getTag()); - throw new SystemException(ErrorCode.FORG0006); - } - else { - // Is only one item - int nodeId = FunctionHelper.getLocalNodeId(tvp2, tpright); - if (nodeId == -1) { - //TODO - } - if (nodes.containsKey(nodeId)) { - sb.addItem(tvp2); - } - } - } - else { - // Is a sequence, so check all the items - tvp2.getValue(seqright); - int seqrightlen = seqright.getEntryCount(); - for (int i = 0; i < seqrightlen; ++i) { - seqright.getEntry(i, tvpright); - if (tvpright.getTag() != ValueTag.NODE_TREE_TAG) { - throw new SystemException(ErrorCode.XPTY0004); - } - int nodeId = FunctionHelper.getLocalNodeId(tvpright, tpright); - if (nodeId == -1) { - //TODO - } - if (nodes.containsKey(nodeId)) { - sb.addItem(tvpright); - } + + // Check if node IDs from right operand are in the hash map + if (tvp2.getTag() == ValueTag.SEQUENCE_TAG) { + tvp2.getValue(seqright); + for (int i = 0; i < seqright.getEntryCount(); ++i) { + seqright.getEntry(i, tvpright); + if (tvpright.getTag() != ValueTag.NODE_TREE_TAG) { + throw new SystemException(ErrorCode.XPTY0004); + } + if (checkItem(tvpright, tpright, nodes)) { + sb.addItem(tvpright); + } + + } + } else { + if (checkItem(tvp2, tpright, nodes)) { + sb.addItem(tvp2); } } sb.finish(); result.set(abvs); - } catch (IOException e) { + } catch (IOException e) { throw new SystemException(ErrorCode.SYSE0001); - } + } } }; } + + /* + * Adds item to nodes (hash map) + * Returns: False if local node id doesn't exist + * True if item added successfully + */ + private boolean addItem(TaggedValuePointable tvp, TypedPointables tp, Map nodes) { + int nodeId = FunctionHelper.getLocalNodeId(tvp, tp); + System.out.println("Node ID: " + nodeId); + if (nodeId == -1) { + System.out.println(new String(tvp.getByteArray())); + return false; + } + nodes.put(nodeId, tvp); + return true; + } + + private boolean checkItem(TaggedValuePointable tvp, TypedPointables tp, Map nodes) { + int nodeId = FunctionHelper.getLocalNodeId(tvp, tp); + if (nodeId == -1) { + return false; + } else if (nodes.containsKey(nodeId)) { + return true; + } + return false; + } } + +/* + $a := 5 union 6 + $b := a + $a intersect $b +*/ \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect01.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect01.txt new file mode 100644 index 000000000..2d8a5317c --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect01.txt @@ -0,0 +1,14 @@ + + Oberg, Bruce + The Poet's First Poem + Poem + 24.95 + The least poetic poems of the decade. + + + Nagata, Suanne + Becoming Somebody + Biography + 39 + A masterpiece of the fine art of gossiping. + \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect02.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect02.txt new file mode 100644 index 000000000..d17890a5a --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect02.txt @@ -0,0 +1,7 @@ + + Oberg, Bruce + The Poet's First Poem + Poem + 24.95 + The least poetic poems of the decade. + \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect03.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect03.txt new file mode 100644 index 000000000..d17890a5a --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect03.txt @@ -0,0 +1,7 @@ + + Oberg, Bruce + The Poet's First Poem + Poem + 24.95 + The least poetic poems of the decade. + \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect04.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect04.txt new file mode 100644 index 000000000..e69de29bb diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect.xq deleted file mode 100644 index e42bf9bc6..000000000 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect.xq +++ /dev/null @@ -1,3 +0,0 @@ -let $a := doc("src/test/resources/TestSources/xml/catalog.xml")/catalog/book[price<50] -let $b := doc("src/test/resources/TestSources/xml/catalog.xml")/catalog/book[price<40] -return $a intersect $b diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect01.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect01.xq new file mode 100644 index 000000000..36a3e38dd --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect01.xq @@ -0,0 +1,21 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +let $c := "catalog" +let $a := doc($c)/catalog/book[price<50] +let $b := doc($c)/catalog/book[price<40] +return $a intersect $b diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect02.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect02.xq new file mode 100644 index 000000000..351d50938 --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect02.xq @@ -0,0 +1,21 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +let $c := "catalog" +let $a := doc($c)/catalog/book[price<25] +let $b := doc($c)/catalog/book[price<50] +return $a intersect $b diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect03.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect03.xq new file mode 100644 index 000000000..4d55752c9 --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect03.xq @@ -0,0 +1,21 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +let $c := "catalog" +let $a := doc($c)/catalog/book[price<25] +let $b := doc($c)/catalog/book[price<50] +return $b intersect $a diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect04.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect04.xq new file mode 100644 index 000000000..4deadb416 --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect04.xq @@ -0,0 +1,21 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +let $c := "catalog" +let $a := doc($c)/catalog/book[price<0] +let $b := doc($c)/catalog/book[price<50] +return $a intersect $b diff --git a/vxquery-xtest/src/test/resources/TestSources/xml/catalog.xml b/vxquery-xtest/src/test/resources/TestSources/xml/catalog.xml index 45d8c7bc3..3e6a3dc65 100644 --- a/vxquery-xtest/src/test/resources/TestSources/xml/catalog.xml +++ b/vxquery-xtest/src/test/resources/TestSources/xml/catalog.xml @@ -37,21 +37,9 @@ Nagata, Suanne Becoming Somebody Biography - 39 + 39 A masterpiece of the fine art of gossiping. - - happy - - Ted - - - - Nye, Bill - Science - - - diff --git a/vxquery-xtest/src/test/resources/cat/SetOperatorQueries.xml b/vxquery-xtest/src/test/resources/cat/SetOperatorQueries.xml index 231f749c3..7dddecfc3 100644 --- a/vxquery-xtest/src/test/resources/cat/SetOperatorQueries.xml +++ b/vxquery-xtest/src/test/resources/cat/SetOperatorQueries.xml @@ -18,9 +18,24 @@ Set Operator Test - + Intersects two sets both with more than 1 item - - intersect.txt + + intersect01.txt + + + Left operand has 1 item, right operand has many + + intersect02.txt + + + Testing reverse of test 2 + + intersect03.txt + + + Left operand has 1 item, right operand has many + + intersect04.txt From 42576645cb91dbc040402f919343e7408da2a835 Mon Sep 17 00:00:00 2001 From: brandon Date: Wed, 16 May 2018 15:41:26 -0700 Subject: [PATCH 06/14] Merge from karen/union --- .../vxquery/functions/builtin-operators.xml | 1 + .../OpUnionScalarEvaluatorFactory.java | 166 ++++++++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpUnionScalarEvaluatorFactory.java diff --git a/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-operators.xml b/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-operators.xml index 46e06b6a1..8da386346 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-operators.xml +++ b/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-operators.xml @@ -494,6 +494,7 @@ + diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpUnionScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpUnionScalarEvaluatorFactory.java new file mode 100644 index 000000000..27ddb00fa --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpUnionScalarEvaluatorFactory.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.vxquery.runtime.functions.sequence; + +import java.io.DataOutput; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; +import org.apache.hyracks.api.context.IHyracksTaskContext; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.data.std.api.IPointable; +import org.apache.hyracks.data.std.primitive.LongPointable; +import org.apache.hyracks.data.std.primitive.UTF8StringPointable; +import org.apache.hyracks.data.std.primitive.VoidPointable; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.vxquery.context.DynamicContext; +import org.apache.vxquery.datamodel.accessors.SequencePointable; +import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; +import org.apache.vxquery.datamodel.accessors.TypedPointables; +import org.apache.vxquery.datamodel.accessors.nodes.ElementNodePointable; +import org.apache.vxquery.datamodel.accessors.nodes.NodeTreePointable; +import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; +import org.apache.vxquery.datamodel.values.ValueTag; +import org.apache.vxquery.datamodel.values.XDMConstants; +import org.apache.vxquery.exceptions.ErrorCode; +import org.apache.vxquery.exceptions.SystemException; +import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator; +import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory; +import org.apache.vxquery.runtime.functions.comparison.AbstractValueComparisonOperation; +import org.apache.vxquery.runtime.functions.comparison.ValueEqComparisonOperation; +import org.apache.vxquery.runtime.functions.comparison.general.AbstractGeneralComparisonScalarEvaluatorFactory; +import org.apache.vxquery.runtime.functions.comparison.general.GeneralEqComparisonScalarEvaluatorFactory; +import org.apache.vxquery.runtime.functions.util.FunctionHelper; + +public class OpUnionScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory { + private static final long serialVersionUID = 1L; + + public OpUnionScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) { + super(args); + } + + @Override + protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) + throws HyracksDataException { + final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); + final SequenceBuilder sb = new SequenceBuilder(); + + final SequencePointable seqleft = (SequencePointable) SequencePointable.FACTORY.createPointable(); + final SequencePointable seqright = (SequencePointable) SequencePointable.FACTORY.createPointable(); + final TaggedValuePointable tvpleft = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); + final TaggedValuePointable tvpright = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); + final TypedPointables tpleft = new TypedPointables(); + final TypedPointables tpright = new TypedPointables(); + + Map arg1_nodes = new HashMap(); + Map intersect_nodes = new HashMap(); + + return new AbstractTaggedValueArgumentScalarEvaluator(args) { + @Override + protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { + try { + abvs.reset(); + sb.reset(abvs); + TaggedValuePointable tvp1 = args[0]; + TaggedValuePointable tvp2 = args[1]; + + // If an operand has one item then it is a node tree + // If an operand has more than one item then it is a sequence + + + // Add items from the left operand into the hash map + if (tvp1.getTag() != ValueTag.SEQUENCE_TAG) { + if (tvp1.getTag() != ValueTag.NODE_TREE_TAG) { + System.out.println("Not a Sequence: " + tvp1.getTag()); + throw new SystemException(ErrorCode.FORG0006); + } + else { + // Is only one item + int nodeId = FunctionHelper.getLocalNodeId(tvp1, tpleft); + if (nodeId == -1) { + //TODO + } + arg1_nodes.put(nodeId, tvp1); + sb.addItem(tvp1); + } + } + else { + // Is a sequence, so add all the items + tvp1.getValue(seqleft); + int seqleftlen = seqleft.getEntryCount(); + for (int i = 0; i < seqleftlen; ++i) { + seqleft.getEntry(i, tvpleft); + if (tvpleft.getTag() != ValueTag.NODE_TREE_TAG) { + throw new SystemException(ErrorCode.XPTY0004); + } + int nodeId = FunctionHelper.getLocalNodeId(tvpleft, tpleft); + if (nodeId == -1) { + //TODO + } + arg1_nodes.put(nodeId, tvpleft); + sb.addItem(tvpleft); + } + } + + // Check if node IDs from right operand is in the hash map + if (tvp2.getTag() != ValueTag.SEQUENCE_TAG) { + if (tvp2.getTag() != ValueTag.NODE_TREE_TAG) { + System.out.println("Not a Sequence: " + tvp2.getTag()); + throw new SystemException(ErrorCode.FORG0006); + } + else { + // Is only one item + int nodeId = FunctionHelper.getLocalNodeId(tvp2, tpright); + if (nodeId == -1) { + //TODO + } + if (!arg1_nodes.containsKey(nodeId)) { + sb.addItem(tvp2); + } + } + } + else { + // Is a sequence, so check all the items + tvp2.getValue(seqright); + int seqrightlen = seqright.getEntryCount(); + for (int i = 0; i < seqrightlen; ++i) { + seqright.getEntry(i, tvpright); + if (tvpright.getTag() != ValueTag.NODE_TREE_TAG) { + throw new SystemException(ErrorCode.XPTY0004); + } + int nodeId = FunctionHelper.getLocalNodeId(tvpright, tpright); + if (nodeId == -1) { + //TODO + } + if (!arg1_nodes.containsKey(nodeId)) { + sb.addItem(tvpright); + } + } + } + + sb.finish(); + result.set(abvs); + } catch (IOException e) { + throw new SystemException(ErrorCode.SYSE0001); + } + } + }; + } +} From 7b65398aff3fedcb83941af786d92e2dd100f590 Mon Sep 17 00:00:00 2001 From: brandon Date: Wed, 23 May 2018 15:12:04 -0700 Subject: [PATCH 07/14] Added a file identifier to TreeNodeIdProvider. The id is based on the file URI. Changed Intersect to use this file id also. --- .../node/FnDocScalarEvaluatorFactory.java | 5 +- .../OpIntersectScalarEvaluatorFactory.java | 95 +++++++++++++------ .../OpUnionScalarEvaluatorFactory.java | 87 ++--------------- .../functions/util/FunctionHelper.java | 7 +- .../vxquery/xmlparser/TreeNodeIdProvider.java | 32 ++++++- .../translator/XMLQueryTranslator.java | 59 ++++++------ .../Queries/XQuery/Intersect/intersect05.xq | 32 +++++++ .../test/resources/cat/SetOperatorQueries.xml | 5 + 8 files changed, 178 insertions(+), 144 deletions(-) create mode 100644 vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect05.xq diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java index 2fd1755ae..2277f636b 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java @@ -17,6 +17,8 @@ package org.apache.vxquery.runtime.functions.node; import java.io.DataInputStream; +import java.util.HashMap; +import java.util.Map; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; @@ -56,7 +58,6 @@ protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvalu final ByteBufferInputStream bbis = new ByteBufferInputStream(); final DataInputStream di = new DataInputStream(bbis); final int partition = ctx.getTaskAttemptId().getTaskId().getPartition(); - final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition); final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @@ -78,8 +79,10 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S tvp.getValue(stringp); try { // Only one document should be parsed so its ok to have a unique parser. + ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition, stringp.toString()); IParser parser = new XMLParser(false, nodeIdProvider, nodeId); FunctionHelper.readInDocFromPointable(stringp, abvs, parser); + System.out.println("Reading from: " + stringp.toString()); } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java index 31adb86de..a64a5fce6 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java @@ -17,9 +17,8 @@ package org.apache.vxquery.runtime.functions.sequence; import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; +import java.util.HashSet; +import java.util.Set; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; @@ -30,7 +29,6 @@ import org.apache.vxquery.datamodel.accessors.SequencePointable; import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; import org.apache.vxquery.datamodel.accessors.TypedPointables; -import org.apache.vxquery.datamodel.accessors.nodes.NodeTreePointable; import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; import org.apache.vxquery.datamodel.values.ValueTag; import org.apache.vxquery.exceptions.ErrorCode; @@ -42,6 +40,40 @@ public class OpIntersectScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory { private static final long serialVersionUID = 1L; + class Pair { + private int rootId, localId; + + Pair(int r, int l) { + rootId = r; + localId = l; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + localId; + result = prime * result + rootId; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Pair other = (Pair) obj; + if (localId != other.localId) + return false; + if (rootId != other.rootId) + return false; + return true; + } + } + public OpIntersectScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) { super(args); } @@ -52,7 +84,6 @@ protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvalu final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final SequenceBuilder sb = new SequenceBuilder(); - final NodeTreePointable temp = (NodeTreePointable) NodeTreePointable.FACTORY.createPointable(); final SequencePointable seqleft = (SequencePointable) SequencePointable.FACTORY.createPointable(); final SequencePointable seqright = (SequencePointable) SequencePointable.FACTORY.createPointable(); final TaggedValuePointable tvpleft = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); @@ -60,7 +91,8 @@ protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvalu final TypedPointables tpleft = new TypedPointables(); final TypedPointables tpright = new TypedPointables(); - Map nodes = new HashMap(); + // a set of unique root node ids and local node ids + Set nodes = new HashSet(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override @@ -77,7 +109,8 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S } // If an operand has one item then it is a node tree - // If an operand has more than one item then it is a sequence + // If an operand has more than one item then it is a sequence + // IF an operand has 0 items then it is an empty sequence // Add items from the left operand into the hash map if (tvp1.getTag() == ValueTag.SEQUENCE_TAG) { @@ -87,21 +120,13 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S if (tvpleft.getTag() != ValueTag.NODE_TREE_TAG) { throw new SystemException(ErrorCode.XPTY0004); } - if (addItem(tvpleft, tpleft, nodes)) { - byte data[] = tvpleft.getByteArray(); - tvpleft.getValue(temp); - byte data2[] = temp.getByteArray(); - if (Arrays.equals(data, data2)) { - System.out.println("true"); - } - else { - System.out.println("false"); - } + if (!addItem(tvpleft, tpleft, nodes)) { + // TODO: What happens when local node id is -1 } } } else { if (!addItem(tvp1, tpleft, nodes)) { - + // TODO: What happens when local node id is -1 } } @@ -115,12 +140,14 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S } if (checkItem(tvpright, tpright, nodes)) { sb.addItem(tvpright); + // TODO } } } else { if (checkItem(tvp2, tpright, nodes)) { sb.addItem(tvp2); + // TODO } } @@ -138,30 +165,36 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S * Returns: False if local node id doesn't exist * True if item added successfully */ - private boolean addItem(TaggedValuePointable tvp, TypedPointables tp, Map nodes) { + private boolean addItem(TaggedValuePointable tvp, TypedPointables tp, Set nodes) { int nodeId = FunctionHelper.getLocalNodeId(tvp, tp); - System.out.println("Node ID: " + nodeId); + int rootNodeId = tp.ntp.getRootNodeId(); + System.out.println("Left Node ID: " + nodeId + " root node id: " + rootNodeId); if (nodeId == -1) { - System.out.println(new String(tvp.getByteArray())); + //TODO return false; } - nodes.put(nodeId, tvp); + nodes.add(new Pair(rootNodeId, nodeId)); return true; } - private boolean checkItem(TaggedValuePointable tvp, TypedPointables tp, Map nodes) { + /* + * Checks if node is in hash map + * Returns: False if local node id doesn't exist + * False if node isn't in hash map + * True if node is in hash map + */ + private boolean checkItem(TaggedValuePointable tvp, TypedPointables tp, Set nodes) { int nodeId = FunctionHelper.getLocalNodeId(tvp, tp); + int rootNodeId = tp.ntp.getRootNodeId(); + + System.out.println("Right Node ID: " + nodeId + " root node id: " + rootNodeId); if (nodeId == -1) { + // TODO return false; - } else if (nodes.containsKey(nodeId)) { + } else if (nodes.contains(new Pair(rootNodeId, nodeId))) { + System.out.println("Node Found!"); return true; } return false; } -} - -/* - $a := 5 union 6 - $b := a - $a intersect $b -*/ \ No newline at end of file +} \ No newline at end of file diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpUnionScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpUnionScalarEvaluatorFactory.java index 27ddb00fa..db5c9cb1c 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpUnionScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpUnionScalarEvaluatorFactory.java @@ -53,7 +53,7 @@ public class OpUnionScalarEvaluatorFactory extends AbstractTaggedValueArgumentSc private static final long serialVersionUID = 1L; public OpUnionScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) { - super(args); + super(args); } @Override @@ -61,18 +61,18 @@ protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvalu throws HyracksDataException { final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final SequenceBuilder sb = new SequenceBuilder(); - + final SequencePointable seqleft = (SequencePointable) SequencePointable.FACTORY.createPointable(); final SequencePointable seqright = (SequencePointable) SequencePointable.FACTORY.createPointable(); final TaggedValuePointable tvpleft = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); final TaggedValuePointable tvpright = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); final TypedPointables tpleft = new TypedPointables(); final TypedPointables tpright = new TypedPointables(); - + Map arg1_nodes = new HashMap(); Map intersect_nodes = new HashMap(); - - return new AbstractTaggedValueArgumentScalarEvaluator(args) { + + return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { try { @@ -80,81 +80,10 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S sb.reset(abvs); TaggedValuePointable tvp1 = args[0]; TaggedValuePointable tvp2 = args[1]; - - // If an operand has one item then it is a node tree - // If an operand has more than one item then it is a sequence - - // Add items from the left operand into the hash map - if (tvp1.getTag() != ValueTag.SEQUENCE_TAG) { - if (tvp1.getTag() != ValueTag.NODE_TREE_TAG) { - System.out.println("Not a Sequence: " + tvp1.getTag()); - throw new SystemException(ErrorCode.FORG0006); - } - else { - // Is only one item - int nodeId = FunctionHelper.getLocalNodeId(tvp1, tpleft); - if (nodeId == -1) { - //TODO - } - arg1_nodes.put(nodeId, tvp1); - sb.addItem(tvp1); - } - } - else { - // Is a sequence, so add all the items - tvp1.getValue(seqleft); - int seqleftlen = seqleft.getEntryCount(); - for (int i = 0; i < seqleftlen; ++i) { - seqleft.getEntry(i, tvpleft); - if (tvpleft.getTag() != ValueTag.NODE_TREE_TAG) { - throw new SystemException(ErrorCode.XPTY0004); - } - int nodeId = FunctionHelper.getLocalNodeId(tvpleft, tpleft); - if (nodeId == -1) { - //TODO - } - arg1_nodes.put(nodeId, tvpleft); - sb.addItem(tvpleft); - } - } - - // Check if node IDs from right operand is in the hash map - if (tvp2.getTag() != ValueTag.SEQUENCE_TAG) { - if (tvp2.getTag() != ValueTag.NODE_TREE_TAG) { - System.out.println("Not a Sequence: " + tvp2.getTag()); - throw new SystemException(ErrorCode.FORG0006); - } - else { - // Is only one item - int nodeId = FunctionHelper.getLocalNodeId(tvp2, tpright); - if (nodeId == -1) { - //TODO - } - if (!arg1_nodes.containsKey(nodeId)) { - sb.addItem(tvp2); - } - } - } - else { - // Is a sequence, so check all the items - tvp2.getValue(seqright); - int seqrightlen = seqright.getEntryCount(); - for (int i = 0; i < seqrightlen; ++i) { - seqright.getEntry(i, tvpright); - if (tvpright.getTag() != ValueTag.NODE_TREE_TAG) { - throw new SystemException(ErrorCode.XPTY0004); - } - int nodeId = FunctionHelper.getLocalNodeId(tvpright, tpright); - if (nodeId == -1) { - //TODO - } - if (!arg1_nodes.containsKey(nodeId)) { - sb.addItem(tvpright); - } - } - } - + sb.addItem(tvp1); + sb.addItem(tvp2); + sb.finish(); result.set(abvs); } catch (IOException e) { diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java index 1d66c4eb8..424181271 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java @@ -1223,12 +1223,13 @@ public static void printUTF8String(UTF8StringPointable stringp) { System.err.println(" printUTF8String END"); } - public static void readInDocFromPointable(UTF8StringPointable stringp, ArrayBackedValueStorage abvs, - IParser parser) throws IOException { + public static void readInDocFromPointable(UTF8StringPointable stringp, ArrayBackedValueStorage abvs, IParser parser) + throws IOException { readInDocFromString(stringp.toString(), abvs, parser); } - public static void readInDocFromString(String fName, ArrayBackedValueStorage abvs, IParser parser) throws IOException { + public static void readInDocFromString(String fName, ArrayBackedValueStorage abvs, IParser parser) + throws IOException { Reader input; if (!fName.contains("hdfs:/")) { File file = new File(fName); diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java index 957b91e55..614e7ea46 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java @@ -16,18 +16,27 @@ */ package org.apache.vxquery.xmlparser; +import java.util.HashMap; +import java.util.Map; + +import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; + public class TreeNodeIdProvider implements ITreeNodeIdProvider { private final short partitionDataSource; private final short dataSouceScanId; private final byte dataSourceBits; private short currentId; + private byte fileId; + private static Map Files = new HashMap(); + private static byte nextFileId = 1; public TreeNodeIdProvider(short partitionDataSource, short dataSouceScanId, short totalDataSources) { this.partitionDataSource = partitionDataSource; this.dataSouceScanId = dataSouceScanId; this.dataSourceBits = getBitsNeeded(totalDataSources); currentId = 0; + fileId = 0; } public TreeNodeIdProvider(short partition) { @@ -35,12 +44,33 @@ public TreeNodeIdProvider(short partition) { dataSouceScanId = 0; dataSourceBits = 0; currentId = 0; + fileId = 0; + } + + public TreeNodeIdProvider(short partition, String uri) { + this.partitionDataSource = partition; + dataSouceScanId = 0; + dataSourceBits = 0; + currentId = 0; + fileId = getFileId(uri); + } + + public byte getFileId(String uri) { + if (Files.containsKey(uri)) { + return Files.get(uri); + } else { + Files.put(uri, nextFileId); + return nextFileId++; + } } public int getId() { + // TODO: We only have 8 bytes for partition and datasourcescanid int p = partitionDataSource; int dssi = dataSouceScanId; - return (p << 16) | (dssi << (16 - dataSourceBits)) | currentId++; + int f = fileId; + System.out.println("File ID: " + fileId); + return (f << 24) | (p << 16) | (dssi << (16 - dataSourceBits)) | currentId++; } private byte getBitsNeeded(int number) { diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/translator/XMLQueryTranslator.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/translator/XMLQueryTranslator.java index 85ade6464..11008ee41 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/translator/XMLQueryTranslator.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/translator/XMLQueryTranslator.java @@ -778,35 +778,35 @@ private LogicalVariable translateExpression(ASTNode value, TranslationContext tC return translateQuantifiedExprNode(tCtx, qeNode); } - /* - case TYPESWITCH_EXPRESSION: { - TypeswitchExprNode teNode = (TypeswitchExprNode) value; - Expression sExpr = translateExpression(teNode.getSwitchExpr()); - ForLetVariable tVar = new ForLetVariable(VarTag.LET, createVarName(), sExpr); - tVar.setDeclaredStaticType(SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR)); + /* + case TYPESWITCH_EXPRESSION: { + TypeswitchExprNode teNode = (TypeswitchExprNode) value; + Expression sExpr = translateExpression(teNode.getSwitchExpr()); + ForLetVariable tVar = new ForLetVariable(VarTag.LET, createVarName(), sExpr); + tVar.setDeclaredStaticType(SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR)); + pushVariableScope(); + varScope.registerVariable(tVar); + List cases = new ArrayList(); + for (CaseClauseNode ccNode : teNode.getCaseClauses()) { + SequenceType type = createSequenceType(ccNode.getType()); + ForLetVariable caseVar = null; pushVariableScope(); - varScope.registerVariable(tVar); - List cases = new ArrayList(); - for (CaseClauseNode ccNode : teNode.getCaseClauses()) { - SequenceType type = createSequenceType(ccNode.getType()); - ForLetVariable caseVar = null; - pushVariableScope(); - if (ccNode.getCaseVar() != null) { - caseVar = new ForLetVariable(VarTag.LET, createQName(ccNode.getCaseVar()), new TreatExpression( - currCtx, new VariableReferenceExpression(currCtx, tVar), type)); - caseVar.setDeclaredStaticType(type); - varScope.registerVariable(caseVar); - } - Expression cExpr = translateExpression(ccNode.getValueExpr()); - TypeswitchExpression.Case c = new TypeswitchExpression.Case(caseVar, type, cExpr); - cases.add(c); - popVariableScope(); + if (ccNode.getCaseVar() != null) { + caseVar = new ForLetVariable(VarTag.LET, createQName(ccNode.getCaseVar()), new TreatExpression( + currCtx, new VariableReferenceExpression(currCtx, tVar), type)); + caseVar.setDeclaredStaticType(type); + varScope.registerVariable(caseVar); } - Expression dExpr = translateExpression(teNode.getDefaultClause()); + Expression cExpr = translateExpression(ccNode.getValueExpr()); + TypeswitchExpression.Case c = new TypeswitchExpression.Case(caseVar, type, cExpr); + cases.add(c); popVariableScope(); - return new TypeswitchExpression(currCtx, tVar, cases, dExpr); } - */ + Expression dExpr = translateExpression(teNode.getDefaultClause()); + popVariableScope(); + return new TypeswitchExpression(currCtx, tVar, cases, dExpr); + } + */ case COMPUTED_TEXT_CONSTRUCTOR: { ComputedTextConstructorNode cNode = (ComputedTextConstructorNode) value; @@ -974,7 +974,8 @@ private LogicalVariable translateValidateExprNode(TranslationContext tCtx, Valid throws SystemException { XQueryConstants.ValidationMode mode = vNode.getMode(); Function fn = mode == null || XQueryConstants.ValidationMode.STRICT.equals(mode) - ? BuiltinOperators.VALIDATE_STRICT : BuiltinOperators.VALIDATE_LAX; + ? BuiltinOperators.VALIDATE_STRICT + : BuiltinOperators.VALIDATE_LAX; LogicalVariable lVar = createAssignment(sfce(fn, vre(translateExpression(vNode.getExpr(), tCtx))), tCtx); return lVar; } @@ -1545,9 +1546,9 @@ private LogicalVariable translateInfixExprNode(TranslationContext tCtx, InfixExp } */ ILogicalExpression result = sfce(operator, arg1, arg2); - if (BuiltinOperators.UNION.equals(operator)) { - result = sfce(BuiltinOperators.SORT_DISTINCT_NODES_ASC, result); - } + // if (BuiltinOperators.UNION.equals(operator)) { + // result = sfce(BuiltinOperators.SORT_DISTINCT_NODES_ASC, result); + // } LogicalVariable lVar = createAssignment(result, tCtx); return lVar; } diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect05.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect05.xq new file mode 100644 index 000000000..046942f00 --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect05.xq @@ -0,0 +1,32 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +let $collection := "ghcnd" +let $a:=(for $i in collection($collection)/dataCollection/data +return $i) + +where $j/value le 20 +return $j) + + + + + + +return $a + + diff --git a/vxquery-xtest/src/test/resources/cat/SetOperatorQueries.xml b/vxquery-xtest/src/test/resources/cat/SetOperatorQueries.xml index 7dddecfc3..e71b91268 100644 --- a/vxquery-xtest/src/test/resources/cat/SetOperatorQueries.xml +++ b/vxquery-xtest/src/test/resources/cat/SetOperatorQueries.xml @@ -37,5 +37,10 @@ Left operand has 1 item, right operand has many intersect04.txt + + + Collection + + intersect05.txt From 53b22c3818419b670281d1146eba6b11c1538b64 Mon Sep 17 00:00:00 2001 From: brandon Date: Wed, 23 May 2018 15:14:53 -0700 Subject: [PATCH 08/14] Cleaned up print statements --- .../runtime/functions/node/FnDocScalarEvaluatorFactory.java | 1 - .../functions/sequence/OpIntersectScalarEvaluatorFactory.java | 3 --- .../java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java | 1 - 3 files changed, 5 deletions(-) diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java index 2277f636b..f4e8f29ec 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java @@ -82,7 +82,6 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition, stringp.toString()); IParser parser = new XMLParser(false, nodeIdProvider, nodeId); FunctionHelper.readInDocFromPointable(stringp, abvs, parser); - System.out.println("Reading from: " + stringp.toString()); } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java index a64a5fce6..05851cbab 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java @@ -168,7 +168,6 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S private boolean addItem(TaggedValuePointable tvp, TypedPointables tp, Set nodes) { int nodeId = FunctionHelper.getLocalNodeId(tvp, tp); int rootNodeId = tp.ntp.getRootNodeId(); - System.out.println("Left Node ID: " + nodeId + " root node id: " + rootNodeId); if (nodeId == -1) { //TODO return false; @@ -187,12 +186,10 @@ private boolean checkItem(TaggedValuePointable tvp, TypedPointables tp, Set Date: Wed, 30 May 2018 15:12:12 -0700 Subject: [PATCH 09/14] Collections are given a unique file ID. --- .../VXQueryCollectionOperatorDescriptor.java | 4 ++- .../OpIntersectScalarEvaluatorFactory.java | 25 ++++++++++++------- .../vxquery/xmlparser/TreeNodeIdProvider.java | 12 +++++++-- .../Queries/XQuery/Intersect/intersect05.xq | 15 ++++++----- 4 files changed, 36 insertions(+), 20 deletions(-) diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java index a3756d5b4..86736d09c 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java @@ -74,6 +74,7 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO private static final long serialVersionUID = 1L; private short dataSourceId; private short totalDataSources; + private final String collectionId; private String[] collectionPartitions; private List childSeq; private List valueSeq; @@ -90,6 +91,7 @@ public VXQueryCollectionOperatorDescriptor(IOperatorDescriptorRegistry spec, Abs collectionPartitions = ds.getPartitions(); dataSourceId = (short) ds.getDataSourceId(); totalDataSources = (short) ds.getTotalDataSources(); + collectionId = ds.getId(); childSeq = ds.getChildSeq(); valueSeq = ds.getValueSeq(); recordDescriptors[0] = rDesc; @@ -107,7 +109,7 @@ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, final IFrame frame = new VSizeFrame(ctx); final IFrameFieldAppender appender = new FrameFixedFieldTupleAppender(fieldOutputCount); final short partitionId = (short) ctx.getTaskAttemptId().getTaskId().getPartition(); - final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider(partitionId, dataSourceId, totalDataSources); + final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider(partitionId, dataSourceId, totalDataSources, collectionId); final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId(); final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); final ArrayBackedValueStorage jsonAbvs = new ArrayBackedValueStorage(); diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java index 05851cbab..ac36bbbaf 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java @@ -41,10 +41,11 @@ public class OpIntersectScalarEvaluatorFactory extends AbstractTaggedValueArgume private static final long serialVersionUID = 1L; class Pair { - private int rootId, localId; + private int localId; + private byte fileId; - Pair(int r, int l) { - rootId = r; + Pair(int l, byte f) { + fileId = f; localId = l; } @@ -53,7 +54,7 @@ public int hashCode() { final int prime = 31; int result = 1; result = prime * result + localId; - result = prime * result + rootId; + result = prime * result + fileId; return result; } @@ -68,7 +69,7 @@ public boolean equals(Object obj) { Pair other = (Pair) obj; if (localId != other.localId) return false; - if (rootId != other.rootId) + if (fileId != other.fileId) return false; return true; } @@ -168,11 +169,15 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S private boolean addItem(TaggedValuePointable tvp, TypedPointables tp, Set nodes) { int nodeId = FunctionHelper.getLocalNodeId(tvp, tp); int rootNodeId = tp.ntp.getRootNodeId(); + byte fileId = (byte) (rootNodeId >> 24); // TODO: Magic number if (nodeId == -1) { - //TODO + // TODO + return false; + } else if (rootNodeId == -1) { + // TODO return false; } - nodes.add(new Pair(rootNodeId, nodeId)); + nodes.add(new Pair(nodeId, fileId)); return true; } @@ -185,11 +190,13 @@ private boolean addItem(TaggedValuePointable tvp, TypedPointables tp, Set private boolean checkItem(TaggedValuePointable tvp, TypedPointables tp, Set nodes) { int nodeId = FunctionHelper.getLocalNodeId(tvp, tp); int rootNodeId = tp.ntp.getRootNodeId(); - + byte fileId = (byte) (rootNodeId >> 24); // TODO: Magic number if (nodeId == -1) { // TODO return false; - } else if (nodes.contains(new Pair(rootNodeId, nodeId))) { + } else if (rootNodeId == -1) { + + } else if (nodes.contains(new Pair(nodeId, fileId))) { return true; } return false; diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java index adae7e17b..00e707ed5 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java @@ -36,7 +36,15 @@ public TreeNodeIdProvider(short partitionDataSource, short dataSouceScanId, shor this.dataSouceScanId = dataSouceScanId; this.dataSourceBits = getBitsNeeded(totalDataSources); currentId = 0; - fileId = 0; + fileId = nextFileId++; + } + + public TreeNodeIdProvider(short partitionDataSource, short dataSouceScanId, short totalDataSources, String collectionId) { + this.partitionDataSource = partitionDataSource; + this.dataSouceScanId = dataSouceScanId; + this.dataSourceBits = getBitsNeeded(totalDataSources); + currentId = 0; + fileId = getFileId(collectionId); } public TreeNodeIdProvider(short partition) { @@ -44,7 +52,7 @@ public TreeNodeIdProvider(short partition) { dataSouceScanId = 0; dataSourceBits = 0; currentId = 0; - fileId = 0; + fileId = nextFileId++; } public TreeNodeIdProvider(short partition, String uri) { diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect05.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect05.xq index 046942f00..e17b47954 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect05.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect05.xq @@ -15,18 +15,17 @@ specific language governing permissions and limitations under the License. :) -let $collection := "ghcnd" -let $a:=(for $i in collection($collection)/dataCollection/data -return $i) +let $a := "ghcnd" +for $i in collection($a)/dataCollection/data -where $j/value le 20 -return $j) +let $b := "ghcnd" +for $j in collection($b)/dataCollection/data +where $i/value < 10 +and $j/value < 20 +return $i intersect $j -return $a - - From 255932e2254845e94d053b578c392f0c0faf5467 Mon Sep 17 00:00:00 2001 From: brandon Date: Thu, 7 Jun 2018 11:10:04 -0700 Subject: [PATCH 10/14] File id for collection files work in some cases. Need to explore other cases (HDFS). --- .../metadata/VXQueryCollectionOperatorDescriptor.java | 10 ++++++---- .../sequence/OpIntersectScalarEvaluatorFactory.java | 10 +++++++--- .../apache/vxquery/xmlparser/TreeNodeIdProvider.java | 4 ++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java index 86736d09c..471b920df 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java @@ -74,7 +74,6 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO private static final long serialVersionUID = 1L; private short dataSourceId; private short totalDataSources; - private final String collectionId; private String[] collectionPartitions; private List childSeq; private List valueSeq; @@ -91,7 +90,6 @@ public VXQueryCollectionOperatorDescriptor(IOperatorDescriptorRegistry spec, Abs collectionPartitions = ds.getPartitions(); dataSourceId = (short) ds.getDataSourceId(); totalDataSources = (short) ds.getTotalDataSources(); - collectionId = ds.getId(); childSeq = ds.getChildSeq(); valueSeq = ds.getValueSeq(); recordDescriptors[0] = rDesc; @@ -109,7 +107,7 @@ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, final IFrame frame = new VSizeFrame(ctx); final IFrameFieldAppender appender = new FrameFixedFieldTupleAppender(fieldOutputCount); final short partitionId = (short) ctx.getTaskAttemptId().getTaskId().getPartition(); - final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider(partitionId, dataSourceId, totalDataSources, collectionId); + final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider(partitionId, dataSourceId, totalDataSources); final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId(); final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); final ArrayBackedValueStorage jsonAbvs = new ArrayBackedValueStorage(); @@ -227,6 +225,7 @@ public void nextFrame(ByteBuffer buffer) throws HyracksDataException { // file currently reading and // send it to parser InputStream in = fs.open(xmlDocument).getWrappedStream(); + parser.parseHDFSElements(in, writer, fta, tupleIndex); in.close(); } @@ -261,7 +260,10 @@ public void xmlAndJsonCollection(File directory) throws HyracksDataException { if (LOGGER.isLoggable(Level.FINE)) { LOGGER.fine("Starting to read XML document: " + file.getAbsolutePath()); } - parser.parseElements(file, writer, tupleIndex); + ITreeNodeIdProvider myNodeIdProvider = new TreeNodeIdProvider(partitionId, dataSourceId, totalDataSources, file.getAbsolutePath()); + XMLParser myparser = new XMLParser(false, myNodeIdProvider, nodeId, appender, childSeq, + dCtx.getStaticContext()); + myparser.parseElements(file, writer, tupleIndex); } else if (fileName.endsWith(".json")) { if (LOGGER.isLoggable(Level.FINE)) { LOGGER.fine("Starting to read JSON document: " + file.getAbsolutePath()); diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java index ac36bbbaf..d186361bd 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java @@ -45,8 +45,8 @@ class Pair { private byte fileId; Pair(int l, byte f) { - fileId = f; localId = l; + fileId = f; } @Override @@ -71,6 +71,8 @@ public boolean equals(Object obj) { return false; if (fileId != other.fileId) return false; + System.out.println("My Equals"); + System.out.flush(); return true; } } @@ -170,11 +172,12 @@ private boolean addItem(TaggedValuePointable tvp, TypedPointables tp, Set int nodeId = FunctionHelper.getLocalNodeId(tvp, tp); int rootNodeId = tp.ntp.getRootNodeId(); byte fileId = (byte) (rootNodeId >> 24); // TODO: Magic number + System.out.println("Left Node ID: " + nodeId + " root node id: " + rootNodeId + " file id: " + fileId); if (nodeId == -1) { // TODO return false; } else if (rootNodeId == -1) { - // TODO + // TODO If rootNodeID is -1 is nodeId also -1? return false; } nodes.add(new Pair(nodeId, fileId)); @@ -191,11 +194,12 @@ private boolean checkItem(TaggedValuePointable tvp, TypedPointables tp, Set> 24); // TODO: Magic number + System.out.println("Right Node ID: " + nodeId + " root node id: " + rootNodeId + " file id: " + fileId); if (nodeId == -1) { // TODO return false; } else if (rootNodeId == -1) { - + // TODO } else if (nodes.contains(new Pair(nodeId, fileId))) { return true; } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java index 00e707ed5..1aedf2374 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java @@ -39,12 +39,12 @@ public TreeNodeIdProvider(short partitionDataSource, short dataSouceScanId, shor fileId = nextFileId++; } - public TreeNodeIdProvider(short partitionDataSource, short dataSouceScanId, short totalDataSources, String collectionId) { + public TreeNodeIdProvider(short partitionDataSource, short dataSouceScanId, short totalDataSources, String uri) { this.partitionDataSource = partitionDataSource; this.dataSouceScanId = dataSouceScanId; this.dataSourceBits = getBitsNeeded(totalDataSources); currentId = 0; - fileId = getFileId(collectionId); + fileId = getFileId(uri); } public TreeNodeIdProvider(short partition) { From e1c108bd3dfafff5af97256f12112ae78b17797c Mon Sep 17 00:00:00 2001 From: brandon Date: Mon, 25 Jun 2018 13:03:03 -0700 Subject: [PATCH 11/14] Gave inline query xml elements unique ids. --- ...bstractNodeConstructorScalarEvaluator.java | 20 ++++- ...tributeNodeConstructorScalarEvaluator.java | 5 ++ ...CommentNodeConstructorScalarEvaluator.java | 5 ++ ...ElementNodeConstructorScalarEvaluator.java | 83 ++++++++++++++++++- .../PINodeConstructorScalarEvaluator.java | 5 ++ .../TextNodeConstructorScalarEvaluator.java | 5 ++ .../vxquery/xmlparser/TreeNodeIdProvider.java | 2 +- .../Queries/XQuery/Intersect/intersect05.xq | 9 +- .../resources/TestSources/xml/catalog.xml | 29 +------ 9 files changed, 129 insertions(+), 34 deletions(-) diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/AbstractNodeConstructorScalarEvaluator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/AbstractNodeConstructorScalarEvaluator.java index dba35946e..6c0c01af2 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/AbstractNodeConstructorScalarEvaluator.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/AbstractNodeConstructorScalarEvaluator.java @@ -26,7 +26,8 @@ import org.apache.vxquery.exceptions.ErrorCode; import org.apache.vxquery.exceptions.SystemException; import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator; - +import org.apache.vxquery.xmlparser.ITreeNodeIdProvider; +import org.apache.vxquery.xmlparser.TreeNodeIdProvider; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.data.std.api.IMutableValueStorage; @@ -42,12 +43,21 @@ public abstract class AbstractNodeConstructorScalarEvaluator extends AbstractTag private final ArrayBackedValueStorage contentAbvs; + protected final ITreeNodeIdProvider nodeIdProvider; + protected int nodeIdCounter; + public AbstractNodeConstructorScalarEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) { super(args); this.ctx = ctx; abvs = new ArrayBackedValueStorage(); db = createsDictionary() ? new DictionaryBuilder() : null; contentAbvs = createsDictionary() ? new ArrayBackedValueStorage() : abvs; + if (createsNodeId()) { + nodeIdProvider = new TreeNodeIdProvider((short) 0); + } else { + nodeIdProvider = null; + } + nodeIdCounter = 0; } @Override @@ -58,7 +68,13 @@ protected final void evaluate(TaggedValuePointable[] args, IPointable result) th DataOutput mainOut = abvs.getDataOutput(); mainOut.write(ValueTag.NODE_TREE_TAG); byte header = (byte) (createsDictionary() ? NodeTreePointable.HEADER_DICTIONARY_EXISTS_MASK : 0); + + header |= (byte) (createsNodeId() ? NodeTreePointable.HEADER_NODEID_EXISTS_MASK : 0); mainOut.write(header); + + if (nodeIdProvider != null) { + mainOut.writeInt(nodeIdProvider.getId()); + } constructNode(db, args, contentAbvs); if (createsDictionary()) { db.write(abvs); @@ -74,4 +90,6 @@ protected abstract void constructNode(DictionaryBuilder db, TaggedValuePointable throws IOException, SystemException; protected abstract boolean createsDictionary(); + + protected abstract boolean createsNodeId(); } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/AttributeNodeConstructorScalarEvaluator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/AttributeNodeConstructorScalarEvaluator.java index 39216f4cb..a41e29b09 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/AttributeNodeConstructorScalarEvaluator.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/AttributeNodeConstructorScalarEvaluator.java @@ -70,4 +70,9 @@ protected void constructNode(DictionaryBuilder db, TaggedValuePointable[] args, protected boolean createsDictionary() { return true; } + + @Override + protected boolean createsNodeId() { + return false; + } } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/CommentNodeConstructorScalarEvaluator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/CommentNodeConstructorScalarEvaluator.java index 917600a3d..7317d955d 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/CommentNodeConstructorScalarEvaluator.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/CommentNodeConstructorScalarEvaluator.java @@ -58,4 +58,9 @@ protected void constructNode(DictionaryBuilder db, TaggedValuePointable[] args, protected boolean createsDictionary() { return false; } + + @Override + protected boolean createsNodeId() { + return false; + } } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/ElementNodeConstructorScalarEvaluator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/ElementNodeConstructorScalarEvaluator.java index cfcdd1358..b5bc2bca3 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/ElementNodeConstructorScalarEvaluator.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/ElementNodeConstructorScalarEvaluator.java @@ -39,7 +39,8 @@ import org.apache.vxquery.datamodel.values.ValueTag; import org.apache.vxquery.exceptions.ErrorCode; import org.apache.vxquery.exceptions.SystemException; - +import org.apache.vxquery.xmlparser.ITreeNodeIdProvider; +import org.apache.vxquery.xmlparser.TreeNodeIdProvider; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.data.std.api.IMutableValueStorage; @@ -71,6 +72,8 @@ public class ElementNodeConstructorScalarEvaluator extends AbstractNodeConstruct private final IMutableValueStorage abvs; + private final boolean createNodeIds; + public ElementNodeConstructorScalarEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) { super(ctx, args); anb = new AttributeNodeBuilder(); @@ -84,6 +87,8 @@ public ElementNodeConstructorScalarEvaluator(IHyracksTaskContext ctx, IScalarEva cqp = (CodedQNamePointable) CodedQNamePointable.FACTORY.createPointable(); strp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable(); seqp = (SequencePointable) SequencePointable.FACTORY.createPointable(); + + createNodeIds = nodeIdProvider != null; } @Override @@ -102,6 +107,10 @@ protected void constructNode(DictionaryBuilder db, TaggedValuePointable[] args, namep.getLocalName(strp); int localCode = db.lookup(strp); enb.setName(uriCode, localCode, prefixCode); + + if (createNodeIds) + enb.setLocalNodeId(nodeIdCounter++); + TaggedValuePointable valueArg = args[1]; enb.startAttributeChunk(); int index = processAttributes(valueArg, db); @@ -173,6 +182,17 @@ private void copyAttribute(ElementNodeBuilder enb, DictionaryBuilder db, NodeTre int newPrefixCode = recode(cqp.getPrefixCode(), ntp, db, strp); int newLocalCode = recode(cqp.getLocalCode(), ntp, db, strp); anb.setName(newURICode, newLocalCode, newPrefixCode); + + if (createNodeIds) { + int localId = -1; + if (ntp.nodeIdExists()) { + localId = anp.getLocalNodeId(ntp); + } else { + localId = nodeIdCounter++; + } + anb.setLocalNodeId(localId); + } + anp.getValue(ntp, vp); anb.setValue(vp); enb.endAttribute(anb); @@ -198,6 +218,17 @@ private void copyElement(ElementNodeBuilder enb, DictionaryBuilder db, NodeTreeP int newPrefixCode = recode(cqp.getPrefixCode(), ntp, db, strp); int newLocalCode = recode(cqp.getLocalCode(), ntp, db, strp); tempEnb.setName(newURICode, newLocalCode, newPrefixCode); + + if (createNodeIds) { + int localId = -1; + if (ntp.nodeIdExists()) { + localId = enp.getLocalNodeId(ntp); + } else { + localId = nodeIdCounter++; + } + tempEnb.setLocalNodeId(localId); + } + tempEnb.startAttributeChunk(); if (enp.attributesChunkExists()) { enp.getAttributeSequence(ntp, seqp); @@ -286,8 +317,8 @@ private int recode(int oldCode, NodeTreePointable ntp, DictionaryBuilder db, UTF return db.lookup(tempStrp); } - private void processChildren(TaggedValuePointable tvp, int start, DictionaryBuilder db) throws IOException, - SystemException { + private void processChildren(TaggedValuePointable tvp, int start, DictionaryBuilder db) + throws IOException, SystemException { if (tvp.getTag() == ValueTag.SEQUENCE_TAG) { tvp.getValue(seqp); TaggedValuePointable tempTvp = ppool.takeOne(TaggedValuePointable.class); @@ -379,6 +410,11 @@ protected boolean createsDictionary() { return true; } + @Override + protected boolean createsNodeId() { + return true; + } + private void copyComment(TaggedValuePointable tvp, NodeTreePointable ntp, IMutableValueStorage mvs) throws IOException { VoidPointable vp = ppool.takeOne(VoidPointable.class); @@ -388,6 +424,17 @@ private void copyComment(TaggedValuePointable tvp, NodeTreePointable ntp, IMutab tcnp.getValue(ntp, vp); cnb.reset(mvs); + + if (createNodeIds) { + int localId = -1; + if (ntp.nodeIdExists()) { + localId = tcnp.getLocalNodeId(ntp); + } else { + localId = nodeIdCounter++; + } + cnb.setLocalNodeId(localId); + } + cnb.setValue(vp); ppool.giveBack(vp); @@ -404,6 +451,17 @@ private void copyPI(TaggedValuePointable tvp, NodeTreePointable ntp, IMutableVal pnp.getTarget(ntp, vp2); pnb.reset(mvs); + + if (createNodeIds) { + int localId = -1; + if (ntp.nodeIdExists()) { + localId = pnp.getLocalNodeId(ntp); + } else { + localId = nodeIdCounter++; + } + pnb.setLocalNodeId(localId); + } + pnb.setContent(vp2); pnb.setTarget(vp1); @@ -412,13 +470,25 @@ private void copyPI(TaggedValuePointable tvp, NodeTreePointable ntp, IMutableVal ppool.giveBack(vp2); } - private void copyText(TaggedValuePointable tvp, NodeTreePointable ntp, IMutableValueStorage mvs) throws IOException { + private void copyText(TaggedValuePointable tvp, NodeTreePointable ntp, IMutableValueStorage mvs) + throws IOException { VoidPointable vp = ppool.takeOne(VoidPointable.class); TextOrCommentNodePointable tcnp = ppool.takeOne(TextOrCommentNodePointable.class); tvp.getValue(tcnp); tcnp.getValue(ntp, vp); tnb.reset(mvs); + + if (createNodeIds) { + int localId = -1; + if (ntp.nodeIdExists()) { + localId = tcnp.getLocalNodeId(ntp); + } else { + localId = nodeIdCounter++; + } + tnb.setLocalNodeId(localId); + } + tnb.setValue(vp); ppool.giveBack(vp); @@ -431,6 +501,11 @@ private void convertToText(TaggedValuePointable tvp, IMutableValueStorage mvs) t TextNodeBuilder tnb = new TextNodeBuilder(); tvp.getValue(vp); tnb.reset(mvs); + + if (createNodeIds) { + tnb.setLocalNodeId(nodeIdCounter++); + } + tnb.setValue(vp); ppool.giveBack(vp); diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/PINodeConstructorScalarEvaluator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/PINodeConstructorScalarEvaluator.java index 839b79b1e..69248afb6 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/PINodeConstructorScalarEvaluator.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/PINodeConstructorScalarEvaluator.java @@ -56,4 +56,9 @@ protected void constructNode(DictionaryBuilder db, TaggedValuePointable[] args, protected boolean createsDictionary() { return false; } + + @Override + protected boolean createsNodeId() { + return false; + } } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/TextNodeConstructorScalarEvaluator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/TextNodeConstructorScalarEvaluator.java index 7d69a14fc..3f5e18ecf 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/TextNodeConstructorScalarEvaluator.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/TextNodeConstructorScalarEvaluator.java @@ -58,4 +58,9 @@ protected void constructNode(DictionaryBuilder db, TaggedValuePointable[] args, protected boolean createsDictionary() { return false; } + + @Override + protected boolean createsNodeId() { + return false; + } } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java index 1aedf2374..104626286 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java @@ -38,7 +38,7 @@ public TreeNodeIdProvider(short partitionDataSource, short dataSouceScanId, shor currentId = 0; fileId = nextFileId++; } - + public TreeNodeIdProvider(short partitionDataSource, short dataSouceScanId, short totalDataSources, String uri) { this.partitionDataSource = partitionDataSource; this.dataSouceScanId = dataSouceScanId; diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect05.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect05.xq index e17b47954..47cc7c92f 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect05.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect05.xq @@ -14,7 +14,8 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. :) - + + (: let $a := "ghcnd" for $i in collection($a)/dataCollection/data @@ -26,6 +27,12 @@ where $i/value < 10 and $j/value < 20 return $i intersect $j +:) + +let $a := 5 +return $a intersect $a + + diff --git a/vxquery-xtest/src/test/resources/TestSources/xml/catalog.xml b/vxquery-xtest/src/test/resources/TestSources/xml/catalog.xml index 3e6a3dc65..fdaa37b02 100644 --- a/vxquery-xtest/src/test/resources/TestSources/xml/catalog.xml +++ b/vxquery-xtest/src/test/resources/TestSources/xml/catalog.xml @@ -1,4 +1,4 @@ - + - - - Hightower, Kim - The First Book - Fiction - 44.95 - 2000-10-01 - An amazing story of nothing. - - - - Oberg, Bruce - The Poet's First Poem - Poem - 24.95 - The least poetic poems of the decade. - - - - Nagata, Suanne - Becoming Somebody - Biography - 39 - A masterpiece of the fine art of gossiping. - - +5 From 92e38ea5038f761e8cbb4d84243804fd46381906 Mon Sep 17 00:00:00 2001 From: brandon Date: Thu, 28 Jun 2018 18:26:15 -0700 Subject: [PATCH 12/14] Elemennt-constructor function no longer gets inlined. This is because when an element is constructed it needs a unqiue ID. --- .../compiler/rewriter/RewriteRuleset.java | 6 ++-- .../InlineNestedVariablesRule.java | 6 ++-- .../VXQueryInlineVariablesRule.java | 29 +++++++++++++++++++ 3 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/algebricksalternatives/VXQueryInlineVariablesRule.java diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java index aa1dd551a..54a466c74 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java @@ -34,7 +34,6 @@ import org.apache.hyracks.algebricks.rewriter.rules.FactorRedundantGroupAndDecorVarsRule; import org.apache.hyracks.algebricks.rewriter.rules.InferTypesRule; import org.apache.hyracks.algebricks.rewriter.rules.InlineAssignIntoAggregateRule; -import org.apache.hyracks.algebricks.rewriter.rules.InlineVariablesRule; import org.apache.hyracks.algebricks.rewriter.rules.IntroJoinInsideSubplanRule; import org.apache.hyracks.algebricks.rewriter.rules.IntroduceAggregateCombinerRule; import org.apache.hyracks.algebricks.rewriter.rules.IntroduceGroupByCombinerRule; @@ -92,6 +91,7 @@ import org.apache.vxquery.compiler.rewriter.rules.algebricksalternatives.ExtractFunctionsFromJoinConditionRule; import org.apache.vxquery.compiler.rewriter.rules.algebricksalternatives.InlineNestedVariablesRule; import org.apache.vxquery.compiler.rewriter.rules.algebricksalternatives.MoveFreeVariableOperatorOutOfSubplanRule; +import org.apache.vxquery.compiler.rewriter.rules.algebricksalternatives.VXQueryInlineVariablesRule; public class RewriteRuleset { RewriteRuleset() { @@ -272,7 +272,7 @@ public static final List buildNormalizationRuleCollection public static final List buildCondPushDownRuleCollection() { List condPushDown = new LinkedList<>(); condPushDown.add(new PushSelectDownRule()); - condPushDown.add(new InlineVariablesRule()); + condPushDown.add(new VXQueryInlineVariablesRule()); condPushDown.add(new SubplanOutOfGroupRule()); condPushDown.add(new RemoveRedundantVariablesRule()); condPushDown.add(new RemoveUnusedAssignAndAggregateRule()); @@ -289,7 +289,7 @@ public static final List buildCondPushDownRuleCollection( public static final List buildJoinInferenceRuleCollection() { List joinInference = new LinkedList<>(); - joinInference.add(new InlineVariablesRule()); + joinInference.add(new VXQueryInlineVariablesRule()); joinInference.add(new ComplexJoinInferenceRule()); return joinInference; } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/algebricksalternatives/InlineNestedVariablesRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/algebricksalternatives/InlineNestedVariablesRule.java index 16e3e9877..32091a713 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/algebricksalternatives/InlineNestedVariablesRule.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/algebricksalternatives/InlineNestedVariablesRule.java @@ -29,16 +29,18 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; +import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator; import org.apache.hyracks.algebricks.rewriter.rules.InlineVariablesRule; +import org.apache.vxquery.functions.BuiltinOperators; /** * Modifies the InlineVariablesRule to also process nested plans. */ -public class InlineNestedVariablesRule extends InlineVariablesRule { - +public class InlineNestedVariablesRule extends VXQueryInlineVariablesRule { + protected boolean inlineVariables(Mutable opRef, IOptimizationContext context) throws AlgebricksException { AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/algebricksalternatives/VXQueryInlineVariablesRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/algebricksalternatives/VXQueryInlineVariablesRule.java new file mode 100644 index 000000000..3f793daa7 --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/algebricksalternatives/VXQueryInlineVariablesRule.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.vxquery.compiler.rewriter.rules.algebricksalternatives; + +import org.apache.hyracks.algebricks.rewriter.rules.InlineVariablesRule; +import org.apache.vxquery.functions.BuiltinOperators; + +// VXQuery implementation of InlineVariablesRule to specify functions we should not inline +public class VXQueryInlineVariablesRule extends InlineVariablesRule { + + public VXQueryInlineVariablesRule() { + // Ignore element constructor because we need to assign each instance a unique ID + doNotInlineFuncs.add(BuiltinOperators.ELEMENT_CONSTRUCTOR.getFunctionIdentifier()); + } +} From 4be5b50ef839fbdc84e431886f9bcccd1742be4b Mon Sep 17 00:00:00 2001 From: brandon Date: Wed, 11 Jul 2018 17:41:22 -0700 Subject: [PATCH 13/14] Optimized the implementation of ids for the element-constructor. Fixed duplicates showing in final result. Cleaned up intersect implementation. --- ...bstractNodeConstructorScalarEvaluator.java | 8 +- ...ElementNodeConstructorScalarEvaluator.java | 42 ++------- .../OpIntersectScalarEvaluatorFactory.java | 90 +++++++------------ .../vxquery/xmlparser/TreeNodeIdProvider.java | 13 +++ 4 files changed, 55 insertions(+), 98 deletions(-) diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/AbstractNodeConstructorScalarEvaluator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/AbstractNodeConstructorScalarEvaluator.java index 6c0c01af2..0a8d82da0 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/AbstractNodeConstructorScalarEvaluator.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/AbstractNodeConstructorScalarEvaluator.java @@ -35,6 +35,8 @@ import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; public abstract class AbstractNodeConstructorScalarEvaluator extends AbstractTaggedValueArgumentScalarEvaluator { + protected final static ITreeNodeIdProvider NodeConstructorIdProvider = new TreeNodeIdProvider((short) 0, true); + protected final IHyracksTaskContext ctx; private final ArrayBackedValueStorage abvs; @@ -44,7 +46,6 @@ public abstract class AbstractNodeConstructorScalarEvaluator extends AbstractTag private final ArrayBackedValueStorage contentAbvs; protected final ITreeNodeIdProvider nodeIdProvider; - protected int nodeIdCounter; public AbstractNodeConstructorScalarEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) { super(args); @@ -53,11 +54,10 @@ public AbstractNodeConstructorScalarEvaluator(IHyracksTaskContext ctx, IScalarEv db = createsDictionary() ? new DictionaryBuilder() : null; contentAbvs = createsDictionary() ? new ArrayBackedValueStorage() : abvs; if (createsNodeId()) { - nodeIdProvider = new TreeNodeIdProvider((short) 0); + nodeIdProvider = NodeConstructorIdProvider; } else { nodeIdProvider = null; } - nodeIdCounter = 0; } @Override @@ -69,7 +69,7 @@ protected final void evaluate(TaggedValuePointable[] args, IPointable result) th mainOut.write(ValueTag.NODE_TREE_TAG); byte header = (byte) (createsDictionary() ? NodeTreePointable.HEADER_DICTIONARY_EXISTS_MASK : 0); - header |= (byte) (createsNodeId() ? NodeTreePointable.HEADER_NODEID_EXISTS_MASK : 0); + header |= (byte) (nodeIdProvider != null ? NodeTreePointable.HEADER_NODEID_EXISTS_MASK : 0); mainOut.write(header); if (nodeIdProvider != null) { diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/ElementNodeConstructorScalarEvaluator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/ElementNodeConstructorScalarEvaluator.java index b5bc2bca3..e7825d1e4 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/ElementNodeConstructorScalarEvaluator.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/ElementNodeConstructorScalarEvaluator.java @@ -74,6 +74,8 @@ public class ElementNodeConstructorScalarEvaluator extends AbstractNodeConstruct private final boolean createNodeIds; + private static int nodeIdCounter = 0; + public ElementNodeConstructorScalarEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) { super(ctx, args); anb = new AttributeNodeBuilder(); @@ -184,13 +186,7 @@ private void copyAttribute(ElementNodeBuilder enb, DictionaryBuilder db, NodeTre anb.setName(newURICode, newLocalCode, newPrefixCode); if (createNodeIds) { - int localId = -1; - if (ntp.nodeIdExists()) { - localId = anp.getLocalNodeId(ntp); - } else { - localId = nodeIdCounter++; - } - anb.setLocalNodeId(localId); + anb.setLocalNodeId(nodeIdCounter++); } anp.getValue(ntp, vp); @@ -220,13 +216,7 @@ private void copyElement(ElementNodeBuilder enb, DictionaryBuilder db, NodeTreeP tempEnb.setName(newURICode, newLocalCode, newPrefixCode); if (createNodeIds) { - int localId = -1; - if (ntp.nodeIdExists()) { - localId = enp.getLocalNodeId(ntp); - } else { - localId = nodeIdCounter++; - } - tempEnb.setLocalNodeId(localId); + tempEnb.setLocalNodeId(nodeIdCounter++); } tempEnb.startAttributeChunk(); @@ -426,13 +416,7 @@ private void copyComment(TaggedValuePointable tvp, NodeTreePointable ntp, IMutab cnb.reset(mvs); if (createNodeIds) { - int localId = -1; - if (ntp.nodeIdExists()) { - localId = tcnp.getLocalNodeId(ntp); - } else { - localId = nodeIdCounter++; - } - cnb.setLocalNodeId(localId); + cnb.setLocalNodeId(nodeIdCounter++); } cnb.setValue(vp); @@ -453,13 +437,7 @@ private void copyPI(TaggedValuePointable tvp, NodeTreePointable ntp, IMutableVal pnb.reset(mvs); if (createNodeIds) { - int localId = -1; - if (ntp.nodeIdExists()) { - localId = pnp.getLocalNodeId(ntp); - } else { - localId = nodeIdCounter++; - } - pnb.setLocalNodeId(localId); + pnb.setLocalNodeId(nodeIdCounter++); } pnb.setContent(vp2); @@ -480,13 +458,7 @@ private void copyText(TaggedValuePointable tvp, NodeTreePointable ntp, IMutableV tnb.reset(mvs); if (createNodeIds) { - int localId = -1; - if (ntp.nodeIdExists()) { - localId = tcnp.getLocalNodeId(ntp); - } else { - localId = nodeIdCounter++; - } - tnb.setLocalNodeId(localId); + tnb.setLocalNodeId(nodeIdCounter++); } tnb.setValue(vp); diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java index d186361bd..64cb0849a 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java @@ -71,8 +71,6 @@ public boolean equals(Object obj) { return false; if (fileId != other.fileId) return false; - System.out.println("My Equals"); - System.out.flush(); return true; } } @@ -91,11 +89,11 @@ protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvalu final SequencePointable seqright = (SequencePointable) SequencePointable.FACTORY.createPointable(); final TaggedValuePointable tvpleft = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); final TaggedValuePointable tvpright = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); - final TypedPointables tpleft = new TypedPointables(); - final TypedPointables tpright = new TypedPointables(); - // a set of unique root node ids and local node ids + // a set of node ids in the left operand Set nodes = new HashSet(); + // a set of node ids that will appear in the final result + Set resultNodes = new HashSet(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override @@ -115,42 +113,44 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S // If an operand has more than one item then it is a sequence // IF an operand has 0 items then it is an empty sequence - // Add items from the left operand into the hash map + // Add items from the left operand into the hash set; this removes duplicates if (tvp1.getTag() == ValueTag.SEQUENCE_TAG) { tvp1.getValue(seqleft); for (int i = 0; i < seqleft.getEntryCount(); ++i) { seqleft.getEntry(i, tvpleft); - if (tvpleft.getTag() != ValueTag.NODE_TREE_TAG) { + if (tvpleft.getTag() != ValueTag.NODE_TREE_TAG) throw new SystemException(ErrorCode.XPTY0004); - } - if (!addItem(tvpleft, tpleft, nodes)) { - // TODO: What happens when local node id is -1 - } + Pair id = getId(tvpleft); + nodes.add(id); } } else { - if (!addItem(tvp1, tpleft, nodes)) { - // TODO: What happens when local node id is -1 - } + Pair id = getId(tvp1); + nodes.add(id); } - // Check if node IDs from right operand are in the hash map + // Check if node IDs from right operand are in the hash set if (tvp2.getTag() == ValueTag.SEQUENCE_TAG) { tvp2.getValue(seqright); for (int i = 0; i < seqright.getEntryCount(); ++i) { seqright.getEntry(i, tvpright); - if (tvpright.getTag() != ValueTag.NODE_TREE_TAG) { + if (tvpright.getTag() != ValueTag.NODE_TREE_TAG) throw new SystemException(ErrorCode.XPTY0004); - } - if (checkItem(tvpright, tpright, nodes)) { - sb.addItem(tvpright); - // TODO - } + Pair temp = getId(tvpright); + if (nodes.contains(temp)) { + if (!resultNodes.contains(temp)) { + resultNodes.add(temp); + sb.addItem(tvpright); + } + } } } else { - if (checkItem(tvp2, tpright, nodes)) { - sb.addItem(tvp2); - // TODO + Pair temp = getId(tvp2); + if (nodes.contains(temp)) { + if (!resultNodes.contains(temp)) { + resultNodes.add(temp); + sb.addItem(tvp2); + } } } @@ -163,46 +163,18 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S }; } - /* - * Adds item to nodes (hash map) - * Returns: False if local node id doesn't exist - * True if item added successfully - */ - private boolean addItem(TaggedValuePointable tvp, TypedPointables tp, Set nodes) { + // Summary: Gets the id of a Node Tree + // Parameter tvp: The Node Tree we want to get the id of + // Return: The local node id, file id pait of tvp + // Error: if the local node id is -1 + private Pair getId(TaggedValuePointable tvp) { + TypedPointables tp = new TypedPointables(); int nodeId = FunctionHelper.getLocalNodeId(tvp, tp); - int rootNodeId = tp.ntp.getRootNodeId(); - byte fileId = (byte) (rootNodeId >> 24); // TODO: Magic number - System.out.println("Left Node ID: " + nodeId + " root node id: " + rootNodeId + " file id: " + fileId); if (nodeId == -1) { // TODO - return false; - } else if (rootNodeId == -1) { - // TODO If rootNodeID is -1 is nodeId also -1? - return false; } - nodes.add(new Pair(nodeId, fileId)); - return true; - } - - /* - * Checks if node is in hash map - * Returns: False if local node id doesn't exist - * False if node isn't in hash map - * True if node is in hash map - */ - private boolean checkItem(TaggedValuePointable tvp, TypedPointables tp, Set nodes) { - int nodeId = FunctionHelper.getLocalNodeId(tvp, tp); int rootNodeId = tp.ntp.getRootNodeId(); byte fileId = (byte) (rootNodeId >> 24); // TODO: Magic number - System.out.println("Right Node ID: " + nodeId + " root node id: " + rootNodeId + " file id: " + fileId); - if (nodeId == -1) { - // TODO - return false; - } else if (rootNodeId == -1) { - // TODO - } else if (nodes.contains(new Pair(nodeId, fileId))) { - return true; - } - return false; + return new Pair(nodeId, fileId); } } \ No newline at end of file diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java index 104626286..a51d7e3d0 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java @@ -29,6 +29,7 @@ public class TreeNodeIdProvider implements ITreeNodeIdProvider { private short currentId; private byte fileId; private static Map Files = new HashMap(); + // File ID 0 is reserved for element constructor private static byte nextFileId = 1; public TreeNodeIdProvider(short partitionDataSource, short dataSouceScanId, short totalDataSources) { @@ -63,6 +64,18 @@ public TreeNodeIdProvider(short partition, String uri) { fileId = getFileId(uri); } + public TreeNodeIdProvider(short partition, boolean isElementConstructor) { + this.partitionDataSource = partition; + dataSouceScanId = 0; + dataSourceBits = 0; + currentId = 0; + if (isElementConstructor) { + fileId = 0; + } else { + fileId = nextFileId++; + } + } + public byte getFileId(String uri) { if (Files.containsKey(uri)) { return Files.get(uri); From 1468e5b3227cbfc6057a618022583f24b8a2f8b6 Mon Sep 17 00:00:00 2001 From: brandon Date: Tue, 23 Oct 2018 20:55:17 -0700 Subject: [PATCH 14/14] bug fixes and unit tests TreeNodeIdProvider now uses canonical paths instead of absolute paths. This removes redundant ./ and ../ items Element Constructor now no longer creates a new fileId each time and isnstead uses a reserved fileID of 0. This is for inline xml elements --- ...bstractNodeConstructorScalarEvaluator.java | 2 +- .../OpIntersectScalarEvaluatorFactory.java | 2 +- .../vxquery/xmlparser/TreeNodeIdProvider.java | 31 +++--- .../Intersect/intersect01.txt | 24 ++--- .../Intersect/intersect02.txt | 12 +-- .../Intersect/intersect03.txt | 12 +-- .../Intersect/intersect05.txt | 1 + .../Queries/XQuery/Intersect/intersect04.xq | 4 +- .../Queries/XQuery/Intersect/intersect05.xq | 17 +--- .../resources/TestSources/xml/catalog.xml | 50 +++++++--- .../test/resources/cat/SetOperatorQueries.xml | 94 ++++++++++--------- 11 files changed, 129 insertions(+), 120 deletions(-) create mode 100644 vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect05.txt diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/AbstractNodeConstructorScalarEvaluator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/AbstractNodeConstructorScalarEvaluator.java index 0a8d82da0..e07a1c9a0 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/AbstractNodeConstructorScalarEvaluator.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/AbstractNodeConstructorScalarEvaluator.java @@ -35,7 +35,7 @@ import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; public abstract class AbstractNodeConstructorScalarEvaluator extends AbstractTaggedValueArgumentScalarEvaluator { - protected final static ITreeNodeIdProvider NodeConstructorIdProvider = new TreeNodeIdProvider((short) 0, true); + protected final static ITreeNodeIdProvider NodeConstructorIdProvider = new TreeNodeIdProvider((short) 0); protected final IHyracksTaskContext ctx; diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java index 64cb0849a..f70cba281 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/sequence/OpIntersectScalarEvaluatorFactory.java @@ -174,7 +174,7 @@ private Pair getId(TaggedValuePointable tvp) { // TODO } int rootNodeId = tp.ntp.getRootNodeId(); - byte fileId = (byte) (rootNodeId >> 24); // TODO: Magic number + byte fileId = (byte) (rootNodeId >> 24); return new Pair(nodeId, fileId); } } \ No newline at end of file diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java index a51d7e3d0..3179d5459 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/TreeNodeIdProvider.java @@ -16,6 +16,7 @@ */ package org.apache.vxquery.xmlparser; +import java.io.File; import java.util.HashMap; import java.util.Map; @@ -37,7 +38,7 @@ public TreeNodeIdProvider(short partitionDataSource, short dataSouceScanId, shor this.dataSouceScanId = dataSouceScanId; this.dataSourceBits = getBitsNeeded(totalDataSources); currentId = 0; - fileId = nextFileId++; + fileId = 0; } public TreeNodeIdProvider(short partitionDataSource, short dataSouceScanId, short totalDataSources, String uri) { @@ -53,7 +54,7 @@ public TreeNodeIdProvider(short partition) { dataSouceScanId = 0; dataSourceBits = 0; currentId = 0; - fileId = nextFileId++; + fileId = 0; } public TreeNodeIdProvider(short partition, String uri) { @@ -64,29 +65,23 @@ public TreeNodeIdProvider(short partition, String uri) { fileId = getFileId(uri); } - public TreeNodeIdProvider(short partition, boolean isElementConstructor) { - this.partitionDataSource = partition; - dataSouceScanId = 0; - dataSourceBits = 0; - currentId = 0; - if (isElementConstructor) { - fileId = 0; - } else { - fileId = nextFileId++; - } - } - public byte getFileId(String uri) { - if (Files.containsKey(uri)) { - return Files.get(uri); + // Try to get canonical path + String path = null; + try { + path = new File(uri).getCanonicalPath(); + } catch (Exception e) { + path = uri; + } + if (Files.containsKey(path)) { + return Files.get(path); } else { - Files.put(uri, nextFileId); + Files.put(path, nextFileId); return nextFileId++; } } public int getId() { - // TODO: We only have 8 bytes for partition and datasourcescanid int p = partitionDataSource; int dssi = dataSouceScanId; int f = fileId; diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect01.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect01.txt index 2d8a5317c..cad67d62e 100644 --- a/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect01.txt +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect01.txt @@ -1,14 +1,14 @@ - Oberg, Bruce - The Poet's First Poem - Poem - 24.95 - The least poetic poems of the decade. - + Oberg, Bruce + The Poet's First Poem + Poem + 24.95 + The least poetic poems of the decade. + - Nagata, Suanne - Becoming Somebody - Biography - 39 - A masterpiece of the fine art of gossiping. - \ No newline at end of file + Nagata, Suanne + Becoming Somebody + Biography + 39 + A masterpiece of the fine art of gossiping. + \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect02.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect02.txt index d17890a5a..343338bcb 100644 --- a/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect02.txt +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect02.txt @@ -1,7 +1,7 @@ - Oberg, Bruce - The Poet's First Poem - Poem - 24.95 - The least poetic poems of the decade. - \ No newline at end of file + Oberg, Bruce + The Poet's First Poem + Poem + 24.95 + The least poetic poems of the decade. + \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect03.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect03.txt index d17890a5a..343338bcb 100644 --- a/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect03.txt +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect03.txt @@ -1,7 +1,7 @@ - Oberg, Bruce - The Poet's First Poem - Poem - 24.95 - The least poetic poems of the decade. - \ No newline at end of file + Oberg, Bruce + The Poet's First Poem + Poem + 24.95 + The least poetic poems of the decade. + \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect05.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect05.txt new file mode 100644 index 000000000..89142d211 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Intersect/intersect05.txt @@ -0,0 +1 @@ +5 \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect04.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect04.xq index 4deadb416..d87fe5048 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect04.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect04.xq @@ -16,6 +16,6 @@ under the License. :) let $c := "catalog" -let $a := doc($c)/catalog/book[price<0] -let $b := doc($c)/catalog/book[price<50] +let $a := doc($c)/catalog/book[price<40] +let $b := doc($c)/catalog/book[price>40] return $a intersect $b diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect05.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect05.xq index 47cc7c92f..f0bc60c13 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect05.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Intersect/intersect05.xq @@ -14,23 +14,10 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. :) - - (: -let $a := "ghcnd" -for $i in collection($a)/dataCollection/data - - -let $b := "ghcnd" -for $j in collection($b)/dataCollection/data - -where $i/value < 10 -and $j/value < 20 - -return $i intersect $j -:) let $a := 5 -return $a intersect $a +let $b := $a +return $a intersect $b diff --git a/vxquery-xtest/src/test/resources/TestSources/xml/catalog.xml b/vxquery-xtest/src/test/resources/TestSources/xml/catalog.xml index fdaa37b02..ffe091f05 100644 --- a/vxquery-xtest/src/test/resources/TestSources/xml/catalog.xml +++ b/vxquery-xtest/src/test/resources/TestSources/xml/catalog.xml @@ -1,20 +1,40 @@ - + + + Hightower, Kim + The First Book + Fiction + 44.95 + 2000-10-01 + An amazing story of nothing. + - http://www.apache.org/licenses/LICENSE-2.0 + + Oberg, Bruce + The Poet's First Poem + Poem + 24.95 + The least poetic poems of the decade. + + + + Nagata, Suanne + Becoming Somebody + Biography + 39 + A masterpiece of the fine art of gossiping. + + - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -5 diff --git a/vxquery-xtest/src/test/resources/cat/SetOperatorQueries.xml b/vxquery-xtest/src/test/resources/cat/SetOperatorQueries.xml index e71b91268..6aa7ba67e 100644 --- a/vxquery-xtest/src/test/resources/cat/SetOperatorQueries.xml +++ b/vxquery-xtest/src/test/resources/cat/SetOperatorQueries.xml @@ -1,46 +1,52 @@ - + - - - Set Operator Test - - - - Intersects two sets both with more than 1 item - - intersect01.txt - - - Left operand has 1 item, right operand has many - - intersect02.txt - - - Testing reverse of test 2 - - intersect03.txt - - - Left operand has 1 item, right operand has many - - intersect04.txt - - - Collection - - intersect05.txt - + + + Set Operator Test + + + + Intersects two sets both with more than 1 item + + + intersect01.txt + + + Left operand has 1 item, right operand has many + + + intersect02.txt + + + Testing reverse of test 2 + + intersect03.txt + + + Left and right operand have items that don't intersect + + + intersect04.txt + + + Text inline xml nodes + + intersect05.txt +