From dc485bba55b87153b18dd733ddab3f37d38818d6 Mon Sep 17 00:00:00 2001 From: Mouli Mukherjee Date: Sat, 13 Jul 2019 13:15:14 -0700 Subject: [PATCH 1/9] Add projectStrict for Dates and Timestamps --- .../org/apache/iceberg/transforms/Dates.java | 7 +++++-- .../iceberg/transforms/ProjectionUtil.java | 17 +++++++++++++++++ .../apache/iceberg/transforms/Timestamps.java | 7 +++++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/transforms/Dates.java b/api/src/main/java/org/apache/iceberg/transforms/Dates.java index a57d6d95eaa4..c7e8e9ea4e97 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Dates.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Dates.java @@ -73,8 +73,11 @@ public UnboundPredicate project(String fieldName, BoundPredicate projectStrict(String fieldName, BoundPredicate predicate) { - return null; + public UnboundPredicate projectStrict(String fieldName, BoundPredicate pred) { + if (pred.op() == NOT_NULL || pred.op() == IS_NULL) { + return Expressions.predicate(pred.op(), fieldName); + } + return ProjectionUtil.truncateStrictToInteger(fieldName, pred, this); } @Override diff --git a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java index 04da036e1637..4f1cf5c8270a 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java +++ b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java @@ -52,6 +52,23 @@ static UnboundPredicate truncateInteger( } } + static UnboundPredicate truncateStrictToInteger( + String name, BoundPredicate pred, Transform transform) { + T boundary = pred.literal().value(); + switch (pred.op()) { + case LT: + case LT_EQ: + return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary) - 1); + case GT: + case GT_EQ: + return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary) + 1); + case NOT_EQ: + return predicate(Expression.Operation.NOT_EQ, name, transform.apply(boundary)); + default: + return null; + } + } + static UnboundPredicate truncateLong( String name, BoundPredicate pred, Transform transform) { long boundary = pred.literal().value(); diff --git a/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java b/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java index f01ea050229c..c01e544c61f9 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java @@ -76,8 +76,11 @@ public UnboundPredicate project(String fieldName, BoundPredicate } @Override - public UnboundPredicate projectStrict(String fieldName, BoundPredicate predicate) { - return null; + public UnboundPredicate projectStrict(String fieldName, BoundPredicate pred) { + if (pred.op() == NOT_NULL || pred.op() == IS_NULL) { + return Expressions.predicate(pred.op(), fieldName); + } + return ProjectionUtil.truncateStrictToInteger(fieldName, pred, this); } @Override From f533ecd9453f4c4b8719aa7ee8f5a50532815c62 Mon Sep 17 00:00:00 2001 From: Mouli Mukherjee Date: Sat, 13 Jul 2019 15:41:09 -0700 Subject: [PATCH 2/9] Add tests for Timestamps --- .../transforms/TestTimestampsProjection.java | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java new file mode 100644 index 000000000000..b452e32c677e --- /dev/null +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java @@ -0,0 +1,162 @@ +package org.apache.iceberg.transforms; + +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Literal; +import org.apache.iceberg.expressions.Projections; +import org.apache.iceberg.expressions.UnboundPredicate; +import org.apache.iceberg.types.Types; +import org.junit.Assert; +import org.junit.Test; + +import static org.apache.iceberg.TestHelpers.assertAndUnwrapUnbound; +import static org.apache.iceberg.expressions.Expressions.*; +import static org.apache.iceberg.types.Types.NestedField.optional; + +public class TestTimestampsProjection { + private static final Types.TimestampType TYPE = Types.TimestampType.withoutZone(); + private static final Schema SCHEMA = new Schema( + optional(1, "timestamp", TYPE)); + + public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate filter, Expression.Operation expectedOp, String expectedLiteral) { + + Expression projection = Projections.strict(spec).project(filter); + UnboundPredicate predicate = assertAndUnwrapUnbound(projection); + + Assert.assertEquals(predicate.op(), expectedOp); + + Literal literal = predicate.literal(); + Timestamps transform = (Timestamps) spec.getFieldsBySourceId(1).get(0).transform(); + String output = transform.toHumanString((int) literal.value()); + Assert.assertEquals(output, expectedLiteral); + + } + + public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate filter, Expression.Operation expectedOp, String expectedLiteral) { + Expression projection = Projections.inclusive(spec).project(filter); + UnboundPredicate predicate = assertAndUnwrapUnbound(projection); + + Assert.assertEquals(predicate.op(), expectedOp); + + Literal literal = predicate.literal(); + Timestamps transform = (Timestamps) spec.getFieldsBySourceId(1).get(0).transform(); + String output = transform.toHumanString((int) literal.value()); + Assert.assertEquals(output, expectedLiteral); + + } + + @Test + public void testMonthStrict() { + Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); + + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build(); + + assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11"); + assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-11"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018-01"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2018-01"); + assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12"); + + } + + @Test + public void testMonthInclusive() { + Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); + + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build(); + + assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12"); + assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12"); + assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12"); + assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12"); + assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.EQ, "2017-12"); + + } + + @Test + public void testDayStrict() { + Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); + + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); + + assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11-30"); + assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-11-30"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02"); + assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01"); + + } + + @Test + public void testDayInclusive() { + Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); + + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); + + assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01"); + assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01"); + assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01"); + assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01"); + assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.EQ, "2017-12-01"); + + } + + @Test + public void testYearStrict() { + Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); + + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("timestamp").build(); + + assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2016"); + assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2016"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2018"); + assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017"); + + } + + @Test + public void testYearInclusive() { + Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); + + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("timestamp").build(); + + assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017"); + assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017"); + assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017"); + assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017"); + assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.EQ, "2017"); + + } + + @Test + public void testHourStrict() { + Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); + + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("timestamp").build(); + + assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09"); + assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11"); + assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01-10"); + + } + + @Test + public void testHourInclusive() { + Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); + + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("timestamp").build(); + + assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-10"); + assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-10"); + assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-10"); + assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-10"); + assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.EQ, "2017-12-01-10"); + + } + + +} From 834303737d17a8e3f6ceb89675d31c39c942de54 Mon Sep 17 00:00:00 2001 From: Mouli Mukherjee Date: Sat, 13 Jul 2019 16:47:31 -0700 Subject: [PATCH 3/9] Adding more tests with lower and upper bounds --- .../transforms/TestTimestampsProjection.java | 112 +++++++++++++++++- 1 file changed, 109 insertions(+), 3 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java index b452e32c677e..6c93ee6fe377 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java @@ -60,10 +60,37 @@ public void testMonthStrict() { } + @Test + public void testMonthStrictLowerBound() { + Long date = (long) Literal.of("2017-12-01T00:00:00.00000").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build(); + + assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11"); + assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-11"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018-01"); + // bound should include the same month for lower bound + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12"); + assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12"); + + } + + @Test + public void testMonthStrictUpperBound() { + Long date = (long) Literal.of("2017-12-31T23:59:59.99999").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build(); + + assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11"); + // bound should include the same month for upper bound + assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018-01"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2018-01"); + assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12"); + + } + @Test public void testMonthInclusive() { Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); - PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build(); assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12"); @@ -77,12 +104,39 @@ public void testMonthInclusive() { @Test public void testDayStrict() { Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); + + assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11-30"); + assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-11-30"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02"); + assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01"); + } + + @Test + public void testDayStrictLowerBound() { + Long date = (long) Literal.of("2017-12-01T00:00:00.00000").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11-30"); assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-11-30"); assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02"); + // bound should include the same day for lower bound + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01"); + assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01"); + + } + + @Test + public void testDayStrictUpperBound() { + Long date = (long) Literal.of("2017-12-01T23:59:59.99999").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); + + assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11-30"); + // bound should include the same day for lower bound + assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02"); assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01"); @@ -91,7 +145,6 @@ public void testDayStrict() { @Test public void testDayInclusive() { Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); - PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01"); @@ -105,12 +158,39 @@ public void testDayInclusive() { @Test public void testYearStrict() { Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("timestamp").build(); + + assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2016"); + assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2016"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2018"); + assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017"); + } + + @Test + public void testYearStrictLowerBound() { + Long date = (long) Literal.of("2017-01-01T00:00:00.00000").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("timestamp").build(); assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2016"); assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2016"); assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018"); + // bound should include the same year for lower bound + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017"); + assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017"); + + } + + @Test + public void testYearStrictUpperBound() { + Long date = (long) Literal.of("2017-12-31T23:59:59.99999").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("timestamp").build(); + + assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2016"); + // bound should include the same year for upper bound + assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018"); assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2018"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017"); @@ -133,12 +213,39 @@ public void testYearInclusive() { @Test public void testHourStrict() { Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("timestamp").build(); + assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09"); + assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11"); + assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01-10"); + + } + + @Test + public void testHourStrictLowerBound() { + Long date = (long) Literal.of("2017-12-01T10:00:00.00000").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("timestamp").build(); assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09"); assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09"); assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11"); + // bound should include the same hour for lower bound + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11"); + assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01-10"); + + } + + @Test + public void testHourStrictUpperBound() { + Long date = (long) Literal.of("2017-12-01T10:59:59.99999").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("timestamp").build(); + + assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09"); + // bound should include the same hour for upper bound + assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-10"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11"); assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01-10"); @@ -147,7 +254,6 @@ public void testHourStrict() { @Test public void testHourInclusive() { Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); - PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("timestamp").build(); assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-10"); From c3ad7e3b7c7f3fb70ccb79206e6862cc0f0e1e43 Mon Sep 17 00:00:00 2001 From: Mouli Mukherjee Date: Sat, 13 Jul 2019 17:19:49 -0700 Subject: [PATCH 4/9] Fixing tests for TimestampsProjections --- .../org/apache/iceberg/transforms/Dates.java | 2 +- .../iceberg/transforms/ProjectionUtil.java | 49 +++++++++++++++++-- .../apache/iceberg/transforms/Timestamps.java | 2 +- .../transforms/TestTimestampsProjection.java | 12 ++--- 4 files changed, 53 insertions(+), 12 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/transforms/Dates.java b/api/src/main/java/org/apache/iceberg/transforms/Dates.java index c7e8e9ea4e97..e1210764928d 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Dates.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Dates.java @@ -77,7 +77,7 @@ public UnboundPredicate projectStrict(String fieldName, BoundPredicate< if (pred.op() == NOT_NULL || pred.op() == IS_NULL) { return Expressions.predicate(pred.op(), fieldName); } - return ProjectionUtil.truncateStrictToInteger(fieldName, pred, this); + return ProjectionUtil.truncateIntegerStrictToInteger(fieldName, pred, this); } @Override diff --git a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java index 4f1cf5c8270a..8eceda45bcfe 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java +++ b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java @@ -52,16 +52,57 @@ static UnboundPredicate truncateInteger( } } - static UnboundPredicate truncateStrictToInteger( - String name, BoundPredicate pred, Transform transform) { - T boundary = pred.literal().value(); + static UnboundPredicate truncateIntegerStrictToInteger( + String name, BoundPredicate pred, Transform transform) { + Integer boundary = pred.literal().value(); switch (pred.op()) { case LT: - case LT_EQ: return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary) - 1); + case LT_EQ: + // Checking if the timestamp is at the date boundary + if (transform.apply(boundary + 1).equals(transform.apply(boundary))) { + return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary) - 1); + } else { + return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary)); + } case GT: + return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary) + 1); case GT_EQ: + // Checking if the timestamp is at the date boundary + if (transform.apply(boundary - 1).equals(transform.apply(boundary))) { + return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary) + 1); + } else { + return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary)); + } + case NOT_EQ: + return predicate(Expression.Operation.NOT_EQ, name, transform.apply(boundary)); + default: + return null; + } + } + + static UnboundPredicate truncateLongStrictToInteger( + String name, BoundPredicate pred, Transform transform) { + Long boundary = pred.literal().value(); + switch (pred.op()) { + case LT: + return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary) - 1); + case LT_EQ: + // Checking if the timestamp is at the date boundary + if (transform.apply(boundary + 1L).equals(transform.apply(boundary))) { + return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary) - 1); + } else { + return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary)); + } + case GT: return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary) + 1); + case GT_EQ: + // Checking if the timestamp is at the date boundary + if (transform.apply(boundary - 1L).equals(transform.apply(boundary))) { + return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary) + 1); + } else { + return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary)); + } case NOT_EQ: return predicate(Expression.Operation.NOT_EQ, name, transform.apply(boundary)); default: diff --git a/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java b/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java index c01e544c61f9..3fe6de4a5b04 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java @@ -80,7 +80,7 @@ public UnboundPredicate projectStrict(String fieldName, BoundPredicate< if (pred.op() == NOT_NULL || pred.op() == IS_NULL) { return Expressions.predicate(pred.op(), fieldName); } - return ProjectionUtil.truncateStrictToInteger(fieldName, pred, this); + return ProjectionUtil.truncateLongStrictToInteger(fieldName, pred, this); } @Override diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java index 6c93ee6fe377..5000865c8f0e 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java @@ -76,7 +76,7 @@ public void testMonthStrictLowerBound() { @Test public void testMonthStrictUpperBound() { - Long date = (long) Literal.of("2017-12-31T23:59:59.99999").to(TYPE).value(); + Long date = (long) Literal.of("2017-12-31T23:59:59.999999").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build(); assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11"); @@ -130,11 +130,11 @@ public void testDayStrictLowerBound() { @Test public void testDayStrictUpperBound() { - Long date = (long) Literal.of("2017-12-01T23:59:59.99999").to(TYPE).value(); + Long date = (long) Literal.of("2017-12-01T23:59:59.999999").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11-30"); - // bound should include the same day for lower bound + // bound should include the same day for upper bound assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01"); assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02"); assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02"); @@ -184,7 +184,7 @@ public void testYearStrictLowerBound() { @Test public void testYearStrictUpperBound() { - Long date = (long) Literal.of("2017-12-31T23:59:59.99999").to(TYPE).value(); + Long date = (long) Literal.of("2017-12-31T23:59:59.999999").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("timestamp").build(); assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2016"); @@ -232,14 +232,14 @@ public void testHourStrictLowerBound() { assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09"); assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11"); // bound should include the same hour for lower bound - assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-10"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01-10"); } @Test public void testHourStrictUpperBound() { - Long date = (long) Literal.of("2017-12-01T10:59:59.99999").to(TYPE).value(); + Long date = (long) Literal.of("2017-12-01T10:59:59.999999").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("timestamp").build(); assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09"); From fae0b36f40dfdfc39cce78ac77b0843815997693 Mon Sep 17 00:00:00 2001 From: Mouli Mukherjee Date: Sat, 13 Jul 2019 17:40:37 -0700 Subject: [PATCH 5/9] Adding tests for Dates Projection --- .../transforms/TestDatesProjection.java | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java new file mode 100644 index 000000000000..bfd9356e20e5 --- /dev/null +++ b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java @@ -0,0 +1,200 @@ +package org.apache.iceberg.transforms; + +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Literal; +import org.apache.iceberg.expressions.Projections; +import org.apache.iceberg.expressions.UnboundPredicate; +import org.apache.iceberg.types.Types; +import org.junit.Assert; +import org.junit.Test; + +import static org.apache.iceberg.TestHelpers.assertAndUnwrapUnbound; +import static org.apache.iceberg.expressions.Expressions.*; +import static org.apache.iceberg.types.Types.NestedField.optional; + +public class TestDatesProjection { + private static final Types.DateType TYPE = Types.DateType.get(); + private static final Schema SCHEMA = new Schema( + optional(1, "date", TYPE)); + + public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate filter, Expression.Operation expectedOp, String expectedLiteral) { + + Expression projection = Projections.strict(spec).project(filter); + UnboundPredicate predicate = assertAndUnwrapUnbound(projection); + + Assert.assertEquals(predicate.op(), expectedOp); + + Literal literal = predicate.literal(); + Dates transform = (Dates) spec.getFieldsBySourceId(1).get(0).transform(); + String output = transform.toHumanString((int) literal.value()); + Assert.assertEquals(output, expectedLiteral); + + } + + public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate filter, Expression.Operation expectedOp, String expectedLiteral) { + Expression projection = Projections.inclusive(spec).project(filter); + UnboundPredicate predicate = assertAndUnwrapUnbound(projection); + + Assert.assertEquals(predicate.op(), expectedOp); + + Literal literal = predicate.literal(); + Dates transform = (Dates) spec.getFieldsBySourceId(1).get(0).transform(); + String output = transform.toHumanString((int) literal.value()); + Assert.assertEquals(output, expectedLiteral); + + } + + @Test + public void testMonthStrictLowerBound() { + Integer date = (Integer) Literal.of("2017-01-01").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build(); + + assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2016-12"); + assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2016-12"); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2017-02"); + // bound should include the same month for lower bound + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017-01"); + assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017-01"); + + } + + @Test + public void testMonthStrictUpperBound() { + Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build(); + + assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2017-11"); + // bound should include the same month for upper bound + assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017-12"); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2018-01"); + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2018-01"); + assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017-12"); + + } + + @Test + public void testMonthInclusiveLowerBound() { + Integer date = (Integer) Literal.of("2017-12-01").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build(); + + assertProjectionInclusive(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2017-11"); + assertProjectionInclusive(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017-12"); + assertProjectionInclusive(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2017-12"); + assertProjectionInclusive(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017-12"); + assertProjectionInclusive(spec, equal("date", date), Expression.Operation.EQ, "2017-12"); + + } + + @Test + public void testMonthInclusiveUpperBound() { + Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build(); + + assertProjectionInclusive(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2017-12"); + assertProjectionInclusive(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017-12"); + assertProjectionInclusive(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2018-01"); + assertProjectionInclusive(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017-12"); + assertProjectionInclusive(spec, equal("date", date), Expression.Operation.EQ, "2017-12"); + + } + + @Test + public void testDayStrictLowerBound() { + Integer date = (Integer) Literal.of("2017-01-01").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("date").build(); + + assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2016-12-31"); + // should be the same date for <= + assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017-01-01"); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2017-01-02"); + // should be the same date for >= + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017-01-01"); + assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017-01-01"); + + } + + @Test + public void testDayStrictUpperBound() { + Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("date").build(); + + assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2017-12-30"); + // should be the same date for <= + assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017-12-31"); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2018-01-01"); + // should be the same date for >= + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017-12-31"); + assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017-12-31"); + + } + + @Test + public void testDayInclusive() { + Integer date = (Integer) Literal.of("2017-12-01").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("date").build(); + + assertProjectionInclusive(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2017-11-30"); + assertProjectionInclusive(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017-12-01"); + assertProjectionInclusive(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2017-12-02"); + assertProjectionInclusive(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017-12-01"); + assertProjectionInclusive(spec, equal("date", date), Expression.Operation.EQ, "2017-12-01"); + + } + + @Test + public void testYearStrictLowerBound() { + Integer date = (Integer) Literal.of("2017-01-01").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build(); + + assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2016"); + assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2016"); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2018"); + // bound should include the same year for lower bound + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017"); + assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017"); + + } + + @Test + public void testYearStrictUpperBound() { + Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build(); + + assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2016"); + // bound should include the same year for upper bound + assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017"); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2018"); + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2018"); + assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017"); + + } + + @Test + public void testYearInclusiveLowerBound() { + Integer date = (Integer) Literal.of("2017-01-01").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build(); + + assertProjectionInclusive(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2016"); + assertProjectionInclusive(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017"); + assertProjectionInclusive(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2017"); + assertProjectionInclusive(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017"); + assertProjectionInclusive(spec, equal("date", date), Expression.Operation.EQ, "2017"); + + } + + @Test + public void testYearInclusiveUpperBound() { + Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build(); + + assertProjectionInclusive(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2017"); + assertProjectionInclusive(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017"); + assertProjectionInclusive(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2018"); + assertProjectionInclusive(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017"); + assertProjectionInclusive(spec, equal("date", date), Expression.Operation.EQ, "2017"); + + } + +} From c3620e55e95eed8e473c0a6aef49f0ba88082ed5 Mon Sep 17 00:00:00 2001 From: Mouli Mukherjee Date: Sat, 13 Jul 2019 17:48:43 -0700 Subject: [PATCH 6/9] Refactor tests --- .../transforms/TestDatesProjection.java | 4 +- .../transforms/TestTimestampsProjection.java | 108 +++++++++--------- 2 files changed, 55 insertions(+), 57 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java index bfd9356e20e5..685889db20ea 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java @@ -29,7 +29,7 @@ public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate filte Literal literal = predicate.literal(); Dates transform = (Dates) spec.getFieldsBySourceId(1).get(0).transform(); String output = transform.toHumanString((int) literal.value()); - Assert.assertEquals(output, expectedLiteral); + Assert.assertEquals(expectedLiteral, output); } @@ -42,7 +42,7 @@ public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate fi Literal literal = predicate.literal(); Dates transform = (Dates) spec.getFieldsBySourceId(1).get(0).transform(); String output = transform.toHumanString((int) literal.value()); - Assert.assertEquals(output, expectedLiteral); + Assert.assertEquals(expectedLiteral, output); } diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java index 5000865c8f0e..8cfe54e65680 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java @@ -29,7 +29,7 @@ public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate filte Literal literal = predicate.literal(); Timestamps transform = (Timestamps) spec.getFieldsBySourceId(1).get(0).transform(); String output = transform.toHumanString((int) literal.value()); - Assert.assertEquals(output, expectedLiteral); + Assert.assertEquals(expectedLiteral, output); } @@ -42,21 +42,7 @@ public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate fi Literal literal = predicate.literal(); Timestamps transform = (Timestamps) spec.getFieldsBySourceId(1).get(0).transform(); String output = transform.toHumanString((int) literal.value()); - Assert.assertEquals(output, expectedLiteral); - - } - - @Test - public void testMonthStrict() { - Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); - - PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build(); - - assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11"); - assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-11"); - assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018-01"); - assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2018-01"); - assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12"); + Assert.assertEquals(expectedLiteral, output); } @@ -76,20 +62,20 @@ public void testMonthStrictLowerBound() { @Test public void testMonthStrictUpperBound() { - Long date = (long) Literal.of("2017-12-31T23:59:59.999999").to(TYPE).value(); + Long date = (long) Literal.of("2017-12-01T00:00:00.00000").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build(); assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11"); - // bound should include the same month for upper bound - assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12"); + assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-11"); assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018-01"); - assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2018-01"); + // bound should include the same month for upper bound + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12"); } @Test - public void testMonthInclusive() { + public void testMonthInclusiveLowerBound() { Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build(); @@ -102,15 +88,15 @@ public void testMonthInclusive() { } @Test - public void testDayStrict() { - Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); - PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); + public void testMonthInclusiveUpperBound() { + Long date = (long) Literal.of("2017-12-01T00:00:00.00000").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build(); - assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11-30"); - assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-11-30"); - assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02"); - assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02"); - assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01"); + assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11"); + assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12"); + assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12"); + assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12"); + assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.EQ, "2017-12"); } @@ -143,11 +129,11 @@ public void testDayStrictUpperBound() { } @Test - public void testDayInclusive() { - Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); + public void testDayInclusiveLowerBound() { + Long date = (long) Literal.of("2017-12-01T00:00:00.00000").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); - assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01"); + assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11-30"); assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01"); assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01"); assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01"); @@ -156,15 +142,15 @@ public void testDayInclusive() { } @Test - public void testYearStrict() { - Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); - PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("timestamp").build(); + public void testDayInclusiveUpperBound() { + Long date = (long) Literal.of("2017-12-01T23:59:59.999999").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); - assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2016"); - assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2016"); - assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018"); - assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2018"); - assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017"); + assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01"); + assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01"); + assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02"); + assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01"); + assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.EQ, "2017-12-01"); } @@ -197,12 +183,11 @@ public void testYearStrictUpperBound() { } @Test - public void testYearInclusive() { - Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); - + public void testYearInclusiveLowerBound() { + Long date = (long) Literal.of("2017-01-01T00:00:00.00000").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("timestamp").build(); - assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017"); + assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2016"); assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017"); assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017"); assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017"); @@ -211,15 +196,15 @@ public void testYearInclusive() { } @Test - public void testHourStrict() { - Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); - PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("timestamp").build(); + public void testYearInclusiveUpperBound() { + Long date = (long) Literal.of("2017-12-31T23:59:59.999999").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("timestamp").build(); - assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09"); - assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09"); - assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11"); - assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11"); - assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01-10"); + assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017"); + assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017"); + assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018"); + assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017"); + assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.EQ, "2017"); } @@ -252,11 +237,11 @@ public void testHourStrictUpperBound() { } @Test - public void testHourInclusive() { - Long date = (long) Literal.of("2017-12-01T10:12:55.038194").to(TYPE).value(); + public void testHourInclusiveLowerBound() { + Long date = (long) Literal.of("2017-12-01T10:00:00.00000").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("timestamp").build(); - assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-10"); + assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09"); assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-10"); assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-10"); assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-10"); @@ -264,5 +249,18 @@ public void testHourInclusive() { } + @Test + public void testHourInclusiveUpperBound() { + Long date = (long) Literal.of("2017-12-01T10:59:59.999999").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("timestamp").build(); + + assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-10"); + assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-10"); + assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11"); + assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-10"); + assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.EQ, "2017-12-01-10"); + + } + } From 0493203d391055e116d08a6ca8da3555dfd72245 Mon Sep 17 00:00:00 2001 From: Mouli Mukherjee Date: Mon, 15 Jul 2019 12:29:44 -0700 Subject: [PATCH 7/9] Implementing strict projection only for boundaries --- .../iceberg/transforms/ProjectionUtil.java | 52 +++++++++++++------ .../transforms/TestDatesProjection.java | 8 --- .../transforms/TestTimestampsProjection.java | 22 ++------ 3 files changed, 39 insertions(+), 43 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java index 8eceda45bcfe..28a096f6c2f9 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java +++ b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java @@ -57,22 +57,32 @@ static UnboundPredicate truncateIntegerStrictToInteger( Integer boundary = pred.literal().value(); switch (pred.op()) { case LT: - return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary) - 1); - case LT_EQ: - // Checking if the timestamp is at the date boundary - if (transform.apply(boundary + 1).equals(transform.apply(boundary))) { + // Checking if the date is at the lower boundary + if (!transform.apply(boundary - 1).equals(transform.apply(boundary))) { return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary) - 1); } else { + return null; + } + case LT_EQ: + // Checking if the date is at the upper boundary + if (!transform.apply(boundary + 1).equals(transform.apply(boundary))) { return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary)); + } else { + return null; } case GT: - return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary) + 1); - case GT_EQ: - // Checking if the timestamp is at the date boundary - if (transform.apply(boundary - 1).equals(transform.apply(boundary))) { + // Checking if the date is at the upper boundary + if (!transform.apply(boundary + 1).equals(transform.apply(boundary))) { return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary) + 1); } else { + return null; + } + case GT_EQ: + // Checking if the date is at the lower boundary + if (!transform.apply(boundary - 1).equals(transform.apply(boundary))) { return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary)); + } else { + return null; } case NOT_EQ: return predicate(Expression.Operation.NOT_EQ, name, transform.apply(boundary)); @@ -86,22 +96,32 @@ static UnboundPredicate truncateLongStrictToInteger( Long boundary = pred.literal().value(); switch (pred.op()) { case LT: - return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary) - 1); - case LT_EQ: - // Checking if the timestamp is at the date boundary - if (transform.apply(boundary + 1L).equals(transform.apply(boundary))) { + // Checking if the timestamp is at the lower boundary + if (!transform.apply(boundary - 1L).equals(transform.apply(boundary))) { return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary) - 1); } else { + return null; + } + case LT_EQ: + // Checking if the timestamp is at the upper boundary + if (!transform.apply(boundary + 1L).equals(transform.apply(boundary))) { return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary)); + } else { + return null; } case GT: - return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary) + 1); - case GT_EQ: - // Checking if the timestamp is at the date boundary - if (transform.apply(boundary - 1L).equals(transform.apply(boundary))) { + // Checking if the timestamp is at the upper boundary + if (!transform.apply(boundary + 1L).equals(transform.apply(boundary))) { return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary) + 1); } else { + return null; + } + case GT_EQ: + // Checking if the timestamp is at the lower boundary + if (!transform.apply(boundary - 1L).equals(transform.apply(boundary))) { return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary)); + } else { + return null; } case NOT_EQ: return predicate(Expression.Operation.NOT_EQ, name, transform.apply(boundary)); diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java index 685889db20ea..02033a7d484c 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java @@ -52,8 +52,6 @@ public void testMonthStrictLowerBound() { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build(); assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2016-12"); - assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2016-12"); - assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2017-02"); // bound should include the same month for lower bound assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017-01"); assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017-01"); @@ -65,11 +63,9 @@ public void testMonthStrictUpperBound() { Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build(); - assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2017-11"); // bound should include the same month for upper bound assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017-12"); assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2018-01"); - assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2018-01"); assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017-12"); } @@ -149,8 +145,6 @@ public void testYearStrictLowerBound() { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build(); assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2016"); - assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2016"); - assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2018"); // bound should include the same year for lower bound assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017"); assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017"); @@ -162,11 +156,9 @@ public void testYearStrictUpperBound() { Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build(); - assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2016"); // bound should include the same year for upper bound assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017"); assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2018"); - assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2018"); assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017"); } diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java index 8cfe54e65680..d136e78b380d 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java @@ -52,8 +52,6 @@ public void testMonthStrictLowerBound() { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build(); assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11"); - assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-11"); - assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018-01"); // bound should include the same month for lower bound assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12"); @@ -62,14 +60,12 @@ public void testMonthStrictLowerBound() { @Test public void testMonthStrictUpperBound() { - Long date = (long) Literal.of("2017-12-01T00:00:00.00000").to(TYPE).value(); + Long date = (long) Literal.of("2017-12-31T23:59:59.999999").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build(); - assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11"); - assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-11"); - assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018-01"); + assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12"); // bound should include the same month for upper bound - assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018-01"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12"); } @@ -106,8 +102,6 @@ public void testDayStrictLowerBound() { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11-30"); - assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-11-30"); - assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02"); // bound should include the same day for lower bound assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01"); @@ -119,11 +113,9 @@ public void testDayStrictUpperBound() { Long date = (long) Literal.of("2017-12-01T23:59:59.999999").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); - assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11-30"); // bound should include the same day for upper bound assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01"); assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02"); - assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01"); } @@ -160,8 +152,6 @@ public void testYearStrictLowerBound() { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("timestamp").build(); assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2016"); - assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2016"); - assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018"); // bound should include the same year for lower bound assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017"); @@ -173,11 +163,9 @@ public void testYearStrictUpperBound() { Long date = (long) Literal.of("2017-12-31T23:59:59.999999").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("timestamp").build(); - assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2016"); // bound should include the same year for upper bound assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017"); assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018"); - assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2018"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017"); } @@ -214,8 +202,6 @@ public void testHourStrictLowerBound() { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("timestamp").build(); assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09"); - assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09"); - assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11"); // bound should include the same hour for lower bound assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-10"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01-10"); @@ -227,11 +213,9 @@ public void testHourStrictUpperBound() { Long date = (long) Literal.of("2017-12-01T10:59:59.999999").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("timestamp").build(); - assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09"); // bound should include the same hour for upper bound assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-10"); assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11"); - assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01-10"); } From c18d4818628f8ec72b5f42271a4a68bd9fc5a556 Mon Sep 17 00:00:00 2001 From: Mouli Mukherjee Date: Mon, 15 Jul 2019 12:35:08 -0700 Subject: [PATCH 8/9] Removing unnecessary test --- .../iceberg/transforms/TestDatesProjection.java | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java index 02033a7d484c..879ed9f39ff8 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java @@ -97,7 +97,7 @@ public void testMonthInclusiveUpperBound() { } @Test - public void testDayStrictLowerBound() { + public void testDayStrict() { Integer date = (Integer) Literal.of("2017-01-01").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("date").build(); @@ -111,21 +111,6 @@ public void testDayStrictLowerBound() { } - @Test - public void testDayStrictUpperBound() { - Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value(); - PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("date").build(); - - assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2017-12-30"); - // should be the same date for <= - assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017-12-31"); - assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2018-01-01"); - // should be the same date for >= - assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017-12-31"); - assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017-12-31"); - - } - @Test public void testDayInclusive() { Integer date = (Integer) Literal.of("2017-12-01").to(TYPE).value(); From 715b76183a164050ec235ccc984112cf863e9fa5 Mon Sep 17 00:00:00 2001 From: Mouli Mukherjee Date: Mon, 15 Jul 2019 12:54:02 -0700 Subject: [PATCH 9/9] Format --- .../transforms/TestDatesProjection.java | 35 ++++++++++++++++--- .../transforms/TestTimestampsProjection.java | 35 ++++++++++++++++--- 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java index 879ed9f39ff8..5607684dcaf0 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iceberg.transforms; import org.apache.iceberg.PartitionSpec; @@ -11,15 +30,20 @@ import org.junit.Test; import static org.apache.iceberg.TestHelpers.assertAndUnwrapUnbound; -import static org.apache.iceberg.expressions.Expressions.*; +import static org.apache.iceberg.expressions.Expressions.equal; +import static org.apache.iceberg.expressions.Expressions.greaterThan; +import static org.apache.iceberg.expressions.Expressions.greaterThanOrEqual; +import static org.apache.iceberg.expressions.Expressions.lessThan; +import static org.apache.iceberg.expressions.Expressions.lessThanOrEqual; +import static org.apache.iceberg.expressions.Expressions.notEqual; import static org.apache.iceberg.types.Types.NestedField.optional; public class TestDatesProjection { private static final Types.DateType TYPE = Types.DateType.get(); - private static final Schema SCHEMA = new Schema( - optional(1, "date", TYPE)); + private static final Schema SCHEMA = new Schema(optional(1, "date", TYPE)); - public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate filter, Expression.Operation expectedOp, String expectedLiteral) { + public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate filter, + Expression.Operation expectedOp, String expectedLiteral) { Expression projection = Projections.strict(spec).project(filter); UnboundPredicate predicate = assertAndUnwrapUnbound(projection); @@ -33,7 +57,8 @@ public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate filte } - public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate filter, Expression.Operation expectedOp, String expectedLiteral) { + public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate filter, + Expression.Operation expectedOp, String expectedLiteral) { Expression projection = Projections.inclusive(spec).project(filter); UnboundPredicate predicate = assertAndUnwrapUnbound(projection); diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java index d136e78b380d..4e44b61c9c75 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iceberg.transforms; import org.apache.iceberg.PartitionSpec; @@ -11,15 +30,20 @@ import org.junit.Test; import static org.apache.iceberg.TestHelpers.assertAndUnwrapUnbound; -import static org.apache.iceberg.expressions.Expressions.*; +import static org.apache.iceberg.expressions.Expressions.equal; +import static org.apache.iceberg.expressions.Expressions.greaterThan; +import static org.apache.iceberg.expressions.Expressions.greaterThanOrEqual; +import static org.apache.iceberg.expressions.Expressions.lessThan; +import static org.apache.iceberg.expressions.Expressions.lessThanOrEqual; +import static org.apache.iceberg.expressions.Expressions.notEqual; import static org.apache.iceberg.types.Types.NestedField.optional; public class TestTimestampsProjection { private static final Types.TimestampType TYPE = Types.TimestampType.withoutZone(); - private static final Schema SCHEMA = new Schema( - optional(1, "timestamp", TYPE)); + private static final Schema SCHEMA = new Schema(optional(1, "timestamp", TYPE)); - public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate filter, Expression.Operation expectedOp, String expectedLiteral) { + public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate filter, + Expression.Operation expectedOp, String expectedLiteral) { Expression projection = Projections.strict(spec).project(filter); UnboundPredicate predicate = assertAndUnwrapUnbound(projection); @@ -33,7 +57,8 @@ public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate filte } - public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate filter, Expression.Operation expectedOp, String expectedLiteral) { + public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate filter, + Expression.Operation expectedOp, String expectedLiteral) { Expression projection = Projections.inclusive(spec).project(filter); UnboundPredicate predicate = assertAndUnwrapUnbound(projection);