diff --git a/docs/content/misc/math-expr.md b/docs/content/misc/math-expr.md index 07dcc5018082..abcebdd3b5e3 100644 --- a/docs/content/misc/math-expr.md +++ b/docs/content/misc/math-expr.md @@ -62,7 +62,7 @@ The following built-in functions are available. |timestamp_ceil|timestamp_ceil(expr, period, \[origin, \[timezone\]\]) rounds up a timestamp, returning it as a new timestamp. Period can be any ISO8601 period, like P3M (quarters) or PT12H (half-days). The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00".| |timestamp_floor|timestamp_floor(expr, period, \[origin, [timezone\]\]) rounds down a timestamp, returning it as a new timestamp. Period can be any ISO8601 period, like P3M (quarters) or PT12H (half-days). The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00".| |timestamp_shift|timestamp_shift(expr, period, step, \[timezone\]) shifts a timestamp by a period (step times), returning it as a new timestamp. Period can be any ISO8601 period. Step may be negative. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00".| -|timestamp_extract|timestamp_extract(expr, unit, \[timezone\]) extracts a time part from expr, returning it as a number. Unit can be EPOCH, SECOND, MINUTE, HOUR, DAY (day of month), DOW (day of week), DOY (day of year), WEEK (week of [week year](https://en.wikipedia.org/wiki/ISO_week_date)), MONTH (1 through 12), QUARTER (1 through 4), or YEAR. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00"| +|timestamp_extract|timestamp_extract(expr, unit, \[timezone\]) extracts a time part from expr, returning it as a number. Unit can be EPOCH (number of seconds since 1970-01-01 00:00:00 UTC), SECOND, MINUTE, HOUR, DAY (day of month), DOW (day of week), DOY (day of year), WEEK (week of [week year](https://en.wikipedia.org/wiki/ISO_week_date)), MONTH (1 through 12), QUARTER (1 through 4), or YEAR. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00"| |timestamp_parse|timestamp_parse(string expr, \[pattern, [timezone\]\]) parses a string into a timestamp using a given [Joda DateTimeFormat pattern](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html). If the pattern is not provided, this parses time strings in either ISO8601 or SQL format. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00", and will be used as the time zone for strings that do not include a time zone offset. Pattern and time zone must be literals. Strings that cannot be parsed as timestamps will be returned as nulls.| |timestamp_format|timestamp_format(expr, \[pattern, \[timezone\]\]) formats a timestamp as a string with a given [Joda DateTimeFormat pattern](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html), or ISO8601 if the pattern is not provided. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00". Pattern and time zone must be literals.| diff --git a/pom.xml b/pom.xml index a4c039532dfc..fe9546068c3d 100644 --- a/pom.xml +++ b/pom.xml @@ -62,7 +62,7 @@ 4.0.0 2.12.0 1.10.0 - 1.14.0 + 1.15.0 16.0.1 4.1.0 9.3.19.v20170502 diff --git a/processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java b/processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java index 19c64415e5fa..bb1b2ca5f243 100644 --- a/processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java @@ -91,7 +91,7 @@ public ExprEval eval(final ObjectBinding bindings) final DateTime dateTime = new DateTime(arg.eval(bindings).asLong(), chronology); switch (unit) { case EPOCH: - return ExprEval.of(dateTime.getMillis()); + return ExprEval.of(dateTime.getMillis() / 1000); case SECOND: return ExprEval.of(dateTime.secondOfMinute().get()); case MINUTE: diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java b/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java index 86475a390492..b779fdd03a0c 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java @@ -35,7 +35,7 @@ import java.util.function.Function; /** - * Represents three kinds of expression-like concepts that native Druid queries support: + * Represents two kinds of expression-like concepts that native Druid queries support: * * (1) SimpleExtractions, which are direct column access, possibly with an extractionFn * (2) native Druid expressions. diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/builtin/ExtractOperatorConversion.java b/sql/src/main/java/io/druid/sql/calcite/expression/builtin/ExtractOperatorConversion.java index 8b6de78be82b..e2cd31005e5a 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/builtin/ExtractOperatorConversion.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/builtin/ExtractOperatorConversion.java @@ -39,6 +39,7 @@ public class ExtractOperatorConversion implements SqlOperatorConversion { private static final Map EXTRACT_UNIT_MAP = ImmutableMap.builder() + .put(TimeUnitRange.EPOCH, TimestampExtractExprMacro.Unit.EPOCH) .put(TimeUnitRange.SECOND, TimestampExtractExprMacro.Unit.SECOND) .put(TimeUnitRange.MINUTE, TimestampExtractExprMacro.Unit.MINUTE) .put(TimeUnitRange.HOUR, TimestampExtractExprMacro.Unit.HOUR) @@ -75,11 +76,6 @@ public DruidExpression toDruidExpression( return null; } - if (call.getOperator().getName().equals("EXTRACT_DATE")) { - // Arg will be in number of days since the epoch. Can't translate. - return null; - } - final TimestampExtractExprMacro.Unit druidUnit = EXTRACT_UNIT_MAP.get(calciteUnit); if (druidUnit == null) { // Don't know how to extract this time unit. diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index 97502ea8c8cd..7939ebec1acc 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -2514,7 +2514,6 @@ public void testCaseFilteredAggregationWithGroupBy() throws Exception } @Test - @Ignore // https://issues.apache.org/jira/browse/CALCITE-1910 public void testFilteredAggregationWithNotIn() throws Exception { testQuery( @@ -2527,12 +2526,26 @@ public void testFilteredAggregationWithNotIn() throws Exception .dataSource(CalciteTests.DATASOURCE1) .intervals(QSS(Filtration.eternity())) .granularity(Granularities.ALL) - .aggregators(AGGS()) + .aggregators( + AGGS( + new FilteredAggregatorFactory( + new CountAggregatorFactory("a0"), + NOT(SELECTOR("dim1", "1", null)) + ), + new FilteredAggregatorFactory( + new CountAggregatorFactory("a1"), + AND( + NOT(SELECTOR("dim2", null, null)), + NOT(SELECTOR("dim1", "1", null)) + ) + ) + ) + ) .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), ImmutableList.of( - new Object[]{1L, 5L} + new Object[]{5L, 2L} ) ); } @@ -2817,6 +2830,27 @@ public void testCountStarWithNotOfDegenerateFilter() throws Exception ); } + @Test + public void testCountStarWithBoundFilterSimplifyOnMetric() throws Exception + { + testQuery( + "SELECT COUNT(*) FROM druid.foo WHERE 2.5 < m1 AND m1 < 3.5", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(QSS(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(BOUND("m1", "2.5", "3.5", true, true, null, StringComparators.NUMERIC)) + .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .context(TIMESERIES_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{1L} + ) + ); + } + @Test public void testCountStarWithBoundFilterSimplifyOr() throws Exception { @@ -3201,6 +3235,39 @@ public void testCountStarWithTimeFilterOnLongColumnUsingExtractEpoch() throws Ex ); } + @Test + public void testCountStarWithTimeFilterOnLongColumnUsingExtractEpochFromDate() throws Exception + { + testQuery( + "SELECT COUNT(*) FROM druid.foo WHERE " + + "cnt >= EXTRACT(EPOCH FROM DATE '1970-01-01') * 1000 " + + "AND cnt < EXTRACT(EPOCH FROM DATE '1970-01-02') * 1000", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(QSS(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters( + BOUND( + "cnt", + String.valueOf(DateTimes.of("1970-01-01").getMillis()), + String.valueOf(DateTimes.of("1970-01-02").getMillis()), + false, + true, + null, + StringComparators.NUMERIC + ) + ) + .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .context(TIMESERIES_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{6L} + ) + ); + } + @Test public void testCountStarWithTimeFilterOnLongColumnUsingTimestampToMillis() throws Exception { @@ -4918,7 +4985,6 @@ public void testFilterOnTimeFloorComparisonMisaligned() throws Exception } @Test - @Ignore // https://issues.apache.org/jira/browse/CALCITE-1601 public void testFilterOnTimeExtract() throws Exception { testQuery( @@ -4928,9 +4994,15 @@ public void testFilterOnTimeExtract() throws Exception ImmutableList.of( Druids.newTimeseriesQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) - .intervals(QSS(Intervals.of("2000/P1M"))) + .intervals(QSS(Filtration.eternity())) .granularity(Granularities.ALL) .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .filters( + AND( + EXPRESSION_FILTER("(timestamp_extract(\"__time\",'YEAR','UTC') == 2000)"), + EXPRESSION_FILTER("(timestamp_extract(\"__time\",'MONTH','UTC') == 1)") + ) + ) .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), @@ -4941,24 +5013,33 @@ public void testFilterOnTimeExtract() throws Exception } @Test - @Ignore // https://issues.apache.org/jira/browse/CALCITE-1601 - public void testFilterOnTimeExtractWithMultipleMonths() throws Exception + public void testFilterOnTimeExtractWithMultipleDays() throws Exception { testQuery( "SELECT COUNT(*) FROM druid.foo\n" + "WHERE EXTRACT(YEAR FROM __time) = 2000\n" - + "AND EXTRACT(MONTH FROM __time) IN (2, 3, 5)", + + "AND EXTRACT(DAY FROM __time) IN (2, 3, 5)", ImmutableList.of( Druids.newTimeseriesQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) - .intervals(QSS(Intervals.of("2000-02-01/P2M"), Intervals.of("2000-05-01/P1M"))) + .intervals(QSS(Filtration.eternity())) .granularity(Granularities.ALL) .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .filters( + AND( + EXPRESSION_FILTER("(timestamp_extract(\"__time\",'YEAR','UTC') == 2000)"), + OR( + EXPRESSION_FILTER("(timestamp_extract(\"__time\",'DAY','UTC') == 2)"), + EXPRESSION_FILTER("(timestamp_extract(\"__time\",'DAY','UTC') == 3)"), + EXPRESSION_FILTER("(timestamp_extract(\"__time\",'DAY','UTC') == 5)") + ) + ) + ) .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), ImmutableList.of( - new Object[]{3L} + new Object[]{2L} ) ); }