Upgrade to Calcite 1.15.0

docs/content/misc/math-expr.md

@@ -62,7 +62,7 @@ The following built-in functions are available.
 |timestamp_ceil|timestamp_ceil(expr, period, \[origin, \[timezone\]\]) rounds up a timestamp, returning it as a new timestamp. Period can be any ISO8601 period, like P3M (quarters) or PT12H (half-days). The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00".|
 |timestamp_floor|timestamp_floor(expr, period, \[origin, [timezone\]\]) rounds down a timestamp, returning it as a new timestamp. Period can be any ISO8601 period, like P3M (quarters) or PT12H (half-days). The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00".|
 |timestamp_shift|timestamp_shift(expr, period, step, \[timezone\]) shifts a timestamp by a period (step times), returning it as a new timestamp. Period can be any ISO8601 period. Step may be negative. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00".|
-|timestamp_extract|timestamp_extract(expr, unit, \[timezone\]) extracts a time part from expr, returning it as a number. Unit can be EPOCH, SECOND, MINUTE, HOUR, DAY (day of month), DOW (day of week), DOY (day of year), WEEK (week of [week year](https://en.wikipedia.org/wiki/ISO_week_date)), MONTH (1 through 12), QUARTER (1 through 4), or YEAR. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00"|
+|timestamp_extract|timestamp_extract(expr, unit, \[timezone\]) extracts a time part from expr, returning it as a number. Unit can be EPOCH (number of seconds since 1970-01-01 00:00:00 UTC), SECOND, MINUTE, HOUR, DAY (day of month), DOW (day of week), DOY (day of year), WEEK (week of [week year](https://en.wikipedia.org/wiki/ISO_week_date)), MONTH (1 through 12), QUARTER (1 through 4), or YEAR. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00"|
 |timestamp_parse|timestamp_parse(string expr, \[pattern, [timezone\]\]) parses a string into a timestamp using a given [Joda DateTimeFormat pattern](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html). If the pattern is not provided, this parses time strings in either ISO8601 or SQL format. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00", and will be used as the time zone for strings that do not include a time zone offset. Pattern and time zone must be literals. Strings that cannot be parsed as timestamps will be returned as nulls.|
 |timestamp_format|timestamp_format(expr, \[pattern, \[timezone\]\]) formats a timestamp as a string with a given [Joda DateTimeFormat pattern](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html), or ISO8601 if the pattern is not provided. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00". Pattern and time zone must be literals.|
 

pom.xml

@@ -62,7 +62,7 @@
         <apache.curator.version>4.0.0</apache.curator.version>
         <apache.curator.test.version>2.12.0</apache.curator.test.version>
         <avatica.version>1.10.0</avatica.version>
-        <calcite.version>1.14.0</calcite.version>
+        <calcite.version>1.15.0</calcite.version>
         <guava.version>16.0.1</guava.version>
         <guice.version>4.1.0</guice.version>
         <jetty.version>9.3.19.v20170502</jetty.version>

processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java

@@ -91,7 +91,7 @@ public ExprEval eval(final ObjectBinding bindings)
         final DateTime dateTime = new DateTime(arg.eval(bindings).asLong(), chronology);
         switch (unit) {
           case EPOCH:
-            return ExprEval.of(dateTime.getMillis());
+            return ExprEval.of(dateTime.getMillis() / 1000);
           case SECOND:
             return ExprEval.of(dateTime.secondOfMinute().get());
           case MINUTE:

sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java

@@ -35,7 +35,7 @@
 import java.util.function.Function;
 
 /**
- * Represents three kinds of expression-like concepts that native Druid queries support:
+ * Represents two kinds of expression-like concepts that native Druid queries support:
  *
  * (1) SimpleExtractions, which are direct column access, possibly with an extractionFn
  * (2) native Druid expressions.

sql/src/main/java/io/druid/sql/calcite/expression/builtin/ExtractOperatorConversion.java

@@ -39,6 +39,7 @@ public class ExtractOperatorConversion implements SqlOperatorConversion
 {
   private static final Map<TimeUnitRange, TimestampExtractExprMacro.Unit> EXTRACT_UNIT_MAP =
       ImmutableMap.<TimeUnitRange, TimestampExtractExprMacro.Unit>builder()
+          .put(TimeUnitRange.EPOCH, TimestampExtractExprMacro.Unit.EPOCH)
           .put(TimeUnitRange.SECOND, TimestampExtractExprMacro.Unit.SECOND)
           .put(TimeUnitRange.MINUTE, TimestampExtractExprMacro.Unit.MINUTE)
           .put(TimeUnitRange.HOUR, TimestampExtractExprMacro.Unit.HOUR)
@@ -75,11 +76,6 @@ public DruidExpression toDruidExpression(
       return null;
     }
 
-    if (call.getOperator().getName().equals("EXTRACT_DATE")) {
-      // Arg will be in number of days since the epoch. Can't translate.
-      return null;
-    }
-
     final TimestampExtractExprMacro.Unit druidUnit = EXTRACT_UNIT_MAP.get(calciteUnit);
     if (druidUnit == null) {
       // Don't know how to extract this time unit.

sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java

@@ -2514,7 +2514,6 @@ public void testCaseFilteredAggregationWithGroupBy() throws Exception
   }
 
   @Test
-  @Ignore // https://issues.apache.org/jira/browse/CALCITE-1910
   public void testFilteredAggregationWithNotIn() throws Exception
   {
     testQuery(
@@ -2527,12 +2526,26 @@ public void testFilteredAggregationWithNotIn() throws Exception
                   .dataSource(CalciteTests.DATASOURCE1)
                   .intervals(QSS(Filtration.eternity()))
                   .granularity(Granularities.ALL)
-                  .aggregators(AGGS())
+                  .aggregators(
+                      AGGS(
+                          new FilteredAggregatorFactory(
+                              new CountAggregatorFactory("a0"),
+                              NOT(SELECTOR("dim1", "1", null))
+                          ),
+                          new FilteredAggregatorFactory(
+                              new CountAggregatorFactory("a1"),
+                              AND(
+                                  NOT(SELECTOR("dim2", null, null)),
+                                  NOT(SELECTOR("dim1", "1", null))
+                              )
+                          )
+                      )
+                  )
                   .context(TIMESERIES_CONTEXT_DEFAULT)
                   .build()
         ),
         ImmutableList.of(
-            new Object[]{1L, 5L}
+            new Object[]{5L, 2L}
         )
     );
   }
@@ -2817,6 +2830,27 @@ public void testCountStarWithNotOfDegenerateFilter() throws Exception
     );
   }
 
+  @Test
+  public void testCountStarWithBoundFilterSimplifyOnMetric() throws Exception
+  {
+    testQuery(
+        "SELECT COUNT(*) FROM druid.foo WHERE 2.5 < m1 AND m1 < 3.5",
+        ImmutableList.of(
+            Druids.newTimeseriesQueryBuilder()
+                  .dataSource(CalciteTests.DATASOURCE1)
+                  .intervals(QSS(Filtration.eternity()))
+                  .granularity(Granularities.ALL)
+                  .filters(BOUND("m1", "2.5", "3.5", true, true, null, StringComparators.NUMERIC))
+                  .aggregators(AGGS(new CountAggregatorFactory("a0")))
+                  .context(TIMESERIES_CONTEXT_DEFAULT)
+                  .build()
+        ),
+        ImmutableList.of(
+            new Object[]{1L}
+        )
+    );
+  }
+
   @Test
   public void testCountStarWithBoundFilterSimplifyOr() throws Exception
   {
@@ -3201,6 +3235,39 @@ public void testCountStarWithTimeFilterOnLongColumnUsingExtractEpoch() throws Ex
     );
   }
 
+  @Test
+  public void testCountStarWithTimeFilterOnLongColumnUsingExtractEpochFromDate() throws Exception
+  {
+    testQuery(
+        "SELECT COUNT(*) FROM druid.foo WHERE "
+        + "cnt >= EXTRACT(EPOCH FROM DATE '1970-01-01') * 1000 "
+        + "AND cnt < EXTRACT(EPOCH FROM DATE '1970-01-02') * 1000",
+        ImmutableList.of(
+            Druids.newTimeseriesQueryBuilder()
+                  .dataSource(CalciteTests.DATASOURCE1)
+                  .intervals(QSS(Filtration.eternity()))
+                  .granularity(Granularities.ALL)
+                  .filters(
+                      BOUND(
+                          "cnt",
+                          String.valueOf(DateTimes.of("1970-01-01").getMillis()),
+                          String.valueOf(DateTimes.of("1970-01-02").getMillis()),
+                          false,
+                          true,
+                          null,
+                          StringComparators.NUMERIC
+                      )
+                  )
+                  .aggregators(AGGS(new CountAggregatorFactory("a0")))
+                  .context(TIMESERIES_CONTEXT_DEFAULT)
+                  .build()
+        ),
+        ImmutableList.of(
+            new Object[]{6L}
+        )
+    );
+  }
+
   @Test
   public void testCountStarWithTimeFilterOnLongColumnUsingTimestampToMillis() throws Exception
   {
@@ -4918,7 +4985,6 @@ public void testFilterOnTimeFloorComparisonMisaligned() throws Exception
   }
 
   @Test
-  @Ignore // https://issues.apache.org/jira/browse/CALCITE-1601
   public void testFilterOnTimeExtract() throws Exception
   {
     testQuery(
@@ -4928,9 +4994,15 @@ public void testFilterOnTimeExtract() throws Exception
         ImmutableList.of(
             Druids.newTimeseriesQueryBuilder()
                   .dataSource(CalciteTests.DATASOURCE1)
-                  .intervals(QSS(Intervals.of("2000/P1M")))
+                  .intervals(QSS(Filtration.eternity()))
                   .granularity(Granularities.ALL)
                   .aggregators(AGGS(new CountAggregatorFactory("a0")))
+                  .filters(
+                      AND(
+                          EXPRESSION_FILTER("(timestamp_extract(\"__time\",'YEAR','UTC') == 2000)"),
+                          EXPRESSION_FILTER("(timestamp_extract(\"__time\",'MONTH','UTC') == 1)")
+                      )
+                  )
                   .context(TIMESERIES_CONTEXT_DEFAULT)
                   .build()
         ),
@@ -4941,24 +5013,33 @@ public void testFilterOnTimeExtract() throws Exception
   }
 
   @Test
-  @Ignore // https://issues.apache.org/jira/browse/CALCITE-1601
-  public void testFilterOnTimeExtractWithMultipleMonths() throws Exception
+  public void testFilterOnTimeExtractWithMultipleDays() throws Exception
   {
     testQuery(
         "SELECT COUNT(*) FROM druid.foo\n"
         + "WHERE EXTRACT(YEAR FROM __time) = 2000\n"
-        + "AND EXTRACT(MONTH FROM __time) IN (2, 3, 5)",
+        + "AND EXTRACT(DAY FROM __time) IN (2, 3, 5)",
         ImmutableList.of(
             Druids.newTimeseriesQueryBuilder()
                   .dataSource(CalciteTests.DATASOURCE1)
-                  .intervals(QSS(Intervals.of("2000-02-01/P2M"), Intervals.of("2000-05-01/P1M")))
+                  .intervals(QSS(Filtration.eternity()))
                   .granularity(Granularities.ALL)
                   .aggregators(AGGS(new CountAggregatorFactory("a0")))
+                  .filters(
+                      AND(
+                          EXPRESSION_FILTER("(timestamp_extract(\"__time\",'YEAR','UTC') == 2000)"),
+                          OR(
+                              EXPRESSION_FILTER("(timestamp_extract(\"__time\",'DAY','UTC') == 2)"),
+                              EXPRESSION_FILTER("(timestamp_extract(\"__time\",'DAY','UTC') == 3)"),
+                              EXPRESSION_FILTER("(timestamp_extract(\"__time\",'DAY','UTC') == 5)")
+                          )
+                      )
+                  )
                   .context(TIMESERIES_CONTEXT_DEFAULT)
                   .build()
         ),
         ImmutableList.of(
-            new Object[]{3L}
+            new Object[]{2L}
         )
     );
   }