Caching parsed expressions in sql planner

docs/configuration/index.md

@@ -1680,6 +1680,7 @@ The Druid SQL server is configured through the following properties on the Broke
 |`druid.sql.planner.sqlTimeZone`|Sets the default time zone for the server, which will affect how time functions and timestamp literals behave. Should be a time zone name like "America/Los_Angeles" or offset like "-08:00".|UTC|
 |`druid.sql.planner.metadataSegmentCacheEnable`|Whether to keep a cache of published segments in broker. If true, broker polls coordinator in background to get segments from metadata store and maintains a local cache. If false, coordinator's REST API will be invoked when broker needs published segments info.|false|
 |`druid.sql.planner.metadataSegmentPollPeriod`|How often to poll coordinator for published segments list if `druid.sql.planner.metadataSegmentCacheEnable` is set to true. Poll period is in milliseconds. |60000|
+|`druid.sql.planner.useParsedExprCache`|Whether to use a cache for parsed expressions. This cache is created per query and stored on heap memory. Enabling cache can be useful when planning time takes long.|false|
 
 > Previous versions of Druid had properties named `druid.sql.planner.maxQueryCount` and `druid.sql.planner.maxSemiJoinRowsInMemory`.
 > These properties are no longer available. Since Druid 0.18.0, you can use `druid.server.http.maxSubqueryRows` to control the maximum

docs/querying/sql.md

@@ -1008,6 +1008,7 @@ Connection context can be specified as JDBC connection properties or as a "conte
 |`sqlTimeZone`|Sets the time zone for this connection, which will affect how time functions and timestamp literals behave. Should be a time zone name like "America/Los_Angeles" or offset like "-08:00".|druid.sql.planner.sqlTimeZone on the Broker (default: UTC)|
 |`useApproximateCountDistinct`|Whether to use an approximate cardinality algorithm for `COUNT(DISTINCT foo)`.|druid.sql.planner.useApproximateCountDistinct on the Broker (default: true)|
 |`useApproximateTopN`|Whether to use approximate [TopN queries](topnquery.md) when a SQL query could be expressed as such. If false, exact [GroupBy queries](groupbyquery.md) will be used instead.|druid.sql.planner.useApproximateTopN on the Broker (default: true)|
+|`useParsedExprCache`|Whether to use a cache for parsed expressions. This cache is created per query and stored on heap memory. Enabling cache can be useful when planning time takes long.|`druid.sql.planner.useParsedExprCache` on the Broker (default: false)|
 
 ## Metadata tables
 

processing/src/main/java/org/apache/druid/query/JoinDataSource.java

@@ -23,10 +23,14 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableList;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.math.expr.Expr;
 import org.apache.druid.math.expr.ExprMacroTable;
+import org.apache.druid.math.expr.Parser;
 import org.apache.druid.query.filter.DimFilter;
 import org.apache.druid.segment.join.JoinConditionAnalysis;
 import org.apache.druid.segment.join.JoinPrefixUtils;
@@ -99,6 +103,27 @@ public static JoinDataSource create(
       @Nullable @JsonProperty("leftFilter") DimFilter leftFilter,
       @JacksonInject ExprMacroTable macroTable
   )
+  {
+    return create(
+        left,
+        right,
+        rightPrefix,
+        condition,
+        joinType,
+        leftFilter,
+        Suppliers.memoize(() -> Parser.parse(condition, macroTable))
+    );
+  }
+
+  public static JoinDataSource create(
+      DataSource left,
+      DataSource right,
+      String rightPrefix,
+      String condition,
+      JoinType joinType,
+      DimFilter leftFilter,
+      Supplier<Expr> conditionExprSupplier
+  )
   {
     return new JoinDataSource(
         left,
@@ -107,7 +132,7 @@ public static JoinDataSource create(
         JoinConditionAnalysis.forExpression(
             Preconditions.checkNotNull(condition, "condition"),
             StringUtils.nullToEmptyNonDruidDataString(rightPrefix),
-            macroTable
+            conditionExprSupplier
         ),
         joinType,
         leftFilter

processing/src/main/java/org/apache/druid/query/aggregation/DoubleMaxAggregatorFactory.java

@@ -22,7 +22,9 @@
 import com.fasterxml.jackson.annotation.JacksonInject;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Supplier;
 import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.math.expr.Expr;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.segment.BaseDoubleColumnValueSelector;
 import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
@@ -48,6 +50,17 @@ public DoubleMaxAggregatorFactory(
     super(macroTable, name, fieldName, expression);
   }
 
+  public DoubleMaxAggregatorFactory(
+      String name,
+      String fieldName,
+      @Nullable String expression,
+      Supplier<Expr> expressionSupplier,
+      ExprMacroTable macroTable
+  )
+  {
+    super(macroTable, name, fieldName, expression, expressionSupplier);
+  }
+
   public DoubleMaxAggregatorFactory(String name, String fieldName)
   {
     this(name, fieldName, null, ExprMacroTable.nil());

processing/src/main/java/org/apache/druid/query/aggregation/DoubleMinAggregatorFactory.java

@@ -22,7 +22,9 @@
 import com.fasterxml.jackson.annotation.JacksonInject;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Supplier;
 import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.math.expr.Expr;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.segment.BaseDoubleColumnValueSelector;
 import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
@@ -48,6 +50,17 @@ public DoubleMinAggregatorFactory(
     super(macroTable, name, fieldName, expression);
   }
 
+  public DoubleMinAggregatorFactory(
+      String name,
+      String fieldName,
+      @Nullable String expression,
+      Supplier<Expr> expressionSupplier,
+      ExprMacroTable macroTable
+  )
+  {
+    super(macroTable, name, fieldName, expression, expressionSupplier);
+  }
+
   public DoubleMinAggregatorFactory(String name, String fieldName)
   {
     this(name, fieldName, null, ExprMacroTable.nil());

processing/src/main/java/org/apache/druid/query/aggregation/DoubleSumAggregatorFactory.java

@@ -22,7 +22,9 @@
 import com.fasterxml.jackson.annotation.JacksonInject;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Supplier;
 import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.math.expr.Expr;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.segment.BaseDoubleColumnValueSelector;
 import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
@@ -48,6 +50,17 @@ public DoubleSumAggregatorFactory(
     super(macroTable, name, fieldName, expression);
   }
 
+  public DoubleSumAggregatorFactory(
+      String name,
+      String fieldName,
+      @Nullable String expression,
+      Supplier<Expr> expressionSupplier,
+      ExprMacroTable macroTable
+  )
+  {
+    super(macroTable, name, fieldName, expression, expressionSupplier);
+  }
+
   public DoubleSumAggregatorFactory(String name, String fieldName)
   {
     this(name, fieldName, null, ExprMacroTable.nil());

processing/src/main/java/org/apache/druid/query/aggregation/FloatMaxAggregatorFactory.java

@@ -22,7 +22,9 @@
 import com.fasterxml.jackson.annotation.JacksonInject;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Supplier;
 import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.math.expr.Expr;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.segment.BaseFloatColumnValueSelector;
 import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
@@ -48,6 +50,17 @@ public FloatMaxAggregatorFactory(
     super(macroTable, name, fieldName, expression);
   }
 
+  public FloatMaxAggregatorFactory(
+      String name,
+      String fieldName,
+      @Nullable String expression,
+      Supplier<Expr> expressionSupplier,
+      ExprMacroTable macroTable
+  )
+  {
+    super(macroTable, name, fieldName, expression, expressionSupplier);
+  }
+
   public FloatMaxAggregatorFactory(String name, String fieldName)
   {
     this(name, fieldName, null, ExprMacroTable.nil());

processing/src/main/java/org/apache/druid/query/aggregation/FloatMinAggregatorFactory.java

@@ -22,7 +22,9 @@
 import com.fasterxml.jackson.annotation.JacksonInject;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Supplier;
 import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.math.expr.Expr;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.segment.BaseFloatColumnValueSelector;
 import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
@@ -48,6 +50,17 @@ public FloatMinAggregatorFactory(
     super(macroTable, name, fieldName, expression);
   }
 
+  public FloatMinAggregatorFactory(
+      String name,
+      String fieldName,
+      @Nullable String expression,
+      Supplier<Expr> expressionSupplier,
+      ExprMacroTable macroTable
+  )
+  {
+    super(macroTable, name, fieldName, expression, expressionSupplier);
+  }
+
   public FloatMinAggregatorFactory(String name, String fieldName)
   {
     this(name, fieldName, null, ExprMacroTable.nil());

processing/src/main/java/org/apache/druid/query/aggregation/FloatSumAggregatorFactory.java

@@ -22,7 +22,9 @@
 import com.fasterxml.jackson.annotation.JacksonInject;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Supplier;
 import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.math.expr.Expr;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.segment.BaseFloatColumnValueSelector;
 import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
@@ -48,6 +50,17 @@ public FloatSumAggregatorFactory(
     super(macroTable, name, fieldName, expression);
   }
 
+  public FloatSumAggregatorFactory(
+      String name,
+      String fieldName,
+      @Nullable String expression,
+      Supplier<Expr> expressionSupplier,
+      ExprMacroTable macroTable
+  )
+  {
+    super(macroTable, name, fieldName, expression, expressionSupplier);
+  }
+
   public FloatSumAggregatorFactory(String name, String fieldName)
   {
     this(name, fieldName, null, ExprMacroTable.nil());

processing/src/main/java/org/apache/druid/query/aggregation/LongMaxAggregatorFactory.java

@@ -22,7 +22,9 @@
 import com.fasterxml.jackson.annotation.JacksonInject;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Supplier;
 import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.math.expr.Expr;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.segment.BaseLongColumnValueSelector;
 import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
@@ -48,6 +50,17 @@ public LongMaxAggregatorFactory(
     super(macroTable, name, fieldName, expression);
   }
 
+  public LongMaxAggregatorFactory(
+      String name,
+      String fieldName,
+      @Nullable String expression,
+      Supplier<Expr> expressionSupplier,
+      ExprMacroTable macroTable
+  )
+  {
+    super(macroTable, name, fieldName, expression, expressionSupplier);
+  }
+
   public LongMaxAggregatorFactory(String name, String fieldName)
   {
     this(name, fieldName, null, ExprMacroTable.nil());

processing/src/main/java/org/apache/druid/query/aggregation/LongMinAggregatorFactory.java

@@ -22,7 +22,9 @@
 import com.fasterxml.jackson.annotation.JacksonInject;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Supplier;
 import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.math.expr.Expr;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.segment.BaseLongColumnValueSelector;
 import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
@@ -48,6 +50,17 @@ public LongMinAggregatorFactory(
     super(macroTable, name, fieldName, expression);
   }
 
+  public LongMinAggregatorFactory(
+      String name,
+      String fieldName,
+      @Nullable String expression,
+      Supplier<Expr> expressionSupplier,
+      ExprMacroTable macroTable
+  )
+  {
+    super(macroTable, name, fieldName, expression, expressionSupplier);
+  }
+
   public LongMinAggregatorFactory(String name, String fieldName)
   {
     this(name, fieldName, null, ExprMacroTable.nil());

processing/src/main/java/org/apache/druid/query/aggregation/LongSumAggregatorFactory.java

@@ -22,7 +22,9 @@
 import com.fasterxml.jackson.annotation.JacksonInject;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Supplier;
 import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.math.expr.Expr;
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.segment.BaseLongColumnValueSelector;
 import org.apache.druid.segment.vector.VectorColumnSelectorFactory;
@@ -48,6 +50,17 @@ public LongSumAggregatorFactory(
     super(macroTable, name, fieldName, expression);
   }
 
+  public LongSumAggregatorFactory(
+      String name,
+      String fieldName,
+      @Nullable String expression,
+      Supplier<Expr> expressionSupplier,
+      ExprMacroTable macroTable
+  )
+  {
+    super(macroTable, name, fieldName, expression, expressionSupplier);
+  }
+
   public LongSumAggregatorFactory(String name, String fieldName)
   {
     this(name, fieldName, null, ExprMacroTable.nil());

processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java

@@ -66,13 +66,30 @@ public SimpleDoubleAggregatorFactory(
       @Nullable final String fieldName,
       @Nullable String expression
   )
+  {
+    this(
+        macroTable,
+        name,
+        fieldName,
+        expression,
+        Suppliers.memoize(() -> expression == null ? null : Parser.parse(expression, macroTable))
+    );
+  }
+
+  public SimpleDoubleAggregatorFactory(
+      ExprMacroTable macroTable,
+      String name,
+      String fieldName,
+      @Nullable String expression,
+      Supplier<Expr> expressionSupplier
+  )
   {
     this.macroTable = macroTable;
     this.name = name;
     this.fieldName = fieldName;
     this.expression = expression;
     this.storeDoubleAsFloat = ColumnHolder.storeDoubleAsFloat();
-    this.fieldExpression = Suppliers.memoize(() -> expression == null ? null : Parser.parse(expression, macroTable));
+    this.fieldExpression = expressionSupplier;
     Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name");
     Preconditions.checkArgument(
         fieldName == null ^ expression == null,

processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java

@@ -58,12 +58,29 @@ public SimpleFloatAggregatorFactory(
       @Nullable final String fieldName,
       @Nullable String expression
   )
+  {
+    this(
+        macroTable,
+        name,
+        fieldName,
+        expression,
+        Suppliers.memoize(() -> expression == null ? null : Parser.parse(expression, macroTable))
+    );
+  }
+
+  public SimpleFloatAggregatorFactory(
+      ExprMacroTable macroTable,
+      String name,
+      String fieldName,
+      @Nullable String expression,
+      Supplier<Expr> expressionSupplier
+  )
   {
     this.macroTable = macroTable;
     this.name = name;
     this.fieldName = fieldName;
     this.expression = expression;
-    this.fieldExpression = Suppliers.memoize(() -> expression == null ? null : Parser.parse(expression, macroTable));
+    this.fieldExpression = expressionSupplier;
     Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name");
     Preconditions.checkArgument(
         fieldName == null ^ expression == null,