diff --git a/common/src/main/java/io/druid/common/utils/StringUtils.java b/common/src/main/java/io/druid/common/utils/StringUtils.java index 6499ebc5817c..3e9cf8af477a 100644 --- a/common/src/main/java/io/druid/common/utils/StringUtils.java +++ b/common/src/main/java/io/druid/common/utils/StringUtils.java @@ -23,7 +23,7 @@ */ public class StringUtils extends com.metamx.common.StringUtils { - public static final String EMPTY = ""; + private static final byte[] EMPTY_BYTES = new byte[0]; // should be used only for estimation // returns the same result with StringUtils.fromUtf8(value).length for valid string values @@ -46,4 +46,9 @@ public static int estimatedBinaryLengthAsUTF8(String value) } return length; } + + public static byte[] toUtf8WithNullToEmpty(final String string) + { + return string == null ? EMPTY_BYTES : toUtf8(string); + } } diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java index ac7aa321137e..f375d2d60ddf 100644 --- a/common/src/main/java/io/druid/math/expr/Expr.java +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -22,13 +22,17 @@ import com.google.common.math.LongMath; import java.util.List; -import java.util.Map; /** */ public interface Expr { - Number eval(Map bindings); + Number eval(NumericBinding bindings); + + interface NumericBinding + { + Number get(String name); + } } class LongExpr implements Expr @@ -47,7 +51,7 @@ public String toString() } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { return value; } @@ -69,7 +73,7 @@ public String toString() } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { return value; } @@ -91,7 +95,7 @@ public String toString() } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { Number val = bindings.get(value); if (val == null) { @@ -104,8 +108,8 @@ public Number eval(Map bindings) class FunctionExpr implements Expr { - private final String name; - private final List args; + final String name; + final List args; public FunctionExpr(String name, List args) { @@ -120,7 +124,7 @@ public String toString() } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { return Parser.func.get(name.toLowerCase()).apply(args, bindings); } @@ -128,7 +132,7 @@ public Number eval(Map bindings) class UnaryMinusExpr implements Expr { - private final Expr expr; + final Expr expr; UnaryMinusExpr(Expr expr) { @@ -136,7 +140,7 @@ class UnaryMinusExpr implements Expr } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { Number valObj = expr.eval(bindings); if (valObj instanceof Long) { @@ -155,7 +159,7 @@ public String toString() class UnaryNotExpr implements Expr { - private final Expr expr; + final Expr expr; UnaryNotExpr(Expr expr) { @@ -163,7 +167,7 @@ class UnaryNotExpr implements Expr } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { Number valObj = expr.eval(bindings); return valObj.doubleValue() > 0 ? 0.0d : 1.0d; @@ -210,7 +214,7 @@ class BinMinusExpr extends BinaryOpExprBase } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); @@ -231,7 +235,7 @@ class BinPowExpr extends BinaryOpExprBase } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); @@ -252,7 +256,7 @@ class BinMulExpr extends BinaryOpExprBase } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); @@ -273,7 +277,7 @@ class BinDivExpr extends BinaryOpExprBase } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); @@ -294,7 +298,7 @@ class BinModuloExpr extends BinaryOpExprBase } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); @@ -315,7 +319,7 @@ class BinPlusExpr extends BinaryOpExprBase } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); @@ -336,7 +340,7 @@ class BinLtExpr extends BinaryOpExprBase } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); @@ -357,7 +361,7 @@ class BinLeqExpr extends BinaryOpExprBase } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); @@ -378,7 +382,7 @@ class BinGtExpr extends BinaryOpExprBase } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); @@ -399,7 +403,7 @@ class BinGeqExpr extends BinaryOpExprBase } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); @@ -420,7 +424,7 @@ class BinEqExpr extends BinaryOpExprBase } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); @@ -441,7 +445,7 @@ class BinNeqExpr extends BinaryOpExprBase } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); @@ -462,7 +466,7 @@ class BinAndExpr extends BinaryOpExprBase } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); @@ -495,7 +499,7 @@ class BinOrExpr extends BinaryOpExprBase } @Override - public Number eval(Map bindings) + public Number eval(NumericBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java index 9ccd73fc34a9..931da8599740 100644 --- a/common/src/main/java/io/druid/math/expr/Function.java +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -20,7 +20,6 @@ package io.druid.math.expr; import java.util.List; -import java.util.Map; /** */ @@ -28,12 +27,12 @@ interface Function { String name(); - Number apply(List args, Map bindings); + Number apply(List args, Expr.NumericBinding bindings); abstract class SingleParam implements Function { @Override - public Number apply(List args, Map bindings) + public Number apply(List args, Expr.NumericBinding bindings) { if (args.size() != 1) { throw new RuntimeException("function '" + name() + "' needs 1 argument"); @@ -48,7 +47,7 @@ public Number apply(List args, Map bindings) abstract class DoubleParam implements Function { @Override - public Number apply(List args, Map bindings) + public Number apply(List args, Expr.NumericBinding bindings) { if (args.size() != 2) { throw new RuntimeException("function '" + name() + "' needs 1 argument"); @@ -616,7 +615,7 @@ public String name() } @Override - public Number apply(List args, Map bindings) + public Number apply(List args, Expr.NumericBinding bindings) { if (args.size() != 3) { throw new RuntimeException("function 'if' needs 3 argument"); diff --git a/common/src/main/java/io/druid/math/expr/Parser.java b/common/src/main/java/io/druid/math/expr/Parser.java index 7e72fcc2dcee..8f54ed98af07 100644 --- a/common/src/main/java/io/druid/math/expr/Parser.java +++ b/common/src/main/java/io/druid/math/expr/Parser.java @@ -20,9 +20,13 @@ package io.druid.math.expr; +import com.google.common.base.Strings; +import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.metamx.common.logger.Logger; +import io.druid.data.input.Row; import io.druid.math.expr.antlr.ExprLexer; import io.druid.math.expr.antlr.ExprParser; import org.antlr.v4.runtime.ANTLRInputStream; @@ -31,6 +35,7 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker; import java.lang.reflect.Modifier; +import java.util.List; import java.util.Map; public class Parser @@ -43,7 +48,7 @@ public class Parser for (Class clazz : Function.class.getClasses()) { if (!Modifier.isAbstract(clazz.getModifiers()) && Function.class.isAssignableFrom(clazz)) { try { - Function function = (Function)clazz.newInstance(); + Function function = (Function) clazz.newInstance(); functionMap.put(function.name().toLowerCase(), function); } catch (Exception e) { @@ -66,4 +71,102 @@ public static Expr parse(String in) walker.walk(listener, parseTree); return listener.getAST(); } + + public static List findRequiredBindings(String in) + { + return findRequiredBindings(parse(in)); + } + + public static List findRequiredBindings(Expr parsed) + { + return findRecursive(parsed, Lists.newArrayList()); + } + + private static List findRecursive(Expr expr, List found) + { + if (expr instanceof IdentifierExpr) { + found.add(expr.toString()); + } else if (expr instanceof BinaryOpExprBase) { + BinaryOpExprBase binary = (BinaryOpExprBase) expr; + findRecursive(binary.left, found); + findRecursive(binary.right, found); + } else if (expr instanceof UnaryMinusExpr) { + findRecursive(((UnaryMinusExpr) expr).expr, found); + } else if (expr instanceof UnaryNotExpr) { + findRecursive(((UnaryNotExpr) expr).expr, found); + } else if (expr instanceof FunctionExpr) { + for (Expr child : ((FunctionExpr) expr).args) { + findRecursive(child, found); + } + } + return found; + } + + public static Expr.NumericBinding withMap(final Map bindings) + { + return new Expr.NumericBinding() + { + @Override + public Number get(String name) + { + return (Number) bindings.get(name); + } + }; + } + + public static Expr.NumericBinding withSuppliers(final Map> bindings) + { + return new Expr.NumericBinding() + { + @Override + public Number get(String name) + { + Supplier supplier = bindings.get(name); + return supplier == null ? null : supplier.get(); + } + }; + } + + public static Expr.NumericBinding withRow(final ThreadLocal convey) + { + return new Expr.NumericBinding() + { + @Override + public Number get(String name) + { + Row row = convey.get(); + if (name.equals("__time")) { + return row.getTimestampFromEpoch(); + } + return toNumeric(row.getRaw(name)); + } + }; + } + + private static Number toNumeric(Object value) + { + if (value == null || value instanceof Number) { + return (Number) value; + } + String stringVal = String.valueOf(value); + if (Strings.isNullOrEmpty(stringVal)) { + return null; + } + + final String target = stringVal.trim(); + int i = 0; + char first = target.charAt(i); + if (!Character.isDigit(first)) { + if (first != '+' && first != '-') { + return null; + } + i++; + } + for (; i < target.length(); i++) { + if (!Character.isDigit(target.charAt(i))) { + return Double.valueOf(target); + } + } + return Long.valueOf(target); + } } diff --git a/common/src/test/java/io/druid/math/expr/EvalTest.java b/common/src/test/java/io/druid/math/expr/EvalTest.java index bb24ec826e35..f64bc4373f61 100644 --- a/common/src/test/java/io/druid/math/expr/EvalTest.java +++ b/common/src/test/java/io/druid/math/expr/EvalTest.java @@ -19,6 +19,7 @@ package io.druid.math.expr; +import com.google.common.base.Supplier; import org.junit.Assert; import org.junit.Test; @@ -29,81 +30,103 @@ */ public class EvalTest { + private Supplier constantSupplier(final Number number) + { + return new Supplier() + { + @Override + public Number get() + { + return number; + } + }; + } + @Test public void testDoubleEval() { - Map bindings = new HashMap<>(); - bindings.put("x", 2.0d); - - Assert.assertEquals(2.0, Parser.parse("x").eval(bindings).doubleValue(), 0.0001); - - Assert.assertFalse(Parser.parse("1.0 && 0.0").eval(bindings).doubleValue() > 0.0); - Assert.assertTrue(Parser.parse("1.0 && 2.0").eval(bindings).doubleValue() > 0.0); - - Assert.assertTrue(Parser.parse("1.0 || 0.0").eval(bindings).doubleValue() > 0.0); - Assert.assertFalse(Parser.parse("0.0 || 0.0").eval(bindings).doubleValue() > 0.0); - - Assert.assertTrue(Parser.parse("2.0 > 1.0").eval(bindings).doubleValue() > 0.0); - Assert.assertTrue(Parser.parse("2.0 >= 2.0").eval(bindings).doubleValue() > 0.0); - Assert.assertTrue(Parser.parse("1.0 < 2.0").eval(bindings).doubleValue() > 0.0); - Assert.assertTrue(Parser.parse("2.0 <= 2.0").eval(bindings).doubleValue() > 0.0); - Assert.assertTrue(Parser.parse("2.0 == 2.0").eval(bindings).doubleValue() > 0.0); - Assert.assertTrue(Parser.parse("2.0 != 1.0").eval(bindings).doubleValue() > 0.0); - - Assert.assertEquals(3.5, Parser.parse("2.0 + 1.5").eval(bindings).doubleValue(), 0.0001); - Assert.assertEquals(0.5, Parser.parse("2.0 - 1.5").eval(bindings).doubleValue(), 0.0001); - Assert.assertEquals(3.0, Parser.parse("2.0 * 1.5").eval(bindings).doubleValue(), 0.0001); - Assert.assertEquals(4.0, Parser.parse("2.0 / 0.5").eval(bindings).doubleValue(), 0.0001); - Assert.assertEquals(0.2, Parser.parse("2.0 % 0.3").eval(bindings).doubleValue(), 0.0001); - Assert.assertEquals(8.0, Parser.parse("2.0 ^ 3.0").eval(bindings).doubleValue(), 0.0001); - Assert.assertEquals(-1.5, Parser.parse("-1.5").eval(bindings).doubleValue(), 0.0001); - - Assert.assertTrue(Parser.parse("!-1.0").eval(bindings).doubleValue() > 0.0); - Assert.assertTrue(Parser.parse("!0.0").eval(bindings).doubleValue() > 0.0); - Assert.assertFalse(Parser.parse("!2.0").eval(bindings).doubleValue() > 0.0); - - Assert.assertEquals(2.0, Parser.parse("sqrt(4.0)").eval(bindings).doubleValue(), 0.0001); - Assert.assertEquals(2.0, Parser.parse("if(1.0, 2.0, 3.0)").eval(bindings).doubleValue(), 0.0001); - Assert.assertEquals(3.0, Parser.parse("if(0.0, 2.0, 3.0)").eval(bindings).doubleValue(), 0.0001); + Map> bindings = new HashMap<>(); + bindings.put( "x", constantSupplier(2.0d)); + + Assert.assertEquals(2.0, evaluate("x", bindings).doubleValue(), 0.0001); + + Assert.assertFalse(evaluate("1.0 && 0.0", bindings).doubleValue() > 0.0); + Assert.assertTrue(evaluate("1.0 && 2.0", bindings).doubleValue() > 0.0); + + Assert.assertTrue(evaluate("1.0 || 0.0", bindings).doubleValue() > 0.0); + Assert.assertFalse(evaluate("0.0 || 0.0", bindings).doubleValue() > 0.0); + + Assert.assertTrue(evaluate("2.0 > 1.0", bindings).doubleValue() > 0.0); + Assert.assertTrue(evaluate("2.0 >= 2.0", bindings).doubleValue() > 0.0); + Assert.assertTrue(evaluate("1.0 < 2.0", bindings).doubleValue() > 0.0); + Assert.assertTrue(evaluate("2.0 <= 2.0", bindings).doubleValue() > 0.0); + Assert.assertTrue(evaluate("2.0 == 2.0", bindings).doubleValue() > 0.0); + Assert.assertTrue(evaluate("2.0 != 1.0", bindings).doubleValue() > 0.0); + + Assert.assertEquals(3.5, evaluate("2.0 + 1.5", bindings).doubleValue(), 0.0001); + Assert.assertEquals(0.5, evaluate("2.0 - 1.5", bindings).doubleValue(), 0.0001); + Assert.assertEquals(3.0, evaluate("2.0 * 1.5", bindings).doubleValue(), 0.0001); + Assert.assertEquals(4.0, evaluate("2.0 / 0.5", bindings).doubleValue(), 0.0001); + Assert.assertEquals(0.2, evaluate("2.0 % 0.3", bindings).doubleValue(), 0.0001); + Assert.assertEquals(8.0, evaluate("2.0 ^ 3.0", bindings).doubleValue(), 0.0001); + Assert.assertEquals(-1.5, evaluate("-1.5", bindings).doubleValue(), 0.0001); + + Assert.assertTrue(evaluate("!-1.0", bindings).doubleValue() > 0.0); + Assert.assertTrue(evaluate("!0.0", bindings).doubleValue() > 0.0); + Assert.assertFalse(evaluate("!2.0", bindings).doubleValue() > 0.0); + + Assert.assertEquals(2.0, evaluate("sqrt(4.0)", bindings).doubleValue(), 0.0001); + Assert.assertEquals(2.0, evaluate("if(1.0, 2.0, 3.0)", bindings).doubleValue(), 0.0001); + Assert.assertEquals(3.0, evaluate("if(0.0, 2.0, 3.0)", bindings).doubleValue(), 0.0001); + } + + private Number evaluate(String in, Map> bindings) { + return Parser.parse(in).eval(Parser.withSuppliers(bindings)); } @Test public void testLongEval() { - Map bindings = new HashMap<>(); - bindings.put("x", 9223372036854775807L); - - Assert.assertEquals(9223372036854775807L, Parser.parse("x").eval(bindings).longValue()); - - Assert.assertFalse(Parser.parse("9223372036854775807 && 0").eval(bindings).longValue() > 0); - Assert.assertTrue(Parser.parse("9223372036854775807 && 9223372036854775806").eval(bindings).longValue() > 0); - - Assert.assertTrue(Parser.parse("9223372036854775807 || 0").eval(bindings).longValue() > 0); - Assert.assertFalse(Parser.parse("-9223372036854775807 || -9223372036854775807").eval(bindings).longValue() > 0); - Assert.assertTrue(Parser.parse("-9223372036854775807 || 9223372036854775807").eval(bindings).longValue() > 0); - Assert.assertFalse(Parser.parse("0 || 0").eval(bindings).longValue() > 0); - - Assert.assertTrue(Parser.parse("9223372036854775807 > 9223372036854775806").eval(bindings).longValue() > 0); - Assert.assertTrue(Parser.parse("9223372036854775807 >= 9223372036854775807").eval(bindings).longValue() > 0); - Assert.assertTrue(Parser.parse("9223372036854775806 < 9223372036854775807").eval(bindings).longValue() > 0); - Assert.assertTrue(Parser.parse("9223372036854775807 <= 9223372036854775807").eval(bindings).longValue() > 0); - Assert.assertTrue(Parser.parse("9223372036854775807 == 9223372036854775807").eval(bindings).longValue() > 0); - Assert.assertTrue(Parser.parse("9223372036854775807 != 9223372036854775806").eval(bindings).longValue() > 0); - - Assert.assertEquals(9223372036854775807L, Parser.parse("9223372036854775806 + 1").eval(bindings).longValue()); - Assert.assertEquals(9223372036854775806L, Parser.parse("9223372036854775807 - 1").eval(bindings).longValue()); - Assert.assertEquals(9223372036854775806L, Parser.parse("4611686018427387903 * 2").eval(bindings).longValue()); - Assert.assertEquals(4611686018427387903L, Parser.parse("9223372036854775806 / 2").eval(bindings).longValue()); - Assert.assertEquals(7L, Parser.parse("9223372036854775807 % 9223372036854775800").eval(bindings).longValue()); - Assert.assertEquals( 9223372030926249001L, Parser.parse("3037000499 ^ 2").eval(bindings).longValue()); - Assert.assertEquals(-9223372036854775807L, Parser.parse("-9223372036854775807").eval(bindings).longValue()); - - Assert.assertTrue(Parser.parse("!-9223372036854775807").eval(bindings).longValue() > 0); - Assert.assertTrue(Parser.parse("!0").eval(bindings).longValue() > 0); - Assert.assertFalse(Parser.parse("!9223372036854775807").eval(bindings).longValue() > 0); - - Assert.assertEquals(3037000499L, Parser.parse("sqrt(9223372036854775807)").eval(bindings).longValue()); - Assert.assertEquals(9223372036854775807L, Parser.parse("if(9223372036854775807, 9223372036854775807, 9223372036854775806)").eval(bindings).longValue()); - Assert.assertEquals(9223372036854775806L, Parser.parse("if(0, 9223372036854775807, 9223372036854775806)").eval(bindings).longValue()); + Map> bindings = new HashMap<>(); + bindings.put("x", constantSupplier(9223372036854775807L)); + + Assert.assertEquals(9223372036854775807L, evaluate("x", bindings).longValue()); + + Assert.assertFalse(evaluate("9223372036854775807 && 0", bindings).longValue() > 0); + Assert.assertTrue(evaluate("9223372036854775807 && 9223372036854775806", bindings).longValue() > 0); + + Assert.assertTrue(evaluate("9223372036854775807 || 0", bindings).longValue() > 0); + Assert.assertFalse(evaluate("-9223372036854775807 || -9223372036854775807", bindings).longValue() > 0); + Assert.assertTrue(evaluate("-9223372036854775807 || 9223372036854775807", bindings).longValue() > 0); + Assert.assertFalse(evaluate("0 || 0", bindings).longValue() > 0); + + Assert.assertTrue(evaluate("9223372036854775807 > 9223372036854775806", bindings).longValue() > 0); + Assert.assertTrue(evaluate("9223372036854775807 >= 9223372036854775807", bindings).longValue() > 0); + Assert.assertTrue(evaluate("9223372036854775806 < 9223372036854775807", bindings).longValue() > 0); + Assert.assertTrue(evaluate("9223372036854775807 <= 9223372036854775807", bindings).longValue() > 0); + Assert.assertTrue(evaluate("9223372036854775807 == 9223372036854775807", bindings).longValue() > 0); + Assert.assertTrue(evaluate("9223372036854775807 != 9223372036854775806", bindings).longValue() > 0); + + Assert.assertEquals(9223372036854775807L, evaluate("9223372036854775806 + 1", bindings).longValue()); + Assert.assertEquals(9223372036854775806L, evaluate("9223372036854775807 - 1", bindings).longValue()); + Assert.assertEquals(9223372036854775806L, evaluate("4611686018427387903 * 2", bindings).longValue()); + Assert.assertEquals(4611686018427387903L, evaluate("9223372036854775806 / 2", bindings).longValue()); + Assert.assertEquals(7L, evaluate("9223372036854775807 % 9223372036854775800", bindings).longValue()); + Assert.assertEquals( 9223372030926249001L, evaluate("3037000499 ^ 2", bindings).longValue()); + Assert.assertEquals(-9223372036854775807L, evaluate("-9223372036854775807", bindings).longValue()); + + Assert.assertTrue(evaluate("!-9223372036854775807", bindings).longValue() > 0); + Assert.assertTrue(evaluate("!0", bindings).longValue() > 0); + Assert.assertFalse(evaluate("!9223372036854775807", bindings).longValue() > 0); + + Assert.assertEquals(3037000499L, evaluate("sqrt(9223372036854775807)", bindings).longValue()); + Assert.assertEquals(9223372036854775807L, evaluate( + "if(9223372036854775807, 9223372036854775807, 9223372036854775806)", + bindings + ).longValue()); + Assert.assertEquals(9223372036854775806L, evaluate( + "if(0, 9223372036854775807, 9223372036854775806)", + bindings + ).longValue()); } } diff --git a/common/src/test/java/io/druid/math/expr/ParserTest.java b/common/src/test/java/io/druid/math/expr/ParserTest.java index 512bcbd04266..b06f01a4eb73 100644 --- a/common/src/test/java/io/druid/math/expr/ParserTest.java +++ b/common/src/test/java/io/druid/math/expr/ParserTest.java @@ -70,88 +70,47 @@ public void testSimpleUnaryOps2() Assert.assertEquals(expected, actual); } + private void validateParser(String expression, String expected, String identifiers) + { + Assert.assertEquals(expected, Parser.parse(expression).toString()); + Assert.assertEquals(identifiers, Parser.findRequiredBindings(expression).toString()); + } + @Test public void testSimpleLogicalOps1() { - String actual = Parser.parse("x>y").toString(); - String expected = "(> x y)"; - Assert.assertEquals(expected, actual); - - actual = Parser.parse("x=y").toString(); - expected = "(>= x y)"; - Assert.assertEquals(expected, actual); - - actual = Parser.parse("x==y").toString(); - expected = "(== x y)"; - Assert.assertEquals(expected, actual); - - actual = Parser.parse("x!=y").toString(); - expected = "(!= x y)"; - Assert.assertEquals(expected, actual); - - actual = Parser.parse("x && y").toString(); - expected = "(&& x y)"; - Assert.assertEquals(expected, actual); - - actual = Parser.parse("x || y").toString(); - expected = "(|| x y)"; - Assert.assertEquals(expected, actual); + validateParser("x>y", "(> x y)", "[x, y]"); + validateParser("x=y", "(>= x y)", "[x, y]"); + validateParser("x==y", "(== x y)", "[x, y]"); + validateParser("x!=y", "(!= x y)", "[x, y]"); + validateParser("x && y", "(&& x y)", "[x, y]"); + validateParser("x || y", "(|| x y)", "[x, y]"); } @Test public void testSimpleAdditivityOp1() { - String actual = Parser.parse("x+y").toString(); - String expected = "(+ x y)"; - Assert.assertEquals(expected, actual); - - actual = Parser.parse("x-y").toString(); - expected = "(- x y)"; - Assert.assertEquals(expected, actual); + validateParser("x+y", "(+ x y)", "[x, y]"); + validateParser("x-y", "(- x y)", "[x, y]"); } @Test public void testSimpleAdditivityOp2() { - String actual = Parser.parse("x+y+z").toString(); - String expected = "(+ (+ x y) z)"; - Assert.assertEquals(expected, actual); - - actual = Parser.parse("x+y-z").toString(); - expected = "(- (+ x y) z)"; - Assert.assertEquals(expected, actual); - - actual = Parser.parse("x-y+z").toString(); - expected = "(+ (- x y) z)"; - Assert.assertEquals(expected, actual); - - actual = Parser.parse("x-y-z").toString(); - expected = "(- (- x y) z)"; - Assert.assertEquals(expected, actual); + validateParser("x+y+z", "(+ (+ x y) z)", "[x, y, z]"); + validateParser("x+y-z", "(- (+ x y) z)", "[x, y, z]"); + validateParser("x-y+z", "(+ (- x y) z)", "[x, y, z]"); + validateParser("x-y-z", "(- (- x y) z)", "[x, y, z]"); } @Test public void testSimpleMultiplicativeOp1() { - String actual = Parser.parse("x*y").toString(); - String expected = "(* x y)"; - Assert.assertEquals(expected, actual); - - actual = Parser.parse("x/y").toString(); - expected = "(/ x y)"; - Assert.assertEquals(expected, actual); - - actual = Parser.parse("x%y").toString(); - expected = "(% x y)"; - Assert.assertEquals(expected, actual); + validateParser("x*y", "(* x y)", "[x, y]"); + validateParser("x/y", "(/ x y)", "[x, y]"); + validateParser("x%y", "(% x y)", "[x, y]"); } @Test @@ -255,12 +214,7 @@ public void testMixed() @Test public void testFunctions() { - String actual = Parser.parse("sqrt(x)").toString(); - String expected = "(sqrt [x])"; - Assert.assertEquals(expected, actual); - - actual = Parser.parse("if(cond,then,else)").toString(); - expected = "(if [cond, then, else])"; - Assert.assertEquals(expected, actual); + validateParser("sqrt(x)", "(sqrt [x])", "[x]"); + validateParser("if(cond,then,else)", "(if [cond, then, else])", "[cond, then, else]"); } } diff --git a/processing/src/main/java/io/druid/jackson/AggregatorsModule.java b/processing/src/main/java/io/druid/jackson/AggregatorsModule.java index 2e9582da89c6..93e03418b2c8 100644 --- a/processing/src/main/java/io/druid/jackson/AggregatorsModule.java +++ b/processing/src/main/java/io/druid/jackson/AggregatorsModule.java @@ -43,6 +43,7 @@ import io.druid.query.aggregation.post.ConstantPostAggregator; import io.druid.query.aggregation.post.FieldAccessPostAggregator; import io.druid.query.aggregation.post.JavaScriptPostAggregator; +import io.druid.query.aggregation.post.MathPostAggregator; import io.druid.segment.serde.ComplexMetrics; /** @@ -82,6 +83,7 @@ public static interface AggregatorFactoryMixin @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes(value = { + @JsonSubTypes.Type(name = "math", value = MathPostAggregator.class), @JsonSubTypes.Type(name = "arithmetic", value = ArithmeticPostAggregator.class), @JsonSubTypes.Type(name = "fieldAccess", value = FieldAccessPostAggregator.class), @JsonSubTypes.Type(name = "constant", value = ConstantPostAggregator.class), diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java index 8c08cceebb65..687b8eed07fa 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java @@ -21,6 +21,10 @@ import com.google.common.collect.Lists; import com.metamx.common.Pair; +import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.LongColumnSelector; +import io.druid.segment.NumericColumnSelector; import java.util.HashSet; import java.util.LinkedList; @@ -84,4 +88,44 @@ public static Pair, List> condensedAggre } return new Pair(condensedAggs, condensedPostAggs); } + + public static FloatColumnSelector getFloatColumnSelector( + ColumnSelectorFactory metricFactory, + String fieldName, + String fieldExpression + ) + { + if (fieldName != null) { + return metricFactory.makeFloatColumnSelector(fieldName); + } + final NumericColumnSelector numeric = metricFactory.makeMathExpressionSelector(fieldExpression); + return new FloatColumnSelector() + { + @Override + public float get() + { + return numeric.get().floatValue(); + } + }; + } + + public static LongColumnSelector getLongColumnSelector( + ColumnSelectorFactory metricFactory, + String fieldName, + String fieldExpression + ) + { + if (fieldName != null) { + return metricFactory.makeLongColumnSelector(fieldName); + } + final NumericColumnSelector numeric = metricFactory.makeMathExpressionSelector(fieldExpression); + return new LongColumnSelector() + { + @Override + public long get() + { + return numeric.get().longValue(); + } + }; + } } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java index 029bc86afafb..b014d8034d16 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java @@ -23,13 +23,16 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; -import com.metamx.common.StringUtils; +import io.druid.common.utils.StringUtils; +import io.druid.math.expr.Parser; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.FloatColumnSelector; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.Objects; /** */ @@ -37,32 +40,48 @@ public class DoubleMaxAggregatorFactory extends AggregatorFactory { private static final byte CACHE_TYPE_ID = 0x3; - private final String fieldName; private final String name; + private final String fieldName; + private final String fieldExpression; @JsonCreator public DoubleMaxAggregatorFactory( @JsonProperty("name") String name, - @JsonProperty("fieldName") final String fieldName + @JsonProperty("fieldName") final String fieldName, + @JsonProperty("fieldExpression") String fieldExpression ) { Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); - Preconditions.checkNotNull(fieldName, "Must have a valid, non-null fieldName"); + Preconditions.checkArgument( + fieldName == null ^ fieldExpression == null, + "Must have a valid, non-null fieldName or fieldExpression" + ); this.name = name; this.fieldName = fieldName; + this.fieldExpression = fieldExpression; + } + + public DoubleMaxAggregatorFactory(String name, String fieldName) + { + this(name, fieldName, null); } @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new DoubleMaxAggregator(name, metricFactory.makeFloatColumnSelector(fieldName)); + return new DoubleMaxAggregator(name, getFloatColumnSelector(metricFactory)); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new DoubleMaxBufferAggregator(metricFactory.makeFloatColumnSelector(fieldName)); + return new DoubleMaxBufferAggregator(getFloatColumnSelector(metricFactory)); + } + + private FloatColumnSelector getFloatColumnSelector(ColumnSelectorFactory metricFactory) + { + return AggregatorUtil.getFloatColumnSelector(metricFactory, fieldName, fieldExpression); } @Override @@ -80,7 +99,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregatorFactory getCombiningFactory() { - return new DoubleMaxAggregatorFactory(name, name); + return new DoubleMaxAggregatorFactory(name, name, null); } @Override @@ -96,7 +115,7 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre @Override public List getRequiredColumns() { - return Arrays.asList(new DoubleMaxAggregatorFactory(fieldName, fieldName)); + return Arrays.asList(new DoubleMaxAggregatorFactory(fieldName, fieldName, fieldExpression)); } @Override @@ -121,6 +140,12 @@ public String getFieldName() return fieldName; } + @JsonProperty + public String getFieldExpression() + { + return fieldExpression; + } + @Override @JsonProperty public String getName() @@ -131,15 +156,17 @@ public String getName() @Override public List requiredFields() { - return Arrays.asList(fieldName); + return fieldName != null ? Arrays.asList(fieldName) : Parser.findRequiredBindings(fieldExpression); } @Override public byte[] getCacheKey() { - byte[] fieldNameBytes = StringUtils.toUtf8(fieldName); + byte[] fieldNameBytes = StringUtils.toUtf8WithNullToEmpty(fieldName); + byte[] fieldExpressionBytes = StringUtils.toUtf8WithNullToEmpty(fieldExpression); - return ByteBuffer.allocate(1 + fieldNameBytes.length).put(CACHE_TYPE_ID).put(fieldNameBytes).array(); + return ByteBuffer.allocate(1 + fieldNameBytes.length + fieldExpressionBytes.length) + .put(CACHE_TYPE_ID).put(fieldNameBytes).put(fieldExpressionBytes).array(); } @Override @@ -165,6 +192,7 @@ public String toString() { return "DoubleMaxAggregatorFactory{" + "fieldName='" + fieldName + '\'' + + ", fieldExpression='" + fieldExpression + '\'' + ", name='" + name + '\'' + '}'; } @@ -172,13 +200,24 @@ public String toString() @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } DoubleMaxAggregatorFactory that = (DoubleMaxAggregatorFactory) o; - if (fieldName != null ? !fieldName.equals(that.fieldName) : that.fieldName != null) return false; - if (name != null ? !name.equals(that.name) : that.name != null) return false; + if (!Objects.equals(fieldName, that.fieldName)) { + return false; + } + if (!Objects.equals(fieldExpression, that.fieldExpression)) { + return false; + } + if (!Objects.equals(name, that.name)) { + return false; + } return true; } @@ -187,6 +226,7 @@ public boolean equals(Object o) public int hashCode() { int result = fieldName != null ? fieldName.hashCode() : 0; + result = 31 * result + (fieldExpression != null ? fieldExpression.hashCode() : 0); result = 31 * result + (name != null ? name.hashCode() : 0); return result; } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java index 04f1de0ede39..4023f44ec7d9 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java @@ -23,13 +23,16 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; -import com.metamx.common.StringUtils; +import io.druid.common.utils.StringUtils; +import io.druid.math.expr.Parser; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.FloatColumnSelector; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.Objects; /** */ @@ -37,32 +40,48 @@ public class DoubleMinAggregatorFactory extends AggregatorFactory { private static final byte CACHE_TYPE_ID = 0x4; - private final String fieldName; private final String name; + private final String fieldName; + private final String fieldExpression; @JsonCreator public DoubleMinAggregatorFactory( @JsonProperty("name") String name, - @JsonProperty("fieldName") final String fieldName + @JsonProperty("fieldName") final String fieldName, + @JsonProperty("fieldExpression") String fieldExpression ) { Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); - Preconditions.checkNotNull(fieldName, "Must have a valid, non-null fieldName"); + Preconditions.checkArgument( + fieldName == null ^ fieldExpression == null, + "Must have a valid, non-null fieldName or fieldExpression" + ); this.name = name; this.fieldName = fieldName; + this.fieldExpression = fieldExpression; + } + + public DoubleMinAggregatorFactory(String name, String fieldName) + { + this(name, fieldName, null); } @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new DoubleMinAggregator(name, metricFactory.makeFloatColumnSelector(fieldName)); + return new DoubleMinAggregator(name, getFloatColumnSelector(metricFactory)); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new DoubleMinBufferAggregator(metricFactory.makeFloatColumnSelector(fieldName)); + return new DoubleMinBufferAggregator(getFloatColumnSelector(metricFactory)); + } + + private FloatColumnSelector getFloatColumnSelector(ColumnSelectorFactory metricFactory) + { + return AggregatorUtil.getFloatColumnSelector(metricFactory, fieldName, fieldExpression); } @Override @@ -80,7 +99,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregatorFactory getCombiningFactory() { - return new DoubleMinAggregatorFactory(name, name); + return new DoubleMinAggregatorFactory(name, name, null); } @Override @@ -96,7 +115,7 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre @Override public List getRequiredColumns() { - return Arrays.asList(new DoubleMinAggregatorFactory(fieldName, fieldName)); + return Arrays.asList(new DoubleMinAggregatorFactory(fieldName, fieldName, fieldExpression)); } @Override @@ -121,6 +140,12 @@ public String getFieldName() return fieldName; } + @JsonProperty + public String getFieldExpression() + { + return fieldExpression; + } + @Override @JsonProperty public String getName() @@ -131,15 +156,17 @@ public String getName() @Override public List requiredFields() { - return Arrays.asList(fieldName); + return fieldName != null ? Arrays.asList(fieldName) : Parser.findRequiredBindings(fieldExpression); } @Override public byte[] getCacheKey() { - byte[] fieldNameBytes = StringUtils.toUtf8(fieldName); + byte[] fieldNameBytes = StringUtils.toUtf8WithNullToEmpty(fieldName); + byte[] fieldExpressionBytes = StringUtils.toUtf8WithNullToEmpty(fieldExpression); - return ByteBuffer.allocate(1 + fieldNameBytes.length).put(CACHE_TYPE_ID).put(fieldNameBytes).array(); + return ByteBuffer.allocate(1 + fieldNameBytes.length + fieldExpressionBytes.length) + .put(CACHE_TYPE_ID).put(fieldNameBytes).put(fieldExpressionBytes).array(); } @Override @@ -165,6 +192,7 @@ public String toString() { return "DoubleMinAggregatorFactory{" + "fieldName='" + fieldName + '\'' + + ", fieldExpression='" + fieldExpression + '\'' + ", name='" + name + '\'' + '}'; } @@ -172,13 +200,24 @@ public String toString() @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } DoubleMinAggregatorFactory that = (DoubleMinAggregatorFactory) o; - if (fieldName != null ? !fieldName.equals(that.fieldName) : that.fieldName != null) return false; - if (name != null ? !name.equals(that.name) : that.name != null) return false; + if (!Objects.equals(fieldName, that.fieldName)) { + return false; + } + if (!Objects.equals(fieldExpression, that.fieldExpression)) { + return false; + } + if (!Objects.equals(name, that.name)) { + return false; + } return true; } @@ -187,6 +226,7 @@ public boolean equals(Object o) public int hashCode() { int result = fieldName != null ? fieldName.hashCode() : 0; + result = 31 * result + (fieldExpression != null ? fieldExpression.hashCode() : 0); result = 31 * result + (name != null ? name.hashCode() : 0); return result; } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java index a05d76a10948..eb7dd1bd5b36 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java @@ -23,13 +23,16 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; -import com.metamx.common.StringUtils; +import io.druid.common.utils.StringUtils; +import io.druid.math.expr.Parser; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.FloatColumnSelector; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.Objects; /** */ @@ -37,35 +40,48 @@ public class DoubleSumAggregatorFactory extends AggregatorFactory { private static final byte CACHE_TYPE_ID = 0x2; - private final String fieldName; private final String name; + private final String fieldName; + private final String fieldExpression; @JsonCreator public DoubleSumAggregatorFactory( @JsonProperty("name") String name, - @JsonProperty("fieldName") final String fieldName + @JsonProperty("fieldName") String fieldName, + @JsonProperty("fieldExpression") String fieldExpression ) { Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); - Preconditions.checkNotNull(fieldName, "Must have a valid, non-null fieldName"); + Preconditions.checkArgument( + fieldName == null ^ fieldExpression == null, + "Must have a valid, non-null fieldName or fieldExpression" + ); this.name = name; this.fieldName = fieldName; + this.fieldExpression = fieldExpression; + } + + public DoubleSumAggregatorFactory(String name, String fieldName) + { + this(name, fieldName, null); } @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new DoubleSumAggregator( - name, - metricFactory.makeFloatColumnSelector(fieldName) - ); + return new DoubleSumAggregator(name, getFloatColumnSelector(metricFactory)); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new DoubleSumBufferAggregator(metricFactory.makeFloatColumnSelector(fieldName)); + return new DoubleSumBufferAggregator(getFloatColumnSelector(metricFactory)); + } + + private FloatColumnSelector getFloatColumnSelector(ColumnSelectorFactory metricFactory) + { + return AggregatorUtil.getFloatColumnSelector(metricFactory, fieldName, fieldExpression); } @Override @@ -83,7 +99,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregatorFactory getCombiningFactory() { - return new DoubleSumAggregatorFactory(name, name); + return new DoubleSumAggregatorFactory(name, name, null); } @Override @@ -99,7 +115,7 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre @Override public List getRequiredColumns() { - return Arrays.asList(new DoubleSumAggregatorFactory(fieldName, fieldName)); + return Arrays.asList(new DoubleSumAggregatorFactory(fieldName, fieldName, fieldExpression)); } @Override @@ -124,6 +140,12 @@ public String getFieldName() return fieldName; } + @JsonProperty + public String getFieldExpression() + { + return fieldExpression; + } + @Override @JsonProperty public String getName() @@ -134,15 +156,17 @@ public String getName() @Override public List requiredFields() { - return Arrays.asList(fieldName); + return fieldName != null ? Arrays.asList(fieldName) : Parser.findRequiredBindings(fieldExpression); } @Override public byte[] getCacheKey() { - byte[] fieldNameBytes = StringUtils.toUtf8(fieldName); + byte[] fieldNameBytes = StringUtils.toUtf8WithNullToEmpty(fieldName); + byte[] fieldExpressionBytes = StringUtils.toUtf8WithNullToEmpty(fieldExpression); - return ByteBuffer.allocate(1 + fieldNameBytes.length).put(CACHE_TYPE_ID).put(fieldNameBytes).array(); + return ByteBuffer.allocate(1 + fieldNameBytes.length + fieldExpressionBytes.length) + .put(CACHE_TYPE_ID).put(fieldNameBytes).put(fieldExpressionBytes).array(); } @Override @@ -168,6 +192,7 @@ public String toString() { return "DoubleSumAggregatorFactory{" + "fieldName='" + fieldName + '\'' + + ", fieldExpression='" + fieldExpression + '\'' + ", name='" + name + '\'' + '}'; } @@ -184,10 +209,13 @@ public boolean equals(Object o) DoubleSumAggregatorFactory that = (DoubleSumAggregatorFactory) o; - if (fieldName != null ? !fieldName.equals(that.fieldName) : that.fieldName != null) { + if (!Objects.equals(fieldName, that.fieldName)) { + return false; + } + if (!Objects.equals(fieldExpression, that.fieldExpression)) { return false; } - if (name != null ? !name.equals(that.name) : that.name != null) { + if (!Objects.equals(name, that.name)) { return false; } @@ -198,6 +226,7 @@ public boolean equals(Object o) public int hashCode() { int result = fieldName != null ? fieldName.hashCode() : 0; + result = 31 * result + (fieldExpression != null ? fieldExpression.hashCode() : 0); result = 31 * result + (name != null ? name.hashCode() : 0); return result; } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java index 91f8c11fee34..af971e982d44 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java @@ -23,13 +23,16 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; -import com.metamx.common.StringUtils; +import io.druid.common.utils.StringUtils; +import io.druid.math.expr.Parser; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.LongColumnSelector; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.Objects; /** */ @@ -37,32 +40,47 @@ public class LongMaxAggregatorFactory extends AggregatorFactory { private static final byte CACHE_TYPE_ID = 0xA; - private final String fieldName; private final String name; + private final String fieldName; + private final String fieldExpression; @JsonCreator public LongMaxAggregatorFactory( @JsonProperty("name") String name, - @JsonProperty("fieldName") final String fieldName + @JsonProperty("fieldName") final String fieldName, + @JsonProperty("fieldExpression") String fieldExpression ) { Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); - Preconditions.checkNotNull(fieldName, "Must have a valid, non-null fieldName"); + Preconditions.checkArgument( + fieldName == null ^ fieldExpression == null, + "Must have a valid, non-null fieldName or fieldExpression"); this.name = name; this.fieldName = fieldName; + this.fieldExpression = fieldExpression; + } + + public LongMaxAggregatorFactory(String name, String fieldName) + { + this(name, fieldName, null); } @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new LongMaxAggregator(name, metricFactory.makeLongColumnSelector(fieldName)); + return new LongMaxAggregator(name, getLongColumnSelector(metricFactory)); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new LongMaxBufferAggregator(metricFactory.makeLongColumnSelector(fieldName)); + return new LongMaxBufferAggregator(getLongColumnSelector(metricFactory)); + } + + private LongColumnSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) + { + return AggregatorUtil.getLongColumnSelector(metricFactory, fieldName, fieldExpression); } @Override @@ -80,7 +98,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregatorFactory getCombiningFactory() { - return new LongMaxAggregatorFactory(name, name); + return new LongMaxAggregatorFactory(name, name, null); } @Override @@ -96,7 +114,7 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre @Override public List getRequiredColumns() { - return Arrays.asList(new LongMaxAggregatorFactory(fieldName, fieldName)); + return Arrays.asList(new LongMaxAggregatorFactory(fieldName, fieldName, fieldExpression)); } @Override @@ -117,6 +135,12 @@ public String getFieldName() return fieldName; } + @JsonProperty + public String getFieldExpression() + { + return fieldExpression; + } + @Override @JsonProperty public String getName() @@ -127,15 +151,17 @@ public String getName() @Override public List requiredFields() { - return Arrays.asList(fieldName); + return fieldName != null ? Arrays.asList(fieldName) : Parser.findRequiredBindings(fieldExpression); } @Override public byte[] getCacheKey() { - byte[] fieldNameBytes = StringUtils.toUtf8(fieldName); + byte[] fieldNameBytes = StringUtils.toUtf8WithNullToEmpty(fieldName); + byte[] fieldExpressionBytes = StringUtils.toUtf8WithNullToEmpty(fieldExpression); - return ByteBuffer.allocate(1 + fieldNameBytes.length).put(CACHE_TYPE_ID).put(fieldNameBytes).array(); + return ByteBuffer.allocate(1 + fieldNameBytes.length + fieldExpressionBytes.length) + .put(CACHE_TYPE_ID).put(fieldNameBytes).put(fieldExpressionBytes).array(); } @Override @@ -161,6 +187,7 @@ public String toString() { return "LongMaxAggregatorFactory{" + "fieldName='" + fieldName + '\'' + + ", fieldExpression='" + fieldExpression + '\'' + ", name='" + name + '\'' + '}'; } @@ -173,8 +200,15 @@ public boolean equals(Object o) LongMaxAggregatorFactory that = (LongMaxAggregatorFactory) o; - if (fieldName != null ? !fieldName.equals(that.fieldName) : that.fieldName != null) return false; - if (name != null ? !name.equals(that.name) : that.name != null) return false; + if (!Objects.equals(fieldName, that.fieldName)) { + return false; + } + if (!Objects.equals(fieldExpression, that.fieldExpression)) { + return false; + } + if (!Objects.equals(name, that.name)) { + return false; + } return true; } @@ -183,6 +217,7 @@ public boolean equals(Object o) public int hashCode() { int result = fieldName != null ? fieldName.hashCode() : 0; + result = 31 * result + (fieldExpression != null ? fieldExpression.hashCode() : 0); result = 31 * result + (name != null ? name.hashCode() : 0); return result; } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java index 82f2111a43a5..e61436cb58ce 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java @@ -23,13 +23,16 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; -import com.metamx.common.StringUtils; +import io.druid.common.utils.StringUtils; +import io.druid.math.expr.Parser; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.LongColumnSelector; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.Objects; /** */ @@ -37,32 +40,47 @@ public class LongMinAggregatorFactory extends AggregatorFactory { private static final byte CACHE_TYPE_ID = 0xB; - private final String fieldName; private final String name; + private final String fieldName; + private final String fieldExpression; @JsonCreator public LongMinAggregatorFactory( @JsonProperty("name") String name, - @JsonProperty("fieldName") final String fieldName + @JsonProperty("fieldName") final String fieldName, + @JsonProperty("fieldExpression") String fieldExpression ) { Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); - Preconditions.checkNotNull(fieldName, "Must have a valid, non-null fieldName"); + Preconditions.checkArgument( + fieldName == null ^ fieldExpression == null, + "Must have a valid, non-null fieldName or fieldExpression"); this.name = name; this.fieldName = fieldName; + this.fieldExpression = fieldExpression; + } + + public LongMinAggregatorFactory(String name, String fieldName) + { + this(name, fieldName, null); } @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new LongMinAggregator(name, metricFactory.makeLongColumnSelector(fieldName)); + return new LongMinAggregator(name, getLongColumnSelector(metricFactory)); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new LongMinBufferAggregator(metricFactory.makeLongColumnSelector(fieldName)); + return new LongMinBufferAggregator(getLongColumnSelector(metricFactory)); + } + + private LongColumnSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) + { + return AggregatorUtil.getLongColumnSelector(metricFactory, fieldName, fieldExpression); } @Override @@ -80,7 +98,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregatorFactory getCombiningFactory() { - return new LongMinAggregatorFactory(name, name); + return new LongMinAggregatorFactory(name, name, null); } @Override @@ -96,7 +114,7 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre @Override public List getRequiredColumns() { - return Arrays.asList(new LongMinAggregatorFactory(fieldName, fieldName)); + return Arrays.asList(new LongMinAggregatorFactory(fieldName, fieldName, fieldExpression)); } @Override @@ -117,6 +135,12 @@ public String getFieldName() return fieldName; } + @JsonProperty + public String getFieldExpression() + { + return fieldExpression; + } + @Override @JsonProperty public String getName() @@ -127,15 +151,17 @@ public String getName() @Override public List requiredFields() { - return Arrays.asList(fieldName); + return fieldName != null ? Arrays.asList(fieldName) : Parser.findRequiredBindings(fieldExpression); } @Override public byte[] getCacheKey() { - byte[] fieldNameBytes = StringUtils.toUtf8(fieldName); + byte[] fieldNameBytes = StringUtils.toUtf8WithNullToEmpty(fieldName); + byte[] fieldExpressionBytes = StringUtils.toUtf8WithNullToEmpty(fieldExpression); - return ByteBuffer.allocate(1 + fieldNameBytes.length).put(CACHE_TYPE_ID).put(fieldNameBytes).array(); + return ByteBuffer.allocate(1 + fieldNameBytes.length + fieldExpressionBytes.length) + .put(CACHE_TYPE_ID).put(fieldNameBytes).put(fieldExpressionBytes).array(); } @Override @@ -161,6 +187,7 @@ public String toString() { return "LongMinAggregatorFactory{" + "fieldName='" + fieldName + '\'' + + ", fieldExpression='" + fieldExpression + '\'' + ", name='" + name + '\'' + '}'; } @@ -173,8 +200,15 @@ public boolean equals(Object o) LongMinAggregatorFactory that = (LongMinAggregatorFactory) o; - if (fieldName != null ? !fieldName.equals(that.fieldName) : that.fieldName != null) return false; - if (name != null ? !name.equals(that.name) : that.name != null) return false; + if (!Objects.equals(fieldName, that.fieldName)) { + return false; + } + if (!Objects.equals(fieldExpression, that.fieldExpression)) { + return false; + } + if (!Objects.equals(name, that.name)) { + return false; + } return true; } @@ -183,6 +217,7 @@ public boolean equals(Object o) public int hashCode() { int result = fieldName != null ? fieldName.hashCode() : 0; + result = 31 * result + (fieldExpression != null ? fieldExpression.hashCode() : 0); result = 31 * result + (name != null ? name.hashCode() : 0); return result; } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index 33b65d2fb61f..3d79b4a088c8 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -23,13 +23,16 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; -import com.metamx.common.StringUtils; +import io.druid.common.utils.StringUtils; +import io.druid.math.expr.Parser; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.LongColumnSelector; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.Objects; /** */ @@ -37,35 +40,47 @@ public class LongSumAggregatorFactory extends AggregatorFactory { private static final byte CACHE_TYPE_ID = 0x1; - private final String fieldName; private final String name; + private final String fieldName; + private final String fieldExpression; @JsonCreator public LongSumAggregatorFactory( @JsonProperty("name") String name, - @JsonProperty("fieldName") final String fieldName + @JsonProperty("fieldName") String fieldName, + @JsonProperty("fieldExpression") String fieldExpression ) { Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); - Preconditions.checkNotNull(fieldName, "Must have a valid, non-null fieldName"); + Preconditions.checkArgument( + fieldName == null ^ fieldExpression == null, + "Must have a valid, non-null fieldName or fieldExpression"); this.name = name; this.fieldName = fieldName; + this.fieldExpression = fieldExpression; + } + + public LongSumAggregatorFactory(String name, String fieldName) + { + this(name, fieldName, null); } @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new LongSumAggregator( - name, - metricFactory.makeLongColumnSelector(fieldName) - ); + return new LongSumAggregator(name, getLongColumnSelector(metricFactory)); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new LongSumBufferAggregator(metricFactory.makeLongColumnSelector(fieldName)); + return new LongSumBufferAggregator(getLongColumnSelector(metricFactory)); + } + + private LongColumnSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) + { + return AggregatorUtil.getLongColumnSelector(metricFactory, fieldName, fieldExpression); } @Override @@ -83,7 +98,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregatorFactory getCombiningFactory() { - return new LongSumAggregatorFactory(name, name); + return new LongSumAggregatorFactory(name, name, null); } @Override @@ -99,7 +114,7 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre @Override public List getRequiredColumns() { - return Arrays.asList(new LongSumAggregatorFactory(fieldName, fieldName)); + return Arrays.asList(new LongSumAggregatorFactory(fieldName, fieldName, fieldExpression)); } @Override @@ -120,6 +135,12 @@ public String getFieldName() return fieldName; } + @JsonProperty + public String getFieldExpression() + { + return fieldExpression; + } + @Override @JsonProperty public String getName() @@ -130,15 +151,17 @@ public String getName() @Override public List requiredFields() { - return Arrays.asList(fieldName); + return fieldName != null ? Arrays.asList(fieldName) : Parser.findRequiredBindings(fieldExpression); } @Override public byte[] getCacheKey() { - byte[] fieldNameBytes = StringUtils.toUtf8(fieldName); + byte[] fieldNameBytes = StringUtils.toUtf8WithNullToEmpty(fieldName); + byte[] fieldExpressionBytes = StringUtils.toUtf8WithNullToEmpty(fieldExpression); - return ByteBuffer.allocate(1 + fieldNameBytes.length).put(CACHE_TYPE_ID).put(fieldNameBytes).array(); + return ByteBuffer.allocate(1 + fieldNameBytes.length + fieldExpressionBytes.length) + .put(CACHE_TYPE_ID).put(fieldNameBytes).put(fieldExpressionBytes).array(); } @Override @@ -164,6 +187,7 @@ public String toString() { return "LongSumAggregatorFactory{" + "fieldName='" + fieldName + '\'' + + ", fieldExpression='" + fieldExpression + '\'' + ", name='" + name + '\'' + '}'; } @@ -180,10 +204,13 @@ public boolean equals(Object o) LongSumAggregatorFactory that = (LongSumAggregatorFactory) o; - if (fieldName != null ? !fieldName.equals(that.fieldName) : that.fieldName != null) { + if (!Objects.equals(fieldName, that.fieldName)) { + return false; + } + if (!Objects.equals(fieldExpression, that.fieldExpression)) { return false; } - if (name != null ? !name.equals(that.name) : that.name != null) { + if (!Objects.equals(name, that.name)) { return false; } @@ -194,6 +221,7 @@ public boolean equals(Object o) public int hashCode() { int result = fieldName != null ? fieldName.hashCode() : 0; + result = 31 * result + (fieldExpression != null ? fieldExpression.hashCode() : 0); result = 31 * result + (name != null ? name.hashCode() : 0); return result; } diff --git a/processing/src/main/java/io/druid/query/aggregation/post/MathPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/MathPostAggregator.java new file mode 100644 index 000000000000..c78edd90db7e --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/post/MathPostAggregator.java @@ -0,0 +1,194 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.post; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; +import io.druid.math.expr.Expr; +import io.druid.math.expr.Parser; +import io.druid.query.aggregation.PostAggregator; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + */ +public class MathPostAggregator implements PostAggregator +{ + private static final Comparator DEFAULT_COMPARATOR = new Comparator() + { + @Override + public int compare(Number o1, Number o2) + { + if (o1 instanceof Long && o2 instanceof Long) { + return Long.compare(o1.longValue(), o2.longValue()); + } + return Double.compare(o1.doubleValue(), o2.doubleValue()); + } + }; + + private final String name; + private final String expression; + private final String ordering; + + private final Expr parsed; + private final Comparator comparator; + private final List dependentFields; + + public MathPostAggregator( + String name, + String expression + ) + { + this(name, expression, null); + } + + @JsonCreator + public MathPostAggregator( + @JsonProperty("name") String name, + @JsonProperty("expression") String expression, + @JsonProperty("ordering") String ordering + ) + { + Preconditions.checkArgument(expression != null, "expression cannot not be null"); + + this.name = name; + this.expression = expression; + this.ordering = ordering; + this.comparator = ordering == null ? DEFAULT_COMPARATOR : Ordering.valueOf(ordering); + + this.parsed = Parser.parse(expression); + this.dependentFields = Parser.findRequiredBindings(parsed); + } + + @Override + public Set getDependentFields() + { + return Sets.newHashSet(dependentFields); + } + + @Override + public Comparator getComparator() + { + return comparator; + } + + @Override + public Object compute(Map values) + { + return parsed.eval(Parser.withMap(values)); + } + + @Override + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty("expression") + public String getExpression() + { + return expression; + } + + @JsonProperty("ordering") + public String getOrdering() + { + return ordering; + } + + @Override + public String toString() + { + return "MathPostAggregator{" + + "name='" + name + '\'' + + ", expression='" + expression + '\'' + + ", ordering=" + ordering + + '}'; + } + + public static enum Ordering implements Comparator + { + // ensures the following order: numeric > NaN > Infinite + numericFirst { + public int compare(Number lhs, Number rhs) + { + if (lhs instanceof Long && rhs instanceof Long) { + return Long.compare(lhs.longValue(), rhs.longValue()); + } + double d1 = lhs.doubleValue(); + double d2 = rhs.doubleValue(); + if (isFinite(d1) && !isFinite(d2)) { + return 1; + } + if (!isFinite(d1) && isFinite(d2)) { + return -1; + } + return Double.compare(d1, d2); + } + + // Double.isFinite only exist in JDK8 + private boolean isFinite(double value) + { + return !Double.isInfinite(value) && !Double.isNaN(value); + } + } + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + MathPostAggregator that = (MathPostAggregator) o; + + if (!Objects.equals(name, that.name)) { + return false; + } + if (!Objects.equals(expression, that.expression)) { + return false; + } + if (!Objects.equals(ordering, that.ordering)) { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + int result = Objects.hashCode(name); + result = 31 * result + Objects.hashCode(expression); + result = 31 * result + Objects.hashCode(ordering); + return result; + } +} diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java index f9c3a1b37a3f..48f729bd666f 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java @@ -402,6 +402,40 @@ public GroupByQuery withLimitSpec(final LimitSpec limitSpec) ); } + public GroupByQuery withAggregatorSpecs(final List aggregatorSpecs) + { + return new GroupByQuery( + getDataSource(), + getQuerySegmentSpec(), + getDimFilter(), + getGranularity(), + getDimensions(), + aggregatorSpecs, + getPostAggregatorSpecs(), + getHavingSpec(), + getLimitSpec(), + limitFn, + getContext() + ); + } + + public GroupByQuery withPostAggregatorSpecs(final List postAggregatorSpecs) + { + return new GroupByQuery( + getDataSource(), + getQuerySegmentSpec(), + getDimFilter(), + getGranularity(), + getDimensions(), + getAggregatorSpecs(), + postAggregatorSpecs, + getHavingSpec(), + getLimitSpec(), + limitFn, + getContext() + ); + } + public static class Builder { private DataSource dataSource; diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java index a3c7f6e3889f..6205f39170a7 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java @@ -25,8 +25,10 @@ import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.base.Supplier; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.google.common.primitives.Chars; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; @@ -35,6 +37,8 @@ import io.druid.data.input.MapBasedRow; import io.druid.data.input.Row; import io.druid.granularity.AllGranularity; +import io.druid.math.expr.Expr; +import io.druid.math.expr.Parser; import io.druid.query.QueryInterruptedException; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.dimension.DimensionSpec; @@ -46,6 +50,7 @@ import io.druid.segment.DimensionSelector; import io.druid.segment.FloatColumnSelector; import io.druid.segment.LongColumnSelector; +import io.druid.segment.NumericColumnSelector; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; @@ -60,6 +65,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; // this class contains shared code between GroupByMergingQueryRunnerV2 and GroupByRowProcessor public class RowBasedGrouperHelper @@ -650,6 +656,38 @@ public Object get() }; } + @Override + public NumericColumnSelector makeMathExpressionSelector(String expression) + { + final Expr parsed = Parser.parse(expression); + + final Set required = Sets.newHashSet(Parser.findRequiredBindings(parsed)); + final Map> values = Maps.newHashMapWithExpectedSize(required.size()); + + for (final String columnName : required) { + values.put( + columnName, new Supplier() + { + @Override + public Number get() + { + return row.get().getFloatMetric(columnName); + } + } + ); + } + final Expr.NumericBinding binding = Parser.withSuppliers(values); + + return new NumericColumnSelector() + { + @Override + public Number get() + { + return parsed.eval(binding); + } + }; + } + @Override public ColumnCapabilities getColumnCapabilities(String columnName) { diff --git a/processing/src/main/java/io/druid/query/topn/TopNQuery.java b/processing/src/main/java/io/druid/query/topn/TopNQuery.java index 71f4c23d3bdf..c81a1cc4cbfe 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNQuery.java +++ b/processing/src/main/java/io/druid/query/topn/TopNQuery.java @@ -169,7 +169,8 @@ public TopNQuery withQuerySegmentSpec(QuerySegmentSpec querySegmentSpec) ); } - public TopNQuery withDimensionSpec(DimensionSpec spec){ + public TopNQuery withDimensionSpec(DimensionSpec spec) + { return new TopNQuery( getDataSource(), spec, @@ -183,7 +184,25 @@ public TopNQuery withDimensionSpec(DimensionSpec spec){ getContext() ); } - public TopNQuery withPostAggregatorSpecs(List postAggregatorSpecs){ + + public TopNQuery withAggregatorSpecs(List aggregatorSpecs) + { + return new TopNQuery( + getDataSource(), + getDimensionSpec(), + topNMetricSpec, + threshold, + getQuerySegmentSpec(), + dimFilter, + granularity, + aggregatorSpecs, + postAggregatorSpecs, + getContext() + ); + } + + public TopNQuery withPostAggregatorSpecs(List postAggregatorSpecs) + { return new TopNQuery( getDataSource(), getDimensionSpec(), diff --git a/processing/src/main/java/io/druid/segment/ColumnSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelector.java index 47a02be97dcc..dbc37355a88c 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelector.java @@ -18,10 +18,12 @@ */ package io.druid.segment;import io.druid.segment.column.Column; +import io.druid.segment.data.Indexed; /** */ public interface ColumnSelector { + public Indexed getColumnNames(); public Column getColumn(String columnName); } diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java index f550fef14e5c..afb38476c048 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java @@ -31,5 +31,6 @@ public interface ColumnSelectorFactory public FloatColumnSelector makeFloatColumnSelector(String columnName); public LongColumnSelector makeLongColumnSelector(String columnName); public ObjectColumnSelector makeObjectColumnSelector(String columnName); + public NumericColumnSelector makeMathExpressionSelector(String expression); public ColumnCapabilities getColumnCapabilities(String columnName); } diff --git a/processing/src/main/java/io/druid/segment/NumericColumnSelector.java b/processing/src/main/java/io/druid/segment/NumericColumnSelector.java new file mode 100644 index 000000000000..576211d38e48 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/NumericColumnSelector.java @@ -0,0 +1,27 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +/** + */ +public interface NumericColumnSelector +{ + Number get(); +} diff --git a/processing/src/main/java/io/druid/segment/QueryableIndex.java b/processing/src/main/java/io/druid/segment/QueryableIndex.java index d2a2c516cae0..c18913946223 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndex.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndex.java @@ -33,7 +33,6 @@ public interface QueryableIndex extends ColumnSelector, Closeable { public Interval getDataInterval(); public int getNumRows(); - public Indexed getColumnNames(); public Indexed getAvailableDimensions(); public BitmapFactory getBitmapFactoryForDimensions(); public Metadata getMetadata(); diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index db724373a51a..0252a6dc0314 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -23,18 +23,19 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.base.Strings; +import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.common.IAE; -import com.metamx.common.UOE; import com.metamx.common.guava.CloseQuietly; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import io.druid.granularity.QueryGranularity; +import io.druid.math.expr.Expr; +import io.druid.math.expr.Parser; import io.druid.query.QueryInterruptedException; import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.dimension.DimensionSpec; @@ -69,6 +70,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; /** */ @@ -812,6 +814,56 @@ public ColumnCapabilities getColumnCapabilities(String columnName) { return getColumnCapabilites(index, columnName); } + + @Override + public NumericColumnSelector makeMathExpressionSelector(String expression) + { + final Expr parsed = Parser.parse(expression); + final Set required = Sets.newHashSet(Parser.findRequiredBindings(parsed)); + + final Map> values = Maps.newHashMapWithExpectedSize(required.size()); + for (String columnName : index.getColumnNames()) { + if (!required.contains(columnName)) { + continue; + } + final GenericColumn column = index.getColumn(columnName).getGenericColumn(); + if (column == null) { + continue; + } + if (column.getType() == ValueType.FLOAT) { + values.put( + columnName, new Supplier() + { + @Override + public Number get() + { + return column.getFloatSingleValueRow(cursorOffset.getOffset()); + } + } + ); + } else if (column.getType() == ValueType.LONG) { + values.put( + columnName, new Supplier() + { + @Override + public Number get() + { + return column.getLongSingleValueRow(cursorOffset.getOffset()); + } + } + ); + } + } + final Expr.NumericBinding binding = Parser.withSuppliers(values); + return new NumericColumnSelector() + { + @Override + public Number get() + { + return parsed.eval(binding); + } + }; + } } if (postFilter == null) { diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java index c8a2fe9b3654..30945b730bdc 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java @@ -28,6 +28,7 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import com.metamx.common.IAE; @@ -39,6 +40,8 @@ import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.SpatialDimensionSchema; import io.druid.granularity.QueryGranularity; +import io.druid.math.expr.Expr; +import io.druid.math.expr.Parser; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.query.dimension.DimensionSpec; @@ -51,6 +54,7 @@ import io.druid.segment.FloatColumnSelector; import io.druid.segment.LongColumnSelector; import io.druid.segment.Metadata; +import io.druid.segment.NumericColumnSelector; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; @@ -77,6 +81,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.Set; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; @@ -206,6 +211,38 @@ public ColumnCapabilities getColumnCapabilities(String columnName) return null; } + @Override + public NumericColumnSelector makeMathExpressionSelector(String expression) + { + final Expr parsed = Parser.parse(expression); + + final Set required = Sets.newHashSet(Parser.findRequiredBindings(parsed)); + final Map> values = Maps.newHashMapWithExpectedSize(required.size()); + + for (final String columnName : required) { + values.put( + columnName, new Supplier() + { + @Override + public Number get() + { + return in.get().getFloatMetric(columnName); + } + } + ); + } + final Expr.NumericBinding binding = Parser.withSuppliers(values); + + return new NumericColumnSelector() + { + @Override + public Number get() + { + return parsed.eval(binding); + } + }; + } + @Override public DimensionSelector makeDimensionSelector( DimensionSpec dimensionSpec diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java index 0af3152c4ca1..615fc71eacdc 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java @@ -21,12 +21,17 @@ import com.google.common.base.Function; import com.google.common.base.Strings; +import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import io.druid.granularity.QueryGranularity; +import io.druid.math.expr.Expr; +import io.druid.math.expr.Parser; import io.druid.query.QueryInterruptedException; import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.dimension.DimensionSpec; @@ -45,6 +50,7 @@ import io.druid.segment.LongColumnSelector; import io.druid.segment.Metadata; import io.druid.segment.NullDimensionSelector; +import io.druid.segment.NumericColumnSelector; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.SingleScanTimeDimSelector; import io.druid.segment.StorageAdapter; @@ -61,6 +67,7 @@ import javax.annotation.Nullable; import java.util.Iterator; import java.util.Map; +import java.util.Set; /** */ @@ -68,10 +75,10 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter { private static final NullDimensionSelector NULL_DIMENSION_SELECTOR = new NullDimensionSelector(); - private final IncrementalIndex index; + private final IncrementalIndex index; public IncrementalIndexStorageAdapter( - IncrementalIndex index + IncrementalIndex index ) { this.index = index; @@ -537,6 +544,55 @@ public ColumnCapabilities getColumnCapabilities(String columnName) { return index.getCapabilities(columnName); } + + @Override + public NumericColumnSelector makeMathExpressionSelector(String expression) + { + final Expr parsed = Parser.parse(expression); + + final Set required = Sets.newHashSet(Parser.findRequiredBindings(parsed)); + final Map> values = Maps.newHashMapWithExpectedSize(required.size()); + + for (String columnName : index.getMetricNames()) { + if (!required.contains(columnName)) { + continue; + } + ValueType type = index.getCapabilities(columnName).getType(); + if (type == ValueType.FLOAT) { + final int metricIndex = index.getMetricIndex(columnName); + values.put( + columnName, new Supplier() + { + @Override + public Number get() + { + return index.getMetricFloatValue(currEntry.getValue(), metricIndex); + } + } + ); + } else if (type == ValueType.LONG) { + final int metricIndex = index.getMetricIndex(columnName); + values.put( + columnName, new Supplier() + { + @Override + public Number get() + { + return index.getMetricLongValue(currEntry.getValue(), metricIndex); + } + } + ); + } + } + final Expr.NumericBinding binding = Parser.withSuppliers(values); + return new NumericColumnSelector() { + @Override + public Number get() + { + return parsed.eval(binding); + } + }; + } }; } } diff --git a/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java index 3080aa713e26..b84c3d60d574 100644 --- a/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java @@ -20,7 +20,6 @@ package io.druid.segment.incremental; import com.google.common.base.Supplier; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.metamx.common.logger.Logger; import com.metamx.common.parsers.ParseException; @@ -33,10 +32,10 @@ import io.druid.segment.DimensionSelector; import io.druid.segment.FloatColumnSelector; import io.druid.segment.LongColumnSelector; +import io.druid.segment.NumericColumnSelector; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.ColumnCapabilities; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -403,6 +402,12 @@ public ColumnCapabilities getColumnCapabilities(String columnName) { return delegate.getColumnCapabilities(columnName); } + + @Override + public NumericColumnSelector makeMathExpressionSelector(String expression) + { + return delegate.makeMathExpressionSelector(expression); + } } } diff --git a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java index 73523879b22e..862eb21a01d0 100644 --- a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java @@ -40,6 +40,7 @@ import io.druid.segment.DimensionSelector; import io.druid.segment.FloatColumnSelector; import io.druid.segment.LongColumnSelector; +import io.druid.segment.NumericColumnSelector; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnCapabilitiesImpl; @@ -184,6 +185,12 @@ public ColumnCapabilities getColumnCapabilities(String columnName) } return caps; } + + @Override + public NumericColumnSelector makeMathExpressionSelector(String expression) + { + throw new UnsupportedOperationException(); + } }; } diff --git a/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java index fb8004e89ed8..379b0be96933 100644 --- a/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java @@ -28,6 +28,7 @@ import io.druid.segment.DimensionSelector; import io.druid.segment.FloatColumnSelector; import io.druid.segment.LongColumnSelector; +import io.druid.segment.NumericColumnSelector; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.ColumnCapabilities; import org.junit.Assert; @@ -76,6 +77,12 @@ public ColumnCapabilities getColumnCapabilities(String columnName) { return null; } + + @Override + public NumericColumnSelector makeMathExpressionSelector(String expression) + { + return null; + } }; static { diff --git a/processing/src/test/java/io/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java index 07d8f4840048..da3bff3f9c6e 100644 --- a/processing/src/test/java/io/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java @@ -40,6 +40,7 @@ public class ArithmeticPostAggregatorTest public void testCompute() { ArithmeticPostAggregator arithmeticPostAggregator; + MathPostAggregator mathPostAggregator; CountAggregator agg = new CountAggregator("rows"); agg.aggregate(); agg.aggregate(); @@ -57,17 +58,29 @@ public void testCompute() ) ); + for (PostAggregator postAggregator : postAggregatorList) { + metricValues.put(postAggregator.getName(), postAggregator.compute(metricValues)); + } + arithmeticPostAggregator = new ArithmeticPostAggregator("add", "+", postAggregatorList); + mathPostAggregator = new MathPostAggregator("add", "roku + rows"); Assert.assertEquals(9.0, arithmeticPostAggregator.compute(metricValues)); + Assert.assertEquals(9.0, mathPostAggregator.compute(metricValues)); arithmeticPostAggregator = new ArithmeticPostAggregator("subtract", "-", postAggregatorList); + mathPostAggregator = new MathPostAggregator("add", "roku - rows"); Assert.assertEquals(3.0, arithmeticPostAggregator.compute(metricValues)); + Assert.assertEquals(3.0, mathPostAggregator.compute(metricValues)); arithmeticPostAggregator = new ArithmeticPostAggregator("multiply", "*", postAggregatorList); + mathPostAggregator = new MathPostAggregator("add", "roku * rows"); Assert.assertEquals(18.0, arithmeticPostAggregator.compute(metricValues)); + Assert.assertEquals(18.0, mathPostAggregator.compute(metricValues)); arithmeticPostAggregator = new ArithmeticPostAggregator("divide", "/", postAggregatorList); + mathPostAggregator = new MathPostAggregator("add", "roku / rows"); Assert.assertEquals(2.0, arithmeticPostAggregator.compute(metricValues)); + Assert.assertEquals(2.0, mathPostAggregator.compute(metricValues)); } @Test diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index b2f416a4742e..d3efc30f8597 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -48,6 +48,7 @@ import io.druid.query.Druids; import io.druid.query.FinalizeResultsQueryRunner; import io.druid.query.Query; +import io.druid.query.QueryDataSource; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerTestHelper; import io.druid.query.QueryToolChest; @@ -67,6 +68,7 @@ import io.druid.query.aggregation.post.ArithmeticPostAggregator; import io.druid.query.aggregation.post.ConstantPostAggregator; import io.druid.query.aggregation.post.FieldAccessPostAggregator; +import io.druid.query.aggregation.post.MathPostAggregator; import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.dimension.DimensionSpec; import io.druid.query.dimension.ExtractionDimensionSpec; @@ -2044,7 +2046,7 @@ private void doTestMergeResultsWithValidLimit(final int limit) ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) - .setLimit(Integer.valueOf(limit)); + .setLimit(limit); final GroupByQuery fullQuery = builder.build(); @@ -2087,7 +2089,7 @@ public void testMergeResultsAcrossMultipleDaysWithLimitAndOrderBy() .setLimit(limit) .addOrderByColumn("idx", OrderByColumnSpec.Direction.DESCENDING); - final GroupByQuery fullQuery = builder.build(); + GroupByQuery fullQuery = builder.build(); List expectedResults = Arrays.asList( GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 3L, "idx", 2900L), @@ -2113,8 +2115,39 @@ public void testMergeResultsAcrossMultipleDaysWithLimitAndOrderBy() TestHelper.assertExpectedObjects( Iterables.limit(expectedResults, limit), mergeRunner.run(fullQuery, context), String.format("limit: %d", limit) ); - } + builder.setAggregatorSpecs( + Arrays.asList( + QueryRunnerTestHelper.rowsCount, + new LongSumAggregatorFactory("idx", null, "index * 2 + indexMin / 10") + ) + ); + fullQuery = builder.build(); + + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 3L, "idx", 6090L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 6030L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 1L, "idx", 333L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 1L, "idx", 285L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 255L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 1L, "idx", 252L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 1L, "idx", 251L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 1L, "idx", 248L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 1L, "idx", 165L), + + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "rows", 3L, "idx", 5262L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 5141L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 1L, "idx", 348L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "rows", 1L, "idx", 309L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 265L) + ); + + mergeRunner = factory.getToolchest().mergeResults(runner); + + TestHelper.assertExpectedObjects( + Iterables.limit(expectedResults, limit), mergeRunner.run(fullQuery, context), String.format("limit: %d", limit) + ); + } @Test(expected = IllegalArgumentException.class) public void testMergeResultsWithNegativeLimit() @@ -2131,7 +2164,7 @@ public void testMergeResultsWithNegativeLimit() ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) - .setLimit(Integer.valueOf(-1)); + .setLimit(-1); builder.build(); } @@ -2282,6 +2315,32 @@ public void testGroupByOrderLimit() throws Exception TestHelper.assertExpectedObjects( Iterables.limit(expectedResults, 5), mergeRunner.run(builder.limit(5).build(), context), "limited" ); + + builder.limit(Integer.MAX_VALUE) + .setAggregatorSpecs( + Arrays.asList( + QueryRunnerTestHelper.rowsCount, + new DoubleSumAggregatorFactory("idx", null, "index / 2 + indexMin") + ) + ); + + expectedResults = GroupByQueryRunnerTestHelper.createExpectedRows( + new String[]{"__time", "alias", "rows", "idx"}, + new Object[]{"2011-04-01", "travel", 2L, 365.4876403808594D}, + new Object[]{"2011-04-01", "technology", 2L, 267.3737487792969D}, + new Object[]{"2011-04-01", "news", 2L, 333.3147277832031D}, + new Object[]{"2011-04-01", "health", 2L, 325.467529296875D}, + new Object[]{"2011-04-01", "entertainment", 2L, 479.916015625D}, + new Object[]{"2011-04-01", "business", 2L, 328.083740234375D}, + new Object[]{"2011-04-01", "automotive", 2L, 405.5966796875D}, + new Object[]{"2011-04-01", "premium", 6L, 6627.927734375D}, + new Object[]{"2011-04-01", "mezzanine", 6L, 6635.47998046875D} + ); + + TestHelper.assertExpectedObjects(expectedResults, mergeRunner.run(builder.build(), context), "no-limit"); + TestHelper.assertExpectedObjects( + Iterables.limit(expectedResults, 5), mergeRunner.run(builder.limit(5).build(), context), "limited" + ); } @Test @@ -2342,90 +2401,19 @@ public void testGroupByWithOrderLimit3() throws Exception .addOrderByColumn("alias", OrderByColumnSpec.Direction.DESCENDING) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)); - final GroupByQuery query = builder.build(); - - List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "alias", - "mezzanine", - "rows", - 6L, - "idx", - 4423.6533203125D - ), - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "alias", - "premium", - "rows", - 6L, - "idx", - 4418.61865234375D - ), - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "alias", - "entertainment", - "rows", - 2L, - "idx", - 319.94403076171875D - ), - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "alias", - "automotive", - "rows", - 2L, - "idx", - 270.3977966308594D - ), - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "alias", - "travel", - "rows", - 2L, - "idx", - 243.65843200683594D - ), - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "alias", - "news", - "rows", - 2L, - "idx", - 222.20980834960938D - ), - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "alias", - "business", - "rows", - 2L, - "idx", - 218.7224884033203D - ), - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "alias", - "health", - "rows", - 2L, - "idx", - 216.97836303710938D - ), - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "alias", - "technology", - "rows", - 2L, - "idx", - 178.24917602539062D - ) + GroupByQuery query = builder.build(); + + List expectedResults = GroupByQueryRunnerTestHelper.createExpectedRows( + new String[]{"__time", "alias", "rows", "idx"}, + new Object[]{"2011-04-01", "mezzanine", 6L, 4423.6533203125D}, + new Object[]{"2011-04-01", "premium", 6L, 4418.61865234375D}, + new Object[]{"2011-04-01", "entertainment", 2L, 319.94403076171875D}, + new Object[]{"2011-04-01", "automotive", 2L, 270.3977966308594D}, + new Object[]{"2011-04-01", "travel", 2L, 243.65843200683594D}, + new Object[]{"2011-04-01", "news", 2L, 222.20980834960938D}, + new Object[]{"2011-04-01", "business", 2L, 218.7224884033203D}, + new Object[]{"2011-04-01", "health", 2L, 216.97836303710938D}, + new Object[]{"2011-04-01", "technology", 2L, 178.24917602539062D} ); Map context = Maps.newHashMap(); @@ -3270,7 +3258,7 @@ public void testMergedHavingSpec() ) ); - final GroupByQuery fullQuery = builder.build(); + GroupByQuery fullQuery = builder.build(); QueryRunner mergedRunner = factory.getToolchest().mergeResults( new QueryRunner() @@ -3379,7 +3367,7 @@ public void testMergedPostAggHavingSpec() ) ); - final GroupByQuery fullQuery = builder.build(); + GroupByQuery fullQuery = builder.build(); QueryRunner mergedRunner = factory.getToolchest().mergeResults( new QueryRunner() @@ -3417,6 +3405,22 @@ public Sequence run( ).run(fullQuery, context), "merged" ); + + fullQuery = fullQuery.withPostAggregatorSpecs( + Arrays.asList( + new MathPostAggregator("rows_times_10", "rows * 10.0") + ) + ); + + TestHelper.assertExpectedObjects( + expectedResults, + factory.getToolchest().postMergeQueryDecoration( + factory.getToolchest().mergeResults( + factory.getToolchest().preMergeQueryDecoration(mergedRunner) + ) + ).run(fullQuery, context), + "merged" + ); } @Test @@ -3582,7 +3586,8 @@ public void testIdenticalSubquery() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index"), + new LongSumAggregatorFactory("indexMaxPlusTen", "indexMaxPlusTen") ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -3646,7 +3651,8 @@ public void testSubqueryWithMultipleIntervalsInOuterQuery() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index"), + new LongSumAggregatorFactory("indexMaxPlusTen", "indexMaxPlusTen") ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -3719,7 +3725,8 @@ public void testSubqueryWithExtractionFnInOuterQuery() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index"), + new LongSumAggregatorFactory("indexMaxPlusTen", "indexMaxPlusTen") ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -3773,7 +3780,8 @@ public void testDifferentGroupingSubquery() .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") + new LongSumAggregatorFactory("idx", "index"), + new LongSumAggregatorFactory("indexMaxPlusTen", "indexMaxPlusTen") ) ) .setGranularity(QueryRunnerTestHelper.dayGran) @@ -3785,19 +3793,44 @@ public void testDifferentGroupingSubquery() .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) .setAggregatorSpecs( Arrays.asList( - new DoubleMaxAggregatorFactory("idx", "idx") + QueryRunnerTestHelper.rowsCount, + new DoubleMaxAggregatorFactory("idx", "idx"), + new DoubleMaxAggregatorFactory("indexMaxPlusTen", "indexMaxPlusTen") ) ) .setGranularity(QueryRunnerTestHelper.dayGran) .build(); - List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "idx", 2900.0), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "idx", 2505.0) + List expectedResults = GroupByQueryRunnerTestHelper.createExpectedRows( + new String[]{"__time", "rows", "idx", "indexMaxPlusTen"}, + new Object[]{"2011-04-01", 9L, 2900.0, 2930.0}, + new Object[]{"2011-04-02", 9L, 2505.0, 2535.0} ); - Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); - TestHelper.assertExpectedObjects(expectedResults, results, ""); + TestHelper.assertExpectedObjects( + expectedResults, + GroupByQueryRunnerTestHelper.runQuery(factory, runner, query), "" + ); + + subquery = subquery.withAggregatorSpecs( + Arrays.asList( + QueryRunnerTestHelper.rowsCount, + new LongSumAggregatorFactory("idx", null, "-index + 100"), + new LongSumAggregatorFactory("indexMaxPlusTen", "indexMaxPlusTen") + ) + ); + query = (GroupByQuery) query.withDataSource(new QueryDataSource(subquery)); + + expectedResults = GroupByQueryRunnerTestHelper.createExpectedRows( + new String[]{"__time", "rows", "idx", "indexMaxPlusTen"}, + new Object[]{"2011-04-01", 9L, 21.0, 2930.0}, + new Object[]{"2011-04-02", 9L, 2.0, 2535.0} + ); + + TestHelper.assertExpectedObjects( + expectedResults, + GroupByQueryRunnerTestHelper.runQuery(factory, runner, query), "" + ); } @Test diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTestHelper.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTestHelper.java index a1976bce7762..761fdc683d3b 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTestHelper.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTestHelper.java @@ -31,8 +31,11 @@ import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerFactory; import io.druid.query.QueryToolChest; +import io.druid.segment.column.Column; import org.joda.time.DateTime; +import java.util.Arrays; +import java.util.List; import java.util.Map; /** @@ -70,4 +73,23 @@ public static Row createExpectedRow(final DateTime timestamp, Object... vals) return new MapBasedRow(ts, theVals); } + public static List createExpectedRows(String[] columnNames, Object[]... values) + { + int timeIndex = Arrays.asList(columnNames).indexOf(Column.TIME_COLUMN_NAME); + Preconditions.checkArgument(timeIndex >= 0); + + List expected = Lists.newArrayList(); + for (Object[] value : values) { + Preconditions.checkArgument(value.length == columnNames.length); + Map theVals = Maps.newHashMapWithExpectedSize(value.length); + for (int i = 0; i < columnNames.length; i++) { + if (i != timeIndex) { + theVals.put(columnNames[i], value[i]); + } + } + expected.add(new MapBasedRow(new DateTime(value[timeIndex]), theVals)); + } + return expected; + } + } diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java index 77c57c0bf3b7..7aeb583755cd 100644 --- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java +++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java @@ -20,11 +20,14 @@ package io.druid.query.groupby.epinephelinae; import io.druid.data.input.Row; +import io.druid.math.expr.Expr; +import io.druid.math.expr.Parser; import io.druid.query.dimension.DimensionSpec; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.DimensionSelector; import io.druid.segment.FloatColumnSelector; import io.druid.segment.LongColumnSelector; +import io.druid.segment.NumericColumnSelector; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.ColumnCapabilities; @@ -93,4 +96,19 @@ public ColumnCapabilities getColumnCapabilities(String columnName) { return null; } + + @Override + public NumericColumnSelector makeMathExpressionSelector(String expression) + { + final Expr parsed = Parser.parse(expression); + final Expr.NumericBinding binding = Parser.withRow(row); + return new NumericColumnSelector() + { + @Override + public Number get() + { + return parsed.eval(binding); + } + }; + } } diff --git a/processing/src/test/java/io/druid/query/groupby/orderby/DefaultLimitSpecTest.java b/processing/src/test/java/io/druid/query/groupby/orderby/DefaultLimitSpecTest.java index af5249caa1cb..2610fee31698 100644 --- a/processing/src/test/java/io/druid/query/groupby/orderby/DefaultLimitSpecTest.java +++ b/processing/src/test/java/io/druid/query/groupby/orderby/DefaultLimitSpecTest.java @@ -33,6 +33,7 @@ import io.druid.query.aggregation.PostAggregator; import io.druid.query.aggregation.post.ArithmeticPostAggregator; import io.druid.query.aggregation.post.ConstantPostAggregator; +import io.druid.query.aggregation.post.MathPostAggregator; import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.dimension.DimensionSpec; import io.druid.query.ordering.StringComparators; @@ -254,6 +255,17 @@ public void testBuildWithExplicitOrder() ImmutableList.of(testRowsList.get(2), testRowsList.get(0)), Sequences.toList(limitFn.apply(testRowsSequence), new ArrayList()) ); + + limitFn = limitSpec.build( + ImmutableList.of(new DefaultDimensionSpec("k1", "k1")), + ImmutableList.of(new LongSumAggregatorFactory("k2", "k2")), + ImmutableList.of(new MathPostAggregator("k1", "1 + 1")) + ); + + Assert.assertEquals( + ImmutableList.of(testRowsList.get(2), testRowsList.get(0)), + Sequences.toList(limitFn.apply(testRowsSequence), new ArrayList()) + ); } private Row createRow(String timestamp, Object... vals) diff --git a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java index 63dff115fbe9..f67a9f949d05 100644 --- a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java +++ b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java @@ -194,7 +194,7 @@ public SegmentMetadataQueryTest( null, null ) - ), mmap1 ? 71982 : 72755, + ), mmap1 ? 93744 : 94517, 1209, null, null, @@ -238,7 +238,7 @@ public SegmentMetadataQueryTest( null ) // null_column will be included only for incremental index, which makes a little bigger result than expected - ), mmap2 ? 71982 : 72755, + ), mmap2 ? 93744 : 94517, 1209, null, null, diff --git a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataUnionQueryTest.java b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataUnionQueryTest.java index 929fdfcb506e..069bfc3d1754 100644 --- a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataUnionQueryTest.java +++ b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataUnionQueryTest.java @@ -110,7 +110,7 @@ public void testSegmentMetadataUnionQuery() null ) ), - mmap ? 287928 : 291020, + mmap ? 374976 : 378068, 4836, null, null, diff --git a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java index c7b59ef9c617..10ef802ddaca 100644 --- a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java @@ -156,9 +156,9 @@ public void testFullOnSelect() PagingOffset offset = query.getPagingOffset(QueryRunnerTestHelper.segmentId); List> expectedResults = toExpected( - toEvents(new String[]{EventHolder.timestampKey + ":TIME"}, V_0112_0114), + toFullEvents(V_0112_0114), Lists.newArrayList("market", "quality", "placement", "placementish", "partial_null_column", "null_column"), - Lists.newArrayList("index", "quality_uniques"), + Lists.newArrayList("index", "quality_uniques", "indexMin", "indexMaxPlusTen"), offset.startOffset(), offset.threshold() ); @@ -247,7 +247,7 @@ public void testFullOnSelectWithDimensionSpec() new SelectResultValue( ImmutableMap.of(QueryRunnerTestHelper.segmentId, 2), Sets.newHashSet("mar", "qual", "place"), - Sets.newHashSet("index", "quality_uniques"), + Sets.newHashSet("index", "quality_uniques", "indexMin", "indexMaxPlusTen"), Arrays.asList( new EventHolder( QueryRunnerTestHelper.segmentId, @@ -258,6 +258,8 @@ public void testFullOnSelectWithDimensionSpec() .put("qual", "automotive0") .put("place", "preferred") .put(QueryRunnerTestHelper.indexMetric, 100.000000F) + .put("indexMin", 100.000000F) + .put("indexMaxPlusTen", 110.000000F) .build() ), new EventHolder( @@ -269,6 +271,8 @@ public void testFullOnSelectWithDimensionSpec() .put("qual", "business0") .put("place", "preferred") .put(QueryRunnerTestHelper.indexMetric, 100.000000F) + .put("indexMin", 100.000000F) + .put("indexMaxPlusTen", 110.000000F) .build() ), new EventHolder( @@ -280,6 +284,8 @@ public void testFullOnSelectWithDimensionSpec() .put("qual", "entertainment0") .put("place", "preferred") .put(QueryRunnerTestHelper.indexMetric, 100.000000F) + .put("indexMin", 100.000000F) + .put("indexMaxPlusTen", 110.000000F) .build() ) ) @@ -293,7 +299,7 @@ public void testFullOnSelectWithDimensionSpec() new SelectResultValue( ImmutableMap.of(QueryRunnerTestHelper.segmentId, -3), Sets.newHashSet("mar", "qual", "place"), - Sets.newHashSet("index", "quality_uniques"), + Sets.newHashSet("index", "quality_uniques", "indexMin", "indexMaxPlusTen"), Arrays.asList( new EventHolder( QueryRunnerTestHelper.segmentId, @@ -304,6 +310,8 @@ public void testFullOnSelectWithDimensionSpec() .put("qual", "premium0") .put("place", "preferred") .put(QueryRunnerTestHelper.indexMetric, 780.27197265625F) + .put("indexMin", 780.27197265625F) + .put("indexMaxPlusTen", 790.27197265625F) .build() ), new EventHolder( @@ -315,6 +323,8 @@ public void testFullOnSelectWithDimensionSpec() .put("qual", "mezzanine0") .put("place", "preferred") .put(QueryRunnerTestHelper.indexMetric, 962.731201171875F) + .put("indexMin", 962.731201171875F) + .put("indexMaxPlusTen", 972.731201171875F) .build() ), new EventHolder( @@ -326,6 +336,8 @@ public void testFullOnSelectWithDimensionSpec() .put("qual", "premium0") .put("place", "preferred") .put(QueryRunnerTestHelper.indexMetric, 1029.0570068359375F) + .put("indexMin", 1029.0570068359375F) + .put("indexMaxPlusTen", 1039.0570068359375F) .build() ) ) @@ -333,6 +345,9 @@ public void testFullOnSelectWithDimensionSpec() ) ); + for (Object x: results) { + System.out.println(x); + } verify(descending ? expectedResultsDsc : expectedResultsAsc, results); } @@ -554,7 +569,7 @@ public void testFullSelectNoResults() new SelectResultValue( ImmutableMap.of(), Sets.newHashSet("market", "quality", "placement", "placementish", "partial_null_column", "null_column"), - Sets.newHashSet("index", "quality_uniques"), + Sets.newHashSet("index", "quality_uniques", "indexMin", "indexMaxPlusTen"), Lists.newArrayList() ) ) @@ -605,6 +620,18 @@ private Map toPagingIdentifier(int startDelta, boolean descendi ); } + private List>> toFullEvents(final String[]... valueSet) + { + return toEvents(new String[]{EventHolder.timestampKey + ":TIME", + QueryRunnerTestHelper.marketDimension + ":STRING", + QueryRunnerTestHelper.qualityDimension + ":STRING", + QueryRunnerTestHelper.placementDimension + ":STRING", + QueryRunnerTestHelper.placementishDimension + ":STRINGS", + QueryRunnerTestHelper.indexMetric + ":FLOAT", + QueryRunnerTestHelper.partialNullDimension + ":STRING"}, + valueSet); + } + private List>> toEvents(final String[] dimSpecs, final String[]... valueSet) { List>> events = Lists.newArrayList(); @@ -620,17 +647,19 @@ public Map apply(String input) Map event = Maps.newHashMap(); String[] values = input.split("\\t"); for (int i = 0; i < dimSpecs.length; i++) { - if (dimSpecs[i] == null || i >= dimSpecs.length) { + if (dimSpecs[i] == null || i >= dimSpecs.length || i >= values.length) { continue; } String[] specs = dimSpecs[i].split(":"); event.put( specs[0], + specs.length == 1 || specs[1].equals("STRING") ? values[i] : specs[1].equals("TIME") ? new DateTime(values[i]) : specs[1].equals("FLOAT") ? Float.valueOf(values[i]) : specs[1].equals("DOUBLE") ? Double.valueOf(values[i]) : specs[1].equals("LONG") ? Long.valueOf(values[i]) : specs[1].equals("NULL") ? null : + specs[1].equals("STRINGS") ? Arrays.asList(values[i].split("\u0001")) : values[i] ); } diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java index 3160ad2ca570..3bd8db35419c 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java @@ -45,6 +45,7 @@ import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.DoubleMaxAggregatorFactory; import io.druid.query.aggregation.DoubleMinAggregatorFactory; +import io.druid.query.aggregation.DoubleSumAggregatorFactory; import io.druid.query.aggregation.FilteredAggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.query.aggregation.cardinality.CardinalityAggregatorFactory; @@ -1641,6 +1642,27 @@ public void testTopNCollapsingDimExtraction() ) ) ); + + assertExpectedResults(expectedResults, query); + + query = query.withAggregatorSpecs( + Arrays.asList( + QueryRunnerTestHelper.rowsCount, + new DoubleSumAggregatorFactory("index", null, "-index + 100") + ) + ); + + expectedResults = Arrays.asList( + TopNQueryRunnerTestHelper.createExpectedRows( + "2011-01-12T00:00:00.000Z", + new String[]{QueryRunnerTestHelper.qualityDimension, "rows", "index", "addRowsIndexConstant"}, + Arrays.asList( + new Object[]{"n", 93L, -2786.472755432129, -2692.472755432129}, + new Object[]{"u", 186L, -3949.824363708496, -3762.824363708496} + ) + ) + ); + assertExpectedResults(expectedResults, query); } diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTestHelper.java b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTestHelper.java new file mode 100644 index 000000000000..35d541486c1b --- /dev/null +++ b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTestHelper.java @@ -0,0 +1,48 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.topn; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import io.druid.query.Result; +import org.joda.time.DateTime; + +import java.util.List; +import java.util.Map; + +/** + */ +public class TopNQueryRunnerTestHelper +{ + public static Result createExpectedRows(String date, String[] columnNames, Iterable values) + { + List expected = Lists.newArrayList(); + for (Object[] value : values) { + Preconditions.checkArgument(value.length == columnNames.length); + Map theVals = Maps.newHashMapWithExpectedSize(value.length); + for (int i = 0; i < columnNames.length; i++) { + theVals.put(columnNames[i], value[i]); + } + expected.add(theVals); + } + return new Result(new DateTime(date), new TopNResultValue(expected)); + } +} diff --git a/processing/src/test/java/io/druid/segment/TestIndex.java b/processing/src/test/java/io/druid/segment/TestIndex.java index 0b3055bcd022..1eb382ecc7ba 100644 --- a/processing/src/test/java/io/druid/segment/TestIndex.java +++ b/processing/src/test/java/io/druid/segment/TestIndex.java @@ -32,6 +32,8 @@ import io.druid.data.input.impl.TimestampSpec; import io.druid.granularity.QueryGranularities; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.DoubleMaxAggregatorFactory; +import io.druid.query.aggregation.DoubleMinAggregatorFactory; import io.druid.query.aggregation.DoubleSumAggregatorFactory; import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import io.druid.query.aggregation.hyperloglog.HyperUniquesSerde; @@ -61,7 +63,9 @@ public class TestIndex "index", "partial_null_column", "null_column", - "quality_uniques" + "quality_uniques", + "indexMin", + "indexMaxPlusTen" }; public static final String[] DIMENSIONS = new String[]{ "market", @@ -71,11 +75,13 @@ public class TestIndex "partial_null_column", "null_column", }; - public static final String[] METRICS = new String[]{"index"}; + public static final String[] METRICS = new String[]{"index", "indexMin", "indexMaxPlusTen"}; private static final Logger log = new Logger(TestIndex.class); private static final Interval DATA_INTERVAL = new Interval("2011-01-12T00:00:00.000Z/2011-05-01T00:00:00.000Z"); public static final AggregatorFactory[] METRIC_AGGS = new AggregatorFactory[]{ new DoubleSumAggregatorFactory(METRICS[0], METRICS[0]), + new DoubleMinAggregatorFactory(METRICS[1], METRICS[0]), + new DoubleMaxAggregatorFactory(METRICS[2], null, "index + 10"), new HyperUniquesAggregatorFactory("quality_uniques", "quality") }; private static final IndexSpec indexSpec = new IndexSpec();