diff --git a/common/pom.xml b/common/pom.xml index 3d025e56f121..1df84723f8d3 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -161,6 +161,10 @@ jets3t 0.9.4 + + org.antlr + antlr4-runtime + @@ -199,6 +203,17 @@ + + org.antlr + antlr4-maven-plugin + + + + antlr4 + + + + diff --git a/common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4 b/common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4 new file mode 100644 index 000000000000..f478911dd552 --- /dev/null +++ b/common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4 @@ -0,0 +1,38 @@ +grammar Expr; + +expr : ('-'|'!') expr # unaryOpExpr + | expr '^' expr # powOpExpr + | expr ('*'|'/'|'%') expr # mulDivModuloExpr + | expr ('+'|'-') expr # addSubExpr + | expr ('<'|'<='|'>'|'>='|'=='|'!=') expr # logicalOpExpr + | expr ('&&'|'||') expr # logicalAndOrExpr + | '(' expr ')' # nestedExpr + | IDENTIFIER '(' fnArgs? ')' # functionExpr + | IDENTIFIER # identifierExpr + | DOUBLE # doubleExpr + | LONG # longExpr + ; + +fnArgs : expr (',' expr)* # functionArgs + ; + +IDENTIFIER : [_$a-zA-Z][_$a-zA-Z0-9]* ; +LONG : [0-9]+ ; +DOUBLE : [0-9]+ '.' [0-9]* ; +WS : [ \t\r\n]+ -> skip ; + +MINUS : '-' ; +NOT : '!' ; +POW : '^' ; +MUL : '*' ; +DIV : '/' ; +MODULO : '%' ; +PLUS : '+' ; +LT : '<' ; +LEQ : '<=' ; +GT : '>' ; +GEQ : '>=' ; +EQ : '==' ; +NEQ : '!=' ; +AND : '&&' ; +OR : '||' ; diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java new file mode 100644 index 000000000000..2ee2cdae7c6d --- /dev/null +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -0,0 +1,520 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +import com.google.common.math.LongMath; + +import java.util.List; +import java.util.Map; + +/** + */ +public interface Expr +{ + Number eval(Map bindings); +} + +class LongExpr implements Expr +{ + private final long value; + + public LongExpr(long value) + { + this.value = value; + } + + @Override + public String toString() + { + return String.valueOf(value); + } + + @Override + public Number eval(Map bindings) + { + return value; + } +} + +class DoubleExpr implements Expr +{ + private final double value; + + public DoubleExpr(double value) + { + this.value = value; + } + + @Override + public String toString() + { + return String.valueOf(value); + } + + @Override + public Number eval(Map bindings) + { + return value; + } +} + +class IdentifierExpr implements Expr +{ + private final String value; + + public IdentifierExpr(String value) + { + this.value = value; + } + + @Override + public String toString() + { + return value; + } + + @Override + public Number eval(Map bindings) + { + Number val = bindings.get(value); + if (val == null) { + throw new RuntimeException("No binding found for " + value); + } else { + return val instanceof Long ? val : val.doubleValue(); + } + } +} + +class FunctionExpr implements Expr +{ + private final String name; + private final List args; + + public FunctionExpr(String name, List args) + { + this.name = name; + this.args = args; + } + + @Override + public String toString() + { + return "(" + name + " " + args + ")"; + } + + @Override + public Number eval(Map bindings) + { + return Parser.func.get(name).apply(args, bindings); + } +} + +class UnaryMinusExpr implements Expr +{ + private final Expr expr; + + UnaryMinusExpr(Expr expr) + { + this.expr = expr; + } + + @Override + public Number eval(Map bindings) + { + Number valObj = expr.eval(bindings); + if (valObj instanceof Long) { + return -1 * valObj.longValue(); + } else { + return -1 * valObj.doubleValue(); + } + } + + @Override + public String toString() + { + return "-" + expr.toString(); + } +} + +class UnaryNotExpr implements Expr +{ + private final Expr expr; + + UnaryNotExpr(Expr expr) + { + this.expr = expr; + } + + @Override + public Number eval(Map bindings) + { + Number valObj = expr.eval(bindings); + return valObj.doubleValue() > 0 ? 0.0d : 1.0d; + } + + @Override + public String toString() + { + return "!" + expr.toString(); + } +} + +abstract class BinaryOpExprBase implements Expr +{ + protected final String op; + protected final Expr left; + protected final Expr right; + + public BinaryOpExprBase(String op, Expr left, Expr right) + { + this.op = op; + this.left = left; + this.right = right; + } + + protected boolean isLong(Number left, Number right) + { + return left instanceof Long && right instanceof Long; + } + + @Override + public String toString() + { + return "(" + op + " " + left + " " + right + ")"; + } +} + +class BinMinusExpr extends BinaryOpExprBase +{ + + BinMinusExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() - rightVal.longValue(); + } else { + return leftVal.doubleValue() - rightVal.doubleValue(); + } + } +} + +class BinPowExpr extends BinaryOpExprBase +{ + + BinPowExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return LongMath.pow(leftVal.longValue(), rightVal.intValue()); + } else { + return Math.pow(leftVal.doubleValue(), rightVal.doubleValue()); + } + } +} + +class BinMulExpr extends BinaryOpExprBase +{ + + BinMulExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() * rightVal.longValue(); + } else { + return leftVal.doubleValue() * rightVal.doubleValue(); + } + } +} + +class BinDivExpr extends BinaryOpExprBase +{ + + BinDivExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() / rightVal.longValue(); + } else { + return leftVal.doubleValue() / rightVal.doubleValue(); + } + } +} + +class BinModuloExpr extends BinaryOpExprBase +{ + + BinModuloExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() % rightVal.longValue(); + } else { + return leftVal.doubleValue() % rightVal.doubleValue(); + } + } +} + +class BinPlusExpr extends BinaryOpExprBase +{ + + BinPlusExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() + rightVal.longValue(); + } else { + return leftVal.doubleValue() + rightVal.doubleValue(); + } + } +} + +class BinLtExpr extends BinaryOpExprBase +{ + + BinLtExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() < rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() < rightVal.doubleValue() ? 1.0d : 0.0d; + } + } +} + +class BinLeqExpr extends BinaryOpExprBase +{ + + BinLeqExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() <= rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() <= rightVal.doubleValue() ? 1.0d : 0.0d; + } + } +} + +class BinGtExpr extends BinaryOpExprBase +{ + + BinGtExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() > rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() > rightVal.doubleValue() ? 1.0d : 0.0d; + } + } +} + +class BinGeqExpr extends BinaryOpExprBase +{ + + BinGeqExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() >= rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() >= rightVal.doubleValue() ? 1.0d : 0.0d; + } + } +} + +class BinEqExpr extends BinaryOpExprBase +{ + + BinEqExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() == rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() == rightVal.doubleValue() ? 1.0d : 0.0d; + } + } +} + +class BinNeqExpr extends BinaryOpExprBase +{ + + BinNeqExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + return leftVal.longValue() != rightVal.longValue() ? 1 : 0; + } else { + return leftVal.doubleValue() != rightVal.doubleValue() ? 1.0d : 0.0d; + } + } +} + +class BinAndExpr extends BinaryOpExprBase +{ + + BinAndExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + long lval = leftVal.longValue(); + if (lval > 0) { + long rval = rightVal.longValue(); + return rval > 0 ? 1 : 0; + } else { + return 0; + } + } else { + double lval = leftVal.doubleValue(); + if (lval > 0) { + double rval = rightVal.doubleValue(); + return rval > 0 ? 1.0d : 0.0d; + } else { + return 0.0d; + } + } + } +} + +class BinOrExpr extends BinaryOpExprBase +{ + + BinOrExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + public Number eval(Map bindings) + { + Number leftVal = left.eval(bindings); + Number rightVal = right.eval(bindings); + if (isLong(leftVal, rightVal)) { + long lval = leftVal.longValue(); + if (lval > 0) { + return 1; + } else { + long rval = rightVal.longValue(); + return rval > 0 ? 1 : 0; + } + } else { + double lval = leftVal.doubleValue(); + if (lval > 0) { + return 1.0d; + } else { + double rval = rightVal.doubleValue(); + return rval > 0 ? 1.0d : 0.0d; + } + } + } +} diff --git a/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java b/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java new file mode 100644 index 000000000000..39fecc779c90 --- /dev/null +++ b/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java @@ -0,0 +1,312 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +import io.druid.math.expr.antlr.ExprBaseListener; +import io.druid.math.expr.antlr.ExprParser; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.TerminalNode; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + */ +public class ExprListenerImpl extends ExprBaseListener +{ + private final Map nodes; + private final ParseTree rootNodeKey; + + ExprListenerImpl(ParseTree rootNodeKey) + { + this.rootNodeKey = rootNodeKey; + this.nodes = new HashMap<>(); + } + + Expr getAST() + { + return (Expr) nodes.get(rootNodeKey); + } + + @Override + public void exitUnaryOpExpr(ExprParser.UnaryOpExprContext ctx) + { + int opCode = ((TerminalNode) ctx.getChild(0)).getSymbol().getType(); + switch (opCode) { + case ExprParser.MINUS: + nodes.put(ctx, new UnaryMinusExpr((Expr) nodes.get(ctx.getChild(1)))); + break; + case ExprParser.NOT: + nodes.put(ctx, new UnaryNotExpr((Expr) nodes.get(ctx.getChild(1)))); + break; + default: + throw new RuntimeException("Unrecognized unary operator " + ctx.getChild(0).getText()); + } + } + + @Override + public void exitDoubleExpr(ExprParser.DoubleExprContext ctx) + { + nodes.put( + ctx, + new DoubleExpr(Double.parseDouble(ctx.getText())) + ); + } + + @Override + public void exitAddSubExpr(ExprParser.AddSubExprContext ctx) + { + int opCode = ((TerminalNode) ctx.getChild(1)).getSymbol().getType(); + switch (opCode) { + case ExprParser.PLUS: + nodes.put( + ctx, + new BinPlusExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.MINUS: + nodes.put( + ctx, + new BinMinusExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + default: + throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText()); + } + } + + @Override + public void exitLongExpr(ExprParser.LongExprContext ctx) + { + nodes.put( + ctx, + new LongExpr(Long.parseLong(ctx.getText())) + ); + } + + @Override + public void exitLogicalAndOrExpr(ExprParser.LogicalAndOrExprContext ctx) + { + int opCode = ((TerminalNode) ctx.getChild(1)).getSymbol().getType(); + switch (opCode) { + case ExprParser.AND: + nodes.put( + ctx, + new BinAndExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.OR: + nodes.put( + ctx, + new BinOrExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + default: + throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText()); + } + } + + @Override + public void exitNestedExpr(ExprParser.NestedExprContext ctx) + { + nodes.put(ctx, nodes.get(ctx.getChild(1))); + } + + @Override + public void exitLogicalOpExpr(ExprParser.LogicalOpExprContext ctx) + { + int opCode = ((TerminalNode) ctx.getChild(1)).getSymbol().getType(); + switch (opCode) { + case ExprParser.LT: + nodes.put( + ctx, + new BinLtExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.LEQ: + nodes.put( + ctx, + new BinLeqExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.GT: + nodes.put( + ctx, + new BinGtExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.GEQ: + nodes.put( + ctx, + new BinGeqExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.EQ: + nodes.put( + ctx, + new BinEqExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.NEQ: + nodes.put( + ctx, + new BinNeqExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + default: + throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText()); + } + } + + @Override + public void exitMulDivModuloExpr(ExprParser.MulDivModuloExprContext ctx) + { + int opCode = ((TerminalNode) ctx.getChild(1)).getSymbol().getType(); + switch (opCode) { + case ExprParser.MUL: + nodes.put( + ctx, + new BinMulExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.DIV: + nodes.put( + ctx, + new BinDivExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.MODULO: + nodes.put( + ctx, + new BinModuloExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + default: + throw new RuntimeException("Unrecognized binary operator " + ctx.getChild(1).getText()); + } + } + + @Override + public void exitPowOpExpr(ExprParser.PowOpExprContext ctx) + { + nodes.put( + ctx, + new BinPowExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + } + + @Override + public void exitFunctionExpr(ExprParser.FunctionExprContext ctx) + { + String fnName = ctx.getChild(0).getText(); + if (!Parser.func.containsKey(fnName)) { + throw new RuntimeException("function " + fnName + " is not defined."); + } + + List args = ctx.getChildCount() > 3 ? (List) nodes.get(ctx.getChild(2)) : Collections.emptyList(); + nodes.put( + ctx, + new FunctionExpr(fnName, args) + ); + } + + @Override + public void exitIdentifierExpr(ExprParser.IdentifierExprContext ctx) + { + nodes.put( + ctx, + new IdentifierExpr(ctx.getText()) + ); + } + + @Override + public void exitFunctionArgs(ExprParser.FunctionArgsContext ctx) + { + List args = new ArrayList<>(); + args.add((Expr) nodes.get(ctx.getChild(0))); + + if (ctx.getChildCount() > 1) { + for (int i = 1; i <= ctx.getChildCount() / 2; i++) { + args.add((Expr) nodes.get(ctx.getChild(2 * i))); + } + } + + nodes.put(ctx, args); + } +} diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java new file mode 100644 index 000000000000..630642ba867c --- /dev/null +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -0,0 +1,64 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +import java.util.List; +import java.util.Map; + +/** + */ +interface Function +{ + Number apply(List args, Map bindings); +} + +class SqrtFunc implements Function +{ + + @Override + public Number apply(List args, Map bindings) + { + if (args.size() != 1) { + throw new RuntimeException("function 'sqrt' needs 1 argument"); + } + + Number x = args.get(0).eval(bindings); + return Math.sqrt(x.doubleValue()); + } +} + +class ConditionFunc implements Function +{ + + @Override + public Number apply(List args, Map bindings) + { + if (args.size() != 3) { + throw new RuntimeException("function 'if' needs 3 argument"); + } + + Number x = args.get(0).eval(bindings); + if (x instanceof Long) { + return x.longValue() > 0 ? args.get(1).eval(bindings) : args.get(2).eval(bindings); + } else { + return x.doubleValue() > 0 ? args.get(1).eval(bindings) : args.get(2).eval(bindings); + } + } +} diff --git a/common/src/main/java/io/druid/math/expr/Parser.java b/common/src/main/java/io/druid/math/expr/Parser.java new file mode 100644 index 000000000000..9c962e116688 --- /dev/null +++ b/common/src/main/java/io/druid/math/expr/Parser.java @@ -0,0 +1,56 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + + +import com.google.common.collect.ImmutableMap; +import io.druid.math.expr.antlr.ExprLexer; +import io.druid.math.expr.antlr.ExprParser; +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.ParseTreeWalker; + +import java.util.Map; + +public class Parser +{ + static final Map func; + + static { + func = ImmutableMap.builder() + .put("sqrt", new SqrtFunc()) + .put("if", new ConditionFunc()) + .build(); + } + + public static Expr parse(String in) + { + ExprLexer lexer = new ExprLexer(new ANTLRInputStream(in)); + CommonTokenStream tokens = new CommonTokenStream(lexer); + ExprParser parser = new ExprParser(tokens); + parser.setBuildParseTree(true); + ParseTree parseTree = parser.expr(); + ParseTreeWalker walker = new ParseTreeWalker(); + ExprListenerImpl listener = new ExprListenerImpl(parseTree); + walker.walk(listener, parseTree); + return listener.getAST(); + } +} diff --git a/common/src/test/java/io/druid/math/expr/EvalTest.java b/common/src/test/java/io/druid/math/expr/EvalTest.java new file mode 100644 index 000000000000..bb24ec826e35 --- /dev/null +++ b/common/src/test/java/io/druid/math/expr/EvalTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +/** + */ +public class EvalTest +{ + @Test + public void testDoubleEval() + { + Map bindings = new HashMap<>(); + bindings.put("x", 2.0d); + + Assert.assertEquals(2.0, Parser.parse("x").eval(bindings).doubleValue(), 0.0001); + + Assert.assertFalse(Parser.parse("1.0 && 0.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("1.0 && 2.0").eval(bindings).doubleValue() > 0.0); + + Assert.assertTrue(Parser.parse("1.0 || 0.0").eval(bindings).doubleValue() > 0.0); + Assert.assertFalse(Parser.parse("0.0 || 0.0").eval(bindings).doubleValue() > 0.0); + + Assert.assertTrue(Parser.parse("2.0 > 1.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("2.0 >= 2.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("1.0 < 2.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("2.0 <= 2.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("2.0 == 2.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("2.0 != 1.0").eval(bindings).doubleValue() > 0.0); + + Assert.assertEquals(3.5, Parser.parse("2.0 + 1.5").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(0.5, Parser.parse("2.0 - 1.5").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(3.0, Parser.parse("2.0 * 1.5").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(4.0, Parser.parse("2.0 / 0.5").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(0.2, Parser.parse("2.0 % 0.3").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(8.0, Parser.parse("2.0 ^ 3.0").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(-1.5, Parser.parse("-1.5").eval(bindings).doubleValue(), 0.0001); + + Assert.assertTrue(Parser.parse("!-1.0").eval(bindings).doubleValue() > 0.0); + Assert.assertTrue(Parser.parse("!0.0").eval(bindings).doubleValue() > 0.0); + Assert.assertFalse(Parser.parse("!2.0").eval(bindings).doubleValue() > 0.0); + + Assert.assertEquals(2.0, Parser.parse("sqrt(4.0)").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(2.0, Parser.parse("if(1.0, 2.0, 3.0)").eval(bindings).doubleValue(), 0.0001); + Assert.assertEquals(3.0, Parser.parse("if(0.0, 2.0, 3.0)").eval(bindings).doubleValue(), 0.0001); + } + + @Test + public void testLongEval() + { + Map bindings = new HashMap<>(); + bindings.put("x", 9223372036854775807L); + + Assert.assertEquals(9223372036854775807L, Parser.parse("x").eval(bindings).longValue()); + + Assert.assertFalse(Parser.parse("9223372036854775807 && 0").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775807 && 9223372036854775806").eval(bindings).longValue() > 0); + + Assert.assertTrue(Parser.parse("9223372036854775807 || 0").eval(bindings).longValue() > 0); + Assert.assertFalse(Parser.parse("-9223372036854775807 || -9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("-9223372036854775807 || 9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertFalse(Parser.parse("0 || 0").eval(bindings).longValue() > 0); + + Assert.assertTrue(Parser.parse("9223372036854775807 > 9223372036854775806").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775807 >= 9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775806 < 9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775807 <= 9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775807 == 9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("9223372036854775807 != 9223372036854775806").eval(bindings).longValue() > 0); + + Assert.assertEquals(9223372036854775807L, Parser.parse("9223372036854775806 + 1").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775806L, Parser.parse("9223372036854775807 - 1").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775806L, Parser.parse("4611686018427387903 * 2").eval(bindings).longValue()); + Assert.assertEquals(4611686018427387903L, Parser.parse("9223372036854775806 / 2").eval(bindings).longValue()); + Assert.assertEquals(7L, Parser.parse("9223372036854775807 % 9223372036854775800").eval(bindings).longValue()); + Assert.assertEquals( 9223372030926249001L, Parser.parse("3037000499 ^ 2").eval(bindings).longValue()); + Assert.assertEquals(-9223372036854775807L, Parser.parse("-9223372036854775807").eval(bindings).longValue()); + + Assert.assertTrue(Parser.parse("!-9223372036854775807").eval(bindings).longValue() > 0); + Assert.assertTrue(Parser.parse("!0").eval(bindings).longValue() > 0); + Assert.assertFalse(Parser.parse("!9223372036854775807").eval(bindings).longValue() > 0); + + Assert.assertEquals(3037000499L, Parser.parse("sqrt(9223372036854775807)").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775807L, Parser.parse("if(9223372036854775807, 9223372036854775807, 9223372036854775806)").eval(bindings).longValue()); + Assert.assertEquals(9223372036854775806L, Parser.parse("if(0, 9223372036854775807, 9223372036854775806)").eval(bindings).longValue()); + } +} diff --git a/common/src/test/java/io/druid/math/expr/ParserTest.java b/common/src/test/java/io/druid/math/expr/ParserTest.java new file mode 100644 index 000000000000..512bcbd04266 --- /dev/null +++ b/common/src/test/java/io/druid/math/expr/ParserTest.java @@ -0,0 +1,266 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.math.expr; + +import org.junit.Assert; +import org.junit.Test; + +/** + */ +public class ParserTest +{ + @Test + public void testSimple() + { + String actual = Parser.parse("1").toString(); + String expected = "1"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleUnaryOps1() + { + String actual = Parser.parse("-x").toString(); + String expected = "-x"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("!x").toString(); + expected = "!x"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleUnaryOps2() + { + String actual = Parser.parse("-1").toString(); + String expected = "-1"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("--1").toString(); + expected = "--1"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("-1+2").toString(); + expected = "(+ -1 2)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("-1*2").toString(); + expected = "(* -1 2)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("-1^2").toString(); + expected = "(^ -1 2)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleLogicalOps1() + { + String actual = Parser.parse("x>y").toString(); + String expected = "(> x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x=y").toString(); + expected = "(>= x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x==y").toString(); + expected = "(== x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x!=y").toString(); + expected = "(!= x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x && y").toString(); + expected = "(&& x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x || y").toString(); + expected = "(|| x y)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleAdditivityOp1() + { + String actual = Parser.parse("x+y").toString(); + String expected = "(+ x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x-y").toString(); + expected = "(- x y)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleAdditivityOp2() + { + String actual = Parser.parse("x+y+z").toString(); + String expected = "(+ (+ x y) z)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x+y-z").toString(); + expected = "(- (+ x y) z)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x-y+z").toString(); + expected = "(+ (- x y) z)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x-y-z").toString(); + expected = "(- (- x y) z)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleMultiplicativeOp1() + { + String actual = Parser.parse("x*y").toString(); + String expected = "(* x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x/y").toString(); + expected = "(/ x y)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("x%y").toString(); + expected = "(% x y)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleMultiplicativeOp2() + { + String actual = Parser.parse("1*2*3").toString(); + String expected = "(* (* 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1*2/3").toString(); + expected = "(/ (* 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1/2*3").toString(); + expected = "(* (/ 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1/2/3").toString(); + expected = "(/ (/ 1 2) 3)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleCarrot1() + { + String actual = Parser.parse("1^2").toString(); + String expected = "(^ 1 2)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testSimpleCarrot2() + { + String actual = Parser.parse("1^2^3").toString(); + String expected = "(^ 1 (^ 2 3))"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testMixed() + { + String actual = Parser.parse("1+2*3").toString(); + String expected = "(+ 1 (* 2 3))"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1+(2*3)").toString(); + Assert.assertEquals(expected, actual); + + actual = Parser.parse("(1+2)*3").toString(); + expected = "(* (+ 1 2) 3)"; + Assert.assertEquals(expected, actual); + + + actual = Parser.parse("1*2+3").toString(); + expected = "(+ (* 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("(1*2)+3").toString(); + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1*(2+3)").toString(); + expected = "(* 1 (+ 2 3))"; + Assert.assertEquals(expected, actual); + + + actual = Parser.parse("1+2^3").toString(); + expected = "(+ 1 (^ 2 3))"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1+(2^3)").toString(); + expected = "(+ 1 (^ 2 3))"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("(1+2)^3").toString(); + expected = "(^ (+ 1 2) 3)"; + Assert.assertEquals(expected, actual); + + + actual = Parser.parse("1^2+3").toString(); + expected = "(+ (^ 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("(1^2)+3").toString(); + expected = "(+ (^ 1 2) 3)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("1^(2+3)").toString(); + expected = "(^ 1 (+ 2 3))"; + Assert.assertEquals(expected, actual); + + + actual = Parser.parse("1^2*3+4").toString(); + expected = "(+ (* (^ 1 2) 3) 4)"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("-1^-2*-3+-4").toString(); + expected = "(+ (* (^ -1 -2) -3) -4)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void testFunctions() + { + String actual = Parser.parse("sqrt(x)").toString(); + String expected = "(sqrt [x])"; + Assert.assertEquals(expected, actual); + + actual = Parser.parse("if(cond,then,else)").toString(); + expected = "(if [cond, then, else])"; + Assert.assertEquals(expected, actual); + } +} diff --git a/docs/content/misc/math-expr.md b/docs/content/misc/math-expr.md new file mode 100644 index 000000000000..b338bf37c21d --- /dev/null +++ b/docs/content/misc/math-expr.md @@ -0,0 +1,28 @@ +--- +layout: doc_page +--- + +This expression language supports the following operators (listed in decreasing order of precedence). + +|Operators|Description| +|---------|-----------| +|!, -|Unary NOT and Minus| +|^|Binary power op| +|*, /, %|Binary multiplicative| +|+, -|Binary additive| +|<, <=, >, >=, ==, !=|Binary Comparison| +|&&,\|\||Binary Logical AND, OR| + +Long and double data types are supported. If a number contains a dot, it is interpreted as a double, otherwise it is interpreted as a long. That means, always add a '.' to your number if you want it intepreted as a double value. + +Expressions can contain variables. Variable names may contain letters, digits, '\_' and '$'. Variable names must not begin with a digit. + +For logical operators, a number is true if and only if it is positive. (0 means false) + +Also, the following in-built functions are supported. + +|name|description| +|----|-----------| +|sqrt|sqrt(x) would return square root of x| +|if|if(predicate,then,else) returns 'then' if 'predicate' evaluates to a positive number, otherwise it returns 'else'| + diff --git a/pom.xml b/pom.xml index b5a06fe3d95f..08b7afc03596 100644 --- a/pom.xml +++ b/pom.xml @@ -444,12 +444,12 @@ org.antlr antlr4-runtime - 4.0 + 4.5.1 org.antlr antlr4-coordinator - 4.0 + 4.5.1 commons-cli @@ -740,7 +740,7 @@ org.antlr antlr4-maven-plugin - 4.0 + 4.5.1 org.apache.maven.plugins diff --git a/server/pom.xml b/server/pom.xml index deffcea39ebf..17dc68d3108f 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -132,10 +132,6 @@ org.eclipse.aether aether-api - - org.antlr - antlr4-runtime - net.spy spymemcached @@ -222,17 +218,6 @@ - - org.antlr - antlr4-maven-plugin - - - - antlr4 - - - - diff --git a/server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4 b/server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4 deleted file mode 100644 index b6e781f64eee..000000000000 --- a/server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4 +++ /dev/null @@ -1,343 +0,0 @@ -grammar DruidSQL; - -@header { -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; -import io.druid.granularity.PeriodGranularity; -import io.druid.granularity.QueryGranularity; -import io.druid.query.aggregation.AggregatorFactory; -import io.druid.query.aggregation.CountAggregatorFactory; -import io.druid.query.aggregation.DoubleSumAggregatorFactory; -import io.druid.query.aggregation.DoubleMaxAggregatorFactory; -import io.druid.query.aggregation.DoubleMinAggregatorFactory; -import io.druid.query.aggregation.PostAggregator; -import io.druid.query.aggregation.post.ArithmeticPostAggregator; -import io.druid.query.aggregation.post.ConstantPostAggregator; -import io.druid.query.aggregation.post.FieldAccessPostAggregator; -import io.druid.query.dimension.DefaultDimensionSpec; -import io.druid.query.dimension.DimensionSpec; -import io.druid.query.filter.AndDimFilter; -import io.druid.query.filter.DimFilter; -import io.druid.query.filter.NotDimFilter; -import io.druid.query.filter.OrDimFilter; -import io.druid.query.filter.RegexDimFilter; -import io.druid.query.filter.SelectorDimFilter; -import org.antlr.v4.runtime.NoViableAltException; -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.atn.ATN; -import org.antlr.v4.runtime.atn.ATNSimulator; -import org.antlr.v4.runtime.atn.ParserATNSimulator; -import org.antlr.v4.runtime.atn.PredictionContextCache; -import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.tree.ParseTreeListener; -import org.antlr.v4.runtime.tree.TerminalNode; -import org.joda.time.DateTime; -import org.joda.time.Period; - -import java.text.NumberFormat; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -} - -@parser::members { - public Map aggregators = new LinkedHashMap(); - public List postAggregators = new LinkedList(); - public DimFilter filter; - public List intervals; - public List fields = new LinkedList(); - public QueryGranularity granularity = QueryGranularity.ALL; - public Map groupByDimensions = new LinkedHashMap(); - - String dataSourceName = null; - - public String getDataSource() { - return dataSourceName; - } - - public String unescape(String quoted) { - String unquote = quoted.trim().replaceFirst("^'(.*)'\$", "\$1"); - return unquote.replace("''", "'"); - } - - AggregatorFactory evalAgg(String name, int fn) { - switch (fn) { - case SUM: return new DoubleSumAggregatorFactory("sum("+name+")", name); - case MIN: return new DoubleMinAggregatorFactory("min("+name+")", name); - case MAX: return new DoubleMaxAggregatorFactory("max("+name+")", name); - case COUNT: return new CountAggregatorFactory(name); - } - throw new IllegalArgumentException("Unknown function [" + fn + "]"); - } - - PostAggregator evalArithmeticPostAggregator(PostAggregator a, List ops, List b) { - if(b.isEmpty()) return a; - else { - int i = 0; - - PostAggregator root = a; - while(i < ops.size()) { - List list = new LinkedList(); - List names = new LinkedList(); - - names.add(root.getName()); - list.add(root); - - Token op = ops.get(i); - - while(i < ops.size() && ops.get(i).getType() == op.getType()) { - PostAggregator e = b.get(i); - list.add(e); - names.add(e.getName()); - i++; - } - - root = new ArithmeticPostAggregator("("+Joiner.on(op.getText()).join(names)+")", op.getText(), list); - } - - return root; - } - } -} - - -AND: 'and'; -OR: 'or'; -SUM: 'sum'; -MIN: 'min'; -MAX: 'max'; -COUNT: 'count'; -AS: 'as'; -OPEN: '('; -CLOSE: ')'; -STAR: '*'; -NOT: '!' ; -PLUS: '+'; -MINUS: '-'; -DIV: '/'; -COMMA: ','; -EQ: '='; -NEQ: '!='; -MATCH: '~'; -GROUP: 'group'; - -IDENT : (LETTER)(LETTER | DIGIT | '_')* ; -QUOTED_STRING : '\'' ( ESC | ~'\'' )*? '\'' ; -ESC : '\'' '\''; - -NUMBER: DIGIT*'.'?DIGIT+(EXPONENT)?; -EXPONENT: ('e') ('+'|'-')? ('0'..'9')+; -fragment DIGIT : '0'..'9'; -fragment LETTER : 'a'..'z' | 'A'..'Z'; - -LINE_COMMENT : '--' .*? '\r'? '\n' -> skip ; -COMMENT : '/*' .*? '*/' -> skip ; -WS : (' '| '\t' | '\r' '\n' | '\n' | '\r')+ -> skip; - - - -query - : select_stmt where_stmt (groupby_stmt)? - ; - -select_stmt - : 'select' e+=aliasedExpression (',' e+=aliasedExpression)* 'from' datasource { - for(AliasedExpressionContext a : $e) { - postAggregators.add(a.p); - fields.add(a.p.getName()); - } - this.dataSourceName = $datasource.text; - } - ; - -where_stmt - : 'where' f=timeAndDimFilter { - if($f.filter != null) this.filter = $f.filter; - this.intervals = Lists.newArrayList($f.interval); - } - ; - -groupby_stmt - : GROUP 'by' groupByExpression ( COMMA! groupByExpression )* - ; - -groupByExpression - : gran=granularityFn {this.granularity = $gran.granularity;} - | dim=IDENT { this.groupByDimensions.put($dim.text, new DefaultDimensionSpec($dim.text, $dim.text)); } - ; - -datasource - : IDENT - ; - -aliasedExpression returns [PostAggregator p] - : expression ( AS^ name=IDENT )? { - if($name != null) { - postAggregators.add($expression.p); - $p = new FieldAccessPostAggregator($name.text, $expression.p.getName()); - } - else $p = $expression.p; - } - ; - -expression returns [PostAggregator p] - : additiveExpression { $p = $additiveExpression.p; } - ; - -additiveExpression returns [PostAggregator p] - : a=multiplyExpression (( ops+=PLUS^ | ops+=MINUS^ ) b+=multiplyExpression)* { - List rhs = new LinkedList(); - for(MultiplyExpressionContext e : $b) rhs.add(e.p); - $p = evalArithmeticPostAggregator($a.p, $ops, rhs); - } - ; - -multiplyExpression returns [PostAggregator p] - : a=unaryExpression ((ops+= STAR | ops+=DIV ) b+=unaryExpression)* { - List rhs = new LinkedList(); - for(UnaryExpressionContext e : $b) rhs.add(e.p); - $p = evalArithmeticPostAggregator($a.p, $ops, rhs); - } - ; - -unaryExpression returns [PostAggregator p] - : MINUS e=unaryExpression { - if($e.p instanceof ConstantPostAggregator) { - ConstantPostAggregator c = (ConstantPostAggregator)$e.p; - double v = c.getConstantValue().doubleValue() * -1; - $p = new ConstantPostAggregator(Double.toString(v), v); - } else { - $p = new ArithmeticPostAggregator( - "-"+$e.p.getName(), - "*", - Lists.newArrayList($e.p, new ConstantPostAggregator("-1", -1.0)) - ); - } - } - | PLUS e=unaryExpression { $p = $e.p; } - | primaryExpression { $p = $primaryExpression.p; } - ; - -primaryExpression returns [PostAggregator p] - : constant { $p = $constant.c; } - | aggregate { - aggregators.put($aggregate.agg.getName(), $aggregate.agg); - $p = new FieldAccessPostAggregator($aggregate.agg.getName(), $aggregate.agg.getName()); - } - | OPEN! e=expression CLOSE! { $p = $e.p; } - ; - -aggregate returns [AggregatorFactory agg] - : fn=( SUM^ | MIN^ | MAX^ ) OPEN! name=(IDENT|COUNT) CLOSE! { $agg = evalAgg($name.text, $fn.type); } - | fn=COUNT OPEN! STAR CLOSE! { $agg = evalAgg("count(*)", $fn.type); } - ; - -constant returns [ConstantPostAggregator c] - : value=NUMBER { double v = Double.parseDouble($value.text); $c = new ConstantPostAggregator(Double.toString(v), v); } - ; - -/* time filters must be top level filters */ -timeAndDimFilter returns [DimFilter filter, org.joda.time.Interval interval] - : (f1=dimFilter AND)? t=timeFilter (AND f2=dimFilter)? { - if($f1.ctx != null || $f2.ctx != null) { - if($f1.ctx != null && $f2.ctx != null) { - $filter = new AndDimFilter(Lists.newArrayList($f1.filter, $f2.filter)); - } else if($f1.ctx != null) { - $filter = $f1.filter; - } else { - $filter = $f2.filter; - } - } - $interval = $t.interval; - } - ; - -dimFilter returns [DimFilter filter] - : e=orDimFilter { $filter = $e.filter; } - ; - -orDimFilter returns [DimFilter filter] - : a=andDimFilter (OR^ b+=andDimFilter)* { - if($b.isEmpty()) $filter = $a.filter; - else { - List rest = new ArrayList(); - for(AndDimFilterContext e : $b) rest.add(e.filter); - $filter = new OrDimFilter(Lists.asList($a.filter, rest.toArray(new DimFilter[]{}))); - } - } - ; - -andDimFilter returns [DimFilter filter] - : a=primaryDimFilter (AND^ b+=primaryDimFilter)* { - if($b.isEmpty()) $filter = $a.filter; - else { - List rest = new ArrayList(); - for(PrimaryDimFilterContext e : $b) rest.add(e.filter); - $filter = new AndDimFilter(Lists.asList($a.filter, rest.toArray(new DimFilter[]{}))); - } - } - ; - -primaryDimFilter returns [DimFilter filter] - : e=selectorDimFilter { $filter = $e.filter; } - | l=inListDimFilter { $filter = $l.filter; } - | NOT f=dimFilter { $filter = new NotDimFilter($f.filter); } - | OPEN! f=dimFilter CLOSE! { $filter = $f.filter; } - ; - -selectorDimFilter returns [DimFilter filter] - : dimension=IDENT op=(EQ|NEQ|MATCH) value=QUOTED_STRING { - String dim = $dimension.text; - String val = unescape($value.text); - switch($op.type) { - case(EQ): $filter = new SelectorDimFilter(dim, val, null); break; - case(NEQ): $filter = new NotDimFilter(new SelectorDimFilter(dim, val, null)); break; - case(MATCH): $filter = new RegexDimFilter(dim, val, null); break; - } - } - ; - -inListDimFilter returns [DimFilter filter] - : dimension=IDENT 'in' (OPEN! ( (list+=QUOTED_STRING (COMMA! list+=QUOTED_STRING)*) ) CLOSE!) { - List filterList = new LinkedList(); - for(Token e : $list) filterList.add(new SelectorDimFilter($dimension.text, unescape(e.getText()), null)); - $filter = new OrDimFilter(filterList); - } - ; - -timeFilter returns [org.joda.time.Interval interval, QueryGranularity granularity] - : 'timestamp' 'between' s=timestamp AND e=timestamp { - $interval = new org.joda.time.Interval($s.t, $e.t); - } - ; - -granularityFn returns [QueryGranularity granularity] - : 'granularity' OPEN! 'timestamp' ',' str=QUOTED_STRING CLOSE! { - String granStr = unescape($str.text); - try { - $granularity = QueryGranularity.fromString(granStr); - } catch(IllegalArgumentException e) { - $granularity = new PeriodGranularity(new Period(granStr), null, null); - } - } - ; - -timestamp returns [DateTime t] - : NUMBER { - String str = $NUMBER.text.trim(); - try { - $t = new DateTime(NumberFormat.getInstance().parse(str)); - } - catch(ParseException e) { - throw new IllegalArgumentException("Unable to parse number [" + str + "]"); - } - } - | QUOTED_STRING { $t = new DateTime(unescape($QUOTED_STRING.text)); } - ; diff --git a/server/src/main/java/io/druid/server/sql/SQLRunner.java b/server/src/main/java/io/druid/server/sql/SQLRunner.java deleted file mode 100644 index eda71827f0a0..000000000000 --- a/server/src/main/java/io/druid/server/sql/SQLRunner.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.server.sql; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.google.common.base.Charsets; -import com.google.common.base.Function; -import com.google.common.base.Joiner; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.metamx.common.StringUtils; -import com.metamx.common.guava.CloseQuietly; -import io.druid.data.input.Row; -import io.druid.jackson.DefaultObjectMapper; -import io.druid.query.Druids; -import io.druid.query.Query; -import io.druid.query.Result; -import io.druid.query.aggregation.AggregatorFactory; -import io.druid.query.dimension.DimensionSpec; -import io.druid.query.groupby.GroupByQuery; -import io.druid.query.timeseries.TimeseriesResultValue; -import io.druid.sql.antlr4.DruidSQLLexer; -import io.druid.sql.antlr4.DruidSQLParser; -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.ConsoleErrorListener; -import org.antlr.v4.runtime.TokenStream; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.GnuParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; - -import javax.annotation.Nullable; -import javax.ws.rs.core.MediaType; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.net.URL; -import java.net.URLConnection; -import java.util.ArrayList; -import java.util.List; - -public class SQLRunner -{ - private static final String STATEMENT = "select count(*), (1 - count(*) / sum(count)) * 100 as ratio from wikipedia where" - + " timestamp between '2013-02-01' and '2013-02-14'" - + " and (namespace = 'article' or page ~ 'Talk:.*')" - + " and language in ( 'en', 'fr' ) " - + " and user ~ '(?i)^david.*'" - + " group by granularity(timestamp, 'day'), language"; - - public static void main(String[] args) throws Exception - { - - Options options = new Options(); - options.addOption("h", "help", false, "help"); - options.addOption("v", false, "verbose"); - options.addOption("e", "host", true, "endpoint [hostname:port]"); - - CommandLine cmd = new GnuParser().parse(options, args); - - if(cmd.hasOption("h")) { - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("SQLRunner", options); - System.exit(2); - } - - String hostname = cmd.getOptionValue("e", "localhost:8080"); - String sql = cmd.getArgs().length > 0 ? cmd.getArgs()[0] : STATEMENT; - - ObjectMapper objectMapper = new DefaultObjectMapper(); - ObjectWriter jsonWriter = objectMapper.writerWithDefaultPrettyPrinter(); - - CharStream stream = new ANTLRInputStream(sql); - DruidSQLLexer lexer = new DruidSQLLexer(stream); - TokenStream tokenStream = new CommonTokenStream(lexer); - DruidSQLParser parser = new DruidSQLParser(tokenStream); - lexer.removeErrorListeners(); - parser.removeErrorListeners(); - - lexer.addErrorListener(ConsoleErrorListener.INSTANCE); - parser.addErrorListener(ConsoleErrorListener.INSTANCE); - - try { - DruidSQLParser.QueryContext queryContext = parser.query(); - if(parser.getNumberOfSyntaxErrors() > 0) throw new IllegalStateException(); -// parser.setBuildParseTree(true); -// System.err.println(q.toStringTree(parser)); - } catch(Exception e) { - String msg = e.getMessage(); - if(msg != null) System.err.println(e); - System.exit(1); - } - - final Query query; - final TypeReference typeRef; - boolean groupBy = false; - if(parser.groupByDimensions.isEmpty()) { - query = Druids.newTimeseriesQueryBuilder() - .dataSource(parser.getDataSource()) - .aggregators(new ArrayList(parser.aggregators.values())) - .postAggregators(parser.postAggregators) - .intervals(parser.intervals) - .granularity(parser.granularity) - .filters(parser.filter) - .build(); - - typeRef = new TypeReference>>(){}; - } else { - query = GroupByQuery.builder() - .setDataSource(parser.getDataSource()) - .setAggregatorSpecs(new ArrayList(parser.aggregators.values())) - .setPostAggregatorSpecs(parser.postAggregators) - .setInterval(parser.intervals) - .setGranularity(parser.granularity) - .setDimFilter(parser.filter) - .setDimensions(new ArrayList(parser.groupByDimensions.values())) - .build(); - - typeRef = new TypeReference>(){}; - groupBy = true; - } - - String queryStr = jsonWriter.writeValueAsString(query); - if(cmd.hasOption("v")) System.err.println(queryStr); - - URL url = new URL(String.format("http://%s/druid/v2/?pretty", hostname)); - final URLConnection urlConnection = url.openConnection(); - urlConnection.addRequestProperty("content-type", MediaType.APPLICATION_JSON); - urlConnection.getOutputStream().write(StringUtils.toUtf8(queryStr)); - BufferedReader stdInput = new BufferedReader(new InputStreamReader(urlConnection.getInputStream(), Charsets.UTF_8)); - - Object res = objectMapper.readValue(stdInput, typeRef); - - Joiner tabJoiner = Joiner.on("\t"); - - if(groupBy) { - List rows = (List)res; - Iterable dimensions = Iterables.transform(parser.groupByDimensions.values(), new Function() - { - @Override - public String apply(@Nullable DimensionSpec input) - { - return input.getOutputName(); - } - }); - - System.out.println(tabJoiner.join(Iterables.concat( - Lists.newArrayList("timestamp"), - dimensions, - parser.fields - ))); - for(final Row r : rows) { - System.out.println( - tabJoiner.join( - Iterables.concat( - Lists.newArrayList(parser.granularity.toDateTime(r.getTimestampFromEpoch())), - Iterables.transform( - parser.groupByDimensions.values(), new Function() - { - @Override - public String apply(@Nullable DimensionSpec input) - { - return Joiner.on(",").join(r.getDimension(input.getOutputName())); - } - }), - Iterables.transform(parser.fields, new Function() - { - @Override - public Object apply(@Nullable String input) - { - return r.getFloatMetric(input); - } - }) - ) - ) - ); - } - } - else { - List> rows = (List>)res; - System.out.println(tabJoiner.join(Iterables.concat( - Lists.newArrayList("timestamp"), - parser.fields - ))); - for(final Result r : rows) { - System.out.println( - tabJoiner.join( - Iterables.concat( - Lists.newArrayList(r.getTimestamp()), - Lists.transform( - parser.fields, - new Function() - { - @Override - public Object apply(@Nullable String input) - { - return r.getValue().getMetric(input); - } - } - ) - ) - ) - ); - } - } - - CloseQuietly.close(stdInput); - } -}