diff --git a/core/src/main/antlr4/org/apache/druid/math/expr/antlr/Expr.g4 b/core/src/main/antlr4/org/apache/druid/math/expr/antlr/Expr.g4 index db336a2d54e0..11853a161586 100644 --- a/core/src/main/antlr4/org/apache/druid/math/expr/antlr/Expr.g4 +++ b/core/src/main/antlr4/org/apache/druid/math/expr/antlr/Expr.g4 @@ -16,8 +16,9 @@ grammar Expr; expr : NULL # null - | ('-'|'!') expr # unaryOpExpr + | ('-'|'!'|'~') expr # unaryOpExpr | expr '^' expr # powOpExpr + | expr ('&'|'|') expr # bitwiseOpExpr | expr ('*'|'/'|'%') expr # mulDivModuloExpr | expr ('+'|'-') expr # addSubExpr | expr ('<'|'<='|'>'|'>='|'=='|'!=') expr # logicalOpExpr @@ -76,3 +77,6 @@ EQ : '==' ; NEQ : '!=' ; AND : '&&' ; OR : '||' ; +BITAND : '&' ; +BITOR : '|' ; +BITNEG : '~' ; diff --git a/core/src/main/java/org/apache/druid/math/expr/Expr.java b/core/src/main/java/org/apache/druid/math/expr/Expr.java index 1cd7d6253ff8..906f2b28d6f2 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Expr.java +++ b/core/src/main/java/org/apache/druid/math/expr/Expr.java @@ -29,6 +29,7 @@ import org.apache.commons.lang.StringEscapeUtils; import org.apache.druid.annotations.SubclassesMustOverrideEqualsAndHashCode; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.common.guava.GuavaUtils; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; @@ -1441,6 +1442,42 @@ public String toString() } } +class UnaryBitwiseNegateExpr extends UnaryExpr +{ + UnaryBitwiseNegateExpr(Expr expr) + { + super(expr); + } + + @Override + UnaryExpr copy(Expr expr) + { + return new UnaryBitwiseNegateExpr(expr); + } + + @Override + public ExprEval eval(ObjectBinding bindings) + { + ExprEval ret = expr.eval(bindings); + if ((NullHandling.sqlCompatible() && (ret.value() == null)) || ret.isNumericNull()) { + return ExprEval.of(null); + } + return ExprEval.of(~ret.asLong()); + } + + @Override + public String stringify() + { + return StringUtils.format("~%s", expr.stringify()); + } + + @Override + public String toString() + { + return StringUtils.format("~%s", expr); + } +} + /** * Base type for all binary operators, this {@link Expr} has two children {@link Expr} for the left and right side * operands. @@ -1971,6 +2008,78 @@ public ExprEval eval(ObjectBinding bindings) ExprEval leftVal = left.eval(bindings); return leftVal.asBoolean() ? leftVal : right.eval(bindings); } +} +class BinBitwiseAndExpr extends BinaryEvalOpExprBase +{ + BinBitwiseAndExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + protected BinaryOpExprBase copy(Expr left, Expr right) + { + return new BinBitwiseAndExpr(op, left, right); + } + + @Override + protected final long evalLong(long left, long right) + { + return left & right; + } + + @Override + protected final double evalDouble(double left, double right) + { + throw new IllegalArgumentException("unsupported type " + ExprType.DOUBLE); + } + + @Override + protected ExprEval evalString(@Nullable String left, @Nullable String right) + { + Long l1 = GuavaUtils.tryParseLong(left); + Long l2 = GuavaUtils.tryParseLong(right); + if (l1 == null || l2 == null) { + return ExprEval.of(null); + } + return ExprEval.of(l1 & l2); + } } +class BinBitwiseOrExpr extends BinaryEvalOpExprBase +{ + BinBitwiseOrExpr(String op, Expr left, Expr right) + { + super(op, left, right); + } + + @Override + protected BinaryOpExprBase copy(Expr left, Expr right) + { + return new BinBitwiseOrExpr(op, left, right); + } + + @Override + protected final long evalLong(long left, long right) + { + return left | right; + } + + @Override + protected final double evalDouble(double left, double right) + { + throw new IllegalArgumentException("unsupported type " + ExprType.DOUBLE); + } + + @Override + protected ExprEval evalString(@Nullable String left, @Nullable String right) + { + Long l1 = GuavaUtils.tryParseLong(left); + Long l2 = GuavaUtils.tryParseLong(right); + if (l1 == null || l2 == null) { + return ExprEval.of(null); + } + return ExprEval.of(l1 | l2); + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/ExprListenerImpl.java b/core/src/main/java/org/apache/druid/math/expr/ExprListenerImpl.java index ae41653950f9..25226e7989ef 100644 --- a/core/src/main/java/org/apache/druid/math/expr/ExprListenerImpl.java +++ b/core/src/main/java/org/apache/druid/math/expr/ExprListenerImpl.java @@ -78,6 +78,9 @@ public void exitUnaryOpExpr(ExprParser.UnaryOpExprContext ctx) case ExprParser.NOT: nodes.put(ctx, new UnaryNotExpr((Expr) nodes.get(ctx.getChild(1)))); break; + case ExprParser.BITNEG: + nodes.put(ctx, new UnaryBitwiseNegateExpr((Expr) nodes.get(ctx.getChild(1)))); + break; default: throw new RE("Unrecognized unary operator %s", ctx.getChild(0).getText()); } @@ -301,6 +304,36 @@ public void exitMulDivModuloExpr(ExprParser.MulDivModuloExprContext ctx) } } + @Override + public void exitBitwiseOpExpr(ExprParser.BitwiseOpExprContext ctx) + { + int opCode = ((TerminalNode) ctx.getChild(1)).getSymbol().getType(); + switch (opCode) { + case ExprParser.BITAND: + nodes.put( + ctx, + new BinBitwiseAndExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + case ExprParser.BITOR: + nodes.put( + ctx, + new BinBitwiseOrExpr( + ctx.getChild(1).getText(), + (Expr) nodes.get(ctx.getChild(0)), + (Expr) nodes.get(ctx.getChild(2)) + ) + ); + break; + default: + throw new RE("Unrecognized bitwise operator %s", ctx.getChild(1).getText()); + } + } + @Override public void exitPowOpExpr(ExprParser.PowOpExprContext ctx) { diff --git a/core/src/test/java/org/apache/druid/math/expr/ExprTest.java b/core/src/test/java/org/apache/druid/math/expr/ExprTest.java index ff12669bbc6c..9479675e3a40 100644 --- a/core/src/test/java/org/apache/druid/math/expr/ExprTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/ExprTest.java @@ -102,6 +102,14 @@ public void testEqualsContractForBinAndExpr() EqualsVerifier.forClass(BinAndExpr.class).usingGetClass().verify(); } + @Test + public void testEqualsContractForBitwiseExpr() + { + EqualsVerifier.forClass(BinBitwiseAndExpr.class).usingGetClass().verify(); + EqualsVerifier.forClass(BinBitwiseOrExpr.class).usingGetClass().verify(); + EqualsVerifier.forClass(UnaryBitwiseNegateExpr.class).usingGetClass().verify(); + } + @Test public void testEqualsContractForFunctionExpr() { diff --git a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java index bd755ba7e0ec..3c5c214de657 100644 --- a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java @@ -468,7 +468,7 @@ public void testGreatest() { // Same types assertExpr("greatest(y, 0)", 2L); - assertExpr("greatest(34.0, z, 5.0, 767.0", 767.0); + assertExpr("greatest(34.0, z, 5.0, 767.0)", 767.0); assertExpr("greatest('B', x, 'A')", "foo"); // Different types @@ -496,7 +496,7 @@ public void testLeast() { // Same types assertExpr("least(y, 0)", 0L); - assertExpr("least(34.0, z, 5.0, 767.0", 3.1); + assertExpr("least(34.0, z, 5.0, 767.0)", 3.1); assertExpr("least('B', x, 'A')", "A"); // Different types diff --git a/core/src/test/java/org/apache/druid/math/expr/ParserTest.java b/core/src/test/java/org/apache/druid/math/expr/ParserTest.java index b1ef6736ed5a..f2fe07b3d511 100644 --- a/core/src/test/java/org/apache/druid/math/expr/ParserTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/ParserTest.java @@ -29,6 +29,7 @@ import org.junit.Test; import org.junit.rules.ExpectedException; +import javax.annotation.Nullable; import java.util.Collections; import java.util.List; import java.util.Set; @@ -172,6 +173,43 @@ public void testMixed() validateFlatten("min(1, max(3, 4))", "(min [1, (max [3, 4])])", "1"); } + @Test + public void testBitwiseOps() + { + validateFlatten("3 & 1", "(& 3 1)", "1"); + validateFlatten("3 & 1", "(& 3 1)", "1"); + validateFlatten("2 & 1", "(& 2 1)", "0"); + validateFlatten("3 | 1", "(| 3 1)", "3"); + validateFlatten("2 | 1", "(| 2 1)", "3"); + validateFlatten("(~1) & 7", "(& ~1 7)", "6"); + + validateFlatten("'2' & '1'", "(& 2 1)", "0"); + validateFlatten("'3' | '1'", "(| 3 1)", "3"); + validateFlatten("(~'1') & 7", "(& ~1 7)", "6"); + + validateFlatten("'notanumber' & '1'", "(& notanumber 1)", null); + validateFlatten("'3' | 'notanumber'", "(| 3 notanumber)", null); + validateFlatten("~'notanumber'", "~notanumber", null); + validateFlatten("(~'notanumber') & '7'", "(& ~notanumber 7)", null); + validateFlatten("(~'notanumber') | '7'", "(| ~notanumber 7)", null); + } + + @Test + public void testBitwiseDoubleAndExplosion() + { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("DOUBLE"); + validateFlatten("3.0 & 1", "(& 3.0 1)", "1"); + } + + @Test + public void testBitwiseDoubleOrExplosion() + { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("DOUBLE"); + validateFlatten("3.0 | 1", "(| 3.0 1)", "1"); + } + @Test public void testIdentifiers() { @@ -543,7 +581,7 @@ public void testUniquify() } - private void validateFlatten(String expression, String withoutFlatten, String withFlatten) + private void validateFlatten(String expression, String withoutFlatten, @Nullable String withFlatten) { Expr notFlat = Parser.parse(expression, ExprMacroTable.nil(), false); Expr flat = Parser.parse(expression, ExprMacroTable.nil(), true); diff --git a/docs/misc/math-expr.md b/docs/misc/math-expr.md index dc356479ad58..014d1c3e4a92 100644 --- a/docs/misc/math-expr.md +++ b/docs/misc/math-expr.md @@ -35,13 +35,15 @@ This expression language supports the following operators (listed in decreasing |Operators|Description| |---------|-----------| -|!, -|Unary NOT and Minus| +|!, -, ^|Unary NOT, Minus, and bitwise Negate| |^|Binary power op| +|&, ||Binary bitwise AND, OR| |*, /, %|Binary multiplicative| |+, -|Binary additive| |<, <=, >, >=, ==, !=|Binary Comparison| |&&, |||Binary Logical AND, OR| + Long, double, and string data types are supported. If a number contains a dot, it is interpreted as a double, otherwise it is interpreted as a long. That means, always add a '.' to your number if you want it interpreted as a double value. String literals should be quoted by single quotation marks. Additionally, the expression language supports long, double, and string arrays. Array literals are created by wrapping square brackets around a list of scalar literals values delimited by a comma or space character. All values in an array literal must be the same type, however null values are accepted. Typed empty arrays may be defined by prefixing with their type in angle brackets: `[]`, `[]`, or `[]`. diff --git a/website/.spelling b/website/.spelling index f7fe62b4a26c..0420831d25c3 100644 --- a/website/.spelling +++ b/website/.spelling @@ -196,6 +196,7 @@ backfills backpressure base64 big-endian +bitwise blobstore boolean breakpoint