Skip to content

string -> expression -> string -> expression #9367

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jon-wei merged 11 commits into from
Feb 21, 2020
Merged

string -> expression -> string -> expression #9367

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
bbc5800
add Expr.stringify which produces parseable expression strings, parse…
clintropolis Feb 15, 2020
7f35b8f
oops, macros are expressions too
clintropolis Feb 15, 2020
6745c34
style
clintropolis Feb 16, 2020
9a8e23e
spotbugs
clintropolis Feb 16, 2020
7389ded
qualified type arrays
clintropolis Feb 18, 2020
9a84920
Merge remote-tracking branch 'upstream/master' into reversible-expres…
clintropolis Feb 19, 2020
8aafe2e
review stuffs
clintropolis Feb 19, 2020
ad0b01f
simplify grammar
clintropolis Feb 20, 2020
276a4c5
more permissive array parsing
clintropolis Feb 21, 2020
7efa7d6
reuse expr joiner
clintropolis Feb 21, 2020
56bf7c3
fix it
clintropolis Feb 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 29 additions & 19 deletions core/src/main/antlr4/org/apache/druid/math/expr/antlr/Expr.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,32 +15,42 @@

grammar Expr;

expr : 'null' # null
| ('-'|'!') expr # unaryOpExpr
|<assoc=right> expr '^' expr # powOpExpr
| expr ('*'|'/'|'%') expr # mulDivModuloExpr
| expr ('+'|'-') expr # addSubExpr
| expr ('<'|'<='|'>'|'>='|'=='|'!=') expr # logicalOpExpr
| expr ('&&'|'||') expr # logicalAndOrExpr
| '(' expr ')' # nestedExpr
| IDENTIFIER '(' lambda ',' fnArgs ')' # applyFunctionExpr
| IDENTIFIER '(' fnArgs? ')' # functionExpr
| IDENTIFIER # identifierExpr
| DOUBLE # doubleExpr
| LONG # longExpr
| STRING # string
| '[' DOUBLE (',' DOUBLE)* ']' # doubleArray
| '[' LONG (',' LONG)* ']' # longArray
| '[' STRING (',' STRING)* ']' # stringArray
| '[]' # emptyArray
expr : NULL # null
| ('-'|'!') expr # unaryOpExpr
|<assoc=right> expr '^' expr # powOpExpr
| expr ('*'|'/'|'%') expr # mulDivModuloExpr
| expr ('+'|'-') expr # addSubExpr
| expr ('<'|'<='|'>'|'>='|'=='|'!=') expr # logicalOpExpr
| expr ('&&'|'||') expr # logicalAndOrExpr
| '(' expr ')' # nestedExpr
| IDENTIFIER '(' lambda ',' fnArgs ')' # applyFunctionExpr
| IDENTIFIER '(' fnArgs? ')' # functionExpr
| IDENTIFIER # identifierExpr
| DOUBLE # doubleExpr
| LONG # longExpr
| STRING # string
| '[' (stringElement (',' stringElement)*)? ']' # stringArray
| '[' longElement (',' longElement)*']' # longArray
| '<LONG>' '[' (numericElement (',' numericElement)*)? ']' # explicitLongArray
| '<DOUBLE>'? '[' (numericElement (',' numericElement)*)? ']' # doubleArray
| '<STRING>' '[' (literalElement (',' literalElement)*)? ']' # explicitStringArray
;

lambda : (IDENTIFIER | '(' ')' | '(' IDENTIFIER (',' IDENTIFIER)* ')') '->' expr
;

fnArgs : expr (',' expr)* # functionArgs
fnArgs : expr (',' expr)* # functionArgs
;

stringElement : (STRING | NULL);

longElement : (LONG | NULL);

numericElement : (LONG | DOUBLE | NULL);

literalElement : (STRING | LONG | DOUBLE | NULL);

NULL : 'null';
IDENTIFIER : [_$a-zA-Z][_$a-zA-Z0-9]* | '"' (ESC | ~ [\"\\])* '"';
LONG : [0-9]+ ;
DOUBLE : [0-9]+ '.' [0-9]* ;
Expand Down
40 changes: 40 additions & 0 deletions core/src/main/java/org/apache/druid/java/util/common/Numbers.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ public static double tryParseDouble(@Nullable Object val, double nullValue)
}
}


/**
* Try parsing the given Number or String object val as long.
* @param val
Expand Down Expand Up @@ -167,6 +168,45 @@ public static float tryParseFloat(@Nullable Object val, float nullValue)
}
}

/**
* Like {@link #tryParseDouble}, but does not produce a primitive and will explode if unable to produce a Double
* similar to {@link Double#parseDouble}
*/
@Nullable
public static Double parseDoubleObject(@Nullable String val)
{
if (val == null) {
return null;
}
Double d = Doubles.tryParse(val);
if (d != null) {
return d;
}
throw new NumberFormatException("Cannot parse string to double");
}

/**
* Like {@link #tryParseLong} but does not produce a primitive and will explode if unable to produce a Long
* similar to {@link Long#parseLong}
*/
@Nullable
public static Long parseLongObject(@Nullable String val)
{
if (val == null) {
return null;
}
Long lobj = Longs.tryParse(val);
if (lobj != null) {
return lobj;
}
// try as a double, for "ddd.dd" , Longs.tryParse(..) returns null
Double dobj = Doubles.tryParse((String) val);
if (dobj != null) {
return dobj.longValue();
}
throw new NumberFormatException("Cannot parse string to long");
}

public static int toIntExact(long value, String error)
{
if ((int) value != value) {
Expand Down
112 changes: 111 additions & 1 deletion core/src/main/java/org/apache/druid/math/expr/Expr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,14 @@

package org.apache.druid.math.expr;

import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.common.math.LongMath;
import com.google.common.primitives.Ints;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.ISE;
Expand All @@ -46,6 +48,8 @@
*/
public interface Expr
{
String NULL_LITERAL = "null";
Joiner ARG_JOINER = Joiner.on(", ");
/**
* Indicates expression is a constant whose literal value can be extracted by {@link Expr#getLiteralValue()},
* making evaluating with arguments and bindings unecessary
Expand Down Expand Up @@ -109,6 +113,12 @@ default String getBindingIfIdentifier()
*/
ExprEval eval(ObjectBinding bindings);

/**
* Convert the {@link Expr} back into parseable string that when parsed with
* {@link Parser#parse(String, ExprMacroTable)} will produce an equivalent {@link Expr}.
*/
String stringify();

/**
* Programmatically inspect the {@link Expr} tree with a {@link Visitor}. Each {@link Expr} is responsible for
* ensuring the {@link Visitor} can visit all of its {@link Expr} children before visiting itself
Expand Down Expand Up @@ -463,6 +473,12 @@ public BindingDetails analyzeInputs()
{
return new BindingDetails();
}

@Override
public String stringify()
{
return toString();
}
}

abstract class NullNumericConstantExpr extends ConstantExpr
Expand All @@ -476,7 +492,7 @@ public Object getLiteralValue()
@Override
public String toString()
{
return "null";
return NULL_LITERAL;
}
}

Expand Down Expand Up @@ -544,6 +560,15 @@ public ExprEval eval(ObjectBinding bindings)
{
return ExprEval.ofLongArray(value);
}

@Override
public String stringify()
{
if (value.length == 0) {
return "<LONG>[]";
}
return StringUtils.format("<LONG>%s", toString());
}
}

class StringExpr extends ConstantExpr
Expand Down Expand Up @@ -574,6 +599,13 @@ public ExprEval eval(ObjectBinding bindings)
{
return ExprEval.of(value);
}

@Override
public String stringify()
{
// escape as javascript string since string literals are wrapped in single quotes
return value == null ? NULL_LITERAL : StringUtils.format("'%s'", StringEscapeUtils.escapeJavaScript(value));
Copy link
Copy Markdown
Contributor

@jon-wei jon-wei Feb 19, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a comment about why escapeJavaScript is used (or escapeJava for IdentifierExpr )?

}
}

class StringArrayExpr extends ConstantExpr
Expand Down Expand Up @@ -602,6 +634,27 @@ public ExprEval eval(ObjectBinding bindings)
{
return ExprEval.ofStringArray(value);
}

@Override
public String stringify()
{
if (value.length == 0) {
return "<STRING>[]";
}

return StringUtils.format(
"<STRING>[%s]",
ARG_JOINER.join(
Arrays.stream(value)
.map(s -> s == null
? NULL_LITERAL
// escape as javascript string since string literals are wrapped in single quotes
: StringUtils.format("'%s'", StringEscapeUtils.escapeJavaScript(s))
)
.iterator()
)
);
}
}

class DoubleExpr extends ConstantExpr
Expand Down Expand Up @@ -667,6 +720,15 @@ public ExprEval eval(ObjectBinding bindings)
{
return ExprEval.ofDoubleArray(value);
}

@Override
public String stringify()
{
if (value.length == 0) {
return "<DOUBLE>[]";
}
return StringUtils.format("<DOUBLE>%s", toString());
}
}

/**
Expand Down Expand Up @@ -757,6 +819,13 @@ public ExprEval eval(ObjectBinding bindings)
return ExprEval.bestEffortOf(bindings.get(binding));
}

@Override
public String stringify()
{
// escape as java strings since identifiers are wrapped in double quotes
return StringUtils.format("\"%s\"", StringEscapeUtils.escapeJava(binding));
}

@Override
public void visit(Visitor visitor)
{
Expand Down Expand Up @@ -823,6 +892,12 @@ public ExprEval eval(ObjectBinding bindings)
return expr.eval(bindings);
}

@Override
public String stringify()
{
return StringUtils.format("(%s) -> %s", ARG_JOINER.join(getIdentifiers()), expr.stringify());
}

@Override
public void visit(Visitor visitor)
{
Expand Down Expand Up @@ -879,6 +954,12 @@ public ExprEval eval(ObjectBinding bindings)
return function.apply(args, bindings);
}

@Override
public String stringify()
{
return StringUtils.format("%s(%s)", name, ARG_JOINER.join(args.stream().map(Expr::stringify).iterator()));
}

@Override
public void visit(Visitor visitor)
{
Expand Down Expand Up @@ -965,6 +1046,17 @@ public ExprEval eval(ObjectBinding bindings)
return function.apply(lambdaExpr, argsExpr, bindings);
}

@Override
public String stringify()
{
return StringUtils.format(
"%s(%s, %s)",
name,
lambdaExpr.stringify(),
ARG_JOINER.join(argsExpr.stream().map(Expr::stringify).iterator())
);
}

@Override
public void visit(Visitor visitor)
{
Expand Down Expand Up @@ -1059,6 +1151,12 @@ public ExprEval eval(ObjectBinding bindings)
throw new IAE("unsupported type " + ret.type());
}

@Override
public String stringify()
{
return StringUtils.format("-%s", expr.stringify());
}

@Override
public String toString()
{
Expand Down Expand Up @@ -1091,6 +1189,12 @@ public ExprEval eval(ObjectBinding bindings)
return ExprEval.of(!ret.asBoolean(), retType);
}

@Override
public String stringify()
{
return StringUtils.format("!%s", expr.stringify());
}

@Override
public String toString()
{
Expand Down Expand Up @@ -1144,6 +1248,12 @@ public String toString()
return StringUtils.format("(%s %s %s)", op, left, right);
}

@Override
public String stringify()
{
return StringUtils.format("(%s %s %s)", left.stringify(), op, right.stringify());
}

protected abstract BinaryOpExprBase copy(Expr left, Expr right);

@Override
Expand Down
Loading