Skip to content

Fixes for safe_divide with vectorize and datatypes #15839

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
abhishekagarwal87 merged 3 commits into from
Feb 8, 2024
Merged

Fixes for safe_divide with vectorize and datatypes #15839

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9dbafed
Fix for save_divide with vectorize
somu-imply Feb 6, 2024
efaf8ef
More fixes
somu-imply Feb 6, 2024
c86d0e3
Update to use expr.eval(null) for both cases when denominator is 0
somu-imply Feb 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions processing/src/main/java/org/apache/druid/math/expr/Function.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1173,14 +1173,17 @@ public ExpressionType getOutputType(Expr.InputBindingInspector inspector, List<E
);
}

@Override
public boolean canVectorize(Expr.InputBindingInspector inspector, List<Expr> args)
{
return false;
}

@Override
protected ExprEval eval(final long x, final long y)
{
if (y == 0) {
if (x != 0) {
return ExprEval.ofLong(null);
}
return ExprEval.ofLong(0);
return ExprEval.ofLong(null);
}
return ExprEval.ofLong(x / y);
}
Expand Down
9 changes: 6 additions & 3 deletions processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -857,11 +857,14 @@ public void testSafeDivide()
assertExpr("safe_divide(4.5, 2)", 2.25);
assertExpr("safe_divide(3, 0)", null);
assertExpr("safe_divide(1, 0.0)", null);
// NaN and Infinity cases
// NaN, Infinity and other weird cases
assertExpr("safe_divide(NaN, 0.0)", null);
assertExpr("safe_divide(0, NaN)", 0.0);
assertExpr("safe_divide(0, POSITIVE_INFINITY)", NullHandling.defaultLongValue());
assertExpr("safe_divide(POSITIVE_INFINITY,0)", NullHandling.defaultLongValue());
assertExpr("safe_divide(0, maxLong)", 0L);
assertExpr("safe_divide(maxLong,0)", null);
assertExpr("safe_divide(0.0, inf)", 0.0);
assertExpr("safe_divide(0.0, -inf)", -0.0);
assertExpr("safe_divide(0,0)", null);
}

@Test
Expand Down
8 changes: 5 additions & 3 deletions ...in/java/org/apache/druid/sql/calcite/expression/builtin/SafeDivideOperatorConversion.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,9 @@
import org.apache.calcite.sql.SqlFunction;
import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.type.OperandTypes;
import org.apache.calcite.sql.type.ReturnTypes;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeTransforms;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.Function;
import org.apache.druid.sql.calcite.expression.DirectOperatorConversion;
Expand All @@ -33,9 +34,10 @@ public class SafeDivideOperatorConversion extends DirectOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder(StringUtils.toUpperCase(Function.SafeDivide.NAME))
.operandTypeChecker(OperandTypes.ANY_NUMERIC)
Copy link
Copy Markdown
Contributor Author

@somu-imply somu-imply Feb 6, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Refactored this slightly and added the proper return type inference. Quotient depended on if the numerator and denominators are null and would throw an exception of failing datatype EXPR$0: INTEGER NOT NULL -> INTEGER

.returnTypeInference(ReturnTypes.QUOTIENT_NULLABLE)
.operandTypes(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)
.returnTypeInference(ReturnTypes.LEAST_RESTRICTIVE.andThen(SqlTypeTransforms.FORCE_NULLABLE))
.functionCategory(SqlFunctionCategory.USER_DEFINED_FUNCTION)
.requiredOperandCount(2)
.build();

public SafeDivideOperatorConversion()
Expand Down
25 changes: 25 additions & 0 deletions sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -570,6 +570,31 @@ public void testGroupByWithForceLimitPushDown()
);
}

@Test
public void testSafeDivide()
{
skipVectorize();
cannotVectorize();
final Map<String, Object> context = new HashMap<>(QUERY_CONTEXT_DEFAULT);

testQuery(
"select count(*) c from foo where ((floor(safe_divide(cast(cast(m1 as char) as bigint), 2))) = 0)",
context,
ImmutableList.of(
Druids.newTimeseriesQueryBuilder()
.dataSource(CalciteTests.DATASOURCE1)
.intervals(querySegmentSpec(Filtration.eternity()))
.virtualColumns(expressionVirtualColumn("v0", "floor(safe_divide(CAST(CAST(\"m1\", 'STRING'), 'LONG'),2))", ColumnType.LONG))
.filters(equality("v0", 0L, ColumnType.LONG))
.granularity(Granularities.ALL)
.aggregators(new CountAggregatorFactory("a0"))
.context(context)
.build()
),
ImmutableList.of(new Object[]{1L})
);
}

@Test
public void testGroupByLimitWrappingOrderByAgg()
{
Expand Down
50 changes: 48 additions & 2 deletions sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,52 @@ public void testBitwiseExpressions()
);
}

@Test
public void testSafeDivideWithoutTable()
{
skipVectorize();
cannotVectorize();
final Map<String, Object> context = new HashMap<>(QUERY_CONTEXT_DEFAULT);

testQuery(
"select SAFE_DIVIDE(0, 0), SAFE_DIVIDE(1,0), SAFE_DIVIDE(10,2.5), "
+ " SAFE_DIVIDE(10.5,3.5), SAFE_DIVIDE(10.5,3), SAFE_DIVIDE(10,2)",
context,
ImmutableList.of(
Druids.newScanQueryBuilder()
.dataSource(
InlineDataSource.fromIterable(
ImmutableList.of(
new Object[]{0L}
),
RowSignature.builder().add("ZERO", ColumnType.LONG).build()
)
)
.intervals(querySegmentSpec(Filtration.eternity()))
.columns("v0", "v1", "v2", "v3", "v4")
.virtualColumns(
expressionVirtualColumn("v0", NullHandling.sqlCompatible() ? "null" : "0", ColumnType.LONG),
expressionVirtualColumn("v1", "4.0", ColumnType.DOUBLE),
expressionVirtualColumn("v2", "3.0", ColumnType.DOUBLE),
expressionVirtualColumn("v3", "3.5", ColumnType.DOUBLE),
expressionVirtualColumn("v4", "5", ColumnType.LONG)
)
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.legacy(false)
.context(context)
.build()
),
ImmutableList.of(new Object[]{
NullHandling.sqlCompatible() ? null : 0,
NullHandling.sqlCompatible() ? null : 0,
4.0D,
3.0D,
3.5D,
5
})
);
}

@Test
public void testSafeDivideExpressions()
{
Expand All @@ -498,8 +544,8 @@ public void testSafeDivideExpressions()
} else {
expected = ImmutableList.of(
new Object[]{null, null, null, 7.0F},
new Object[]{1.0F, 1L, 1.0, 3253230.0F},
new Object[]{0.0F, 0L, 0.0, 0.0F},
new Object[]{1.0F, 1L, 1.0D, 3253230.0F},
new Object[]{0.0F, null, 0.0D, 0.0F},
Copy link
Copy Markdown
Contributor Author

@somu-imply somu-imply Feb 6, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This change is due to change in eval handling SQL compatibility mode correctly

new Object[]{null, null, null, null},
new Object[]{null, null, null, null},
new Object[]{null, null, null, null}
Expand Down