From 6dd891ef72cd782f22e02ccb9899f633f461b77d Mon Sep 17 00:00:00 2001 From: frank chen Date: Mon, 16 Nov 2020 18:15:58 +0800 Subject: [PATCH 01/20] add binary_byte_format/decimal_byte_format/decimal_format --- .../java/util/common/HumanReadableBytes.java | 103 ++++++++++ .../org/apache/druid/math/expr/Function.java | 84 ++++++++ .../util/common/HumanReadableBytesTest.java | 101 ++++++++++ .../apache/druid/math/expr/FunctionTest.java | 64 ++++++ docs/misc/math-expr.md | 10 + docs/querying/sql.md | 3 + .../builtin/SizeFormatOperatorConversion.java | 117 +++++++++++ .../calcite/planner/DruidOperatorTable.java | 9 + .../druid/sql/calcite/CalciteQueryTest.java | 50 ++++- .../calcite/expression/ExpressionsTest.java | 189 ++++++++++++++++++ 10 files changed, 728 insertions(+), 2 deletions(-) create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/SizeFormatOperatorConversion.java diff --git a/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java b/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java index 9d9385442173..49140237e4c5 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java +++ b/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java @@ -208,4 +208,107 @@ private static long parseInner(String rawNumber) throw new IAE("Invalid format or out of range of long: %s", rawNumber); } } + + public enum UnitSystem + { + /** + * also known as IEC format + * eg: B, KiB, MiB, GiB ... + */ + BINARY_BYTE, + + /** + * also known as SI format + * eg: B, KB, MB ... + */ + DECIMAL_BYTE, + + /** + * simplified SI format without 'B' indicator + * eg: B, K, M, G ... + */ + DECIMAL + } + + /** + * Returns a human-readable string version of input value + * + * @param bytes input value. Negative value is also allowed + * @param precision [0,3] + * @param unitSystem which unit system is adopted to format the input value, see {@link UnitSystem} + * @param hasSpace if it's true, there's an extra space between the number and the unit suffix + */ + public static String format(long bytes, int precision, UnitSystem unitSystem, boolean hasSpace) + { + if (precision < 0 || precision > 3) { + throw new IAE("precision [%d] must be in the range of [0,3]", precision); + } + String pattern = "%." + precision + (hasSpace ? "f %s%s" : "f%s%s"); + switch (unitSystem) { + case BINARY_BYTE: + return BinaryFormatter.format(bytes, pattern, "B"); + case DECIMAL_BYTE: + return DecimalFormatter.format(bytes, pattern, "B"); + case DECIMAL: + return DecimalFormatter.format(bytes, pattern, ""); + default: + throw new IAE("Unkonwn unit system[%s]", unitSystem); + } + } + + static class BinaryFormatter + { + private static final String[] UNITS = {"", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei"}; + + static String format(long bytes, String pattern, String suffix) + { + if (bytes == Long.MIN_VALUE) { + /** + * special path for Long.MIN_VALUE + * + * Long.MIN_VALUE = 2^63 = (2^60=1EiB) * 2^3 + */ + return StringUtils.format(pattern, -8.0, UNITS[UNITS.length - 1], suffix); + } + + /** + * A number and its binary bits are listed as fellows + * [0, 1KiB) = [0, 2^10) + * [1KiB, 1MiB) = [2^10, 2^20), + * [1MiB, 1GiB) = [2^20, 2^30), + * [1GiB, 1PiB) = [2^30, 2^40), + * + * So, expression (63 - Long.numberOfLeadingZeros(absValue))) helps us to get the right number of bits of the given input + * + * Internal implementaion of Long.numberOfLeadingZeros uses bit operations to do calculation so the cost is very cheap + */ + int unitIndex = (63 - Long.numberOfLeadingZeros(Math.abs(bytes))) / 10; + return StringUtils.format(pattern, (double) bytes / (1L << (unitIndex * 10)), UNITS[unitIndex], suffix); + } + } + + static class DecimalFormatter + { + private static final String[] UNITS = {"K", "M", "G", "T", "P", "E"}; + + static String format(long bytes, String pattern, String suffix) + { + /** + * handle number between (-1000, 1000) first to simply further processing + */ + if (bytes > -1000 && bytes < 1000) { + return StringUtils.format(pattern, (double) bytes, "", suffix); + } + + /** + * because max precision is 3, extra fraction can be ignored by use of integer division which might be a little more efficient + */ + int unitIndex = 0; + while (bytes <= -1000_000 || bytes >= 1000_000) { + bytes /= 1000; + unitIndex++; + } + return StringUtils.format(pattern, bytes / 1000.0, UNITS[unitIndex], suffix); + } + } } diff --git a/core/src/main/java/org/apache/druid/math/expr/Function.java b/core/src/main/java/org/apache/druid/math/expr/Function.java index 2c4ff6853749..96378b1f0dc2 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Function.java +++ b/core/src/main/java/org/apache/druid/math/expr/Function.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableSet; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.HumanReadableBytes; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.RE; import org.apache.druid.java.util.common.StringUtils; @@ -3275,4 +3276,87 @@ private Stream prepend(T val, T[] array) return l.stream(); } } + + abstract class SizeFormatFunc implements Function + { + protected abstract HumanReadableBytes.UnitSystem getUnitSystem(); + + @Override + public ExprEval apply(List args, Expr.ObjectBinding bindings) + { + final long bytes = args.get(0).eval(bindings).asLong(); + + int precision = 2; + if (args.size() > 1) { + precision = args.get(1).eval(bindings).asInt(); + } + + boolean hasSpace = false; + if (args.size() > 2) { + hasSpace = args.get(2).eval(bindings).asBoolean(); + } + + return ExprEval.of(HumanReadableBytes.format(bytes, precision, this.getUnitSystem(), hasSpace)); + } + + @Override + public void validateArguments(List args) + { + if (args.size() < 1 || args.size() > 3) { + throw new IAE("Function[%s] needs 1 or 2 or 3 arguments", name()); + } + } + + @Nullable + @Override + public ExprType getOutputType(Expr.InputBindingTypes inputTypes, List args) + { + return ExprType.STRING; + } + } + + class DecimalByteFormatFunc extends SizeFormatFunc + { + @Override + public String name() + { + return "decimal_byte_format"; + } + + @Override + protected HumanReadableBytes.UnitSystem getUnitSystem() + { + return HumanReadableBytes.UnitSystem.DECIMAL_BYTE; + } + } + + class BinaryByteFormatFunc extends SizeFormatFunc + { + @Override + public String name() + { + return "binary_byte_format"; + } + + @Override + protected HumanReadableBytes.UnitSystem getUnitSystem() + { + return HumanReadableBytes.UnitSystem.BINARY_BYTE; + } + } + + class DecimalFormatFunc extends SizeFormatFunc + { + @Override + public String name() + { + return "decimal_format"; + } + + @Override + protected HumanReadableBytes.UnitSystem getUnitSystem() + { + return HumanReadableBytes.UnitSystem.DECIMAL; + } + } } diff --git a/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java b/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java index 3d1d8922fd80..8e482fc67561 100644 --- a/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java +++ b/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java @@ -395,6 +395,107 @@ public void testBytesRange() Assert.assertEquals("value must be in the range of [0, 5]", message); } + @Test + public void testFormatInBinaryByte() + { + Assert.assertEquals("-8.00 EiB", HumanReadableBytes.format(Long.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("-8.000 EiB", HumanReadableBytes.format(Long.MIN_VALUE, 3, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + + Assert.assertEquals("-2.00 GiB", HumanReadableBytes.format(Integer.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("-32.00 KiB", HumanReadableBytes.format(Short.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("-128.00 B", HumanReadableBytes.format(Byte.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("-1.00 B", HumanReadableBytes.format(-1, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("0.00 B", HumanReadableBytes.format(0, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("1.00 B", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("1.00 KiB", HumanReadableBytes.format(1024L, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("1.00 MiB", HumanReadableBytes.format(1024L * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("1.00 GiB", HumanReadableBytes.format(1024L * 1024 * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("1.00 TiB", HumanReadableBytes.format(1024L * 1024 * 1024 * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("1.00 PiB", HumanReadableBytes.format(1024L * 1024 * 1024 * 1024 * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("8.00 EiB", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + } + + @Test + public void testPrecisionInBinaryFormat() + { + Assert.assertEquals("1 KiB", HumanReadableBytes.format(1500, 0, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("1.5 KiB", HumanReadableBytes.format(1500, 1, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("1.46 KiB", HumanReadableBytes.format(1500, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("1.465 KiB", HumanReadableBytes.format(1500, 3, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + } + + @Test + public void testPrecisionInDecimalFormat() + { + Assert.assertEquals("1 KB", HumanReadableBytes.format(1456, 0, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + Assert.assertEquals("1.5 KB", HumanReadableBytes.format(1456, 1, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + Assert.assertEquals("1.46 KB", HumanReadableBytes.format(1456, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + Assert.assertEquals("1.456 KB", HumanReadableBytes.format(1456, 3, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + } + + @Test + public void testFormatInDecimalByte() + { + Assert.assertEquals("1.00 B", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + Assert.assertEquals("1.00 KB", HumanReadableBytes.format(1000L, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + Assert.assertEquals("1.00 MB", HumanReadableBytes.format(1000L * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + Assert.assertEquals("1.00 GB", HumanReadableBytes.format(1000L * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + Assert.assertEquals("1.00 TB", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + Assert.assertEquals("1.00 PB", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + Assert.assertEquals("9.22 EB", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + + Assert.assertEquals("100.00 KB", HumanReadableBytes.format(99999, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + Assert.assertEquals("99.999 KB", HumanReadableBytes.format(99999, 3, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + + Assert.assertEquals("999.9 PB", HumanReadableBytes.format(999_949_999_999_999_999L, 1, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + Assert.assertEquals("999.95 PB", HumanReadableBytes.format(999_949_999_999_999_999L, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + Assert.assertEquals("999.949 PB", HumanReadableBytes.format(999_949_999_999_999_999L, 3, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + } + + @Test + public void testFormatInDecimal() + { + Assert.assertEquals("1.00 ", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); + Assert.assertEquals("1.00 K", HumanReadableBytes.format(1000L, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); + Assert.assertEquals("1.00 M", HumanReadableBytes.format(1000L * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); + Assert.assertEquals("1.00 G", HumanReadableBytes.format(1000L * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); + Assert.assertEquals("1.00 T", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); + Assert.assertEquals("1.00 P", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); + Assert.assertEquals("9.22 E", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); + } + + @Test + public void testFormatInDecimalNoSpace() + { + Assert.assertEquals("1.00", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.DECIMAL, false)); + Assert.assertEquals("1.00K", HumanReadableBytes.format(1000L, 2, HumanReadableBytes.UnitSystem.DECIMAL, false)); + Assert.assertEquals("1.00M", HumanReadableBytes.format(1000L * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, false)); + Assert.assertEquals("1.00G", HumanReadableBytes.format(1000L * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, false)); + Assert.assertEquals("1.00T", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, false)); + Assert.assertEquals("1.00P", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, false)); + Assert.assertEquals("9.22E", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL, false)); + + Assert.assertEquals("1P", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000 * 1000, 0, HumanReadableBytes.UnitSystem.DECIMAL, false)); + Assert.assertEquals("9E", HumanReadableBytes.format(Long.MAX_VALUE, 0, HumanReadableBytes.UnitSystem.DECIMAL, false)); + } + + @Test + public void testInvalidPrecisionArgumentLowerBound() + { + expectedException.expect(IAE.class); + expectedException.expectMessage("precision [-1] must be in the range of [0,3]"); + Assert.assertEquals("1.00", HumanReadableBytes.format(1, -1, HumanReadableBytes.UnitSystem.DECIMAL, false)); + } + + @Test + public void testInvalidPrecisionArgumentUpperBound() + { + expectedException.expect(IAE.class); + expectedException.expectMessage("precision [4] must be in the range of [0,3]"); + Assert.assertEquals("1.000", HumanReadableBytes.format(1, 3, HumanReadableBytes.UnitSystem.DECIMAL, false)); + Assert.assertEquals("1.00", HumanReadableBytes.format(1, 4, HumanReadableBytes.UnitSystem.DECIMAL, false)); + } + private static String validate(T obj) { Validator validator = Validation.buildDefaultValidatorFactory() diff --git a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java index 3c5c214de657..241eb5750653 100644 --- a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java @@ -22,11 +22,14 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.Pair; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import javax.annotation.Nullable; import java.math.BigDecimal; @@ -36,6 +39,9 @@ public class FunctionTest extends InitializedNullHandlingTest { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + private Expr.ObjectBinding bindings; @Before @@ -519,6 +525,64 @@ public void testLeast() assertExpr("least(1, null, 'A')", "1"); } + @Test + public void testSizeFormat() + { + assertExpr("binary_byte_format(-1024)", "-1.00KiB"); + assertExpr("binary_byte_format(1024)", "1.00KiB"); + assertExpr("binary_byte_format(1024*1024)", "1.00MiB"); + assertExpr("binary_byte_format(1024*1024*1024)", "1.00GiB"); + assertExpr("binary_byte_format(1024*1024*1024*1024)", "1.00TiB"); + assertExpr("binary_byte_format(1024*1024*1024*1024*1024)", "1.00PiB"); + + assertExpr("decimal_byte_format(-1000)", "-1.00KB"); + assertExpr("decimal_byte_format(1000)", "1.00KB"); + assertExpr("decimal_byte_format(1000*1000)", "1.00MB"); + assertExpr("decimal_byte_format(1000*1000*1000)", "1.00GB"); + assertExpr("decimal_byte_format(1000*1000*1000*1000)", "1.00TB"); + + assertExpr("decimal_format(-1000)", "-1.00K"); + assertExpr("decimal_format(1000)", "1.00K"); + assertExpr("decimal_format(1000*1000)", "1.00M"); + assertExpr("decimal_format(1000*1000*1000)", "1.00G"); + assertExpr("decimal_format(1000*1000*1000*1000)", "1.00T"); + } + + @Test + public void testSizeFormatWithDifferentPrecision() + { + assertExpr("binary_byte_format(1024, 0)", "1KiB"); + assertExpr("binary_byte_format(1024*1024, 1)", "1.0MiB"); + assertExpr("binary_byte_format(1024*1024*1024, 2)", "1.00GiB"); + assertExpr("binary_byte_format(1024*1024*1024*1024, 3)", "1.000TiB"); + + assertExpr("decimal_byte_format(1234, 0)", "1KB"); + assertExpr("decimal_byte_format(1234*1000, 1)", "1.2MB"); + assertExpr("decimal_byte_format(1234*1000*1000, 2)", "1.23GB"); + assertExpr("decimal_byte_format(1234*1000*1000*1000, 3)", "1.234TB"); + + assertExpr("decimal_format(1234, 0)", "1K"); + assertExpr("decimal_format(1234*1000,1)", "1.2M"); + assertExpr("decimal_format(1234*1000*1000,2)", "1.23G"); + assertExpr("decimal_format(1234*1000*1000*1000,3)", "1.234T"); + } + + @Test + public void testSizeFormatInvalidArgumentLowerBound() + { + expectedException.expect(IAE.class); + expectedException.expectMessage("precision [-1] must be in the range of [0,4]"); + assertExpr("binary_byte_format(1024, -1)", "1KiB"); + } + + @Test + public void testSizeFormatInvalidArgumentUpperBound() + { + expectedException.expect(IAE.class); + expectedException.expectMessage("precision [4] must be in the range of [0,3]"); + assertExpr("binary_byte_format(1024, 4)", "1KiB"); + } + private void assertExpr(final String expression, @Nullable final Object expectedResult) { final Expr expr = Parser.parse(expression, ExprMacroTable.nil()); diff --git a/docs/misc/math-expr.md b/docs/misc/math-expr.md index 916d9cb827f4..f50d64d8c339 100644 --- a/docs/misc/math-expr.md +++ b/docs/misc/math-expr.md @@ -227,3 +227,13 @@ supported features: * math functions: `abs`, `acos`, `asin`, `atan`, `cbrt`, `ceil`, `cos`, `cosh`, `cot`, `exp`, `expm1`, `floor`, `getExponent`, `log`, `log10`, `log1p`, `nextUp`, `rint`, `signum`, `sin`, `sinh`, `sqrt`, `tan`, `tanh`, `toDegrees`, `toRadians`, `ulp`, `atan2`, `copySign`, `div`, `hypot`, `max`, `min`, `nextAfter`, `pow`, `remainder`, `scalb` are supported for numeric types * time functions: `timestamp_floor` (with constant granularity argument) is supported for numeric types * other: `parse_long` is supported for numeric and string types + + +## Other functions + +| function | description | +| --- | --- | +| binary_byte_format(value[, precision]) | Returns the value in human-readable [IEC](https://en.wikipedia.org/wiki/Binary_prefix) format. Supported unit suffix: `B`, `KiB`, `MiB`, `GiB`, `TiB`, `PiB`, `EiB`. `precision` must be in the range of [0,3] (default: 2). | +| decimal_byte_format(value[, precision]) | Returns the value in human-readable [SI](https://en.wikipedia.org/wiki/Binary_prefix) format. Supported unit suffix: `B`, `KB`, `MB`, `GB`, `TB`, `PB`, `EB`. `precision` must be in the range of [0,3] (default: 2). | +| decimal_format(value[, precision]) | Returns the value in human-readable SI format. Supported unit suffix: `B`, `K`, `M`, `G`, `T`, `P`, `E`. `precision` must be in the range of [0,3] (default: 2). | + diff --git a/docs/querying/sql.md b/docs/querying/sql.md index 3e83115477fa..50cfaf0a0b17 100644 --- a/docs/querying/sql.md +++ b/docs/querying/sql.md @@ -566,6 +566,9 @@ The [DataSketches extension](../development/extensions-core/datasketches-extensi |`COALESCE(value1, value2, ...)`|Returns the first value that is neither NULL nor empty string.| |`NVL(expr,expr-for-null)`|Returns 'expr-for-null' if 'expr' is null (or empty string for string type).| |`BLOOM_FILTER_TEST(, )`|Returns true if the value is contained in a Base64-serialized bloom filter. See the [Bloom filter extension](../development/extensions-core/bloom-filter.md) documentation for additional details.| +|`BINARY_BYTE_FORMAT(value, [precision])`|Returns the value in human-readable [IEC](https://en.wikipedia.org/wiki/Binary_prefix) format. Supported unit suffix: `B`, `KiB`, `MiB`, `GiB`, `TiB`, `PiB`, `EiB`. `precision` must be in the range of [0,3] (default: 2).| +|`DECIMAL_BYTE_FORMAT(value, [precision])`|Returns the value in human-readable [SI](https://en.wikipedia.org/wiki/Binary_prefix) format.Supported unit suffix: `B`, `KB`, `MB`, `GB`, `TB`, `PB`, `EB`. `precision` must be in the range of [0,3] (default: 2).| +|`DECIMAL_FORMAT(value, [precision])`|Returns the value in human-readable SI format. Supported unit suffix: `B`, `K`, `M`, `G`, `T`, `P`, `E`. `precision` must be in the range of [0,3] (default: 2).| ## Multi-value string functions diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/SizeFormatOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/SizeFormatOperatorConversion.java new file mode 100644 index 000000000000..aeb570c4c0c2 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/SizeFormatOperatorConversion.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.expression.builtin; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.sql.calcite.expression.DruidExpression; +import org.apache.druid.sql.calcite.expression.OperatorConversions; +import org.apache.druid.sql.calcite.expression.SqlOperatorConversion; +import org.apache.druid.sql.calcite.planner.PlannerContext; + +public class SizeFormatOperatorConversion implements SqlOperatorConversion +{ + public static final SqlOperatorConversion BINARY_BYTE_FORMAT = new SizeFormatOperatorConversion("binary_byte_format"); + public static final SqlOperatorConversion DECIMAL_BYTE_FORMAT = new SizeFormatOperatorConversion("decimal_byte_format"); + public static final SqlOperatorConversion DECIMAL_FORMAT = new SizeFormatOperatorConversion("decimal_format"); + + private final String name; + private final SqlFunction sqlFunction; + + private SizeFormatOperatorConversion(String name) + { + this.sqlFunction = OperatorConversions + .operatorBuilder(StringUtils.toUpperCase(name)) + .operandTypeChecker(new StringFormatOperandTypeChecker()) + .functionCategory(SqlFunctionCategory.STRING) + .returnTypeNonNull(SqlTypeName.VARCHAR) + .build(); + + this.name = name; + } + + @Override + public SqlOperator calciteOperator() + { + return sqlFunction; + } + + @Override + public DruidExpression toDruidExpression( + final PlannerContext plannerContext, + final RowSignature rowSignature, + final RexNode rexNode + ) + { + return OperatorConversions.convertCall(plannerContext, rowSignature, rexNode, name); + } + + private static class StringFormatOperandTypeChecker implements SqlOperandTypeChecker + { + @Override + public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) + { + final RelDataType firstArgType = callBinding.getOperandType(0); + if (SqlTypeName.NUMERIC_TYPES.contains(firstArgType.getSqlTypeName())) { + return true; + } else { + if (throwOnFailure) { + throw callBinding.newValidationSignatureError(); + } else { + return false; + } + } + } + + @Override + public SqlOperandCountRange getOperandCountRange() + { + return SqlOperandCountRanges.between(1, 2); + } + + @Override + public String getAllowedSignatures(SqlOperator op, String opName) + { + return StringUtils.format("%s(Number, [Precision])", opName); + } + + @Override + public Consistency getConsistency() + { + return Consistency.NONE; + } + + @Override + public boolean isOptional(int i) + { + return i > 0; + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java index 9fbc8dffb53a..6c1563427bd4 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java @@ -91,6 +91,7 @@ import org.apache.druid.sql.calcite.expression.builtin.ReverseOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.RightOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.RoundOperatorConversion; +import org.apache.druid.sql.calcite.expression.builtin.SizeFormatOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.StringFormatOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.StringToMultiValueStringOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.StrposOperatorConversion; @@ -237,6 +238,13 @@ public class DruidOperatorTable implements SqlOperatorTable .add(new IPv4AddressStringifyOperatorConversion()) .build(); + private static final List FORMAT_OPERATOR_CONVERSIONS = + ImmutableList.builder() + .add(SizeFormatOperatorConversion.BINARY_BYTE_FORMAT) + .add(SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT) + .add(SizeFormatOperatorConversion.DECIMAL_FORMAT) + .build(); + private static final List STANDARD_OPERATOR_CONVERSIONS = ImmutableList.builder() .add(new DirectOperatorConversion(SqlStdOperatorTable.ABS, "abs")) @@ -295,6 +303,7 @@ public class DruidOperatorTable implements SqlOperatorTable .addAll(MULTIVALUE_STRING_OPERATOR_CONVERSIONS) .addAll(REDUCTION_OPERATOR_CONVERSIONS) .addAll(IPV4ADDRESS_OPERATOR_CONVERSIONS) + .addAll(FORMAT_OPERATOR_CONVERSIONS) .build(); // Operators that have no conversion, but are handled in the convertlet table, so they still need to exist. diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 09a5d02e78f3..8af774d822db 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -16242,7 +16242,11 @@ public void testTimeStampAddConversion() throws Exception .dataSource(CalciteTests.DATASOURCE1) .intervals(querySegmentSpec(Filtration.eternity())) .virtualColumns( - expressionVirtualColumn("v0", "timestamp_shift(\"__time\",'P1M',1,'UTC')", ValueType.LONG) + expressionVirtualColumn( + "v0", + "timestamp_shift(\"__time\",'P1M',1,'UTC')", + ValueType.LONG + ) ) .columns("v0") .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) @@ -16266,7 +16270,11 @@ public void testTimeStampAddConversion() throws Exception .dataSource(CalciteTests.DATASOURCE1) .intervals(querySegmentSpec(Filtration.eternity())) .virtualColumns( - expressionVirtualColumn("v0", "timestamp_shift(\"__time\",concat('P', (1 * \"cnt\"), 'M'),1,'UTC')", ValueType.LONG) + expressionVirtualColumn( + "v0", + "timestamp_shift(\"__time\",concat('P', (1 * \"cnt\"), 'M'),1,'UTC')", + ValueType.LONG + ) ) .columns("v0") .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) @@ -16281,6 +16289,44 @@ public void testTimeStampAddConversion() throws Exception ); } + public void testSizeFormatFunction() throws Exception + { + testQuery( + "SELECT m1, " + + "BINARY_BYTE_FORMAT(45678)," + + "BINARY_BYTE_FORMAT(m1*12345)," + + "BINARY_BYTE_FORMAT(m1*12345, 0), " + + "DECIMAL_BYTE_FORMAT(m1*12345), " + + "DECIMAL_FORMAT(m1*12345) " + + "FROM numfoo WHERE f1 = 0.1 LIMIT 1", + ImmutableList.of( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .virtualColumns(expressionVirtualColumn("v0", "'44.61KiB'", ValueType.STRING), + expressionVirtualColumn("v1", "binary_byte_format((\"m1\" * 12345))", ValueType.STRING), + expressionVirtualColumn("v2", "binary_byte_format((\"m1\" * 12345),0)", ValueType.STRING), + expressionVirtualColumn("v3", "decimal_byte_format((\"m1\" * 12345))", ValueType.STRING), + expressionVirtualColumn("v4", "decimal_format((\"m1\" * 12345))", ValueType.STRING)) + .columns("m1", "v0", "v1", "v2", "v3", "v4") + .filters(selector("f1", "0.1", null)) + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .limit(1) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{(float) 2.0, + "44.61KiB", // 45678 / 1024 + "24.11KiB", // = m1(2.0) * 12345 / 1024 + "24KiB", // = m1(2.0) * 12345 / 1024, precision = 0 + "24.69KB", // decimal byte format, m1(2.0) * 12345 / 1000, + "24.69K" // decimal format, m1(2.0) * 12345 / 1000, + } + ) + ); + } + /** * This is a provider of query contexts that should be used by join tests. * It tests various configs that can be passed to join queries. All the configs provided by this provider should diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java index 49ed15994f6a..2c90ef924d3d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java @@ -52,6 +52,7 @@ import org.apache.druid.sql.calcite.expression.builtin.ReverseOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.RightOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.RoundOperatorConversion; +import org.apache.druid.sql.calcite.expression.builtin.SizeFormatOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.StringFormatOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.StrposOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.TimeCeilOperatorConversion; @@ -76,6 +77,7 @@ public class ExpressionsTest extends ExpressionTestBase .add("t", ValueType.LONG) .add("a", ValueType.LONG) .add("b", ValueType.LONG) + .add("p", ValueType.LONG) .add("x", ValueType.FLOAT) .add("y", ValueType.LONG) .add("z", ValueType.FLOAT) @@ -98,6 +100,7 @@ public class ExpressionsTest extends ExpressionTestBase .put("t", DateTimes.of("2000-02-03T04:05:06").getMillis()) .put("a", 10) .put("b", 25) + .put("p", 3) .put("x", 2.25) .put("y", 3.0) .put("z", -2.25) @@ -1982,4 +1985,190 @@ public void testAbnormalRepeatWithWrongType() null ); } + + @Test + public void testBinaryByteFormat() + { + /** + * Basic Test + */ + testHelper.testExpression( + SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), + ImmutableList.of( + testHelper.makeLiteral(1000) + ), + DruidExpression.fromExpression("binary_byte_format(1000)"), + "1000.00B" + ); + testHelper.testExpression( + SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), + ImmutableList.of( + testHelper.makeLiteral(1024) + ), + DruidExpression.fromExpression("binary_byte_format(1024)"), + "1.00KiB" + ); + testHelper.testExpression( + SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), + ImmutableList.of( + testHelper.makeLiteral(Long.MAX_VALUE) + ), + DruidExpression.fromExpression("binary_byte_format(9223372036854775807)"), + "8.00EiB" + ); + + /** + * NOTE: Test for Long.MIN_VALUE is skipped since ExprListnerImpl#exitLongExpr fails to parse Long.MIN_VALUE + */ + + /** + * test input with variable reference + */ + testHelper.testExpression( + SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), + ImmutableList.of( + testHelper.makeInputRef("b"), + testHelper.makeInputRef("p") + ), + DruidExpression.fromExpression("binary_byte_format(\"b\",\"p\")"), + "25.000B" + ); + + /** + * test different precision + */ + testHelper.testExpression( + SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), + ImmutableList.of( + testHelper.makeLiteral(45000), + //precision 0 + testHelper.makeLiteral(0) + ), + DruidExpression.fromExpression("binary_byte_format(45000,0)"), + "44KiB" + ); + testHelper.testExpression( + SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), + ImmutableList.of( + testHelper.makeLiteral(45000), + //precision 1 + testHelper.makeLiteral(1) + ), + DruidExpression.fromExpression("binary_byte_format(45000,1)"), + "43.9KiB" + ); + testHelper.testExpression( + SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), + ImmutableList.of( + testHelper.makeLiteral(45000), + //precision 2 + testHelper.makeLiteral(2) + ), + DruidExpression.fromExpression("binary_byte_format(45000,2)"), + "43.95KiB" + ); + testHelper.testExpression( + SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), + ImmutableList.of( + testHelper.makeLiteral(45000), + //precision 3 + testHelper.makeLiteral(3) + ), + DruidExpression.fromExpression("binary_byte_format(45000,3)"), + "43.945KiB" + ); + } + + @Test + public void testDecimalByteFormat() + { + /** + * Basic Test + */ + testHelper.testExpression( + SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), + ImmutableList.of( + testHelper.makeLiteral(999) + ), + DruidExpression.fromExpression("decimal_byte_format(999)"), + "999.00B" + ); + testHelper.testExpression( + SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), + ImmutableList.of( + testHelper.makeLiteral(1024) + ), + DruidExpression.fromExpression("decimal_byte_format(1024)"), + "1.02KB" + ); + testHelper.testExpression( + SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), + ImmutableList.of( + testHelper.makeLiteral(Long.MAX_VALUE) + ), + DruidExpression.fromExpression("decimal_byte_format(9223372036854775807)"), + "9.22EB" + ); + + /** + * NOTE: Test for Long.MIN_VALUE is skipped since ExprListnerImpl#exitLongExpr fails to parse Long.MIN_VALUE + */ + + /** + * test input with variable reference + */ + testHelper.testExpression( + SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), + ImmutableList.of( + testHelper.makeInputRef("b"), + testHelper.makeInputRef("p") + ), + DruidExpression.fromExpression("decimal_byte_format(\"b\",\"p\")"), + "25.000B" + ); + + /** + * test different precision + */ + testHelper.testExpression( + SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), + ImmutableList.of( + testHelper.makeLiteral(45678), + //precision 0 + testHelper.makeLiteral(0) + ), + DruidExpression.fromExpression("decimal_byte_format(45678,0)"), + "46KB" + ); + testHelper.testExpression( + SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), + ImmutableList.of( + testHelper.makeLiteral(45678), + //precision 1 + testHelper.makeLiteral(1) + ), + DruidExpression.fromExpression("decimal_byte_format(45678,1)"), + "45.7KB" + ); + testHelper.testExpression( + SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), + ImmutableList.of( + testHelper.makeLiteral(45678), + //precision 2 + testHelper.makeLiteral(2) + ), + DruidExpression.fromExpression("decimal_byte_format(45678,2)"), + "45.68KB" + ); + testHelper.testExpression( + SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), + ImmutableList.of( + testHelper.makeLiteral(45678), + //precision 3 + testHelper.makeLiteral(3) + ), + DruidExpression.fromExpression("decimal_byte_format(45678,3)"), + "45.678KB" + ); + } } From 1ecbbfc65776d02e32a868300b07741bd95ef4d0 Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 4 Dec 2020 16:26:03 +0800 Subject: [PATCH 02/20] clean code --- .../java/org/apache/druid/math/expr/Function.java | 13 ++++--------- .../org/apache/druid/math/expr/FunctionTest.java | 9 +++++++++ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/apache/druid/math/expr/Function.java b/core/src/main/java/org/apache/druid/math/expr/Function.java index 96378b1f0dc2..cffcc4454042 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Function.java +++ b/core/src/main/java/org/apache/druid/math/expr/Function.java @@ -3291,25 +3291,20 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) precision = args.get(1).eval(bindings).asInt(); } - boolean hasSpace = false; - if (args.size() > 2) { - hasSpace = args.get(2).eval(bindings).asBoolean(); - } - - return ExprEval.of(HumanReadableBytes.format(bytes, precision, this.getUnitSystem(), hasSpace)); + return ExprEval.of(HumanReadableBytes.format(bytes, precision, this.getUnitSystem(), true)); } @Override public void validateArguments(List args) { - if (args.size() < 1 || args.size() > 3) { - throw new IAE("Function[%s] needs 1 or 2 or 3 arguments", name()); + if (args.size() < 1 || args.size() > 2) { + throw new IAE("Function[%s] needs 1 or 2 arguments", name()); } } @Nullable @Override - public ExprType getOutputType(Expr.InputBindingTypes inputTypes, List args) + public ExprType getOutputType(Expr.InputBindingInspector inputTypes, List args) { return ExprType.STRING; } diff --git a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java index 241eb5750653..fdcf730a6b90 100644 --- a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java @@ -583,6 +583,15 @@ public void testSizeFormatInvalidArgumentUpperBound() assertExpr("binary_byte_format(1024, 4)", "1KiB"); } + @Test + public void testSizeFormatInvalidArgumentSize() + { + expectedException.expect(IAE.class); + expectedException.expectMessage("Function[binary_byte_format] needs 1 or 2 arguments"); + final Expr expr = Parser.parse("binary_byte_format(1024, 2, 3)", ExprMacroTable.nil()); + expr.eval(bindings).value(); + } + private void assertExpr(final String expression, @Nullable final Object expectedResult) { final Expr expr = Parser.parse(expression, ExprMacroTable.nil()); From a6afe8521c2c49b2187a802fc4f9675a6d88455f Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 4 Dec 2020 16:45:15 +0800 Subject: [PATCH 03/20] fix doc --- .../org/apache/druid/java/util/common/HumanReadableBytes.java | 3 ++- docs/misc/math-expr.md | 2 +- docs/querying/sql.md | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java b/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java index 49140237e4c5..a04cb82c127a 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java +++ b/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java @@ -225,7 +225,7 @@ public enum UnitSystem /** * simplified SI format without 'B' indicator - * eg: B, K, M, G ... + * eg: K, M, G ... */ DECIMAL } @@ -277,6 +277,7 @@ static String format(long bytes, String pattern, String suffix) * [1KiB, 1MiB) = [2^10, 2^20), * [1MiB, 1GiB) = [2^20, 2^30), * [1GiB, 1PiB) = [2^30, 2^40), + * ... * * So, expression (63 - Long.numberOfLeadingZeros(absValue))) helps us to get the right number of bits of the given input * diff --git a/docs/misc/math-expr.md b/docs/misc/math-expr.md index f50d64d8c339..32f5652085a6 100644 --- a/docs/misc/math-expr.md +++ b/docs/misc/math-expr.md @@ -235,5 +235,5 @@ supported features: | --- | --- | | binary_byte_format(value[, precision]) | Returns the value in human-readable [IEC](https://en.wikipedia.org/wiki/Binary_prefix) format. Supported unit suffix: `B`, `KiB`, `MiB`, `GiB`, `TiB`, `PiB`, `EiB`. `precision` must be in the range of [0,3] (default: 2). | | decimal_byte_format(value[, precision]) | Returns the value in human-readable [SI](https://en.wikipedia.org/wiki/Binary_prefix) format. Supported unit suffix: `B`, `KB`, `MB`, `GB`, `TB`, `PB`, `EB`. `precision` must be in the range of [0,3] (default: 2). | -| decimal_format(value[, precision]) | Returns the value in human-readable SI format. Supported unit suffix: `B`, `K`, `M`, `G`, `T`, `P`, `E`. `precision` must be in the range of [0,3] (default: 2). | +| decimal_format(value[, precision]) | Returns the value in human-readable SI format. Supported unit suffix: `K`, `M`, `G`, `T`, `P`, `E`. `precision` must be in the range of [0,3] (default: 2). | diff --git a/docs/querying/sql.md b/docs/querying/sql.md index 50cfaf0a0b17..24ef2e86123d 100644 --- a/docs/querying/sql.md +++ b/docs/querying/sql.md @@ -568,7 +568,7 @@ The [DataSketches extension](../development/extensions-core/datasketches-extensi |`BLOOM_FILTER_TEST(, )`|Returns true if the value is contained in a Base64-serialized bloom filter. See the [Bloom filter extension](../development/extensions-core/bloom-filter.md) documentation for additional details.| |`BINARY_BYTE_FORMAT(value, [precision])`|Returns the value in human-readable [IEC](https://en.wikipedia.org/wiki/Binary_prefix) format. Supported unit suffix: `B`, `KiB`, `MiB`, `GiB`, `TiB`, `PiB`, `EiB`. `precision` must be in the range of [0,3] (default: 2).| |`DECIMAL_BYTE_FORMAT(value, [precision])`|Returns the value in human-readable [SI](https://en.wikipedia.org/wiki/Binary_prefix) format.Supported unit suffix: `B`, `KB`, `MB`, `GB`, `TB`, `PB`, `EB`. `precision` must be in the range of [0,3] (default: 2).| -|`DECIMAL_FORMAT(value, [precision])`|Returns the value in human-readable SI format. Supported unit suffix: `B`, `K`, `M`, `G`, `T`, `P`, `E`. `precision` must be in the range of [0,3] (default: 2).| +|`DECIMAL_FORMAT(value, [precision])`|Returns the value in human-readable SI format. Supported unit suffix: `K`, `M`, `G`, `T`, `P`, `E`. `precision` must be in the range of [0,3] (default: 2).| ## Multi-value string functions From f5fab7159fff0d21186524d74d32a61689cbef26 Mon Sep 17 00:00:00 2001 From: frank chen Date: Mon, 7 Dec 2020 13:32:43 +0800 Subject: [PATCH 04/20] fix review comments --- .../java/util/common/HumanReadableBytes.java | 23 +++++--- .../org/apache/druid/math/expr/Function.java | 6 ++- .../util/common/HumanReadableBytesTest.java | 6 ++- .../apache/druid/math/expr/FunctionTest.java | 2 +- .../builtin/SizeFormatOperatorConversion.java | 24 +++++---- .../druid/sql/calcite/CalciteQueryTest.java | 53 +++++++++++++++++++ 6 files changed, 96 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java b/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java index a04cb82c127a..d5635ba6f165 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java +++ b/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java @@ -243,14 +243,25 @@ public static String format(long bytes, int precision, UnitSystem unitSystem, bo if (precision < 0 || precision > 3) { throw new IAE("precision [%d] must be in the range of [0,3]", precision); } - String pattern = "%." + precision + (hasSpace ? "f %s%s" : "f%s%s"); + switch (unitSystem) { - case BINARY_BYTE: + case BINARY_BYTE: { + String pattern = "%." + precision + (hasSpace ? "f %s%s" : "f%s%s"); return BinaryFormatter.format(bytes, pattern, "B"); - case DECIMAL_BYTE: + } + case DECIMAL_BYTE: { + String pattern = "%." + precision + (hasSpace ? "f %s%s" : "f%s%s"); return DecimalFormatter.format(bytes, pattern, "B"); - case DECIMAL: - return DecimalFormatter.format(bytes, pattern, ""); + } + case DECIMAL: { + /** + * For the case of a number lower than 1000 and format of UnitSystem.DECIMAL + * there's no unit suffix at the end of result, + * so we have to handle the extra space introduced by 'hasSpace' argument + */ + String pattern = "%." + precision + (hasSpace && (bytes >= 1000 || bytes <= -1000) ? "f %s%s" : "f%s%s"); + return DecimalFormatter.format(bytes, pattern, "").trim(); + } default: throw new IAE("Unkonwn unit system[%s]", unitSystem); } @@ -298,7 +309,7 @@ static String format(long bytes, String pattern, String suffix) * handle number between (-1000, 1000) first to simply further processing */ if (bytes > -1000 && bytes < 1000) { - return StringUtils.format(pattern, (double) bytes, "", suffix); + return StringUtils.format(pattern, (double) bytes, "", suffix).trim(); } /** diff --git a/core/src/main/java/org/apache/druid/math/expr/Function.java b/core/src/main/java/org/apache/druid/math/expr/Function.java index cffcc4454042..7da808042407 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Function.java +++ b/core/src/main/java/org/apache/druid/math/expr/Function.java @@ -3281,6 +3281,10 @@ abstract class SizeFormatFunc implements Function { protected abstract HumanReadableBytes.UnitSystem getUnitSystem(); + /** + * Evaluate given expression + * By default, 'precision' is 2 and 'hasSpace' is false + */ @Override public ExprEval apply(List args, Expr.ObjectBinding bindings) { @@ -3291,7 +3295,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) precision = args.get(1).eval(bindings).asInt(); } - return ExprEval.of(HumanReadableBytes.format(bytes, precision, this.getUnitSystem(), true)); + return ExprEval.of(HumanReadableBytes.format(bytes, precision, this.getUnitSystem(), false)); } @Override diff --git a/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java b/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java index 8e482fc67561..8523f12b5d57 100644 --- a/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java +++ b/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java @@ -455,12 +455,16 @@ public void testFormatInDecimalByte() @Test public void testFormatInDecimal() { - Assert.assertEquals("1.00 ", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); + Assert.assertEquals("1.00", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); + Assert.assertEquals("999.00", HumanReadableBytes.format(999, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); + Assert.assertEquals("-999.00", HumanReadableBytes.format(-999, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); + Assert.assertEquals("-1.00 K", HumanReadableBytes.format(-1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); Assert.assertEquals("1.00 K", HumanReadableBytes.format(1000L, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); Assert.assertEquals("1.00 M", HumanReadableBytes.format(1000L * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); Assert.assertEquals("1.00 G", HumanReadableBytes.format(1000L * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); Assert.assertEquals("1.00 T", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); Assert.assertEquals("1.00 P", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); + Assert.assertEquals("-9.22 E", HumanReadableBytes.format(Long.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); Assert.assertEquals("9.22 E", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); } diff --git a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java index fdcf730a6b90..05745223df95 100644 --- a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java @@ -571,7 +571,7 @@ public void testSizeFormatWithDifferentPrecision() public void testSizeFormatInvalidArgumentLowerBound() { expectedException.expect(IAE.class); - expectedException.expectMessage("precision [-1] must be in the range of [0,4]"); + expectedException.expectMessage("precision [-1] must be in the range of [0,3]"); assertExpr("binary_byte_format(1024, -1)", "1KiB"); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/SizeFormatOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/SizeFormatOperatorConversion.java index aeb570c4c0c2..f4deca31c58a 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/SizeFormatOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/SizeFormatOperatorConversion.java @@ -49,7 +49,7 @@ private SizeFormatOperatorConversion(String name) { this.sqlFunction = OperatorConversions .operatorBuilder(StringUtils.toUpperCase(name)) - .operandTypeChecker(new StringFormatOperandTypeChecker()) + .operandTypeChecker(new SizeFormatOperandTypeChecker()) .functionCategory(SqlFunctionCategory.STRING) .returnTypeNonNull(SqlTypeName.VARCHAR) .build(); @@ -73,21 +73,27 @@ public DruidExpression toDruidExpression( return OperatorConversions.convertCall(plannerContext, rowSignature, rexNode, name); } - private static class StringFormatOperandTypeChecker implements SqlOperandTypeChecker + private static class SizeFormatOperandTypeChecker implements SqlOperandTypeChecker { @Override public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) { + boolean isSigatureError = false; final RelDataType firstArgType = callBinding.getOperandType(0); - if (SqlTypeName.NUMERIC_TYPES.contains(firstArgType.getSqlTypeName())) { - return true; - } else { - if (throwOnFailure) { - throw callBinding.newValidationSignatureError(); - } else { - return false; + if (!SqlTypeName.NUMERIC_TYPES.contains(firstArgType.getSqlTypeName())) { + isSigatureError = true; + } + if (callBinding.getOperandCount() > 1) { + final RelDataType secondArgType = callBinding.getOperandType(1); + if (!SqlTypeName.NUMERIC_TYPES.contains(secondArgType.getSqlTypeName())) { + isSigatureError = true; } } + if (isSigatureError && throwOnFailure) { + throw callBinding.newValidationSignatureError(); + } else { + return isSigatureError; + } } @Override diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 8af774d822db..a087d602e17f 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -16289,6 +16289,7 @@ public void testTimeStampAddConversion() throws Exception ); } + @Test public void testSizeFormatFunction() throws Exception { testQuery( @@ -16303,6 +16304,10 @@ public void testSizeFormatFunction() throws Exception newScanQueryBuilder() .dataSource(CalciteTests.DATASOURCE3) .intervals(querySegmentSpec(Filtration.eternity())) + // + // NOTE: the first expression BINARY_BYTE_FORMAT(45678) in SQL is calculated during SQL parse phase, + // so the converted Druid native query is its result intead of the raw function call + // .virtualColumns(expressionVirtualColumn("v0", "'44.61KiB'", ValueType.STRING), expressionVirtualColumn("v1", "binary_byte_format((\"m1\" * 12345))", ValueType.STRING), expressionVirtualColumn("v2", "binary_byte_format((\"m1\" * 12345),0)", ValueType.STRING), @@ -16327,6 +16332,54 @@ public void testSizeFormatFunction() throws Exception ); } + @Test + public void testSizeFormatFunctionExceptionWithWrongNumberType() throws Exception + { + this.expectedException.expect(ValidationException.class); + this.expectedException.expectMessage("Supported form(s): BINARY_BYTE_FORMAT(Number, [Precision])"); + testQuery( + "SELECT " + + "BINARY_BYTE_FORMAT('45678')", + Collections.emptyList(), + Collections.emptyList() + ); + } + + @Test + public void testSizeFormatFunctionWithWrongPrecisionType() throws Exception + { + this.expectedException.expect(ValidationException.class); + this.expectedException.expectMessage("Supported form(s): BINARY_BYTE_FORMAT(Number, [Precision])"); + testQuery( + "SELECT " + + "BINARY_BYTE_FORMAT(45678, '2')", + Collections.emptyList(), + Collections.emptyList() + ); + } + + @Test + public void testSizeFormatFunctionWithInvalidNumberOfArguments() throws Exception + { + this.expectedException.expect(ValidationException.class); + + /** + * frankly speaking, the exception message thrown here is a little bit confusion + * it says it's expecting 1 arguments but acturally BINARY_BYTE_FORMAT supports 1 or 2 arguments + * + * The message is return from {@link org.apache.calcite.sql.validate.SqlValidatorImpl#handleUnresolvedFunction}, + * and we can see from its implementation that it gets the min number arguments to format the exception message. + * + */ + this.expectedException.expectMessage("Invalid number of arguments to function 'BINARY_BYTE_FORMAT'. Was expecting 1 arguments"); + testQuery( + "SELECT " + + "BINARY_BYTE_FORMAT(45678, 2, 1)", + Collections.emptyList(), + Collections.emptyList() + ); + } + /** * This is a provider of query contexts that should be used by join tests. * It tests various configs that can be passed to join queries. All the configs provided by this provider should From be3c333612e7aad74f4b2e90b0bc25b0b33da6af Mon Sep 17 00:00:00 2001 From: frank chen Date: Mon, 7 Dec 2020 17:10:48 +0800 Subject: [PATCH 05/20] add spelling check rules --- docs/querying/sql.md | 2 +- website/.spelling | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/querying/sql.md b/docs/querying/sql.md index 24ef2e86123d..9974eb1e25c1 100644 --- a/docs/querying/sql.md +++ b/docs/querying/sql.md @@ -567,7 +567,7 @@ The [DataSketches extension](../development/extensions-core/datasketches-extensi |`NVL(expr,expr-for-null)`|Returns 'expr-for-null' if 'expr' is null (or empty string for string type).| |`BLOOM_FILTER_TEST(, )`|Returns true if the value is contained in a Base64-serialized bloom filter. See the [Bloom filter extension](../development/extensions-core/bloom-filter.md) documentation for additional details.| |`BINARY_BYTE_FORMAT(value, [precision])`|Returns the value in human-readable [IEC](https://en.wikipedia.org/wiki/Binary_prefix) format. Supported unit suffix: `B`, `KiB`, `MiB`, `GiB`, `TiB`, `PiB`, `EiB`. `precision` must be in the range of [0,3] (default: 2).| -|`DECIMAL_BYTE_FORMAT(value, [precision])`|Returns the value in human-readable [SI](https://en.wikipedia.org/wiki/Binary_prefix) format.Supported unit suffix: `B`, `KB`, `MB`, `GB`, `TB`, `PB`, `EB`. `precision` must be in the range of [0,3] (default: 2).| +|`DECIMAL_BYTE_FORMAT(value, [precision])`|Returns the value in human-readable [SI](https://en.wikipedia.org/wiki/Binary_prefix) format. Supported unit suffix: `B`, `KB`, `MB`, `GB`, `TB`, `PB`, `EB`. `precision` must be in the range of [0,3] (default: 2).| |`DECIMAL_FORMAT(value, [precision])`|Returns the value in human-readable SI format. Supported unit suffix: `K`, `M`, `G`, `T`, `P`, `E`. `precision` must be in the range of [0,3] (default: 2).| ## Multi-value string functions diff --git a/website/.spelling b/website/.spelling index 1bcd280f0759..d8e3692fc7bc 100644 --- a/website/.spelling +++ b/website/.spelling @@ -1144,6 +1144,10 @@ unix_timestamp value1 value2 valueOf +IEC +binary_byte_format +decimal_byte_format +decimal_format - ../docs/misc/papers-and-talks.md RADStack - ../docs/operations/api-reference.md @@ -1542,6 +1546,7 @@ total_size useApproximateCountDistinct useApproximateTopN wikipedia +IEC - ../docs/querying/timeseriesquery.md fieldName1 fieldName2 From e02809a2a976c32328f2be86bd42533cc0c1469f Mon Sep 17 00:00:00 2001 From: frank chen Date: Mon, 7 Dec 2020 17:39:50 +0800 Subject: [PATCH 06/20] remove extra param --- .../java/util/common/HumanReadableBytes.java | 21 +--- .../org/apache/druid/math/expr/Function.java | 2 +- .../util/common/HumanReadableBytesTest.java | 111 ++++++++---------- 3 files changed, 54 insertions(+), 80 deletions(-) diff --git a/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java b/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java index d5635ba6f165..a29d7d3fd6dd 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java +++ b/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java @@ -236,32 +236,21 @@ public enum UnitSystem * @param bytes input value. Negative value is also allowed * @param precision [0,3] * @param unitSystem which unit system is adopted to format the input value, see {@link UnitSystem} - * @param hasSpace if it's true, there's an extra space between the number and the unit suffix */ - public static String format(long bytes, int precision, UnitSystem unitSystem, boolean hasSpace) + public static String format(long bytes, int precision, UnitSystem unitSystem) { if (precision < 0 || precision > 3) { throw new IAE("precision [%d] must be in the range of [0,3]", precision); } + String pattern = "%." + precision + "f%s%s"; switch (unitSystem) { - case BINARY_BYTE: { - String pattern = "%." + precision + (hasSpace ? "f %s%s" : "f%s%s"); + case BINARY_BYTE: return BinaryFormatter.format(bytes, pattern, "B"); - } - case DECIMAL_BYTE: { - String pattern = "%." + precision + (hasSpace ? "f %s%s" : "f%s%s"); + case DECIMAL_BYTE: return DecimalFormatter.format(bytes, pattern, "B"); - } - case DECIMAL: { - /** - * For the case of a number lower than 1000 and format of UnitSystem.DECIMAL - * there's no unit suffix at the end of result, - * so we have to handle the extra space introduced by 'hasSpace' argument - */ - String pattern = "%." + precision + (hasSpace && (bytes >= 1000 || bytes <= -1000) ? "f %s%s" : "f%s%s"); + case DECIMAL: return DecimalFormatter.format(bytes, pattern, "").trim(); - } default: throw new IAE("Unkonwn unit system[%s]", unitSystem); } diff --git a/core/src/main/java/org/apache/druid/math/expr/Function.java b/core/src/main/java/org/apache/druid/math/expr/Function.java index 7da808042407..144e7c595589 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Function.java +++ b/core/src/main/java/org/apache/druid/math/expr/Function.java @@ -3295,7 +3295,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) precision = args.get(1).eval(bindings).asInt(); } - return ExprEval.of(HumanReadableBytes.format(bytes, precision, this.getUnitSystem(), false)); + return ExprEval.of(HumanReadableBytes.format(bytes, precision, this.getUnitSystem())); } @Override diff --git a/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java b/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java index 8523f12b5d57..58a1718fa64d 100644 --- a/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java +++ b/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java @@ -398,89 +398,74 @@ public void testBytesRange() @Test public void testFormatInBinaryByte() { - Assert.assertEquals("-8.00 EiB", HumanReadableBytes.format(Long.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); - Assert.assertEquals("-8.000 EiB", HumanReadableBytes.format(Long.MIN_VALUE, 3, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("-8.00EiB", HumanReadableBytes.format(Long.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("-8.000EiB", HumanReadableBytes.format(Long.MIN_VALUE, 3, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("-2.00 GiB", HumanReadableBytes.format(Integer.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); - Assert.assertEquals("-32.00 KiB", HumanReadableBytes.format(Short.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); - Assert.assertEquals("-128.00 B", HumanReadableBytes.format(Byte.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); - Assert.assertEquals("-1.00 B", HumanReadableBytes.format(-1, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); - Assert.assertEquals("0.00 B", HumanReadableBytes.format(0, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); - Assert.assertEquals("1.00 B", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); - Assert.assertEquals("1.00 KiB", HumanReadableBytes.format(1024L, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); - Assert.assertEquals("1.00 MiB", HumanReadableBytes.format(1024L * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); - Assert.assertEquals("1.00 GiB", HumanReadableBytes.format(1024L * 1024 * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); - Assert.assertEquals("1.00 TiB", HumanReadableBytes.format(1024L * 1024 * 1024 * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); - Assert.assertEquals("1.00 PiB", HumanReadableBytes.format(1024L * 1024 * 1024 * 1024 * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); - Assert.assertEquals("8.00 EiB", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("-2.00GiB", HumanReadableBytes.format(Integer.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("-32.00KiB", HumanReadableBytes.format(Short.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("-128.00B", HumanReadableBytes.format(Byte.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("-1.00B", HumanReadableBytes.format(-1, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("0.00B", HumanReadableBytes.format(0, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.00B", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.00KiB", HumanReadableBytes.format(1024L, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.00MiB", HumanReadableBytes.format(1024L * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.00GiB", HumanReadableBytes.format(1024L * 1024 * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.00TiB", HumanReadableBytes.format(1024L * 1024 * 1024 * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.00PiB", HumanReadableBytes.format(1024L * 1024 * 1024 * 1024 * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("8.00EiB", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); } @Test public void testPrecisionInBinaryFormat() { - Assert.assertEquals("1 KiB", HumanReadableBytes.format(1500, 0, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); - Assert.assertEquals("1.5 KiB", HumanReadableBytes.format(1500, 1, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); - Assert.assertEquals("1.46 KiB", HumanReadableBytes.format(1500, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); - Assert.assertEquals("1.465 KiB", HumanReadableBytes.format(1500, 3, HumanReadableBytes.UnitSystem.BINARY_BYTE, true)); + Assert.assertEquals("1KiB", HumanReadableBytes.format(1500, 0, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.5KiB", HumanReadableBytes.format(1500, 1, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.46KiB", HumanReadableBytes.format(1500, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.465KiB", HumanReadableBytes.format(1500, 3, HumanReadableBytes.UnitSystem.BINARY_BYTE)); } @Test public void testPrecisionInDecimalFormat() { - Assert.assertEquals("1 KB", HumanReadableBytes.format(1456, 0, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); - Assert.assertEquals("1.5 KB", HumanReadableBytes.format(1456, 1, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); - Assert.assertEquals("1.46 KB", HumanReadableBytes.format(1456, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); - Assert.assertEquals("1.456 KB", HumanReadableBytes.format(1456, 3, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + Assert.assertEquals("1KB", HumanReadableBytes.format(1456, 0, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1.5KB", HumanReadableBytes.format(1456, 1, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1.46KB", HumanReadableBytes.format(1456, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1.456KB", HumanReadableBytes.format(1456, 3, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); } @Test public void testFormatInDecimalByte() { - Assert.assertEquals("1.00 B", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); - Assert.assertEquals("1.00 KB", HumanReadableBytes.format(1000L, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); - Assert.assertEquals("1.00 MB", HumanReadableBytes.format(1000L * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); - Assert.assertEquals("1.00 GB", HumanReadableBytes.format(1000L * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); - Assert.assertEquals("1.00 TB", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); - Assert.assertEquals("1.00 PB", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); - Assert.assertEquals("9.22 EB", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + Assert.assertEquals("1.00B", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1.00KB", HumanReadableBytes.format(1000L, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1.00MB", HumanReadableBytes.format(1000L * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1.00GB", HumanReadableBytes.format(1000L * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1.00TB", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1.00PB", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("9.22EB", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); - Assert.assertEquals("100.00 KB", HumanReadableBytes.format(99999, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); - Assert.assertEquals("99.999 KB", HumanReadableBytes.format(99999, 3, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + Assert.assertEquals("100.00KB", HumanReadableBytes.format(99999, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("99.999KB", HumanReadableBytes.format(99999, 3, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); - Assert.assertEquals("999.9 PB", HumanReadableBytes.format(999_949_999_999_999_999L, 1, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); - Assert.assertEquals("999.95 PB", HumanReadableBytes.format(999_949_999_999_999_999L, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); - Assert.assertEquals("999.949 PB", HumanReadableBytes.format(999_949_999_999_999_999L, 3, HumanReadableBytes.UnitSystem.DECIMAL_BYTE, true)); + Assert.assertEquals("999.9PB", HumanReadableBytes.format(999_949_999_999_999_999L, 1, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("999.95PB", HumanReadableBytes.format(999_949_999_999_999_999L, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("999.949PB", HumanReadableBytes.format(999_949_999_999_999_999L, 3, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); } @Test public void testFormatInDecimal() { - Assert.assertEquals("1.00", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); - Assert.assertEquals("999.00", HumanReadableBytes.format(999, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); - Assert.assertEquals("-999.00", HumanReadableBytes.format(-999, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); - Assert.assertEquals("-1.00 K", HumanReadableBytes.format(-1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); - Assert.assertEquals("1.00 K", HumanReadableBytes.format(1000L, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); - Assert.assertEquals("1.00 M", HumanReadableBytes.format(1000L * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); - Assert.assertEquals("1.00 G", HumanReadableBytes.format(1000L * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); - Assert.assertEquals("1.00 T", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); - Assert.assertEquals("1.00 P", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); - Assert.assertEquals("-9.22 E", HumanReadableBytes.format(Long.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); - Assert.assertEquals("9.22 E", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL, true)); - } - - @Test - public void testFormatInDecimalNoSpace() - { - Assert.assertEquals("1.00", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.DECIMAL, false)); - Assert.assertEquals("1.00K", HumanReadableBytes.format(1000L, 2, HumanReadableBytes.UnitSystem.DECIMAL, false)); - Assert.assertEquals("1.00M", HumanReadableBytes.format(1000L * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, false)); - Assert.assertEquals("1.00G", HumanReadableBytes.format(1000L * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, false)); - Assert.assertEquals("1.00T", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, false)); - Assert.assertEquals("1.00P", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL, false)); - Assert.assertEquals("9.22E", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL, false)); - - Assert.assertEquals("1P", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000 * 1000, 0, HumanReadableBytes.UnitSystem.DECIMAL, false)); - Assert.assertEquals("9E", HumanReadableBytes.format(Long.MAX_VALUE, 0, HumanReadableBytes.UnitSystem.DECIMAL, false)); + Assert.assertEquals("1.00", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("999.00", HumanReadableBytes.format(999, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("-999.00", HumanReadableBytes.format(-999, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("-1.00K", HumanReadableBytes.format(-1000, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("1.00K", HumanReadableBytes.format(1000L, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("1.00M", HumanReadableBytes.format(1000L * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("1.00G", HumanReadableBytes.format(1000L * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("1.00T", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("1.00P", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("-9.22E", HumanReadableBytes.format(Long.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("9.22E", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL)); } @Test @@ -488,7 +473,7 @@ public void testInvalidPrecisionArgumentLowerBound() { expectedException.expect(IAE.class); expectedException.expectMessage("precision [-1] must be in the range of [0,3]"); - Assert.assertEquals("1.00", HumanReadableBytes.format(1, -1, HumanReadableBytes.UnitSystem.DECIMAL, false)); + Assert.assertEquals("1.00", HumanReadableBytes.format(1, -1, HumanReadableBytes.UnitSystem.DECIMAL)); } @Test @@ -496,8 +481,8 @@ public void testInvalidPrecisionArgumentUpperBound() { expectedException.expect(IAE.class); expectedException.expectMessage("precision [4] must be in the range of [0,3]"); - Assert.assertEquals("1.000", HumanReadableBytes.format(1, 3, HumanReadableBytes.UnitSystem.DECIMAL, false)); - Assert.assertEquals("1.00", HumanReadableBytes.format(1, 4, HumanReadableBytes.UnitSystem.DECIMAL, false)); + Assert.assertEquals("1.000", HumanReadableBytes.format(1, 3, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("1.00", HumanReadableBytes.format(1, 4, HumanReadableBytes.UnitSystem.DECIMAL)); } private static String validate(T obj) From b5a3756633260a0ec55721f47f518abcaaf727f5 Mon Sep 17 00:00:00 2001 From: frank chen Date: Wed, 9 Dec 2020 18:20:39 +0800 Subject: [PATCH 07/20] improve type handling and null handling --- .../druid/common/config/NullHandling.java | 9 ++ .../java/util/common/HumanReadableBytes.java | 4 +- .../org/apache/druid/math/expr/Function.java | 33 +++- .../apache/druid/math/expr/FunctionTest.java | 150 ++++++++++++++++-- .../druid/sql/calcite/CalciteQueryTest.java | 4 +- 5 files changed, 178 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/org/apache/druid/common/config/NullHandling.java b/core/src/main/java/org/apache/druid/common/config/NullHandling.java index 51cb2658628b..152ad7dd6a0a 100644 --- a/core/src/main/java/org/apache/druid/common/config/NullHandling.java +++ b/core/src/main/java/org/apache/druid/common/config/NullHandling.java @@ -61,6 +61,15 @@ public static void initializeForTests() INSTANCE = new NullValueHandlingConfig(null); } + /** + * This function should only be called by unit tests + */ + @VisibleForTesting + public static void updateForTests(boolean useDefaultValuesForNull) + { + INSTANCE = new NullValueHandlingConfig(useDefaultValuesForNull); + } + /** * whether nulls should be replaced with default value. */ diff --git a/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java b/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java index a29d7d3fd6dd..1c22b13ba504 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java +++ b/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java @@ -237,7 +237,7 @@ public enum UnitSystem * @param precision [0,3] * @param unitSystem which unit system is adopted to format the input value, see {@link UnitSystem} */ - public static String format(long bytes, int precision, UnitSystem unitSystem) + public static String format(long bytes, long precision, UnitSystem unitSystem) { if (precision < 0 || precision > 3) { throw new IAE("precision [%d] must be in the range of [0,3]", precision); @@ -298,7 +298,7 @@ static String format(long bytes, String pattern, String suffix) * handle number between (-1000, 1000) first to simply further processing */ if (bytes > -1000 && bytes < 1000) { - return StringUtils.format(pattern, (double) bytes, "", suffix).trim(); + return StringUtils.format(pattern, (double) bytes, "", suffix); } /** diff --git a/core/src/main/java/org/apache/druid/math/expr/Function.java b/core/src/main/java/org/apache/druid/math/expr/Function.java index 144e7c595589..d0441bc88204 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Function.java +++ b/core/src/main/java/org/apache/druid/math/expr/Function.java @@ -3281,21 +3281,38 @@ abstract class SizeFormatFunc implements Function { protected abstract HumanReadableBytes.UnitSystem getUnitSystem(); - /** - * Evaluate given expression - * By default, 'precision' is 2 and 'hasSpace' is false - */ @Override public ExprEval apply(List args, Expr.ObjectBinding bindings) { - final long bytes = args.get(0).eval(bindings).asLong(); + final ExprEval valueParam = args.get(0).eval(bindings); + if (NullHandling.sqlCompatible() && valueParam.isNumericNull()) { + return ExprEval.of(null); + } - int precision = 2; + /** + * only LONG and DOUBLE are allowed + * For a DOUBLE, it will be cast to LONG before format + */ + if (valueParam.value() != null && valueParam.type() != ExprType.LONG && valueParam.type() != ExprType.DOUBLE) { + throw new IAE("Function[%s] needs a number as its first argument", name()); + } + + /** + * By default, precision is 2 + */ + long precision = 2; if (args.size() > 1) { - precision = args.get(1).eval(bindings).asInt(); + ExprEval precisionParam = args.get(1).eval(bindings); + if (precisionParam.type() != ExprType.LONG) { + throw new IAE("Function[%s] needs an integer as its second argument", name()); + } + precision = precisionParam.asLong(); + if (precision < 0 || precision > 3) { + throw new IAE("Given precision[%d] of Function[%s] must be in the range of [0,3]", precision, name()); + } } - return ExprEval.of(HumanReadableBytes.format(bytes, precision, this.getUnitSystem())); + return ExprEval.of(HumanReadableBytes.format(valueParam.asLong(), precision, this.getUnitSystem())); } @Override diff --git a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java index 05745223df95..000cb76ef828 100644 --- a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java @@ -568,19 +568,123 @@ public void testSizeFormatWithDifferentPrecision() } @Test - public void testSizeFormatInvalidArgumentLowerBound() + public void testSizeFormatWithEdgeCases() { - expectedException.expect(IAE.class); - expectedException.expectMessage("precision [-1] must be in the range of [0,3]"); - assertExpr("binary_byte_format(1024, -1)", "1KiB"); + //a nonexist value is null which is treated as 0 + assertExpr("binary_byte_format(nonexist)", "0.00B"); + + //f = 12.34 + assertExpr("binary_byte_format(f)", "12.00B"); + + //nan is Double.NaN + assertExpr("binary_byte_format(nan)", "0.00B"); + + //inf = Double.POSITIVE_INFINITY + assertExpr("binary_byte_format(inf)", "8.00EiB"); + + //inf = Double.NEGATIVE_INFINITY + assertExpr("binary_byte_format(-inf)", "-8.00EiB"); + + // o = 0 + assertExpr("binary_byte_format(o)", "0.00B"); + + // od = 0D + assertExpr("binary_byte_format(od)", "0.00B"); + + // of = 0F + assertExpr("binary_byte_format(of)", "0.00B"); } @Test - public void testSizeFormatInvalidArgumentUpperBound() + public void testSizeForatInvalidArgumentType() { - expectedException.expect(IAE.class); - expectedException.expectMessage("precision [4] must be in the range of [0,3]"); - assertExpr("binary_byte_format(1024, 4)", "1KiB"); + try { + //x = "foo" + Parser.parse("binary_byte_format(x)", ExprMacroTable.nil()) + .eval(bindings); + + //must not go to here + Assert.assertTrue(false); + } + catch (IAE e) { + Assert.assertEquals("Function[binary_byte_format] needs a number as its first argument", e.getMessage()); + } + + try { + //x = "foo" + Parser.parse("binary_byte_format(1024, x)", ExprMacroTable.nil()) + .eval(bindings); + + //must not go to here + Assert.assertTrue(false); + } + catch (IAE e) { + Assert.assertEquals("Function[binary_byte_format] needs an integer as its second argument", e.getMessage()); + } + + try { + //of = 0F + Parser.parse("binary_byte_format(1024, of)", ExprMacroTable.nil()) + .eval(bindings); + + //must not go to here + Assert.assertTrue(false); + } + catch (IAE e) { + Assert.assertEquals("Function[binary_byte_format] needs an integer as its second argument", e.getMessage()); + } + + try { + //of = 0F + Parser.parse("binary_byte_format(1024, nonexist)", ExprMacroTable.nil()) + .eval(bindings); + + //must not go to here + Assert.assertTrue(false); + } + catch (IAE e) { + Assert.assertEquals("Function[binary_byte_format] needs an integer as its second argument", e.getMessage()); + } + } + + @Test + public void testSizeFormatInvalidPrecision() + { + try { + Parser.parse("binary_byte_format(1024, maxLong)", ExprMacroTable.nil()) + .eval(bindings); + Assert.assertTrue(false); + } + catch (IAE e) { + Assert.assertEquals("Given precision[9223372036854775807] of Function[binary_byte_format] must be in the range of [0,3]", e.getMessage()); + } + + try { + Parser.parse("binary_byte_format(1024, minLong)", ExprMacroTable.nil()) + .eval(bindings); + Assert.assertTrue(false); + } + catch (IAE e) { + Assert.assertEquals("Given precision[-9223372036854775808] of Function[binary_byte_format] must be in the range of [0,3]", e.getMessage()); + } + + try { + Parser.parse("binary_byte_format(1024, -1)", ExprMacroTable.nil()) + .eval(bindings); + Assert.assertTrue(false); + } + catch (IAE e) { + Assert.assertEquals("Given precision[-1] of Function[binary_byte_format] must be in the range of [0,3]", e.getMessage()); + } + + try { + Parser.parse("binary_byte_format(1024, 4)", ExprMacroTable.nil()) + .eval(bindings); + Assert.assertTrue(false); + } + catch (IAE e) { + Assert.assertEquals("Given precision[4] of Function[binary_byte_format] must be in the range of [0,3]", e.getMessage()); + } } @Test @@ -588,10 +692,36 @@ public void testSizeFormatInvalidArgumentSize() { expectedException.expect(IAE.class); expectedException.expectMessage("Function[binary_byte_format] needs 1 or 2 arguments"); - final Expr expr = Parser.parse("binary_byte_format(1024, 2, 3)", ExprMacroTable.nil()); - expr.eval(bindings).value(); + Parser.parse("binary_byte_format(1024, 2, 3)", ExprMacroTable.nil()) + .eval(bindings); } + @Test + public void testSizeFormatWithNoDefaultValueForNull() + { + NullHandling.updateForTests(false); + { + // + // normal cases + // y is not null, the function returns correctly + // + assertExpr("binary_byte_format(y)", "2.00B"); + assertExpr("decimal_byte_format(y)", "2.00B"); + assertExpr("decimal_format(y)", "2.00"); + + // + // since 'druid.generic.useDefaultValueForNull' has been disabled above, + // the 'nonexist' below returns null, and the function calls also return null + // + assertExpr("binary_byte_format(nonexist)", null); + assertExpr("decimal_byte_format(nonexist)", null); + assertExpr("decimal_format(nonexist)", null); + + } + NullHandling.updateForTests(true); + } + + private void assertExpr(final String expression, @Nullable final Object expectedResult) { final Expr expr = Parser.parse(expression, ExprMacroTable.nil()); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index a087d602e17f..636e5a8089b1 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -16365,9 +16365,9 @@ public void testSizeFormatFunctionWithInvalidNumberOfArguments() throws Exceptio /** * frankly speaking, the exception message thrown here is a little bit confusion - * it says it's expecting 1 arguments but acturally BINARY_BYTE_FORMAT supports 1 or 2 arguments + * it says it's 'expecting 1 arguments' but acturally BINARY_BYTE_FORMAT supports 1 or 2 arguments * - * The message is return from {@link org.apache.calcite.sql.validate.SqlValidatorImpl#handleUnresolvedFunction}, + * The message is returned from {@link org.apache.calcite.sql.validate.SqlValidatorImpl#handleUnresolvedFunction}, * and we can see from its implementation that it gets the min number arguments to format the exception message. * */ From e279c18a05c6b93246ab00a46f2d743225cad3ca Mon Sep 17 00:00:00 2001 From: frank chen Date: Wed, 9 Dec 2020 18:52:59 +0800 Subject: [PATCH 08/20] remove extra zeros --- .../java/util/common/HumanReadableBytes.java | 6 ++++- .../util/common/HumanReadableBytesTest.java | 22 ++++++++++--------- .../apache/druid/math/expr/FunctionTest.java | 18 +++++++-------- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java b/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java index 1c22b13ba504..a6c2422f5e6e 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java +++ b/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java @@ -262,6 +262,10 @@ static class BinaryFormatter static String format(long bytes, String pattern, String suffix) { + if (bytes > -1024 && bytes < 1024) { + return bytes + suffix; + } + if (bytes == Long.MIN_VALUE) { /** * special path for Long.MIN_VALUE @@ -298,7 +302,7 @@ static String format(long bytes, String pattern, String suffix) * handle number between (-1000, 1000) first to simply further processing */ if (bytes > -1000 && bytes < 1000) { - return StringUtils.format(pattern, (double) bytes, "", suffix); + return bytes + suffix; } /** diff --git a/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java b/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java index 58a1718fa64d..f5a896b31f19 100644 --- a/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java +++ b/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java @@ -403,10 +403,12 @@ public void testFormatInBinaryByte() Assert.assertEquals("-2.00GiB", HumanReadableBytes.format(Integer.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); Assert.assertEquals("-32.00KiB", HumanReadableBytes.format(Short.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("-128.00B", HumanReadableBytes.format(Byte.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("-1.00B", HumanReadableBytes.format(-1, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("0.00B", HumanReadableBytes.format(0, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("1.00B", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + + Assert.assertEquals("-128B", HumanReadableBytes.format(Byte.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("-1B", HumanReadableBytes.format(-1, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("0B", HumanReadableBytes.format(0, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1B", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.00KiB", HumanReadableBytes.format(1024L, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); Assert.assertEquals("1.00MiB", HumanReadableBytes.format(1024L * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); Assert.assertEquals("1.00GiB", HumanReadableBytes.format(1024L * 1024 * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); @@ -436,7 +438,7 @@ public void testPrecisionInDecimalFormat() @Test public void testFormatInDecimalByte() { - Assert.assertEquals("1.00B", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1B", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); Assert.assertEquals("1.00KB", HumanReadableBytes.format(1000L, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); Assert.assertEquals("1.00MB", HumanReadableBytes.format(1000L * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); Assert.assertEquals("1.00GB", HumanReadableBytes.format(1000L * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); @@ -455,9 +457,9 @@ public void testFormatInDecimalByte() @Test public void testFormatInDecimal() { - Assert.assertEquals("1.00", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.DECIMAL)); - Assert.assertEquals("999.00", HumanReadableBytes.format(999, 2, HumanReadableBytes.UnitSystem.DECIMAL)); - Assert.assertEquals("-999.00", HumanReadableBytes.format(-999, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("1", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("999", HumanReadableBytes.format(999, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("-999", HumanReadableBytes.format(-999, 2, HumanReadableBytes.UnitSystem.DECIMAL)); Assert.assertEquals("-1.00K", HumanReadableBytes.format(-1000, 2, HumanReadableBytes.UnitSystem.DECIMAL)); Assert.assertEquals("1.00K", HumanReadableBytes.format(1000L, 2, HumanReadableBytes.UnitSystem.DECIMAL)); Assert.assertEquals("1.00M", HumanReadableBytes.format(1000L * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL)); @@ -481,8 +483,8 @@ public void testInvalidPrecisionArgumentUpperBound() { expectedException.expect(IAE.class); expectedException.expectMessage("precision [4] must be in the range of [0,3]"); - Assert.assertEquals("1.000", HumanReadableBytes.format(1, 3, HumanReadableBytes.UnitSystem.DECIMAL)); - Assert.assertEquals("1.00", HumanReadableBytes.format(1, 4, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("1", HumanReadableBytes.format(1, 3, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("1", HumanReadableBytes.format(1, 4, HumanReadableBytes.UnitSystem.DECIMAL)); } private static String validate(T obj) diff --git a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java index 000cb76ef828..684c423f660c 100644 --- a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java @@ -571,13 +571,13 @@ public void testSizeFormatWithDifferentPrecision() public void testSizeFormatWithEdgeCases() { //a nonexist value is null which is treated as 0 - assertExpr("binary_byte_format(nonexist)", "0.00B"); + assertExpr("binary_byte_format(nonexist)", "0B"); //f = 12.34 - assertExpr("binary_byte_format(f)", "12.00B"); + assertExpr("binary_byte_format(f)", "12B"); //nan is Double.NaN - assertExpr("binary_byte_format(nan)", "0.00B"); + assertExpr("binary_byte_format(nan)", "0B"); //inf = Double.POSITIVE_INFINITY assertExpr("binary_byte_format(inf)", "8.00EiB"); @@ -586,13 +586,13 @@ public void testSizeFormatWithEdgeCases() assertExpr("binary_byte_format(-inf)", "-8.00EiB"); // o = 0 - assertExpr("binary_byte_format(o)", "0.00B"); + assertExpr("binary_byte_format(o)", "0B"); // od = 0D - assertExpr("binary_byte_format(od)", "0.00B"); + assertExpr("binary_byte_format(od)", "0B"); // of = 0F - assertExpr("binary_byte_format(of)", "0.00B"); + assertExpr("binary_byte_format(of)", "0B"); } @Test @@ -705,9 +705,9 @@ public void testSizeFormatWithNoDefaultValueForNull() // normal cases // y is not null, the function returns correctly // - assertExpr("binary_byte_format(y)", "2.00B"); - assertExpr("decimal_byte_format(y)", "2.00B"); - assertExpr("decimal_format(y)", "2.00"); + assertExpr("binary_byte_format(y)", "2B"); + assertExpr("decimal_byte_format(y)", "2B"); + assertExpr("decimal_format(y)", "2"); // // since 'druid.generic.useDefaultValueForNull' has been disabled above, From 91e6a550ff8dbacae2d6da91262784df0a52d6c5 Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 11 Dec 2020 09:37:39 +0800 Subject: [PATCH 09/20] fix tests and add space between unit suffix and number as most size-format functions do --- .../java/util/common/HumanReadableBytes.java | 6 +- .../util/common/HumanReadableBytesTest.java | 84 +++++++++---------- .../apache/druid/math/expr/FunctionTest.java | 76 ++++++++--------- .../druid/sql/calcite/CalciteQueryTest.java | 12 +-- .../calcite/expression/ExpressionsTest.java | 21 ++--- 5 files changed, 100 insertions(+), 99 deletions(-) diff --git a/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java b/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java index a6c2422f5e6e..b63e0f207c7e 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java +++ b/core/src/main/java/org/apache/druid/java/util/common/HumanReadableBytes.java @@ -243,7 +243,7 @@ public static String format(long bytes, long precision, UnitSystem unitSystem) throw new IAE("precision [%d] must be in the range of [0,3]", precision); } - String pattern = "%." + precision + "f%s%s"; + String pattern = "%." + precision + "f %s%s"; switch (unitSystem) { case BINARY_BYTE: return BinaryFormatter.format(bytes, pattern, "B"); @@ -263,7 +263,7 @@ static class BinaryFormatter static String format(long bytes, String pattern, String suffix) { if (bytes > -1024 && bytes < 1024) { - return bytes + suffix; + return bytes + " " + suffix; } if (bytes == Long.MIN_VALUE) { @@ -302,7 +302,7 @@ static String format(long bytes, String pattern, String suffix) * handle number between (-1000, 1000) first to simply further processing */ if (bytes > -1000 && bytes < 1000) { - return bytes + suffix; + return bytes + " " + suffix; } /** diff --git a/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java b/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java index f5a896b31f19..a49533d086a7 100644 --- a/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java +++ b/core/src/test/java/org/apache/druid/java/util/common/HumanReadableBytesTest.java @@ -398,60 +398,60 @@ public void testBytesRange() @Test public void testFormatInBinaryByte() { - Assert.assertEquals("-8.00EiB", HumanReadableBytes.format(Long.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("-8.000EiB", HumanReadableBytes.format(Long.MIN_VALUE, 3, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("-8.00 EiB", HumanReadableBytes.format(Long.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("-8.000 EiB", HumanReadableBytes.format(Long.MIN_VALUE, 3, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("-2.00GiB", HumanReadableBytes.format(Integer.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("-32.00KiB", HumanReadableBytes.format(Short.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("-2.00 GiB", HumanReadableBytes.format(Integer.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("-32.00 KiB", HumanReadableBytes.format(Short.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("-128B", HumanReadableBytes.format(Byte.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("-1B", HumanReadableBytes.format(-1, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("0B", HumanReadableBytes.format(0, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("1B", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("-128 B", HumanReadableBytes.format(Byte.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("-1 B", HumanReadableBytes.format(-1, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("0 B", HumanReadableBytes.format(0, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1 B", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("1.00KiB", HumanReadableBytes.format(1024L, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("1.00MiB", HumanReadableBytes.format(1024L * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("1.00GiB", HumanReadableBytes.format(1024L * 1024 * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("1.00TiB", HumanReadableBytes.format(1024L * 1024 * 1024 * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("1.00PiB", HumanReadableBytes.format(1024L * 1024 * 1024 * 1024 * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("8.00EiB", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.00 KiB", HumanReadableBytes.format(1024L, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.00 MiB", HumanReadableBytes.format(1024L * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.00 GiB", HumanReadableBytes.format(1024L * 1024 * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.00 TiB", HumanReadableBytes.format(1024L * 1024 * 1024 * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.00 PiB", HumanReadableBytes.format(1024L * 1024 * 1024 * 1024 * 1024, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("8.00 EiB", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); } @Test public void testPrecisionInBinaryFormat() { - Assert.assertEquals("1KiB", HumanReadableBytes.format(1500, 0, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("1.5KiB", HumanReadableBytes.format(1500, 1, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("1.46KiB", HumanReadableBytes.format(1500, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); - Assert.assertEquals("1.465KiB", HumanReadableBytes.format(1500, 3, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1 KiB", HumanReadableBytes.format(1500, 0, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.5 KiB", HumanReadableBytes.format(1500, 1, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.46 KiB", HumanReadableBytes.format(1500, 2, HumanReadableBytes.UnitSystem.BINARY_BYTE)); + Assert.assertEquals("1.465 KiB", HumanReadableBytes.format(1500, 3, HumanReadableBytes.UnitSystem.BINARY_BYTE)); } @Test public void testPrecisionInDecimalFormat() { - Assert.assertEquals("1KB", HumanReadableBytes.format(1456, 0, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); - Assert.assertEquals("1.5KB", HumanReadableBytes.format(1456, 1, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); - Assert.assertEquals("1.46KB", HumanReadableBytes.format(1456, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); - Assert.assertEquals("1.456KB", HumanReadableBytes.format(1456, 3, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1 KB", HumanReadableBytes.format(1456, 0, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1.5 KB", HumanReadableBytes.format(1456, 1, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1.46 KB", HumanReadableBytes.format(1456, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1.456 KB", HumanReadableBytes.format(1456, 3, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); } @Test public void testFormatInDecimalByte() { - Assert.assertEquals("1B", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); - Assert.assertEquals("1.00KB", HumanReadableBytes.format(1000L, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); - Assert.assertEquals("1.00MB", HumanReadableBytes.format(1000L * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); - Assert.assertEquals("1.00GB", HumanReadableBytes.format(1000L * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); - Assert.assertEquals("1.00TB", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); - Assert.assertEquals("1.00PB", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); - Assert.assertEquals("9.22EB", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1 B", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1.00 KB", HumanReadableBytes.format(1000L, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1.00 MB", HumanReadableBytes.format(1000L * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1.00 GB", HumanReadableBytes.format(1000L * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1.00 TB", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("1.00 PB", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("9.22 EB", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); - Assert.assertEquals("100.00KB", HumanReadableBytes.format(99999, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); - Assert.assertEquals("99.999KB", HumanReadableBytes.format(99999, 3, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("100.00 KB", HumanReadableBytes.format(99999, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("99.999 KB", HumanReadableBytes.format(99999, 3, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); - Assert.assertEquals("999.9PB", HumanReadableBytes.format(999_949_999_999_999_999L, 1, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); - Assert.assertEquals("999.95PB", HumanReadableBytes.format(999_949_999_999_999_999L, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); - Assert.assertEquals("999.949PB", HumanReadableBytes.format(999_949_999_999_999_999L, 3, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("999.9 PB", HumanReadableBytes.format(999_949_999_999_999_999L, 1, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("999.95 PB", HumanReadableBytes.format(999_949_999_999_999_999L, 2, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); + Assert.assertEquals("999.949 PB", HumanReadableBytes.format(999_949_999_999_999_999L, 3, HumanReadableBytes.UnitSystem.DECIMAL_BYTE)); } @Test @@ -460,14 +460,14 @@ public void testFormatInDecimal() Assert.assertEquals("1", HumanReadableBytes.format(1, 2, HumanReadableBytes.UnitSystem.DECIMAL)); Assert.assertEquals("999", HumanReadableBytes.format(999, 2, HumanReadableBytes.UnitSystem.DECIMAL)); Assert.assertEquals("-999", HumanReadableBytes.format(-999, 2, HumanReadableBytes.UnitSystem.DECIMAL)); - Assert.assertEquals("-1.00K", HumanReadableBytes.format(-1000, 2, HumanReadableBytes.UnitSystem.DECIMAL)); - Assert.assertEquals("1.00K", HumanReadableBytes.format(1000L, 2, HumanReadableBytes.UnitSystem.DECIMAL)); - Assert.assertEquals("1.00M", HumanReadableBytes.format(1000L * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL)); - Assert.assertEquals("1.00G", HumanReadableBytes.format(1000L * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL)); - Assert.assertEquals("1.00T", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL)); - Assert.assertEquals("1.00P", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL)); - Assert.assertEquals("-9.22E", HumanReadableBytes.format(Long.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL)); - Assert.assertEquals("9.22E", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("-1.00 K", HumanReadableBytes.format(-1000, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("1.00 K", HumanReadableBytes.format(1000L, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("1.00 M", HumanReadableBytes.format(1000L * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("1.00 G", HumanReadableBytes.format(1000L * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("1.00 T", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("1.00 P", HumanReadableBytes.format(1000L * 1000 * 1000 * 1000 * 1000, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("-9.22 E", HumanReadableBytes.format(Long.MIN_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL)); + Assert.assertEquals("9.22 E", HumanReadableBytes.format(Long.MAX_VALUE, 2, HumanReadableBytes.UnitSystem.DECIMAL)); } @Test diff --git a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java index 684c423f660c..eed83c6e25c5 100644 --- a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java @@ -528,71 +528,71 @@ public void testLeast() @Test public void testSizeFormat() { - assertExpr("binary_byte_format(-1024)", "-1.00KiB"); - assertExpr("binary_byte_format(1024)", "1.00KiB"); - assertExpr("binary_byte_format(1024*1024)", "1.00MiB"); - assertExpr("binary_byte_format(1024*1024*1024)", "1.00GiB"); - assertExpr("binary_byte_format(1024*1024*1024*1024)", "1.00TiB"); - assertExpr("binary_byte_format(1024*1024*1024*1024*1024)", "1.00PiB"); + assertExpr("binary_byte_format(-1024)", "-1.00 KiB"); + assertExpr("binary_byte_format(1024)", "1.00 KiB"); + assertExpr("binary_byte_format(1024*1024)", "1.00 MiB"); + assertExpr("binary_byte_format(1024*1024*1024)", "1.00 GiB"); + assertExpr("binary_byte_format(1024*1024*1024*1024)", "1.00 TiB"); + assertExpr("binary_byte_format(1024*1024*1024*1024*1024)", "1.00 PiB"); - assertExpr("decimal_byte_format(-1000)", "-1.00KB"); - assertExpr("decimal_byte_format(1000)", "1.00KB"); - assertExpr("decimal_byte_format(1000*1000)", "1.00MB"); - assertExpr("decimal_byte_format(1000*1000*1000)", "1.00GB"); - assertExpr("decimal_byte_format(1000*1000*1000*1000)", "1.00TB"); + assertExpr("decimal_byte_format(-1000)", "-1.00 KB"); + assertExpr("decimal_byte_format(1000)", "1.00 KB"); + assertExpr("decimal_byte_format(1000*1000)", "1.00 MB"); + assertExpr("decimal_byte_format(1000*1000*1000)", "1.00 GB"); + assertExpr("decimal_byte_format(1000*1000*1000*1000)", "1.00 TB"); - assertExpr("decimal_format(-1000)", "-1.00K"); - assertExpr("decimal_format(1000)", "1.00K"); - assertExpr("decimal_format(1000*1000)", "1.00M"); - assertExpr("decimal_format(1000*1000*1000)", "1.00G"); - assertExpr("decimal_format(1000*1000*1000*1000)", "1.00T"); + assertExpr("decimal_format(-1000)", "-1.00 K"); + assertExpr("decimal_format(1000)", "1.00 K"); + assertExpr("decimal_format(1000*1000)", "1.00 M"); + assertExpr("decimal_format(1000*1000*1000)", "1.00 G"); + assertExpr("decimal_format(1000*1000*1000*1000)", "1.00 T"); } @Test public void testSizeFormatWithDifferentPrecision() { - assertExpr("binary_byte_format(1024, 0)", "1KiB"); - assertExpr("binary_byte_format(1024*1024, 1)", "1.0MiB"); - assertExpr("binary_byte_format(1024*1024*1024, 2)", "1.00GiB"); - assertExpr("binary_byte_format(1024*1024*1024*1024, 3)", "1.000TiB"); + assertExpr("binary_byte_format(1024, 0)", "1 KiB"); + assertExpr("binary_byte_format(1024*1024, 1)", "1.0 MiB"); + assertExpr("binary_byte_format(1024*1024*1024, 2)", "1.00 GiB"); + assertExpr("binary_byte_format(1024*1024*1024*1024, 3)", "1.000 TiB"); - assertExpr("decimal_byte_format(1234, 0)", "1KB"); - assertExpr("decimal_byte_format(1234*1000, 1)", "1.2MB"); - assertExpr("decimal_byte_format(1234*1000*1000, 2)", "1.23GB"); - assertExpr("decimal_byte_format(1234*1000*1000*1000, 3)", "1.234TB"); + assertExpr("decimal_byte_format(1234, 0)", "1 KB"); + assertExpr("decimal_byte_format(1234*1000, 1)", "1.2 MB"); + assertExpr("decimal_byte_format(1234*1000*1000, 2)", "1.23 GB"); + assertExpr("decimal_byte_format(1234*1000*1000*1000, 3)", "1.234 TB"); - assertExpr("decimal_format(1234, 0)", "1K"); - assertExpr("decimal_format(1234*1000,1)", "1.2M"); - assertExpr("decimal_format(1234*1000*1000,2)", "1.23G"); - assertExpr("decimal_format(1234*1000*1000*1000,3)", "1.234T"); + assertExpr("decimal_format(1234, 0)", "1 K"); + assertExpr("decimal_format(1234*1000,1)", "1.2 M"); + assertExpr("decimal_format(1234*1000*1000,2)", "1.23 G"); + assertExpr("decimal_format(1234*1000*1000*1000,3)", "1.234 T"); } @Test public void testSizeFormatWithEdgeCases() { //a nonexist value is null which is treated as 0 - assertExpr("binary_byte_format(nonexist)", "0B"); + assertExpr("binary_byte_format(nonexist)", "0 B"); //f = 12.34 - assertExpr("binary_byte_format(f)", "12B"); + assertExpr("binary_byte_format(f)", "12 B"); //nan is Double.NaN - assertExpr("binary_byte_format(nan)", "0B"); + assertExpr("binary_byte_format(nan)", "0 B"); //inf = Double.POSITIVE_INFINITY - assertExpr("binary_byte_format(inf)", "8.00EiB"); + assertExpr("binary_byte_format(inf)", "8.00 EiB"); //inf = Double.NEGATIVE_INFINITY - assertExpr("binary_byte_format(-inf)", "-8.00EiB"); + assertExpr("binary_byte_format(-inf)", "-8.00 EiB"); // o = 0 - assertExpr("binary_byte_format(o)", "0B"); + assertExpr("binary_byte_format(o)", "0 B"); // od = 0D - assertExpr("binary_byte_format(od)", "0B"); + assertExpr("binary_byte_format(od)", "0 B"); // of = 0F - assertExpr("binary_byte_format(of)", "0B"); + assertExpr("binary_byte_format(of)", "0 B"); } @Test @@ -705,8 +705,8 @@ public void testSizeFormatWithNoDefaultValueForNull() // normal cases // y is not null, the function returns correctly // - assertExpr("binary_byte_format(y)", "2B"); - assertExpr("decimal_byte_format(y)", "2B"); + assertExpr("binary_byte_format(y)", "2 B"); + assertExpr("decimal_byte_format(y)", "2 B"); assertExpr("decimal_format(y)", "2"); // diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 636e5a8089b1..3583f1abdc68 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -16308,7 +16308,7 @@ public void testSizeFormatFunction() throws Exception // NOTE: the first expression BINARY_BYTE_FORMAT(45678) in SQL is calculated during SQL parse phase, // so the converted Druid native query is its result intead of the raw function call // - .virtualColumns(expressionVirtualColumn("v0", "'44.61KiB'", ValueType.STRING), + .virtualColumns(expressionVirtualColumn("v0", "'44.61 KiB'", ValueType.STRING), expressionVirtualColumn("v1", "binary_byte_format((\"m1\" * 12345))", ValueType.STRING), expressionVirtualColumn("v2", "binary_byte_format((\"m1\" * 12345),0)", ValueType.STRING), expressionVirtualColumn("v3", "decimal_byte_format((\"m1\" * 12345))", ValueType.STRING), @@ -16322,11 +16322,11 @@ public void testSizeFormatFunction() throws Exception ), ImmutableList.of( new Object[]{(float) 2.0, - "44.61KiB", // 45678 / 1024 - "24.11KiB", // = m1(2.0) * 12345 / 1024 - "24KiB", // = m1(2.0) * 12345 / 1024, precision = 0 - "24.69KB", // decimal byte format, m1(2.0) * 12345 / 1000, - "24.69K" // decimal format, m1(2.0) * 12345 / 1000, + "44.61 KiB", // 45678 / 1024 + "24.11 KiB", // = m1(2.0) * 12345 / 1024 + "24 KiB", // = m1(2.0) * 12345 / 1024, precision = 0 + "24.69 KB", // decimal byte format, m1(2.0) * 12345 / 1000, + "24.69 K" // decimal format, m1(2.0) * 12345 / 1000, } ) ); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java index 2c90ef924d3d..49012697c0f9 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java @@ -1998,7 +1998,7 @@ public void testBinaryByteFormat() testHelper.makeLiteral(1000) ), DruidExpression.fromExpression("binary_byte_format(1000)"), - "1000.00B" + "1000 B" ); testHelper.testExpression( SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), @@ -2006,7 +2006,7 @@ public void testBinaryByteFormat() testHelper.makeLiteral(1024) ), DruidExpression.fromExpression("binary_byte_format(1024)"), - "1.00KiB" + "1.00 KiB" ); testHelper.testExpression( SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), @@ -2014,11 +2014,12 @@ public void testBinaryByteFormat() testHelper.makeLiteral(Long.MAX_VALUE) ), DruidExpression.fromExpression("binary_byte_format(9223372036854775807)"), - "8.00EiB" + "8.00 EiB" ); /** * NOTE: Test for Long.MIN_VALUE is skipped since ExprListnerImpl#exitLongExpr fails to parse Long.MIN_VALUE + * This cases has also been verified in the tests of underlying implementation */ /** @@ -2031,7 +2032,7 @@ public void testBinaryByteFormat() testHelper.makeInputRef("p") ), DruidExpression.fromExpression("binary_byte_format(\"b\",\"p\")"), - "25.000B" + "25 B" ); /** @@ -2045,7 +2046,7 @@ public void testBinaryByteFormat() testHelper.makeLiteral(0) ), DruidExpression.fromExpression("binary_byte_format(45000,0)"), - "44KiB" + "44 KiB" ); testHelper.testExpression( SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), @@ -2055,7 +2056,7 @@ public void testBinaryByteFormat() testHelper.makeLiteral(1) ), DruidExpression.fromExpression("binary_byte_format(45000,1)"), - "43.9KiB" + "43.9 KiB" ); testHelper.testExpression( SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), @@ -2065,7 +2066,7 @@ public void testBinaryByteFormat() testHelper.makeLiteral(2) ), DruidExpression.fromExpression("binary_byte_format(45000,2)"), - "43.95KiB" + "43.95 KiB" ); testHelper.testExpression( SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), @@ -2075,7 +2076,7 @@ public void testBinaryByteFormat() testHelper.makeLiteral(3) ), DruidExpression.fromExpression("binary_byte_format(45000,3)"), - "43.945KiB" + "43.945 KiB" ); } @@ -2091,7 +2092,7 @@ public void testDecimalByteFormat() testHelper.makeLiteral(999) ), DruidExpression.fromExpression("decimal_byte_format(999)"), - "999.00B" + "999B" ); testHelper.testExpression( SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), @@ -2124,7 +2125,7 @@ public void testDecimalByteFormat() testHelper.makeInputRef("p") ), DruidExpression.fromExpression("decimal_byte_format(\"b\",\"p\")"), - "25.000B" + "25B" ); /** From 5d0fe124f5438e608d3d3eda1f80f39fa17a56ff Mon Sep 17 00:00:00 2001 From: frank chen Date: Wed, 23 Dec 2020 09:12:47 +0800 Subject: [PATCH 10/20] fix tests --- .../sql/calcite/expression/ExpressionsTest.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java index 49012697c0f9..cd9b445e97bb 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java @@ -2092,7 +2092,7 @@ public void testDecimalByteFormat() testHelper.makeLiteral(999) ), DruidExpression.fromExpression("decimal_byte_format(999)"), - "999B" + "999 B" ); testHelper.testExpression( SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), @@ -2100,7 +2100,7 @@ public void testDecimalByteFormat() testHelper.makeLiteral(1024) ), DruidExpression.fromExpression("decimal_byte_format(1024)"), - "1.02KB" + "1.02 KB" ); testHelper.testExpression( SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), @@ -2108,7 +2108,7 @@ public void testDecimalByteFormat() testHelper.makeLiteral(Long.MAX_VALUE) ), DruidExpression.fromExpression("decimal_byte_format(9223372036854775807)"), - "9.22EB" + "9.22 EB" ); /** @@ -2125,7 +2125,7 @@ public void testDecimalByteFormat() testHelper.makeInputRef("p") ), DruidExpression.fromExpression("decimal_byte_format(\"b\",\"p\")"), - "25B" + "25 B" ); /** @@ -2139,7 +2139,7 @@ public void testDecimalByteFormat() testHelper.makeLiteral(0) ), DruidExpression.fromExpression("decimal_byte_format(45678,0)"), - "46KB" + "46 KB" ); testHelper.testExpression( SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), @@ -2149,7 +2149,7 @@ public void testDecimalByteFormat() testHelper.makeLiteral(1) ), DruidExpression.fromExpression("decimal_byte_format(45678,1)"), - "45.7KB" + "45.7 KB" ); testHelper.testExpression( SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), @@ -2159,7 +2159,7 @@ public void testDecimalByteFormat() testHelper.makeLiteral(2) ), DruidExpression.fromExpression("decimal_byte_format(45678,2)"), - "45.68KB" + "45.68 KB" ); testHelper.testExpression( SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), @@ -2169,7 +2169,7 @@ public void testDecimalByteFormat() testHelper.makeLiteral(3) ), DruidExpression.fromExpression("decimal_byte_format(45678,3)"), - "45.678KB" + "45.678 KB" ); } } From 94206b693ce67cf66741f340360e95268aeb3246 Mon Sep 17 00:00:00 2001 From: frank chen Date: Mon, 22 Feb 2021 15:39:33 +0800 Subject: [PATCH 11/20] add examples --- docs/misc/math-expr.md | 6 +++--- docs/querying/sql.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/misc/math-expr.md b/docs/misc/math-expr.md index e0d3f2187865..bf9cb3bf5eaa 100644 --- a/docs/misc/math-expr.md +++ b/docs/misc/math-expr.md @@ -241,7 +241,7 @@ supported features: | function | description | | --- | --- | -| binary_byte_format(value[, precision]) | Returns the value in human-readable [IEC](https://en.wikipedia.org/wiki/Binary_prefix) format. Supported unit suffix: `B`, `KiB`, `MiB`, `GiB`, `TiB`, `PiB`, `EiB`. `precision` must be in the range of [0,3] (default: 2). | -| decimal_byte_format(value[, precision]) | Returns the value in human-readable [SI](https://en.wikipedia.org/wiki/Binary_prefix) format. Supported unit suffix: `B`, `KB`, `MB`, `GB`, `TB`, `PB`, `EB`. `precision` must be in the range of [0,3] (default: 2). | -| decimal_format(value[, precision]) | Returns the value in human-readable SI format. Supported unit suffix: `K`, `M`, `G`, `T`, `P`, `E`. `precision` must be in the range of [0,3] (default: 2). | +| binary_byte_format(value[, precision]) | Returns the value in human-readable [IEC](https://en.wikipedia.org/wiki/Binary_prefix) format. For example, binary_byte_format(1048576) returns 1.00MiB. `precision` must be in the range of [0,3] (default: 2). | +| decimal_byte_format(value[, precision]) | Returns the value in human-readable [SI](https://en.wikipedia.org/wiki/Binary_prefix) format. decimal_byte_format(1048576) returns 1.04MB. `precision` must be in the range of [0,3] (default: 2). `precision` must be in the range of [0,3] (default: 2). | +| decimal_format(value[, precision]) | Returns the value in human-readable SI format. For example, DECIMAL_FORMAT(1048576) returns 1.04M. `precision` must be in the range of [0,3] (default: 2). | diff --git a/docs/querying/sql.md b/docs/querying/sql.md index b77aaf177d89..df6e46125298 100644 --- a/docs/querying/sql.md +++ b/docs/querying/sql.md @@ -387,6 +387,9 @@ to FLOAT. At runtime, Druid will widen 32-bit floats to 64-bit for most expressi |`BITWISE_SHIFT_LEFT(expr1, expr2)`|Returns the result of `expr1 << expr2`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles| |`BITWISE_SHIFT_RIGHT(expr1, expr2)`|Returns the result of `expr1 >> expr2`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles| |`BITWISE_XOR(expr1, expr2)`|Returns the result of `expr1 ^ expr2`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles| +|`BINARY_BYTE_FORMAT(value, [precision])`|Returns the value in human-readable [IEC](https://en.wikipedia.org/wiki/Binary_prefix) format. For example, BINARY_BYTE_FORMAT(1048576) returns 1.00MiB. `precision` must be in the range of [0,3] (default: 2).| +|`DECIMAL_BYTE_FORMAT(value, [precision])`|Returns the value in human-readable [SI](https://en.wikipedia.org/wiki/Binary_prefix) format. For example, DECIMAL_BYTE_FORMAT(1048576) returns 1.04MB. `precision` must be in the range of [0,3] (default: 2).| +|`DECIMAL_FORMAT(value, [precision])`|Returns the value in human-readable SI format. For example, DECIMAL_FORMAT(1048576) returns 1.04M. `precision` must be in the range of [0,3] (default: 2).| ### String functions @@ -574,9 +577,6 @@ The [DataSketches extension](../development/extensions-core/datasketches-extensi |`COALESCE(value1, value2, ...)`|Returns the first value that is neither NULL nor empty string.| |`NVL(expr,expr-for-null)`|Returns 'expr-for-null' if 'expr' is null (or empty string for string type).| |`BLOOM_FILTER_TEST(, )`|Returns true if the value is contained in a Base64-serialized bloom filter. See the [Bloom filter extension](../development/extensions-core/bloom-filter.md) documentation for additional details.| -|`BINARY_BYTE_FORMAT(value, [precision])`|Returns the value in human-readable [IEC](https://en.wikipedia.org/wiki/Binary_prefix) format. Supported unit suffix: `B`, `KiB`, `MiB`, `GiB`, `TiB`, `PiB`, `EiB`. `precision` must be in the range of [0,3] (default: 2).| -|`DECIMAL_BYTE_FORMAT(value, [precision])`|Returns the value in human-readable [SI](https://en.wikipedia.org/wiki/Binary_prefix) format. Supported unit suffix: `B`, `KB`, `MB`, `GB`, `TB`, `PB`, `EB`. `precision` must be in the range of [0,3] (default: 2).| -|`DECIMAL_FORMAT(value, [precision])`|Returns the value in human-readable SI format. Supported unit suffix: `K`, `M`, `G`, `T`, `P`, `E`. `precision` must be in the range of [0,3] (default: 2).| ## Multi-value string functions From f19e6a59ef82e4d056e05a6954b3a9203a040eec Mon Sep 17 00:00:00 2001 From: frank chen Date: Mon, 22 Feb 2021 16:27:09 +0800 Subject: [PATCH 12/20] change function names according to review comments --- .../org/apache/druid/math/expr/Function.java | 12 +- .../apache/druid/math/expr/FunctionTest.java | 120 +++++++++--------- docs/misc/math-expr.md | 7 +- docs/querying/sql.md | 7 +- ...umanReadableFormatOperatorConversion.java} | 14 +- .../calcite/planner/DruidOperatorTable.java | 8 +- .../druid/sql/calcite/CalciteQueryTest.java | 107 ++++++++-------- .../calcite/expression/ExpressionsTest.java | 70 +++++----- website/.spelling | 6 +- 9 files changed, 175 insertions(+), 176 deletions(-) rename sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/{SizeFormatOperatorConversion.java => HumanReadableFormatOperatorConversion.java} (87%) diff --git a/core/src/main/java/org/apache/druid/math/expr/Function.java b/core/src/main/java/org/apache/druid/math/expr/Function.java index 73e78c80fd1e..1eb5e26a25ee 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Function.java +++ b/core/src/main/java/org/apache/druid/math/expr/Function.java @@ -3562,12 +3562,12 @@ public ExprType getOutputType(Expr.InputBindingInspector inputTypes, List } } - class DecimalByteFormatFunc extends SizeFormatFunc + class HumanReadableDecimalByteFormatFunc extends SizeFormatFunc { @Override public String name() { - return "decimal_byte_format"; + return "human_readable_decimal_byte_format"; } @Override @@ -3577,12 +3577,12 @@ protected HumanReadableBytes.UnitSystem getUnitSystem() } } - class BinaryByteFormatFunc extends SizeFormatFunc + class HumanReadableBinaryByteFormatFunc extends SizeFormatFunc { @Override public String name() { - return "binary_byte_format"; + return "human_readable_binary_byte_format"; } @Override @@ -3592,12 +3592,12 @@ protected HumanReadableBytes.UnitSystem getUnitSystem() } } - class DecimalFormatFunc extends SizeFormatFunc + class HumanReadableDecimalFormatFunc extends SizeFormatFunc { @Override public String name() { - return "decimal_format"; + return "human_readable_decimal_format"; } @Override diff --git a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java index 316396e31599..354bffd953be 100644 --- a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java @@ -528,71 +528,71 @@ public void testLeast() @Test public void testSizeFormat() { - assertExpr("binary_byte_format(-1024)", "-1.00 KiB"); - assertExpr("binary_byte_format(1024)", "1.00 KiB"); - assertExpr("binary_byte_format(1024*1024)", "1.00 MiB"); - assertExpr("binary_byte_format(1024*1024*1024)", "1.00 GiB"); - assertExpr("binary_byte_format(1024*1024*1024*1024)", "1.00 TiB"); - assertExpr("binary_byte_format(1024*1024*1024*1024*1024)", "1.00 PiB"); + assertExpr("human_readable_binary_byte_format(-1024)", "-1.00 KiB"); + assertExpr("human_readable_binary_byte_format(1024)", "1.00 KiB"); + assertExpr("human_readable_binary_byte_format(1024*1024)", "1.00 MiB"); + assertExpr("human_readable_binary_byte_format(1024*1024*1024)", "1.00 GiB"); + assertExpr("human_readable_binary_byte_format(1024*1024*1024*1024)", "1.00 TiB"); + assertExpr("human_readable_binary_byte_format(1024*1024*1024*1024*1024)", "1.00 PiB"); - assertExpr("decimal_byte_format(-1000)", "-1.00 KB"); - assertExpr("decimal_byte_format(1000)", "1.00 KB"); - assertExpr("decimal_byte_format(1000*1000)", "1.00 MB"); - assertExpr("decimal_byte_format(1000*1000*1000)", "1.00 GB"); - assertExpr("decimal_byte_format(1000*1000*1000*1000)", "1.00 TB"); + assertExpr("human_readable_decimal_byte_format(-1000)", "-1.00 KB"); + assertExpr("human_readable_decimal_byte_format(1000)", "1.00 KB"); + assertExpr("human_readable_decimal_byte_format(1000*1000)", "1.00 MB"); + assertExpr("human_readable_decimal_byte_format(1000*1000*1000)", "1.00 GB"); + assertExpr("human_readable_decimal_byte_format(1000*1000*1000*1000)", "1.00 TB"); - assertExpr("decimal_format(-1000)", "-1.00 K"); - assertExpr("decimal_format(1000)", "1.00 K"); - assertExpr("decimal_format(1000*1000)", "1.00 M"); - assertExpr("decimal_format(1000*1000*1000)", "1.00 G"); - assertExpr("decimal_format(1000*1000*1000*1000)", "1.00 T"); + assertExpr("human_readable_decimal_format(-1000)", "-1.00 K"); + assertExpr("human_readable_decimal_format(1000)", "1.00 K"); + assertExpr("human_readable_decimal_format(1000*1000)", "1.00 M"); + assertExpr("human_readable_decimal_format(1000*1000*1000)", "1.00 G"); + assertExpr("human_readable_decimal_format(1000*1000*1000*1000)", "1.00 T"); } @Test public void testSizeFormatWithDifferentPrecision() { - assertExpr("binary_byte_format(1024, 0)", "1 KiB"); - assertExpr("binary_byte_format(1024*1024, 1)", "1.0 MiB"); - assertExpr("binary_byte_format(1024*1024*1024, 2)", "1.00 GiB"); - assertExpr("binary_byte_format(1024*1024*1024*1024, 3)", "1.000 TiB"); + assertExpr("human_readable_binary_byte_format(1024, 0)", "1 KiB"); + assertExpr("human_readable_binary_byte_format(1024*1024, 1)", "1.0 MiB"); + assertExpr("human_readable_binary_byte_format(1024*1024*1024, 2)", "1.00 GiB"); + assertExpr("human_readable_binary_byte_format(1024*1024*1024*1024, 3)", "1.000 TiB"); - assertExpr("decimal_byte_format(1234, 0)", "1 KB"); - assertExpr("decimal_byte_format(1234*1000, 1)", "1.2 MB"); - assertExpr("decimal_byte_format(1234*1000*1000, 2)", "1.23 GB"); - assertExpr("decimal_byte_format(1234*1000*1000*1000, 3)", "1.234 TB"); + assertExpr("human_readable_decimal_byte_format(1234, 0)", "1 KB"); + assertExpr("human_readable_decimal_byte_format(1234*1000, 1)", "1.2 MB"); + assertExpr("human_readable_decimal_byte_format(1234*1000*1000, 2)", "1.23 GB"); + assertExpr("human_readable_decimal_byte_format(1234*1000*1000*1000, 3)", "1.234 TB"); - assertExpr("decimal_format(1234, 0)", "1 K"); - assertExpr("decimal_format(1234*1000,1)", "1.2 M"); - assertExpr("decimal_format(1234*1000*1000,2)", "1.23 G"); - assertExpr("decimal_format(1234*1000*1000*1000,3)", "1.234 T"); + assertExpr("human_readable_decimal_format(1234, 0)", "1 K"); + assertExpr("human_readable_decimal_format(1234*1000,1)", "1.2 M"); + assertExpr("human_readable_decimal_format(1234*1000*1000,2)", "1.23 G"); + assertExpr("human_readable_decimal_format(1234*1000*1000*1000,3)", "1.234 T"); } @Test public void testSizeFormatWithEdgeCases() { //a nonexist value is null which is treated as 0 - assertExpr("binary_byte_format(nonexist)", "0 B"); + assertExpr("human_readable_binary_byte_format(nonexist)", "0 B"); //f = 12.34 - assertExpr("binary_byte_format(f)", "12 B"); + assertExpr("human_readable_binary_byte_format(f)", "12 B"); //nan is Double.NaN - assertExpr("binary_byte_format(nan)", "0 B"); + assertExpr("human_readable_binary_byte_format(nan)", "0 B"); //inf = Double.POSITIVE_INFINITY - assertExpr("binary_byte_format(inf)", "8.00 EiB"); + assertExpr("human_readable_binary_byte_format(inf)", "8.00 EiB"); //inf = Double.NEGATIVE_INFINITY - assertExpr("binary_byte_format(-inf)", "-8.00 EiB"); + assertExpr("human_readable_binary_byte_format(-inf)", "-8.00 EiB"); // o = 0 - assertExpr("binary_byte_format(o)", "0 B"); + assertExpr("human_readable_binary_byte_format(o)", "0 B"); // od = 0D - assertExpr("binary_byte_format(od)", "0 B"); + assertExpr("human_readable_binary_byte_format(od)", "0 B"); // of = 0F - assertExpr("binary_byte_format(of)", "0 B"); + assertExpr("human_readable_binary_byte_format(of)", "0 B"); } @Test @@ -600,50 +600,50 @@ public void testSizeForatInvalidArgumentType() { try { //x = "foo" - Parser.parse("binary_byte_format(x)", ExprMacroTable.nil()) + Parser.parse("human_readable_binary_byte_format(x)", ExprMacroTable.nil()) .eval(bindings); //must not go to here Assert.assertTrue(false); } catch (IAE e) { - Assert.assertEquals("Function[binary_byte_format] needs a number as its first argument", e.getMessage()); + Assert.assertEquals("Function[human_readable_binary_byte_format] needs a number as its first argument", e.getMessage()); } try { //x = "foo" - Parser.parse("binary_byte_format(1024, x)", ExprMacroTable.nil()) + Parser.parse("human_readable_binary_byte_format(1024, x)", ExprMacroTable.nil()) .eval(bindings); //must not go to here Assert.assertTrue(false); } catch (IAE e) { - Assert.assertEquals("Function[binary_byte_format] needs an integer as its second argument", e.getMessage()); + Assert.assertEquals("Function[human_readable_binary_byte_format] needs an integer as its second argument", e.getMessage()); } try { //of = 0F - Parser.parse("binary_byte_format(1024, of)", ExprMacroTable.nil()) + Parser.parse("human_readable_binary_byte_format(1024, of)", ExprMacroTable.nil()) .eval(bindings); //must not go to here Assert.assertTrue(false); } catch (IAE e) { - Assert.assertEquals("Function[binary_byte_format] needs an integer as its second argument", e.getMessage()); + Assert.assertEquals("Function[human_readable_binary_byte_format] needs an integer as its second argument", e.getMessage()); } try { //of = 0F - Parser.parse("binary_byte_format(1024, nonexist)", ExprMacroTable.nil()) + Parser.parse("human_readable_binary_byte_format(1024, nonexist)", ExprMacroTable.nil()) .eval(bindings); //must not go to here Assert.assertTrue(false); } catch (IAE e) { - Assert.assertEquals("Function[binary_byte_format] needs an integer as its second argument", e.getMessage()); + Assert.assertEquals("Function[human_readable_binary_byte_format] needs an integer as its second argument", e.getMessage()); } } @@ -651,39 +651,39 @@ public void testSizeForatInvalidArgumentType() public void testSizeFormatInvalidPrecision() { try { - Parser.parse("binary_byte_format(1024, maxLong)", ExprMacroTable.nil()) + Parser.parse("human_readable_binary_byte_format(1024, maxLong)", ExprMacroTable.nil()) .eval(bindings); Assert.assertTrue(false); } catch (IAE e) { - Assert.assertEquals("Given precision[9223372036854775807] of Function[binary_byte_format] must be in the range of [0,3]", e.getMessage()); + Assert.assertEquals("Given precision[9223372036854775807] of Function[human_readable_binary_byte_format] must be in the range of [0,3]", e.getMessage()); } try { - Parser.parse("binary_byte_format(1024, minLong)", ExprMacroTable.nil()) + Parser.parse("human_readable_binary_byte_format(1024, minLong)", ExprMacroTable.nil()) .eval(bindings); Assert.assertTrue(false); } catch (IAE e) { - Assert.assertEquals("Given precision[-9223372036854775808] of Function[binary_byte_format] must be in the range of [0,3]", e.getMessage()); + Assert.assertEquals("Given precision[-9223372036854775808] of Function[human_readable_binary_byte_format] must be in the range of [0,3]", e.getMessage()); } try { - Parser.parse("binary_byte_format(1024, -1)", ExprMacroTable.nil()) + Parser.parse("human_readable_binary_byte_format(1024, -1)", ExprMacroTable.nil()) .eval(bindings); Assert.assertTrue(false); } catch (IAE e) { - Assert.assertEquals("Given precision[-1] of Function[binary_byte_format] must be in the range of [0,3]", e.getMessage()); + Assert.assertEquals("Given precision[-1] of Function[human_readable_binary_byte_format] must be in the range of [0,3]", e.getMessage()); } try { - Parser.parse("binary_byte_format(1024, 4)", ExprMacroTable.nil()) + Parser.parse("human_readable_binary_byte_format(1024, 4)", ExprMacroTable.nil()) .eval(bindings); Assert.assertTrue(false); } catch (IAE e) { - Assert.assertEquals("Given precision[4] of Function[binary_byte_format] must be in the range of [0,3]", e.getMessage()); + Assert.assertEquals("Given precision[4] of Function[human_readable_binary_byte_format] must be in the range of [0,3]", e.getMessage()); } } @@ -691,8 +691,8 @@ public void testSizeFormatInvalidPrecision() public void testSizeFormatInvalidArgumentSize() { expectedException.expect(IAE.class); - expectedException.expectMessage("Function[binary_byte_format] needs 1 or 2 arguments"); - Parser.parse("binary_byte_format(1024, 2, 3)", ExprMacroTable.nil()) + expectedException.expectMessage("Function[human_readable_binary_byte_format] needs 1 or 2 arguments"); + Parser.parse("human_readable_binary_byte_format(1024, 2, 3)", ExprMacroTable.nil()) .eval(bindings); } @@ -705,17 +705,17 @@ public void testSizeFormatWithNoDefaultValueForNull() // normal cases // y is not null, the function returns correctly // - assertExpr("binary_byte_format(y)", "2 B"); - assertExpr("decimal_byte_format(y)", "2 B"); - assertExpr("decimal_format(y)", "2"); + assertExpr("human_readable_binary_byte_format(y)", "2 B"); + assertExpr("human_readable_decimal_byte_format(y)", "2 B"); + assertExpr("human_readable_decimal_format(y)", "2"); // // since 'druid.generic.useDefaultValueForNull' has been disabled above, // the 'nonexist' below returns null, and the function calls also return null // - assertExpr("binary_byte_format(nonexist)", null); - assertExpr("decimal_byte_format(nonexist)", null); - assertExpr("decimal_format(nonexist)", null); + assertExpr("human_readable_binary_byte_format(nonexist)", null); + assertExpr("human_readable_decimal_byte_format(nonexist)", null); + assertExpr("human_readable_decimal_format(nonexist)", null); } NullHandling.updateForTests(true); diff --git a/docs/misc/math-expr.md b/docs/misc/math-expr.md index bf9cb3bf5eaa..66c5a6761922 100644 --- a/docs/misc/math-expr.md +++ b/docs/misc/math-expr.md @@ -241,7 +241,6 @@ supported features: | function | description | | --- | --- | -| binary_byte_format(value[, precision]) | Returns the value in human-readable [IEC](https://en.wikipedia.org/wiki/Binary_prefix) format. For example, binary_byte_format(1048576) returns 1.00MiB. `precision` must be in the range of [0,3] (default: 2). | -| decimal_byte_format(value[, precision]) | Returns the value in human-readable [SI](https://en.wikipedia.org/wiki/Binary_prefix) format. decimal_byte_format(1048576) returns 1.04MB. `precision` must be in the range of [0,3] (default: 2). `precision` must be in the range of [0,3] (default: 2). | -| decimal_format(value[, precision]) | Returns the value in human-readable SI format. For example, DECIMAL_FORMAT(1048576) returns 1.04M. `precision` must be in the range of [0,3] (default: 2). | - +| human_readable_binary_byte_format(value[, precision]) | Format a number in human-readable [IEC](https://en.wikipedia.org/wiki/Binary_prefix) format. For example, human_readable_binary_byte_format(1048576) returns `1.00 MiB`. `precision` must be in the range of [0,3] (default: 2). | +| human_readable_decimal_byte_format(value[, precision]) | Format a number in human-readable [SI](https://en.wikipedia.org/wiki/Binary_prefix) format. human_readable_decimal_byte_format(1048576) returns `1.04 MB`. `precision` must be in the range of [0,3] (default: 2). `precision` must be in the range of [0,3] (default: 2). | +| human_readable_decimal_format(value[, precision]) | Format a number in human-readable SI format. For example, HUMAN_READABLE_DECIMAL_BYTE_FORMAT(1048576) returns `1.04 M`. `precision` must be in the range of [0,3] (default: 2). | diff --git a/docs/querying/sql.md b/docs/querying/sql.md index df6e46125298..8a0a5b8b3880 100644 --- a/docs/querying/sql.md +++ b/docs/querying/sql.md @@ -387,9 +387,10 @@ to FLOAT. At runtime, Druid will widen 32-bit floats to 64-bit for most expressi |`BITWISE_SHIFT_LEFT(expr1, expr2)`|Returns the result of `expr1 << expr2`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles| |`BITWISE_SHIFT_RIGHT(expr1, expr2)`|Returns the result of `expr1 >> expr2`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles| |`BITWISE_XOR(expr1, expr2)`|Returns the result of `expr1 ^ expr2`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles| -|`BINARY_BYTE_FORMAT(value, [precision])`|Returns the value in human-readable [IEC](https://en.wikipedia.org/wiki/Binary_prefix) format. For example, BINARY_BYTE_FORMAT(1048576) returns 1.00MiB. `precision` must be in the range of [0,3] (default: 2).| -|`DECIMAL_BYTE_FORMAT(value, [precision])`|Returns the value in human-readable [SI](https://en.wikipedia.org/wiki/Binary_prefix) format. For example, DECIMAL_BYTE_FORMAT(1048576) returns 1.04MB. `precision` must be in the range of [0,3] (default: 2).| -|`DECIMAL_FORMAT(value, [precision])`|Returns the value in human-readable SI format. For example, DECIMAL_FORMAT(1048576) returns 1.04M. `precision` must be in the range of [0,3] (default: 2).| +|`HUMAN_READABLE_BINARY_BYTE_FORMAT(value[, precision])`| Format a number in human-readable [IEC](https://en.wikipedia.org/wiki/Binary_prefix) format. For example, HUMAN_READABLE_BINARY_BYTE_FORMAT(1048576) returns `1.00 MiB`. `precision` must be in the range of [0,3] (default: 2). | +|`HUMAN_READABLE_DECIMAL_BYTE_FORMAT(value[, precision])`| Format a number in human-readable [SI](https://en.wikipedia.org/wiki/Binary_prefix) format. HUMAN_READABLE_DECIMAL_BYTE_FORMAT(1048576) returns `1.04 MB`. `precision` must be in the range of [0,3] (default: 2). `precision` must be in the range of [0,3] (default: 2). | +|`HUMAN_READABLE_DECIMAL_FORMAT(value[, precision])`| Format a number in human-readable SI format. For example, HUMAN_READABLE_DECIMAL_FORMAT(1048576) returns `1.04 M`. `precision` must be in the range of [0,3] (default: 2). | + ### String functions diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/SizeFormatOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/HumanReadableFormatOperatorConversion.java similarity index 87% rename from sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/SizeFormatOperatorConversion.java rename to sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/HumanReadableFormatOperatorConversion.java index f4deca31c58a..4002f625b52a 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/SizeFormatOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/HumanReadableFormatOperatorConversion.java @@ -36,20 +36,20 @@ import org.apache.druid.sql.calcite.expression.SqlOperatorConversion; import org.apache.druid.sql.calcite.planner.PlannerContext; -public class SizeFormatOperatorConversion implements SqlOperatorConversion +public class HumanReadableFormatOperatorConversion implements SqlOperatorConversion { - public static final SqlOperatorConversion BINARY_BYTE_FORMAT = new SizeFormatOperatorConversion("binary_byte_format"); - public static final SqlOperatorConversion DECIMAL_BYTE_FORMAT = new SizeFormatOperatorConversion("decimal_byte_format"); - public static final SqlOperatorConversion DECIMAL_FORMAT = new SizeFormatOperatorConversion("decimal_format"); + public static final SqlOperatorConversion BINARY_BYTE_FORMAT = new HumanReadableFormatOperatorConversion("human_readable_binary_byte_format"); + public static final SqlOperatorConversion DECIMAL_BYTE_FORMAT = new HumanReadableFormatOperatorConversion("human_readable_decimal_byte_format"); + public static final SqlOperatorConversion DECIMAL_FORMAT = new HumanReadableFormatOperatorConversion("human_readable_decimal_format"); private final String name; private final SqlFunction sqlFunction; - private SizeFormatOperatorConversion(String name) + private HumanReadableFormatOperatorConversion(String name) { this.sqlFunction = OperatorConversions .operatorBuilder(StringUtils.toUpperCase(name)) - .operandTypeChecker(new SizeFormatOperandTypeChecker()) + .operandTypeChecker(new HumanReadableFormatOperandTypeChecker()) .functionCategory(SqlFunctionCategory.STRING) .returnTypeNonNull(SqlTypeName.VARCHAR) .build(); @@ -73,7 +73,7 @@ public DruidExpression toDruidExpression( return OperatorConversions.convertCall(plannerContext, rowSignature, rexNode, name); } - private static class SizeFormatOperandTypeChecker implements SqlOperandTypeChecker + private static class HumanReadableFormatOperandTypeChecker implements SqlOperandTypeChecker { @Override public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java index eec109f89aee..28e20bf3e98a 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java @@ -68,6 +68,7 @@ import org.apache.druid.sql.calcite.expression.builtin.ExtractOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.FloorOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.GreatestOperatorConversion; +import org.apache.druid.sql.calcite.expression.builtin.HumanReadableFormatOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.IPv4AddressMatchOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.IPv4AddressParseOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.IPv4AddressStringifyOperatorConversion; @@ -92,7 +93,6 @@ import org.apache.druid.sql.calcite.expression.builtin.ReverseOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.RightOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.RoundOperatorConversion; -import org.apache.druid.sql.calcite.expression.builtin.SizeFormatOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.StringFormatOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.StringToMultiValueStringOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.StrposOperatorConversion; @@ -241,9 +241,9 @@ public class DruidOperatorTable implements SqlOperatorTable private static final List FORMAT_OPERATOR_CONVERSIONS = ImmutableList.builder() - .add(SizeFormatOperatorConversion.BINARY_BYTE_FORMAT) - .add(SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT) - .add(SizeFormatOperatorConversion.DECIMAL_FORMAT) + .add(HumanReadableFormatOperatorConversion.BINARY_BYTE_FORMAT) + .add(HumanReadableFormatOperatorConversion.DECIMAL_BYTE_FORMAT) + .add(HumanReadableFormatOperatorConversion.DECIMAL_FORMAT) .build(); private static final List BITWISE_OPERATOR_CONVERSIONS = diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index c1032bef6f04..2c20fa9bf4be 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -123,7 +123,6 @@ import org.junit.Test; import org.junit.runner.RunWith; -import javax.validation.ValidationException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -16643,49 +16642,6 @@ public void testGroupingSetsWithLimit() throws Exception ); } - @Test - public void testSizeFormatFunction() throws Exception - { - testQuery( - "SELECT m1, " - + "BINARY_BYTE_FORMAT(45678)," - + "BINARY_BYTE_FORMAT(m1*12345)," - + "BINARY_BYTE_FORMAT(m1*12345, 0), " - + "DECIMAL_BYTE_FORMAT(m1*12345), " - + "DECIMAL_FORMAT(m1*12345) " - + "FROM numfoo WHERE f1 = 0.1 LIMIT 1", - ImmutableList.of( - newScanQueryBuilder() - .dataSource(CalciteTests.DATASOURCE3) - .intervals(querySegmentSpec(Filtration.eternity())) - // - // NOTE: the first expression BINARY_BYTE_FORMAT(45678) in SQL is calculated during SQL parse phase, - // so the converted Druid native query is its result intead of the raw function call - // - .virtualColumns(expressionVirtualColumn("v0", "'44.61 KiB'", ValueType.STRING), - expressionVirtualColumn("v1", "binary_byte_format((\"m1\" * 12345))", ValueType.STRING), - expressionVirtualColumn("v2", "binary_byte_format((\"m1\" * 12345),0)", ValueType.STRING), - expressionVirtualColumn("v3", "decimal_byte_format((\"m1\" * 12345))", ValueType.STRING), - expressionVirtualColumn("v4", "decimal_format((\"m1\" * 12345))", ValueType.STRING)) - .columns("m1", "v0", "v1", "v2", "v3", "v4") - .filters(selector("f1", "0.1", null)) - .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) - .limit(1) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - new Object[]{(float) 2.0, - "44.61 KiB", // 45678 / 1024 - "24.11 KiB", // = m1(2.0) * 12345 / 1024 - "24 KiB", // = m1(2.0) * 12345 / 1024, precision = 0 - "24.69 KB", // decimal byte format, m1(2.0) * 12345 / 1000, - "24.69 K" // decimal format, m1(2.0) * 12345 / 1000, - } - ) - ); - } - public void testGroupingSetsWithLimitOrderByGran() throws Exception { // Cannot vectorize due to virtual columns. @@ -16761,46 +16717,89 @@ public void testGroupingSetsWithLimitOrderByGran() throws Exception } @Test - public void testSizeFormatFunctionExceptionWithWrongNumberType() throws Exception + public void testHumanReadableFormatFunction() throws Exception + { + testQuery( + "SELECT m1, " + + "HUMAN_READABLE_BINARY_BYTE_FORMAT(45678)," + + "HUMAN_READABLE_BINARY_BYTE_FORMAT(m1*12345)," + + "HUMAN_READABLE_BINARY_BYTE_FORMAT(m1*12345, 0), " + + "HUMAN_READABLE_DECIMAL_BYTE_FORMAT(m1*12345), " + + "HUMAN_READABLE_DECIMAL_FORMAT(m1*12345) " + + "FROM numfoo WHERE f1 = 0.1 LIMIT 1", + ImmutableList.of( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + // + // NOTE: the first expression HUMAN_READABLE_BINARY_BYTE_FORMAT(45678) in SQL is calculated during SQL parse phase, + // so the converted Druid native query is its result intead of the raw function call + // + .virtualColumns(expressionVirtualColumn("v0", "'44.61 KiB'", ValueType.STRING), + expressionVirtualColumn("v1", "human_readable_binary_byte_format((\"m1\" * 12345))", ValueType.STRING), + expressionVirtualColumn("v2", "human_readable_binary_byte_format((\"m1\" * 12345),0)", ValueType.STRING), + expressionVirtualColumn("v3", "human_readable_decimal_byte_format((\"m1\" * 12345))", ValueType.STRING), + expressionVirtualColumn("v4", "human_readable_decimal_format((\"m1\" * 12345))", ValueType.STRING)) + .columns("m1", "v0", "v1", "v2", "v3", "v4") + .filters(selector("f1", "0.1", null)) + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .limit(1) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{(float) 2.0, + "44.61 KiB", // 45678 / 1024 + "24.11 KiB", // = m1(2.0) * 12345 / 1024 + "24 KiB", // = m1(2.0) * 12345 / 1024, precision = 0 + "24.69 KB", // decimal byte format, m1(2.0) * 12345 / 1000, + "24.69 K" // decimal format, m1(2.0) * 12345 / 1000, + } + ) + ); + } + + @Test + public void testHumanReadableFormatFunctionExceptionWithWrongNumberType() throws Exception { this.expectedException.expect(SqlPlanningException.class); - this.expectedException.expectMessage("Supported form(s): BINARY_BYTE_FORMAT(Number, [Precision])"); + this.expectedException.expectMessage("Supported form(s): HUMAN_READABLE_BINARY_BYTE_FORMAT(Number, [Precision])"); testQuery( - "SELECT BINARY_BYTE_FORMAT('45678')", + "SELECT HUMAN_READABLE_BINARY_BYTE_FORMAT('45678')", Collections.emptyList(), Collections.emptyList() ); } @Test - public void testSizeFormatFunctionWithWrongPrecisionType() throws Exception + public void testHumanReadableFormatFunctionWithWrongPrecisionType() throws Exception { this.expectedException.expect(SqlPlanningException.class); - this.expectedException.expectMessage("Supported form(s): BINARY_BYTE_FORMAT(Number, [Precision])"); + this.expectedException.expectMessage("Supported form(s): HUMAN_READABLE_BINARY_BYTE_FORMAT(Number, [Precision])"); testQuery( - "SELECT BINARY_BYTE_FORMAT(45678, '2')", + "SELECT HUMAN_READABLE_BINARY_BYTE_FORMAT(45678, '2')", Collections.emptyList(), Collections.emptyList() ); } @Test - public void testSizeFormatFunctionWithInvalidNumberOfArguments() throws Exception + public void testHumanReadableFormatFunctionWithInvalidNumberOfArguments() throws Exception { this.expectedException.expect(SqlPlanningException.class); /* * frankly speaking, the exception message thrown here is a little bit confusion - * it says it's 'expecting 1 arguments' but acturally BINARY_BYTE_FORMAT supports 1 or 2 arguments + * it says it's 'expecting 1 arguments' but acturally HUMAN_READABLE_BINARY_BYTE_FORMAT supports 1 or 2 arguments * * The message is returned from {@link org.apache.calcite.sql.validate.SqlValidatorImpl#handleUnresolvedFunction}, * and we can see from its implementation that it gets the min number arguments to format the exception message. * */ this.expectedException.expectMessage( - "Invalid number of arguments to function 'BINARY_BYTE_FORMAT'. Was expecting 1 arguments"); + "Invalid number of arguments to function 'HUMAN_READABLE_BINARY_BYTE_FORMAT'. Was expecting 1 arguments"); testQuery( - "SELECT BINARY_BYTE_FORMAT(45678, 2, 1)", + "SELECT HUMAN_READABLE_BINARY_BYTE_FORMAT(45678, 2, 1)", Collections.emptyList(), Collections.emptyList() ); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java index d264ecaac7fb..36ab65998b02 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java @@ -42,6 +42,7 @@ import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.sql.calcite.expression.builtin.ContainsOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.DateTruncOperatorConversion; +import org.apache.druid.sql.calcite.expression.builtin.HumanReadableFormatOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.LPadOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.LeftOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.ParseLongOperatorConversion; @@ -52,7 +53,6 @@ import org.apache.druid.sql.calcite.expression.builtin.ReverseOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.RightOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.RoundOperatorConversion; -import org.apache.druid.sql.calcite.expression.builtin.SizeFormatOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.StringFormatOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.StrposOperatorConversion; import org.apache.druid.sql.calcite.expression.builtin.TimeCeilOperatorConversion; @@ -2083,33 +2083,33 @@ public void testOperatorConversionsDruidBinaryLongFn() } @Test - public void testBinaryByteFormat() + public void testHumanReadableBinaryByteFormat() { /* * Basic Test */ testHelper.testExpression( - SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), + HumanReadableFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), ImmutableList.of( testHelper.makeLiteral(1000) ), - DruidExpression.fromExpression("binary_byte_format(1000)"), + DruidExpression.fromExpression("human_readable_binary_byte_format(1000)"), "1000 B" ); testHelper.testExpression( - SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), + HumanReadableFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), ImmutableList.of( testHelper.makeLiteral(1024) ), - DruidExpression.fromExpression("binary_byte_format(1024)"), + DruidExpression.fromExpression("human_readable_binary_byte_format(1024)"), "1.00 KiB" ); testHelper.testExpression( - SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), + HumanReadableFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), ImmutableList.of( testHelper.makeLiteral(Long.MAX_VALUE) ), - DruidExpression.fromExpression("binary_byte_format(9223372036854775807)"), + DruidExpression.fromExpression("human_readable_binary_byte_format(9223372036854775807)"), "8.00 EiB" ); @@ -2122,12 +2122,12 @@ public void testBinaryByteFormat() * test input with variable reference */ testHelper.testExpression( - SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), + HumanReadableFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), ImmutableList.of( testHelper.makeInputRef("b"), testHelper.makeInputRef("p") ), - DruidExpression.fromExpression("binary_byte_format(\"b\",\"p\")"), + DruidExpression.fromExpression("human_readable_binary_byte_format(\"b\",\"p\")"), "25 B" ); @@ -2135,75 +2135,75 @@ public void testBinaryByteFormat() * test different precision */ testHelper.testExpression( - SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), + HumanReadableFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), ImmutableList.of( testHelper.makeLiteral(45000), //precision 0 testHelper.makeLiteral(0) ), - DruidExpression.fromExpression("binary_byte_format(45000,0)"), + DruidExpression.fromExpression("human_readable_binary_byte_format(45000,0)"), "44 KiB" ); testHelper.testExpression( - SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), + HumanReadableFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), ImmutableList.of( testHelper.makeLiteral(45000), //precision 1 testHelper.makeLiteral(1) ), - DruidExpression.fromExpression("binary_byte_format(45000,1)"), + DruidExpression.fromExpression("human_readable_binary_byte_format(45000,1)"), "43.9 KiB" ); testHelper.testExpression( - SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), + HumanReadableFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), ImmutableList.of( testHelper.makeLiteral(45000), //precision 2 testHelper.makeLiteral(2) ), - DruidExpression.fromExpression("binary_byte_format(45000,2)"), + DruidExpression.fromExpression("human_readable_binary_byte_format(45000,2)"), "43.95 KiB" ); testHelper.testExpression( - SizeFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), + HumanReadableFormatOperatorConversion.BINARY_BYTE_FORMAT.calciteOperator(), ImmutableList.of( testHelper.makeLiteral(45000), //precision 3 testHelper.makeLiteral(3) ), - DruidExpression.fromExpression("binary_byte_format(45000,3)"), + DruidExpression.fromExpression("human_readable_binary_byte_format(45000,3)"), "43.945 KiB" ); } @Test - public void testDecimalByteFormat() + public void testHumanReadableDecimalByteFormat() { /* * Basic Test */ testHelper.testExpression( - SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), + HumanReadableFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), ImmutableList.of( testHelper.makeLiteral(999) ), - DruidExpression.fromExpression("decimal_byte_format(999)"), + DruidExpression.fromExpression("human_readable_decimal_byte_format(999)"), "999 B" ); testHelper.testExpression( - SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), + HumanReadableFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), ImmutableList.of( testHelper.makeLiteral(1024) ), - DruidExpression.fromExpression("decimal_byte_format(1024)"), + DruidExpression.fromExpression("human_readable_decimal_byte_format(1024)"), "1.02 KB" ); testHelper.testExpression( - SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), + HumanReadableFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), ImmutableList.of( testHelper.makeLiteral(Long.MAX_VALUE) ), - DruidExpression.fromExpression("decimal_byte_format(9223372036854775807)"), + DruidExpression.fromExpression("human_readable_decimal_byte_format(9223372036854775807)"), "9.22 EB" ); @@ -2215,12 +2215,12 @@ public void testDecimalByteFormat() * test input with variable reference */ testHelper.testExpression( - SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), + HumanReadableFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), ImmutableList.of( testHelper.makeInputRef("b"), testHelper.makeInputRef("p") ), - DruidExpression.fromExpression("decimal_byte_format(\"b\",\"p\")"), + DruidExpression.fromExpression("human_readable_decimal_byte_format(\"b\",\"p\")"), "25 B" ); @@ -2228,43 +2228,43 @@ public void testDecimalByteFormat() * test different precision */ testHelper.testExpression( - SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), + HumanReadableFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), ImmutableList.of( testHelper.makeLiteral(45678), //precision 0 testHelper.makeLiteral(0) ), - DruidExpression.fromExpression("decimal_byte_format(45678,0)"), + DruidExpression.fromExpression("human_readable_decimal_byte_format(45678,0)"), "46 KB" ); testHelper.testExpression( - SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), + HumanReadableFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), ImmutableList.of( testHelper.makeLiteral(45678), //precision 1 testHelper.makeLiteral(1) ), - DruidExpression.fromExpression("decimal_byte_format(45678,1)"), + DruidExpression.fromExpression("human_readable_decimal_byte_format(45678,1)"), "45.7 KB" ); testHelper.testExpression( - SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), + HumanReadableFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), ImmutableList.of( testHelper.makeLiteral(45678), //precision 2 testHelper.makeLiteral(2) ), - DruidExpression.fromExpression("decimal_byte_format(45678,2)"), + DruidExpression.fromExpression("human_readable_decimal_byte_format(45678,2)"), "45.68 KB" ); testHelper.testExpression( - SizeFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), + HumanReadableFormatOperatorConversion.DECIMAL_BYTE_FORMAT.calciteOperator(), ImmutableList.of( testHelper.makeLiteral(45678), //precision 3 testHelper.makeLiteral(3) ), - DruidExpression.fromExpression("decimal_byte_format(45678,3)"), + DruidExpression.fromExpression("human_readable_decimal_byte_format(45678,3)"), "45.678 KB" ); } diff --git a/website/.spelling b/website/.spelling index 6a4fea142a67..9e071b3f6e76 100644 --- a/website/.spelling +++ b/website/.spelling @@ -1158,9 +1158,9 @@ value1 value2 valueOf IEC -binary_byte_format -decimal_byte_format -decimal_format +human_readable_binary_byte_format +human_readable_decimal_byte_format +human_readable_decimal_format - ../docs/misc/papers-and-talks.md RADStack - ../docs/operations/api-reference.md From 47d22f9b0f479fa920a2315e1429c890859391c7 Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 5 Mar 2021 18:21:40 +0800 Subject: [PATCH 13/20] fix merge Signed-off-by: frank chen --- .../druid/sql/calcite/CalciteQueryTest.java | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 5d20f0eaa831..5202d5ab5200 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -25,6 +25,7 @@ import junitparams.JUnitParamsRunner; import junitparams.Parameters; import org.apache.calcite.plan.RelOptPlanner; +import org.apache.druid.annotations.UsedByJUnitParamsRunner; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.IAE; @@ -16463,11 +16464,7 @@ public void testTimeStampAddConversion() throws Exception .dataSource(CalciteTests.DATASOURCE1) .intervals(querySegmentSpec(Filtration.eternity())) .virtualColumns( - expressionVirtualColumn( - "v0", - "timestamp_shift(\"__time\",'P1M',1,'UTC')", - ValueType.LONG - ) + expressionVirtualColumn("v0", "timestamp_shift(\"__time\",'P1M',1,'UTC')", ValueType.LONG) ) .columns("v0") .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) @@ -16491,11 +16488,7 @@ public void testTimeStampAddConversion() throws Exception .dataSource(CalciteTests.DATASOURCE1) .intervals(querySegmentSpec(Filtration.eternity())) .virtualColumns( - expressionVirtualColumn( - "v0", - "timestamp_shift(\"__time\",concat('P', (1 * \"cnt\"), 'M'),1,'UTC')", - ValueType.LONG - ) + expressionVirtualColumn("v0", "timestamp_shift(\"__time\",concat('P', (1 * \"cnt\"), 'M'),1,'UTC')", ValueType.LONG) ) .columns("v0") .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) @@ -16510,6 +16503,7 @@ public void testTimeStampAddConversion() throws Exception ); } + @Test public void testGroupingSetsWithLimit() throws Exception { // Cannot vectorize due to virtual columns. @@ -16576,6 +16570,7 @@ public void testGroupingSetsWithLimit() throws Exception ); } + @Test public void testGroupingSetsWithLimitOrderByGran() throws Exception { // Cannot vectorize due to virtual columns. From ad7d8312c46c22db73dada4f5e14caf894e976c0 Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 5 Mar 2021 18:29:28 +0800 Subject: [PATCH 14/20] no need to configure NullHandling explicitly for tests Signed-off-by: frank chen --- .../druid/common/config/NullHandling.java | 9 ------- .../apache/druid/math/expr/FunctionTest.java | 25 ------------------- 2 files changed, 34 deletions(-) diff --git a/core/src/main/java/org/apache/druid/common/config/NullHandling.java b/core/src/main/java/org/apache/druid/common/config/NullHandling.java index 152ad7dd6a0a..51cb2658628b 100644 --- a/core/src/main/java/org/apache/druid/common/config/NullHandling.java +++ b/core/src/main/java/org/apache/druid/common/config/NullHandling.java @@ -61,15 +61,6 @@ public static void initializeForTests() INSTANCE = new NullValueHandlingConfig(null); } - /** - * This function should only be called by unit tests - */ - @VisibleForTesting - public static void updateForTests(boolean useDefaultValuesForNull) - { - INSTANCE = new NullValueHandlingConfig(useDefaultValuesForNull); - } - /** * whether nulls should be replaced with default value. */ diff --git a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java index 354bffd953be..2a5bcb59435e 100644 --- a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java @@ -696,31 +696,6 @@ public void testSizeFormatInvalidArgumentSize() .eval(bindings); } - @Test - public void testSizeFormatWithNoDefaultValueForNull() - { - NullHandling.updateForTests(false); - { - // - // normal cases - // y is not null, the function returns correctly - // - assertExpr("human_readable_binary_byte_format(y)", "2 B"); - assertExpr("human_readable_decimal_byte_format(y)", "2 B"); - assertExpr("human_readable_decimal_format(y)", "2"); - - // - // since 'druid.generic.useDefaultValueForNull' has been disabled above, - // the 'nonexist' below returns null, and the function calls also return null - // - assertExpr("human_readable_binary_byte_format(nonexist)", null); - assertExpr("human_readable_decimal_byte_format(nonexist)", null); - assertExpr("human_readable_decimal_format(nonexist)", null); - - } - NullHandling.updateForTests(true); - } - @Test public void testBitwise() { From c3dbb1e5e0d5fabfdb116de89f0657a772605d3d Mon Sep 17 00:00:00 2001 From: frank chen Date: Mon, 8 Mar 2021 17:32:25 +0800 Subject: [PATCH 15/20] fix tests in SQL-Compatible mode Signed-off-by: frank chen --- .../test/java/org/apache/druid/math/expr/FunctionTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java index 2a5bcb59435e..ed1bd8165311 100644 --- a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java @@ -571,7 +571,7 @@ public void testSizeFormatWithDifferentPrecision() public void testSizeFormatWithEdgeCases() { //a nonexist value is null which is treated as 0 - assertExpr("human_readable_binary_byte_format(nonexist)", "0 B"); + assertExpr("human_readable_binary_byte_format(nonexist)", NullHandling.sqlCompatible() ? null : "0 B"); //f = 12.34 assertExpr("human_readable_binary_byte_format(f)", "12 B"); @@ -603,8 +603,9 @@ public void testSizeForatInvalidArgumentType() Parser.parse("human_readable_binary_byte_format(x)", ExprMacroTable.nil()) .eval(bindings); - //must not go to here - Assert.assertTrue(false); + // for sqlCompatible, function above returns null and goes here + // but for non-sqlCompatible, it must not go to here + Assert.assertTrue(NullHandling.sqlCompatible() ? true : false); } catch (IAE e) { Assert.assertEquals("Function[human_readable_binary_byte_format] needs a number as its first argument", e.getMessage()); From fc7cc9bd95aeaab565ce1ad9b05b41c34935e7de Mon Sep 17 00:00:00 2001 From: frank chen Date: Wed, 23 Jun 2021 16:47:05 +0800 Subject: [PATCH 16/20] Resolve review comments --- .../builtin/HumanReadableFormatOperatorConversion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/HumanReadableFormatOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/HumanReadableFormatOperatorConversion.java index 4002f625b52a..f9f06e10f3a1 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/HumanReadableFormatOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/HumanReadableFormatOperatorConversion.java @@ -51,7 +51,7 @@ private HumanReadableFormatOperatorConversion(String name) .operatorBuilder(StringUtils.toUpperCase(name)) .operandTypeChecker(new HumanReadableFormatOperandTypeChecker()) .functionCategory(SqlFunctionCategory.STRING) - .returnTypeNonNull(SqlTypeName.VARCHAR) + .returnTypeCascadeNullable(SqlTypeName.VARCHAR) .build(); this.name = name; From fa1f05795e2f829f977ed89cf10627f2727552d7 Mon Sep 17 00:00:00 2001 From: frank chen Date: Wed, 23 Jun 2021 16:49:46 +0800 Subject: [PATCH 17/20] Update SQL test case to check null handling --- .../druid/sql/calcite/CalciteQueryTest.java | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 4bbedf75861e..b668219ae247 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -17612,17 +17612,24 @@ public void testExpressionCounts() throws Exception ); } + /** + * see {@link org.apache.druid.sql.calcite.util.CalciteTests#RAW_ROWS1_WITH_NUMERIC_DIMS} for the input data source of this test + */ @Test public void testHumanReadableFormatFunction() throws Exception { + // For the row where dim1 = '1', m1 = 4.0 and l1 is null testQuery( "SELECT m1, " + "HUMAN_READABLE_BINARY_BYTE_FORMAT(45678)," + "HUMAN_READABLE_BINARY_BYTE_FORMAT(m1*12345)," + "HUMAN_READABLE_BINARY_BYTE_FORMAT(m1*12345, 0), " + "HUMAN_READABLE_DECIMAL_BYTE_FORMAT(m1*12345), " - + "HUMAN_READABLE_DECIMAL_FORMAT(m1*12345) " - + "FROM numfoo WHERE f1 = 0.1 LIMIT 1", + + "HUMAN_READABLE_DECIMAL_FORMAT(m1*12345), " + + "HUMAN_READABLE_BINARY_BYTE_FORMAT(l1)," + + "HUMAN_READABLE_DECIMAL_BYTE_FORMAT(l1), " + + "HUMAN_READABLE_DECIMAL_FORMAT(l1) " + + "FROM numfoo WHERE dim1 = '1' LIMIT 1", ImmutableList.of( newScanQueryBuilder() .dataSource(CalciteTests.DATASOURCE3) @@ -17635,21 +17642,28 @@ public void testHumanReadableFormatFunction() throws Exception expressionVirtualColumn("v1", "human_readable_binary_byte_format((\"m1\" * 12345))", ValueType.STRING), expressionVirtualColumn("v2", "human_readable_binary_byte_format((\"m1\" * 12345),0)", ValueType.STRING), expressionVirtualColumn("v3", "human_readable_decimal_byte_format((\"m1\" * 12345))", ValueType.STRING), - expressionVirtualColumn("v4", "human_readable_decimal_format((\"m1\" * 12345))", ValueType.STRING)) - .columns("m1", "v0", "v1", "v2", "v3", "v4") - .filters(selector("f1", "0.1", null)) + expressionVirtualColumn("v4", "human_readable_decimal_format((\"m1\" * 12345))", ValueType.STRING), + expressionVirtualColumn("v5", "human_readable_binary_byte_format(\"l1\")", ValueType.STRING), + expressionVirtualColumn("v6", "human_readable_decimal_byte_format(\"l1\")", ValueType.STRING), + expressionVirtualColumn("v7", "human_readable_decimal_format(\"l1\")", ValueType.STRING) + ) + .columns("m1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7") + .filters(selector("dim1", "1", null)) .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) .limit(1) .context(QUERY_CONTEXT_DEFAULT) .build() ), ImmutableList.of( - new Object[]{(float) 2.0, + new Object[]{(float) 4.0, "44.61 KiB", // 45678 / 1024 - "24.11 KiB", // = m1(2.0) * 12345 / 1024 - "24 KiB", // = m1(2.0) * 12345 / 1024, precision = 0 - "24.69 KB", // decimal byte format, m1(2.0) * 12345 / 1000, - "24.69 K" // decimal format, m1(2.0) * 12345 / 1000, + "48.22 KiB", // = m1(4.0) * 12345 / 1024 + "48 KiB", // = m1(4.0) * 12345 / 1024, precision = 0 + "49.38 KB", // decimal byte format, m1(4.0) * 12345 / 1000, + "49.38 K", // decimal format, m1(4.0) * 12345 / 1000, + NullHandling.replaceWithDefault() ? "0 B" : null, + NullHandling.replaceWithDefault() ? "0 B" : null, + NullHandling.replaceWithDefault() ? "0" : null } ) ); From 12edaabc96f385a2cd400f00ca71af74d0c7e160 Mon Sep 17 00:00:00 2001 From: frank chen Date: Thu, 24 Jun 2021 09:54:35 +0800 Subject: [PATCH 18/20] Fix intellij inspections --- .../java/org/apache/druid/sql/calcite/CalciteQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index b668219ae247..e4ef08d1e26f 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -17613,7 +17613,7 @@ public void testExpressionCounts() throws Exception } /** - * see {@link org.apache.druid.sql.calcite.util.CalciteTests#RAW_ROWS1_WITH_NUMERIC_DIMS} for the input data source of this test + * see {@link CalciteTests#RAW_ROWS1_WITH_NUMERIC_DIMS} for the input data source of this test */ @Test public void testHumanReadableFormatFunction() throws Exception From 30eb6eda3c521dc65bc36ef5f607332cee074bd2 Mon Sep 17 00:00:00 2001 From: frank chen Date: Tue, 29 Jun 2021 17:08:58 +0800 Subject: [PATCH 19/20] Add more examples --- docs/misc/math-expr.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/misc/math-expr.md b/docs/misc/math-expr.md index 6207b4f0045c..d0d94211df3a 100644 --- a/docs/misc/math-expr.md +++ b/docs/misc/math-expr.md @@ -264,6 +264,6 @@ supported features: | function | description | | --- | --- | -| human_readable_binary_byte_format(value[, precision]) | Format a number in human-readable [IEC](https://en.wikipedia.org/wiki/Binary_prefix) format. For example, human_readable_binary_byte_format(1048576) returns `1.00 MiB`. `precision` must be in the range of [0,3] (default: 2). | -| human_readable_decimal_byte_format(value[, precision]) | Format a number in human-readable [SI](https://en.wikipedia.org/wiki/Binary_prefix) format. human_readable_decimal_byte_format(1048576) returns `1.04 MB`. `precision` must be in the range of [0,3] (default: 2). `precision` must be in the range of [0,3] (default: 2). | -| human_readable_decimal_format(value[, precision]) | Format a number in human-readable SI format. For example, HUMAN_READABLE_DECIMAL_BYTE_FORMAT(1048576) returns `1.04 M`. `precision` must be in the range of [0,3] (default: 2). | +| human_readable_binary_byte_format(value[, precision]) | Format a number in human-readable [IEC](https://en.wikipedia.org/wiki/Binary_prefix) format. `precision` must be in the range of [0,3] (default: 2). For example:
  • human_readable_binary_byte_format(1048576) returns `1.00 MiB`
  • human_readable_binary_byte_format(1048576, 1) returns `1.000 MiB`
    • | +| human_readable_decimal_byte_format(value[, precision]) | Format a number in human-readable [SI](https://en.wikipedia.org/wiki/Binary_prefix) format. `precision` must be in the range of [0,3] (default: 2). For example:
  • human_readable_decimal_byte_format(1000000) returns `1.00 MB`
  • human_readable_decimal_byte_format(1000000, 3) returns `1.000 MB`
    • | +| human_readable_decimal_format(value[, precision]) | Format a number in human-readable SI format. `precision` must be in the range of [0,3] (default: 2). For example:
  • human_readable_decimal_format(1000000) returns `1.00 M`
  • human_readable_decimal_format(1000000, 3) returns `1.000 M`
    • | From e773dac201b3c56834ae31b88cc336b3cbf500a0 Mon Sep 17 00:00:00 2001 From: frank chen Date: Tue, 29 Jun 2021 17:15:33 +0800 Subject: [PATCH 20/20] Fix example --- docs/misc/math-expr.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/misc/math-expr.md b/docs/misc/math-expr.md index d0d94211df3a..9afccdf77a63 100644 --- a/docs/misc/math-expr.md +++ b/docs/misc/math-expr.md @@ -264,6 +264,6 @@ supported features: | function | description | | --- | --- | -| human_readable_binary_byte_format(value[, precision]) | Format a number in human-readable [IEC](https://en.wikipedia.org/wiki/Binary_prefix) format. `precision` must be in the range of [0,3] (default: 2). For example:
  • human_readable_binary_byte_format(1048576) returns `1.00 MiB`
  • human_readable_binary_byte_format(1048576, 1) returns `1.000 MiB`
    • | +| human_readable_binary_byte_format(value[, precision]) | Format a number in human-readable [IEC](https://en.wikipedia.org/wiki/Binary_prefix) format. `precision` must be in the range of [0,3] (default: 2). For example:
  • human_readable_binary_byte_format(1048576) returns `1.00 MiB`
  • human_readable_binary_byte_format(1048576, 3) returns `1.000 MiB`
    • | | human_readable_decimal_byte_format(value[, precision]) | Format a number in human-readable [SI](https://en.wikipedia.org/wiki/Binary_prefix) format. `precision` must be in the range of [0,3] (default: 2). For example:
  • human_readable_decimal_byte_format(1000000) returns `1.00 MB`
  • human_readable_decimal_byte_format(1000000, 3) returns `1.000 MB`
    • | | human_readable_decimal_format(value[, precision]) | Format a number in human-readable SI format. `precision` must be in the range of [0,3] (default: 2). For example:
  • human_readable_decimal_format(1000000) returns `1.00 M`
  • human_readable_decimal_format(1000000, 3) returns `1.000 M`
    • |