diff --git a/docs/content/misc/math-expr.md b/docs/content/misc/math-expr.md index 3fab03aa5abd..0aa58b37cf6d 100644 --- a/docs/content/misc/math-expr.md +++ b/docs/content/misc/math-expr.md @@ -195,3 +195,17 @@ See javadoc of java.lang.Math for detailed explanation for each function. | cartesian_fold(lambda,arr1,arr2,...) | folds a multi argument lambda across the cartesian product of all input arrays. The first arguments of the lambda is the array element and the last is the accumulator, returning a single accumulated value. | | any(lambda,arr) | returns 1 if any element in the array matches the lambda expression, else 0 | | all(lambda,arr) | returns 1 if all elements in the array matches the lambda expression, else 0 | + + +## IP Address Functions + +For the IPv4 address functions, the `address` argument can either be an IPv4 dotted-decimal string +(e.g., "192.168.0.1") or an IP address represented as a long (e.g., 3232235521). The `subnet` +argument should be a string formatted as an IPv4 address subnet in CIDR notation (e.g., +"192.168.0.0/16"). + +| function | description | +| --- | --- | +| ipv4_match(address, subnet) | Returns 1 if the `address` belongs to the `subnet` literal, else 0. If `address` is not a valid IPv4 address, then 0 is returned. This function is more efficient if `address` is a long instead of a string.| +| ipv4_parse(address) | Parses `address` into an IPv4 address stored as a long. If `address` is a long that is a valid IPv4 address, then it is passed through. Returns null if `address` cannot be represented as an IPv4 address. | +| ipv4_stringify(address) | Converts `address` into an IPv4 address dotted-decimal string. If `address` is a string that is a valid IPv4 address, then it is passed through. Returns null if `address` cannot be represented as an IPv4 address.| diff --git a/licenses.yaml b/licenses.yaml index 15458fcdc65c..10986dc9a2b2 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -362,6 +362,16 @@ libraries: --- +name: Apache Commons Net +license_category: binary +module: java-core +license_name: Apache License version 2.0 +version: 3.6 +libraries: + - commons-net: commons-net + +--- + name: Apache Commons Pool license_category: binary module: java-core diff --git a/pom.xml b/pom.xml index 1a802c9f5e15..dca5d24addda 100644 --- a/pom.xml +++ b/pom.xml @@ -212,6 +212,11 @@ commons-lang 2.6 + + commons-net + commons-net + 3.6 + com.amazonaws aws-java-sdk-ec2 diff --git a/processing/pom.xml b/processing/pom.xml index 53c1d23f3ed1..5e29987724bf 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -79,6 +79,10 @@ commons-io commons-io + + commons-net + commons-net + com.google.errorprone error_prone_annotations diff --git a/processing/src/main/java/org/apache/druid/query/expression/ExprUtils.java b/processing/src/main/java/org/apache/druid/query/expression/ExprUtils.java index 1b9cb9df16ab..75fe2740057c 100644 --- a/processing/src/main/java/org/apache/druid/query/expression/ExprUtils.java +++ b/processing/src/main/java/org/apache/druid/query/expression/ExprUtils.java @@ -19,6 +19,7 @@ package org.apache.druid.query.expression; +import com.google.common.base.Preconditions; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.IAE; @@ -41,7 +42,7 @@ public static Expr.ObjectBinding nilBindings() return NIL_BINDINGS; } - public static DateTimeZone toTimeZone(final Expr timeZoneArg) + static DateTimeZone toTimeZone(final Expr timeZoneArg) { if (!timeZoneArg.isLiteral()) { throw new IAE("Time zone must be a literal"); @@ -51,7 +52,7 @@ public static DateTimeZone toTimeZone(final Expr timeZoneArg) return literalValue == null ? DateTimeZone.UTC : DateTimes.inferTzFromString((String) literalValue); } - public static PeriodGranularity toPeriodGranularity( + static PeriodGranularity toPeriodGranularity( final Expr periodArg, @Nullable final Expr originArg, @Nullable final Expr timeZoneArg, @@ -87,4 +88,14 @@ public static PeriodGranularity toPeriodGranularity( return new PeriodGranularity(period, origin, timeZone); } + static String createErrMsg(String functionName, String msg) + { + String prefix = "Function[" + functionName + "] "; + return prefix + msg; + } + + static void checkLiteralArgument(String functionName, Expr arg, String argName) + { + Preconditions.checkArgument(arg.isLiteral(), createErrMsg(functionName, argName + " arg must be a literal")); + } } diff --git a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java new file mode 100644 index 000000000000..4d87b38b50f4 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import com.google.common.net.InetAddresses; + +import javax.annotation.Nullable; +import java.net.Inet4Address; +import java.net.InetAddress; +import java.util.regex.Pattern; + +class IPv4AddressExprUtils +{ + private static final Pattern IPV4_PATTERN = Pattern.compile( + "^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$" + ); + + /** + * @return True if argument cannot be represented by an unsigned integer (4 bytes), else false + */ + static boolean overflowsUnsignedInt(long value) + { + return value < 0L || 0xff_ff_ff_ffL < value; + } + + /** + * @return True if argument is a valid IPv4 address dotted-decimal string + */ + static boolean isValidAddress(@Nullable String string) + { + return string != null && IPV4_PATTERN.matcher(string).matches(); + } + + @Nullable + static Inet4Address parse(@Nullable String string) + { + // Explicitly check for valid address to avoid overhead of InetAddresses#forString() potentially + // throwing IllegalArgumentException + if (isValidAddress(string)) { + // Do not use java.lang.InetAddress#getByName() as it may do DNS lookups + InetAddress address = InetAddresses.forString(string); + if (address instanceof Inet4Address) { + return (Inet4Address) address; + } + } + return null; + } + + static Inet4Address parse(int value) + { + return InetAddresses.fromInteger(value); + } + + /** + * @return IPv4 address dotted-decimal notated string + */ + static String toString(Inet4Address address) + { + return address.getHostAddress(); + } + + static long toLong(Inet4Address address) + { + int value = InetAddresses.coerceToInteger(address); + return Integer.toUnsignedLong(value); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java new file mode 100644 index 000000000000..6cc94ac61133 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import org.apache.commons.net.util.SubnetUtils; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.math.expr.ExprType; + +import javax.annotation.Nonnull; +import java.util.List; + +/** + *
+ * Implements an expression that checks if an IPv4 address belongs to a particular subnet.
+ *
+ * Expression signatures:
+ * - long ipv4_match(string address, string subnet)
+ * - long ipv4_match(long address, string subnet)
+ *
+ * Valid "address" argument formats are:
+ * - unsigned int long (e.g., 3232235521)
+ * - IPv4 address dotted-decimal string (e.g., "198.168.0.1")
+ *
+ * The argument format for the "subnet" argument should be a literal in CIDR notation
+ * (e.g., "198.168.0.0/16").
+ *
+ * If the "address" argument does not represent an IPv4 address then false is returned.
+ * 
+ * + * @see IPv4AddressParseExprMacro + * @see IPv4AddressStringifyExprMacro + */ +public class IPv4AddressMatchExprMacro implements ExprMacroTable.ExprMacro +{ + public static final String NAME = "ipv4_match"; + private static final int ARG_SUBNET = 1; + + @Override + public String name() + { + return NAME; + } + + @Override + public Expr apply(final List args) + { + if (args.size() != 2) { + throw new IAE(ExprUtils.createErrMsg(name(), "must have 2 arguments")); + } + + SubnetUtils.SubnetInfo subnetInfo = getSubnetInfo(args); + Expr arg = args.get(0); + + class IPv4AddressMatchExpr extends ExprMacroTable.BaseScalarUnivariateMacroFunctionExpr + { + private final SubnetUtils.SubnetInfo subnetInfo; + + private IPv4AddressMatchExpr(Expr arg, SubnetUtils.SubnetInfo subnetInfo) + { + super(arg); + this.subnetInfo = subnetInfo; + } + + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + ExprEval eval = arg.eval(bindings); + boolean match; + switch (eval.type()) { + case STRING: + match = isStringMatch(eval.asString()); + break; + case LONG: + match = !eval.isNumericNull() && isLongMatch(eval.asLong()); + break; + default: + match = false; + } + return ExprEval.of(match, ExprType.LONG); + } + + private boolean isStringMatch(String stringValue) + { + return IPv4AddressExprUtils.isValidAddress(stringValue) && subnetInfo.isInRange(stringValue); + } + + private boolean isLongMatch(long longValue) + { + return !IPv4AddressExprUtils.overflowsUnsignedInt(longValue) && subnetInfo.isInRange((int) longValue); + } + + @Override + public Expr visit(Shuttle shuttle) + { + Expr newArg = arg.visit(shuttle); + return shuttle.visit(new IPv4AddressMatchExpr(newArg, subnetInfo)); + } + } + + return new IPv4AddressMatchExpr(arg, subnetInfo); + } + + private SubnetUtils.SubnetInfo getSubnetInfo(List args) + { + String subnetArgName = "subnet"; + Expr arg = args.get(ARG_SUBNET); + ExprUtils.checkLiteralArgument(name(), arg, subnetArgName); + String subnet = (String) arg.getLiteralValue(); + + SubnetUtils subnetUtils; + try { + subnetUtils = new SubnetUtils(subnet); + } + catch (IllegalArgumentException e) { + throw new IAE(e, ExprUtils.createErrMsg(name(), subnetArgName + " arg has an invalid format: " + subnet)); + } + subnetUtils.setInclusiveHostCount(true); // make network and broadcast addresses match + + return subnetUtils.getInfo(); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressParseExprMacro.java b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressParseExprMacro.java new file mode 100644 index 000000000000..569c037148ee --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressParseExprMacro.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.math.expr.ExprMacroTable; + +import javax.annotation.Nonnull; +import java.net.Inet4Address; +import java.util.List; + +/** + *
+ * Implements an expression that parses a string or long into an IPv4 address stored (as an unsigned
+ * int) in a long.
+ *
+ * Expression signatures:
+ * - long ipv4_parse(string)
+ * - long ipv4_parse(long)
+ *
+ * String arguments should be formatted as a dotted-decimal.
+ * Long arguments that can be represented as an IPv4 address are passed through.
+ * Invalid arguments return null.
+ * 
+ * + * @see IPv4AddressStringifyExprMacro + * @see IPv4AddressMatchExprMacro + */ +public class IPv4AddressParseExprMacro implements ExprMacroTable.ExprMacro +{ + public static final String NAME = "ipv4_parse"; + + @Override + public String name() + { + return NAME; + } + + @Override + public Expr apply(final List args) + { + if (args.size() != 1) { + throw new IAE(ExprUtils.createErrMsg(name(), "must have 1 argument")); + } + + Expr arg = args.get(0); + + class IPv4AddressParseExpr extends ExprMacroTable.BaseScalarUnivariateMacroFunctionExpr + { + private IPv4AddressParseExpr(Expr arg) + { + super(arg); + } + + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + ExprEval eval = arg.eval(bindings); + switch (eval.type()) { + case STRING: + return evalAsString(eval); + case LONG: + return evalAsLong(eval); + default: + return ExprEval.ofLong(null); + } + } + + @Override + public Expr visit(Shuttle shuttle) + { + Expr newArg = arg.visit(shuttle); + return shuttle.visit(new IPv4AddressParseExpr(newArg)); + } + } + + return new IPv4AddressParseExpr(arg); + } + + private static ExprEval evalAsString(ExprEval eval) + { + Inet4Address address = IPv4AddressExprUtils.parse(eval.asString()); + Long value = address == null ? null : IPv4AddressExprUtils.toLong(address); + return ExprEval.ofLong(value); + } + + private static ExprEval evalAsLong(ExprEval eval) + { + if (eval.isNumericNull() || !IPv4AddressExprUtils.overflowsUnsignedInt(eval.asLong())) { + return eval; + } + return ExprEval.ofLong(null); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacro.java b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacro.java new file mode 100644 index 000000000000..d9afa5872777 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacro.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.math.expr.ExprMacroTable; + +import javax.annotation.Nonnull; +import java.net.Inet4Address; +import java.util.List; + +/** + *
+ * Implements an expression that converts a long or a string into an IPv4 address dotted-decimal string.
+ *
+ * Expression signatures:
+ * - string ipv4_stringify(long)
+ * - string ipv4_stringify(string)
+ *
+ * Long arguments that can be represented as an IPv4 address are converted to a dotted-decimal string.
+ * String arguments that are dotted-decimal IPv4 addresses are passed through.
+ * Invalid arguments return null.
+ * 
+ * + * @see IPv4AddressParseExprMacro + * @see IPv4AddressMatchExprMacro + */ +public class IPv4AddressStringifyExprMacro implements ExprMacroTable.ExprMacro +{ + public static final String NAME = "ipv4_stringify"; + + @Override + public String name() + { + return NAME; + } + + @Override + public Expr apply(final List args) + { + if (args.size() != 1) { + throw new IAE(ExprUtils.createErrMsg(name(), "must have 1 argument")); + } + + Expr arg = args.get(0); + + class IPv4AddressStringifyExpr extends ExprMacroTable.BaseScalarUnivariateMacroFunctionExpr + { + private IPv4AddressStringifyExpr(Expr arg) + { + super(arg); + } + + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + ExprEval eval = arg.eval(bindings); + switch (eval.type()) { + case STRING: + return evalAsString(eval); + case LONG: + return evalAsLong(eval); + default: + return ExprEval.of(null); + } + } + + @Override + public Expr visit(Shuttle shuttle) + { + Expr newArg = arg.visit(shuttle); + return shuttle.visit(new IPv4AddressStringifyExpr(newArg)); + } + } + + return new IPv4AddressStringifyExpr(arg); + } + + private static ExprEval evalAsString(ExprEval eval) + { + if (IPv4AddressExprUtils.isValidAddress(eval.asString())) { + return eval; + } + return ExprEval.of(null); + } + + private static ExprEval evalAsLong(ExprEval eval) + { + if (eval.isNumericNull()) { + return ExprEval.of(null); + } + + long longValue = eval.asLong(); + if (IPv4AddressExprUtils.overflowsUnsignedInt(longValue)) { + return ExprEval.of(null); + } + + Inet4Address address = IPv4AddressExprUtils.parse((int) longValue); + return ExprEval.of(IPv4AddressExprUtils.toString(address)); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressExprUtilsTest.java b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressExprUtilsTest.java new file mode 100644 index 000000000000..a13ee9b94dab --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressExprUtilsTest.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import org.junit.Assert; +import org.junit.Test; + +import java.net.Inet4Address; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.List; + +public class IPv4AddressExprUtilsTest +{ + private static final List VALID_IPV4_ADDRESSES = Arrays.asList( + "192.168.0.1", + "0.0.0.0", + "255.255.255.255", + "255.0.0.0", + "0.255.0.0", + "0.0.255.0", + "0.0.0.255" + ); + private static final List INVALID_IPV4_ADDRESSES = Arrays.asList( + "druid.apache.org", // no octets are numbers + "a.b.c.d", // no octets are numbers + "abc.def.ghi.jkl", // no octets are numbers + "1..3.4", // missing octet + "1.2..4", // missing octet + "1.2.3..", // missing octet + "1", // missing octets + "1.2", // missing octets + "1.2.3", // missing octet + "1.2.3.4.5", // too many octets + "256.0.0.0", // first octet too large + "0.265.0.0", // second octet too large + "0.0.266.0", // third octet too large + "0.0.0.355", // fourth octet too large + "a.2.3.4", // first octet not number + "1.a.3.4", // second octet not number + "1.2.c.4", // third octet not number + "1.2.3.d" // fourth octet not number + ); + private static final String IPV6_MAPPED = "::ffff:192.168.0.1"; + private static final String IPV6_COMPATIBLE = "::192.168.0.1"; + + @Test + public void testOverflowsUnsignedIntTooLow() + { + Assert.assertTrue(IPv4AddressExprUtils.overflowsUnsignedInt(-1L)); + } + + @Test + public void testOverflowsUnsignedIntLowest() + { + Assert.assertFalse(IPv4AddressExprUtils.overflowsUnsignedInt(0L)); + } + + @Test + public void testOverflowsUnsignedIntMiddle() + { + Assert.assertFalse(IPv4AddressExprUtils.overflowsUnsignedInt(0xff_ffL)); + } + + @Test + public void testOverflowsUnsignedIntHighest() + { + Assert.assertFalse(IPv4AddressExprUtils.overflowsUnsignedInt(0xff_ff_ff_ffL)); + } + + @Test + public void testOverflowsUnsignedIntTooHigh() + { + Assert.assertTrue(IPv4AddressExprUtils.overflowsUnsignedInt(0x1_00_00_00_00L)); + } + + @Test + public void testIsValidAddressNull() + { + Assert.assertFalse(IPv4AddressExprUtils.isValidAddress(null)); + } + + @Test + public void testIsValidAddressIPv4() + { + for (String address : VALID_IPV4_ADDRESSES) { + Assert.assertTrue(getErrMsg(address), IPv4AddressExprUtils.isValidAddress(address)); + } + } + + @Test + public void testIsValidAddressIPv6Mapped() + { + Assert.assertFalse(IPv4AddressExprUtils.isValidAddress(IPV6_MAPPED)); + } + + @Test + public void testIsValidAddressIPv6Compatible() + { + Assert.assertFalse(IPv4AddressExprUtils.isValidAddress(IPV6_COMPATIBLE)); + } + + @Test + public void testIsValidAddressNotIpAddress() + { + for (String address : INVALID_IPV4_ADDRESSES) { + Assert.assertFalse(getErrMsg(address), IPv4AddressExprUtils.isValidAddress(address)); + } + } + + @Test + public void testParseNull() + { + Assert.assertNull(IPv4AddressExprUtils.parse(null)); + } + + @Test + public void testParseIPv4() + { + for (String string : VALID_IPV4_ADDRESSES) { + String errMsg = getErrMsg(string); + Inet4Address address = IPv4AddressExprUtils.parse(string); + Assert.assertNotNull(errMsg, address); + Assert.assertEquals(errMsg, string, address.getHostAddress()); + } + } + + @Test + public void testParseIPv6Mapped() + { + Assert.assertNull(IPv4AddressExprUtils.parse(IPV6_MAPPED)); + } + + @Test + public void testParseIPv6Compatible() + { + Assert.assertNull(IPv4AddressExprUtils.parse(IPV6_COMPATIBLE)); + } + + @Test + public void testParseNotIpAddress() + { + for (String address : INVALID_IPV4_ADDRESSES) { + Assert.assertNull(getErrMsg(address), IPv4AddressExprUtils.parse(address)); + } + } + + @Test + public void testParseInt() + { + Inet4Address address = IPv4AddressExprUtils.parse((int) 0xC0A80001L); + Assert.assertArrayEquals(new byte[]{(byte) 0xC0, (byte) 0xA8, 0x00, 0x01}, address.getAddress()); + } + + @Test + public void testToString() throws UnknownHostException + { + byte[] bytes = new byte[]{(byte) 192, (byte) 168, 0, 1}; + InetAddress address = InetAddress.getByAddress(bytes); + Assert.assertEquals("192.168.0.1", IPv4AddressExprUtils.toString((Inet4Address) address)); + } + + @Test + public void testToLong() throws UnknownHostException + { + byte[] bytes = new byte[]{(byte) 0xC0, (byte) 0xA8, 0x00, 0x01}; + InetAddress address = InetAddress.getByAddress(bytes); + Assert.assertEquals(0xC0A80001L, IPv4AddressExprUtils.toLong((Inet4Address) address)); + } + + private String getErrMsg(String msg) + { + String prefix = "Failed: "; + return prefix + msg; + } +} diff --git a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacroTest.java b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacroTest.java new file mode 100644 index 000000000000..c4f7d9c12a2c --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacroTest.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.math.expr.ExprMacroTable; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +public class IPv4AddressMatchExprMacroTest extends MacroTestBase +{ + private static final Expr IPV4 = ExprEval.of("192.168.0.1").toExpr(); + private static final Expr IPV4_LONG = ExprEval.of(3232235521L).toExpr(); + private static final Expr IPV4_UINT = ExprEval.of("3232235521").toExpr(); + private static final Expr IPV4_NETWORK = ExprEval.of("192.168.0.0").toExpr(); + private static final Expr IPV4_BROADCAST = ExprEval.of("192.168.255.255").toExpr(); + private static final Expr IPV6_COMPATIBLE = ExprEval.of("::192.168.0.1").toExpr(); + private static final Expr IPV6_MAPPED = ExprEval.of("::ffff:192.168.0.1").toExpr(); + private static final Expr SUBNET_192_168 = ExprEval.of("192.168.0.0/16").toExpr(); + private static final Expr SUBNET_10 = ExprEval.of("10.0.0.0/8").toExpr(); + private static final Expr NOT_LITERAL = new NotLiteralExpr(null); + + private IPv4AddressMatchExprMacro target; + + @Before + public void setUp() + { + target = new IPv4AddressMatchExprMacro(); + } + + @Test + public void testTooFewArgs() + { + expectException(IllegalArgumentException.class, "must have 2 arguments"); + + target.apply(Collections.emptyList()); + } + + @Test + public void testTooManyArgs() + { + expectException(IllegalArgumentException.class, "must have 2 arguments"); + + target.apply(Arrays.asList(IPV4, SUBNET_192_168, NOT_LITERAL)); + } + + @Test + public void testSubnetArgNotLiteral() + { + expectException(IllegalArgumentException.class, "subnet arg must be a literal"); + + target.apply(Arrays.asList(IPV4, NOT_LITERAL)); + } + + @Test + public void testSubnetArgInvalid() + { + expectException(IllegalArgumentException.class, "subnet arg has an invalid format"); + + Expr invalidSubnet = ExprEval.of("192.168.0.1/invalid").toExpr(); + target.apply(Arrays.asList(IPV4, invalidSubnet)); + } + + @Test + public void testNullStringArg() + { + Expr nullString = ExprEval.of(null).toExpr(); + Assert.assertFalse(eval(nullString, SUBNET_192_168)); + } + + @Test + public void testNullLongArg() + { + Expr nullLong = ExprEval.ofLong(null).toExpr(); + Assert.assertFalse(eval(nullLong, SUBNET_192_168)); + } + + @Test + public void testInvalidArgType() + { + Expr longArray = ExprEval.ofLongArray(new Long[]{1L, 2L}).toExpr(); + Assert.assertFalse(eval(longArray, SUBNET_192_168)); + } + + @Test + public void testMatchingStringArgIPv4() + { + Assert.assertTrue(eval(IPV4, SUBNET_192_168)); + } + + @Test + public void testNotMatchingStringArgIPv4() + { + Assert.assertFalse(eval(IPV4, SUBNET_10)); + } + + @Test + public void testMatchingStringArgIPv6Mapped() + { + Assert.assertFalse(eval(IPV6_MAPPED, SUBNET_192_168)); + } + + @Test + public void testNotMatchingStringArgIPv6Mapped() + { + Assert.assertFalse(eval(IPV6_MAPPED, SUBNET_10)); + } + + @Test + public void testMatchingStringArgIPv6Compatible() + { + Assert.assertFalse(eval(IPV6_COMPATIBLE, SUBNET_192_168)); + } + + @Test + public void testNotMatchingStringArgIPv6Compatible() + { + Assert.assertFalse(eval(IPV6_COMPATIBLE, SUBNET_10)); + } + + @Test + public void testNotIpAddress() + { + Expr notIpAddress = ExprEval.of("druid.apache.org").toExpr(); + Assert.assertFalse(eval(notIpAddress, SUBNET_192_168)); + } + + @Test + public void testMatchingLongArg() + { + Assert.assertTrue(eval(IPV4_LONG, SUBNET_192_168)); + } + + @Test + public void testNotMatchingLongArg() + { + Assert.assertFalse(eval(IPV4_LONG, SUBNET_10)); + } + + @Test + public void testMatchingStringArgUnsignedInt() + { + Assert.assertFalse(eval(IPV4_UINT, SUBNET_192_168)); + } + + @Test + public void testNotMatchingStringArgUnsignedInt() + { + Assert.assertFalse(eval(IPV4_UINT, SUBNET_10)); + } + + @Test + public void testInclusive() + { + Expr subnet = SUBNET_192_168; + Assert.assertTrue(eval(IPV4_NETWORK, subnet)); + Assert.assertTrue(eval(IPV4, subnet)); + Assert.assertTrue(eval(IPV4_BROADCAST, subnet)); + } + + private boolean eval(Expr... args) + { + Expr expr = target.apply(Arrays.asList(args)); + ExprEval eval = expr.eval(ExprUtils.nilBindings()); + return eval.asBoolean(); + } + + /* Helper for tests */ + @SuppressWarnings({"ReturnOfNull", "NullableProblems"}) // suppressed since this is a test helper class + private static class NotLiteralExpr extends ExprMacroTable.BaseScalarUnivariateMacroFunctionExpr + { + NotLiteralExpr(Expr arg) + { + super(arg); + } + + @Override + public ExprEval eval(ObjectBinding bindings) + { + return null; + } + + @Override + public Expr visit(Shuttle shuttle) + { + return null; + } + } +} diff --git a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressParseExprMacroTest.java b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressParseExprMacroTest.java new file mode 100644 index 000000000000..2bf392141d51 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressParseExprMacroTest.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprEval; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +public class IPv4AddressParseExprMacroTest extends MacroTestBase +{ + private static final Expr VALID = ExprEval.of("192.168.0.1").toExpr(); + private static final long EXPECTED = 3232235521L; + private static final Long NULL = NullHandling.replaceWithDefault() ? NullHandling.ZERO_LONG : null; + + private IPv4AddressParseExprMacro target; + + @Before + public void setUp() + { + target = new IPv4AddressParseExprMacro(); + } + + @Test + public void testTooFewArgs() + { + expectException(IllegalArgumentException.class, "must have 1 argument"); + + target.apply(Collections.emptyList()); + } + + @Test + public void testTooManyArgs() + { + expectException(IllegalArgumentException.class, "must have 1 argument"); + + target.apply(Arrays.asList(VALID, VALID)); + } + + @Test + public void testNullStringArg() + { + Expr nullString = ExprEval.of(null).toExpr(); + Assert.assertSame(NULL, eval(nullString)); + } + + @Test + public void testNullLongArg() + { + Expr nullLong = ExprEval.ofLong(null).toExpr(); + Assert.assertEquals(NULL, eval(nullLong)); + } + + @Test + public void testInvalidArgType() + { + Expr longArray = ExprEval.ofLongArray(new Long[]{1L, 2L}).toExpr(); + Assert.assertEquals(NULL, eval(longArray)); + } + + @Test + public void testInvalidStringArgNotIPAddress() + { + Expr notIpAddress = ExprEval.of("druid.apache.org").toExpr(); + Assert.assertEquals(NULL, eval(notIpAddress)); + } + + @Test + public void testInvalidStringArgIPv6Compatible() + { + Expr ipv6Compatible = ExprEval.of("::192.168.0.1").toExpr(); + Assert.assertEquals(NULL, eval(ipv6Compatible)); + } + + @Test + public void testValidStringArgIPv6Mapped() + { + Expr ipv6Mapped = ExprEval.of("::ffff:192.168.0.1").toExpr(); + Assert.assertEquals(NULL, eval(ipv6Mapped)); + } + + @Test + public void testValidStringArgIPv4() + { + Assert.assertEquals(EXPECTED, eval(VALID)); + } + + @Test + public void testValidStringArgUnsignedInt() + { + Expr unsignedInt = ExprEval.of("3232235521").toExpr(); + Assert.assertEquals(NULL, eval(unsignedInt)); + } + + @Test + public void testInvalidLongArgTooLow() + { + Expr tooLow = ExprEval.ofLong(-1L).toExpr(); + Assert.assertEquals(NULL, eval(tooLow)); + } + + @Test + public void testValidLongArgLowest() + { + long lowest = 0L; + Expr tooLow = ExprEval.ofLong(lowest).toExpr(); + Assert.assertEquals(lowest, eval(tooLow)); + } + + @Test + public void testValidLongArgHighest() + { + long highest = 0xff_ff_ff_ffL; + Expr tooLow = ExprEval.ofLong(highest).toExpr(); + Assert.assertEquals(highest, eval(tooLow)); + } + + @Test + public void testInvalidLongArgTooHigh() + { + Expr tooHigh = ExprEval.ofLong(0x1_00_00_00_00L).toExpr(); + Assert.assertEquals(NULL, eval(tooHigh)); + } + + @Test + public void testValidLongArg() + { + long value = EXPECTED; + Expr valid = ExprEval.ofLong(value).toExpr(); + Assert.assertEquals(value, eval(valid)); + } + + private Object eval(Expr arg) + { + Expr expr = target.apply(Collections.singletonList(arg)); + ExprEval eval = expr.eval(ExprUtils.nilBindings()); + return eval.value(); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacroTest.java b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacroTest.java new file mode 100644 index 000000000000..602d00cfcc4e --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacroTest.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprEval; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +public class IPv4AddressStringifyExprMacroTest extends MacroTestBase +{ + private static final Expr VALID = ExprEval.of(3232235521L).toExpr(); + private static final String EXPECTED = "192.168.0.1"; + private static final String NULL = NullHandling.replaceWithDefault() ? "0.0.0.0" : null; + + private IPv4AddressStringifyExprMacro target; + + @Before + public void setUp() + { + target = new IPv4AddressStringifyExprMacro(); + } + + @Test + public void testTooFewArgs() + { + expectException(IllegalArgumentException.class, "must have 1 argument"); + + target.apply(Collections.emptyList()); + } + + @Test + public void testTooManyArgs() + { + expectException(IllegalArgumentException.class, "must have 1 argument"); + + target.apply(Arrays.asList(VALID, VALID)); + } + + @Test + public void testNullLongArg() + { + Expr nullNumeric = ExprEval.ofLong(null).toExpr(); + Assert.assertEquals(NULL, eval(nullNumeric)); + } + + @Test + public void testInvalidArgType() + { + Expr longArray = ExprEval.ofLongArray(new Long[]{1L, 2L}).toExpr(); + Assert.assertNull(eval(longArray)); + } + + @Test + public void testInvalidLongArgTooSmall() + { + Expr tooSmall = ExprEval.ofLong(-1L).toExpr(); + Assert.assertNull(eval(tooSmall)); + } + + @Test + public void testValidLongArgLowest() + { + Expr tooSmall = ExprEval.ofLong(0L).toExpr(); + Assert.assertEquals("0.0.0.0", eval(tooSmall)); + } + + @Test + public void testValidLongArg() + { + Assert.assertEquals(EXPECTED, eval(VALID)); + } + + @Test + public void testValidLongArgHighest() + { + Expr tooSmall = ExprEval.ofLong(0xff_ff_ff_ffL).toExpr(); + Assert.assertEquals("255.255.255.255", eval(tooSmall)); + } + + @Test + public void testInvalidLongArgTooLarge() + { + Expr tooLarge = ExprEval.ofLong(0x1_00_00_00_00L).toExpr(); + Assert.assertNull(eval(tooLarge)); + } + + @Test + public void testNullStringArg() + { + Expr nullString = ExprEval.of(null).toExpr(); + Assert.assertNull(NULL, eval(nullString)); + } + + @Test + public void testInvalidStringArgNotIPAddress() + { + Expr notIpAddress = ExprEval.of("druid.apache.org").toExpr(); + Assert.assertNull(eval(notIpAddress)); + } + + @Test + public void testInvalidStringArgIPv6Compatible() + { + Expr ipv6Compatible = ExprEval.of("::192.168.0.1").toExpr(); + Assert.assertNull(eval(ipv6Compatible)); + } + + @Test + public void testValidStringArgIPv6Mapped() + { + Expr ipv6Mapped = ExprEval.of("::ffff:192.168.0.1").toExpr(); + Assert.assertNull(eval(ipv6Mapped)); + } + + @Test + public void testValidStringArgIPv4() + { + Assert.assertEquals(EXPECTED, eval(VALID)); + } + + @Test + public void testValidStringArgUnsignedInt() + { + Expr unsignedInt = ExprEval.of("3232235521").toExpr(); + Assert.assertNull(eval(unsignedInt)); + } + + private Object eval(Expr arg) + { + Expr expr = target.apply(Collections.singletonList(arg)); + ExprEval eval = expr.eval(ExprUtils.nilBindings()); + return eval.value(); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/expression/MacroTestBase.java b/processing/src/test/java/org/apache/druid/query/expression/MacroTestBase.java new file mode 100644 index 000000000000..2c203abe5942 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/expression/MacroTestBase.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import org.junit.Rule; +import org.junit.rules.ExpectedException; + +public abstract class MacroTestBase +{ + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + void expectException(Class type, String message) + { + expectedException.expect(type); + expectedException.expectMessage(message); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/expression/TestExprMacroTable.java b/processing/src/test/java/org/apache/druid/query/expression/TestExprMacroTable.java index ba0758f8d3b1..c617099e3aed 100644 --- a/processing/src/test/java/org/apache/druid/query/expression/TestExprMacroTable.java +++ b/processing/src/test/java/org/apache/druid/query/expression/TestExprMacroTable.java @@ -30,6 +30,9 @@ private TestExprMacroTable() { super( ImmutableList.of( + new IPv4AddressMatchExprMacro(), + new IPv4AddressParseExprMacro(), + new IPv4AddressStringifyExprMacro(), new LikeExprMacro(), new RegexpExtractExprMacro(), new TimestampCeilExprMacro(), diff --git a/server/src/main/java/org/apache/druid/guice/ExpressionModule.java b/server/src/main/java/org/apache/druid/guice/ExpressionModule.java index 29d216d81b77..f695563d352f 100644 --- a/server/src/main/java/org/apache/druid/guice/ExpressionModule.java +++ b/server/src/main/java/org/apache/druid/guice/ExpressionModule.java @@ -26,6 +26,9 @@ import org.apache.druid.initialization.DruidModule; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.query.expression.GuiceExprMacroTable; +import org.apache.druid.query.expression.IPv4AddressMatchExprMacro; +import org.apache.druid.query.expression.IPv4AddressParseExprMacro; +import org.apache.druid.query.expression.IPv4AddressStringifyExprMacro; import org.apache.druid.query.expression.LikeExprMacro; import org.apache.druid.query.expression.RegexpExtractExprMacro; import org.apache.druid.query.expression.TimestampCeilExprMacro; @@ -44,6 +47,9 @@ public class ExpressionModule implements DruidModule { public static final List> EXPR_MACROS = ImmutableList.>builder() + .add(IPv4AddressMatchExprMacro.class) + .add(IPv4AddressParseExprMacro.class) + .add(IPv4AddressStringifyExprMacro.class) .add(LikeExprMacro.class) .add(RegexpExtractExprMacro.class) .add(TimestampCeilExprMacro.class) diff --git a/server/src/test/java/org/apache/druid/query/expression/ExprMacroTest.java b/server/src/test/java/org/apache/druid/query/expression/ExprMacroTest.java index d79cff18b474..74e74394570a 100644 --- a/server/src/test/java/org/apache/druid/query/expression/ExprMacroTest.java +++ b/server/src/test/java/org/apache/druid/query/expression/ExprMacroTest.java @@ -31,6 +31,8 @@ public class ExprMacroTest { + private static final String IPV4_STRING = "192.168.0.1"; + private static final long IPV4_LONG = 3232235521L; private static final Expr.ObjectBinding BINDINGS = Parser.withMap( ImmutableMap.builder() .put("t", DateTimes.of("2000-02-03T04:05:06").getMillis()) @@ -42,6 +44,10 @@ public class ExprMacroTest .put("z", 3.1) .put("CityOfAngels", "America/Los_Angeles") .put("spacey", " hey there ") + .put("ipv4_string", IPV4_STRING) + .put("ipv4_long", IPV4_LONG) + .put("ipv4_network", "192.168.0.0") + .put("ipv4_broadcast", "192.168.255.255") .build() ); @@ -187,6 +193,34 @@ public void testRTrim() assertExpr("rtrim(spacey, substring(spacey, 0, 4))", " hey ther"); } + @Test + public void testIPv4AddressParse() + { + Long nullLong = NullHandling.replaceWithDefault() ? NullHandling.ZERO_LONG : null; + assertExpr("ipv4_parse(x)", nullLong); + assertExpr("ipv4_parse(ipv4_string)", IPV4_LONG); + assertExpr("ipv4_parse(ipv4_long)", IPV4_LONG); + assertExpr("ipv4_parse(ipv4_stringify(ipv4_long))", IPV4_LONG); + } + + @Test + public void testIPv4AddressStringify() + { + assertExpr("ipv4_stringify(x)", null); + assertExpr("ipv4_stringify(ipv4_long)", IPV4_STRING); + assertExpr("ipv4_stringify(ipv4_string)", IPV4_STRING); + assertExpr("ipv4_stringify(ipv4_parse(ipv4_string))", IPV4_STRING); + } + + @Test + public void testIPv4AddressMatch() + { + assertExpr("ipv4_match(ipv4_string, '10.0.0.0/8')", 0L); + assertExpr("ipv4_match(ipv4_string, '192.168.0.0/16')", 1L); + assertExpr("ipv4_match(ipv4_network, '192.168.0.0/16')", 1L); + assertExpr("ipv4_match(ipv4_broadcast, '192.168.0.0/16')", 1L); + } + private void assertExpr(final String expression, final Object expectedResult) { final Expr expr = Parser.parse(expression, LookupEnabledTestExprMacroTable.INSTANCE);