From 06660f31912f41e93ec593c654538ed5d8be74c9 Mon Sep 17 00:00:00 2001 From: Chi Cao Minh Date: Tue, 30 Jul 2019 11:01:40 -0700 Subject: [PATCH 1/5] Add IPv4 druid expressions New druid expressions for filtering IPv4 addresses: - ipv4address_match: Check if IP address belongs to a subnet - ipv4address_parse: Convert string IP address to long - ipv4address_stringify: Convert long IP address to string These expressions operate on IP addresses represented as either strings or longs, so that they can be applied to dimensions with mixed representation of IP addresses. The filtering is more efficient when operating on IP addresses as longs. In other words, the intended use case is: 1) Use ipv4address_parse to convert to long at ingestion time 2) Use ipv4address_match to filter (on longs) at query time 3) Use ipv4adress_stringify to convert to (readable) string at query time --- docs/content/misc/math-expr.md | 8 + pom.xml | 5 + processing/pom.xml | 4 + .../expression/IPv4AddressExprUtils.java | 59 ++++ .../expression/IPv4AddressMatchExprMacro.java | 198 +++++++++++++ .../expression/IPv4AddressParseExprMacro.java | 146 ++++++++++ .../IPv4AddressStringifyExprMacro.java | 148 ++++++++++ .../expression/IPv4AddressExprUtilsTest.java | 91 ++++++ .../expression/IPv4AddressMatchMacroTest.java | 259 ++++++++++++++++++ .../IPv4AddressParseExprMacroTest.java | 161 +++++++++++ .../IPv4AddressStringifyMacroTest.java | 157 +++++++++++ .../druid/query/expression/MacroTestBase.java | 35 +++ .../query/expression/TestExprMacroTable.java | 3 + .../apache/druid/guice/ExpressionModule.java | 6 + .../druid/query/expression/ExprMacroTest.java | 38 +++ 15 files changed, 1318 insertions(+) create mode 100644 processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java create mode 100644 processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java create mode 100644 processing/src/main/java/org/apache/druid/query/expression/IPv4AddressParseExprMacro.java create mode 100644 processing/src/main/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacro.java create mode 100644 processing/src/test/java/org/apache/druid/query/expression/IPv4AddressExprUtilsTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchMacroTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/expression/IPv4AddressParseExprMacroTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/expression/IPv4AddressStringifyMacroTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/expression/MacroTestBase.java diff --git a/docs/content/misc/math-expr.md b/docs/content/misc/math-expr.md index 3fab03aa5abd..fc72a5f1773c 100644 --- a/docs/content/misc/math-expr.md +++ b/docs/content/misc/math-expr.md @@ -195,3 +195,11 @@ See javadoc of java.lang.Math for detailed explanation for each function. | cartesian_fold(lambda,arr1,arr2,...) | folds a multi argument lambda across the cartesian product of all input arrays. The first arguments of the lambda is the array element and the last is the accumulator, returning a single accumulated value. | | any(lambda,arr) | returns 1 if any element in the array matches the lambda expression, else 0 | | all(lambda,arr) | returns 1 if all elements in the array matches the lambda expression, else 0 | + + +## IP Address Functions +| function | description | +| --- | --- | +| ipv4address_match(address, subnet [,inclusive]) | Returns 1 if the string or long IP `address` belongs to the `subnet` CIDR notation literal string, else 0. The optional literal boolean `inclusive` parameter indicates whether the network and broadcast IP addresses should be included in the subnet and defaults to `false`. This function is more efficient when operating on IP addresses as longs instead of strings.| +| ipv4address_parse(address) | Parses string or long into an IPv4 address as a long. Returns null if `address` cannot be represented as an IPv4 address. | +| ipv4address_stringify(address) | Converts string or long IPv4 address into an IPv4 address dotted-decimal notated string. Return null if `address` cannot be represented as an IPv4 address.| diff --git a/pom.xml b/pom.xml index 1a802c9f5e15..dca5d24addda 100644 --- a/pom.xml +++ b/pom.xml @@ -212,6 +212,11 @@ commons-lang 2.6 + + commons-net + commons-net + 3.6 + com.amazonaws aws-java-sdk-ec2 diff --git a/processing/pom.xml b/processing/pom.xml index 53c1d23f3ed1..5e29987724bf 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -79,6 +79,10 @@ commons-io commons-io + + commons-net + commons-net + com.google.errorprone error_prone_annotations diff --git a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java new file mode 100644 index 000000000000..74bf58706339 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import com.google.common.net.InetAddresses; + +import javax.annotation.Nullable; +import java.net.Inet4Address; +import java.net.InetAddress; + +class IPv4AddressExprUtils +{ + /** + * @return True if argument cannot be represented by an unsigned integer (4 bytes) + */ + static boolean overflowsUnsignedInt(long value) + { + return value < 0L || 0xff_ff_ff_ffL < value; + } + + /** + * @return IPv4 address dotted-decimal notated string or null if the argument is not a valid IPv4 address string or + * IPv6 IPv4-mapped address string. + */ + @Nullable + static String extractIPv4Address(String string) + { + if (string != null) { + try { + InetAddress address = InetAddresses.forString(string); + if (address instanceof Inet4Address) { + return address.getHostAddress(); + } + } + catch (IllegalArgumentException ignored) { + // fall through + } + } + + return null; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java new file mode 100644 index 000000000000..81b44e1eb101 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import com.google.common.base.Preconditions; +import org.apache.commons.net.util.SubnetUtils; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.math.expr.ExprType; + +import javax.annotation.Nonnull; +import java.util.List; + +/** + * 
+ * Implements an expression that checks if an IPv4 address belongs to a particular subnet.
+ *
+ * Expression signatures:
+ * - boolean ipv4address_match(string address, string subnet)
+ * - boolean ipv4address_match(string address, string subnet, boolean inclusive)
+ * - boolean ipv4address_match(long address, string subnet)
+ * - boolean ipv4address_match(long address, string subnet, boolean inclusive)
+ *
+ * Valid "address" argument formats are:
+ * - unsigned int long (e.g., 3232235521)
+ * - unsigned int string (e.g., "3232235521")
+ * - IPv4 address dotted-decimal notation string (e.g., "198.168.0.1")
+ * - IPv6 IPv4-mapped address string (e.g., "::ffff:192.168.0.1")
+ *
+ * The argument format for the "subnet" argument should be a literal in CIDR notation
+ * (e.g., "198.168.0.0/16").
+ *
+ * An optional "inclusive" argument should be a boolean literal (e.g., 1, 0, "true", or "false") that
+ * indicates whether the network and the broadcast addresses should be considered part of the
+ * subnet. When this argument is absent, its value defaults to "false".
+ *
+ * The overloaded signature allows applying the expression to a dimension with mixed string and long
+ * representations of IPv4 addresses.
+ *
+ * + * @see IPv4AddressParseExprMacro + * @see IPv4AddressStringifyExprMacro + */ +public class IPv4AddressMatchExprMacro implements ExprMacroTable.ExprMacro +{ + public static final String NAME = "ipv4address_match"; + private static final int ARG_SUBNET = 1; + private static final int ARG_INCLUSIVE = 2; + + @Override + public String name() + { + return NAME; + } + + @Override + public Expr apply(final List args) + { + if (args.size() < 2 || 3 < args.size()) { + throw new IAE(createErrMsg("must have 2-3 arguments")); + } + + boolean inclusive = getInclusive(args); + SubnetUtils.SubnetInfo subnetInfo = getSubnetInfo(args, inclusive); + + Expr arg = args.get(0); + + class IPv4AddressMatchExpr extends ExprMacroTable.BaseScalarUnivariateMacroFunctionExpr + { + private final SubnetUtils.SubnetInfo subnetInfo; + + private IPv4AddressMatchExpr(Expr arg, SubnetUtils.SubnetInfo subnetInfo) + { + super(arg); + this.subnetInfo = subnetInfo; + } + + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + ExprEval eval = arg.eval(bindings); + boolean match; + switch (eval.type()) { + case STRING: + match = isStringMatch(eval.asString()); + break; + case LONG: + match = isLongMatch(eval.asLong()); + break; + default: + match = false; + } + return ExprEval.of(match, ExprType.LONG); + } + + private boolean isStringMatch(String stringValue) + { + boolean match; + String ipv4 = IPv4AddressExprUtils.extractIPv4Address(stringValue); + if (ipv4 == null) { + match = isLongMatch(stringValue); + } else { + match = subnetInfo.isInRange(ipv4); + } + return match; + } + + private boolean isLongMatch(String stringValue) + { + boolean match; + try { + long longValue = Long.parseLong(stringValue); + match = isLongMatch(longValue); + } + catch (NumberFormatException ignored) { + match = false; + } + return match; + } + + private boolean isLongMatch(long longValue) + { + return !IPv4AddressExprUtils.overflowsUnsignedInt(longValue) && subnetInfo.isInRange((int) longValue); + } + + @Override + public Expr visit(Shuttle shuttle) + { + Expr newArg = arg.visit(shuttle); + return shuttle.visit(new IPv4AddressMatchExpr(newArg, subnetInfo)); + } + } + + return new IPv4AddressMatchExpr(arg, subnetInfo); + } + + private String createErrMsg(String msg) + { + String prefix = "Function[" + name() + "] "; + return prefix + msg; + } + + private boolean getInclusive(List args) + { + boolean inclusive = false; + if (ARG_INCLUSIVE < args.size()) { + Expr arg = args.get(ARG_INCLUSIVE); + checkLiteralArgument(arg, "inclusive"); + ExprEval eval = arg.eval(ExprUtils.nilBindings()); + inclusive = eval.asBoolean(); + } + return inclusive; + } + + private void checkLiteralArgument(Expr arg, String name) + { + Preconditions.checkArgument(arg.isLiteral(), createErrMsg(name + " arg must be a literal")); + } + + private SubnetUtils.SubnetInfo getSubnetInfo(List args, boolean inclusive) + { + String subnetArgName = "subnet"; + Expr arg = args.get(ARG_SUBNET); + checkLiteralArgument(arg, subnetArgName); + String subnet = (String) arg.getLiteralValue(); + + SubnetUtils subnetUtils; + try { + subnetUtils = new SubnetUtils(subnet); + } + catch (IllegalArgumentException e) { + throw new IAE(e, createErrMsg(subnetArgName + " arg has an invalid format: " + subnet)); + } + subnetUtils.setInclusiveHostCount(inclusive); + + return subnetUtils.getInfo(); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressParseExprMacro.java b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressParseExprMacro.java new file mode 100644 index 000000000000..f410a98ceb49 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressParseExprMacro.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import com.google.common.net.InetAddresses; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.math.expr.ExprMacroTable; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.net.Inet4Address; +import java.net.InetAddress; +import java.util.List; + +/** + * 
+ * Implements an expression that parses string or long into an IPv4 address stored (as an unsigned
+ * int) in a long.
+ *
+ * Expression signatures:
+ * - long ipv4address_parse(string)
+ * - long ipv4address_parse(long)
+ *
+ * Valid argument formats are:
+ * - IPv4 address dotted-decimal notation string (e.g., "198.168.0.1")
+ * - IPv6 IPv4-mapped adress string (e.g., "::ffff:192.168.0.1")
+ * - unsigned int long (e.g., 3232235521)
+ * - unsigned int string (e.g., "3232235521")
+ *
+ * Invalid arguments return null.
+ *
+ * The overloaded signature allows applying the expression to a dimension with mixed string and long
+ * representations of IPv4 addresses.
+ *
+ * + * @see IPv4AddressStringifyExprMacro + * @see IPv4AddressMatchExprMacro + */ +public class IPv4AddressParseExprMacro implements ExprMacroTable.ExprMacro +{ + public static final String NAME = "ipv4address_parse"; + + @Override + public String name() + { + return NAME; + } + + @Override + public Expr apply(final List args) + { + if (args.size() != 1) { + throw new IAE("Function[%s] must have 1 argument", name()); + } + + Expr arg = args.get(0); + + class IPv4AddressParseExpr extends ExprMacroTable.BaseScalarUnivariateMacroFunctionExpr + { + private IPv4AddressParseExpr(Expr arg) + { + super(arg); + } + + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + String stringValue = arg.eval(bindings).asString(); + if (stringValue == null) { + return ExprEval.ofLong(null); + } + + // Assume use cases in order of most frequent to least are: + // 1) convert string to long + // 2) convert long to long + Long value = parseAsString(stringValue); + if (value == null) { + value = parseAsLong(stringValue); + } + + return ExprEval.ofLong(value); + } + + @Override + public Expr visit(Shuttle shuttle) + { + Expr newArg = arg.visit(shuttle); + return shuttle.visit(new IPv4AddressParseExpr(newArg)); + } + } + + return new IPv4AddressParseExpr(arg); + } + + @Nullable + private static Long parseAsString(String stringValue) + { + try { + // Do not use java.lang.InetAddress#getByName() as it may do DNS lookups + InetAddress address = InetAddresses.forString(stringValue); + if (address instanceof Inet4Address) { + int value = InetAddresses.coerceToInteger(address); + return Integer.toUnsignedLong(value); + } + } + catch (IllegalArgumentException ignored) { + // fall through (Invalid IPv4 adddress string) + } + return null; + } + + @Nullable + private static Long parseAsLong(String stringValue) + { + try { + Long value = Long.valueOf(stringValue); + if (!IPv4AddressExprUtils.overflowsUnsignedInt(value)) { + return value; + } + } + catch (NumberFormatException ignored) { + // fall through + } + return null; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacro.java b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacro.java new file mode 100644 index 000000000000..c70557992964 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacro.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import com.google.common.net.InetAddresses; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.math.expr.ExprMacroTable; + +import javax.annotation.Nonnull; +import java.net.Inet4Address; +import java.util.List; + +/** + * 
+ * Implements an expression that converts an IPv4 address stored (as an unsigned int) in a long or
+ * stored as a string into an IPv4 address dotted-decimal notated string (e.g., "192.168.0.1").
+ *
+ * Expression signatures:
+ * - string ipv4address_stringify(long)
+ * - string ipv4address_stringify(string)
+ *
+ * Valid argument formats are:
+ * - unsigned int long (e.g., 3232235521)
+ * - unsigned int string (e.g., "3232235521")
+ * - IPv4 address dotted-decimal notation string (e.g., "198.168.0.1")
+ * - IPv6 IPv4-mapped address string (e.g., "::ffff:192.168.0.1")
+ *
+ * Invalid arguments return null.
+ *
+ * The overloaded signature allows applying the expression to a dimension with mixed string and long
+ * representations of IPv4 addresses.
+ *
+ * + * @see IPv4AddressParseExprMacro + * @see IPv4AddressMatchExprMacro + */ +public class IPv4AddressStringifyExprMacro implements ExprMacroTable.ExprMacro +{ + public static final String NAME = "ipv4address_stringify"; + + @Override + public String name() + { + return NAME; + } + + @Override + public Expr apply(final List args) + { + if (args.size() != 1) { + throw new IAE("Function[%s] must have 1 argument", name()); + } + + Expr arg = args.get(0); + + class IPv4AddressStringifyExpr extends ExprMacroTable.BaseScalarUnivariateMacroFunctionExpr + { + private IPv4AddressStringifyExpr(Expr arg) + { + super(arg); + } + + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + ExprEval eval = arg.eval(bindings); + switch (eval.type()) { + case STRING: + return evalAsString(eval); + case LONG: + return evalAsLong(eval); + default: + return ExprEval.of(null); + } + } + + @Override + public Expr visit(Shuttle shuttle) + { + Expr newArg = arg.visit(shuttle); + return shuttle.visit(new IPv4AddressStringifyExpr(newArg)); + } + } + + return new IPv4AddressStringifyExpr(arg); + } + + private static ExprEval evalAsString(ExprEval eval) + { + String stringValue = eval.asString(); + if (stringValue == null) { + return eval; + } + + // Assume use cases in order of most frequent to least: + // 1) convert long to string + // 2) convert string to string + + try { + long longValue = Long.parseLong(stringValue); + return evalLong(longValue); + } + catch (NumberFormatException ignored) { + // fall through + } + + return ExprEval.of(IPv4AddressExprUtils.extractIPv4Address(stringValue)); + } + + private static ExprEval evalAsLong(ExprEval eval) + { + if (eval.isNumericNull()) { + return eval; + } + + return evalLong(eval.asLong()); + } + + private static ExprEval evalLong(long longValue) + { + if (IPv4AddressExprUtils.overflowsUnsignedInt(longValue)) { + return ExprEval.of(null); + } + + Inet4Address address = InetAddresses.fromInteger((int) longValue); + return ExprEval.of(address.getHostAddress()); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressExprUtilsTest.java b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressExprUtilsTest.java new file mode 100644 index 000000000000..fc239f6f0796 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressExprUtilsTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import org.junit.Assert; +import org.junit.Test; + +public class IPv4AddressExprUtilsTest +{ + @Test + public void testOverflowsUnsignedIntTooLow() + { + Assert.assertTrue(IPv4AddressExprUtils.overflowsUnsignedInt(-1L)); + } + + @Test + public void testOverflowsUnsignedIntLowest() + { + Assert.assertFalse(IPv4AddressExprUtils.overflowsUnsignedInt(0L)); + } + + @Test + public void testOverflowsUnsignedIntMiddle() + { + Assert.assertFalse(IPv4AddressExprUtils.overflowsUnsignedInt(0xff_ffL)); + } + + @Test + public void testOverflowsUnsignedIntHighest() + { + Assert.assertFalse(IPv4AddressExprUtils.overflowsUnsignedInt(0xff_ff_ff_ffL)); + } + + @Test + public void testOverflowsUnsignedIntTooHigh() + { + Assert.assertTrue(IPv4AddressExprUtils.overflowsUnsignedInt(0x1_00_00_00_00L)); + } + + @Test + public void testExtractIPv4AddressNull() + { + Assert.assertNull(IPv4AddressExprUtils.extractIPv4Address(null)); + } + + @Test + public void testExtractIPv4AddressIPv4() + { + String ipv4 = "192.168.0.1"; + Assert.assertEquals(ipv4, IPv4AddressExprUtils.extractIPv4Address(ipv4)); + } + + @Test + public void testExtractIPv4AddressIPv6Mapped() + { + String ipv4 = "192.168.0.1"; + String ipv6Mapped = "::ffff:" + ipv4; + Assert.assertEquals(ipv4, IPv4AddressExprUtils.extractIPv4Address(ipv6Mapped)); + } + + @Test + public void testExtractIPv4AddressIPv6Compatible() + { + String ipv6Compatible = "::192.168.0.1"; + Assert.assertNull(IPv4AddressExprUtils.extractIPv4Address(ipv6Compatible)); + } + + @Test + public void testExtractIPv4AddressInvalid() + { + String notIpAddress = "druid.apache.org"; + Assert.assertNull(IPv4AddressExprUtils.extractIPv4Address(notIpAddress)); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchMacroTest.java b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchMacroTest.java new file mode 100644 index 000000000000..9f3b66bda096 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchMacroTest.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.math.expr.ExprMacroTable; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +public class IPv4AddressMatchMacroTest extends MacroTestBase +{ + private static final Expr IPV4 = ExprEval.of("192.168.0.1").toExpr(); + private static final Expr IPV4_LONG = ExprEval.of(3232235521L).toExpr(); + private static final Expr IPV4_UINT = ExprEval.of("3232235521").toExpr(); + private static final Expr IPV4_NETWORK = ExprEval.of("192.168.0.0").toExpr(); + private static final Expr IPV4_BROADCAST = ExprEval.of("192.168.255.255").toExpr(); + private static final Expr IPV6_COMPATIBLE = ExprEval.of("::192.168.0.1").toExpr(); + private static final Expr IPV6_MAPPED = ExprEval.of("::ffff:192.168.0.1").toExpr(); + private static final Expr SUBNET_192_168 = ExprEval.of("192.168.0.0/16").toExpr(); + private static final Expr SUBNET_10 = ExprEval.of("10.0.0.0/8").toExpr(); + private static final Expr INCLUSIVE = ExprEval.of("true").toExpr(); + private static final Expr NOT_INCLUSIVE = ExprEval.of("false").toExpr(); + private static final Expr NOT_LITERAL = new NotLiteralExpr(null); + + private IPv4AddressMatchExprMacro target; + + @Before + public void setUp() + { + target = new IPv4AddressMatchExprMacro(); + } + + @Test + public void testTooFewArgs() + { + expectException(IllegalArgumentException.class, "must have 2-3 arguments"); + + target.apply(Collections.emptyList()); + } + + @Test + public void testTooManyArgs() + { + expectException(IllegalArgumentException.class, "must have 2-3 arguments"); + + target.apply(Arrays.asList(IPV4, SUBNET_192_168, INCLUSIVE, NOT_LITERAL)); + } + + @Test + public void testSubnetArgNotLiteral() + { + expectException(IllegalArgumentException.class, "subnet arg must be a literal"); + + target.apply(Arrays.asList(IPV4, NOT_LITERAL)); + } + + @Test + public void testSubnetArgInvalid() + { + expectException(IllegalArgumentException.class, "subnet arg has an invalid format"); + + Expr invalidSubnet = ExprEval.of("192.168.0.1/invalid").toExpr(); + target.apply(Arrays.asList(IPV4, invalidSubnet)); + } + + @Test + public void testInclusiveArgNotLiteral() + { + expectException(IllegalArgumentException.class, "inclusive arg must be a literal"); + + target.apply(Arrays.asList(IPV4, SUBNET_192_168, NOT_LITERAL)); + } + + @Test + public void testNullStringArg() + { + Expr nullString = ExprEval.of(null).toExpr(); + Assert.assertFalse(eval(nullString, SUBNET_192_168)); + } + + @Test + public void testNullLongArg() + { + Expr nullLong = ExprEval.ofLong(null).toExpr(); + Assert.assertFalse(eval(nullLong, SUBNET_192_168)); + } + + @Test + public void testInvalidArgType() + { + Expr longArray = ExprEval.ofLongArray(new Long[]{1L, 2L}).toExpr(); + Assert.assertFalse(eval(longArray, SUBNET_192_168)); + } + + @Test + public void testMatchingStringArgIPv4() + { + Assert.assertTrue(eval(IPV4, SUBNET_192_168)); + } + + @Test + public void testNotMatchingStringArgIPv4() + { + Assert.assertFalse(eval(IPV4, SUBNET_10)); + } + + @Test + public void testMatchingStringArgIPv6Mapped() + { + Assert.assertTrue(eval(IPV6_MAPPED, SUBNET_192_168)); + } + + @Test + public void testNotMatchingStringArgIPv6Mapped() + { + Assert.assertFalse(eval(IPV6_MAPPED, SUBNET_10)); + } + + @Test + public void testMatchingStringArgIPv6Compatible() + { + Assert.assertFalse(eval(IPV6_COMPATIBLE, SUBNET_192_168)); + } + + @Test + public void testNotMatchingStringArgIPv6Compatible() + { + Assert.assertFalse(eval(IPV6_COMPATIBLE, SUBNET_10)); + } + + @Test + public void testNotIpAddress() + { + Expr notIpAddress = ExprEval.of("druid.apache.org").toExpr(); + Assert.assertFalse(eval(notIpAddress, SUBNET_192_168)); + } + + @Test + public void testMatchingLongArg() + { + Assert.assertTrue(eval(IPV4_LONG, SUBNET_192_168)); + } + + @Test + public void testNotMatchingLongArg() + { + Assert.assertFalse(eval(IPV4_LONG, SUBNET_10)); + } + + @Test + public void testMatchingStringArgUnsignedInt() + { + Assert.assertTrue(eval(IPV4_UINT, SUBNET_192_168)); + } + + @Test + public void testNotMatchingStringArgUnsignedInt() + { + Assert.assertFalse(eval(IPV4_UINT, SUBNET_10)); + } + + @Test + public void testInclusive() + { + Expr subnet = SUBNET_192_168; + Assert.assertTrue(eval(IPV4_NETWORK, subnet, INCLUSIVE)); + Assert.assertTrue(eval(IPV4, subnet, INCLUSIVE)); + Assert.assertTrue(eval(IPV4_BROADCAST, subnet, INCLUSIVE)); + } + + @Test + public void testNotInclusive() + { + Expr subnet = SUBNET_192_168; + Assert.assertFalse(eval(IPV4_NETWORK, subnet, NOT_INCLUSIVE)); + Assert.assertTrue(eval(IPV4, subnet, NOT_INCLUSIVE)); + Assert.assertFalse(eval(IPV4_BROADCAST, subnet, NOT_INCLUSIVE)); + } + + @Test + public void testDefaultNotInclusive() + { + Expr subnet = SUBNET_192_168; + Assert.assertFalse(eval(IPV4_NETWORK, subnet)); + Assert.assertTrue(eval(IPV4, subnet)); + Assert.assertFalse(eval(IPV4_BROADCAST, subnet)); + } + + @Test + public void testInclusiveAsLong() + { + Expr subnet = SUBNET_192_168; + Expr inclusive = ExprEval.of(1L).toExpr(); + Assert.assertTrue(eval(IPV4_NETWORK, subnet, inclusive)); + Assert.assertTrue(eval(IPV4, subnet, inclusive)); + Assert.assertTrue(eval(IPV4_BROADCAST, subnet, inclusive)); + } + + @Test + public void testNotInclusiveAsLong() + { + Expr subnet = SUBNET_192_168; + Expr notInclusive = ExprEval.of(0L).toExpr(); + Assert.assertFalse(eval(IPV4_NETWORK, subnet, notInclusive)); + Assert.assertTrue(eval(IPV4, subnet, notInclusive)); + Assert.assertFalse(eval(IPV4_BROADCAST, subnet, notInclusive)); + } + + private boolean eval(Expr... args) + { + Expr expr = target.apply(Arrays.asList(args)); + ExprEval eval = expr.eval(ExprUtils.nilBindings()); + return eval.asBoolean(); + } + + /* Helper for tests */ + @SuppressWarnings({"ReturnOfNull", "NullableProblems"}) // suppressed since this is a test helper class + private static class NotLiteralExpr extends ExprMacroTable.BaseScalarUnivariateMacroFunctionExpr + { + NotLiteralExpr(Expr arg) + { + super(arg); + } + + @Override + public ExprEval eval(ObjectBinding bindings) + { + return null; + } + + @Override + public Expr visit(Shuttle shuttle) + { + return null; + } + } +} diff --git a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressParseExprMacroTest.java b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressParseExprMacroTest.java new file mode 100644 index 000000000000..5951ff6aa844 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressParseExprMacroTest.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprEval; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +public class IPv4AddressParseExprMacroTest extends MacroTestBase +{ + private static final Expr VALID = ExprEval.of("192.168.0.1").toExpr(); + private static final long EXPECTED = 3232235521L; + private static final Long NULL = NullHandling.replaceWithDefault() ? NullHandling.ZERO_LONG : null; + + private IPv4AddressParseExprMacro target; + + @Before + public void setUp() + { + target = new IPv4AddressParseExprMacro(); + } + + @Test + public void testTooFewArgs() + { + expectException(IllegalArgumentException.class, "must have 1 argument"); + + target.apply(Collections.emptyList()); + } + + @Test + public void testTooManyArgs() + { + expectException(IllegalArgumentException.class, "must have 1 argument"); + + target.apply(Arrays.asList(VALID, VALID)); + } + + @Test + public void testNullStringArg() + { + Expr nullString = ExprEval.of(null).toExpr(); + Assert.assertEquals(NULL, eval(nullString)); + } + + @Test + public void testNullLongArg() + { + Expr nullLong = ExprEval.ofLong(null).toExpr(); + Assert.assertEquals(NULL, eval(nullLong)); + } + + @Test + public void testInvalidArgType() + { + Expr longArray = ExprEval.ofLongArray(new Long[]{1L, 2L}).toExpr(); + Assert.assertEquals(NULL, eval(longArray)); + } + + @Test + public void testInvalidStringArgNotIPAddress() + { + Expr notIpAddress = ExprEval.of("druid.apache.org").toExpr(); + Assert.assertEquals(NULL, eval(notIpAddress)); + } + + @Test + public void testInvalidStringArgIPv6Compatible() + { + Expr ipv6Compatible = ExprEval.of("::192.168.0.1").toExpr(); + Assert.assertEquals(NULL, eval(ipv6Compatible)); + } + + @Test + public void testValidStringArgIPv6Mapped() + { + Expr ipv6Mapped = ExprEval.of("::ffff:192.168.0.1").toExpr(); + Assert.assertEquals(EXPECTED, eval(ipv6Mapped)); + } + + @Test + public void testValidStringArgIPv4() + { + Assert.assertEquals(EXPECTED, eval(VALID)); + } + + @Test + public void testValidStringArgUnsignedInt() + { + Expr unsignedInt = ExprEval.of("3232235521").toExpr(); + Assert.assertEquals(EXPECTED, eval(unsignedInt)); + } + + @Test + public void testInvalidLongArgTooLow() + { + Expr tooLow = ExprEval.ofLong(-1L).toExpr(); + Assert.assertEquals(NULL, eval(tooLow)); + } + + @Test + public void testValidLongArgLowest() + { + long lowest = 0L; + Expr tooLow = ExprEval.ofLong(lowest).toExpr(); + Assert.assertEquals(lowest, eval(tooLow)); + } + + @Test + public void testValidLongArgHighest() + { + long highest = 0xff_ff_ff_ffL; + Expr tooLow = ExprEval.ofLong(highest).toExpr(); + Assert.assertEquals(highest, eval(tooLow)); + } + + @Test + public void testInvalidLongArgTooHigh() + { + Expr tooHigh = ExprEval.ofLong(0x1_00_00_00_00L).toExpr(); + Assert.assertEquals(NULL, eval(tooHigh)); + } + + @Test + public void testValidLongArg() + { + long value = EXPECTED; + Expr valid = ExprEval.ofLong(value).toExpr(); + Assert.assertEquals(value, eval(valid)); + } + + private Object eval(Expr arg) + { + Expr expr = target.apply(Collections.singletonList(arg)); + ExprEval eval = expr.eval(ExprUtils.nilBindings()); + return eval.value(); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressStringifyMacroTest.java b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressStringifyMacroTest.java new file mode 100644 index 000000000000..2a20c95f1149 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressStringifyMacroTest.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprEval; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +public class IPv4AddressStringifyMacroTest extends MacroTestBase +{ + private static final Expr VALID = ExprEval.of(3232235521L).toExpr(); + private static final String EXPECTED = "192.168.0.1"; + private static final String NULL = NullHandling.replaceWithDefault() ? "0.0.0.0" : null; + + private IPv4AddressStringifyExprMacro target; + + @Before + public void setUp() + { + target = new IPv4AddressStringifyExprMacro(); + } + + @Test + public void testTooFewArgs() + { + expectException(IllegalArgumentException.class, "must have 1 argument"); + + target.apply(Collections.emptyList()); + } + + @Test + public void testTooManyArgs() + { + expectException(IllegalArgumentException.class, "must have 1 argument"); + + target.apply(Arrays.asList(VALID, VALID)); + } + + @Test + public void testNullLongArg() + { + Expr nullNumeric = ExprEval.ofLong(null).toExpr(); + Assert.assertEquals(NULL, eval(nullNumeric)); + } + + @Test + public void testInvalidArgType() + { + Expr longArray = ExprEval.ofLongArray(new Long[]{1L, 2L}).toExpr(); + Assert.assertNull(eval(longArray)); + } + + @Test + public void testInvalidLongArgTooSmall() + { + Expr tooSmall = ExprEval.ofLong(-1L).toExpr(); + Assert.assertNull(eval(tooSmall)); + } + + @Test + public void testValidLongArgLowest() + { + Expr tooSmall = ExprEval.ofLong(0L).toExpr(); + Assert.assertEquals("0.0.0.0", eval(tooSmall)); + } + + @Test + public void testValidLongArg() + { + Assert.assertEquals(EXPECTED, eval(VALID)); + } + + @Test + public void testValidLongArgHighest() + { + Expr tooSmall = ExprEval.ofLong(0xff_ff_ff_ffL).toExpr(); + Assert.assertEquals("255.255.255.255", eval(tooSmall)); + } + + @Test + public void testInvalidLongArgTooLarge() + { + Expr tooLarge = ExprEval.ofLong(0x1_00_00_00_00L).toExpr(); + Assert.assertNull(eval(tooLarge)); + } + + @Test + public void testNullStringArg() + { + Expr nullString = ExprEval.of(null).toExpr(); + Assert.assertNull(NULL, eval(nullString)); + } + + @Test + public void testInvalidStringArgNotIPAddress() + { + Expr notIpAddress = ExprEval.of("druid.apache.org").toExpr(); + Assert.assertNull(eval(notIpAddress)); + } + + @Test + public void testInvalidStringArgIPv6Compatible() + { + Expr ipv6Compatible = ExprEval.of("::192.168.0.1").toExpr(); + Assert.assertNull(eval(ipv6Compatible)); + } + + @Test + public void testValidStringArgIPv6Mapped() + { + Expr ipv6Mapped = ExprEval.of("::ffff:192.168.0.1").toExpr(); + Assert.assertEquals(EXPECTED, eval(ipv6Mapped)); + } + + @Test + public void testValidStringArgIPv4() + { + Assert.assertEquals(EXPECTED, eval(VALID)); + } + + @Test + public void testValidStringArgUnsignedInt() + { + Expr unsignedInt = ExprEval.of("3232235521").toExpr(); + Assert.assertEquals(EXPECTED, eval(unsignedInt)); + } + + private Object eval(Expr arg) + { + Expr expr = target.apply(Collections.singletonList(arg)); + ExprEval eval = expr.eval(ExprUtils.nilBindings()); + return eval.value(); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/expression/MacroTestBase.java b/processing/src/test/java/org/apache/druid/query/expression/MacroTestBase.java new file mode 100644 index 000000000000..2c203abe5942 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/expression/MacroTestBase.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.expression; + +import org.junit.Rule; +import org.junit.rules.ExpectedException; + +public abstract class MacroTestBase +{ + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + void expectException(Class type, String message) + { + expectedException.expect(type); + expectedException.expectMessage(message); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/expression/TestExprMacroTable.java b/processing/src/test/java/org/apache/druid/query/expression/TestExprMacroTable.java index ba0758f8d3b1..c617099e3aed 100644 --- a/processing/src/test/java/org/apache/druid/query/expression/TestExprMacroTable.java +++ b/processing/src/test/java/org/apache/druid/query/expression/TestExprMacroTable.java @@ -30,6 +30,9 @@ private TestExprMacroTable() { super( ImmutableList.of( + new IPv4AddressMatchExprMacro(), + new IPv4AddressParseExprMacro(), + new IPv4AddressStringifyExprMacro(), new LikeExprMacro(), new RegexpExtractExprMacro(), new TimestampCeilExprMacro(), diff --git a/server/src/main/java/org/apache/druid/guice/ExpressionModule.java b/server/src/main/java/org/apache/druid/guice/ExpressionModule.java index 29d216d81b77..f695563d352f 100644 --- a/server/src/main/java/org/apache/druid/guice/ExpressionModule.java +++ b/server/src/main/java/org/apache/druid/guice/ExpressionModule.java @@ -26,6 +26,9 @@ import org.apache.druid.initialization.DruidModule; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.query.expression.GuiceExprMacroTable; +import org.apache.druid.query.expression.IPv4AddressMatchExprMacro; +import org.apache.druid.query.expression.IPv4AddressParseExprMacro; +import org.apache.druid.query.expression.IPv4AddressStringifyExprMacro; import org.apache.druid.query.expression.LikeExprMacro; import org.apache.druid.query.expression.RegexpExtractExprMacro; import org.apache.druid.query.expression.TimestampCeilExprMacro; @@ -44,6 +47,9 @@ public class ExpressionModule implements DruidModule { public static final List> EXPR_MACROS = ImmutableList.>builder() + .add(IPv4AddressMatchExprMacro.class) + .add(IPv4AddressParseExprMacro.class) + .add(IPv4AddressStringifyExprMacro.class) .add(LikeExprMacro.class) .add(RegexpExtractExprMacro.class) .add(TimestampCeilExprMacro.class) diff --git a/server/src/test/java/org/apache/druid/query/expression/ExprMacroTest.java b/server/src/test/java/org/apache/druid/query/expression/ExprMacroTest.java index d79cff18b474..cb5fc2fb910b 100644 --- a/server/src/test/java/org/apache/druid/query/expression/ExprMacroTest.java +++ b/server/src/test/java/org/apache/druid/query/expression/ExprMacroTest.java @@ -31,6 +31,8 @@ public class ExprMacroTest { + private static final String IPV4_STRING = "192.168.0.1"; + private static final long IPV4_LONG = 3232235521L; private static final Expr.ObjectBinding BINDINGS = Parser.withMap( ImmutableMap.builder() .put("t", DateTimes.of("2000-02-03T04:05:06").getMillis()) @@ -42,6 +44,10 @@ public class ExprMacroTest .put("z", 3.1) .put("CityOfAngels", "America/Los_Angeles") .put("spacey", " hey there ") + .put("ipv4_string", IPV4_STRING) + .put("ipv4_long", IPV4_LONG) + .put("ipv4_network", "192.168.0.0") + .put("ipv4_broadcast", "192.168.255.255") .build() ); @@ -187,6 +193,38 @@ public void testRTrim() assertExpr("rtrim(spacey, substring(spacey, 0, 4))", " hey ther"); } + @Test + public void testIPv4AddressParse() + { + Long nullLong = NullHandling.replaceWithDefault() ? NullHandling.ZERO_LONG : null; + assertExpr("ipv4address_parse(x)", nullLong); + assertExpr("ipv4address_parse(ipv4_string)", IPV4_LONG); + assertExpr("ipv4address_parse(ipv4_long)", IPV4_LONG); + assertExpr("ipv4address_parse(ipv4address_stringify(ipv4_long))", IPV4_LONG); + } + + @Test + public void testIPv4AddressStringify() + { + assertExpr("ipv4address_stringify(x)", null); + assertExpr("ipv4address_stringify(ipv4_long)", IPV4_STRING); + assertExpr("ipv4address_stringify(ipv4_string)", IPV4_STRING); + assertExpr("ipv4address_stringify(ipv4address_parse(ipv4_string))", IPV4_STRING); + } + + @Test + public void testIPv4AddressMatch() + { + assertExpr("ipv4address_match(ipv4_string, '10.0.0.0/8')", 0L); + assertExpr("ipv4address_match(ipv4_string, '192.168.0.0/16')", 1L); + assertExpr("ipv4address_match(ipv4_network, '192.168.0.0/16')", 0L); + assertExpr("ipv4address_match(ipv4_broadcast, '192.168.0.0/16')", 0L); + assertExpr("ipv4address_match(ipv4_network, '192.168.0.0/16', 'false')", 0L); + assertExpr("ipv4address_match(ipv4_broadcast, '192.168.0.0/16', 'false')", 0L); + assertExpr("ipv4address_match(ipv4_network, '192.168.0.0/16', 'true')", 1L); + assertExpr("ipv4address_match(ipv4_broadcast, '192.168.0.0/16', 'true')", 1L); + } + private void assertExpr(final String expression, final Object expectedResult) { final Expr expr = Parser.parse(expression, LookupEnabledTestExprMacroTable.INSTANCE); From c8bcc12cb293a3a977a7fc866ff67d5a129f8670 Mon Sep 17 00:00:00 2001 From: Chi Cao Minh Date: Tue, 30 Jul 2019 12:55:07 -0700 Subject: [PATCH 2/5] Fix licenses and null handling --- licenses.yaml | 10 ++++++++++ .../query/expression/IPv4AddressMatchExprMacro.java | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/licenses.yaml b/licenses.yaml index 15458fcdc65c..10986dc9a2b2 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -362,6 +362,16 @@ libraries: --- +name: Apache Commons Net +license_category: binary +module: java-core +license_name: Apache License version 2.0 +version: 3.6 +libraries: + - commons-net: commons-net + +--- + name: Apache Commons Pool license_category: binary module: java-core diff --git a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java index 81b44e1eb101..140468867121 100644 --- a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java +++ b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java @@ -105,7 +105,7 @@ public ExprEval eval(final ObjectBinding bindings) match = isStringMatch(eval.asString()); break; case LONG: - match = isLongMatch(eval.asLong()); + match = !eval.isNumericNull() && isLongMatch(eval.asLong()); break; default: match = false; From 76e6c7533380265f8bd168206810d91e2cd78c99 Mon Sep 17 00:00:00 2001 From: Chi Cao Minh Date: Wed, 31 Jul 2019 15:11:29 -0700 Subject: [PATCH 3/5] Simplify IPv4 expressions --- docs/content/misc/math-expr.md | 12 ++- .../druid/query/expression/ExprUtils.java | 15 +++- .../expression/IPv4AddressExprUtils.java | 51 ++++++++--- .../expression/IPv4AddressMatchExprMacro.java | 82 +++-------------- .../expression/IPv4AddressParseExprMacro.java | 78 +++++----------- .../IPv4AddressStringifyExprMacro.java | 53 +++-------- .../expression/IPv4AddressExprUtilsTest.java | 89 +++++++++++++++---- ...ava => IPv4AddressMatchExprMacroTest.java} | 64 ++----------- .../IPv4AddressParseExprMacroTest.java | 6 +- ...=> IPv4AddressStringifyExprMacroTest.java} | 6 +- 10 files changed, 196 insertions(+), 260 deletions(-) rename processing/src/test/java/org/apache/druid/query/expression/{IPv4AddressMatchMacroTest.java => IPv4AddressMatchExprMacroTest.java} (74%) rename processing/src/test/java/org/apache/druid/query/expression/{IPv4AddressStringifyMacroTest.java => IPv4AddressStringifyExprMacroTest.java} (95%) diff --git a/docs/content/misc/math-expr.md b/docs/content/misc/math-expr.md index fc72a5f1773c..0aa58b37cf6d 100644 --- a/docs/content/misc/math-expr.md +++ b/docs/content/misc/math-expr.md @@ -198,8 +198,14 @@ See javadoc of java.lang.Math for detailed explanation for each function. ## IP Address Functions + +For the IPv4 address functions, the `address` argument can either be an IPv4 dotted-decimal string +(e.g., "192.168.0.1") or an IP address represented as a long (e.g., 3232235521). The `subnet` +argument should be a string formatted as an IPv4 address subnet in CIDR notation (e.g., +"192.168.0.0/16"). + | function | description | | --- | --- | -| ipv4address_match(address, subnet [,inclusive]) | Returns 1 if the string or long IP `address` belongs to the `subnet` CIDR notation literal string, else 0. The optional literal boolean `inclusive` parameter indicates whether the network and broadcast IP addresses should be included in the subnet and defaults to `false`. This function is more efficient when operating on IP addresses as longs instead of strings.| -| ipv4address_parse(address) | Parses string or long into an IPv4 address as a long. Returns null if `address` cannot be represented as an IPv4 address. | -| ipv4address_stringify(address) | Converts string or long IPv4 address into an IPv4 address dotted-decimal notated string. Return null if `address` cannot be represented as an IPv4 address.| +| ipv4_match(address, subnet) | Returns 1 if the `address` belongs to the `subnet` literal, else 0. If `address` is not a valid IPv4 address, then 0 is returned. This function is more efficient if `address` is a long instead of a string.| +| ipv4_parse(address) | Parses `address` into an IPv4 address stored as a long. If `address` is a long that is a valid IPv4 address, then it is passed through. Returns null if `address` cannot be represented as an IPv4 address. | +| ipv4_stringify(address) | Converts `address` into an IPv4 address dotted-decimal string. If `address` is a string that is a valid IPv4 address, then it is passed through. Returns null if `address` cannot be represented as an IPv4 address.| diff --git a/processing/src/main/java/org/apache/druid/query/expression/ExprUtils.java b/processing/src/main/java/org/apache/druid/query/expression/ExprUtils.java index 1b9cb9df16ab..75fe2740057c 100644 --- a/processing/src/main/java/org/apache/druid/query/expression/ExprUtils.java +++ b/processing/src/main/java/org/apache/druid/query/expression/ExprUtils.java @@ -19,6 +19,7 @@ package org.apache.druid.query.expression; +import com.google.common.base.Preconditions; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.IAE; @@ -41,7 +42,7 @@ public static Expr.ObjectBinding nilBindings() return NIL_BINDINGS; } - public static DateTimeZone toTimeZone(final Expr timeZoneArg) + static DateTimeZone toTimeZone(final Expr timeZoneArg) { if (!timeZoneArg.isLiteral()) { throw new IAE("Time zone must be a literal"); @@ -51,7 +52,7 @@ public static DateTimeZone toTimeZone(final Expr timeZoneArg) return literalValue == null ? DateTimeZone.UTC : DateTimes.inferTzFromString((String) literalValue); } - public static PeriodGranularity toPeriodGranularity( + static PeriodGranularity toPeriodGranularity( final Expr periodArg, @Nullable final Expr originArg, @Nullable final Expr timeZoneArg, @@ -87,4 +88,14 @@ public static PeriodGranularity toPeriodGranularity( return new PeriodGranularity(period, origin, timeZone); } + static String createErrMsg(String functionName, String msg) + { + String prefix = "Function[" + functionName + "] "; + return prefix + msg; + } + + static void checkLiteralArgument(String functionName, Expr arg, String argName) + { + Preconditions.checkArgument(arg.isLiteral(), createErrMsg(functionName, argName + " arg must be a literal")); + } } diff --git a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java index 74bf58706339..8e42b58e6b35 100644 --- a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java +++ b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java @@ -24,11 +24,14 @@ import javax.annotation.Nullable; import java.net.Inet4Address; import java.net.InetAddress; +import java.util.regex.Pattern; class IPv4AddressExprUtils { + private static final Pattern IPV4_PATTERN = Pattern.compile("(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})"); + /** - * @return True if argument cannot be represented by an unsigned integer (4 bytes) + * @return True if argument cannot be represented by an unsigned integer (4 bytes), else false */ static boolean overflowsUnsignedInt(long value) { @@ -36,24 +39,44 @@ static boolean overflowsUnsignedInt(long value) } /** - * @return IPv4 address dotted-decimal notated string or null if the argument is not a valid IPv4 address string or - * IPv6 IPv4-mapped address string. + * @return True if argument is a valid IPv4 address dotted-decimal string */ + static boolean isValidAddress(@Nullable String string) + { + return string != null && IPV4_PATTERN.matcher(string).matches(); + } + @Nullable - static String extractIPv4Address(String string) + static Inet4Address parse(@Nullable String string) { - if (string != null) { - try { - InetAddress address = InetAddresses.forString(string); - if (address instanceof Inet4Address) { - return address.getHostAddress(); - } - } - catch (IllegalArgumentException ignored) { - // fall through + // Explicitly check for valid address to avoid overhead of InetAddresses#forString() potentially + // throwing IllegalArgumentException + if (isValidAddress(string)) { + // Do not use java.lang.InetAddress#getByName() as it may do DNS lookups + InetAddress address = InetAddresses.forString(string); + if (address instanceof Inet4Address) { + return (Inet4Address) address; } } - return null; } + + static Inet4Address parse(int value) + { + return InetAddresses.fromInteger(value); + } + + /** + * @return IPv4 address dotted-decimal notated string + */ + static String toString(Inet4Address address) + { + return address.getHostAddress(); + } + + static long toLong(Inet4Address address) + { + int value = InetAddresses.coerceToInteger(address); + return Integer.toUnsignedLong(value); + } } diff --git a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java index 140468867121..6cc94ac61133 100644 --- a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java +++ b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java @@ -19,7 +19,6 @@ package org.apache.druid.query.expression; -import com.google.common.base.Preconditions; import org.apache.commons.net.util.SubnetUtils; import org.apache.druid.java.util.common.IAE; import org.apache.druid.math.expr.Expr; @@ -35,26 +34,17 @@ * Implements an expression that checks if an IPv4 address belongs to a particular subnet. * * Expression signatures: - * - boolean ipv4address_match(string address, string subnet) - * - boolean ipv4address_match(string address, string subnet, boolean inclusive) - * - boolean ipv4address_match(long address, string subnet) - * - boolean ipv4address_match(long address, string subnet, boolean inclusive) + * - long ipv4_match(string address, string subnet) + * - long ipv4_match(long address, string subnet) * * Valid "address" argument formats are: * - unsigned int long (e.g., 3232235521) - * - unsigned int string (e.g., "3232235521") - * - IPv4 address dotted-decimal notation string (e.g., "198.168.0.1") - * - IPv6 IPv4-mapped address string (e.g., "::ffff:192.168.0.1") + * - IPv4 address dotted-decimal string (e.g., "198.168.0.1") * * The argument format for the "subnet" argument should be a literal in CIDR notation * (e.g., "198.168.0.0/16"). * - * An optional "inclusive" argument should be a boolean literal (e.g., 1, 0, "true", or "false") that - * indicates whether the network and the broadcast addresses should be considered part of the - * subnet. When this argument is absent, its value defaults to "false". - * - * The overloaded signature allows applying the expression to a dimension with mixed string and long - * representations of IPv4 addresses. + * If the "address" argument does not represent an IPv4 address then false is returned. * * * @see IPv4AddressParseExprMacro @@ -62,9 +52,8 @@ */ public class IPv4AddressMatchExprMacro implements ExprMacroTable.ExprMacro { - public static final String NAME = "ipv4address_match"; + public static final String NAME = "ipv4_match"; private static final int ARG_SUBNET = 1; - private static final int ARG_INCLUSIVE = 2; @Override public String name() @@ -75,13 +64,11 @@ public String name() @Override public Expr apply(final List args) { - if (args.size() < 2 || 3 < args.size()) { - throw new IAE(createErrMsg("must have 2-3 arguments")); + if (args.size() != 2) { + throw new IAE(ExprUtils.createErrMsg(name(), "must have 2 arguments")); } - boolean inclusive = getInclusive(args); - SubnetUtils.SubnetInfo subnetInfo = getSubnetInfo(args, inclusive); - + SubnetUtils.SubnetInfo subnetInfo = getSubnetInfo(args); Expr arg = args.get(0); class IPv4AddressMatchExpr extends ExprMacroTable.BaseScalarUnivariateMacroFunctionExpr @@ -115,27 +102,7 @@ public ExprEval eval(final ObjectBinding bindings) private boolean isStringMatch(String stringValue) { - boolean match; - String ipv4 = IPv4AddressExprUtils.extractIPv4Address(stringValue); - if (ipv4 == null) { - match = isLongMatch(stringValue); - } else { - match = subnetInfo.isInRange(ipv4); - } - return match; - } - - private boolean isLongMatch(String stringValue) - { - boolean match; - try { - long longValue = Long.parseLong(stringValue); - match = isLongMatch(longValue); - } - catch (NumberFormatException ignored) { - match = false; - } - return match; + return IPv4AddressExprUtils.isValidAddress(stringValue) && subnetInfo.isInRange(stringValue); } private boolean isLongMatch(long longValue) @@ -154,34 +121,11 @@ public Expr visit(Shuttle shuttle) return new IPv4AddressMatchExpr(arg, subnetInfo); } - private String createErrMsg(String msg) - { - String prefix = "Function[" + name() + "] "; - return prefix + msg; - } - - private boolean getInclusive(List args) - { - boolean inclusive = false; - if (ARG_INCLUSIVE < args.size()) { - Expr arg = args.get(ARG_INCLUSIVE); - checkLiteralArgument(arg, "inclusive"); - ExprEval eval = arg.eval(ExprUtils.nilBindings()); - inclusive = eval.asBoolean(); - } - return inclusive; - } - - private void checkLiteralArgument(Expr arg, String name) - { - Preconditions.checkArgument(arg.isLiteral(), createErrMsg(name + " arg must be a literal")); - } - - private SubnetUtils.SubnetInfo getSubnetInfo(List args, boolean inclusive) + private SubnetUtils.SubnetInfo getSubnetInfo(List args) { String subnetArgName = "subnet"; Expr arg = args.get(ARG_SUBNET); - checkLiteralArgument(arg, subnetArgName); + ExprUtils.checkLiteralArgument(name(), arg, subnetArgName); String subnet = (String) arg.getLiteralValue(); SubnetUtils subnetUtils; @@ -189,9 +133,9 @@ private SubnetUtils.SubnetInfo getSubnetInfo(List args, boolean inclusive) subnetUtils = new SubnetUtils(subnet); } catch (IllegalArgumentException e) { - throw new IAE(e, createErrMsg(subnetArgName + " arg has an invalid format: " + subnet)); + throw new IAE(e, ExprUtils.createErrMsg(name(), subnetArgName + " arg has an invalid format: " + subnet)); } - subnetUtils.setInclusiveHostCount(inclusive); + subnetUtils.setInclusiveHostCount(true); // make network and broadcast addresses match return subnetUtils.getInfo(); } diff --git a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressParseExprMacro.java b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressParseExprMacro.java index f410a98ceb49..569c037148ee 100644 --- a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressParseExprMacro.java +++ b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressParseExprMacro.java @@ -19,37 +19,27 @@ package org.apache.druid.query.expression; -import com.google.common.net.InetAddresses; import org.apache.druid.java.util.common.IAE; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprEval; import org.apache.druid.math.expr.ExprMacroTable; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.net.Inet4Address; -import java.net.InetAddress; import java.util.List; /** * 
- * Implements an expression that parses string or long into an IPv4 address stored (as an unsigned
+ * Implements an expression that parses a string or long into an IPv4 address stored (as an unsigned
  * int) in a long.
  *
  * Expression signatures:
- * - long ipv4address_parse(string)
- * - long ipv4address_parse(long)
- *
- * Valid argument formats are:
- * - IPv4 address dotted-decimal notation string (e.g., "198.168.0.1")
- * - IPv6 IPv4-mapped adress string (e.g., "::ffff:192.168.0.1")
- * - unsigned int long (e.g., 3232235521)
- * - unsigned int string (e.g., "3232235521")
+ * - long ipv4_parse(string)
+ * - long ipv4_parse(long)
  *
+ * String arguments should be formatted as a dotted-decimal.
+ * Long arguments that can be represented as an IPv4 address are passed through.
  * Invalid arguments return null.
- *
- * The overloaded signature allows applying the expression to a dimension with mixed string and long
- * representations of IPv4 addresses.
  *
* * @see IPv4AddressStringifyExprMacro @@ -57,7 +47,7 @@ */ public class IPv4AddressParseExprMacro implements ExprMacroTable.ExprMacro { - public static final String NAME = "ipv4address_parse"; + public static final String NAME = "ipv4_parse"; @Override public String name() @@ -69,7 +59,7 @@ public String name() public Expr apply(final List args) { if (args.size() != 1) { - throw new IAE("Function[%s] must have 1 argument", name()); + throw new IAE(ExprUtils.createErrMsg(name(), "must have 1 argument")); } Expr arg = args.get(0); @@ -85,20 +75,15 @@ private IPv4AddressParseExpr(Expr arg) @Override public ExprEval eval(final ObjectBinding bindings) { - String stringValue = arg.eval(bindings).asString(); - if (stringValue == null) { - return ExprEval.ofLong(null); + ExprEval eval = arg.eval(bindings); + switch (eval.type()) { + case STRING: + return evalAsString(eval); + case LONG: + return evalAsLong(eval); + default: + return ExprEval.ofLong(null); } - - // Assume use cases in order of most frequent to least are: - // 1) convert string to long - // 2) convert long to long - Long value = parseAsString(stringValue); - if (value == null) { - value = parseAsLong(stringValue); - } - - return ExprEval.ofLong(value); } @Override @@ -112,35 +97,18 @@ public Expr visit(Shuttle shuttle) return new IPv4AddressParseExpr(arg); } - @Nullable - private static Long parseAsString(String stringValue) + private static ExprEval evalAsString(ExprEval eval) { - try { - // Do not use java.lang.InetAddress#getByName() as it may do DNS lookups - InetAddress address = InetAddresses.forString(stringValue); - if (address instanceof Inet4Address) { - int value = InetAddresses.coerceToInteger(address); - return Integer.toUnsignedLong(value); - } - } - catch (IllegalArgumentException ignored) { - // fall through (Invalid IPv4 adddress string) - } - return null; + Inet4Address address = IPv4AddressExprUtils.parse(eval.asString()); + Long value = address == null ? null : IPv4AddressExprUtils.toLong(address); + return ExprEval.ofLong(value); } - @Nullable - private static Long parseAsLong(String stringValue) + private static ExprEval evalAsLong(ExprEval eval) { - try { - Long value = Long.valueOf(stringValue); - if (!IPv4AddressExprUtils.overflowsUnsignedInt(value)) { - return value; - } - } - catch (NumberFormatException ignored) { - // fall through + if (eval.isNumericNull() || !IPv4AddressExprUtils.overflowsUnsignedInt(eval.asLong())) { + return eval; } - return null; + return ExprEval.ofLong(null); } } diff --git a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacro.java b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacro.java index c70557992964..d9afa5872777 100644 --- a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacro.java +++ b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacro.java @@ -19,7 +19,6 @@ package org.apache.druid.query.expression; -import com.google.common.net.InetAddresses; import org.apache.druid.java.util.common.IAE; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprEval; @@ -31,23 +30,15 @@ /** * 
- * Implements an expression that converts an IPv4 address stored (as an unsigned int) in a long or
- * stored as a string into an IPv4 address dotted-decimal notated string (e.g., "192.168.0.1").
+ * Implements an expression that converts a long or a string into an IPv4 address dotted-decimal string.
  *
  * Expression signatures:
- * - string ipv4address_stringify(long)
- * - string ipv4address_stringify(string)
- *
- * Valid argument formats are:
- * - unsigned int long (e.g., 3232235521)
- * - unsigned int string (e.g., "3232235521")
- * - IPv4 address dotted-decimal notation string (e.g., "198.168.0.1")
- * - IPv6 IPv4-mapped address string (e.g., "::ffff:192.168.0.1")
+ * - string ipv4_stringify(long)
+ * - string ipv4_stringify(string)
  *
+ * Long arguments that can be represented as an IPv4 address are converted to a dotted-decimal string.
+ * String arguments that are dotted-decimal IPv4 addresses are passed through.
  * Invalid arguments return null.
- *
- * The overloaded signature allows applying the expression to a dimension with mixed string and long
- * representations of IPv4 addresses.
  *
* * @see IPv4AddressParseExprMacro @@ -55,7 +46,7 @@ */ public class IPv4AddressStringifyExprMacro implements ExprMacroTable.ExprMacro { - public static final String NAME = "ipv4address_stringify"; + public static final String NAME = "ipv4_stringify"; @Override public String name() @@ -67,7 +58,7 @@ public String name() public Expr apply(final List args) { if (args.size() != 1) { - throw new IAE("Function[%s] must have 1 argument", name()); + throw new IAE(ExprUtils.createErrMsg(name(), "must have 1 argument")); } Expr arg = args.get(0); @@ -107,42 +98,24 @@ public Expr visit(Shuttle shuttle) private static ExprEval evalAsString(ExprEval eval) { - String stringValue = eval.asString(); - if (stringValue == null) { + if (IPv4AddressExprUtils.isValidAddress(eval.asString())) { return eval; } - - // Assume use cases in order of most frequent to least: - // 1) convert long to string - // 2) convert string to string - - try { - long longValue = Long.parseLong(stringValue); - return evalLong(longValue); - } - catch (NumberFormatException ignored) { - // fall through - } - - return ExprEval.of(IPv4AddressExprUtils.extractIPv4Address(stringValue)); + return ExprEval.of(null); } private static ExprEval evalAsLong(ExprEval eval) { if (eval.isNumericNull()) { - return eval; + return ExprEval.of(null); } - return evalLong(eval.asLong()); - } - - private static ExprEval evalLong(long longValue) - { + long longValue = eval.asLong(); if (IPv4AddressExprUtils.overflowsUnsignedInt(longValue)) { return ExprEval.of(null); } - Inet4Address address = InetAddresses.fromInteger((int) longValue); - return ExprEval.of(address.getHostAddress()); + Inet4Address address = IPv4AddressExprUtils.parse((int) longValue); + return ExprEval.of(IPv4AddressExprUtils.toString(address)); } } diff --git a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressExprUtilsTest.java b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressExprUtilsTest.java index fc239f6f0796..653430ce7122 100644 --- a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressExprUtilsTest.java +++ b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressExprUtilsTest.java @@ -22,8 +22,17 @@ import org.junit.Assert; import org.junit.Test; +import java.net.Inet4Address; +import java.net.InetAddress; +import java.net.UnknownHostException; + public class IPv4AddressExprUtilsTest { + private static final String IPV4 = "192.168.0.1"; + private static final String IPV6_MAPPED = "::ffff:192.168.0.1"; + private static final String IPV6_COMPATIBLE = "::192.168.0.1"; + private static final String NOT_IP_ADDRESS = "druid.apache.org"; + @Test public void testOverflowsUnsignedIntTooLow() { @@ -55,37 +64,87 @@ public void testOverflowsUnsignedIntTooHigh() } @Test - public void testExtractIPv4AddressNull() + public void testIsValidAddressNull() + { + Assert.assertFalse(IPv4AddressExprUtils.isValidAddress(null)); + } + + @Test + public void testIsValidAddressIPv4() + { + Assert.assertTrue(IPv4AddressExprUtils.isValidAddress(IPV4)); + } + + @Test + public void testIsValidAddressIPv6Mapped() + { + Assert.assertFalse(IPv4AddressExprUtils.isValidAddress(IPV6_MAPPED)); + } + + @Test + public void testIsValidAddressIPv6Compatible() + { + Assert.assertFalse(IPv4AddressExprUtils.isValidAddress(IPV6_COMPATIBLE)); + } + + @Test + public void testIsValidAddressNotIpAddress() + { + Assert.assertFalse(IPv4AddressExprUtils.isValidAddress(NOT_IP_ADDRESS)); + } + + @Test + public void testParseNull() + { + Assert.assertNull(IPv4AddressExprUtils.parse(null)); + } + + @Test + public void testParseIPv4() + { + Inet4Address address = IPv4AddressExprUtils.parse(IPV4); + Assert.assertNotNull(address); + Assert.assertEquals(IPV4, address.getHostAddress()); + } + + @Test + public void testParseIPv6Mapped() + { + Assert.assertNull(IPv4AddressExprUtils.parse(IPV6_MAPPED)); + } + + @Test + public void testParseIPv6Compatible() { - Assert.assertNull(IPv4AddressExprUtils.extractIPv4Address(null)); + Assert.assertNull(IPv4AddressExprUtils.parse(IPV6_COMPATIBLE)); } @Test - public void testExtractIPv4AddressIPv4() + public void testParseNotIpAddress() { - String ipv4 = "192.168.0.1"; - Assert.assertEquals(ipv4, IPv4AddressExprUtils.extractIPv4Address(ipv4)); + Assert.assertNull(IPv4AddressExprUtils.parse(NOT_IP_ADDRESS)); } @Test - public void testExtractIPv4AddressIPv6Mapped() + public void testParseInt() { - String ipv4 = "192.168.0.1"; - String ipv6Mapped = "::ffff:" + ipv4; - Assert.assertEquals(ipv4, IPv4AddressExprUtils.extractIPv4Address(ipv6Mapped)); + Inet4Address address = IPv4AddressExprUtils.parse((int) 0xC0A80001L); + Assert.assertArrayEquals(new byte[]{(byte) 0xC0, (byte) 0xA8, 0x00, 0x01}, address.getAddress()); } @Test - public void testExtractIPv4AddressIPv6Compatible() + public void testToString() throws UnknownHostException { - String ipv6Compatible = "::192.168.0.1"; - Assert.assertNull(IPv4AddressExprUtils.extractIPv4Address(ipv6Compatible)); + byte[] bytes = new byte[]{(byte) 192, (byte) 168, 0, 1}; + InetAddress address = InetAddress.getByAddress(bytes); + Assert.assertEquals("192.168.0.1", IPv4AddressExprUtils.toString((Inet4Address) address)); } @Test - public void testExtractIPv4AddressInvalid() + public void testToLong() throws UnknownHostException { - String notIpAddress = "druid.apache.org"; - Assert.assertNull(IPv4AddressExprUtils.extractIPv4Address(notIpAddress)); + byte[] bytes = new byte[]{(byte) 0xC0, (byte) 0xA8, 0x00, 0x01}; + InetAddress address = InetAddress.getByAddress(bytes); + Assert.assertEquals(0xC0A80001L, IPv4AddressExprUtils.toLong((Inet4Address) address)); } } diff --git a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchMacroTest.java b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacroTest.java similarity index 74% rename from processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchMacroTest.java rename to processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacroTest.java index 9f3b66bda096..c4f7d9c12a2c 100644 --- a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchMacroTest.java +++ b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacroTest.java @@ -29,7 +29,7 @@ import java.util.Arrays; import java.util.Collections; -public class IPv4AddressMatchMacroTest extends MacroTestBase +public class IPv4AddressMatchExprMacroTest extends MacroTestBase { private static final Expr IPV4 = ExprEval.of("192.168.0.1").toExpr(); private static final Expr IPV4_LONG = ExprEval.of(3232235521L).toExpr(); @@ -40,8 +40,6 @@ public class IPv4AddressMatchMacroTest extends MacroTestBase private static final Expr IPV6_MAPPED = ExprEval.of("::ffff:192.168.0.1").toExpr(); private static final Expr SUBNET_192_168 = ExprEval.of("192.168.0.0/16").toExpr(); private static final Expr SUBNET_10 = ExprEval.of("10.0.0.0/8").toExpr(); - private static final Expr INCLUSIVE = ExprEval.of("true").toExpr(); - private static final Expr NOT_INCLUSIVE = ExprEval.of("false").toExpr(); private static final Expr NOT_LITERAL = new NotLiteralExpr(null); private IPv4AddressMatchExprMacro target; @@ -55,7 +53,7 @@ public void setUp() @Test public void testTooFewArgs() { - expectException(IllegalArgumentException.class, "must have 2-3 arguments"); + expectException(IllegalArgumentException.class, "must have 2 arguments"); target.apply(Collections.emptyList()); } @@ -63,9 +61,9 @@ public void testTooFewArgs() @Test public void testTooManyArgs() { - expectException(IllegalArgumentException.class, "must have 2-3 arguments"); + expectException(IllegalArgumentException.class, "must have 2 arguments"); - target.apply(Arrays.asList(IPV4, SUBNET_192_168, INCLUSIVE, NOT_LITERAL)); + target.apply(Arrays.asList(IPV4, SUBNET_192_168, NOT_LITERAL)); } @Test @@ -85,14 +83,6 @@ public void testSubnetArgInvalid() target.apply(Arrays.asList(IPV4, invalidSubnet)); } - @Test - public void testInclusiveArgNotLiteral() - { - expectException(IllegalArgumentException.class, "inclusive arg must be a literal"); - - target.apply(Arrays.asList(IPV4, SUBNET_192_168, NOT_LITERAL)); - } - @Test public void testNullStringArg() { @@ -129,7 +119,7 @@ public void testNotMatchingStringArgIPv4() @Test public void testMatchingStringArgIPv6Mapped() { - Assert.assertTrue(eval(IPV6_MAPPED, SUBNET_192_168)); + Assert.assertFalse(eval(IPV6_MAPPED, SUBNET_192_168)); } @Test @@ -172,7 +162,7 @@ public void testNotMatchingLongArg() @Test public void testMatchingStringArgUnsignedInt() { - Assert.assertTrue(eval(IPV4_UINT, SUBNET_192_168)); + Assert.assertFalse(eval(IPV4_UINT, SUBNET_192_168)); } @Test @@ -185,47 +175,9 @@ public void testNotMatchingStringArgUnsignedInt() public void testInclusive() { Expr subnet = SUBNET_192_168; - Assert.assertTrue(eval(IPV4_NETWORK, subnet, INCLUSIVE)); - Assert.assertTrue(eval(IPV4, subnet, INCLUSIVE)); - Assert.assertTrue(eval(IPV4_BROADCAST, subnet, INCLUSIVE)); - } - - @Test - public void testNotInclusive() - { - Expr subnet = SUBNET_192_168; - Assert.assertFalse(eval(IPV4_NETWORK, subnet, NOT_INCLUSIVE)); - Assert.assertTrue(eval(IPV4, subnet, NOT_INCLUSIVE)); - Assert.assertFalse(eval(IPV4_BROADCAST, subnet, NOT_INCLUSIVE)); - } - - @Test - public void testDefaultNotInclusive() - { - Expr subnet = SUBNET_192_168; - Assert.assertFalse(eval(IPV4_NETWORK, subnet)); + Assert.assertTrue(eval(IPV4_NETWORK, subnet)); Assert.assertTrue(eval(IPV4, subnet)); - Assert.assertFalse(eval(IPV4_BROADCAST, subnet)); - } - - @Test - public void testInclusiveAsLong() - { - Expr subnet = SUBNET_192_168; - Expr inclusive = ExprEval.of(1L).toExpr(); - Assert.assertTrue(eval(IPV4_NETWORK, subnet, inclusive)); - Assert.assertTrue(eval(IPV4, subnet, inclusive)); - Assert.assertTrue(eval(IPV4_BROADCAST, subnet, inclusive)); - } - - @Test - public void testNotInclusiveAsLong() - { - Expr subnet = SUBNET_192_168; - Expr notInclusive = ExprEval.of(0L).toExpr(); - Assert.assertFalse(eval(IPV4_NETWORK, subnet, notInclusive)); - Assert.assertTrue(eval(IPV4, subnet, notInclusive)); - Assert.assertFalse(eval(IPV4_BROADCAST, subnet, notInclusive)); + Assert.assertTrue(eval(IPV4_BROADCAST, subnet)); } private boolean eval(Expr... args) diff --git a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressParseExprMacroTest.java b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressParseExprMacroTest.java index 5951ff6aa844..2bf392141d51 100644 --- a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressParseExprMacroTest.java +++ b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressParseExprMacroTest.java @@ -63,7 +63,7 @@ public void testTooManyArgs() public void testNullStringArg() { Expr nullString = ExprEval.of(null).toExpr(); - Assert.assertEquals(NULL, eval(nullString)); + Assert.assertSame(NULL, eval(nullString)); } @Test @@ -98,7 +98,7 @@ public void testInvalidStringArgIPv6Compatible() public void testValidStringArgIPv6Mapped() { Expr ipv6Mapped = ExprEval.of("::ffff:192.168.0.1").toExpr(); - Assert.assertEquals(EXPECTED, eval(ipv6Mapped)); + Assert.assertEquals(NULL, eval(ipv6Mapped)); } @Test @@ -111,7 +111,7 @@ public void testValidStringArgIPv4() public void testValidStringArgUnsignedInt() { Expr unsignedInt = ExprEval.of("3232235521").toExpr(); - Assert.assertEquals(EXPECTED, eval(unsignedInt)); + Assert.assertEquals(NULL, eval(unsignedInt)); } @Test diff --git a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressStringifyMacroTest.java b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacroTest.java similarity index 95% rename from processing/src/test/java/org/apache/druid/query/expression/IPv4AddressStringifyMacroTest.java rename to processing/src/test/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacroTest.java index 2a20c95f1149..602d00cfcc4e 100644 --- a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressStringifyMacroTest.java +++ b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressStringifyExprMacroTest.java @@ -29,7 +29,7 @@ import java.util.Arrays; import java.util.Collections; -public class IPv4AddressStringifyMacroTest extends MacroTestBase +public class IPv4AddressStringifyExprMacroTest extends MacroTestBase { private static final Expr VALID = ExprEval.of(3232235521L).toExpr(); private static final String EXPECTED = "192.168.0.1"; @@ -132,7 +132,7 @@ public void testInvalidStringArgIPv6Compatible() public void testValidStringArgIPv6Mapped() { Expr ipv6Mapped = ExprEval.of("::ffff:192.168.0.1").toExpr(); - Assert.assertEquals(EXPECTED, eval(ipv6Mapped)); + Assert.assertNull(eval(ipv6Mapped)); } @Test @@ -145,7 +145,7 @@ public void testValidStringArgIPv4() public void testValidStringArgUnsignedInt() { Expr unsignedInt = ExprEval.of("3232235521").toExpr(); - Assert.assertEquals(EXPECTED, eval(unsignedInt)); + Assert.assertNull(eval(unsignedInt)); } private Object eval(Expr arg) From 28fddbb456d02aa6a299f8c9b301b9e4459810de Mon Sep 17 00:00:00 2001 From: Chi Cao Minh Date: Wed, 31 Jul 2019 15:49:02 -0700 Subject: [PATCH 4/5] Fix tests --- .../druid/query/expression/ExprMacroTest.java | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/server/src/test/java/org/apache/druid/query/expression/ExprMacroTest.java b/server/src/test/java/org/apache/druid/query/expression/ExprMacroTest.java index cb5fc2fb910b..74e74394570a 100644 --- a/server/src/test/java/org/apache/druid/query/expression/ExprMacroTest.java +++ b/server/src/test/java/org/apache/druid/query/expression/ExprMacroTest.java @@ -197,32 +197,28 @@ public void testRTrim() public void testIPv4AddressParse() { Long nullLong = NullHandling.replaceWithDefault() ? NullHandling.ZERO_LONG : null; - assertExpr("ipv4address_parse(x)", nullLong); - assertExpr("ipv4address_parse(ipv4_string)", IPV4_LONG); - assertExpr("ipv4address_parse(ipv4_long)", IPV4_LONG); - assertExpr("ipv4address_parse(ipv4address_stringify(ipv4_long))", IPV4_LONG); + assertExpr("ipv4_parse(x)", nullLong); + assertExpr("ipv4_parse(ipv4_string)", IPV4_LONG); + assertExpr("ipv4_parse(ipv4_long)", IPV4_LONG); + assertExpr("ipv4_parse(ipv4_stringify(ipv4_long))", IPV4_LONG); } @Test public void testIPv4AddressStringify() { - assertExpr("ipv4address_stringify(x)", null); - assertExpr("ipv4address_stringify(ipv4_long)", IPV4_STRING); - assertExpr("ipv4address_stringify(ipv4_string)", IPV4_STRING); - assertExpr("ipv4address_stringify(ipv4address_parse(ipv4_string))", IPV4_STRING); + assertExpr("ipv4_stringify(x)", null); + assertExpr("ipv4_stringify(ipv4_long)", IPV4_STRING); + assertExpr("ipv4_stringify(ipv4_string)", IPV4_STRING); + assertExpr("ipv4_stringify(ipv4_parse(ipv4_string))", IPV4_STRING); } @Test public void testIPv4AddressMatch() { - assertExpr("ipv4address_match(ipv4_string, '10.0.0.0/8')", 0L); - assertExpr("ipv4address_match(ipv4_string, '192.168.0.0/16')", 1L); - assertExpr("ipv4address_match(ipv4_network, '192.168.0.0/16')", 0L); - assertExpr("ipv4address_match(ipv4_broadcast, '192.168.0.0/16')", 0L); - assertExpr("ipv4address_match(ipv4_network, '192.168.0.0/16', 'false')", 0L); - assertExpr("ipv4address_match(ipv4_broadcast, '192.168.0.0/16', 'false')", 0L); - assertExpr("ipv4address_match(ipv4_network, '192.168.0.0/16', 'true')", 1L); - assertExpr("ipv4address_match(ipv4_broadcast, '192.168.0.0/16', 'true')", 1L); + assertExpr("ipv4_match(ipv4_string, '10.0.0.0/8')", 0L); + assertExpr("ipv4_match(ipv4_string, '192.168.0.0/16')", 1L); + assertExpr("ipv4_match(ipv4_network, '192.168.0.0/16')", 1L); + assertExpr("ipv4_match(ipv4_broadcast, '192.168.0.0/16')", 1L); } private void assertExpr(final String expression, final Object expectedResult) From 4bd2a5738a1b13e312c40c57c7a278c210b3adb1 Mon Sep 17 00:00:00 2001 From: Chi Cao Minh Date: Wed, 31 Jul 2019 22:13:06 -0700 Subject: [PATCH 5/5] Fix check for valid ipv4 address string --- .../expression/IPv4AddressExprUtils.java | 4 +- .../expression/IPv4AddressExprUtilsTest.java | 60 ++++++++++++++++--- 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java index 8e42b58e6b35..4d87b38b50f4 100644 --- a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java +++ b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java @@ -28,7 +28,9 @@ class IPv4AddressExprUtils { - private static final Pattern IPV4_PATTERN = Pattern.compile("(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})"); + private static final Pattern IPV4_PATTERN = Pattern.compile( + "^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$" + ); /** * @return True if argument cannot be represented by an unsigned integer (4 bytes), else false diff --git a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressExprUtilsTest.java b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressExprUtilsTest.java index 653430ce7122..a13ee9b94dab 100644 --- a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressExprUtilsTest.java +++ b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressExprUtilsTest.java @@ -25,13 +25,42 @@ import java.net.Inet4Address; import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.List; public class IPv4AddressExprUtilsTest { - private static final String IPV4 = "192.168.0.1"; + private static final List VALID_IPV4_ADDRESSES = Arrays.asList( + "192.168.0.1", + "0.0.0.0", + "255.255.255.255", + "255.0.0.0", + "0.255.0.0", + "0.0.255.0", + "0.0.0.255" + ); + private static final List INVALID_IPV4_ADDRESSES = Arrays.asList( + "druid.apache.org", // no octets are numbers + "a.b.c.d", // no octets are numbers + "abc.def.ghi.jkl", // no octets are numbers + "1..3.4", // missing octet + "1.2..4", // missing octet + "1.2.3..", // missing octet + "1", // missing octets + "1.2", // missing octets + "1.2.3", // missing octet + "1.2.3.4.5", // too many octets + "256.0.0.0", // first octet too large + "0.265.0.0", // second octet too large + "0.0.266.0", // third octet too large + "0.0.0.355", // fourth octet too large + "a.2.3.4", // first octet not number + "1.a.3.4", // second octet not number + "1.2.c.4", // third octet not number + "1.2.3.d" // fourth octet not number + ); private static final String IPV6_MAPPED = "::ffff:192.168.0.1"; private static final String IPV6_COMPATIBLE = "::192.168.0.1"; - private static final String NOT_IP_ADDRESS = "druid.apache.org"; @Test public void testOverflowsUnsignedIntTooLow() @@ -72,7 +101,9 @@ public void testIsValidAddressNull() @Test public void testIsValidAddressIPv4() { - Assert.assertTrue(IPv4AddressExprUtils.isValidAddress(IPV4)); + for (String address : VALID_IPV4_ADDRESSES) { + Assert.assertTrue(getErrMsg(address), IPv4AddressExprUtils.isValidAddress(address)); + } } @Test @@ -90,7 +121,9 @@ public void testIsValidAddressIPv6Compatible() @Test public void testIsValidAddressNotIpAddress() { - Assert.assertFalse(IPv4AddressExprUtils.isValidAddress(NOT_IP_ADDRESS)); + for (String address : INVALID_IPV4_ADDRESSES) { + Assert.assertFalse(getErrMsg(address), IPv4AddressExprUtils.isValidAddress(address)); + } } @Test @@ -102,9 +135,12 @@ public void testParseNull() @Test public void testParseIPv4() { - Inet4Address address = IPv4AddressExprUtils.parse(IPV4); - Assert.assertNotNull(address); - Assert.assertEquals(IPV4, address.getHostAddress()); + for (String string : VALID_IPV4_ADDRESSES) { + String errMsg = getErrMsg(string); + Inet4Address address = IPv4AddressExprUtils.parse(string); + Assert.assertNotNull(errMsg, address); + Assert.assertEquals(errMsg, string, address.getHostAddress()); + } } @Test @@ -122,7 +158,9 @@ public void testParseIPv6Compatible() @Test public void testParseNotIpAddress() { - Assert.assertNull(IPv4AddressExprUtils.parse(NOT_IP_ADDRESS)); + for (String address : INVALID_IPV4_ADDRESSES) { + Assert.assertNull(getErrMsg(address), IPv4AddressExprUtils.parse(address)); + } } @Test @@ -147,4 +185,10 @@ public void testToLong() throws UnknownHostException InetAddress address = InetAddress.getByAddress(bytes); Assert.assertEquals(0xC0A80001L, IPv4AddressExprUtils.toLong((Inet4Address) address)); } + + private String getErrMsg(String msg) + { + String prefix = "Failed: "; + return prefix + msg; + } }