From 218cf1f0b83431f5323df2852d38ce52f2c03ae6 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Thu, 15 May 2025 22:26:00 -0700 Subject: [PATCH 1/3] fixed ip address conversion --- .../internal/AbstractBinaryFormatReader.java | 9 +-- .../internal/InetAddressConverter.java | 72 +++++++++++++++++++ .../internal/MapBackedRecord.java | 8 +-- .../internal/SerializerUtils.java | 2 + .../clickhouse/client/query/QueryTests.java | 30 +++++++- .../com/clickhouse/jdbc/DataTypeTests.java | 6 +- 6 files changed, 117 insertions(+), 10 deletions(-) create mode 100644 client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/InetAddressConverter.java diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java index c031f4b69..a3decdbfe 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java @@ -28,6 +28,7 @@ import java.math.BigInteger; import java.net.Inet4Address; import java.net.Inet6Address; +import java.net.InetAddress; import java.time.Duration; import java.time.Instant; import java.time.LocalDate; @@ -486,12 +487,12 @@ public TemporalAmount getTemporalAmount(String colName) { @Override public Inet4Address getInet4Address(String colName) { - return readValue(colName); + return InetAddressConverter.convertToIpv4(readValue(colName)); } @Override public Inet6Address getInet6Address(String colName) { - return readValue(colName); + return InetAddressConverter.convertToIpv6(readValue(colName)); } @Override @@ -651,12 +652,12 @@ public TemporalAmount getTemporalAmount(int index) { @Override public Inet4Address getInet4Address(int index) { - return readValue(index); + return InetAddressConverter.convertToIpv4(readValue(index)); } @Override public Inet6Address getInet6Address(int index) { - return readValue(index); + return InetAddressConverter.convertToIpv6(readValue(index)); } @Override diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/InetAddressConverter.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/InetAddressConverter.java new file mode 100644 index 000000000..5c43a3843 --- /dev/null +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/InetAddressConverter.java @@ -0,0 +1,72 @@ +package com.clickhouse.client.api.data_formats.internal; + +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.UnknownHostException; + +public class InetAddressConverter { + + /** + * Converts IPv4 address to IPv6 address. + * + * @param value IPv4 address + * @return IPv6 address + * @throws IllegalArgumentException when failed to convert to IPv6 address + */ + public static Inet6Address convertToIpv6(InetAddress value) { + if (value == null || value instanceof Inet6Address) { + return (Inet6Address) value; + } + + // https://en.wikipedia.org/wiki/IPv6#IPv4-mapped_IPv6_addresses + byte[] bytes = new byte[16]; + bytes[10] = (byte) 0xFF; + bytes[11] = (byte) 0xFF; + System.arraycopy(value.getAddress(), 0, bytes, 12, 4); + + try { + return Inet6Address.getByAddress(null, bytes, null); + } catch (UnknownHostException e) { + throw new IllegalArgumentException(e); + } + } + + /** + * Converts IPv6 address to IPv4 address if applicable. + * + * @param value IPv6 address + * @return IPv4 address + * @throws IllegalArgumentException when failed to convert to IPv4 address + */ + public static Inet4Address convertToIpv4(InetAddress value) { + if (value == null || value instanceof Inet4Address) { + return (Inet4Address) value; + } + + byte[] bytes = value.getAddress(); + boolean invalid = false; + for (int i = 0; i < 10; i++) { + if (bytes[i] != (byte) 0) { + invalid = true; + break; + } + } + + if (!invalid) { + invalid = bytes[10] != 0xFF || bytes[11] != 0xFF; + } + + if (invalid) { + throw new IllegalArgumentException("Failed to convert IPv6 to IPv4"); + } + + byte[] addr = new byte[4]; + System.arraycopy(bytes, 12, addr, 0, 4); + try { + return (Inet4Address) InetAddress.getByAddress(addr); + } catch (UnknownHostException e) { + throw new IllegalArgumentException(e); + } + } +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/MapBackedRecord.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/MapBackedRecord.java index 813584994..3d375d168 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/MapBackedRecord.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/MapBackedRecord.java @@ -162,12 +162,12 @@ public TemporalAmount getTemporalAmount(String colName) { @Override public Inet4Address getInet4Address(String colName) { - return readValue(colName); + return InetAddressConverter.convertToIpv4(readValue(colName)); } @Override public Inet6Address getInet6Address(String colName) { - return readValue(colName); + return InetAddressConverter.convertToIpv6(readValue(colName)); } @Override @@ -323,12 +323,12 @@ public TemporalAmount getTemporalAmount(int index) { @Override public Inet4Address getInet4Address(int index) { - return readValue(index); + return InetAddressConverter.convertToIpv4(readValue(index)); } @Override public Inet6Address getInet6Address(int index) { - return readValue(index); + return InetAddressConverter.convertToIpv6(readValue(index)); } @Override diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java index 4fdade64c..a4c05c99f 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java @@ -28,6 +28,8 @@ import java.math.BigInteger; import java.net.Inet4Address; import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.sql.Timestamp; import java.time.*; import java.time.temporal.TemporalUnit; diff --git a/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java b/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java index 884137cf7..fd3f45f24 100644 --- a/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java @@ -44,6 +44,7 @@ import org.testng.annotations.Test; import org.testng.util.Strings; +import javax.management.Query; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.ByteArrayInputStream; @@ -781,7 +782,34 @@ public void testIPAddresses() throws Exception { testDataTypes(columns, valueGenerators, verifiers); } - @Test + @Test(groups = {"integration"}) + public void testConversionOfIpAddresses() throws Exception { + + try (QueryResponse response = client.query("SELECT toIPv6('::ffff:90.176.75.97') ipv4, toIPv6('2001:db8:85a3::8a2e:370:7334') ipv6").get()) { + ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(response); + + reader.next(); + InetAddress ipv4 = reader.getInet4Address(1); + Assert.assertNotNull(ipv4); + InetAddress ipv6 = reader.getInet6Address(2); + Assert.assertNotNull(ipv6); + InetAddress ipv4_as_ipv6 = reader.getInet6Address(1); + Assert.assertEquals(Inet4Address.getByAddress(ipv4_as_ipv6.getAddress()), ipv4); + Assert.assertThrows(() -> reader.getInet4Address(2)); + } + + List records = client.queryAll("SELECT toIPv6('::ffff:90.176.75.97') ipv4, toIPv6('2001:db8:85a3::8a2e:370:7334') ipv6"); + GenericRecord record = records.get(0); + InetAddress ipv4 = record.getInet4Address(1); + Assert.assertNotNull(ipv4); + InetAddress ipv6 = record.getInet6Address(2); + Assert.assertNotNull(ipv6); + InetAddress ipv4_as_ipv6 = record.getInet6Address(1); + Assert.assertEquals(Inet4Address.getByAddress(ipv4_as_ipv6.getAddress()), ipv4); + Assert.assertThrows(() -> record.getInet4Address(2)); + } + + @Test(groups = {"integration"}) public void testDateTimeDataTypes() { final List columns = Arrays.asList( "min_date Date", diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java index 470f95bba..d24e3fd76 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java @@ -519,7 +519,7 @@ public void testStringTypes() throws SQLException { @Test(groups = { "integration" }) public void testIpAddressTypes() throws SQLException, UnknownHostException { runQuery("CREATE TABLE test_ips (order Int8, " - + "ipv4_ip IPv4, ipv4_name IPv4, ipv6 IPv6" + + "ipv4_ip IPv4, ipv4_name IPv4, ipv6 IPv6, ipv4_as_ipv6 IPv6" + ") ENGINE = MergeTree ORDER BY ()"); // Insert random (valid) values @@ -529,12 +529,14 @@ public void testIpAddressTypes() throws SQLException, UnknownHostException { InetAddress ipv4AddressByIp = Inet4Address.getByName(rand.nextInt(256) + "." + rand.nextInt(256) + "." + rand.nextInt(256) + "." + rand.nextInt(256)); InetAddress ipv4AddressByName = Inet4Address.getByName("www.example.com"); InetAddress ipv6Address = Inet6Address.getByName("2001:adb8:85a3:1:2:8a2e:370:7334"); + InetAddress ipv4AsIpv6 = Inet4Address.getByName("90.176.75.97"); try (Connection conn = getConnection()) { try (PreparedStatement stmt = conn.prepareStatement("INSERT INTO test_ips VALUES ( 1, ?, ?, ? )")) { stmt.setObject(1, ipv4AddressByIp); stmt.setObject(2, ipv4AddressByName); stmt.setObject(3, ipv6Address); + stmt.setObject(4, ipv4AsIpv6); stmt.executeUpdate(); } } @@ -549,6 +551,8 @@ public void testIpAddressTypes() throws SQLException, UnknownHostException { assertEquals(rs.getObject("ipv4_name"), ipv4AddressByName); assertEquals(rs.getObject("ipv6"), ipv6Address); assertEquals(rs.getString("ipv6"), ipv6Address.toString()); + String value = rs.getObject("ipv4_as_ipv6").toString(); + System.out.println("ip: " + value); assertFalse(rs.next()); } } From 0d3e01074f9b55ee61b342c5a67697be11f315c3 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Thu, 15 May 2025 22:30:07 -0700 Subject: [PATCH 2/3] added JDBC tests --- .../src/test/java/com/clickhouse/jdbc/DataTypeTests.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java index d24e3fd76..b127443f2 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java @@ -28,7 +28,6 @@ import java.time.OffsetDateTime; import java.time.ZoneId; import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; import java.util.GregorianCalendar; import java.util.HashMap; import java.util.Map; @@ -532,7 +531,7 @@ public void testIpAddressTypes() throws SQLException, UnknownHostException { InetAddress ipv4AsIpv6 = Inet4Address.getByName("90.176.75.97"); try (Connection conn = getConnection()) { - try (PreparedStatement stmt = conn.prepareStatement("INSERT INTO test_ips VALUES ( 1, ?, ?, ? )")) { + try (PreparedStatement stmt = conn.prepareStatement("INSERT INTO test_ips VALUES ( 1, ?, ?, ?, ? )")) { stmt.setObject(1, ipv4AddressByIp); stmt.setObject(2, ipv4AddressByName); stmt.setObject(3, ipv6Address); @@ -551,8 +550,7 @@ public void testIpAddressTypes() throws SQLException, UnknownHostException { assertEquals(rs.getObject("ipv4_name"), ipv4AddressByName); assertEquals(rs.getObject("ipv6"), ipv6Address); assertEquals(rs.getString("ipv6"), ipv6Address.toString()); - String value = rs.getObject("ipv4_as_ipv6").toString(); - System.out.println("ip: " + value); + assertEquals(rs.getObject("ipv4_as_ipv6"), ipv4AsIpv6); assertFalse(rs.next()); } } From d19eab5378187cf1891df2f0c1c7e0345228579d Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Thu, 15 May 2025 22:46:18 -0700 Subject: [PATCH 3/3] fixed review comments --- .../api/data_formats/internal/AbstractBinaryFormatReader.java | 2 +- .../src/test/java/com/clickhouse/client/query/QueryTests.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java index a3decdbfe..8f920447c 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java @@ -652,7 +652,7 @@ public TemporalAmount getTemporalAmount(int index) { @Override public Inet4Address getInet4Address(int index) { - return InetAddressConverter.convertToIpv4(readValue(index)); + return InetAddressConverter.convertToIpv4(readValue(index)); } @Override diff --git a/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java b/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java index fd3f45f24..08c8fc0c9 100644 --- a/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java @@ -44,7 +44,6 @@ import org.testng.annotations.Test; import org.testng.util.Strings; -import javax.management.Query; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.ByteArrayInputStream;