From 76a5d3dee3d70f46228a1e0e497f4824a82e69e7 Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Wed, 21 Apr 2021 19:32:19 +0800 Subject: [PATCH] Move query to request body --- .../clickhouse/ClickHouseStatementImpl.java | 67 +++++++++++++++++-- .../integration/BatchInsertsTest.java | 27 ++++++++ .../ClickHouseStatementImplTest.java | 34 ++++++++++ 3 files changed, 122 insertions(+), 6 deletions(-) diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java index 9e8da7c73..d2eb21e2d 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java @@ -3,6 +3,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; @@ -22,10 +23,12 @@ import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.NameValuePair; +import org.apache.http.StatusLine; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.AbstractHttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.entity.mime.MultipartEntityBuilder; @@ -57,6 +60,50 @@ public class ClickHouseStatementImpl extends ConfigurableApi getUrlQueryParams( ) { List result = new ArrayList<>(); - if (sql != null) { + if (sql != null && !sql.isEmpty()) { result.add(new BasicNameValuePair("query", sql)); } @@ -1020,10 +1067,14 @@ void sendStream(Writer writer, HttpEntity content) throws ClickHouseException { HttpEntity entity = null; // TODO no parser involved so user can execute arbitray statement here try { - URI uri = buildRequestUri(writer.getSql(), null, writer.getAdditionalDBParams(), writer.getRequestParams(), false); + String sql = writer.getSql(); + boolean isContentCompressed = writer.getCompression() != ClickHouseCompression.none; + URI uri = buildRequestUri( + isContentCompressed ? sql : null, null, writer.getAdditionalDBParams(), writer.getRequestParams(), false); uri = followRedirects(uri); - content = applyRequestBodyCompression(content); + content = applyRequestBodyCompression( + new WrappedHttpEntity(isContentCompressed ? null : sql, content)); HttpPost httpPost = new HttpPost(uri); @@ -1050,7 +1101,8 @@ void sendStream(Writer writer, HttpEntity content) throws ClickHouseException { } private void checkForErrorAndThrow(HttpEntity entity, HttpResponse response) throws IOException, ClickHouseException { - if (response.getStatusLine().getStatusCode() != HttpURLConnection.HTTP_OK) { + StatusLine line = response.getStatusLine(); + if (line.getStatusCode() != HttpURLConnection.HTTP_OK) { InputStream messageStream = entity.getContent(); byte[] bytes = Utils.toByteArray(messageStream); if (properties.isCompress()) { @@ -1062,8 +1114,11 @@ private void checkForErrorAndThrow(HttpEntity entity, HttpResponse response) thr } } EntityUtils.consumeQuietly(entity); - String chMessage = new String(bytes, StandardCharsets.UTF_8); - throw ClickHouseExceptionSpecifier.specify(chMessage, properties.getHost(), properties.getPort()); + if (bytes.length == 0) { + throw ClickHouseExceptionSpecifier.specify(new IllegalStateException(line.toString()), properties.getHost(), properties.getPort()); + } else { + throw ClickHouseExceptionSpecifier.specify(new String(bytes, StandardCharsets.UTF_8), properties.getHost(), properties.getPort()); + } } } diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/BatchInsertsTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/BatchInsertsTest.java index beae7a255..2065d7295 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/BatchInsertsTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/BatchInsertsTest.java @@ -5,12 +5,15 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.sql.Timestamp; import java.sql.Types; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Collections; import java.util.TimeZone; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.testng.Assert; import org.testng.annotations.AfterTest; @@ -289,4 +292,28 @@ public void testNullParameters() throws SQLException { st.addBatch(); } + @Test + public void testBatchInsertWithLongQuery() throws SQLException { + int columnCount = 200; + try (Statement s = connection.createStatement()) { + String createColumns = IntStream.range(0, columnCount).mapToObj( + i -> "`looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongnaaaaameeeeeeee" + i + "` String " + ).collect(Collectors.joining(",")); + s.execute("DROP TABLE IF EXISTS test.batch_insert_with_long_query"); + s.execute("CREATE TABLE test.batch_insert_with_long_query (" + createColumns + ") ENGINE = Memory"); + } + + String values = IntStream.range(0, columnCount).mapToObj(i -> "?").collect(Collectors.joining(",")); + String columns = IntStream.range(0, columnCount).mapToObj( + i -> "looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongnaaaaameeeeeeee" + i + ).collect(Collectors.joining(",")); + int index = 1; + try (PreparedStatement s = connection.prepareStatement("INSERT INTO test.batch_insert_with_long_query (" + columns + ") VALUES (" + values + ")")) { + for (int i = 0; i < columnCount; i++) { + s.setString(index++, "12345"); + } + s.addBatch(); + s.executeBatch(); + } + } } diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseStatementImplTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseStatementImplTest.java index a51ef7ee2..0720bc21e 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseStatementImplTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseStatementImplTest.java @@ -154,6 +154,40 @@ public void testExternalData() throws SQLException, UnsupportedEncodingException } } + // reproduce issue #634 + public void testLargeQueryWithExternalData() throws Exception { + String serverVersion = connection.getServerVersion(); + String[] rows = ClickHouseVersionNumberUtil.getMajorVersion(serverVersion) >= 21 + && ClickHouseVersionNumberUtil.getMinorVersion(serverVersion) >= 3 + ? new String[] { "1\tGroup\n" } + : new String[] { "1\tGroup", "1\tGroup\n" }; + + int length = 160000; + StringBuilder builder = new StringBuilder(length); + for (int i = 0; i < length; i++) { + builder.append('u'); + } + String user = builder.toString(); + for (String row : rows) { + try (ClickHouseStatement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery( + "select UserName, GroupName from (select '" + + user + + "' as UserName, 1 as GroupId) as g" + + "any left join groups using GroupId", null, + Collections.singletonList(new ClickHouseExternalData( + "groups", new ByteArrayInputStream(row.getBytes()) + ).withStructure("GroupId UInt8, GroupName String")))) { + Assert.assertTrue(rs.next()); + String userName = rs.getString("UserName"); + String groupName = rs.getString("GroupName"); + + Assert.assertEquals(userName, user); + Assert.assertEquals(groupName, "Group"); + } + } + } + private InputStream getTSVStream(final int rowsCount) { return new InputStream() {