From 3767957ca0f534e8db597b6e8167a3d0bab21940 Mon Sep 17 00:00:00 2001 From: bgranvea Date: Mon, 5 Oct 2020 13:56:45 +0200 Subject: [PATCH 1/7] give access to stats returned by server in X-ClickHouse-Summary --- .../clickhouse/ClickHouseStatement.java | 3 + .../clickhouse/ClickHouseStatementImpl.java | 19 +++++ .../response/ClickHouseResponseSummary.java | 40 +++++++++ .../settings/ClickHouseQueryParam.java | 2 + .../integration/ResultSetSummary.java | 85 +++++++++++++++++++ 5 files changed, 149 insertions(+) create mode 100644 src/main/java/ru/yandex/clickhouse/response/ClickHouseResponseSummary.java create mode 100644 src/test/java/ru/yandex/clickhouse/integration/ResultSetSummary.java diff --git a/src/main/java/ru/yandex/clickhouse/ClickHouseStatement.java b/src/main/java/ru/yandex/clickhouse/ClickHouseStatement.java index 262257a3d..1773698ae 100644 --- a/src/main/java/ru/yandex/clickhouse/ClickHouseStatement.java +++ b/src/main/java/ru/yandex/clickhouse/ClickHouseStatement.java @@ -1,6 +1,7 @@ package ru.yandex.clickhouse; import ru.yandex.clickhouse.response.ClickHouseResponse; +import ru.yandex.clickhouse.response.ClickHouseResponseSummary; import ru.yandex.clickhouse.settings.ClickHouseQueryParam; import ru.yandex.clickhouse.util.ClickHouseRowBinaryInputStream; import ru.yandex.clickhouse.util.ClickHouseStreamCallback; @@ -105,4 +106,6 @@ ResultSet executeQuery(String sql, * Returns extended write-API */ Writer write(); + + ClickHouseResponseSummary getResponseSummary(); } diff --git a/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java b/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java index 3df0ea2cb..531a6d6c5 100644 --- a/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java +++ b/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java @@ -1,6 +1,7 @@ package ru.yandex.clickhouse; import com.google.common.base.Strings; +import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.NameValuePair; @@ -52,6 +53,8 @@ public class ClickHouseStatementImpl implements ClickHouseStatement { private ClickHouseRowBinaryInputStream currentRowBinaryResult; + private ClickHouseResponseSummary currentSummary; + private int currentUpdateCount = -1; private int queryTimeout; @@ -217,6 +220,7 @@ public int executeUpdate(String sql) throws SQLException { } finally { StreamUtils.close(is); } + // should we return currentSummary.writtenRows? return 1; } @@ -444,6 +448,11 @@ public boolean isWrapperFor(Class iface) throws SQLException { return iface.isAssignableFrom(getClass()); } + @Override + public ClickHouseResponseSummary getResponseSummary() { + return currentSummary; + } + static String clickhousifySql(String sql) { return addFormatIfAbsent(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes); } @@ -623,6 +632,11 @@ private InputStream getInputStream( entity.writeTo(baos); is = baos.convertToInputStream(); } + + // retrieve response summary + Header summaryHeader = response.getFirstHeader("X-ClickHouse-Summary"); + currentSummary = summaryHeader != null ? Jackson.getObjectMapper().readValue(summaryHeader.getValue(), ClickHouseResponseSummary.class) : null; + return is; } catch (ClickHouseException e) { throw e; @@ -850,6 +864,11 @@ void sendStream(Writer writer, HttpEntity content) throws ClickHouseException { HttpResponse response = client.execute(httpPost); entity = response.getEntity(); checkForErrorAndThrow(entity, response); + + // retrieve response summary + Header summaryHeader = response.getFirstHeader("X-ClickHouse-Summary"); + currentSummary = summaryHeader != null ? Jackson.getObjectMapper().readValue(summaryHeader.getValue(), ClickHouseResponseSummary.class) : null; + } catch (ClickHouseException e) { throw e; } catch (Exception e) { diff --git a/src/main/java/ru/yandex/clickhouse/response/ClickHouseResponseSummary.java b/src/main/java/ru/yandex/clickhouse/response/ClickHouseResponseSummary.java new file mode 100644 index 000000000..cffe4894b --- /dev/null +++ b/src/main/java/ru/yandex/clickhouse/response/ClickHouseResponseSummary.java @@ -0,0 +1,40 @@ +package ru.yandex.clickhouse.response; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class ClickHouseResponseSummary { + final private long readRows; + final private long writtenRows; + final private long readBytes; + final private long writtenBytes; + final private long totalRowsToRead; + + public ClickHouseResponseSummary(@JsonProperty("read_rows") long readRows, @JsonProperty("written_rows") long writtenRows, @JsonProperty("read_bytes") long readBytes, + @JsonProperty("written_bytes") long writtenBytes, @JsonProperty("total_rows_to_read") long totalRowsToRead) { + this.readRows = readRows; + this.writtenRows = writtenRows; + this.readBytes = readBytes; + this.writtenBytes = writtenBytes; + this.totalRowsToRead = totalRowsToRead; + } + + public long getReadRows() { + return readRows; + } + + public long getWrittenRows() { + return writtenRows; + } + + public long getReadBytes() { + return readBytes; + } + + public long getWrittenBytes() { + return writtenBytes; + } + + public long getTotalRowsToRead() { + return totalRowsToRead; + } +} diff --git a/src/main/java/ru/yandex/clickhouse/settings/ClickHouseQueryParam.java b/src/main/java/ru/yandex/clickhouse/settings/ClickHouseQueryParam.java index c1d1a991a..e2418a83e 100644 --- a/src/main/java/ru/yandex/clickhouse/settings/ClickHouseQueryParam.java +++ b/src/main/java/ru/yandex/clickhouse/settings/ClickHouseQueryParam.java @@ -217,6 +217,8 @@ public enum ClickHouseQueryParam implements DriverPropertyCreator { SELECT_SEQUENTIAL_CONSISTENCY("select_sequential_consistency", null, Long.class, ""), + SEND_PROGRESS_IN_HTTP_HEADERS("send_progress_in_http_headers", null, Boolean.class, "Allow to populate summary in ClickHouseStatement with read/written rows/bytes"), + SEND_TIMEOUT("send_timeout", null, Integer.class, ""), SESSION_CHECK("session_check", false, Boolean.class, ""), diff --git a/src/test/java/ru/yandex/clickhouse/integration/ResultSetSummary.java b/src/test/java/ru/yandex/clickhouse/integration/ResultSetSummary.java new file mode 100644 index 000000000..746f6aeb1 --- /dev/null +++ b/src/test/java/ru/yandex/clickhouse/integration/ResultSetSummary.java @@ -0,0 +1,85 @@ +package ru.yandex.clickhouse.integration; + +import org.testng.annotations.AfterTest; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; +import ru.yandex.clickhouse.ClickHouseConnection; +import ru.yandex.clickhouse.ClickHouseDataSource; +import ru.yandex.clickhouse.ClickHousePreparedStatement; +import ru.yandex.clickhouse.ClickHouseStatement; +import ru.yandex.clickhouse.settings.ClickHouseProperties; +import ru.yandex.clickhouse.settings.ClickHouseQueryParam; + +import java.util.Collections; + +import static org.testng.Assert.assertEquals; +import static org.testng.AssertJUnit.assertTrue; + +public class ResultSetSummary { + private ClickHouseConnection connection; + + @BeforeTest + public void setUp() throws Exception { + ClickHouseProperties properties = new ClickHouseProperties(); + ClickHouseDataSource dataSource = new ClickHouseDataSource("jdbc:clickhouse://localhost:8123", properties); + connection = dataSource.getConnection(); + connection.createStatement().execute("CREATE DATABASE IF NOT EXISTS test"); + connection.createStatement().execute("DROP TABLE IF EXISTS test.insert_test"); + connection.createStatement().execute("CREATE TABLE IF NOT EXISTS test.insert_test (value UInt32) ENGINE = TinyLog"); + } + + @AfterTest + public void tearDown() throws Exception { + connection.createStatement().execute("DROP DATABASE test"); + } + + @Test + public void select() throws Exception { + ClickHouseStatement st = connection.createStatement(); + st.executeQuery("SELECT * FROM numbers(10)", Collections.singletonMap(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, "true")); + + assertEquals(st.getResponseSummary().getReadRows(), 10); + assertTrue(st.getResponseSummary().getReadBytes() > 0); + } + + @Test + public void selectWithoutParam() throws Exception { + ClickHouseStatement st = connection.createStatement(); + st.executeQuery("SELECT * FROM numbers(10)", Collections.singletonMap(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, "true")); + + assertEquals(st.getResponseSummary().getReadRows(), 10); + assertTrue(st.getResponseSummary().getReadBytes() > 0); + } + + @Test + public void insertSingle() throws Exception { + ClickHousePreparedStatement ps = (ClickHousePreparedStatement) connection.prepareStatement("INSERT INTO test.insert_test VALUES(?)"); + ps.setLong(1, 1); + ps.executeQuery(Collections.singletonMap(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, "true")); + + assertEquals(ps.getResponseSummary().getWrittenRows(), 1); + assertTrue(ps.getResponseSummary().getWrittenBytes() > 0); + } + + @Test + public void insertBatch() throws Exception { + ClickHousePreparedStatement ps = (ClickHousePreparedStatement) connection.prepareStatement("INSERT INTO test.insert_test VALUES(?)"); + for (long i = 0; i < 10; i++) { + ps.setLong(1, i); + ps.addBatch(); + } + ps.executeBatch(Collections.singletonMap(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, "true")); + + assertEquals(ps.getResponseSummary().getWrittenRows(), 10); + assertTrue(ps.getResponseSummary().getWrittenBytes() > 0); + } + + @Test + public void insertSelect() throws Exception { + ClickHousePreparedStatement ps = (ClickHousePreparedStatement) connection.prepareStatement("INSERT INTO test.insert_test SELECT number FROM numbers(10)"); + ps.executeQuery(Collections.singletonMap(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, "true")); + + assertEquals(ps.getResponseSummary().getWrittenRows(), 10); + assertTrue(ps.getResponseSummary().getWrittenBytes() > 0); + } +} From 52d3ce51211769103f8bdbdb401c48609ab217cc Mon Sep 17 00:00:00 2001 From: bgranvea Date: Mon, 5 Oct 2020 14:47:21 +0200 Subject: [PATCH 2/7] - added WAIT_END_OF_QUERY param - executeUpdate returns real written rows if possible - better unit tests --- .../clickhouse/ClickHouseStatementImpl.java | 4 +- .../response/ClickHouseResponseSummary.java | 4 +- .../settings/ClickHouseQueryParam.java | 2 + .../integration/ResultSetSummary.java | 40 ++++++++++++++++++- 4 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java b/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java index 531a6d6c5..626ab4c94 100644 --- a/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java +++ b/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java @@ -220,8 +220,8 @@ public int executeUpdate(String sql) throws SQLException { } finally { StreamUtils.close(is); } - // should we return currentSummary.writtenRows? - return 1; + + return currentSummary != null ? (int) currentSummary.getWrittenRows() : 1; } @Override diff --git a/src/main/java/ru/yandex/clickhouse/response/ClickHouseResponseSummary.java b/src/main/java/ru/yandex/clickhouse/response/ClickHouseResponseSummary.java index cffe4894b..5b0702758 100644 --- a/src/main/java/ru/yandex/clickhouse/response/ClickHouseResponseSummary.java +++ b/src/main/java/ru/yandex/clickhouse/response/ClickHouseResponseSummary.java @@ -3,8 +3,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; public class ClickHouseResponseSummary { - final private long readRows; - final private long writtenRows; + final private long readRows; // number of read rows for selects (may be more than rows in result set) + final private long writtenRows; // number of written rows for inserts final private long readBytes; final private long writtenBytes; final private long totalRowsToRead; diff --git a/src/main/java/ru/yandex/clickhouse/settings/ClickHouseQueryParam.java b/src/main/java/ru/yandex/clickhouse/settings/ClickHouseQueryParam.java index e2418a83e..4d6e57baa 100644 --- a/src/main/java/ru/yandex/clickhouse/settings/ClickHouseQueryParam.java +++ b/src/main/java/ru/yandex/clickhouse/settings/ClickHouseQueryParam.java @@ -255,6 +255,8 @@ public enum ClickHouseQueryParam implements DriverPropertyCreator { PREFERRED_BLOCK_SIZE_BYTES("preferred_block_size_bytes", null, Long.class, "Adaptively estimates number of required rows in a block."), ENABLE_OPTIMIZE_PREDICATE_EXPRESSION("enable_optimize_predicate_expression", null, Boolean.class, "See Clickhouse server description for this parameter. Default value is null so that server setting is taken."), + + WAIT_END_OF_QUERY("wait_end_of_query", null, Boolean.class, "Buffer the response server-side before sending to client. Useful when using SEND_PROGRESS_IN_HTTP_HEADERS to get accurate stats."), ; private final String key; diff --git a/src/test/java/ru/yandex/clickhouse/integration/ResultSetSummary.java b/src/test/java/ru/yandex/clickhouse/integration/ResultSetSummary.java index 746f6aeb1..1899c8605 100644 --- a/src/test/java/ru/yandex/clickhouse/integration/ResultSetSummary.java +++ b/src/test/java/ru/yandex/clickhouse/integration/ResultSetSummary.java @@ -11,6 +11,8 @@ import ru.yandex.clickhouse.settings.ClickHouseQueryParam; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import static org.testng.Assert.assertEquals; import static org.testng.AssertJUnit.assertTrue; @@ -38,16 +40,41 @@ public void select() throws Exception { ClickHouseStatement st = connection.createStatement(); st.executeQuery("SELECT * FROM numbers(10)", Collections.singletonMap(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, "true")); - assertEquals(st.getResponseSummary().getReadRows(), 10); + assertTrue(st.getResponseSummary().getReadRows() >= 10); assertTrue(st.getResponseSummary().getReadBytes() > 0); } + @Test + public void largeSelect() throws Exception { + ClickHouseStatement st = connection.createStatement(); + st.executeQuery("SELECT * FROM numbers(10000000)", Collections.singletonMap(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, "true")); + + assertTrue(st.getResponseSummary().getReadRows() < 10000000); + assertTrue(st.getResponseSummary().getReadBytes() > 0); + } + + @Test + public void largeSelectWaitEndOfQuery() throws Exception { + ClickHouseStatement st = connection.createStatement(); + st.executeQuery("SELECT * FROM numbers(10000000)", largeSelectWaitEndOfQueryParams()); + + assertTrue(st.getResponseSummary().getReadRows() >= 10000000); + assertTrue(st.getResponseSummary().getReadBytes() > 0); + } + + private Map largeSelectWaitEndOfQueryParams() { + Map res = new HashMap<>(); + res.put(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, "true"); + res.put(ClickHouseQueryParam.WAIT_END_OF_QUERY, "true"); + return res; + } + @Test public void selectWithoutParam() throws Exception { ClickHouseStatement st = connection.createStatement(); st.executeQuery("SELECT * FROM numbers(10)", Collections.singletonMap(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, "true")); - assertEquals(st.getResponseSummary().getReadRows(), 10); + assertTrue(st.getResponseSummary().getReadRows() >= 10); assertTrue(st.getResponseSummary().getReadBytes() > 0); } @@ -82,4 +109,13 @@ public void insertSelect() throws Exception { assertEquals(ps.getResponseSummary().getWrittenRows(), 10); assertTrue(ps.getResponseSummary().getWrittenBytes() > 0); } + + @Test + public void insertLargeSelect() throws Exception { + ClickHousePreparedStatement ps = (ClickHousePreparedStatement) connection.prepareStatement("INSERT INTO test.insert_test SELECT number FROM numbers(10000000)"); + ps.executeQuery(Collections.singletonMap(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, "true")); + + assertEquals(ps.getResponseSummary().getWrittenRows(), 10000000); + assertTrue(ps.getResponseSummary().getWrittenBytes() > 0); + } } From 2505101bbb4ff0e4f34627706bb2afce841d078c Mon Sep 17 00:00:00 2001 From: bgranvea Date: Mon, 5 Oct 2020 16:17:51 +0200 Subject: [PATCH 3/7] updated ClickHouseProperties with new query params --- .../settings/ClickHouseProperties.java | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/main/java/ru/yandex/clickhouse/settings/ClickHouseProperties.java b/src/main/java/ru/yandex/clickhouse/settings/ClickHouseProperties.java index c99014c82..14ce0774f 100644 --- a/src/main/java/ru/yandex/clickhouse/settings/ClickHouseProperties.java +++ b/src/main/java/ru/yandex/clickhouse/settings/ClickHouseProperties.java @@ -94,7 +94,8 @@ public class ClickHouseProperties { private Boolean insertDeduplicate; private Boolean insertDistributedSync; private Boolean anyJoinDistinctRightTableKeys; - + private Boolean sendProgressInHttpHeaders; + private Boolean waitEndOfQuery; public ClickHouseProperties() { this(new Properties()); @@ -162,6 +163,8 @@ public ClickHouseProperties(Properties info) { this.insertDeduplicate = getSetting(info, ClickHouseQueryParam.INSERT_DEDUPLICATE); this.insertDistributedSync = getSetting(info, ClickHouseQueryParam.INSERT_DISTRIBUTED_SYNC); this.anyJoinDistinctRightTableKeys = getSetting(info, ClickHouseQueryParam.ANY_JOIN_DISTINCT_RIGHT_TABLE_KEYS); + this.sendProgressInHttpHeaders = (Boolean)getSetting(info, ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS); + this.waitEndOfQuery = (Boolean)getSetting(info, ClickHouseQueryParam.WAIT_END_OF_QUERY); } public Properties asProperties() { @@ -226,6 +229,8 @@ public Properties asProperties() { ret.put(ClickHouseQueryParam.INSERT_DEDUPLICATE.getKey(), insertDeduplicate); ret.put(ClickHouseQueryParam.INSERT_DISTRIBUTED_SYNC.getKey(), insertDistributedSync); ret.put(ClickHouseQueryParam.ANY_JOIN_DISTINCT_RIGHT_TABLE_KEYS.getKey(), anyJoinDistinctRightTableKeys); + ret.put(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS.getKey(), sendProgressInHttpHeaders); + ret.put(ClickHouseQueryParam.WAIT_END_OF_QUERY.getKey(), waitEndOfQuery); return ret.getProperties(); } @@ -292,6 +297,8 @@ public ClickHouseProperties(ClickHouseProperties properties) { setInsertDeduplicate(properties.insertDeduplicate); setInsertDistributedSync(properties.insertDistributedSync); setAnyJoinDistinctRightTableKeys(properties.anyJoinDistinctRightTableKeys); + setSendProgressInHttpHeaders(properties.sendProgressInHttpHeaders); + setWaitEndOfQuery(properties.waitEndOfQuery); } public Map buildQueryParams(boolean ignoreDatabase){ @@ -379,6 +386,9 @@ public Map buildQueryParams(boolean ignoreDatabase params.put(ClickHouseQueryParam.ENABLE_OPTIMIZE_PREDICATE_EXPRESSION, enableOptimizePredicateExpression ? "1" : "0"); } + addQueryParam(sendProgressInHttpHeaders, ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, params); + addQueryParam(waitEndOfQuery, ClickHouseQueryParam.WAIT_END_OF_QUERY, params); + return params; } @@ -907,6 +917,22 @@ public Boolean getAnyJoinDistinctRightTableKeys() { return anyJoinDistinctRightTableKeys; } + public Boolean getSendProgressInHttpHeaders() { + return sendProgressInHttpHeaders; + } + + public void setSendProgressInHttpHeaders(Boolean sendProgressInHttpHeaders) { + this.sendProgressInHttpHeaders = sendProgressInHttpHeaders; + } + + public Boolean getWaitEndOfQuery() { + return waitEndOfQuery; + } + + public void setWaitEndOfQuery(Boolean waitEndOfQuery) { + this.waitEndOfQuery = waitEndOfQuery; + } + private static class PropertiesBuilder { private final Properties properties; public PropertiesBuilder() { From 00e531b6df70a93ed2b865313f36cb39a54a915b Mon Sep 17 00:00:00 2001 From: bgranvea Date: Mon, 5 Oct 2020 18:34:13 +0200 Subject: [PATCH 4/7] parse X-ClickHouse-Summary only if it asked in query param (otherwise we always get read/writtenRows=0) --- .../clickhouse/ClickHouseStatementImpl.java | 30 +++++++++++++++---- ...sultSetSummary.java => ResultSummary.java} | 11 ++++++- 2 files changed, 35 insertions(+), 6 deletions(-) rename src/test/java/ru/yandex/clickhouse/integration/{ResultSetSummary.java => ResultSummary.java} (94%) diff --git a/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java b/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java index 626ab4c94..6bf7f4664 100644 --- a/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java +++ b/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java @@ -634,8 +634,10 @@ private InputStream getInputStream( } // retrieve response summary - Header summaryHeader = response.getFirstHeader("X-ClickHouse-Summary"); - currentSummary = summaryHeader != null ? Jackson.getObjectMapper().readValue(summaryHeader.getValue(), ClickHouseResponseSummary.class) : null; + if (isClickhouseSummarySet(additionalClickHouseDBParams, additionalRequestParams)) { + Header summaryHeader = response.getFirstHeader("X-ClickHouse-Summary"); + currentSummary = summaryHeader != null ? Jackson.getObjectMapper().readValue(summaryHeader.getValue(), ClickHouseResponseSummary.class) : null; + } return is; } catch (ClickHouseException e) { @@ -738,6 +740,23 @@ private List getUrlQueryParams( return result; } + private boolean isClickhouseSummarySet(Map additionalClickHouseDBParams, Map additionalRequestParams) { + if (Boolean.TRUE.equals(properties.getSendProgressInHttpHeaders())) + return true; + + if (additionalClickHouseDBParams != null && isQueryParamSet(additionalClickHouseDBParams.get(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS))) + return true; + + if (additionalRequestParams != null && isQueryParamSet(additionalRequestParams.get(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS.getKey()))) + return true; + + return false; + } + + private boolean isQueryParamSet(String param) { + return "true".equals(param) || "1".equals(param); + } + private URI followRedirects(URI uri) throws IOException, URISyntaxException { if (properties.isCheckForRedirects()) { int redirects = 0; @@ -866,9 +885,10 @@ void sendStream(Writer writer, HttpEntity content) throws ClickHouseException { checkForErrorAndThrow(entity, response); // retrieve response summary - Header summaryHeader = response.getFirstHeader("X-ClickHouse-Summary"); - currentSummary = summaryHeader != null ? Jackson.getObjectMapper().readValue(summaryHeader.getValue(), ClickHouseResponseSummary.class) : null; - + if (isClickhouseSummarySet(writer.getAdditionalDBParams(), writer.getRequestParams())) { + Header summaryHeader = response.getFirstHeader("X-ClickHouse-Summary"); + currentSummary = summaryHeader != null ? Jackson.getObjectMapper().readValue(summaryHeader.getValue(), ClickHouseResponseSummary.class) : null; + } } catch (ClickHouseException e) { throw e; } catch (Exception e) { diff --git a/src/test/java/ru/yandex/clickhouse/integration/ResultSetSummary.java b/src/test/java/ru/yandex/clickhouse/integration/ResultSummary.java similarity index 94% rename from src/test/java/ru/yandex/clickhouse/integration/ResultSetSummary.java rename to src/test/java/ru/yandex/clickhouse/integration/ResultSummary.java index 1899c8605..ceb497f38 100644 --- a/src/test/java/ru/yandex/clickhouse/integration/ResultSetSummary.java +++ b/src/test/java/ru/yandex/clickhouse/integration/ResultSummary.java @@ -15,9 +15,10 @@ import java.util.Map; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; import static org.testng.AssertJUnit.assertTrue; -public class ResultSetSummary { +public class ResultSummary { private ClickHouseConnection connection; @BeforeTest @@ -118,4 +119,12 @@ public void insertLargeSelect() throws Exception { assertEquals(ps.getResponseSummary().getWrittenRows(), 10000000); assertTrue(ps.getResponseSummary().getWrittenBytes() > 0); } + + @Test + public void noSummary() throws Exception { + ClickHouseStatement st = connection.createStatement(); + st.executeQuery("SELECT * FROM numbers(10)"); + + assertNull(st.getResponseSummary()); + } } From 85d1fcc53ea2fb320f4a824397787bd257ab8435 Mon Sep 17 00:00:00 2001 From: bgranvea Date: Tue, 6 Oct 2020 09:29:28 +0200 Subject: [PATCH 5/7] cleaner test of query param --- .../clickhouse/ClickHouseStatementImpl.java | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java b/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java index 6bf7f4664..2edb37a52 100644 --- a/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java +++ b/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java @@ -634,7 +634,7 @@ private InputStream getInputStream( } // retrieve response summary - if (isClickhouseSummarySet(additionalClickHouseDBParams, additionalRequestParams)) { + if (isQueryParamSet(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, additionalClickHouseDBParams, additionalRequestParams)) { Header summaryHeader = response.getFirstHeader("X-ClickHouse-Summary"); currentSummary = summaryHeader != null ? Jackson.getObjectMapper().readValue(summaryHeader.getValue(), ClickHouseResponseSummary.class) : null; } @@ -740,21 +740,20 @@ private List getUrlQueryParams( return result; } - private boolean isClickhouseSummarySet(Map additionalClickHouseDBParams, Map additionalRequestParams) { - if (Boolean.TRUE.equals(properties.getSendProgressInHttpHeaders())) - return true; + private boolean isQueryParamSet(ClickHouseQueryParam param, Map additionalClickHouseDBParams, Map additionalRequestParams) { + String value = getQueryParamValue(param, additionalClickHouseDBParams, additionalRequestParams); - if (additionalClickHouseDBParams != null && isQueryParamSet(additionalClickHouseDBParams.get(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS))) - return true; + return "true".equals(value) || "1".equals(value); + } - if (additionalRequestParams != null && isQueryParamSet(additionalRequestParams.get(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS.getKey()))) - return true; + private String getQueryParamValue(ClickHouseQueryParam param, Map additionalClickHouseDBParams, Map additionalRequestParams) { + if (additionalRequestParams != null && additionalRequestParams.containsKey(param.getKey()) && !Strings.isNullOrEmpty(additionalRequestParams.get(param.getKey()))) + return additionalRequestParams.get(param.getKey()); - return false; - } + if (additionalClickHouseDBParams != null && additionalClickHouseDBParams.containsKey(param) && !Strings.isNullOrEmpty(additionalClickHouseDBParams.get(param))) + return additionalClickHouseDBParams.get(param); - private boolean isQueryParamSet(String param) { - return "true".equals(param) || "1".equals(param); + return properties.asProperties().getProperty(param.getKey()); } private URI followRedirects(URI uri) throws IOException, URISyntaxException { @@ -885,7 +884,7 @@ void sendStream(Writer writer, HttpEntity content) throws ClickHouseException { checkForErrorAndThrow(entity, response); // retrieve response summary - if (isClickhouseSummarySet(writer.getAdditionalDBParams(), writer.getRequestParams())) { + if (isQueryParamSet(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, writer.getAdditionalDBParams(), writer.getRequestParams())) { Header summaryHeader = response.getFirstHeader("X-ClickHouse-Summary"); currentSummary = summaryHeader != null ? Jackson.getObjectMapper().readValue(summaryHeader.getValue(), ClickHouseResponseSummary.class) : null; } From 12319c63f16ad8589738c57e86be6fb1ae5aec66 Mon Sep 17 00:00:00 2001 From: bgranvea Date: Tue, 13 Oct 2020 09:23:56 +0200 Subject: [PATCH 6/7] make ClickHouseStatementImpl extend ConfigurableApi --- .../clickhouse/ClickHouseStatementImpl.java | 20 ++++++-- .../ru/yandex/clickhouse/ConfigurableApi.java | 10 ++++ .../clickhouse/ClickHouseStatementTest.java | 49 ++++++++++++++++++- 3 files changed, 74 insertions(+), 5 deletions(-) diff --git a/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java b/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java index 2edb37a52..a6ce8afc1 100644 --- a/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java +++ b/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java @@ -39,7 +39,7 @@ import java.util.*; -public class ClickHouseStatementImpl implements ClickHouseStatement { +public class ClickHouseStatementImpl extends ConfigurableApi implements ClickHouseStatement { private static final Logger log = LoggerFactory.getLogger(ClickHouseStatementImpl.class); @@ -81,6 +81,7 @@ public class ClickHouseStatementImpl implements ClickHouseStatement { public ClickHouseStatementImpl(CloseableHttpClient client, ClickHouseConnection connection, ClickHouseProperties properties, int resultSetType) { + super(null); this.client = client; this.connection = connection; this.properties = properties == null ? new ClickHouseProperties() : properties; @@ -716,6 +717,8 @@ private List getUrlQueryParams( params.put(ClickHouseQueryParam.DATABASE, initialDatabase); } + params.putAll(getAdditionalDBParams()); + if (additionalClickHouseDBParams != null && !additionalClickHouseDBParams.isEmpty()) { params.putAll(additionalClickHouseDBParams); } @@ -728,6 +731,12 @@ private List getUrlQueryParams( } } + for (Map.Entry entry : getRequestParams().entrySet()) { + if (!Strings.isNullOrEmpty(entry.getValue())) { + result.add(new BasicNameValuePair(entry.getKey(), entry.getValue())); + } + } + if (additionalRequestParams != null) { for (Map.Entry entry : additionalRequestParams.entrySet()) { if (!Strings.isNullOrEmpty(entry.getValue())) { @@ -736,7 +745,6 @@ private List getUrlQueryParams( } } - return result; } @@ -750,9 +758,15 @@ private String getQueryParamValue(ClickHouseQueryParam param, Map addQueryIdTo(Map dbParams) { this.additionalDBParams = new HashMap(); if (null != dbParams) { @@ -49,4 +54,9 @@ public T option(String key, String value) { return (T) this; } + public T removeOption(String key) { + additionalRequestParams.remove(key); + return (T) this; + } + } diff --git a/src/test/java/ru/yandex/clickhouse/ClickHouseStatementTest.java b/src/test/java/ru/yandex/clickhouse/ClickHouseStatementTest.java index 36762fa19..c45738411 100644 --- a/src/test/java/ru/yandex/clickhouse/ClickHouseStatementTest.java +++ b/src/test/java/ru/yandex/clickhouse/ClickHouseStatementTest.java @@ -5,6 +5,7 @@ import java.net.URISyntaxException; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Collections; import java.util.Properties; import org.apache.http.impl.client.HttpClientBuilder; @@ -13,6 +14,7 @@ import com.google.common.collect.ImmutableMap; import ru.yandex.clickhouse.settings.ClickHouseProperties; +import ru.yandex.clickhouse.settings.ClickHouseQueryParam; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; @@ -109,7 +111,7 @@ public void testMaxMemoryUsage() throws Exception { } @Test - public void testAdditionalRequestParams() throws Exception { + public void testAdditionalRequestParams() { ClickHouseProperties properties = new ClickHouseProperties(); ClickHouseStatementImpl statement = new ClickHouseStatementImpl( HttpClientBuilder.create().build(), @@ -118,15 +120,58 @@ public void testAdditionalRequestParams() throws Exception { ResultSet.TYPE_FORWARD_ONLY ); + statement.option("cache_namespace", "aaaa"); URI uri = statement.buildRequestUri( null, null, null, - ImmutableMap.of("cache_namespace", "aaaa"), + null, false ); String query = uri.getQuery(); assertTrue(query.contains("cache_namespace=aaaa"), "cache_namespace param is missing in URL"); + + uri = statement.buildRequestUri( + null, + null, + null, + ImmutableMap.of("cache_namespace", "bbbb"), + false + ); + query = uri.getQuery(); + assertTrue(query.contains("cache_namespace=bbbb"), "cache_namespace param is missing in URL"); + + // check that statement level params are given to Writer + assertEquals(statement.write().getRequestParams().get("cache_namespace"), "aaaa"); + } + + @Test + public void testAdditionalDBParams() { + ClickHouseProperties properties = new ClickHouseProperties(); + properties.setMaxThreads(1); + + ClickHouseStatementImpl statement = new ClickHouseStatementImpl( + HttpClientBuilder.create().build(), + null, + properties, + ResultSet.TYPE_FORWARD_ONLY + ); + + URI uri = statement.buildRequestUri(null, null, null, null, false); + assertTrue(uri.getQuery().contains("max_threads=1")); + + // override on statement level + statement.addDbParam(ClickHouseQueryParam.MAX_THREADS, "2"); + + uri = statement.buildRequestUri(null, null, null, null, false); + assertTrue(uri.getQuery().contains("max_threads=2")); + + // override on method level + uri = statement.buildRequestUri(null, null, Collections.singletonMap(ClickHouseQueryParam.MAX_THREADS, "3"), null, false); + assertTrue(uri.getQuery().contains("max_threads=3")); + + // check that statement level params are given to Writer + assertEquals(statement.write().getAdditionalDBParams().get(ClickHouseQueryParam.MAX_THREADS), "2"); } @Test From e44a78c4b576002078f4a33b1d9d15772f276651 Mon Sep 17 00:00:00 2001 From: bgranvea Date: Mon, 1 Feb 2021 15:26:19 +0100 Subject: [PATCH 7/7] use TestContainers --- ...ultSummary.java => ResultSummaryTest.java} | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) rename src/test/java/ru/yandex/clickhouse/integration/{ResultSummary.java => ResultSummaryTest.java} (89%) diff --git a/src/test/java/ru/yandex/clickhouse/integration/ResultSummary.java b/src/test/java/ru/yandex/clickhouse/integration/ResultSummaryTest.java similarity index 89% rename from src/test/java/ru/yandex/clickhouse/integration/ResultSummary.java rename to src/test/java/ru/yandex/clickhouse/integration/ResultSummaryTest.java index ceb497f38..79ad4661a 100644 --- a/src/test/java/ru/yandex/clickhouse/integration/ResultSummary.java +++ b/src/test/java/ru/yandex/clickhouse/integration/ResultSummaryTest.java @@ -3,13 +3,10 @@ import org.testng.annotations.AfterTest; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import ru.yandex.clickhouse.ClickHouseConnection; -import ru.yandex.clickhouse.ClickHouseDataSource; -import ru.yandex.clickhouse.ClickHousePreparedStatement; -import ru.yandex.clickhouse.ClickHouseStatement; -import ru.yandex.clickhouse.settings.ClickHouseProperties; +import ru.yandex.clickhouse.*; import ru.yandex.clickhouse.settings.ClickHouseQueryParam; +import java.sql.SQLException; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -18,22 +15,18 @@ import static org.testng.Assert.assertNull; import static org.testng.AssertJUnit.assertTrue; -public class ResultSummary { +public class ResultSummaryTest { private ClickHouseConnection connection; @BeforeTest public void setUp() throws Exception { - ClickHouseProperties properties = new ClickHouseProperties(); - ClickHouseDataSource dataSource = new ClickHouseDataSource("jdbc:clickhouse://localhost:8123", properties); - connection = dataSource.getConnection(); + connection = ClickHouseContainerForTest.newDataSource().getConnection(); connection.createStatement().execute("CREATE DATABASE IF NOT EXISTS test"); - connection.createStatement().execute("DROP TABLE IF EXISTS test.insert_test"); - connection.createStatement().execute("CREATE TABLE IF NOT EXISTS test.insert_test (value UInt32) ENGINE = TinyLog"); } @AfterTest public void tearDown() throws Exception { - connection.createStatement().execute("DROP DATABASE test"); + connection.createStatement().execute("DROP DATABASE IF EXISTS test"); } @Test @@ -81,6 +74,8 @@ public void selectWithoutParam() throws Exception { @Test public void insertSingle() throws Exception { + createInsertTestTable(); + ClickHousePreparedStatement ps = (ClickHousePreparedStatement) connection.prepareStatement("INSERT INTO test.insert_test VALUES(?)"); ps.setLong(1, 1); ps.executeQuery(Collections.singletonMap(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, "true")); @@ -91,6 +86,8 @@ public void insertSingle() throws Exception { @Test public void insertBatch() throws Exception { + createInsertTestTable(); + ClickHousePreparedStatement ps = (ClickHousePreparedStatement) connection.prepareStatement("INSERT INTO test.insert_test VALUES(?)"); for (long i = 0; i < 10; i++) { ps.setLong(1, i); @@ -104,6 +101,8 @@ public void insertBatch() throws Exception { @Test public void insertSelect() throws Exception { + createInsertTestTable(); + ClickHousePreparedStatement ps = (ClickHousePreparedStatement) connection.prepareStatement("INSERT INTO test.insert_test SELECT number FROM numbers(10)"); ps.executeQuery(Collections.singletonMap(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, "true")); @@ -113,6 +112,8 @@ public void insertSelect() throws Exception { @Test public void insertLargeSelect() throws Exception { + createInsertTestTable(); + ClickHousePreparedStatement ps = (ClickHousePreparedStatement) connection.prepareStatement("INSERT INTO test.insert_test SELECT number FROM numbers(10000000)"); ps.executeQuery(Collections.singletonMap(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, "true")); @@ -127,4 +128,9 @@ public void noSummary() throws Exception { assertNull(st.getResponseSummary()); } + + private void createInsertTestTable() throws SQLException { + connection.createStatement().execute("DROP TABLE IF EXISTS test.insert_test"); + connection.createStatement().execute("CREATE TABLE IF NOT EXISTS test.insert_test (value UInt32) ENGINE = TinyLog"); + } }