From 813df634dcefb631c44188710d2036638ff24b16 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Tue, 24 Jun 2025 22:23:30 -0700 Subject: [PATCH 1/4] made client settings super stream lined and removed multiple checks for default --- .../client/ClickHouseServerForTest.java | 7 - .../com/clickhouse/client/api/Client.java | 106 +++--- .../client/api/ClientConfigProperties.java | 248 ++++++++++---- .../internal/AbstractBinaryFormatReader.java | 3 +- .../client/api/insert/InsertSettings.java | 4 +- .../api/internal/HttpAPIClientHelper.java | 305 +++++++----------- .../client/api/query/QuerySettings.java | 11 +- .../com/clickhouse/client/ClientTests.java | 6 +- .../clickhouse/client/HttpTransportTests.java | 2 +- .../clickhouse/client/query/QueryTests.java | 1 - .../com/clickhouse/jdbc/DataTypeTests.java | 100 +++--- 11 files changed, 418 insertions(+), 375 deletions(-) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseServerForTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseServerForTest.java index 6796c31c9..d3a32515e 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseServerForTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseServerForTest.java @@ -370,13 +370,6 @@ public static String getDatabase() { public static boolean runQuery(String sql) { LOGGER.info("runQuery: (\"" + sql + "\")"); - - try { - throw new Exception("test"); - } catch (Exception e) { - e.printStackTrace(); - } - if (clickhouseContainer != null) { try { Container.ExecResult res = clickhouseContainer.execInContainer("clickhouse-client", diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index b014dc99f..0c4fd7ef7 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -115,7 +115,7 @@ public class Client implements AutoCloseable { private HttpAPIClientHelper httpClientHelper = null; private final List endpoints; - private final Map configuration; + private final Map configuration; private final Map readOnlyConfig; @@ -145,15 +145,16 @@ private Client(Set endpoints, Map configuration, private Client(Set endpoints, Map configuration, ExecutorService sharedOperationExecutor, ColumnToMethodMatchingStrategy columnToMethodMatchingStrategy, Object metricsRegistry) { // Simple initialization - this.configuration = configuration; - this.readOnlyConfig = Collections.unmodifiableMap(this.configuration); + this.configuration = ClientConfigProperties.parseConfigMap(configuration); + this.readOnlyConfig = Collections.unmodifiableMap(configuration); this.metricsRegistry = metricsRegistry; // Serialization this.pojoSerDe = new POJOSerDe(columnToMethodMatchingStrategy); // Operation Execution - boolean isAsyncEnabled = MapUtils.getFlag(this.configuration, ClientConfigProperties.ASYNC_OPERATIONS.getKey(), false); + boolean isAsyncEnabled = ClientConfigProperties.ASYNC_OPERATIONS.getOrDefault(this.configuration); + if (isAsyncEnabled && sharedOperationExecutor == null) { this.isSharedOpExecutorOwned = true; this.sharedOperationExecutor = Executors.newCachedThreadPool(new DefaultThreadFactory("chc-operation")); @@ -179,7 +180,7 @@ private Client(Set endpoints, Map configuration, } this.endpoints = tmpEndpoints.build(); - this.httpClientHelper = new HttpAPIClientHelper(configuration, metricsRegistry, initSslContext); + this.httpClientHelper = new HttpAPIClientHelper(this.configuration, metricsRegistry, initSslContext); String retry = configuration.get(ClientConfigProperties.RETRY_ON_FAILURE.getKey()); this.retries = retry == null ? 0 : Integer.parseInt(retry); @@ -217,7 +218,7 @@ public void loadServerInfo() { * @return String - actual default database name. */ public String getDefaultDatabase() { - return this.configuration.get("database"); + return (String) this.configuration.get(ClientConfigProperties.DATABASE.getKey()); } @@ -845,7 +846,7 @@ public Builder setMaxRetries(int maxRetries) { * @return */ public Builder allowBinaryReaderToReuseBuffers(boolean reuse) { - this.configuration.put("client_allow_binary_reader_to_reuse_buffers", String.valueOf(reuse)); + this.configuration.put(ClientConfigProperties.BINARY_READER_USE_PREALLOCATED_BUFFERS.getKey(), String.valueOf(reuse)); return this; } @@ -1009,20 +1010,21 @@ public Client build() { throw new IllegalArgumentException("At least one endpoint is required"); } // check if username and password are empty. so can not initiate client? - if (!this.configuration.containsKey("access_token") && - (!this.configuration.containsKey("user") || !this.configuration.containsKey("password")) && - !MapUtils.getFlag(this.configuration, "ssl_authentication", false) && - !this.configuration.containsKey(ClientConfigProperties.httpHeader(HttpHeaders.AUTHORIZATION))) { + boolean useSslAuth = MapUtils.getFlag(this.configuration, ClientConfigProperties.SSL_AUTH.getKey()); + boolean hasAccessToken = this.configuration.containsKey(ClientConfigProperties.ACCESS_TOKEN.getKey()); + boolean hasUser = this.configuration.containsKey(ClientConfigProperties.USER.getKey()); + boolean hasPassword = this.configuration.containsKey(ClientConfigProperties.PASSWORD.getKey()); + boolean customHttpHeaders = this.configuration.containsKey(ClientConfigProperties.httpHeader(HttpHeaders.AUTHORIZATION)); + + if (!(useSslAuth || hasAccessToken || hasUser || hasPassword || customHttpHeaders)) { throw new IllegalArgumentException("Username and password (or access token or SSL authentication or pre-define Authorization header) are required"); } - if (this.configuration.containsKey("ssl_authentication") && - (this.configuration.containsKey("password") || this.configuration.containsKey("access_token"))) { + if (useSslAuth && (hasAccessToken || hasPassword)) { throw new IllegalArgumentException("Only one of password, access token or SSL authentication can be used per client."); } - if (this.configuration.containsKey("ssl_authentication") && - !this.configuration.containsKey(ClientConfigProperties.SSL_CERTIFICATE.getKey())) { + if (useSslAuth && !this.configuration.containsKey(ClientConfigProperties.SSL_CERTIFICATE.getKey())) { throw new IllegalArgumentException("SSL authentication requires a client certificate"); } @@ -1159,17 +1161,16 @@ public CompletableFuture insert(String tableName, List data, if (data == null || data.isEmpty()) { throw new IllegalArgumentException("Data cannot be empty"); } - + //Add format to the settings + if (settings == null) { + settings = new InsertSettings(); + } String operationId = registerOperationMetrics(); settings.setOperationId(operationId); globalClientStats.get(operationId).start(ClientMetrics.OP_DURATION); globalClientStats.get(operationId).start(ClientMetrics.OP_SERIALIZATION); - //Add format to the settings - if (settings == null) { - settings = new InsertSettings(); - } boolean hasDefaults = this.tableSchemaHasDefaults.get(tableName); ClickHouseFormat format = hasDefaults? ClickHouseFormat.RowBinaryWithDefaults : ClickHouseFormat.RowBinary; @@ -1193,11 +1194,11 @@ public CompletableFuture insert(String tableName, List data, } - String retry = configuration.get(ClientConfigProperties.RETRY_ON_FAILURE.getKey()); - final int maxRetries = retry == null ? 0 : Integer.parseInt(retry); + Integer retry = (Integer) configuration.get(ClientConfigProperties.RETRY_ON_FAILURE.getKey()); + final int maxRetries = retry == null ? 0 : retry; settings.setOption(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), format.name()); - final InsertSettings finalSettings = settings; + final InsertSettings finalSettings = new InsertSettings(buildRequestSettings(settings.getAllSettings())); Supplier supplier = () -> { long startTime = System.nanoTime(); // Selecting some node @@ -1319,8 +1320,7 @@ public CompletableFuture insert(String tableName, InsertSettings settings) { final int writeBufferSize = settings.getInputStreamCopyBufferSize() <= 0 ? - Integer.parseInt(configuration.getOrDefault(ClientConfigProperties.CLIENT_NETWORK_BUFFER_SIZE.getKey(), - ClientConfigProperties.CLIENT_NETWORK_BUFFER_SIZE.getDefaultValue())) : + (int) configuration.get(ClientConfigProperties.CLIENT_NETWORK_BUFFER_SIZE.getKey()) : settings.getInputStreamCopyBufferSize(); if (writeBufferSize <= 0) { @@ -1392,9 +1392,8 @@ public CompletableFuture insert(String tableName, Supplier responseSupplier; - final int writeBufferSize = settings.getInputStreamCopyBufferSize() <= 0 ? - Integer.parseInt(configuration.getOrDefault(ClientConfigProperties.CLIENT_NETWORK_BUFFER_SIZE.getKey(), "8192")) : + (int) configuration.get(ClientConfigProperties.CLIENT_NETWORK_BUFFER_SIZE.getKey()) : settings.getInputStreamCopyBufferSize(); if (writeBufferSize <= 0) { @@ -1402,7 +1401,7 @@ public CompletableFuture insert(String tableName, } settings.setOption(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), format.name()); - final InsertSettings finalSettings = settings; + final InsertSettings finalSettings = new InsertSettings(buildRequestSettings(settings.getAllSettings())); StringBuilder sqlStmt = new StringBuilder("INSERT INTO ").append(tableName); if (columnNames != null && !columnNames.isEmpty()) { @@ -1531,14 +1530,13 @@ public CompletableFuture query(String sqlQuery, Map responseSupplier; if (queryParams != null) { settings.setOption("statement_params", queryParams); } - final QuerySettings finalSettings = settings; + final QuerySettings finalSettings = new QuerySettings(buildRequestSettings(settings.getAllSettings())); responseSupplier = () -> { long startTime = System.nanoTime(); // Selecting some node @@ -1916,7 +1914,7 @@ public CompletableFuture execute(String sql) { public ClickHouseBinaryFormatReader newBinaryFormatReader(QueryResponse response, TableSchema schema) { ClickHouseBinaryFormatReader reader = null; // Using caching buffer allocator is risky so this parameter is not exposed to the user - boolean useCachingBufferAllocator = MapUtils.getFlag(configuration, "client_allow_binary_reader_to_reuse_buffers"); + boolean useCachingBufferAllocator = MapUtils.getFlag(configuration, "client_allow_binary_reader_to_reuse_buffers", false); BinaryStreamReader.ByteBufferAllocator byteBufferPool = useCachingBufferAllocator ? new BinaryStreamReader.CachingByteBufferAllocator() : new BinaryStreamReader.DefaultByteBufferAllocator(); @@ -1954,25 +1952,6 @@ private String registerOperationMetrics() { return operationId; } - private void applyDefaults(QuerySettings settings) { - Map settingsMap = settings.getAllSettings(); - - String key = ClientConfigProperties.USE_SERVER_TIMEZONE.getKey(); - if (!settingsMap.containsKey(key) && configuration.containsKey(key)) { - settings.setOption(key, MapUtils.getFlag(configuration, key)); - } - - key = ClientConfigProperties.USE_TIMEZONE.getKey(); - if ( !settings.getUseServerTimeZone() && !settingsMap.containsKey(key) && configuration.containsKey(key)) { - settings.setOption(key, TimeZone.getTimeZone(configuration.get(key))); - } - - key = ClientConfigProperties.SERVER_TIMEZONE.getKey(); - if (!settingsMap.containsKey(key) && configuration.containsKey(key)) { - settings.setOption(key, TimeZone.getTimeZone(configuration.get(key))); - } - } - private CompletableFuture runAsyncOperation(Supplier resultSupplier, Map requestSettings) { boolean isAsync = MapUtils.getFlag(requestSettings, configuration, ClientConfigProperties.ASYNC_OPERATIONS.getKey()); if (isAsync) { @@ -2000,7 +1979,7 @@ public Map getConfiguration() { /** Returns operation timeout in seconds */ protected int getOperationTimeout() { - return Integer.parseInt(configuration.get(ClientConfigProperties.MAX_EXECUTION_TIME.getKey())); + return (int) configuration.get(ClientConfigProperties.MAX_EXECUTION_TIME.getKey()); } /** @@ -2013,7 +1992,7 @@ public Set getEndpoints() { } public String getUser() { - return this.configuration.get(ClientConfigProperties.USER.getKey()); + return (String) this.configuration.get(ClientConfigProperties.USER.getKey()); } public String getServerVersion() { @@ -2021,7 +2000,8 @@ public String getServerVersion() { } public String getServerTimeZone() { - return this.configuration.get(ClientConfigProperties.SERVER_TIMEZONE.getKey()); + TimeZone tz = (TimeZone) this.configuration.get(ClientConfigProperties.SERVER_TIMEZONE.getKey()); + return tz == null ? null : tz.getID(); } public String getClientVersion() { @@ -2034,10 +2014,9 @@ public String getClientVersion() { * @param dbRoles */ public void setDBRoles(Collection dbRoles) { - this.configuration.put(ClientConfigProperties.SESSION_DB_ROLES.getKey(), ClientConfigProperties.commaSeparated(dbRoles)); - this.unmodifiableDbRolesView = - Collections.unmodifiableCollection(ClientConfigProperties.valuesFromCommaSeparated( - this.configuration.get(ClientConfigProperties.SESSION_DB_ROLES.getKey()))); + List tmp = new ArrayList<>(dbRoles); + this.configuration.put(ClientConfigProperties.SESSION_DB_ROLES.getKey(), tmp); + this.unmodifiableDbRolesView = ImmutableList.copyOf(tmp); } public void updateClientName(String name) { @@ -2068,4 +2047,17 @@ private Endpoint getNextAliveNode() { } public static final String VALUES_LIST_DELIMITER = ","; + + /** + * Produces a merge of operation and client settings. + * Operation settings override client settings + * @param opSettings - operation settings + * @return request settings - merged client and operation settings + */ + private Map buildRequestSettings(Map opSettings) { + Map requestSettings = new HashMap<>(); + requestSettings.putAll(configuration); + requestSettings.putAll(opSettings); + return requestSettings; + } } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java index 06e0a9153..5f9239e90 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java @@ -1,12 +1,21 @@ package com.clickhouse.client.api; import com.clickhouse.client.api.internal.ClickHouseLZ4OutputStream; +import com.clickhouse.data.ClickHouseFormat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; +import java.util.TimeZone; +import java.util.function.Consumer; import java.util.stream.Collectors; /** @@ -14,167 +23,187 @@ */ public enum ClientConfigProperties { - SESSION_DB_ROLES("session_db_roles"), + SESSION_DB_ROLES("session_db_roles", List.class), - SETTING_LOG_COMMENT(serverSetting("log_comment")), + SETTING_LOG_COMMENT(serverSetting("log_comment"), String.class), - HTTP_USE_BASIC_AUTH("http_use_basic_auth", "true"), + HTTP_USE_BASIC_AUTH("http_use_basic_auth", Boolean.class, "true"), - USER("user", "default"), + USER("user", String.class, "default"), - PASSWORD("password"), + PASSWORD("password", String.class), /** * Maximum number of active connection in internal connection pool. */ - HTTP_MAX_OPEN_CONNECTIONS("max_open_connections", "10"), + HTTP_MAX_OPEN_CONNECTIONS("max_open_connections", Integer.class, "10"), /** * HTTP keep-alive timeout override. */ - HTTP_KEEP_ALIVE_TIMEOUT("http_keep_alive_timeout"), + HTTP_KEEP_ALIVE_TIMEOUT("http_keep_alive_timeout", Long.class), - USE_SERVER_TIMEZONE("use_server_time_zone", "true"), + USE_SERVER_TIMEZONE("use_server_time_zone", Boolean.class, "true"), - USE_TIMEZONE("use_time_zone"), + USE_TIMEZONE("use_time_zone", TimeZone.class), - SERVER_VERSION("server_version"), + SERVER_VERSION("server_version", String.class), - SERVER_TIMEZONE("server_time_zone", "UTC"), + SERVER_TIMEZONE("server_time_zone", TimeZone.class, "UTC"), - ASYNC_OPERATIONS("async", "false"), + ASYNC_OPERATIONS("async", Boolean.class, "false"), - CONNECTION_TTL("connection_ttl", "-1"), + CONNECTION_TTL("connection_ttl", Long.class, "-1"), - CONNECTION_TIMEOUT("connection_timeout"), + CONNECTION_TIMEOUT("connection_timeout", Long.class), - CONNECTION_REUSE_STRATEGY("connection_reuse_strategy", String.valueOf(ConnectionReuseStrategy.FIFO)), + CONNECTION_REUSE_STRATEGY("connection_reuse_strategy", ConnectionReuseStrategy.class, String.valueOf(ConnectionReuseStrategy.FIFO)), - SOCKET_OPERATION_TIMEOUT("socket_timeout", "0"), + SOCKET_OPERATION_TIMEOUT("socket_timeout", Integer.class, "0"), - SOCKET_RCVBUF_OPT("socket_rcvbuf", "8196"), + SOCKET_RCVBUF_OPT("socket_rcvbuf", Integer.class, "8196"), - SOCKET_SNDBUF_OPT("socket_sndbuf", "8196"), + SOCKET_SNDBUF_OPT("socket_sndbuf", Integer.class,"8196"), - SOCKET_REUSEADDR_OPT("socket_reuseaddr"), + SOCKET_REUSEADDR_OPT("socket_reuseaddr", Boolean.class), - SOCKET_KEEPALIVE_OPT("socket_keepalive"), + SOCKET_KEEPALIVE_OPT("socket_keepalive", Boolean.class), - SOCKET_TCP_NO_DELAY_OPT("socket_tcp_nodelay"), + SOCKET_TCP_NO_DELAY_OPT("socket_tcp_nodelay", Boolean.class), - SOCKET_LINGER_OPT("socket_linger"), + SOCKET_LINGER_OPT("socket_linger", Boolean.class), - DATABASE("database", "default"), + DATABASE("database", String.class, "default"), - COMPRESS_SERVER_RESPONSE("compress", "true"), // actually a server setting, but has client effect too + COMPRESS_SERVER_RESPONSE("compress", Boolean.class, "true"), // actually a server setting, but has client effect too - COMPRESS_CLIENT_REQUEST("decompress", "false"), // actually a server setting, but has client effect too + COMPRESS_CLIENT_REQUEST("decompress", Boolean.class, "false"), // actually a server setting, but has client effect too - USE_HTTP_COMPRESSION("client.use_http_compression", "false"), + USE_HTTP_COMPRESSION("client.use_http_compression", Boolean.class, "false"), - COMPRESSION_LZ4_UNCOMPRESSED_BUF_SIZE("compression.lz4.uncompressed_buffer_size", String.valueOf(ClickHouseLZ4OutputStream.UNCOMPRESSED_BUFF_SIZE)), + COMPRESSION_LZ4_UNCOMPRESSED_BUF_SIZE("compression.lz4.uncompressed_buffer_size", Integer.class, String.valueOf(ClickHouseLZ4OutputStream.UNCOMPRESSED_BUFF_SIZE)), - DISABLE_NATIVE_COMPRESSION("disable_native_compression", "false"), + DISABLE_NATIVE_COMPRESSION("disable_native_compression", Boolean.class, "false"), - PROXY_TYPE("proxy_type"), // "http" + PROXY_TYPE("proxy_type", String.class), // "http" - PROXY_HOST("proxy_host"), + PROXY_HOST("proxy_host", String.class), - PROXY_PORT("proxy_port"), + PROXY_PORT("proxy_port", Integer.class), - PROXY_USER("proxy_user"), + PROXY_USER("proxy_user", String.class), - PROXY_PASSWORD("proxy_password"), + PROXY_PASSWORD("proxy_password", String.class), - MAX_EXECUTION_TIME("max_execution_time", "0"), + MAX_EXECUTION_TIME("max_execution_time", Integer.class,"0"), - SSL_TRUST_STORE("trust_store"), + SSL_TRUST_STORE("trust_store", String.class), - SSL_KEYSTORE_TYPE("key_store_type"), + SSL_KEYSTORE_TYPE("key_store_type", String.class), - SSL_KEY_STORE("ssl_key_store"), + SSL_KEY_STORE("ssl_key_store", String.class), - SSL_KEY_STORE_PASSWORD("key_store_password"), + SSL_KEY_STORE_PASSWORD("key_store_password", String.class), - SSL_KEY("ssl_key"), + SSL_KEY("ssl_key", String.class), - CA_CERTIFICATE("sslrootcert"), + CA_CERTIFICATE("sslrootcert", String.class), - SSL_CERTIFICATE("sslcert"), + SSL_CERTIFICATE("sslcert", String.class), - RETRY_ON_FAILURE("retry", "3"), + RETRY_ON_FAILURE("retry", Integer.class, "3"), - INPUT_OUTPUT_FORMAT("format"), + INPUT_OUTPUT_FORMAT("format", ClickHouseFormat.class), - MAX_THREADS_PER_CLIENT("max_threads_per_client", "0"), + MAX_THREADS_PER_CLIENT("max_threads_per_client", Integer.class, "0"), - QUERY_ID("query_id"), // actually a server setting, but has client effect too + QUERY_ID("query_id", String.class), // actually a server setting, but has client effect too - CLIENT_NETWORK_BUFFER_SIZE("client_network_buffer_size", "300000"), + CLIENT_NETWORK_BUFFER_SIZE("client_network_buffer_size", Integer.class, "300000"), - ACCESS_TOKEN("access_token"), SSL_AUTH("ssl_authentication"), + ACCESS_TOKEN("access_token", String.class), - CONNECTION_POOL_ENABLED("connection_pool_enabled", "true"), + SSL_AUTH("ssl_authentication", Boolean.class, "false"), - CONNECTION_REQUEST_TIMEOUT("connection_request_timeout", "10000"), + CONNECTION_POOL_ENABLED("connection_pool_enabled", Boolean.class, "true"), - CLIENT_RETRY_ON_FAILURE("client_retry_on_failures", + CONNECTION_REQUEST_TIMEOUT("connection_request_timeout", Long.class, "10000"), + + CLIENT_RETRY_ON_FAILURE("client_retry_on_failures", List.class, String.join(",", ClientFaultCause.NoHttpResponse.name(), ClientFaultCause.ConnectTimeout.name(), - ClientFaultCause.ConnectionRequestTimeout.name())), + ClientFaultCause.ConnectionRequestTimeout.name())) { + @Override + public Object parseValue(String value) { + List strValues = (List) super.parseValue(value); + List failures = new ArrayList(); + if (strValues != null) { + for (String strValue : strValues) { + failures.add(ClientFaultCause.valueOf(strValue)); + } + } + return failures; + } + }, - CLIENT_NAME("client_name"), + CLIENT_NAME("client_name", String.class, ""), /** * An old alias to {@link ClientConfigProperties#CLIENT_NAME}. Using the last one is preferred. */ @Deprecated - PRODUCT_NAME("product_name"), + PRODUCT_NAME("product_name", String.class), - BEARERTOKEN_AUTH ("bearer_token"), + BEARERTOKEN_AUTH ("bearer_token", String.class), /** * Indicates that data provided for write operation is compressed by application. */ - APP_COMPRESSED_DATA("app_compressed_data", "false"), + APP_COMPRESSED_DATA("app_compressed_data", Boolean.class, "false"), /** * Name of the group under which client metrics appear */ - METRICS_GROUP_NAME("metrics_name"), + METRICS_GROUP_NAME("metrics_name", String.class, "ch-http-pool"), + + HTTP_SAVE_COOKIES("client.http.cookies_enabled", Boolean.class, "false"), + + BINARY_READER_USE_PREALLOCATED_BUFFERS("client_allow_binary_reader_to_reuse_buffers", Boolean.class, "false"), + ; - private final String key; + private static final Logger LOG = LoggerFactory.getLogger(ClientConfigProperties.class); - private final String defaultValue; + private final String key; - private final List choices; + private final Class valueType; + private final String defaultValue; - ClientConfigProperties(String key) { - this(key, null, Collections.emptyList()); - } + private final Object defaultObjValue; - ClientConfigProperties(String key, String defaultValue) { - this(key, defaultValue, Collections.emptyList()); + ClientConfigProperties(String key, Class valueType) { + this(key, valueType, null); } - ClientConfigProperties(String key, String defaultValue, List choices) { + ClientConfigProperties(String key, Class valueType, String defaultValue) { this.key = key; + this.valueType = valueType; this.defaultValue = defaultValue; - this.choices = Collections.unmodifiableList(choices); + this.defaultObjValue = parseValue(defaultValue); } public String getKey() { return key; } - public List getChoices() { - return choices; - } - public String getDefaultValue() { return defaultValue; } + @SuppressWarnings("unchecked") + public T getDefObjVal() { + return (T) defaultObjValue; + } + public static final String HTTP_HEADER_PREFIX = "http_header_"; public static final String SERVER_SETTING_PREFIX = "clickhouse_setting_"; @@ -207,4 +236,83 @@ public static List valuesFromCommaSeparated(String value) { return Arrays.stream(value.split("(? s.replaceAll("\\\\,", ",")) .collect(Collectors.toList()); } + + public Object parseValue(String value) { + if (value == null) { + return null; + } + + if (valueType.equals(String.class)) { + return value; + } + + if (valueType.equals(Boolean.class)) { + if (value.equals("1")) return true; + if (value.equals("0")) return false; + return Boolean.parseBoolean(value); + } + + if (valueType.equals(Integer.class)) { + return Integer.parseInt(value); + } + + if (valueType.equals(Long.class)) { + return Long.parseLong(value); + } + + if (valueType.equals(List.class)) { + return valuesFromCommaSeparated(value); + } + + if (valueType.isEnum()) { + Object[] constants = valueType.getEnumConstants(); + for (Object constant : constants) { + if (constant.toString().equals(value)) { + return constant; + } + } + throw new IllegalArgumentException("Invalid constant name '" + value + "' for enum " + valueType.getName()); + } + + if (valueType.equals(TimeZone.class)) { + return TimeZone.getTimeZone(value); + } + + return null; + } + + @SuppressWarnings("unchecked") + public T getOrDefault(Map configMap) { + return (T) configMap.getOrDefault(getKey(), getDefObjVal()); + } + + public void applyIfSet(Map configMap, Consumer consumer) { + T value = (T) configMap.get(getKey()); + if (value != null) { + consumer.accept(value); + } + } + + public static Map parseConfigMap(Map configMap) { + Map parsedConfig = new HashMap<>(); + + Map tmpMap = new HashMap<>(configMap); + + for (ClientConfigProperties config : ClientConfigProperties.values()) { + String value = tmpMap.remove(config.getKey()); + if (value != null) { + parsedConfig.put(config.getKey(), config.parseValue(value)); + } + } + + for (String key : new HashSet<>(tmpMap.keySet())) { + if (key.startsWith(HTTP_HEADER_PREFIX) || key.startsWith(SERVER_SETTING_PREFIX)) { + parsedConfig.put(key, tmpMap.remove(key)); + } + } + + LOG.warn("Unknown and unmapped config properties: {}", tmpMap); + + return parsedConfig; + } } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java index 625825a1a..b7f44de82 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java @@ -68,7 +68,8 @@ protected AbstractBinaryFormatReader(InputStream inputStream, QuerySettings quer this.input = inputStream; Map settings = querySettings == null ? Collections.emptyMap() : querySettings.getAllSettings(); Boolean useServerTimeZone = (Boolean) settings.get(ClientConfigProperties.USE_SERVER_TIMEZONE.getKey()); - TimeZone timeZone = useServerTimeZone == Boolean.TRUE && querySettings != null ? querySettings.getServerTimeZone() : + TimeZone timeZone = (useServerTimeZone == Boolean.TRUE && querySettings != null) ? + querySettings.getServerTimeZone() : (TimeZone) settings.get(ClientConfigProperties.USE_TIMEZONE.getKey()); if (timeZone == null) { throw new ClientException("Time zone is not set. (useServerTimezone:" + useServerTimeZone + ")"); diff --git a/client-v2/src/main/java/com/clickhouse/client/api/insert/InsertSettings.java b/client-v2/src/main/java/com/clickhouse/client/api/insert/InsertSettings.java index b178770f1..6179b09ce 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/insert/InsertSettings.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/insert/InsertSettings.java @@ -176,7 +176,7 @@ public boolean isClientRequestEnabled() { * @return same instance of the builder */ public InsertSettings httpHeader(String key, String value) { - rawSettings.put(ClientConfigProperties.HTTP_HEADER_PREFIX + key, value); + rawSettings.put(ClientConfigProperties.httpHeader(key), value); return this; } @@ -187,7 +187,7 @@ public InsertSettings httpHeader(String key, String value) { * @return same instance of the builder */ public InsertSettings httpHeader(String key, Collection values) { - rawSettings.put(ClientConfigProperties.HTTP_HEADER_PREFIX + key, ClientConfigProperties.commaSeparated(values)); + rawSettings.put(ClientConfigProperties.httpHeader(key), ClientConfigProperties.commaSeparated(values)); return this; } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 970a18f03..89247986f 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -9,7 +9,6 @@ import com.clickhouse.client.api.ConnectionInitiationException; import com.clickhouse.client.api.ConnectionReuseStrategy; import com.clickhouse.client.api.ServerException; -import com.clickhouse.client.api.data_formats.internal.SerializerUtils; import com.clickhouse.client.api.enums.ProxyType; import com.clickhouse.client.api.http.ClickHouseHttpProto; import com.clickhouse.client.api.transport.Endpoint; @@ -71,14 +70,14 @@ import java.net.SocketTimeoutException; import java.net.URI; import java.net.URISyntaxException; -import java.net.URL; import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; import java.security.NoSuchAlgorithmException; import java.util.Base64; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Properties; @@ -91,41 +90,37 @@ public class HttpAPIClientHelper { private static final Logger LOG = LoggerFactory.getLogger(Client.class); - private static int ERROR_BODY_BUFFER_SIZE = 1024; // Error messages are usually small + private static final int ERROR_BODY_BUFFER_SIZE = 1024; // Error messages are usually small - private CloseableHttpClient httpClient; + private final CloseableHttpClient httpClient; - private Map chConfiguration; - - private RequestConfig baseRequestConfig; + private final RequestConfig baseRequestConfig; private String proxyAuthHeaderValue; private final Set defaultRetryCauses; - private String defaultUserAgent; - private Object metricsRegistry; + private final String defaultUserAgent; + private final Object metricsRegistry; ConnPoolControl poolControl; - public HttpAPIClientHelper(Map configuration, Object metricsRegistry, boolean initSslContext) { - this.chConfiguration = configuration; + public HttpAPIClientHelper(Map configuration, Object metricsRegistry, boolean initSslContext) { this.metricsRegistry = metricsRegistry; - this.httpClient = createHttpClient(initSslContext); + this.httpClient = createHttpClient(initSslContext, configuration); RequestConfig.Builder reqConfBuilder = RequestConfig.custom(); - MapUtils.applyLong(chConfiguration, "connection_request_timeout", - (t) -> reqConfBuilder - .setConnectionRequestTimeout(t, TimeUnit.MILLISECONDS)); + reqConfBuilder.setConnectionRequestTimeout(ClientConfigProperties.CONNECTION_REQUEST_TIMEOUT.getOrDefault(configuration), TimeUnit.MILLISECONDS); this.baseRequestConfig = reqConfBuilder.build(); - boolean usingClientCompression= chConfiguration.getOrDefault(ClientConfigProperties.COMPRESS_CLIENT_REQUEST.getKey(), "false").equalsIgnoreCase("true"); - boolean usingServerCompression= chConfiguration.getOrDefault(ClientConfigProperties.COMPRESS_SERVER_RESPONSE.getKey(), "false").equalsIgnoreCase("true"); - boolean useHttpCompression = chConfiguration.getOrDefault("client.use_http_compression", "false").equalsIgnoreCase("true"); + boolean usingClientCompression = ClientConfigProperties.COMPRESS_CLIENT_REQUEST.getOrDefault(configuration); + boolean usingServerCompression = ClientConfigProperties.COMPRESS_SERVER_RESPONSE.getOrDefault(configuration); + boolean useHttpCompression = ClientConfigProperties.USE_HTTP_COMPRESSION.getOrDefault(configuration); + LOG.debug("client compression: {}, server compression: {}, http compression: {}", usingClientCompression, usingServerCompression, useHttpCompression); - defaultRetryCauses = SerializerUtils.parseEnumList(chConfiguration.get(ClientConfigProperties.CLIENT_RETRY_ON_FAILURE.getKey()), ClientFaultCause.class); + defaultRetryCauses = new HashSet<>(ClientConfigProperties.CLIENT_RETRY_ON_FAILURE.getOrDefault(configuration)); if (defaultRetryCauses.contains(ClientFaultCause.None)) { defaultRetryCauses.removeIf(c -> c != ClientFaultCause.None); } @@ -135,9 +130,10 @@ public HttpAPIClientHelper(Map configuration, Object metricsRegi /** * Creates or returns default SSL context. + * * @return SSLContext */ - public SSLContext createSSLContext() { + public SSLContext createSSLContext(Map configuration) { SSLContext sslContext; try { sslContext = SSLContext.getDefault(); @@ -145,26 +141,26 @@ public SSLContext createSSLContext() { throw new ClientException("Failed to create default SSL context", e); } ClickHouseSslContextProvider sslContextProvider = ClickHouseSslContextProvider.getProvider(); - String trustStorePath = chConfiguration.get(ClientConfigProperties.SSL_TRUST_STORE.getKey()); - if (trustStorePath != null ) { + String trustStorePath = (String) configuration.get(ClientConfigProperties.SSL_TRUST_STORE.getKey()); + if (trustStorePath != null) { try { sslContext = sslContextProvider.getSslContextFromKeyStore( trustStorePath, - chConfiguration.get(ClientConfigProperties.SSL_KEY_STORE_PASSWORD.getKey()), - chConfiguration.get(ClientConfigProperties.SSL_KEYSTORE_TYPE.getKey()) + (String) configuration.get(ClientConfigProperties.SSL_KEY_STORE_PASSWORD.getKey()), + (String) configuration.get(ClientConfigProperties.SSL_KEYSTORE_TYPE.getKey()) ); } catch (SSLException e) { throw new ClientMisconfigurationException("Failed to create SSL context from a keystore", e); } - } else if (chConfiguration.get(ClientConfigProperties.CA_CERTIFICATE.getKey()) != null || - chConfiguration.get(ClientConfigProperties.SSL_CERTIFICATE.getKey()) != null || - chConfiguration.get(ClientConfigProperties.SSL_KEY.getKey()) != null) { + } else if (configuration.get(ClientConfigProperties.CA_CERTIFICATE.getKey()) != null || + configuration.get(ClientConfigProperties.SSL_CERTIFICATE.getKey()) != null || + configuration.get(ClientConfigProperties.SSL_KEY.getKey()) != null) { try { sslContext = sslContextProvider.getSslContextFromCerts( - chConfiguration.get(ClientConfigProperties.SSL_CERTIFICATE.getKey()), - chConfiguration.get(ClientConfigProperties.SSL_KEY.getKey()), - chConfiguration.get(ClientConfigProperties.CA_CERTIFICATE.getKey()) + (String) configuration.get(ClientConfigProperties.SSL_CERTIFICATE.getKey()), + (String) configuration.get(ClientConfigProperties.SSL_KEY.getKey()), + (String) configuration.get(ClientConfigProperties.CA_CERTIFICATE.getKey()) ); } catch (SSLException e) { throw new ClientMisconfigurationException("Failed to create SSL context from certificates", e); @@ -173,37 +169,35 @@ public SSLContext createSSLContext() { return sslContext; } - private long CONNECTION_INACTIVITY_CHECK = 5000L; + private static final long CONNECTION_INACTIVITY_CHECK = 5000L; - private ConnectionConfig createConnectionConfig() { + private ConnectionConfig createConnectionConfig(Map configuration) { ConnectionConfig.Builder connConfig = ConnectionConfig.custom(); - connConfig.setTimeToLive(MapUtils.getLong(chConfiguration, ClientConfigProperties.CONNECTION_TTL.getKey()), - TimeUnit.MILLISECONDS); - connConfig.setConnectTimeout(MapUtils.getLong(chConfiguration, ClientConfigProperties.CONNECTION_TIMEOUT.getKey()), - TimeUnit.MILLISECONDS); + + ClientConfigProperties.CONNECTION_TTL.applyIfSet(configuration, (t) -> connConfig.setTimeToLive(t, TimeUnit.MILLISECONDS)); + ClientConfigProperties.CONNECTION_TIMEOUT.applyIfSet(configuration, (t) -> connConfig.setConnectTimeout(t, TimeUnit.MILLISECONDS)); connConfig.setValidateAfterInactivity(CONNECTION_INACTIVITY_CHECK, TimeUnit.MILLISECONDS); // non-configurable for now return connConfig.build(); } - private HttpClientConnectionManager basicConnectionManager(LayeredConnectionSocketFactory sslConnectionSocketFactory, SocketConfig socketConfig) { + private HttpClientConnectionManager basicConnectionManager(LayeredConnectionSocketFactory sslConnectionSocketFactory, SocketConfig socketConfig, Map configuration) { RegistryBuilder registryBuilder = RegistryBuilder.create(); registryBuilder.register("http", PlainConnectionSocketFactory.getSocketFactory()); registryBuilder.register("https", sslConnectionSocketFactory); BasicHttpClientConnectionManager connManager = new BasicHttpClientConnectionManager(registryBuilder.build()); - connManager.setConnectionConfig(createConnectionConfig()); + connManager.setConnectionConfig(createConnectionConfig(configuration)); connManager.setSocketConfig(socketConfig); return connManager; } - private HttpClientConnectionManager poolConnectionManager(LayeredConnectionSocketFactory sslConnectionSocketFactory, SocketConfig socketConfig) { + private HttpClientConnectionManager poolConnectionManager(LayeredConnectionSocketFactory sslConnectionSocketFactory, SocketConfig socketConfig, Map configuration) { PoolingHttpClientConnectionManagerBuilder connMgrBuilder = PoolingHttpClientConnectionManagerBuilder.create() .setPoolConcurrencyPolicy(PoolConcurrencyPolicy.LAX); - ConnectionReuseStrategy connectionReuseStrategy = - ConnectionReuseStrategy.valueOf(chConfiguration.get("connection_reuse_strategy")); + ConnectionReuseStrategy connectionReuseStrategy = ClientConfigProperties.CONNECTION_REUSE_STRATEGY.getOrDefault(configuration); switch (connectionReuseStrategy) { case LIFO: connMgrBuilder.setConnPoolPolicy(PoolReusePolicy.LIFO); @@ -216,13 +210,11 @@ private HttpClientConnectionManager poolConnectionManager(LayeredConnectionSocke } LOG.debug("Connection reuse strategy: {}", connectionReuseStrategy); - connMgrBuilder.setDefaultConnectionConfig(createConnectionConfig()); + connMgrBuilder.setDefaultConnectionConfig(createConnectionConfig(configuration)); connMgrBuilder.setMaxConnTotal(Integer.MAX_VALUE); // as we do not know how many routes we will have - MapUtils.applyInt(chConfiguration, ClientConfigProperties.HTTP_MAX_OPEN_CONNECTIONS.getKey(), - connMgrBuilder::setMaxConnPerRoute); - + ClientConfigProperties.HTTP_MAX_OPEN_CONNECTIONS.applyIfSet(configuration, connMgrBuilder::setMaxConnPerRoute); - int networkBufferSize = MapUtils.getInt(chConfiguration, "client_network_buffer_size"); + int networkBufferSize = ClientConfigProperties.CLIENT_NETWORK_BUFFER_SIZE.getOrDefault(configuration); MeteredManagedHttpClientConnectionFactory connectionFactory = new MeteredManagedHttpClientConnectionFactory( Http1Config.custom() .setBufferSize(networkBufferSize) @@ -235,10 +227,9 @@ private HttpClientConnectionManager poolConnectionManager(LayeredConnectionSocke connMgrBuilder.setDefaultSocketConfig(socketConfig); PoolingHttpClientConnectionManager phccm = connMgrBuilder.build(); poolControl = phccm; - if (metricsRegistry != null ) { + if (metricsRegistry != null) { try { - String mGroupName = chConfiguration.getOrDefault(ClientConfigProperties.METRICS_GROUP_NAME.getKey(), - "ch-http-pool"); + String mGroupName = ClientConfigProperties.METRICS_GROUP_NAME.getOrDefault(configuration); Class micrometerLoader = getClass().getClassLoader().loadClass("com.clickhouse.client.api.metrics.MicrometerLoader"); Method applyMethod = micrometerLoader.getDeclaredMethod("applyPoolingMetricsBinder", Object.class, String.class, PoolingHttpClientConnectionManager.class); applyMethod.invoke(micrometerLoader, metricsRegistry, mGroupName, phccm); @@ -252,61 +243,63 @@ private HttpClientConnectionManager poolConnectionManager(LayeredConnectionSocke return phccm; } - public CloseableHttpClient createHttpClient(boolean initSslContext) { + public CloseableHttpClient createHttpClient(boolean initSslContext, Map configuration) { // Top Level builders HttpClientBuilder clientBuilder = HttpClientBuilder.create(); - SSLContext sslContext = initSslContext ? createSSLContext() : null; + SSLContext sslContext = initSslContext ? createSSLContext(configuration) : null; LayeredConnectionSocketFactory sslConnectionSocketFactory = sslContext == null ? new DummySSLConnectionSocketFactory() : new SSLConnectionSocketFactory(sslContext); // Socket configuration SocketConfig.Builder soCfgBuilder = SocketConfig.custom(); - MapUtils.applyInt(chConfiguration, ClientConfigProperties.SOCKET_OPERATION_TIMEOUT.getKey(), + ClientConfigProperties.SOCKET_OPERATION_TIMEOUT.applyIfSet(configuration, (t) -> soCfgBuilder.setSoTimeout(t, TimeUnit.MILLISECONDS)); - MapUtils.applyInt(chConfiguration, ClientConfigProperties.SOCKET_RCVBUF_OPT.getKey(), + + ClientConfigProperties.SOCKET_RCVBUF_OPT.applyIfSet(configuration, soCfgBuilder::setRcvBufSize); - MapUtils.applyInt(chConfiguration, ClientConfigProperties.SOCKET_SNDBUF_OPT.getKey(), + + ClientConfigProperties.SOCKET_SNDBUF_OPT.applyIfSet(configuration, soCfgBuilder::setSndBufSize); - MapUtils.applyInt(chConfiguration, ClientConfigProperties.SOCKET_LINGER_OPT.getKey(), - (v) -> soCfgBuilder.setSoLinger(v, TimeUnit.SECONDS)); - if (MapUtils.getFlag(chConfiguration, ClientConfigProperties.SOCKET_TCP_NO_DELAY_OPT.getKey(), false)) { - soCfgBuilder.setTcpNoDelay(true); - } + ClientConfigProperties.SOCKET_LINGER_OPT.applyIfSet(configuration, + (v) -> soCfgBuilder.setSoLinger(v, TimeUnit.SECONDS)); + + ClientConfigProperties.SOCKET_TCP_NO_DELAY_OPT.applyIfSet(configuration, + soCfgBuilder::setTcpNoDelay); // Proxy - String proxyHost = chConfiguration.get(ClientConfigProperties.PROXY_HOST.getKey()); - String proxyPort = chConfiguration.get(ClientConfigProperties.PROXY_PORT.getKey()); + String proxyHost = (String) configuration.get(ClientConfigProperties.PROXY_HOST.getKey()); + Integer proxyPort = (Integer) configuration.get(ClientConfigProperties.PROXY_PORT.getKey()); HttpHost proxy = null; if (proxyHost != null && proxyPort != null) { - proxy = new HttpHost(proxyHost, Integer.parseInt(proxyPort)); + proxy = new HttpHost(proxyHost, proxyPort); } - String proxyTypeVal = chConfiguration.get(ClientConfigProperties.PROXY_TYPE.getKey()); + String proxyTypeVal = (String) configuration.get(ClientConfigProperties.PROXY_TYPE.getKey()); ProxyType proxyType = proxyTypeVal == null ? null : ProxyType.valueOf(proxyTypeVal); if (proxyType == ProxyType.HTTP) { clientBuilder.setProxy(proxy); - if (chConfiguration.containsKey("proxy_password") && chConfiguration.containsKey("proxy_user")) { - proxyAuthHeaderValue = "Basic " + Base64.getEncoder().encodeToString( - (chConfiguration.get("proxy_user") + ":" + chConfiguration.get("proxy_password")).getBytes()); + String proxyUser = (String) configuration.get(ClientConfigProperties.PROXY_USER.getKey()); + String proxyPassword = (String) configuration.get(ClientConfigProperties.PROXY_PASSWORD.getKey()); + if (proxyUser != null && proxyPassword != null) { + proxyAuthHeaderValue = "Basic " + Base64.getEncoder().encodeToString((proxyUser + ":" + proxyPassword).getBytes(StandardCharsets.UTF_8)); } + } else if (proxyType == ProxyType.SOCKS) { - soCfgBuilder.setSocksProxyAddress(new InetSocketAddress(proxyHost, Integer.parseInt(proxyPort))); + soCfgBuilder.setSocksProxyAddress(new InetSocketAddress(proxyHost, proxyPort)); } - if (chConfiguration.getOrDefault("client.http.cookies_enabled", "true") - .equalsIgnoreCase("false")) { + if (ClientConfigProperties.HTTP_SAVE_COOKIES.getOrDefault(configuration)) { clientBuilder.disableCookieManagement(); } SocketConfig socketConfig = soCfgBuilder.build(); // Connection manager - boolean isConnectionPooling = MapUtils.getFlag(chConfiguration, "connection_pool_enabled"); - if (isConnectionPooling) { - clientBuilder.setConnectionManager(poolConnectionManager(sslConnectionSocketFactory, socketConfig)); + if (ClientConfigProperties.CONNECTION_POOL_ENABLED.getOrDefault(configuration)) { + clientBuilder.setConnectionManager(poolConnectionManager(sslConnectionSocketFactory, socketConfig, configuration)); } else { - clientBuilder.setConnectionManager(basicConnectionManager(sslConnectionSocketFactory, socketConfig)); + clientBuilder.setConnectionManager(basicConnectionManager(sslConnectionSocketFactory, socketConfig, configuration)); } - long keepAliveTimeout = MapUtils.getLong(chConfiguration, ClientConfigProperties.HTTP_KEEP_ALIVE_TIMEOUT.getKey()); - if (keepAliveTimeout > 0) { + Long keepAliveTimeout = ClientConfigProperties.HTTP_KEEP_ALIVE_TIMEOUT.getOrDefault(configuration); + if (keepAliveTimeout != null && keepAliveTimeout > 0) { clientBuilder.setKeepAliveStrategy((response, context) -> TimeValue.ofMilliseconds(keepAliveTimeout)); } @@ -314,18 +307,19 @@ public CloseableHttpClient createHttpClient(boolean initSslContext) { } private static final String ERROR_CODE_PREFIX_PATTERN = "Code: %d. DB::Exception:"; + /** * Reads status line and if error tries to parse response body to get server error message. * * @param httpResponse - HTTP response - * @return + * @return exception object with server code */ public Exception readError(ClassicHttpResponse httpResponse) { int serverCode = getHeaderInt(httpResponse.getFirstHeader(ClickHouseHttpProto.HEADER_EXCEPTION_CODE), 0); try (InputStream body = httpResponse.getEntity().getContent()) { - byte [] buffer = new byte[ERROR_BODY_BUFFER_SIZE]; - byte [] lookUpStr = String.format(ERROR_CODE_PREFIX_PATTERN, serverCode).getBytes(StandardCharsets.UTF_8); + byte[] buffer = new byte[ERROR_BODY_BUFFER_SIZE]; + byte[] lookUpStr = String.format(ERROR_CODE_PREFIX_PATTERN, serverCode).getBytes(StandardCharsets.UTF_8); StringBuilder msgBuilder = new StringBuilder(); boolean found = false; while (true) { @@ -376,7 +370,7 @@ public Exception readError(ClassicHttpResponse httpResponse) { } private static final long POOL_VENT_TIMEOUT = 10000L; - private AtomicLong timeToPoolVent = new AtomicLong(0); + private final AtomicLong timeToPoolVent = new AtomicLong(0); public ClassicHttpResponse executeRequest(Endpoint server, Map requestConfig, LZ4Factory lz4Factory, IOCallback writeCallback) throws IOException { @@ -391,30 +385,30 @@ public ClassicHttpResponse executeRequest(Endpoint server, Map r URI uri; try { URIBuilder uriBuilder = new URIBuilder(server.getBaseURL()); - addQueryParams(uriBuilder, chConfiguration, requestConfig); + addQueryParams(uriBuilder, requestConfig); uri = uriBuilder.normalizeSyntax().build(); } catch (URISyntaxException e) { throw new RuntimeException(e); } HttpPost req = new HttpPost(uri); // req.setVersion(new ProtocolVersion("HTTP", 1, 0)); // to disable chunk transfer encoding - addHeaders(req, chConfiguration, requestConfig); + addHeaders(req, requestConfig); - boolean clientCompression = MapUtils.getFlag(requestConfig, chConfiguration, ClientConfigProperties.COMPRESS_CLIENT_REQUEST.getKey()); - boolean useHttpCompression = MapUtils.getFlag(requestConfig, chConfiguration, ClientConfigProperties.USE_HTTP_COMPRESSION.getKey()); - boolean appCompressedData = MapUtils.getFlag(requestConfig, chConfiguration, ClientConfigProperties.APP_COMPRESSED_DATA.getKey()); + boolean clientCompression = ClientConfigProperties.COMPRESS_CLIENT_REQUEST.getOrDefault(requestConfig); + boolean useHttpCompression = ClientConfigProperties.USE_HTTP_COMPRESSION.getOrDefault(requestConfig); + boolean appCompressedData = ClientConfigProperties.APP_COMPRESSED_DATA.getOrDefault(requestConfig); req.setConfig(baseRequestConfig); // setting entity. wrapping if compression is enabled req.setEntity(wrapRequestEntity(new EntityTemplate(-1, CONTENT_TYPE, null, writeCallback), - clientCompression, useHttpCompression, appCompressedData, lz4Factory)); + clientCompression, useHttpCompression, appCompressedData, lz4Factory, requestConfig)); HttpClientContext context = HttpClientContext.create(); try { ClassicHttpResponse httpResponse = httpClient.executeOpen(null, req, context); - boolean serverCompression = MapUtils.getFlag(requestConfig, chConfiguration, ClientConfigProperties.COMPRESS_SERVER_RESPONSE.getKey()); - httpResponse.setEntity(wrapResponseEntity(httpResponse.getEntity(), httpResponse.getCode(), serverCompression, useHttpCompression, lz4Factory)); + boolean serverCompression = ClientConfigProperties.COMPRESS_SERVER_RESPONSE.getOrDefault(requestConfig); + httpResponse.setEntity(wrapResponseEntity(httpResponse.getEntity(), httpResponse.getCode(), serverCompression, useHttpCompression, lz4Factory, requestConfig)); if (httpResponse.getCode() == HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED) { throw new ClientMisconfigurationException("Proxy authentication required. Please check your proxy settings."); @@ -436,7 +430,8 @@ public ClassicHttpResponse executeRequest(Endpoint server, Map r } catch (ConnectException | NoRouteToHostException e) { LOG.warn("Failed to connect to '{}': {}", server.getBaseURL(), e.getMessage()); throw new ClientException("Failed to connect", e); - } catch (ConnectionRequestTimeoutException | ServerException | NoHttpResponseException | ClientException | SocketTimeoutException e) { + } catch (ConnectionRequestTimeoutException | ServerException | NoHttpResponseException | ClientException | + SocketTimeoutException e) { throw e; } catch (Exception e) { throw new ClientException(e.getMessage(), e); @@ -445,7 +440,7 @@ public ClassicHttpResponse executeRequest(Endpoint server, Map r private static final ContentType CONTENT_TYPE = ContentType.create(ContentType.TEXT_PLAIN.getMimeType(), "UTF-8"); - private void addHeaders(HttpPost req, Map chConfig, Map requestConfig) { + private void addHeaders(HttpPost req, Map requestConfig) { req.addHeader(HttpHeaders.CONTENT_TYPE, CONTENT_TYPE.getMimeType()); if (requestConfig.containsKey(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey())) { req.addHeader(ClickHouseHttpProto.HEADER_FORMAT, requestConfig.get(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey())); @@ -455,31 +450,33 @@ private void addHeaders(HttpPost req, Map chConfig, Map chConfig, Map chConfig, Map chConfig, Map requestConfig) { - for (String key : chConfig.keySet()) { - if (key.startsWith(ClientConfigProperties.SERVER_SETTING_PREFIX)) { - req.addParameter(key.substring(ClientConfigProperties.SERVER_SETTING_PREFIX.length()), chConfig.get(key)); - } - } - + private void addQueryParams(URIBuilder req, Map requestConfig) { if (requestConfig.containsKey(ClientConfigProperties.QUERY_ID.getKey())) { req.addParameter(ClickHouseHttpProto.QPARAM_QUERY_ID, requestConfig.get(ClientConfigProperties.QUERY_ID.getKey()).toString()); } @@ -535,9 +520,9 @@ private void addQueryParams(URIBuilder req, Map chConfig, Map chConfig, Map sessionRoles = (Collection) requestConfig.getOrDefault(ClientConfigProperties.SESSION_DB_ROLES.getKey(), - ClientConfigProperties.valuesFromCommaSeparated(chConfiguration.getOrDefault(ClientConfigProperties.SESSION_DB_ROLES.getKey(), ""))); - if (!sessionRoles.isEmpty()) { + Collection sessionRoles = ClientConfigProperties.SESSION_DB_ROLES.getOrDefault(requestConfig); + if (!(sessionRoles == null || sessionRoles.isEmpty())) { sessionRoles.forEach(r -> req.addParameter(ClickHouseHttpProto.QPARAM_ROLE, r)); } @@ -570,19 +554,20 @@ private void addQueryParams(URIBuilder req, Map chConfig, Map requestConfig) { + LOG.debug("wrapRequestEntity: client compression: {}, http compression: {}", clientCompression, useHttpCompression); if (clientCompression && !appControlledCompression) { + int buffSize = ClientConfigProperties.COMPRESSION_LZ4_UNCOMPRESSED_BUF_SIZE.getOrDefault(requestConfig); return new LZ4Entity(httpEntity, useHttpCompression, false, true, - MapUtils.getInt(chConfiguration, "compression.lz4.uncompressed_buffer_size"), false, lz4Factory); - } else { + buffSize, false, lz4Factory); + } else { return httpEntity; } } - private HttpEntity wrapResponseEntity(HttpEntity httpEntity, int httpStatus, boolean serverCompression, boolean useHttpCompression, LZ4Factory lz4Factory) { - LOG.debug("server compression: {}, http compression: {}", serverCompression, useHttpCompression); + private HttpEntity wrapResponseEntity(HttpEntity httpEntity, int httpStatus, boolean serverCompression, boolean useHttpCompression, LZ4Factory lz4Factory, Map requestConfig) { + LOG.debug("wrapResponseEntity: server compression: {}, http compression: {}", serverCompression, useHttpCompression); if (serverCompression) { // Server doesn't compress certain errors like 403 @@ -597,8 +582,8 @@ private HttpEntity wrapResponseEntity(HttpEntity httpEntity, int httpStatus, boo case HttpStatus.SC_BAD_REQUEST: case HttpStatus.SC_INTERNAL_SERVER_ERROR: case HttpStatus.SC_NOT_FOUND: - return new LZ4Entity(httpEntity, useHttpCompression, true, false, - MapUtils.getInt(chConfiguration, "compression.lz4.uncompressed_buffer_size"), true, lz4Factory); + int buffSize = ClientConfigProperties.COMPRESSION_LZ4_UNCOMPRESSED_BUF_SIZE.getOrDefault(requestConfig); + return new LZ4Entity(httpEntity, useHttpCompression, true, false, buffSize, true, lz4Factory); } } @@ -622,8 +607,7 @@ public static T getHeaderVal(Header header, T defaultValue, Function requestSettings) { - Set retryCauses = (Set) - requestSettings.getOrDefault(ClientConfigProperties.CLIENT_RETRY_ON_FAILURE.getKey(), defaultRetryCauses); + List retryCauses = ClientConfigProperties.CLIENT_RETRY_ON_FAILURE.getOrDefault(requestSettings); if (retryCauses.contains(ClientFaultCause.None)) { return false; @@ -671,54 +655,12 @@ public RuntimeException wrapException(String message, Exception cause) { return new ClientException(message, cause); } - - /** - * Parses URL parameters. - * @param url - * @return Map of parameters - */ - public static Map parseUrlParameters(URL url) { - Map params = new HashMap<>(); - - try { - String path = url.getPath(); - path = path.substring(path.indexOf('/') + 1); - LOG.debug("path: {}", path); - if (!path.trim().isEmpty()) { - params.put("database", path); - } else { - params.put("database", "default"); - } - - String query = url.getQuery(); - if (query != null) { - for (String pair : query.split("&")) { - int idx = pair.indexOf("="); - if (idx > 0) { - params.put(pair.substring(0, idx), pair.substring(idx + 1)); - } - } - } - } catch (Exception e) { - LOG.error("Failed to parse URL parameters", e); - } - - return params; - } - - private void correctUserAgentHeader(HttpRequest request, Map requestConfig) { //TODO: implement cache for user-agent Header userAgentHeader = request.getLastHeader(HttpHeaders.USER_AGENT); request.removeHeaders(HttpHeaders.USER_AGENT); - String clientName = chConfiguration.getOrDefault(ClientConfigProperties.CLIENT_NAME.getKey(), ""); - if (requestConfig != null) { - String reqClientName = (String) requestConfig.get(ClientConfigProperties.CLIENT_NAME.getKey()); - if (reqClientName != null && !reqClientName.isEmpty()) { - clientName = reqClientName; - } - } + String clientName = ClientConfigProperties.CLIENT_NAME.getOrDefault(requestConfig); String userAgentValue = defaultUserAgent; if (userAgentHeader == null && clientName != null && !clientName.isEmpty()) { userAgentValue = clientName + " " + defaultUserAgent; @@ -729,7 +671,7 @@ private void correctUserAgentHeader(HttpRequest request, Map req request.setHeader(HttpHeaders.USER_AGENT, userAgentValue); } - private String buildDefaultUserAgent() { + private String buildDefaultUserAgent() { StringBuilder userAgent = new StringBuilder(); userAgent.append(Client.CLIENT_USER_AGENT); @@ -798,7 +740,7 @@ public Socket connectSocket(TimeValue connectTimeout, Socket socket, HttpHost ho } } - public class MeteredManagedHttpClientConnectionFactory extends ManagedHttpClientConnectionFactory { + public static class MeteredManagedHttpClientConnectionFactory extends ManagedHttpClientConnectionFactory { public MeteredManagedHttpClientConnectionFactory(Http1Config http1Config, CharCodingConfig charCodingConfig, DefaultHttpResponseParserFactory defaultHttpResponseParserFactory) { super(http1Config, charCodingConfig, defaultHttpResponseParserFactory); } @@ -821,7 +763,10 @@ public long getTime() { int count = times.size(); long runningAverage = 0; for (int i = 0; i < count; i++) { - runningAverage += times.poll(); + Long t = times.poll(); + if (t != null) { + runningAverage += t; + } } return count > 0 ? runningAverage / count : 0; diff --git a/client-v2/src/main/java/com/clickhouse/client/api/query/QuerySettings.java b/client-v2/src/main/java/com/clickhouse/client/api/query/QuerySettings.java index b1eb68607..16580d129 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/query/QuerySettings.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/query/QuerySettings.java @@ -20,10 +20,13 @@ public class QuerySettings { public static final int MINIMAL_READ_BUFFER_SIZE = 8192; - private Map rawSettings; + private final Map rawSettings; + public QuerySettings(Map rawSettings) { + this.rawSettings = rawSettings; + } public QuerySettings() { - this.rawSettings = new HashMap<>(); + this(new HashMap<>()); } /** @@ -175,7 +178,7 @@ public TimeZone getServerTimeZone() { * @return same instance of the builder */ public QuerySettings httpHeader(String key, String value) { - rawSettings.put(ClientConfigProperties.HTTP_HEADER_PREFIX + key, value); + rawSettings.put(ClientConfigProperties.httpHeader(key), value); return this; } @@ -186,7 +189,7 @@ public QuerySettings httpHeader(String key, String value) { * @return same instance of the builder */ public QuerySettings httpHeader(String key, Collection values) { - rawSettings.put(ClientConfigProperties.HTTP_HEADER_PREFIX + key, ClientConfigProperties.commaSeparated(values)); + rawSettings.put(ClientConfigProperties.httpHeader(key), ClientConfigProperties.commaSeparated(values)); return this; } diff --git a/client-v2/src/test/java/com/clickhouse/client/ClientTests.java b/client-v2/src/test/java/com/clickhouse/client/ClientTests.java index 3499f1dc1..5471f882b 100644 --- a/client-v2/src/test/java/com/clickhouse/client/ClientTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/ClientTests.java @@ -206,7 +206,7 @@ public void testDefaultSettings() { Assert.assertEquals(config.get(p.getKey()), p.getDefaultValue(), "Default value doesn't match"); } } - Assert.assertEquals(config.size(), 27); // to check everything is set. Increment when new added. + Assert.assertEquals(config.size(), 31); // to check everything is set. Increment when new added. } try (Client client = new Client.Builder() @@ -239,7 +239,7 @@ public void testDefaultSettings() { .setSocketSndbuf(100000) .build()) { Map config = client.getConfiguration(); - Assert.assertEquals(config.size(), 28); // to check everything is set. Increment when new added. + Assert.assertEquals(config.size(), 32); // to check everything is set. Increment when new added. Assert.assertEquals(config.get(ClientConfigProperties.DATABASE.getKey()), "mydb"); Assert.assertEquals(config.get(ClientConfigProperties.MAX_EXECUTION_TIME.getKey()), "10"); Assert.assertEquals(config.get(ClientConfigProperties.COMPRESSION_LZ4_UNCOMPRESSED_BUF_SIZE.getKey()), "300000"); @@ -306,7 +306,7 @@ public void testWithOldDefaults() { Assert.assertEquals(config.get(p.getKey()), p.getDefaultValue(), "Default value doesn't match"); } } - Assert.assertEquals(config.size(), 27); // to check everything is set. Increment when new added. + Assert.assertEquals(config.size(), 31); // to check everything is set. Increment when new added. } } diff --git a/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java b/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java index dbc41304f..6cd2456ae 100644 --- a/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java @@ -574,7 +574,7 @@ public void testServerSettings() { mockServer.addStubMapping(WireMock.post(WireMock.anyUrl()) .withQueryParam("max_threads", WireMock.equalTo("10")) - .withQueryParam("async_insert", WireMock.equalTo("1")) + .withQueryParam("async_insert", WireMock.equalTo("3")) .withQueryParam("roles", WireMock.equalTo("role3,role2")) .withQueryParam("compress", WireMock.equalTo("0")) .willReturn(WireMock.aResponse() diff --git a/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java b/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java index ba9b8f018..749985d12 100644 --- a/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java @@ -122,7 +122,6 @@ public void setUp() { client = newClient().build(); delayForProfiler(0); - System.out.println("Real port: " + node.getPort()); } @AfterMethod(groups = {"integration"}) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java index dd1865ff7..d71b822bc 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java @@ -1,6 +1,7 @@ package com.clickhouse.jdbc; import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.api.internal.ServerSettings; import com.clickhouse.data.Tuple; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,7 +42,7 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; - +@Test(groups = { "integration" }) public class DataTypeTests extends JdbcIntegrationTest { private static final Logger log = LoggerFactory.getLogger(DataTypeTests.class); @@ -49,13 +50,9 @@ public class DataTypeTests extends JdbcIntegrationTest { public static void setUp() throws SQLException { Driver.load(); } - - private Connection getConnection() throws SQLException { - return getJdbcConnection(); - } - + private int insertData(String sql) throws SQLException { - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { return stmt.executeUpdate(sql); } @@ -99,7 +96,7 @@ public void testIntegerTypes() throws SQLException { BigInteger uint128 = new BigInteger(128, rand); BigInteger uint256 = new BigInteger(256, rand); - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (PreparedStatement stmt = conn.prepareStatement("INSERT INTO test_integers VALUES ( 3, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")) { stmt.setInt(1, int8); stmt.setInt(2, int16); @@ -118,7 +115,7 @@ public void testIntegerTypes() throws SQLException { } // Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_integers ORDER BY order")) { assertTrue(rs.next()); @@ -169,7 +166,7 @@ public void testIntegerTypes() throws SQLException { } // Check the with getObject - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_integers ORDER BY order")) { assertTrue(rs.next()); @@ -245,7 +242,7 @@ public void testDecimalTypes() throws SQLException { BigDecimal dec128 = new BigDecimal(new BigInteger(20, rand) + "." + rand.nextLong(100000000000000000L, 1000000000000000000L)); BigDecimal dec256 = new BigDecimal(new BigInteger(58, rand) + "." + rand.nextLong(100000000000000000L, 1000000000000000000L)); - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (PreparedStatement stmt = conn.prepareStatement("INSERT INTO test_decimals VALUES ( 3, ?, ?, ?, ?, ?)")) { stmt.setBigDecimal(1, dec); stmt.setBigDecimal(2, dec32); @@ -258,7 +255,7 @@ public void testDecimalTypes() throws SQLException { // Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_decimals ORDER BY order")) { assertTrue(rs.next()); @@ -288,7 +285,7 @@ public void testDecimalTypes() throws SQLException { } // Check the results with getObject - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_decimals ORDER BY order")) { assertTrue(rs.next()); @@ -352,7 +349,7 @@ public void testDateTypes() throws SQLException { final java.sql.Timestamp dateTime649 = Timestamp.valueOf(LocalDateTime.now(ZoneId.of("America/Los_Angeles"))); dateTime649.setNanos(333333333); - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (PreparedStatement stmt = conn.prepareStatement("INSERT INTO test_dates VALUES ( 4, ?, ?, ?, ?, ?, ?, ?)")) { stmt.setDate(1, date); stmt.setDate(2, date32); @@ -366,7 +363,7 @@ public void testDateTypes() throws SQLException { } // Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_dates ORDER BY order")) { assertTrue(rs.next()); @@ -408,7 +405,7 @@ public void testDateTypes() throws SQLException { } // Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_dates ORDER BY order")) { assertTrue(rs.next()); @@ -464,7 +461,7 @@ public void testStringTypes() throws SQLException { String uuid = UUID.randomUUID().toString(); String escaped = "\\xA3\\xA3\\x12\\xA0\\xDF\\x13\\x4E\\x8C\\x87\\x74\\xD4\\x53\\xDB\\xFC\\x34\\x95"; - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (PreparedStatement stmt = conn.prepareStatement("INSERT INTO test_strings VALUES ( 1, ?, ?, ?, ?, ?, ?, ? )")) { stmt.setString(1, str); stmt.setString(2, fixed); @@ -478,7 +475,7 @@ public void testStringTypes() throws SQLException { } // Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_strings ORDER BY order")) { assertTrue(rs.next()); @@ -498,7 +495,7 @@ public void testStringTypes() throws SQLException { } // Check the results with getObject - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_strings ORDER BY order")) { assertTrue(rs.next()); @@ -530,7 +527,7 @@ public void testIpAddressTypes() throws SQLException, UnknownHostException { InetAddress ipv6Address = Inet6Address.getByName("2001:adb8:85a3:1:2:8a2e:370:7334"); InetAddress ipv4AsIpv6 = Inet4Address.getByName("90.176.75.97"); - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (PreparedStatement stmt = conn.prepareStatement("INSERT INTO test_ips VALUES ( 1, ?, ?, ?, ? )")) { stmt.setObject(1, ipv4AddressByIp); stmt.setObject(2, ipv4AddressByName); @@ -541,7 +538,7 @@ public void testIpAddressTypes() throws SQLException, UnknownHostException { } // Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_ips ORDER BY order")) { assertTrue(rs.next()); @@ -577,7 +574,7 @@ public void testFloatTypes() throws SQLException { Float float32 = rand.nextFloat(); Double float64 = rand.nextDouble(); - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (PreparedStatement stmt = conn.prepareStatement("INSERT INTO test_floats VALUES ( 3, ?, ? )")) { stmt.setFloat(1, float32); stmt.setDouble(2, float64); @@ -586,7 +583,7 @@ public void testFloatTypes() throws SQLException { } // Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_floats ORDER BY order")) { assertTrue(rs.next()); @@ -607,7 +604,7 @@ public void testFloatTypes() throws SQLException { } // Check the results with getObject - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_floats ORDER BY order")) { assertTrue(rs.next()); @@ -643,7 +640,7 @@ public void testBooleanTypes() throws SQLException { boolean bool = rand.nextBoolean(); - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (PreparedStatement stmt = conn.prepareStatement("INSERT INTO test_booleans VALUES ( 1, ? )")) { stmt.setBoolean(1, bool); stmt.executeUpdate(); @@ -651,7 +648,7 @@ public void testBooleanTypes() throws SQLException { } // Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_booleans ORDER BY order")) { assertTrue(rs.next()); @@ -663,7 +660,7 @@ public void testBooleanTypes() throws SQLException { } // Check the results with getObject - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_booleans ORDER BY order")) { assertTrue(rs.next()); @@ -707,7 +704,7 @@ public void testArrayTypes() throws SQLException { } // Insert random (valid) values - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (PreparedStatement stmt = conn.prepareStatement("INSERT INTO test_arrays VALUES ( 1, ?, ?, ?, ?)")) { stmt.setArray(1, conn.createArrayOf("Int8", array)); stmt.setArray(2, conn.createArrayOf("String", arraystr)); @@ -718,7 +715,7 @@ public void testArrayTypes() throws SQLException { } // Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_arrays ORDER BY order")) { assertTrue(rs.next()); @@ -753,7 +750,7 @@ public void testArrayTypes() throws SQLException { } // Check the results with getObject - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_arrays ORDER BY order")) { assertTrue(rs.next()); @@ -811,7 +808,7 @@ public void testMapTypes() throws SQLException { } // Insert random (valid) values - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (PreparedStatement stmt = conn.prepareStatement("INSERT INTO test_maps VALUES ( 1, ?, ? )")) { stmt.setObject(1, integerMap); stmt.setObject(2, stringMap); @@ -820,7 +817,7 @@ public void testMapTypes() throws SQLException { } // Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_maps ORDER BY order")) { assertTrue(rs.next()); @@ -863,7 +860,7 @@ public void testNullableTypesSimpleStatement() throws SQLException { + "NULL, NULL, NULL, NULL)"); //Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_nullable ORDER BY order")) { assertTrue(rs.next()); @@ -894,7 +891,7 @@ public void testLowCardinalityTypeSimpleStatement() throws SQLException { lowcardinality)); // Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_low_cardinality ORDER BY order")) { assertTrue(rs.next()); @@ -925,7 +922,7 @@ public void testSimpleAggregateFunction() throws SQLException { insertData(String.format("INSERT INTO test_aggregate VALUES ( 3, %d, null )", int8)); // Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT sum(int8) FROM test_aggregate")) { assertTrue(rs.next()); @@ -965,7 +962,7 @@ public void testNestedTypeSimpleStatement() throws SQLException { insertData(sql); // Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_nested ORDER BY order")) { assertTrue(rs.next()); @@ -988,7 +985,7 @@ public void testNestedTypeNonFlatten() throws SQLException { if (earlierThan(25,1)){ return; } - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { stmt.execute("SET flatten_nested = 0"); stmt.execute("CREATE TABLE test_nested_not_flatten (order Int8, " @@ -1048,7 +1045,7 @@ public void testTupleTypeSimpleStatement() throws SQLException { insertData(sql); // Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_tuple ORDER BY order")) { assertTrue(rs.next()); @@ -1067,7 +1064,7 @@ public void testTupleTypeSimpleStatement() throws SQLException { - @Test (enabled = false)//TODO: This type is experimental right now + @Test public void testJSONTypeSimpleStatement() throws SQLException { runQuery("CREATE TABLE test_json (order Int8, " + "json JSON" @@ -1078,24 +1075,29 @@ public void testJSONTypeSimpleStatement() throws SQLException { Random rand = new Random(seed); log.info("Random seed was: {}", seed); - String json = "{\"key1\": \"" + rand.nextDouble() + "\", \"key2\": " + rand.nextInt() + ", \"key3\": [\"value3\", 4]}"; + double key1 = rand.nextDouble(); + int key2 = rand.nextInt(); + final String json = "{\"key1\": \"" + key1 + "\", \"key2\": " + key2 + ", \"key3\": [1000, \"value3\", 400000]}"; + final String serverJson = "{\"key1\":\"" + key1 + "\",\"key2\":" + key2 + ",\"key3\":[\"1000\",\"value3\",\"400000\"]}"; insertData(String.format("INSERT INTO test_json VALUES ( 1, '%s' )", json)); // Check the results - try (Connection conn = getConnection()) { + Properties props = new Properties(); + props.setProperty( + ClientConfigProperties.serverSetting(ServerSettings.OUTPUT_FORMAT_BINARY_WRITE_JSON_AS_STRING), + "1"); + props.setProperty(ClientConfigProperties.serverSetting("output_format_json_quote_64bit_integers"), "0"); + try (Connection conn = getJdbcConnection(props)) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_json ORDER BY order")) { assertTrue(rs.next()); - assertEquals(rs.getString("json"), json); - + assertEquals(rs.getString("json"), serverJson); assertFalse(rs.next()); } } } } - - @Test(groups = { "integration" }, enabled = false) public void testGeometricTypesSimpleStatement() throws SQLException { // TODO: add LineString and MultiLineString support @@ -1119,7 +1121,7 @@ public void testGeometricTypesSimpleStatement() throws SQLException { point, ring, linestring, multilinestring, polygon, multipolygon)); // Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_geometric ORDER BY order")) { assertTrue(rs.next()); @@ -1169,7 +1171,7 @@ public void testDynamicTypesSimpleStatement() throws SQLException { insertData(sql); // Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_dynamic ORDER BY order")) { assertTrue(rs.next()); @@ -1190,7 +1192,7 @@ public void testDynamicTypesSimpleStatement() throws SQLException { @Test(groups = { "integration" }) public void testTypeConversions() throws Exception { - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT 1, 'true', '1.0', " + "toDate('2024-12-01'), toDateTime('2024-12-01 12:34:56'), toDateTime64('2024-12-01 12:34:56.789', 3), toDateTime64('2024-12-01 12:34:56.789789', 6), toDateTime64('2024-12-01 12:34:56.789789789', 9)")) { @@ -1282,7 +1284,7 @@ public void testVariantTypesSimpleStatement() throws SQLException { // Check the results - try (Connection conn = getConnection()) { + try (Connection conn = getJdbcConnection()) { try (Statement stmt = conn.createStatement()) { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_variant ORDER BY order")) { assertTrue(rs.next()); From e554769f2d036d609b2755c571db0515c00d9280 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 25 Jun 2025 19:52:29 -0700 Subject: [PATCH 2/4] fixed parsing when table name is keyword --- .../jdbc/internal/ParsedPreparedStatement.java | 9 ++++++++- .../java/com/clickhouse/jdbc/internal/SqlParserTest.java | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java index 00d4aafdb..acd3f19c2 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java @@ -211,11 +211,18 @@ private void appendParameter(int startIndex) { } } + @Override + public void enterTableExprIdentifier(ClickHouseParser.TableExprIdentifierContext ctx) { + if (ctx.tableIdentifier() != null) { + this.table = SqlParser.unquoteIdentifier(ctx.tableIdentifier().getText()); + } + } + @Override public void enterInsertStmt(ClickHouseParser.InsertStmtContext ctx) { ClickHouseParser.TableIdentifierContext tableId = ctx.tableIdentifier(); if (tableId != null) { - this.table = tableId.identifier().IDENTIFIER().getText(); + this.table = SqlParser.unquoteIdentifier(tableId.getText()); } ClickHouseParser.ColumnsClauseContext columns = ctx.columnsClause(); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java index 33bfb5b8b..96a6e3b8d 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java @@ -275,6 +275,7 @@ public void testMiscStatements(String sql, int args) { ParsedPreparedStatement stmt = parser.parsePreparedStatement(sql); Assert.assertFalse(stmt.isHasErrors()); Assert.assertEquals(stmt.getArgCount(), args); + System.out.println(stmt.getTable()); } @DataProvider @@ -294,6 +295,8 @@ public Object[][] testMiscStmtDp() { {"SELECT * FROM table test WHERE ts = ?", 1}, {"SELECT * FROM table view WHERE ts = ?", 1}, {"SELECT * FROM table primary WHERE ts = ?", 1}, + {"insert into events (s) values ('a')", 0}, + {"insert into `events` (s) values ('a')", 0}, }; } } \ No newline at end of file From 0e59b6efc277c647f708bd965c2f5c83b13fbbcc Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Mon, 30 Jun 2025 09:40:54 -0700 Subject: [PATCH 3/4] added version check to the test. fixed NPE getting operation timeout setting --- .../src/main/java/com/clickhouse/client/api/Client.java | 2 +- .../client/api/internal/HttpAPIClientHelper.java | 3 ++- .../src/test/java/com/clickhouse/jdbc/DataTypeTests.java | 9 +++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index f41ed7217..c747904f6 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -1980,7 +1980,7 @@ public Map getConfiguration() { /** Returns operation timeout in seconds */ protected int getOperationTimeout() { - return (int) configuration.get(ClientConfigProperties.MAX_EXECUTION_TIME.getKey()); + return ClientConfigProperties.MAX_EXECUTION_TIME.getOrDefault(configuration); } /** diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 86c68967b..e4bb92590 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -280,7 +280,8 @@ public CloseableHttpClient createHttpClient(boolean initSslContext, Map Date: Mon, 30 Jun 2025 10:10:07 -0700 Subject: [PATCH 4/4] fixed tests --- client-v2/src/test/java/com/clickhouse/client/ProxyTests.java | 4 +++- jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/ProxyTests.java b/client-v2/src/test/java/com/clickhouse/client/ProxyTests.java index af2a510c3..e5de2eeab 100644 --- a/client-v2/src/test/java/com/clickhouse/client/ProxyTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/ProxyTests.java @@ -135,7 +135,9 @@ public void testProxyWithCookies() { return; // to specific setup for cloud, may be later } - client.set(clientBuilder(initProxy(), true).build()); + client.set(clientBuilder(initProxy(), true) + .setHttpCookiesEnabled(true) + .build()); final int targetPort = getServer(ClickHouseProtocol.HTTP).getPort(); proxy.get().addStubMapping(post(urlMatching("/.*")) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java index 13f5b7797..e2ebe772e 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java @@ -1071,9 +1071,11 @@ public void testJSONWritingAsString() throws SQLException { return; // JSON was introduced in 24.10 } + Properties createProperties = new Properties(); + createProperties.put(ClientConfigProperties.serverSetting("allow_experimental_json_type"), "1"); runQuery("CREATE TABLE test_json (order Int8, " + "json JSON" - + ") ENGINE = MergeTree ORDER BY ()"); + + ") ENGINE = MergeTree ORDER BY ()", createProperties); // Insert random (valid) values long seed = System.currentTimeMillis();