diff --git a/.gitignore b/.gitignore index 8e4ab4f9f..2b66649b1 100644 --- a/.gitignore +++ b/.gitignore @@ -36,7 +36,9 @@ target/ **/parser/ClickHouseSqlParserTokenManager.java **/parser/Token*.java **/parser/ParseException.java +java.prof jmh-result.* +profile.html # Shell scripts *.sh diff --git a/CHANGELOG b/CHANGELOG index e9c0034ed..5463a629f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,8 @@ +0.3.1-patch + * fix incorrect database used in DDL + * fix batch insert issue when no VALUES used in SQL statement + * fix issue of handling negative decimal128/256 values + 0.3.1 * BREAKING CHANGE - move query from url to request body * BREAKING CHANGE - always parse SQL(use extended API to skip that) @@ -10,6 +15,7 @@ * fix parser issue when DESC statement contains alias * support batch processing with arbitrary query - update and delete are not recommended so there'll be warnings * support multi-statement sql - session will be used automatically and only the last result will be returned + 0.3.0 * BREAKING CHANGE - dropped JDK 7 support * BREAKING CHANGE - removed Guava dependency(and so is UnsignedLong) @@ -23,6 +29,7 @@ * fix error when using ClickHouseCompression.none against 19.16 * fix NegativeArraySizeException when dealing with large array * fix datetime/date display issue caused by timezone differences(between client and column/server) + 0.2.6 * add new feature for sending compressed files/streams * introduce an experimental SQL parser to fix parsing related issues - set connection setting use_new_parser to false to disable @@ -33,6 +40,7 @@ * upgrade to lz4-java and improve performance of LZ4 stream * use HTTP Basic Auth for credentials instead of query parameters * use static version instead of property-based revision in pom.xml + 0.2.5 * bump dependencies and include lz4 in shaded jar * new API: ClickHouseRowBinaryStream.writeUInt64Array(UnsignedLong[]) @@ -42,20 +50,26 @@ * fix ResultSet.findColumn(String) issue * fix the issue of not being able to use NULL constant in PreparedStatement * fix toLowerCase issue for Turkish + 0.2.4 * fix FORMAT clause append for queries, ending with comment + 0.2.3 * added support for Decimals in RowBinary protocol + 0.2.2 * close certificate keystore * fix for Boolean data type + 0.2.1 * implement some ResultSet metadata methods * added support for "any_join_distinct_right_table_keys" setting * nested array support + 0.2 * new API for writing streams of data * deprecation of send* methods in ClickHouseStatement interface + 0.1.55 NOTE: behavior for byte[] parameters changed. See https://github.com/yandex/clickhouse-jdbc/pull/352 * support for sending stream of CSV data diff --git a/README.md b/README.md index b615e9ed8..8bc5d345b 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,13 @@ ClickHouse JDBC driver This is a basic and restricted implementation of jdbc driver for ClickHouse. It has support of a minimal subset of features to be usable. + ### Usage ```xml ru.yandex.clickhouse clickhouse-jdbc - 0.3.1 + 0.3.2 ``` @@ -46,6 +47,7 @@ try (ClickHouseConnection conn = dataSource.getConnection(); Additionally, if you have a few instances, you can use `BalancedClickhouseDataSource`. + ### Extended API In order to provide non-JDBC complaint data manipulation functionality, proprietary API exists. Entry point for API is `ClickHouseStatement#write()` method. @@ -61,6 +63,7 @@ sth .data(new File("/path/to/file.csv.gz"), ClickHouseFormat.CSV, ClickHouseCompression.gzip) // specify input .send(); ``` + #### Configurable send ```java import ru.yandex.clickhouse.ClickHouseStatement; @@ -73,6 +76,7 @@ sth .addDbParam(ClickHouseQueryParam.MAX_PARALLEL_REPLICAS, 2) .send(); ``` + #### Send data in binary formatted with custom user callback ```java import ru.yandex.clickhouse.ClickHouseStatement; @@ -88,6 +92,12 @@ sth.write().send("INSERT INTO test.writer", new ClickHouseStreamCallback() { }, ClickHouseFormat.RowBinary); // RowBinary or Native are supported ``` + + +### Supported Server Versions +All [active releases](../ClickHouse/pulls?q=is%3Aopen+is%3Apr+label%3Arelease) are supported. You can still use the driver for older versions like 18.14 or 19.16 but please keep in mind that they're no longer supported. + + ### Compiling with maven The driver is built with maven. `mvn package -DskipTests=true` @@ -96,5 +106,6 @@ To build a jar with dependencies use `mvn package assembly:single -DskipTests=true` + ### Build requirements -In order to build the jdbc client one need to have jdk 1.7 or higher. +In order to build the jdbc client one need to have jdk 1.8 or higher. diff --git a/clickhouse-benchmark/pom.xml b/clickhouse-benchmark/pom.xml index fc02afda9..09fd5be64 100644 --- a/clickhouse-benchmark/pom.xml +++ b/clickhouse-benchmark/pom.xml @@ -18,12 +18,12 @@ 1.4.4 - 2.7.2 - 8.0.23 - 2.5.4 - 1.15.2 + 2.7.3 + 8.0.25 + 2.5.6 + 42.2.22 UTF-8 - 1.27 + 1.32 benchmarks @@ -97,6 +97,17 @@ + + org.postgresql + postgresql + ${postgresql-driver.version} + + + * + * + + + org.testcontainers testcontainers diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Basic.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Basic.java index 079001636..f6e6c118b 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Basic.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Basic.java @@ -3,32 +3,27 @@ import java.sql.ResultSet; import java.sql.Statement; import java.util.Collections; -import java.util.Random; import org.openjdk.jmh.annotations.Benchmark; public class Basic extends JdbcBenchmark { @Benchmark - public int selectOneRandomNumber(ClientState state) throws Throwable { - final int num = new Random().nextInt(1000); + public int insertOneRandomNumber(ClientState state) throws Throwable { + final int num = state.getRandomSample(); - try (Statement stmt = executeQuery(state, "select ? as n", num)) { - ResultSet rs = stmt.getResultSet(); + return executeInsert(state, "insert into test_insert(i) values(?)", + Collections.enumeration(Collections.singletonList(new Object[] { num }))); + } - rs.next(); + @Benchmark + public int selectOneRandomNumber(ClientState state) throws Throwable { + final int num = state.getRandomSample(); - if (num != rs.getInt(1)) { + try (Statement stmt = executeQuery(state, "select ? as n", num); ResultSet rs = stmt.getResultSet();) { + if (!rs.next() || num != rs.getInt(1)) { throw new IllegalStateException(); } return num; } } - - @Benchmark - public int insertOneRandomNumber(ClientState state) throws Throwable { - final int num = new Random().nextInt(1000); - - return executeInsert(state, "insert into test_insert(i) values(?)", - Collections.enumeration(Collections.singletonList(new Object[] { num }))); - } } diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java index 13d787529..bf5d49f74 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java @@ -1,9 +1,12 @@ package tech.clickhouse.benchmark; import java.sql.Connection; +import java.sql.Driver; import java.sql.SQLException; import java.sql.Statement; import java.util.Properties; +import java.util.Random; + import org.openjdk.jmh.annotations.Level; import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; @@ -13,27 +16,43 @@ @State(Scope.Thread) public class ClientState { - @Param(value = { "clickhouse4j", Constants.CLICKHOUSE_DRIVER, "clickhouse-native-jdbc-shaded", - "mariadb-java-client", "mysql-connector-java" }) + // sample size used in 10k query/insert + private final int SAMPLE_SIZE = Integer.parseInt(System.getProperty("sampleSize", "10000")); + // floating range(to reduce server-side cache hits) used in 10k query/insert + private final int FLOATING_RANGE = Integer.parseInt(System.getProperty("floatingRange", "100")); + + @Param(value = { "clickhouse4j", "clickhouse-jdbc", "clickhouse-native-jdbc-shaded", "mariadb-java-client", + "mysql-connector-java", "postgresql-jdbc" }) private String client; + @Param(value = { Constants.REUSE_CONNECTION, Constants.NEW_CONNECTION }) + private String connection; + @Param(value = { Constants.NORMAL_STATEMENT, Constants.PREPARED_STATEMENT }) private String statement; + private Driver driver; + private String url; private Connection conn; + private int randomSample; + private int randomNum; + @Setup(Level.Trial) public void doSetup(ServerState serverState) throws Exception { - JdbcDriver driver = JdbcDriver.from(client); + JdbcDriver jdbcDriver = JdbcDriver.from(client); try { - conn = ((java.sql.Driver) Class.forName(driver.getClassName()).getDeclaredConstructor().newInstance()) - .connect(String.format(driver.getUrlTemplate(), serverState.getHost(), - serverState.getPort(driver.getDefaultPort()), serverState.getDatabase(), - serverState.getUser(), serverState.getPassword()), new Properties()); + driver = (java.sql.Driver) Class.forName(jdbcDriver.getClassName()).getDeclaredConstructor().newInstance(); + url = String.format(jdbcDriver.getUrlTemplate(), serverState.getHost(), + serverState.getPort(jdbcDriver.getDefaultPort()), serverState.getDatabase(), serverState.getUser(), + serverState.getPassword()); + conn = driver.connect(url, new Properties()); try (Statement s = conn.createStatement()) { - s.execute("create table if not exists system.test_insert(i Nullable(UInt64), s Nullable(String), t Nullable(DateTime))engine=Memory"); + s.execute("truncate table if exists system.test_insert"); + s.execute( + "create table if not exists system.test_insert(i Nullable(UInt64), s Nullable(String), t Nullable(DateTime))engine=Memory"); } } catch (SQLException e) { e.printStackTrace(); @@ -43,17 +62,56 @@ public void doSetup(ServerState serverState) throws Exception { @TearDown(Level.Trial) public void doTearDown(ServerState serverState) throws SQLException { - try (Statement s = conn.createStatement()) { - s.execute("drop table if exists system.test_insert"); + if (conn != null) { + conn.close(); + } + } + + @Setup(Level.Iteration) + public void prepare() { + if (!Constants.REUSE_CONNECTION.equalsIgnoreCase(connection)) { + try { + conn = driver.connect(url, new Properties()); + } catch (SQLException e) { + throw new IllegalStateException("Failed to create new connection", e); + } } - conn.close(); + + Random random = new Random(); + randomSample = random.nextInt(SAMPLE_SIZE); + randomNum = random.nextInt(FLOATING_RANGE); + } + + @TearDown(Level.Iteration) + public void shutdown() { + if (!Constants.REUSE_CONNECTION.equalsIgnoreCase(connection)) { + try { + conn.close(); + } catch (SQLException e) { + throw new IllegalStateException("Failed to close connection", e); + } finally { + conn = null; + } + } + } + + public int getSampleSize() { + return SAMPLE_SIZE; + } + + public int getRandomSample() { + return randomSample; + } + + public int getRandomNumber() { + return randomNum; } - public Connection getConnection() { - return this.conn; + public Connection getConnection() throws SQLException { + return conn; } public boolean usePreparedStatement() { - return Constants.PREPARED_STATEMENT.equals(this.statement); + return Constants.PREPARED_STATEMENT.equalsIgnoreCase(this.statement); } } diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Constants.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Constants.java index 27220a5ef..9b23e84b1 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Constants.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Constants.java @@ -4,8 +4,6 @@ * Constant interface. */ public interface Constants { - public static final String CLICKHOUSE_DRIVER = "clickhouse-jdbc"; - public static final String DEFAULT_HOST = "127.0.0.1"; public static final String DEFAULT_DB = "system"; public static final String DEFAULT_USER = "default"; @@ -13,9 +11,13 @@ public interface Constants { public static final int GRPC_PORT = 9100; public static final int HTTP_PORT = 8123; - public static final int MYSQL_PORT = 3307; + public static final int MYSQL_PORT = 9004; public static final int NATIVE_PORT = 9000; + public static final int POSTGRESQL_PORT = 9005; public static final String NORMAL_STATEMENT = "normal"; public static final String PREPARED_STATEMENT = "prepared"; + + public static final String REUSE_CONNECTION = "reuse"; + public static final String NEW_CONNECTION = "new"; } diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java index b6b2cfe17..6501680d4 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java @@ -1,23 +1,14 @@ package tech.clickhouse.benchmark; import java.sql.Timestamp; -// import java.util.Collections; import java.util.Enumeration; -import java.util.Random; import org.openjdk.jmh.annotations.Benchmark; public class Insertion extends JdbcBenchmark { - // @Benchmark - // public int insertOneNumber(ClientState state) throws Throwable { - // return executeInsert(state, "insert into test_insert(i) values(?)", - // Collections.enumeration(Collections.singletonList(new Object[] { new - // Random().nextInt(1000) }))); - // } - @Benchmark public int insert10kUInt64Rows(ClientState state) throws Throwable { - final int rows = 10000; - final int num = new Random().nextInt(rows); + final int range = state.getRandomNumber(); + final int rows = state.getSampleSize() + range; return executeInsert(state, "insert into system.test_insert(i) values(?)", new Enumeration() { int counter = 0; @@ -29,15 +20,15 @@ public boolean hasMoreElements() { @Override public Object[] nextElement() { - return new Object[] { num + (counter++) }; + return new Object[] { range + (counter++) }; } }); } @Benchmark public int insert10kStringRows(ClientState state) throws Throwable { - final int rows = 10000; - final int num = new Random().nextInt(rows); + final int range = state.getRandomNumber(); + final int rows = state.getSampleSize() + range; return executeInsert(state, "insert into system.test_insert(s) values(?)", new Enumeration() { int counter = 0; @@ -49,15 +40,15 @@ public boolean hasMoreElements() { @Override public Object[] nextElement() { - return new Object[] { String.valueOf(num + (counter++)) }; + return new Object[] { String.valueOf(range + (counter++)) }; } }); } @Benchmark public int insert10kTimestampRows(ClientState state) throws Throwable { - final int rows = 10000; - final int num = new Random().nextInt(rows); + final int range = state.getRandomNumber(); + final int rows = state.getSampleSize() + range; return executeInsert(state, "insert into system.test_insert(t) values(?)", new Enumeration() { int counter = 0; @@ -69,7 +60,7 @@ public boolean hasMoreElements() { @Override public Object[] nextElement() { - return new Object[] { new Timestamp((long) num + (counter++)) }; + return new Object[] { new Timestamp((long) range + (counter++)) }; } }); } diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcBenchmark.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcBenchmark.java index dac7ce228..39b1bc8f2 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcBenchmark.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcBenchmark.java @@ -29,9 +29,9 @@ @OutputTimeUnit(TimeUnit.SECONDS) public abstract class JdbcBenchmark { // batch size for mutation - private final int batchSize = Integer.parseInt(System.getProperty("batchSize", "1000")); + private final int BATCH_SIZE = Integer.parseInt(System.getProperty("batchSize", "5000")); // fetch size for query - private final int fetchSize = Integer.parseInt(System.getProperty("fetchSize", "1000")); + private final int FETCH_SIZE = Integer.parseInt(System.getProperty("fetchSize", "1000")); protected PreparedStatement setParameters(PreparedStatement s, Object... values) throws SQLException { if (values != null && values.length > 0) { @@ -73,16 +73,30 @@ private int processBatch(Statement s, String sql, Enumeration generato while (generator.hasMoreElements()) { Object[] values = generator.nextElement(); if (ps != null) { - setParameters(ps, values).addBatch(); + setParameters(ps, values); + + if (BATCH_SIZE > 0) { + ps.addBatch(); + } else { + ps.execute(); + rows++; + } } else { - s.addBatch(replaceParameters(sql, values)); + sql = replaceParameters(sql, values); + if (BATCH_SIZE > 0) { + s.addBatch(sql); + } else { + s.execute(sql); + rows++; + } } - if (++counter % batchSize == 0) { + + if (BATCH_SIZE > 0 && ++counter % BATCH_SIZE == 0) { rows += s.executeBatch().length; } } - if (counter % batchSize != 0) { + if (BATCH_SIZE > 0 && counter % BATCH_SIZE != 0) { rows += s.executeBatch().length; } @@ -114,17 +128,14 @@ protected Statement executeQuery(ClientState state, String sql, Object... values final Connection conn = state.getConnection(); if (state.usePreparedStatement()) { - try (PreparedStatement s = conn.prepareStatement(sql)) { - stmt = s; - s.setFetchSize(fetchSize); - setParameters(s, values).executeQuery(); - } + PreparedStatement s = conn.prepareStatement(sql); + stmt = s; + s.setFetchSize(FETCH_SIZE); + setParameters(s, values).executeQuery(); } else { - try (Statement s = conn.createStatement()) { - stmt = s; - stmt.setFetchSize(fetchSize); - stmt.executeQuery(replaceParameters(sql, values)); - } + stmt = conn.createStatement(); + stmt.setFetchSize(FETCH_SIZE); + stmt.executeQuery(replaceParameters(sql, values)); } return stmt; diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcDriver.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcDriver.java index 4e8111a4e..8656638d7 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcDriver.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcDriver.java @@ -24,7 +24,12 @@ public enum JdbcDriver { MysqlConnectorJava("com.mysql.cj.jdbc.Driver", "jdbc:mysql://%s:%s/%s?user=%s&password=%s&useSSL=false&useCompression=true&useServerPrepStmts=false" + "&rewriteBatchedStatements=true&cachePrepStmts=true&connectionTimeZone=UTC", - Constants.MYSQL_PORT); + Constants.MYSQL_PORT), + + // PostgreSQL JDBC Driver + PostgresqlJdbc("org.postgresql.Driver", + "jdbc:postgresql://%s:%s/%s?user=%s&password=%s&ssl=false&sslmode=disable&preferQueryMode=simple", + Constants.POSTGRESQL_PORT); private final String className; private final String urlTemplate; diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Query.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Query.java index 020fa3087..1480a142b 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Query.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Query.java @@ -2,22 +2,21 @@ import java.sql.ResultSet; import java.sql.Statement; -import java.sql.Timestamp; -import java.util.Random; import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.infra.Blackhole; public class Query extends JdbcBenchmark { @Benchmark - public int select10kUInt64Rows(ClientState state) throws Throwable { - int rows = 10000; - int num = new Random().nextInt(rows); - try (Statement stmt = executeQuery(state, "select * from system.numbers where number > ? limit " + rows, num)) { + public int select10kUInt64Rows(Blackhole blackhole, ClientState state) throws Throwable { + int num = state.getRandomNumber(); + int rows = state.getSampleSize() + num; + try (Statement stmt = executeQuery(state, "select * from system.numbers limit ?", rows)) { ResultSet rs = stmt.getResultSet(); - float avg = 0.0F; int count = 0; while (rs.next()) { - avg = (rs.getInt(1) + avg * count) / (++count); + blackhole.consume(rs.getInt(1)); + count++; } if (count != rows) { @@ -29,17 +28,15 @@ public int select10kUInt64Rows(ClientState state) throws Throwable { } @Benchmark - public int select10kStringRows(ClientState state) throws Throwable { - int rows = 10000; - int num = new Random().nextInt(rows); - try (Statement stmt = executeQuery(state, - "select toString(number) as s from system.numbers where number > ? limit " + rows, num)) { + public int select10kStringRows(Blackhole blackhole, ClientState state) throws Throwable { + int num = state.getRandomNumber(); + int rows = state.getSampleSize() + num; + try (Statement stmt = executeQuery(state, "select toString(number) as s from system.numbers limit ?", rows)) { ResultSet rs = stmt.getResultSet(); int count = 0; - String str = null; while (rs.next()) { - str = rs.getString(1); + blackhole.consume(rs.getString(1)); count++; } @@ -52,19 +49,16 @@ public int select10kStringRows(ClientState state) throws Throwable { } @Benchmark - public int select10kTimestampRows(ClientState state) throws Throwable { - int rows = 10000; - int num = new Random().nextInt(rows); + public int select10kTimestampRows(Blackhole blackhole, ClientState state) throws Throwable { + int num = state.getRandomNumber(); + int rows = state.getSampleSize() + num; try (Statement stmt = executeQuery(state, - "select toDateTime('2021-02-20 13:15:20') + number as d from system.numbers where number > ? limit " - + rows, - num)) { + "select toDateTime('2021-02-20 13:15:20') + number as d from system.numbers limit ?", rows)) { ResultSet rs = stmt.getResultSet(); int count = 0; - Timestamp ts = null; while (rs.next()) { - ts = rs.getTimestamp(1); + blackhole.consume(rs.getTimestamp(1)); count++; } diff --git a/pom.xml b/pom.xml index 194135348..6a2728a88 100644 --- a/pom.xml +++ b/pom.xml @@ -1,7 +1,5 @@ - + 4.0.0 tech.clickhouse @@ -74,7 +72,7 @@ - 0.3.1-SNAPSHOT + 0.3.2-SNAPSHOT 2021 UTF-8 UTF-8 @@ -82,14 +80,13 @@ 3.0.0 3.4.2 3.3.1 - 1.7.1 - 2.9.10 - 2.9.10.8 - 0.9.10 - 1.7.30 + 1.8.0 + 2.12.3 + 0.9.15 + 1.7.31 1.10.19 2.27.2 - 1.15.2 + 1.15.3 6.14.3 3.3.0 3.0.0-M1 @@ -113,7 +110,7 @@ ${project.artifactId} clickhouse https://sonarcloud.io - 0.3.1 + 0.3.2 @@ -121,12 +118,12 @@ com.fasterxml.jackson.core jackson-core - ${jackson-core.version} + ${jackson.version} com.fasterxml.jackson.core jackson-databind - ${jackson-databind.version} + ${jackson.version} com.github.ben-manes.caffeine @@ -251,6 +248,7 @@ ${failsafe-plugin.version} **/*.java + ${skipTests} ${skipITs} false @@ -271,6 +269,8 @@ maven-surefire-plugin ${surefire-plugin.version} + + ${skipUTs} false @@ -307,6 +307,10 @@ org.codehaus.mojo flatten-maven-plugin + + org.apache.maven.plugins + maven-failsafe-plugin +