diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 7920a2fc5..dc4341e55 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -6,7 +6,10 @@ on: - master - develop - 0.2.x - pull_request: + pull_request_target: + branches: + - master + - develop types: - opened - synchronize @@ -28,7 +31,7 @@ jobs: uses: actions/checkout@v2 with: ref: ${{ github.event.pull_request.head.sha }} - if: github.event_name == 'pull_request' + if: github.event_name == 'pull_request_target' - name: Check out PR run: | git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 \ @@ -57,7 +60,13 @@ jobs: reporter: 'github-pr-check' # added,diff_context,file,nofilter filter_mode: 'added' - if: github.event_name == 'pull_request' || github.event.inputs.pr != '' + if: github.event_name == 'pull_request_target' || github.event.inputs.pr != '' + continue-on-error: true + - name: Update sonar config + run: | + sed -i -e 's|^\(.*\).*\(\)$|\1ClickHouse_clickhouse-jdbc\2|' \ + -e 's|^\(.*\).*\(\)$|\1clickhouse-java\2|' pom.xml + if: github.repository_owner == 'ClickHouse' - name: Build and analyze env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -65,3 +74,4 @@ jobs: run: | find . -type f -name "log4j.*" -exec rm -fv '{}' \; mvn -q --batch-mode -Panalysis verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar + continue-on-error: true diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d45e36794..669052292 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,7 +1,10 @@ name: Benchmark on: - pull_request: + pull_request_target: + branches: + - master + - develop types: - opened - synchronize @@ -36,7 +39,7 @@ jobs: benchmark-pull-request: runs-on: ubuntu-latest name: Benchmark pull request - if: github.event_name == 'pull_request' + if: github.event_name == 'pull_request_target' steps: - name: Check out repository uses: actions/checkout@v2 @@ -69,13 +72,10 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - if (process.env.BENCHMARK_REPORT) { - github.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: '```\n' + process.env.BENCHMARK_REPORT + '\n```' - }); + const report = process.env.BENCHMARK_REPORT; + const { issue: { number: issue_number }, repo: { owner, repo } } = context; + if (report) { + github.issues.createComment({issue_number, owner, repo, body: '```\n' + report + '\n```'}); } benchmark-on-demand: @@ -108,8 +108,8 @@ jobs: continue-on-error: true - name: Build project run: | + mvn --batch-mode --update-snapshots -DskipTests -pl clickhouse-benchmark -am package cd clickhouse-benchmark - mvn --batch-mode --update-snapshots -DclickhouseVersion=${{ github.event.inputs.clickhouse }} install java -jar target/benchmarks.jar -rf json ${{ github.event.inputs.options }} > output.txt echo "BENCHMARK_REPORT<> $GITHUB_ENV tail -n +$(grep -n '^REMEMBER:' output.txt | tail -1 | awk -F: '{print $1+6}') output.txt | head -n -2 | grep -v ':·' >> $GITHUB_ENV diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f362b2ec8..32ff1dc46 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,9 +32,10 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java: [8, 11] + java: [8, 11, 15] # most recent LTS releases as well as latest stable builds - clickhouse: ["19.14", "20.3", "20.8", "20.10", "20.12", "21.2", "latest"] + clickhouse: ["20.8", "21.3", "latest"] + fail-fast: false name: Build using JDK ${{ matrix.java }} against ClickHouse ${{ matrix.clickhouse }} steps: - name: Check out Git repository @@ -56,25 +57,6 @@ jobs: key: ${{ runner.os }}-build-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-build- - - name: Generate build properties - uses: actions/github-script@v3 - id: props - env: - CURRENT_VERSION: ${{ steps.version.outputs.value }} - with: - script: | - const timezones = [ - 'Asia/Chongqing', 'America/Los_Angeles', 'Etc/UTC', 'Europe/Berlin', 'Europe/Moscow' - ]; - // surprise me - return { - clickhouse: timezones[Math.floor(Math.random() * Math.floor(timezones.length))] || '', - java: timezones[Math.floor(Math.random() * Math.floor(timezones.length))] || '' - }; - name: Build with Maven run: | - find . -type f -name "pom.xml" -exec sed -i -e 's|.*argLine.*timezone=.*||g' '{}' \; - mvn --batch-mode --update-snapshots \ - -DclickhouseVersion=${{ matrix.clickhouse }} \ - -DclickhouseTimezone=${{ fromJSON(steps.props.outputs.result).clickhouse }} \ - -Duser.timezone=${{ fromJSON(steps.props.outputs.result).java }} verify + mvn --batch-mode --update-snapshots -DclickhouseVersion=${{ matrix.clickhouse }} verify diff --git a/.github/workflows/timezone.yml b/.github/workflows/timezone.yml new file mode 100644 index 000000000..78fd5f1f5 --- /dev/null +++ b/.github/workflows/timezone.yml @@ -0,0 +1,62 @@ +name: TimeZone + +on: + push: + branches: + - master + - develop + paths-ignore: + - "**.md" + - "docs/**" + - "**/CHANGELOG" + + pull_request: + types: + - opened + - synchronize + - reopened + paths-ignore: + - "**.md" + - "docs/**" + - "**/CHANGELOG" + + workflow_dispatch: + inputs: + pr: + description: "Pull request#" + required: false + +jobs: + timezone: + runs-on: ubuntu-latest + strategy: + matrix: + serverTz: ["Asia/Chongqing", "America/Los_Angeles", "Etc/UTC", "Europe/Berlin", "Europe/Moscow"] + clientTz: ["Asia/Chongqing", "America/Los_Angeles", "Etc/UTC", "Europe/Berlin", "Europe/Moscow"] + fail-fast: false + name: "Test TimeZones - Server: ${{ matrix.serverTz }}, Client: ${{ matrix.clientTz }}" + steps: + - name: Check out Git repository + uses: actions/checkout@v2 + - name: Check out PR + run: | + git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 \ + origin pull/${{ github.event.inputs.pr }}/merge:merged-pr && git checkout merged-pr + if: github.event.inputs.pr != '' + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Cache maven dependencies + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-build-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-build- + - name: Test using Maven + run: | + find . -type f -name "pom.xml" -exec sed -i -e 's|.*argLine.*timezone=.*||g' '{}' \; + mvn --batch-mode --update-snapshots \ + -DclickhouseTimezone=${{ matrix.serverTz }} \ + -Duser.timezone=${{ matrix.clientTz }} verify diff --git a/.gitignore b/.gitignore index 285aa2d41..507b39f58 100644 --- a/.gitignore +++ b/.gitignore @@ -29,10 +29,10 @@ log/ target/ # Generated files -src/main/java/ru/yandex/clickhouse/jdbc/parser/*CharStream.java -src/main/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlParser*.java -src/main/java/ru/yandex/clickhouse/jdbc/parser/Token*.java -src/main/java/ru/yandex/clickhouse/jdbc/parser/ParseException.java +**/parser/*CharStream.java +**/parser/ClickHouseSqlParser*.java +**/parser/Token*.java +**/parser/ParseException.java # Shell scripts *.sh diff --git a/CHANGELOG b/CHANGELOG index f6ffc6e73..8c4ff5840 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,16 @@ +0.3.0 + * BREAKING CHANGE - dropped JDK 7 support + * BREAKING CHANGE - removed Guava dependency(and so is UnsignedLong) + * JDBC 4.2 support + * add connection setting client_name for load-balancing and troubleshooting + * add writeBytes & writeUUIDArray and remove UnsignedLong related methods in ClickHouseRowBinaryStream + * support more data types: IPv4, IPv6, Int128, UInt128, Int256, UInt256, Decimal256, DateTime*, and Map + * support ORC/Parquet streaming + * support read/write Bitmap from/into AggregateFunction(groupBitmap, UInt[8-64]) column + * throw SQLException instead of RuntimeException when instantiating ClickHouseConnectionImpl + * fix error when using ClickHouseCompression.none against 19.16 + * fix NegativeArraySizeException when dealing with large array + * fix datetime/date display issue caused by timezone differences(between client and column/server) 0.2.6 * add new feature for sending compressed files/streams * introduce an experimental SQL parser to fix parsing related issues - set connection setting use_new_parser to false to disable diff --git a/README.md b/README.md index ff31548bc..7bc8c2536 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ClickHouse JDBC driver =============== -[![clickhouse-jdbc](https://maven-badges.herokuapp.com/maven-central/ru.yandex.clickhouse/clickhouse-jdbc/badge.svg)](https://maven-badges.herokuapp.com/maven-central/ru.yandex.clickhouse/clickhouse-jdbc) ![Build Status(https://github.com/ClickHouse/clickhouse-jdbc/workflows/Build/badge.svg)](https://github.com/ClickHouse/clickhouse-jdbc/workflows/Build/badge.svg) +[![clickhouse-jdbc](https://maven-badges.herokuapp.com/maven-central/ru.yandex.clickhouse/clickhouse-jdbc/badge.svg)](https://maven-badges.herokuapp.com/maven-central/ru.yandex.clickhouse/clickhouse-jdbc) ![Build Status(https://github.com/ClickHouse/clickhouse-jdbc/workflows/Build/badge.svg)](https://github.com/ClickHouse/clickhouse-jdbc/workflows/Build/badge.svg) [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=ClickHouse_clickhouse-jdbc&metric=coverage)](https://sonarcloud.io/dashboard?id=ClickHouse_clickhouse-jdbc) This is a basic and restricted implementation of jdbc driver for ClickHouse. It has support of a minimal subset of features to be usable. @@ -10,7 +10,7 @@ It has support of a minimal subset of features to be usable. ru.yandex.clickhouse clickhouse-jdbc - 0.2.6 + 0.3.0 ``` diff --git a/clickhouse-benchmark/pom.xml b/clickhouse-benchmark/pom.xml index c967c9eb6..c10861b59 100644 --- a/clickhouse-benchmark/pom.xml +++ b/clickhouse-benchmark/pom.xml @@ -2,24 +2,28 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - tech.clickhouse + + tech.clickhouse + clickhouse-java + ${revision} + + + ${parent.groupId} clickhouse-benchmark ${revision} jar clickhouse-benchmark + Benchmarks for ClickHouse clients - 0.3.0-SNAPSHOT 1.4.4 - 2.5.3 2.7.2 8.0.23 - 2.5.3 + 2.5.4 1.15.2 UTF-8 1.27 - 1.8 benchmarks @@ -96,7 +100,6 @@ org.testcontainers testcontainers - ${testcontainers.version} @@ -105,17 +108,15 @@ org.apache.maven.plugins maven-compiler-plugin - 3.8.0 - ${javac.target} - ${javac.target} - ${javac.target} + ${jdk.version} + ${jdk.version} + ${jdk.version} org.apache.maven.plugins maven-shade-plugin - 3.2.1 package diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Basic.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Basic.java index da543ee77..079001636 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Basic.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Basic.java @@ -3,13 +3,13 @@ import java.sql.ResultSet; import java.sql.Statement; import java.util.Collections; - +import java.util.Random; import org.openjdk.jmh.annotations.Benchmark; public class Basic extends JdbcBenchmark { @Benchmark public int selectOneRandomNumber(ClientState state) throws Throwable { - final int num = (int) (Math.random() * 1000); + final int num = new Random().nextInt(1000); try (Statement stmt = executeQuery(state, "select ? as n", num)) { ResultSet rs = stmt.getResultSet(); @@ -26,7 +26,7 @@ public int selectOneRandomNumber(ClientState state) throws Throwable { @Benchmark public int insertOneRandomNumber(ClientState state) throws Throwable { - final int num = (int) (Math.random() * 1000); + final int num = new Random().nextInt(1000); return executeInsert(state, "insert into test_insert(i) values(?)", Collections.enumeration(Collections.singletonList(new Object[] { num }))); diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java index cc22324e7..62d44bf6b 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java @@ -4,7 +4,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Properties; - import org.openjdk.jmh.annotations.Level; import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Constants.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Constants.java index e2873579b..27220a5ef 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Constants.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Constants.java @@ -1,5 +1,8 @@ package tech.clickhouse.benchmark; +/** + * Constant interface. + */ public interface Constants { public static final String CLICKHOUSE_DRIVER = "clickhouse-jdbc"; diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java index 15fa43f49..44eea7efe 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java @@ -3,21 +3,21 @@ import java.sql.Timestamp; // import java.util.Collections; import java.util.Enumeration; - +import java.util.Random; import org.openjdk.jmh.annotations.Benchmark; public class Insertion extends JdbcBenchmark { // @Benchmark // public int insertOneNumber(ClientState state) throws Throwable { // return executeInsert(state, "insert into test_insert(i) values(?)", - // Collections.enumeration(Collections.singletonList(new Object[] { (int) - // (Math.random() * 1000) }))); + // Collections.enumeration(Collections.singletonList(new Object[] { new + // Random().nextInt(1000) }))); // } @Benchmark public int insert10kUInt64Rows(ClientState state) throws Throwable { final int rows = 10000; - final int num = (int) (Math.random() * rows); + final int num = new Random().nextInt(rows); return executeInsert(state, "insert into test_insert(i) values(?)", new Enumeration() { int counter = 0; @@ -37,7 +37,7 @@ public Object[] nextElement() { @Benchmark public int insert10kStringRows(ClientState state) throws Throwable { final int rows = 10000; - final int num = (int) (Math.random() * rows); + final int num = new Random().nextInt(rows); return executeInsert(state, "insert into test_insert(s) values(?)", new Enumeration() { int counter = 0; @@ -57,7 +57,7 @@ public Object[] nextElement() { @Benchmark public int insert10kTimestampRows(ClientState state) throws Throwable { final int rows = 10000; - final int num = (int) (Math.random() * rows); + final int num = new Random().nextInt(rows); return executeInsert(state, "insert into test_insert(t) values(?)", new Enumeration() { int counter = 0; @@ -69,7 +69,7 @@ public boolean hasMoreElements() { @Override public Object[] nextElement() { - return new Object[] { new Timestamp(num + (counter++)) }; + return new Object[] { new Timestamp((long) num + (counter++)) }; } }); } diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcBenchmark.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcBenchmark.java index e1e4afdc3..dac7ce228 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcBenchmark.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcBenchmark.java @@ -1,7 +1,5 @@ package tech.clickhouse.benchmark; -import org.openjdk.jmh.annotations.*; - import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -9,7 +7,19 @@ import java.util.Enumeration; import java.util.Objects; import java.util.concurrent.TimeUnit; - +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +/** + * Base class for JDBC driver benchmarking. + */ @State(Scope.Benchmark) @Warmup(iterations = 10, timeUnit = TimeUnit.SECONDS, time = 1) @Measurement(iterations = 10, timeUnit = TimeUnit.SECONDS, time = 1) @@ -104,14 +114,17 @@ protected Statement executeQuery(ClientState state, String sql, Object... values final Connection conn = state.getConnection(); if (state.usePreparedStatement()) { - PreparedStatement s = conn.prepareStatement(sql); - s.setFetchSize(fetchSize); - setParameters(s, values).executeQuery(); - stmt = s; + try (PreparedStatement s = conn.prepareStatement(sql)) { + stmt = s; + s.setFetchSize(fetchSize); + setParameters(s, values).executeQuery(); + } } else { - stmt = conn.createStatement(); - stmt.setFetchSize(fetchSize); - stmt.executeQuery(replaceParameters(sql, values)); + try (Statement s = conn.createStatement()) { + stmt = s; + stmt.setFetchSize(fetchSize); + stmt.executeQuery(replaceParameters(sql, values)); + } } return stmt; diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcDriver.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcDriver.java index 5476eb9cd..4e8111a4e 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcDriver.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcDriver.java @@ -16,12 +16,14 @@ public enum JdbcDriver { // MariaDB Java Client MariadbJavaClient("org.mariadb.jdbc.Driver", - "jdbc:mariadb://%s:%s/%s?user=%s&password=%s&useSSL=false&useCompression=true&useServerPrepStmts=false&rewriteBatchedStatements=true&cachePrepStmts=true&serverTimezone=UTC", + "jdbc:mariadb://%s:%s/%s?user=%s&password=%s&useSSL=false&useCompression=true&useServerPrepStmts=false" + + "&rewriteBatchedStatements=true&cachePrepStmts=true&serverTimezone=UTC", Constants.MYSQL_PORT), // MySQL Connector/J MysqlConnectorJava("com.mysql.cj.jdbc.Driver", - "jdbc:mysql://%s:%s/%s?user=%s&password=%s&useSSL=false&useCompression=true&useServerPrepStmts=false&rewriteBatchedStatements=true&cachePrepStmts=true&connectionTimeZone=UTC", + "jdbc:mysql://%s:%s/%s?user=%s&password=%s&useSSL=false&useCompression=true&useServerPrepStmts=false" + + "&rewriteBatchedStatements=true&cachePrepStmts=true&connectionTimeZone=UTC", Constants.MYSQL_PORT); private final String className; diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Query.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Query.java index 174b2943e..020fa3087 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Query.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Query.java @@ -3,14 +3,14 @@ import java.sql.ResultSet; import java.sql.Statement; import java.sql.Timestamp; - +import java.util.Random; import org.openjdk.jmh.annotations.Benchmark; public class Query extends JdbcBenchmark { @Benchmark public int select10kUInt64Rows(ClientState state) throws Throwable { int rows = 10000; - int num = (int) (Math.random() * rows); + int num = new Random().nextInt(rows); try (Statement stmt = executeQuery(state, "select * from system.numbers where number > ? limit " + rows, num)) { ResultSet rs = stmt.getResultSet(); @@ -31,7 +31,7 @@ public int select10kUInt64Rows(ClientState state) throws Throwable { @Benchmark public int select10kStringRows(ClientState state) throws Throwable { int rows = 10000; - int num = (int) (Math.random() * rows); + int num = new Random().nextInt(rows); try (Statement stmt = executeQuery(state, "select toString(number) as s from system.numbers where number > ? limit " + rows, num)) { ResultSet rs = stmt.getResultSet(); @@ -54,7 +54,7 @@ public int select10kStringRows(ClientState state) throws Throwable { @Benchmark public int select10kTimestampRows(ClientState state) throws Throwable { int rows = 10000; - int num = (int) (Math.random() * rows); + int num = new Random().nextInt(rows); try (Statement stmt = executeQuery(state, "select toDateTime('2021-02-20 13:15:20') + number as d from system.numbers where number > ? limit " + rows, diff --git a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ServerState.java b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ServerState.java index 71c2ab9c4..6e1436db9 100644 --- a/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ServerState.java +++ b/clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ServerState.java @@ -1,12 +1,13 @@ package tech.clickhouse.benchmark; +import static java.time.temporal.ChronoUnit.SECONDS; + import java.net.Inet4Address; import java.net.InetAddress; import java.net.InterfaceAddress; import java.net.NetworkInterface; import java.time.Duration; import java.util.Enumeration; - import org.openjdk.jmh.annotations.Level; import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; @@ -16,8 +17,6 @@ import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.images.builder.ImageFromDockerfile; -import static java.time.temporal.ChronoUnit.SECONDS; - @State(Scope.Benchmark) public class ServerState { static String getLocalIpAddress() { diff --git a/clickhouse-client/pom.xml b/clickhouse-client/pom.xml new file mode 100644 index 000000000..32b882d42 --- /dev/null +++ b/clickhouse-client/pom.xml @@ -0,0 +1,40 @@ + + 4.0.0 + + + tech.clickhouse + clickhouse-java + ${revision} + + + ${parent.groupId} + clickhouse-client + ${revision} + jar + + + + org.slf4j + slf4j-api + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${jdk.version} + ${jdk.version} + true + + -Xlint:all + -Werror + + + + + + diff --git a/clickhouse-grpc-client/pom.xml b/clickhouse-grpc-client/pom.xml new file mode 100644 index 000000000..41cb9243f --- /dev/null +++ b/clickhouse-grpc-client/pom.xml @@ -0,0 +1,41 @@ + + 4.0.0 + + + tech.clickhouse + clickhouse-java + ${revision} + + + ${parent.groupId} + clickhouse-grpc-client + ${revision} + jar + + + + ${parent.groupId} + clickhouse-client + ${revision} + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${jdk.version} + ${jdk.version} + true + + -Xlint:all + -Werror + + + + + + diff --git a/clickhouse-http-client/pom.xml b/clickhouse-http-client/pom.xml new file mode 100644 index 000000000..a89830997 --- /dev/null +++ b/clickhouse-http-client/pom.xml @@ -0,0 +1,41 @@ + + 4.0.0 + + + tech.clickhouse + clickhouse-java + ${revision} + + + ${parent.groupId} + clickhouse-http-client + ${revision} + jar + + + + ${parent.groupId} + clickhouse-client + ${revision} + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${jdk.version} + ${jdk.version} + true + + -Xlint:all + -Werror + + + + + + diff --git a/AUTHORS b/clickhouse-jdbc/AUTHORS similarity index 100% rename from AUTHORS rename to clickhouse-jdbc/AUTHORS diff --git a/clickhouse-jdbc/docs/datetime.md b/clickhouse-jdbc/docs/datetime.md new file mode 100644 index 000000000..cbb058c4e --- /dev/null +++ b/clickhouse-jdbc/docs/datetime.md @@ -0,0 +1,69 @@ +# ClickHouse Server + +ClickHouse has dedicated data types for [Date](https://clickhouse.tech/docs/en/data_types/date) and [DateTime](https://clickhouse.tech/docs/en/data_types/datetime). A DateTime value is usually to be interpreted in the server time zone, but a DateTime value may also be formatted for a different time zone. Note that the explicit time zone per column only affects inserting and displaying values, _not_ the predicates (see one example [here](https://github.com/ClickHouse/ClickHouse/issues/5206)). + + +# Setting Values + +When setting values via the PreparedStatement setter methods, the JDBC driver does not have any knowledge about the target columns. Its job is to serialize any date time values into a textual representation. One of the aims of this driver is to use a serialization format that makes it easy to use ClickHouse Date or DateTime fields from a Java application. + +In some cases the driver cannot perform the serialization without referring to a relevant time zone. This is how it works: + +If the client supplies a valid `Calendar` object as optional argument, the driver will use the time zone contained therein (`tz_calendar`). + +Two regular time zones are initialized like this: + +* `tz_datetime`: value from `ru.yandex.clickhouse.settings.ClickHouseConnectionSettings.USE_TIME_ZONE`. If null, either ClickHouse server time zone (`ru.yandex.clickhouse.settings.ClickHouseConnectionSettings.USE_SERVER_TIME_ZONE` is `true`) or JVM time zone (else) + +* `tz_date`: same as `tz_datetime` if `ru.yandex.clickhouse.settings.ClickHouseConnectionSettings.USE_SERVER_TIME_ZONE_FOR_DATES` is `true`, JVM time zone else + +The JDBC driver supports all explicit methods, e.g. setDate, setTimestamp etc. with their optional Calendar argument. Providing hints via target SQL type does not have any effect. + +The following table illustrates the serialization format for some popular date time data types, which we consider the most convenient. Clients are of course free to take care of serialization themselves by supplying a String or an Integer parameter, optionally using one of the server's utility methods (e.g. [parseDateTimeBestEffort](https://clickhouse.tech/docs/en/query_language/functions/type_conversion_functions/#type_conversion_functions-parsedatetimebesteffort)). + + Method | Format | Relevant time zone | + ------ | ------ | ------------------- +[setDate(int, Date)](https://docs.oracle.com/javase/8/docs/api/java/sql/PreparedStatement.html#setDate-int-java.sql.Date-) | yyyy-MM-dd | tz_date +[setDate(int, Date, Calendar)](https://docs.oracle.com/javase/8/docs/api/java/sql/PreparedStatement.html#setDate-int-java.sql.Date-java.util.Calendar-) | yyyy-MM-dd | tz_calendar +[setObject(int, Date)](https://docs.oracle.com/javase/8/docs/api/java/sql/PreparedStatement.html#setObject-int-java.lang.Object-) | yyyy-MM-dd | tz_date +[setTime(int, Time)](https://docs.oracle.com/javase/8/docs/api/java/sql/PreparedStatement.html#setTime-int-java.sql.Time-) | HH:mm:ss | tz_datetime +[setTime(int, Time, Calendar)](https://docs.oracle.com/javase/8/docs/api/java/sql/PreparedStatement.html#setTime-int-java.sql.Time-java.util.Calendar-) | HH:mm:ss | tz_calendar +[setObject(int, Time)](https://docs.oracle.com/javase/8/docs/api/java/sql/PreparedStatement.html#setObject-int-java.lang.Object-) | HH:mm:ss | tz_datetime +[setTimestamp(int, Timestamp)](https://docs.oracle.com/javase/8/docs/api/java/sql/PreparedStatement.html#setTimestamp-int-java.sql.Timestamp-) | yyyy-MM-dd HH:mm:ss | tz_datetime +[setTimestamp(int, Timestamp, Calendar)](https://docs.oracle.com/javase/8/docs/api/java/sql/PreparedStatement.html#setTimestamp-int-java.sql.Timestamp-java.util.Calendar-) | yyyy-MM-dd HH:mm:ss | tz_calendar +[setObject(int, Timestamp)](https://docs.oracle.com/javase/8/docs/api/java/sql/PreparedStatement.html#setObject-int-java.lang.Object-) | yyyy-MM-dd HH:mm:ss | tz_datetime +[setObject(int, LocalTime)](https://docs.oracle.com/javase/8/docs/api/java/sql/PreparedStatement.html#setObject-int-java.lang.Object-) | HH:mm:ss | _none_ +[setObject(int, OffsetTime)](https://docs.oracle.com/javase/8/docs/api/java/sql/PreparedStatement.html#setObject-int-java.lang.Object-) | HH:mm:ssZZZZ | _none_ +[setObject(int, LocalDate)](https://docs.oracle.com/javase/8/docs/api/java/sql/PreparedStatement.html#setObject-int-java.lang.Object-) | yyyy-MM-dd | _none_ +[setObject(int, LocalDateTime)](https://docs.oracle.com/javase/8/docs/api/java/sql/PreparedStatement.html#setObject-int-java.lang.Object-) | yyyy-MM-dd HH:mm:ss | _none_ +[setObject(int, OffsetDateTime)](https://docs.oracle.com/javase/8/docs/api/java/sql/PreparedStatement.html#setObject-int-java.lang.Object-) | yyyy-MM-dd HH:mm:ss | tz_datetime +[setObject(int, ZonedDateTime)](https://docs.oracle.com/javase/8/docs/api/java/sql/PreparedStatement.html#setObject-int-java.lang.Object-) | yyyy-MM-dd HH:mm:ss | tz_datetime +[setObject(int, Instant)](https://docs.oracle.com/javase/8/docs/api/java/sql/PreparedStatement.html#setObject-int-java.lang.Object-) | yyyy-MM-dd HH:mm:ss | tz_datetime + +# Retrieving Values + +When retrieving values via the ResultSet's getter methods, the JDBC driver will try to accomodate for some obvious options. If the underlying data field is of type Date or DateTime, the driver knows the implied time zone. This helps during the interpretation of the values retrieved from the server. Users may configure the driver to use a different time zone when reporting results back to the client (via `tz_date` or`tz_datetime`, see above). + +The methods which take a [Calendar]((https://docs.oracle.com/javase/8/docs/api/java.base/java/util/Calendar.html) argument behave the same as the corresponding methods without such an argument. The API documentation says something like + +> This method uses the given calendar to construct an appropriate millisecond value for the _x_ if the underlying database does not store timezone information. + +For Date and DateTime fields, the JDBC driver has enough time zone related information available, so these methods would only be relevant for String or other typed fields. There might be valid use cases, but for now we think that adding such an option would make things even more complicated. + +Requested Type | Number | Date | DateTime | Other +---------------| ---------------------------------- +[Date](https://docs.oracle.com/javase/8/docs/api/java/sql/Date.html) | Seconds or milliseconds past epoch truncated to day in relevant time zone | Date in relevant time zone, midnight | Date time in relevant time zone, rewind to midnight | Try number, date time (with or without offset) truncated to day, date +[Time](https://docs.oracle.com/javase/8/docs/api/java/sql/Time.html) | Local time at 1970-01-01 (e.g. “1337” is “13:37:00” at TZ) | Midnight on 1970-01-01 in relevant time zone | Local time in relevant time zone | Local time in relevant time zone via ISO format or via number, at 1970-01-01 +[Timestamp](https://docs.oracle.com/javase/8/docs/api/java/sql/Timestamp.html) | Seconds or milliseconds past epoch | Local date at midnight in relevant time zone | Local date and time in relevant time zone | Number, date time with or without offset +[LocalTime](https://docs.oracle.com/javase/8/docs/api/java/time/LocalTime.html) | Local time (e.g. "430" is "04:30:00") | Midnight | Local time | ISO format with or without offset, number +[OffsetTime](https://docs.oracle.com/javase/8/docs/api/java/time/OffsetTime.html) | Local time with current (!) offset of relevant time zone | Midnight with offset of relevant time zone on that date | Local time with offset of relevant time zone at value's date | ISO format, number +[LocalDate](https://docs.oracle.com/javase/8/docs/api/java/time/LocalDate.html) | Seconds or milliseconds past epoch as local date in relevant time zone | Local date | Local date (no conversion) | Local date, local time, number +[LocalDateTime](https://docs.oracle.com/javase/8/docs/api/java/time/LocalDateTime.html) | Seconds or milliseconds past epoch as local date time in relevant time zone | Local date midnight | Local date time | Local date time, number +[OffsetDateTime](https://docs.oracle.com/javase/8/docs/api/java/time/OffsetDateTime.html) | Seconds or milliseconds past epoch, offset from relevant time zone | Date midnight in relevant time zone | Date time in relevant time zone | Local date time in relevant time zone, ISO formats, number +[ZonedDdateTime](https://docs.oracle.com/javase/8/docs/api/java/time/ZonedDateTime.html) | Seconds or milliseconds past epoch, offset from relevant time zone | Date midnight in relevant time zone | Date time in relevant time zone | Local date, local date time in relevant time zone, ISO formats, number + +# Summary + +Life as a developer would be boring without time zones: [xkcd Super Villain Plan](https://xkcd.com/1883) Have fun! + +If you think the ClickHouse JDBC driver behaves wrong, please file an issue. Make sure to include some time zone information of your ClickHouse server, the JVM, and the relevant driver settings. \ No newline at end of file diff --git a/clickhouse-jdbc/pom.xml b/clickhouse-jdbc/pom.xml new file mode 100644 index 000000000..b57b275b0 --- /dev/null +++ b/clickhouse-jdbc/pom.xml @@ -0,0 +1,259 @@ + + 4.0.0 + + + tech.clickhouse + clickhouse-java + ${revision} + + + ru.yandex.clickhouse + clickhouse-jdbc + ${revision} + jar + + + + serebrserg + Sergey Serebryanik + serebrserg@yandex-team.ru + + + jkee + Viktor Tarnavsky + jkee@yandex-team.ru + + + orantius + Yuriy Galitskiy + orantius@yandex-team.ru + + + krash + Alexandr Krasheninnikov + krash3@gmail.com + + + zhicwu + Zhichun Wu + zhicwu@gmail.com + + + + + 4.5.13 + 4.1.4 + ru.yandex.clickhouse.jdbc.internal + JDBC + 4.2 + + + + + org.apache.httpcomponents + httpclient + ${httpclient.version} + + + org.apache.httpcomponents + httpmime + ${httpclient.version} + + + org.lz4 + lz4-java + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + com.github.RoaringBitmap + RoaringBitmap + + + org.slf4j + slf4j-api + + + + org.slf4j + slf4j-log4j12 + test + + + org.mockito + mockito-all + test + + + com.github.tomakehurst + wiremock-jre8 + test + + + org.testcontainers + testcontainers + test + + + org.testng + testng + test + + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + com.helger.maven + ph-javacc-maven-plugin + ${javacc-plugin.version} + + + jjc + generate-sources + + javacc + + + ${jdk.version} + true + ru.yandex.clickhouse.jdbc.parser + src/main/javacc + src/main/java + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${jdk.version} + ${jdk.version} + true + + + + org.apache.maven.plugins + maven-jar-plugin + + + + true + true + true + + + ${spec.title} + ${spec.version} + ${project.groupId} + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + jar-with-dependencies + + + + true + true + + + + + + org.apache.maven.plugins + maven-shade-plugin + + true + true + true + shaded + + + com.fasterxml.jackson + ${shade.base}.jackson + + + org.apache + ${shade.base}.apache + + + net.jpountz + ${shade.base}.jpountz + + + org.roaringbitmap + ${shade.base}.bitmap + + + org.slf4j + ${shade.base}.slf4j + + + + + + + + + + ${spec.title} + ${spec.version} + ${project.name} + ${project.version} + ${project.groupId} + + + + + + *:* + + mozilla/** + **/darwin/** + **/linux/** + **/win32/** + META-INF/maven/** + META-INF/native-image/** + META-INF/versions/** + META-INF/*.xml + + + + + + + package + + shade + + + false + + + + + + + diff --git a/src/main/java/ru/yandex/clickhouse/BalancedClickhouseDataSource.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/BalancedClickhouseDataSource.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/BalancedClickhouseDataSource.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/BalancedClickhouseDataSource.java diff --git a/src/main/java/ru/yandex/clickhouse/BodyEntityWrapper.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/BodyEntityWrapper.java similarity index 87% rename from src/main/java/ru/yandex/clickhouse/BodyEntityWrapper.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/BodyEntityWrapper.java index e9bb19a1c..0b7cb8861 100644 --- a/src/main/java/ru/yandex/clickhouse/BodyEntityWrapper.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/BodyEntityWrapper.java @@ -3,13 +3,12 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import org.apache.http.HttpEntity; import org.apache.http.entity.AbstractHttpEntity; import org.apache.http.entity.StringEntity; -import ru.yandex.clickhouse.util.guava.StreamUtils; - /** * Allow to inject sql query in the body, followed by row data */ @@ -18,7 +17,7 @@ public class BodyEntityWrapper extends AbstractHttpEntity { private final HttpEntity delegate; public BodyEntityWrapper(String sql, HttpEntity content) { - this.sql = new StringEntity(sql+"\n", StreamUtils.UTF_8); + this.sql = new StringEntity(sql+"\n", StandardCharsets.UTF_8); this.delegate = content; } diff --git a/src/main/java/ru/yandex/clickhouse/ClickHouseArray.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseArray.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/ClickHouseArray.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseArray.java diff --git a/src/main/java/ru/yandex/clickhouse/ClickHouseConnection.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnection.java similarity index 93% rename from src/main/java/ru/yandex/clickhouse/ClickHouseConnection.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnection.java index 7a3302421..7454d1a09 100644 --- a/src/main/java/ru/yandex/clickhouse/ClickHouseConnection.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnection.java @@ -10,6 +10,8 @@ public interface ClickHouseConnection extends Connection { @Deprecated ClickHouseStatement createClickHouseStatement() throws SQLException; + TimeZone getServerTimeZone(); + TimeZone getTimeZone(); @Override diff --git a/src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java similarity index 91% rename from src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java index e767bd445..4c557511a 100644 --- a/src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java @@ -31,16 +31,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Strings; - import ru.yandex.clickhouse.domain.ClickHouseDataType; import ru.yandex.clickhouse.except.ClickHouseUnknownException; import ru.yandex.clickhouse.settings.ClickHouseConnectionSettings; import ru.yandex.clickhouse.settings.ClickHouseProperties; import ru.yandex.clickhouse.util.ClickHouseHttpClientBuilder; import ru.yandex.clickhouse.util.LogProxy; -import ru.yandex.clickhouse.util.guava.StreamUtils; - +import ru.yandex.clickhouse.util.Utils; public class ClickHouseConnectionImpl implements ClickHouseConnection { @@ -56,14 +53,15 @@ public class ClickHouseConnectionImpl implements ClickHouseConnection { private boolean closed = false; + private TimeZone serverTimeZone; private TimeZone timezone; - private volatile String serverVersion; + private String serverVersion; - public ClickHouseConnectionImpl(String url) { + public ClickHouseConnectionImpl(String url) throws SQLException { this(url, new ClickHouseProperties()); } - public ClickHouseConnectionImpl(String url, ClickHouseProperties properties) { + public ClickHouseConnectionImpl(String url, ClickHouseProperties properties) throws SQLException { this.url = url; try { this.properties = ClickhouseJdbcUrlParser.parse(url, properties.asProperties()); @@ -77,31 +75,35 @@ public ClickHouseConnectionImpl(String url, ClickHouseProperties properties) { }catch (Exception e) { throw new IllegalStateException("cannot initialize http client", e); } - initTimeZone(this.properties); + initConnection(this.properties); } - private void initTimeZone(ClickHouseProperties properties) { - if (properties.isUseServerTimeZone() && !Strings.isNullOrEmpty(properties.getUseTimeZone())) { + private void initConnection(ClickHouseProperties properties) throws SQLException { + // timezone + if (properties.isUseServerTimeZone() && !Utils.isNullOrEmptyString(properties.getUseTimeZone())) { throw new IllegalArgumentException(String.format("only one of %s or %s must be enabled", ClickHouseConnectionSettings.USE_SERVER_TIME_ZONE.getKey(), ClickHouseConnectionSettings.USE_TIME_ZONE.getKey())); } - if (!properties.isUseServerTimeZone() && Strings.isNullOrEmpty(properties.getUseTimeZone())) { + if (!properties.isUseServerTimeZone() && Utils.isNullOrEmptyString(properties.getUseTimeZone())) { throw new IllegalArgumentException(String.format("one of %s or %s must be enabled", ClickHouseConnectionSettings.USE_SERVER_TIME_ZONE.getKey(), ClickHouseConnectionSettings.USE_TIME_ZONE.getKey())); } - if (properties.isUseServerTimeZone()) { - ResultSet rs = null; - try { - timezone = TimeZone.getTimeZone("UTC"); // just for next query - rs = createStatement().executeQuery("select timezone()"); - rs.next(); - String timeZoneName = rs.getString(1); - timezone = TimeZone.getTimeZone(timeZoneName); - } catch (SQLException e) { - throw new RuntimeException(e); - } finally { - StreamUtils.close(rs); + + serverTimeZone = TimeZone.getTimeZone("UTC"); // just for next query + try (Statement s = createStatement(); ResultSet rs = s.executeQuery("select timezone(), version()")) { + if (rs.next()) { + serverTimeZone = TimeZone.getTimeZone(rs.getString(1)); + serverVersion = rs.getString(2); } - } else if (!Strings.isNullOrEmpty(properties.getUseTimeZone())) { - timezone = TimeZone.getTimeZone(properties.getUseTimeZone()); + } + + timezone = serverTimeZone; + if (!properties.isUseServerTimeZone()) { + timezone = Utils.isNullOrEmptyString(properties.getUseTimeZone()) + ? TimeZone.getDefault() + : TimeZone.getTimeZone(properties.getUseTimeZone()); + } + + if (serverVersion == null) { + serverVersion = ""; } } @@ -131,6 +133,11 @@ public TimeZone getTimeZone() { return timezone; } + @Override + public TimeZone getServerTimeZone() { + return serverTimeZone; + } + private ClickHouseStatement createClickHouseStatement(CloseableHttpClient httpClient) throws SQLException { return LogProxy.wrap( ClickHouseStatement.class, @@ -178,12 +185,6 @@ public ClickHouseStatement createStatement(int resultSetType, int resultSetConcu */ @Override public String getServerVersion() throws SQLException { - if (serverVersion == null) { - ResultSet rs = createStatement().executeQuery("select version()"); - rs.next(); - serverVersion = rs.getString(1); - rs.close(); - } return serverVersion; } diff --git a/src/main/java/ru/yandex/clickhouse/ClickHouseDataSource.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseDataSource.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/ClickHouseDataSource.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseDataSource.java diff --git a/src/main/java/ru/yandex/clickhouse/ClickHouseDatabaseMetadata.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseDatabaseMetadata.java similarity index 99% rename from src/main/java/ru/yandex/clickhouse/ClickHouseDatabaseMetadata.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseDatabaseMetadata.java index a89569338..37e0f7795 100644 --- a/src/main/java/ru/yandex/clickhouse/ClickHouseDatabaseMetadata.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseDatabaseMetadata.java @@ -837,7 +837,8 @@ public ResultSet getColumns(String catalog, String schemaPattern, String tableNa //column name ClickHouseColumnInfo columnInfo = ClickHouseColumnInfo.parse( descTable.getString("type"), - descTable.getString("name")); + descTable.getString("name"), + connection.getServerTimeZone()); row.add(columnInfo.getColumnName()); //data type row.add(String.valueOf(columnInfo.getClickHouseDataType().getSqlType())); diff --git a/src/main/java/ru/yandex/clickhouse/ClickHouseDriver.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseDriver.java similarity index 92% rename from src/main/java/ru/yandex/clickhouse/ClickHouseDriver.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseDriver.java index 2713984cf..a7673cafe 100644 --- a/src/main/java/ru/yandex/clickhouse/ClickHouseDriver.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseDriver.java @@ -1,6 +1,5 @@ package ru.yandex.clickhouse; -import com.google.common.collect.MapMaker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -9,10 +8,18 @@ import ru.yandex.clickhouse.settings.ClickHouseQueryParam; import ru.yandex.clickhouse.settings.DriverPropertyCreator; import ru.yandex.clickhouse.util.LogProxy; -import java.sql.*; + +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.DriverPropertyInfo; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Properties; +import java.util.WeakHashMap; import java.util.concurrent.*; /** @@ -30,7 +37,7 @@ public class ClickHouseDriver implements Driver { private static final Logger logger = LoggerFactory.getLogger(ClickHouseDriver.class); - private static final ConcurrentMap connections = new MapMaker().weakKeys().makeMap(); + private static final Map connections = Collections.synchronizedMap(new WeakHashMap<>()); static { ClickHouseDriver driver = new ClickHouseDriver(); diff --git a/src/main/java/ru/yandex/clickhouse/ClickHouseExternalData.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseExternalData.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/ClickHouseExternalData.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseExternalData.java diff --git a/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatement.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatement.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatement.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatement.java diff --git a/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementImpl.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementImpl.java similarity index 95% rename from src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementImpl.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementImpl.java index a0e64175e..27d4e37ec 100644 --- a/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementImpl.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementImpl.java @@ -6,6 +6,7 @@ import java.io.Reader; import java.math.BigDecimal; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.sql.Array; import java.sql.Blob; import java.sql.Clob; @@ -38,12 +39,12 @@ import ru.yandex.clickhouse.jdbc.parser.ClickHouseSqlStatement; import ru.yandex.clickhouse.jdbc.parser.StatementType; + import ru.yandex.clickhouse.response.ClickHouseResponse; import ru.yandex.clickhouse.settings.ClickHouseProperties; import ru.yandex.clickhouse.settings.ClickHouseQueryParam; import ru.yandex.clickhouse.util.ClickHouseArrayUtil; import ru.yandex.clickhouse.util.ClickHouseValueFormatter; -import ru.yandex.clickhouse.util.guava.StreamUtils; public class ClickHousePreparedStatementImpl extends ClickHouseStatementImpl implements ClickHousePreparedStatement { @@ -338,7 +339,7 @@ private List buildBatch() throws SQLException { } sb.append(j < pList.size() - 1 ? "\t" : "\n"); } - newBatches.add(sb.toString().getBytes(StreamUtils.UTF_8)); + newBatches.add(sb.toString().getBytes(StandardCharsets.UTF_8)); sb = new StringBuilder(); } return newBatches; @@ -452,17 +453,38 @@ public ResultSetMetaData getMetaData() throws SQLException { @Override public void setDate(int parameterIndex, Date x, Calendar cal) throws SQLException { - throw new SQLFeatureNotSupportedException(); + if (x != null && cal != null && cal.getTimeZone() != null) { + setBind( + parameterIndex, + ClickHouseValueFormatter.formatDate(x, cal.getTimeZone()), + true); + } else { + setDate(parameterIndex, x); + } } @Override public void setTime(int parameterIndex, Time x, Calendar cal) throws SQLException { - throw new SQLFeatureNotSupportedException(); + if (x != null && cal != null && cal.getTimeZone() != null) { + setBind( + parameterIndex, + ClickHouseValueFormatter.formatTime(x, cal.getTimeZone()), + true); + } else { + setTime(parameterIndex, x); + } } @Override public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) throws SQLException { - throw new SQLFeatureNotSupportedException(); + if (x != null && cal != null && cal.getTimeZone() != null) { + setBind( + parameterIndex, + ClickHouseValueFormatter.formatTimestamp(x, cal.getTimeZone()), + true); + } else { + setTimestamp(parameterIndex, x); + } } @Override diff --git a/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementParameter.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementParameter.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementParameter.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHousePreparedStatementParameter.java diff --git a/src/main/java/ru/yandex/clickhouse/ClickHouseStatement.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatement.java similarity index 90% rename from src/main/java/ru/yandex/clickhouse/ClickHouseStatement.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatement.java index 1773698ae..80dc0971c 100644 --- a/src/main/java/ru/yandex/clickhouse/ClickHouseStatement.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatement.java @@ -42,68 +42,42 @@ ResultSet executeQuery(String sql, List externalData, Map additionalRequestParams) throws SQLException; - /** - * @see #write() - */ @Deprecated void sendStream(InputStream content, String table, Map additionalDBParams) throws SQLException; - /** - * @see #write() - */ @Deprecated void sendStream(InputStream content, String table) throws SQLException; - /** - * @see #write() - */ @Deprecated void sendRowBinaryStream(String sql, Map additionalDBParams, ClickHouseStreamCallback callback) throws SQLException; - /** - * @see #write() - */ @Deprecated void sendRowBinaryStream(String sql, ClickHouseStreamCallback callback) throws SQLException; - /** - * @see #write() - */ @Deprecated void sendNativeStream(String sql, Map additionalDBParams, ClickHouseStreamCallback callback) throws SQLException; - /** - * @see #write() - */ @Deprecated void sendNativeStream(String sql, ClickHouseStreamCallback callback) throws SQLException; - /** - * @see #write() - */ @Deprecated void sendCSVStream(InputStream content, String table, Map additionalDBParams) throws SQLException; - /** - * @see #write() - */ @Deprecated void sendCSVStream(InputStream content, String table) throws SQLException; - /** - * @see #write() - */ @Deprecated void sendStreamSQL(InputStream content, String sql, Map additionalDBParams) throws SQLException; - /** - * @see #write() - */ @Deprecated void sendStreamSQL(InputStream content, String sql) throws SQLException; /** - * Returns extended write-API + * Returns extended write-API, which simplifies uploading larger files or + * data streams + * + * @return a new {@link Writer} builder object which can be used to + * construct a request to the server */ Writer write(); diff --git a/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java similarity index 93% rename from src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java index 947c877f4..906e2f166 100644 --- a/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java @@ -6,6 +6,7 @@ import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLWarning; @@ -17,7 +18,6 @@ import java.util.Map; import java.util.TimeZone; import java.util.UUID; - import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -34,9 +34,7 @@ import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import com.google.common.base.Strings; - +import ru.yandex.clickhouse.domain.ClickHouseCompression; import ru.yandex.clickhouse.domain.ClickHouseFormat; import ru.yandex.clickhouse.except.ClickHouseException; import ru.yandex.clickhouse.except.ClickHouseExceptionSpecifier; @@ -55,7 +53,6 @@ import ru.yandex.clickhouse.util.ClickHouseRowBinaryInputStream; import ru.yandex.clickhouse.util.ClickHouseStreamCallback; import ru.yandex.clickhouse.util.Utils; -import ru.yandex.clickhouse.util.guava.StreamUtils; public class ClickHouseStatementImpl extends ConfigurableApi implements ClickHouseStatement { @@ -194,7 +191,7 @@ public ResultSet executeQuery(String sql, } InputStream is = getInputStream(sql, additionalDBParams, externalData, additionalRequestParams); - + try { if (parsedStmt.isQuery()) { currentUpdateCount = -1; @@ -211,11 +208,19 @@ public ResultSet executeQuery(String sql, return currentResult; } else { currentUpdateCount = 0; - StreamUtils.close(is); + try { + is.close(); + } catch (IOException e) { + log.error("can not close stream: {}", e.getMessage()); + } return null; } } catch (Exception e) { - StreamUtils.close(is); + try { + is.close(); + } catch (IOException ioe) { + log.error("can not close stream: {}", ioe.getMessage()); + } throw ClickHouseExceptionSpecifier.specify(e, properties.getHost(), properties.getPort()); } } @@ -241,21 +246,12 @@ public ClickHouseResponse executeQueryClickhouseResponse(String sql, sql = addFormatIfAbsent(sql, ClickHouseFormat.JSONCompact); } - InputStream is = getInputStream( - sql, - additionalDBParams, - null, - additionalRequestParams - ); - try { - if (properties.isCompress()) { - is = new ClickHouseLZ4Stream(is); - } + try (InputStream is = properties.isCompress() + ? new ClickHouseLZ4Stream(getInputStream(sql, additionalDBParams, null, additionalRequestParams)) + : getInputStream(sql, additionalDBParams, null, additionalRequestParams)) { return Jackson.getObjectMapper().readValue(is, ClickHouseResponse.class); } catch (IOException e) { throw new RuntimeException(e); - } finally { - StreamUtils.close(is); } } @@ -298,11 +294,19 @@ public ClickHouseRowBinaryInputStream executeQueryClickhouseRowBinaryStream(Stri return currentRowBinaryResult; } else { currentUpdateCount = 0; - StreamUtils.close(is); + try { + is.close(); + } catch (IOException e) { + log.error("can not close stream: {}", e.getMessage()); + } return null; } } catch (Exception e) { - StreamUtils.close(is); + try { + is.close(); + } catch (IOException ioe) { + log.error("can not close stream: {}", ioe.getMessage()); + } throw ClickHouseExceptionSpecifier.specify(e, properties.getHost(), properties.getPort()); } } @@ -311,12 +315,10 @@ public ClickHouseRowBinaryInputStream executeQueryClickhouseRowBinaryStream(Stri public int executeUpdate(String sql) throws SQLException { parseSingleStatement(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes); - InputStream is = null; - try { - is = getInputStream(sql, null, null, null); + try (InputStream is = getInputStream(sql, null, null, null)) { //noinspection StatementWithEmptyBody - } finally { - StreamUtils.close(is); + } catch (IOException e) { + log.error("can not close stream: {}", e.getMessage()); } return currentSummary != null ? (int) currentSummary.getWrittenRows() : 1; @@ -335,7 +337,11 @@ public void close() throws SQLException { } if (currentRowBinaryResult != null) { - StreamUtils.close(currentRowBinaryResult); + try { + currentRowBinaryResult.close(); + } catch (IOException e) { + log.error("can not close stream: {}", e.getMessage()); + } } } @@ -701,7 +707,7 @@ private InputStream getInputStream( HttpEntity requestEntity; if (externalData == null || externalData.isEmpty()) { - requestEntity = new StringEntity(sql, StreamUtils.UTF_8); + requestEntity = new StringEntity(sql, StandardCharsets.UTF_8); } else { MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); @@ -712,7 +718,7 @@ private InputStream getInputStream( // TODO do not read stream into memory when this issue is fixed in clickhouse entityBuilder.addBinaryBody( externalDataItem.getName(), - StreamUtils.toByteArray(externalDataItem.getContent()), + Utils.toByteArray(externalDataItem.getContent()), ContentType.APPLICATION_OCTET_STREAM, externalDataItem.getName() ); @@ -837,20 +843,20 @@ private List getUrlQueryParams( setStatementPropertiesToParams(params); for (Map.Entry entry : params.entrySet()) { - if (!Strings.isNullOrEmpty(entry.getValue())) { + if (!Utils.isNullOrEmptyString(entry.getValue())) { result.add(new BasicNameValuePair(entry.getKey().toString(), entry.getValue())); } } for (Map.Entry entry : getRequestParams().entrySet()) { - if (!Strings.isNullOrEmpty(entry.getValue())) { + if (!Utils.isNullOrEmptyString(entry.getValue())) { result.add(new BasicNameValuePair(entry.getKey(), entry.getValue())); } } if (additionalRequestParams != null) { for (Map.Entry entry : additionalRequestParams.entrySet()) { - if (!Strings.isNullOrEmpty(entry.getValue())) { + if (!Utils.isNullOrEmptyString(entry.getValue())) { result.add(new BasicNameValuePair(entry.getKey(), entry.getValue())); } } @@ -866,19 +872,19 @@ private boolean isQueryParamSet(ClickHouseQueryParam param, Map additionalClickHouseDBParams, Map additionalRequestParams) { - if (additionalRequestParams != null && additionalRequestParams.containsKey(param.getKey()) && !Strings.isNullOrEmpty(additionalRequestParams.get(param.getKey()))) { + if (additionalRequestParams != null && additionalRequestParams.containsKey(param.getKey()) && !Utils.isNullOrEmptyString(additionalRequestParams.get(param.getKey()))) { return additionalRequestParams.get(param.getKey()); } - if (getRequestParams().containsKey(param.getKey()) && !Strings.isNullOrEmpty(getRequestParams().get(param.getKey()))) { + if (getRequestParams().containsKey(param.getKey()) && !Utils.isNullOrEmptyString(getRequestParams().get(param.getKey()))) { return getRequestParams().get(param.getKey()); } - if (additionalClickHouseDBParams != null && additionalClickHouseDBParams.containsKey(param) && !Strings.isNullOrEmpty(additionalClickHouseDBParams.get(param))) { + if (additionalClickHouseDBParams != null && additionalClickHouseDBParams.containsKey(param) && !Utils.isNullOrEmptyString(additionalClickHouseDBParams.get(param))) { return additionalClickHouseDBParams.get(param); } - if (getAdditionalDBParams().containsKey(param) && !Strings.isNullOrEmpty(getAdditionalDBParams().get(param))) { + if (getAdditionalDBParams().containsKey(param) && !Utils.isNullOrEmptyString(getAdditionalDBParams().get(param))) { return getAdditionalDBParams().get(param); } @@ -1007,7 +1013,7 @@ void sendStream(Writer writer, HttpEntity content) throws ClickHouseException { HttpPost httpPost = new HttpPost(uri); - if (writer.getCompression() != null) { + if (writer.getCompression() != ClickHouseCompression.none) { httpPost.addHeader("Content-Encoding", writer.getCompression().name()); } httpPost.setEntity(content); @@ -1032,17 +1038,17 @@ void sendStream(Writer writer, HttpEntity content) throws ClickHouseException { private void checkForErrorAndThrow(HttpEntity entity, HttpResponse response) throws IOException, ClickHouseException { if (response.getStatusLine().getStatusCode() != HttpURLConnection.HTTP_OK) { InputStream messageStream = entity.getContent(); - byte[] bytes = StreamUtils.toByteArray(messageStream); + byte[] bytes = Utils.toByteArray(messageStream); if (properties.isCompress()) { try { messageStream = new ClickHouseLZ4Stream(new ByteArrayInputStream(bytes)); - bytes = StreamUtils.toByteArray(messageStream); + bytes = Utils.toByteArray(messageStream); } catch (IOException e) { log.warn("error while read compressed stream {}", e.getMessage()); } } EntityUtils.consumeQuietly(entity); - String chMessage = new String(bytes, StreamUtils.UTF_8); + String chMessage = new String(bytes, StandardCharsets.UTF_8); throw ClickHouseExceptionSpecifier.specify(chMessage, properties.getHost(), properties.getPort()); } } diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseUtil.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseUtil.java new file mode 100644 index 000000000..020f989af --- /dev/null +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseUtil.java @@ -0,0 +1,55 @@ +package ru.yandex.clickhouse; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class ClickHouseUtil { + private static final Map escapeMapping; + + static { + Map map = new HashMap<>(); + map.put('\\', "\\\\"); + map.put('\n', "\\n"); + map.put('\t', "\\t"); + map.put('\b', "\\b"); + map.put('\f', "\\f"); + map.put('\r', "\\r"); + map.put('\0', "\\0"); + map.put('\'', "\\'"); + map.put('`', "\\`"); + escapeMapping = Collections.unmodifiableMap(map); + } + + public static String escape(String s) { + if (s == null) { + return "\\N"; + } + + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < s.length(); i++) { + char ch = s.charAt(i); + + String escaped = escapeMapping.get(ch); + if (escaped != null) { + sb.append(escaped); + } else { + sb.append(ch); + } + } + + return sb.toString(); + } + + public static String quoteIdentifier(String s) { + if (s == null) { + throw new IllegalArgumentException("Can't quote null as identifier"); + } + StringBuilder sb = new StringBuilder(s.length() + 2); + sb.append('`'); + sb.append(escape(s)); + sb.append('`'); + return sb.toString(); + } +} diff --git a/src/main/java/ru/yandex/clickhouse/ClickhouseJdbcUrlParser.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickhouseJdbcUrlParser.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/ClickhouseJdbcUrlParser.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickhouseJdbcUrlParser.java diff --git a/src/main/java/ru/yandex/clickhouse/ConfigurableApi.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ConfigurableApi.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/ConfigurableApi.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ConfigurableApi.java diff --git a/src/main/java/ru/yandex/clickhouse/Jackson.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/Jackson.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/Jackson.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/Jackson.java diff --git a/src/main/java/ru/yandex/clickhouse/LZ4EntityWrapper.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/LZ4EntityWrapper.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/LZ4EntityWrapper.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/LZ4EntityWrapper.java diff --git a/src/main/java/ru/yandex/clickhouse/PreparedStatementParser.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/PreparedStatementParser.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/PreparedStatementParser.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/PreparedStatementParser.java diff --git a/src/main/java/ru/yandex/clickhouse/ResponseFactory.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ResponseFactory.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/ResponseFactory.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ResponseFactory.java diff --git a/src/main/java/ru/yandex/clickhouse/Writer.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/Writer.java similarity index 64% rename from src/main/java/ru/yandex/clickhouse/Writer.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/Writer.java index a93ea8fa6..dd24fccec 100644 --- a/src/main/java/ru/yandex/clickhouse/Writer.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/Writer.java @@ -1,19 +1,23 @@ package ru.yandex.clickhouse; -import org.apache.http.HttpEntity; -import org.apache.http.entity.InputStreamEntity; -import ru.yandex.clickhouse.domain.ClickHouseCompression; -import ru.yandex.clickhouse.domain.ClickHouseFormat; -import ru.yandex.clickhouse.util.ClickHouseStreamCallback; -import ru.yandex.clickhouse.util.ClickHouseStreamHttpEntity; +import static ru.yandex.clickhouse.domain.ClickHouseFormat.Native; +import static ru.yandex.clickhouse.domain.ClickHouseFormat.RowBinary; +import static ru.yandex.clickhouse.domain.ClickHouseFormat.TabSeparated; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.sql.SQLException; +import java.util.Objects; -import static ru.yandex.clickhouse.domain.ClickHouseFormat.*; +import org.apache.http.HttpEntity; +import org.apache.http.entity.InputStreamEntity; +import ru.yandex.clickhouse.domain.ClickHouseCompression; +import ru.yandex.clickhouse.domain.ClickHouseFormat; +import ru.yandex.clickhouse.settings.ClickHouseQueryParam; +import ru.yandex.clickhouse.util.ClickHouseStreamCallback; +import ru.yandex.clickhouse.util.ClickHouseStreamHttpEntity; public class Writer extends ConfigurableApi { @@ -25,10 +29,16 @@ public class Writer extends ConfigurableApi { Writer(ClickHouseStatementImpl statement) { super(statement); + + dataCompression(ClickHouseCompression.none); } /** - * Specifies format for further insert of data via send() + * Specifies format for further insert of data via send(). + * + * @param format + * the format of the data to upload + * @return this writer instance */ public Writer format(ClickHouseFormat format) { if (null == format) { @@ -39,10 +49,11 @@ public Writer format(ClickHouseFormat format) { } /** - * Set table name for data insertion + * Set table name for data insertion. * - * @param table table name - * @return this + * @param table + * name of the table to upload the data to + * @return this writer instance */ public Writer table(String table) { this.sql = null; @@ -51,10 +62,11 @@ public Writer table(String table) { } /** - * Set SQL for data insertion + * Set SQL for data insertion. * - * @param sql in a form "INSERT INTO table_name [(X,Y,Z)] VALUES " - * @return this + * @param sql + * in a form "INSERT INTO table_name [(X,Y,Z)] VALUES " + * @return this writer instance */ public Writer sql(String sql) { this.sql = sql; @@ -63,19 +75,36 @@ public Writer sql(String sql) { } /** - * Specifies data input stream + * Specifies data input stream. + * + * @param stream + * a stream providing the data to upload + * @return this writer instance */ public Writer data(InputStream stream) { streamProvider = new HoldingInputProvider(stream); return this; } + /** + * Specifies data input stream, and the format to use. + * + * @param stream + * a stream providing the data to upload + * @param format + * the format of the data to upload + * @return this writer instance + */ public Writer data(InputStream stream, ClickHouseFormat format) { return format(format).data(stream); } /** - * Shortcut method for specifying a file as an input + * Shortcut method for specifying a file as an input. + * + * @param input + * the file to upload + * @return this writer instance */ public Writer data(File input) { streamProvider = new FileInputProvider(input); @@ -91,10 +120,9 @@ public Writer data(File input, ClickHouseFormat format, ClickHouseCompression co } public Writer dataCompression(ClickHouseCompression compression) { - if (null == compression) { - throw new NullPointerException("Compression can not be null"); - } - this.compression = compression; + this.compression = Objects.requireNonNull(compression, "Compression can not be null"); + this.addDbParam(ClickHouseQueryParam.COMPRESS, String.valueOf(compression != ClickHouseCompression.none)); + return this; } @@ -103,7 +131,7 @@ public Writer data(File input, ClickHouseFormat format) { } /** - * Method to call, when Writer is fully configured + * Method to call, when Writer is fully configured. */ public void send() throws SQLException { HttpEntity entity; @@ -124,31 +152,50 @@ private void send(HttpEntity entity) throws SQLException { } /** - * Allows to send stream of data to ClickHouse + * Allows to send stream of data to ClickHouse. * - * @param sql in a form of "INSERT INTO table_name (X,Y,Z) VALUES " - * @param data where to read data from - * @param format format of data in InputStream + * @param sql + * in a form of "INSERT INTO table_name (X,Y,Z) VALUES " + * @param data + * where to read data from + * @param format + * format of data in InputStream * @throws SQLException + * if the upload fails */ public void send(String sql, InputStream data, ClickHouseFormat format) throws SQLException { sql(sql).data(data).format(format).send(); } /** - * Convenient method for importing the data into table + * Convenient method for importing the data into table. * - * @param table table name - * @param data source data - * @param format format of data in InputStream + * @param table + * table name + * @param data + * source data + * @param format + * format of data in InputStream * @throws SQLException + * if the upload fails */ public void sendToTable(String table, InputStream data, ClickHouseFormat format) throws SQLException { table(table).data(data).format(format).send(); } /** - * Sends the data in RowBinary or in Native formats + * Sends the data in {@link ClickHouseFormat#RowBinary RowBinary} or in + * {@link ClickHouseFormat#Native Native} format. + * + * @param sql + * the SQL statement to execute + * @param callback + * data source for the upload + * @param format + * the format to use, either {@link ClickHouseFormat#RowBinary + * RowBinary} or {@link ClickHouseFormat#Native Native} + * @throws SQLException + * if the upload fails */ public void send(String sql, ClickHouseStreamCallback callback, ClickHouseFormat format) throws SQLException { if (!(RowBinary.equals(format) || Native.equals(format))) { diff --git a/src/main/java/ru/yandex/clickhouse/domain/ClickHouseCompression.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/domain/ClickHouseCompression.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/domain/ClickHouseCompression.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/domain/ClickHouseCompression.java diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/domain/ClickHouseDataType.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/domain/ClickHouseDataType.java new file mode 100644 index 000000000..182120b90 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/domain/ClickHouseDataType.java @@ -0,0 +1,178 @@ +package ru.yandex.clickhouse.domain; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Array; +import java.sql.Date; +import java.sql.JDBCType; +import java.sql.Timestamp; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +/** + * Basic ClickHouse data types. + *

+ * This list is based on the list of data type families returned by + * {@code SELECT * FROM system.data_type_families} + *

+ * {@code LowCardinality} and {@code Nullable} are technically data types in + * ClickHouse, but for the sake of this driver, we treat these data types as + * modifiers for the underlying base data types. + */ +public enum ClickHouseDataType { + // aliases: + // https://clickhouse.tech/docs/en/sql-reference/data-types/multiword-types/ + // https://github.com/ClickHouse/ClickHouse/blob/master/src/DataTypes/DataTypeCustomIPv4AndIPv6.cpp + // https://github.com/ClickHouse/ClickHouse/blob/master/src/DataTypes/registerDataTypeDateTime.cpp + // https://github.com/ClickHouse/ClickHouse/blob/master/src/DataTypes/DataTypesDecimal.cpp + // https://github.com/ClickHouse/ClickHouse/blob/master/src/DataTypes/DataTypeFixedString.cpp + // https://github.com/ClickHouse/ClickHouse/blob/master/src/DataTypes/DataTypesNumber.cpp + // https://github.com/ClickHouse/ClickHouse/blob/master/src/DataTypes/DataTypeString.cpp + IntervalYear (JDBCType.INTEGER, Integer.class, true, 19, 0), + IntervalQuarter (JDBCType.INTEGER, Integer.class, true, 19, 0), + IntervalMonth (JDBCType.INTEGER, Integer.class, true, 19, 0), + IntervalWeek (JDBCType.INTEGER, Integer.class, true, 19, 0), + IntervalDay (JDBCType.INTEGER, Integer.class, true, 19, 0), + IntervalHour (JDBCType.INTEGER, Integer.class, true, 19, 0), + IntervalMinute (JDBCType.INTEGER, Integer.class, true, 19, 0), + IntervalSecond (JDBCType.INTEGER, Integer.class, true, 19, 0), + UInt256 (JDBCType.NUMERIC, BigInteger.class, true, 39, 0), + UInt128 (JDBCType.NUMERIC, BigInteger.class, true, 20, 0), + UInt64 (JDBCType.BIGINT, BigInteger.class, false, 19, 0, + "BIGINT UNSIGNED"), + UInt32 (JDBCType.BIGINT, Long.class, false, 10, 0, + "INT UNSIGNED", "INTEGER UNSIGNED", "MEDIUMINT UNSIGNED"), + UInt16 (JDBCType.SMALLINT, Integer.class, false, 5, 0, + "SMALLINT UNSIGNED"), + UInt8 (JDBCType.TINYINT, Integer.class, false, 3, 0, + "TINYINT UNSIGNED", "INT1 UNSIGNED"), + Int256 (JDBCType.NUMERIC, BigInteger.class, true, 40, 0), + Int128 (JDBCType.NUMERIC, BigInteger.class, true, 20, 0), + Int64 (JDBCType.BIGINT, Long.class, true, 20, 0, + "BIGINT", "BIGINT SIGNED"), + Int32 (JDBCType.INTEGER, Integer.class, true, 11, 0, + "INT", "INTEGER", "MEDIUMINT", "INT SIGNED", "INTEGER SIGNED", "MEDIUMINT SIGNED"), + Int16 (JDBCType.SMALLINT, Integer.class, true, 6, 0, + "SMALLINT", "SMALLINT SIGNED"), + Int8 (JDBCType.TINYINT, Integer.class, true, 4, 0, + "TINYINT", "BOOL", "BOOLEAN", "INT1", "BYTE", "TINYINT SIGNED", "INT1 SIGNED"), + Date (JDBCType.DATE, Date.class, false, 10, 0), + DateTime (JDBCType.TIMESTAMP, Timestamp.class, false, 19, 0, + "TIMESTAMP"), + DateTime32 (JDBCType.TIMESTAMP, Timestamp.class, false, 19, 0), + DateTime64 (JDBCType.TIMESTAMP, Timestamp.class, false, 38, 3), // scale up to 18 + Enum8 (JDBCType.VARCHAR, String.class, false, 0, 0, + "ENUM"), + Enum16 (JDBCType.VARCHAR, String.class, false, 0, 0), + Float32 (JDBCType.REAL, Float.class, true, 8, 8, + "SINGLE", "REAL"), + Float64 (JDBCType.DOUBLE, Double.class, true, 17, 17, + "DOUBLE", "DOUBLE PRECISION"), + Decimal32 (JDBCType.DECIMAL, BigDecimal.class, true, 9, 9), + Decimal64 (JDBCType.DECIMAL, BigDecimal.class, true, 18, 18), + Decimal128 (JDBCType.DECIMAL, BigDecimal.class, true, 38, 38), + Decimal256 (JDBCType.DECIMAL, BigDecimal.class, true, 76, 20), + Decimal (JDBCType.DECIMAL, BigDecimal.class, true, 0, 0, + "DEC", "NUMERIC", "FIXED"), + UUID (JDBCType.OTHER, UUID.class, false, 36, 0), + IPv4 (JDBCType.VARCHAR, String.class, false, 10, 0), + IPv6 (JDBCType.VARCHAR, String.class, false, 0, 0), + String (JDBCType.VARCHAR, String.class, false, 0, 0, + "CHAR", "NCHAR", "CHARACTER", "VARCHAR", "NVARCHAR", "VARCHAR2", + "TEXT", "TINYTEXT", "MEDIUMTEXT", "LONGTEXT", + "BLOB", "CLOB", "TINYBLOB", "MEDIUMBLOB", "LONGBLOB", "BYTEA", + "CHARACTER LARGE OBJECT", "CHARACTER VARYING", "CHAR LARGE OBJECT", "CHAR VARYING", + "NATIONAL CHAR", "NATIONAL CHARACTER", "NATIONAL CHARACTER LARGE OBJECT", + "NATIONAL CHARACTER VARYING", "NATIONAL CHAR VARYING", + "NCHAR VARYING", "NCHAR LARGE OBJECT", "BINARY LARGE OBJECT", "BINARY VARYING"), + FixedString (JDBCType.CHAR, String.class, false, -1, 0, + "BINARY"), + Nothing (JDBCType.NULL, Object.class, false, 0, 0), + Nested (JDBCType.STRUCT, String.class, false, 0, 0), + // TODO use list/collection for Tuple + Tuple (JDBCType.OTHER, String.class, false, 0, 0), + Array (JDBCType.ARRAY, Array.class, false, 0, 0), + Map (JDBCType.OTHER, Map.class, false, 0, 0), + AggregateFunction (JDBCType.OTHER, String.class, false, 0, 0), + Unknown (JDBCType.OTHER, String.class, false, 0, 0); + + private static final Map name2type; + + static { + Map map = new HashMap<>(); + String errorMsg = "[%s] is used by type [%s]"; + ClickHouseDataType used = null; + for (ClickHouseDataType t : ClickHouseDataType.values()) { + used = map.put(t.name(), t); + if (used != null) { + throw new IllegalStateException(java.lang.String.format(errorMsg, t.name(), used.name())); + } + String nameInUpperCase = t.name().toUpperCase(); + if (!nameInUpperCase.equals(t.name())) { + used = map.put(nameInUpperCase, t); + if (used != null) { + throw new IllegalStateException(java.lang.String.format(errorMsg, nameInUpperCase, used.name())); + } + } + for (String alias: t.aliases) { + used = map.put(alias.toUpperCase(), t); + if (used != null) { + throw new IllegalStateException(java.lang.String.format(errorMsg, alias, used.name())); + } + } + } + name2type = Collections.unmodifiableMap(map); + } + + private final JDBCType jdbcType; + private final Class javaClass; + private final boolean signed; + private final int defaultPrecision; + private final int defaultScale; + private final String[] aliases; + + ClickHouseDataType(JDBCType jdbcType, Class javaClass, + boolean signed, int defaultPrecision, int defaultScale, + String... aliases) { + this.jdbcType = jdbcType; + this.javaClass = javaClass; + this.signed = signed; + this.defaultPrecision = defaultPrecision; + this.defaultScale = defaultScale; + this.aliases = aliases; + } + + public int getSqlType() { + return jdbcType.getVendorTypeNumber().intValue(); + } + + public JDBCType getJdbcType() { + return jdbcType; + } + + public Class getJavaClass() { + return javaClass; + } + + public boolean isSigned() { + return signed; + } + + public int getDefaultPrecision() { + return defaultPrecision; + } + + public int getDefaultScale() { + return defaultScale; + } + + public static ClickHouseDataType fromTypeString(String typeString) { + return name2type.getOrDefault(typeString.trim().toUpperCase(), ClickHouseDataType.Unknown); + } + + public static ClickHouseDataType resolveDefaultArrayDataType(String typeName) { + return name2type.getOrDefault(typeName, ClickHouseDataType.String); + } +} diff --git a/src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java similarity index 93% rename from src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java index ef8614cb2..a230d1a89 100644 --- a/src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java @@ -24,7 +24,11 @@ public enum ClickHouseFormat { Vertical, JSON, JSONCompact, + JSONCompactString, JSONEachRow, + JSONStringEachRow, + JSONCompactEachRow, + JSONCompactStringEachRow, TSKV, TSV, Pretty, diff --git a/src/main/java/ru/yandex/clickhouse/except/ClickHouseErrorCode.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/except/ClickHouseErrorCode.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/except/ClickHouseErrorCode.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/except/ClickHouseErrorCode.java diff --git a/src/main/java/ru/yandex/clickhouse/except/ClickHouseException.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/except/ClickHouseException.java similarity index 74% rename from src/main/java/ru/yandex/clickhouse/except/ClickHouseException.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/except/ClickHouseException.java index 37d020855..570da79cb 100644 --- a/src/main/java/ru/yandex/clickhouse/except/ClickHouseException.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/except/ClickHouseException.java @@ -13,4 +13,10 @@ public ClickHouseException(int code, String message, Throwable cause, String hos super("ClickHouse exception, message: " + message + ", host: " + host + ", port: " + port + "; " + (cause == null ? "" : cause.getMessage()), null, code, cause); } + + public ClickHouseException(int code, String message, Throwable cause) { + super("ClickHouse exception, message: " + message + "; " + + (cause == null ? "" : cause.getMessage()), null, code, cause); + } + } diff --git a/src/main/java/ru/yandex/clickhouse/except/ClickHouseExceptionSpecifier.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/except/ClickHouseExceptionSpecifier.java similarity index 96% rename from src/main/java/ru/yandex/clickhouse/except/ClickHouseExceptionSpecifier.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/except/ClickHouseExceptionSpecifier.java index 53f3b87cc..c4bbe0848 100644 --- a/src/main/java/ru/yandex/clickhouse/except/ClickHouseExceptionSpecifier.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/except/ClickHouseExceptionSpecifier.java @@ -1,10 +1,11 @@ package ru.yandex.clickhouse.except; -import com.google.common.base.Strings; import org.apache.http.conn.ConnectTimeoutException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import ru.yandex.clickhouse.util.Utils; + import java.net.ConnectException; import java.net.SocketTimeoutException; @@ -37,7 +38,7 @@ public static ClickHouseException specify(String clickHouseMessage) { * "Code: 10, e.displayText() = DB::Exception: ...". */ private static ClickHouseException specify(String clickHouseMessage, Throwable cause, String host, int port) { - if (Strings.isNullOrEmpty(clickHouseMessage) && cause != null) { + if (Utils.isNullOrEmptyString(clickHouseMessage) && cause != null) { return getException(cause, host, port); } diff --git a/src/main/java/ru/yandex/clickhouse/except/ClickHouseUnknownException.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/except/ClickHouseUnknownException.java similarity index 79% rename from src/main/java/ru/yandex/clickhouse/except/ClickHouseUnknownException.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/except/ClickHouseUnknownException.java index d2c295a19..e1bd79708 100644 --- a/src/main/java/ru/yandex/clickhouse/except/ClickHouseUnknownException.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/except/ClickHouseUnknownException.java @@ -11,6 +11,10 @@ public ClickHouseUnknownException(String message, Throwable cause, String host, super(ClickHouseErrorCode.UNKNOWN_EXCEPTION.code, message, cause, host, port); } + public ClickHouseUnknownException(String message, Throwable cause) { + super(ClickHouseErrorCode.UNKNOWN_EXCEPTION.code, message, cause); + } + public ClickHouseUnknownException(Integer code, Throwable cause, String host, int port) { super(code, cause, host, port); } diff --git a/src/main/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlStatement.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlStatement.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlStatement.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlStatement.java diff --git a/src/main/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlUtils.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlUtils.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlUtils.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/ClickHouseSqlUtils.java diff --git a/src/main/java/ru/yandex/clickhouse/jdbc/parser/LanguageType.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/LanguageType.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/jdbc/parser/LanguageType.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/LanguageType.java diff --git a/src/main/java/ru/yandex/clickhouse/jdbc/parser/OperationType.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/OperationType.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/jdbc/parser/OperationType.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/OperationType.java diff --git a/src/main/java/ru/yandex/clickhouse/jdbc/parser/ParseHandler.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/ParseHandler.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/jdbc/parser/ParseHandler.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/ParseHandler.java diff --git a/src/main/java/ru/yandex/clickhouse/jdbc/parser/StatementType.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/StatementType.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/jdbc/parser/StatementType.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/jdbc/parser/StatementType.java diff --git a/src/main/java/ru/yandex/clickhouse/response/AbstractResultSet.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/AbstractResultSet.java similarity index 96% rename from src/main/java/ru/yandex/clickhouse/response/AbstractResultSet.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/AbstractResultSet.java index 14642c959..9c44e6f6a 100644 --- a/src/main/java/ru/yandex/clickhouse/response/AbstractResultSet.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/AbstractResultSet.java @@ -4,12 +4,27 @@ import java.io.Reader; import java.math.BigDecimal; import java.net.URL; -import java.sql.*; +import java.sql.Array; +import java.sql.Blob; +import java.sql.Clob; +import java.sql.Date; +import java.sql.NClob; +import java.sql.Ref; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.RowId; +import java.sql.SQLException; +import java.sql.SQLWarning; +import java.sql.SQLXML; +import java.sql.Statement; +import java.sql.Time; +import java.sql.Timestamp; import java.util.Calendar; import java.util.Map; public abstract class AbstractResultSet implements ResultSet { + @Override public boolean next() throws SQLException { throw new UnsupportedOperationException(); @@ -952,12 +967,20 @@ public boolean isWrapperFor(Class iface) throws SQLException { throw new UnsupportedOperationException(); } - public long[] getLongArray(String column) throws SQLException { - Array array = getArray(column); - return (long[])array.getArray(); // optimistic - } - - - + /** + * Parse the value in current row at column with label {@code column} as an array + * of long + * + * @param column + * the label, name, alias of the column + * @return an array of longs + * @throws SQLException + * if the value cannot be interpreted as {@code long[]} + * @deprecated prefer to use regular JDBC API, e.g. via + * {@link #getArray(int)} or simply + * {@link #getObject(int, Class)} + */ + @Deprecated + public abstract long[] getLongArray(String column) throws SQLException; } diff --git a/src/main/java/ru/yandex/clickhouse/response/ArrayByteFragment.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ArrayByteFragment.java similarity index 62% rename from src/main/java/ru/yandex/clickhouse/response/ArrayByteFragment.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ArrayByteFragment.java index bf7390b00..602ec42f5 100644 --- a/src/main/java/ru/yandex/clickhouse/response/ArrayByteFragment.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ArrayByteFragment.java @@ -1,12 +1,12 @@ package ru.yandex.clickhouse.response; -class ArrayByteFragment extends ByteFragment { +public final class ArrayByteFragment extends ByteFragment { private ArrayByteFragment(byte[] buf, int start, int len) { super(buf, start, len); } - static ArrayByteFragment wrap(ByteFragment fragment) { + public static ArrayByteFragment wrap(ByteFragment fragment) { return new ArrayByteFragment(fragment.buf, fragment.start, fragment.len); } @@ -16,8 +16,5 @@ public boolean isNull() { return len == 4 && buf[start] == 'N' && buf[start + 1] == 'U' && buf[start + 2] == 'L' && buf[start + 3] == 'L'; } - public boolean isNaN() { - // nan - return len == 3 && buf[start] == 'n' && buf[start + 1] == 'a' && buf[start + 2] == 'n'; - } + } diff --git a/src/main/java/ru/yandex/clickhouse/response/ArrayToStringDeserializer.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ArrayToStringDeserializer.java similarity index 56% rename from src/main/java/ru/yandex/clickhouse/response/ArrayToStringDeserializer.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ArrayToStringDeserializer.java index 9ab7eadbb..a0dbed6f1 100644 --- a/src/main/java/ru/yandex/clickhouse/response/ArrayToStringDeserializer.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ArrayToStringDeserializer.java @@ -5,48 +5,41 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.type.TypeFactory; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; import ru.yandex.clickhouse.Jackson; +import ru.yandex.clickhouse.util.LRUCache; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ExecutionException; - +import java.util.Map; +import java.util.function.Function; class ArrayToStringDeserializer extends JsonDeserializer> { - - private static final LoadingCache> deserializers - = CacheBuilder.newBuilder() - .weakKeys() - .concurrencyLevel(16) - .maximumSize(10000) - .build(new CacheLoader>() { - @Override - public JsonDeserializer load(DeserializationContext ctxt) throws Exception { - return ctxt.findContextualValueDeserializer(TypeFactory.defaultInstance() - .constructType(new TypeReference>() { - }), null); - } - }); + private static final Map> deserializers = LRUCache.create(1000, + new Function>() { + @Override + public JsonDeserializer apply(DeserializationContext ctx) { + try { + return ctx.findContextualValueDeserializer( + TypeFactory.defaultInstance().constructType(new TypeReference>() { + }), null); + } catch (JsonMappingException e) { + throw new IllegalStateException(e); + } + } + }); @Override public List deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException { - JsonDeserializer deserializer; - try { - deserializer = deserializers.get(ctxt); - } catch (ExecutionException e){ - throw new RuntimeException(e); - } + JsonDeserializer deserializer = deserializers.get(ctxt); final Object deserialized = deserializer.deserialize(jp, ctxt); - if (!(deserialized instanceof List)){ + if (!(deserialized instanceof List)) { throw new IllegalStateException(); } - //noinspection unchecked + // noinspection unchecked final List deserializedList = (List) deserialized; List result = new ArrayList(); for (Object x : deserializedList) { diff --git a/src/main/java/ru/yandex/clickhouse/response/ByteFragment.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ByteFragment.java similarity index 90% rename from src/main/java/ru/yandex/clickhouse/response/ByteFragment.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ByteFragment.java index f9e3037f9..4ffcb2c6e 100644 --- a/src/main/java/ru/yandex/clickhouse/response/ByteFragment.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ByteFragment.java @@ -1,10 +1,9 @@ package ru.yandex.clickhouse.response; -import ru.yandex.clickhouse.util.guava.StreamUtils; - import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; public class ByteFragment { @@ -20,18 +19,21 @@ public ByteFragment(byte[] buf, int start, int len) { } public static ByteFragment fromString(String str) { - byte[] bytes = str.getBytes(StreamUtils.UTF_8); + // https://bugs.openjdk.java.net/browse/JDK-6219899 + byte[] bytes = str.getBytes(StandardCharsets.UTF_8); return new ByteFragment(bytes, 0, bytes.length); } public String asString() { - return new String(buf, start, len, StreamUtils.UTF_8); + return new String(buf, start, len, StandardCharsets.UTF_8); } public String asString(boolean unescape) { if(unescape) { - if (isNull()) return null; - return new String(unescape(), StreamUtils.UTF_8); + if (isNull()) { + return null; + } + return new String(unescape(), StandardCharsets.UTF_8); } else { return asString(); } @@ -42,6 +44,15 @@ public boolean isNull() { return len == 2 && buf[start] == '\\' && buf[start + 1] == 'N'; } + public boolean isEmpty() { + return len == 0; + } + + public boolean isNaN() { + // nan + return len == 3 && buf[start] == 'n' && buf[start + 1] == 'a' && buf[start + 2] == 'n'; + } + @Override public String toString() { StringBuilder b = new StringBuilder(); @@ -72,7 +83,9 @@ public ByteFragment[] split(byte sep) { } } catch (IOException ignore) { } - if(res[c-1] == null) res[c-1] = ByteFragment.EMPTY; + if(res[c-1] == null) { + res[c-1] = ByteFragment.EMPTY; + } return res; } // [45, 49, 57, 52, 49, 51, 56, 48, 57, 49, 52, 9, 9, 50, 48, 49, 50, 45, 48, 55, 45, 49, 55, 32, 49, 51, 58, 49, 50, 58, 50, 49, 9, 49, 50, 49, 50, 55, 53, 53, 9, 50, 57, 57, 57, 55, 55, 57, 57, 55, 56, 9, 48, 9, 52, 48, 57, 49, 57, 55, 52, 49, 49, 51, 50, 56, 53, 53, 50, 54, 57, 51, 9, 51, 9, 54, 9, 50, 48, 9, 48, 92, 48, 9, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 97, 118, 105, 116, 111, 46, 114, 117, 47, 99, 97, 116, 97, 108, 111, 103, 47, 103, 97, 114, 97, 122, 104, 105, 95, 105, 95, 109, 97, 115, 104, 105, 110, 111, 109, 101, 115, 116, 97, 45, 56, 53, 47, 116, 97, 116, 97, 114, 115, 116, 97, 110, 45, 54, 53, 48, 49, 51, 48, 47, 112, 97, 103, 101, 56, 9, 104, 116, 116, 112, 58, 47, 47, 119, 119, 119, 46, 97, 118, 105, 116, 111, 46, 114, 117, 47, 99, 97, 116, 97, 108, 111, 103, 47, 103, 97, 114, 97, 122, 104, 105, 95, 105, 95, 109, 97, 115, 104, 105, 110, 111, 109, 101, 115, 116, 97, 45, 56, 53, 47, 116, 97, 116, 97, 114, 115, 116, 97, 110, 45, 54, 53, 48, 49, 51, 48, 47, 112, 97, 103, 101, 55, 9, 48, 9, 48, 9, 50, 56, 53, 55, 48, 56, 48, 9, 45, 49, 9, 48, 9, 9, 48, 9, 48, 9, 48, 9, 45, 49, 9, 48, 48, 48, 48, 45, 48, 48, 45, 48, 48, 32, 48, 48, 58, 48, 48, 58, 48, 48, 9, 9, 48, 9, 48, 9, 103, 9, 45, 49, 9, 45, 49, 9, 45, 49, 9] @@ -174,7 +187,9 @@ public byte[] unescape() { for (int i = 0; i < convert.length; i++) { reverse[i] = -1; byte c = convert[i]; - if (c != -1) reverse[c] = (byte) i; + if (c != -1) { + reverse[c] = (byte) i; + } } } diff --git a/src/main/java/ru/yandex/clickhouse/response/ClickHouseColumnInfo.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseColumnInfo.java similarity index 51% rename from src/main/java/ru/yandex/clickhouse/response/ClickHouseColumnInfo.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseColumnInfo.java index 5783300c7..240dd0d9a 100644 --- a/src/main/java/ru/yandex/clickhouse/response/ClickHouseColumnInfo.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseColumnInfo.java @@ -1,9 +1,11 @@ package ru.yandex.clickhouse.response; import java.util.TimeZone; - import ru.yandex.clickhouse.domain.ClickHouseDataType; +/** + * This class represents a column defined in database. + */ public final class ClickHouseColumnInfo { private static final String KEYWORD_NULLABLE = "Nullable"; @@ -20,8 +22,24 @@ public final class ClickHouseColumnInfo { private TimeZone timeZone; private int precision; private int scale; + private ClickHouseColumnInfo keyInfo; + private ClickHouseColumnInfo valueInfo; + private String functionName; + @Deprecated public static ClickHouseColumnInfo parse(String typeInfo, String columnName) { + return parse(typeInfo, columnName, null); + } + + /** + * Parse given type string. + * + * @param typeInfo type defined in database + * @param columnName column name + * @param serverTimeZone server time zone + * @return parsed type + */ + public static ClickHouseColumnInfo parse(String typeInfo, String columnName, TimeZone serverTimeZone) { ClickHouseColumnInfo column = new ClickHouseColumnInfo(typeInfo, columnName); int currIdx = 0; while (typeInfo.startsWith(KEYWORD_ARRAY, currIdx)) { @@ -52,42 +70,76 @@ public static ClickHouseColumnInfo parse(String typeInfo, String columnName) { } column.precision = dataType.getDefaultPrecision(); column.scale = dataType.getDefaultScale(); + column.timeZone = serverTimeZone; currIdx = endIdx; - if (endIdx == typeInfo.length() - || !typeInfo.startsWith("(", currIdx)) - { + if (endIdx == typeInfo.length() || !typeInfo.startsWith("(", currIdx)) { return column; } switch (dataType) { - case DateTime : - String[] argsTZ = splitArgs(typeInfo, currIdx); - if (argsTZ.length == 1) { - // unfortunately this will fall back to GMT if the time zone - // cannot be resolved - TimeZone tz = TimeZone.getTimeZone(argsTZ[0].replace("'", "")); - column.timeZone = tz; - } - break; - case Decimal : - String[] argsDecimal = splitArgs(typeInfo, currIdx); - if (argsDecimal.length == 2) { - column.precision = Integer.parseInt(argsDecimal[0]); - column.scale = Integer.parseInt(argsDecimal[1]); - } - break; - case Decimal32 : - case Decimal64 : - case Decimal128 : - String[] argsScale = splitArgs(typeInfo, currIdx); - column.scale = Integer.parseInt(argsScale[0]); - break; - case FixedString : - String[] argsPrecision = splitArgs(typeInfo, currIdx); - column.precision = Integer.parseInt(argsPrecision[0]); - break; - default : - break; + case AggregateFunction : + String[] argsAf = splitArgs(typeInfo, currIdx); + column.functionName = argsAf[0]; + column.arrayBaseType = ClickHouseDataType.Unknown; + if (argsAf.length == 2) { + column.arrayBaseType = ClickHouseDataType.fromTypeString(argsAf[1]); + } + break; + case DateTime : + String[] argsDt = splitArgs(typeInfo, currIdx); + if (argsDt.length == 2) { // same as DateTime64 + column.scale = Integer.parseInt(argsDt[0]); + column.timeZone = TimeZone.getTimeZone(argsDt[1].replace("'", "")); + } else if (argsDt.length == 1) { // same as DateTime32 + // unfortunately this will fall back to GMT if the time zone + // cannot be resolved + TimeZone tz = TimeZone.getTimeZone(argsDt[0].replace("'", "")); + column.timeZone = tz; + } + break; + case DateTime32: + String[] argsD32 = splitArgs(typeInfo, currIdx); + if (argsD32.length == 1) { + // unfortunately this will fall back to GMT if the time zone + // cannot be resolved + TimeZone tz = TimeZone.getTimeZone(argsD32[0].replace("'", "")); + column.timeZone = tz; + } + break; + case DateTime64: + String[] argsD64 = splitArgs(typeInfo, currIdx); + if (argsD64.length == 2) { + column.scale = Integer.parseInt(argsD64[0]); + column.timeZone = TimeZone.getTimeZone(argsD64[1].replace("'", "")); + } + break; + case Decimal : + String[] argsDecimal = splitArgs(typeInfo, currIdx); + if (argsDecimal.length == 2) { + column.precision = Integer.parseInt(argsDecimal[0]); + column.scale = Integer.parseInt(argsDecimal[1]); + } + break; + case Decimal32 : + case Decimal64 : + case Decimal128 : + case Decimal256 : + String[] argsScale = splitArgs(typeInfo, currIdx); + column.scale = Integer.parseInt(argsScale[0]); + break; + case FixedString : + String[] argsPrecision = splitArgs(typeInfo, currIdx); + column.precision = Integer.parseInt(argsPrecision[0]); + break; + case Map: + String[] argsMap = splitArgs(typeInfo, currIdx); + if (argsMap.length == 2) { + column.keyInfo = ClickHouseColumnInfo.parse(argsMap[0], columnName + "Key", serverTimeZone); + column.valueInfo = ClickHouseColumnInfo.parse(argsMap[1], columnName + "Value", serverTimeZone); + } + break; + default: + break; } return column; @@ -118,8 +170,9 @@ public String getOriginalTypeName() { } /** - * @return the type name returned from the database, without modifiers, i.e. - * Nullable or LowCardinality + * Get the type name returned from the database, without modifiers, i.e. Nullable or LowCardinality. + * + * @return the type name returned from the database */ public String getCleanTypeName() { if (!nullable && !lowCardinality) { @@ -156,15 +209,23 @@ boolean isLowCardinality() { return lowCardinality; } - int getArrayLevel() { + public int getArrayLevel() { return arrayLevel; } + public boolean isArray() { + return arrayLevel > 0; + } + public ClickHouseDataType getArrayBaseType() { return arrayBaseType; } - TimeZone getTimeZone() { + public ClickHouseDataType getEffectiveClickHouseDataType() { + return arrayLevel > 0 ? arrayBaseType : clickHouseDataType; + } + + public TimeZone getTimeZone() { return timeZone; } @@ -176,4 +237,15 @@ public int getScale() { return scale; } + public ClickHouseColumnInfo getKeyInfo() { + return this.keyInfo; + } + + public ClickHouseColumnInfo getValueInfo() { + return this.valueInfo; + } + + public String getFunctionName() { + return this.functionName; + } } diff --git a/src/main/java/ru/yandex/clickhouse/response/ClickHouseLZ4Stream.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseLZ4Stream.java similarity index 89% rename from src/main/java/ru/yandex/clickhouse/response/ClickHouseLZ4Stream.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseLZ4Stream.java index 75139a5e3..a1ebf2d45 100644 --- a/src/main/java/ru/yandex/clickhouse/response/ClickHouseLZ4Stream.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseLZ4Stream.java @@ -1,10 +1,11 @@ package ru.yandex.clickhouse.response; -import com.google.common.io.LittleEndianDataInputStream; import net.jpountz.lz4.LZ4Factory; import net.jpountz.lz4.LZ4FastDecompressor; import ru.yandex.clickhouse.util.ClickHouseBlockChecksum; +import ru.yandex.clickhouse.util.Utils; +import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; @@ -19,14 +20,14 @@ public class ClickHouseLZ4Stream extends InputStream { public static final int MAGIC = 0x82; private final InputStream stream; - private final LittleEndianDataInputStream dataWrapper; + private final DataInputStream dataWrapper; private byte[] currentBlock; private int pointer; public ClickHouseLZ4Stream(InputStream stream) { this.stream = stream; - dataWrapper = new LittleEndianDataInputStream(stream); + dataWrapper = new DataInputStream(stream); } @Override @@ -85,20 +86,20 @@ private byte[] readNextBlock() throws IOException { byte[] checksum = new byte[16]; checksum[0] = (byte)read; // checksum - 16 bytes. - dataWrapper.readFully(checksum, 1, 15); + Utils.readFully(dataWrapper, checksum, 1, 15); ClickHouseBlockChecksum expected = ClickHouseBlockChecksum.fromBytes(checksum); // header: // 1 byte - 0x82 (shows this is LZ4) int magic = dataWrapper.readUnsignedByte(); if (magic != MAGIC) throw new IOException("Magic is not correct: " + magic); // 4 bytes - size of the compressed data including 9 bytes of the header - int compressedSizeWithHeader = dataWrapper.readInt(); + int compressedSizeWithHeader = Utils.readInt(dataWrapper); // 4 bytes - size of uncompressed data - int uncompressedSize = dataWrapper.readInt(); + int uncompressedSize = Utils.readInt(dataWrapper); int compressedSize = compressedSizeWithHeader - 9; //header byte[] block = new byte[compressedSize]; // compressed data: compressed_size - 9 байт. - dataWrapper.readFully(block); + Utils.readFully(dataWrapper, block); ClickHouseBlockChecksum real = ClickHouseBlockChecksum.calculateForBlock((byte)magic, compressedSizeWithHeader, uncompressedSize, block, compressedSize); if (!real.equals(expected)) { diff --git a/src/main/java/ru/yandex/clickhouse/response/ClickHouseResponse.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResponse.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/response/ClickHouseResponse.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResponse.java diff --git a/src/main/java/ru/yandex/clickhouse/response/ClickHouseResponseFactory.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResponseFactory.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/response/ClickHouseResponseFactory.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResponseFactory.java diff --git a/src/main/java/ru/yandex/clickhouse/response/ClickHouseResponseSummary.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResponseSummary.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/response/ClickHouseResponseSummary.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResponseSummary.java diff --git a/src/main/java/ru/yandex/clickhouse/response/ClickHouseResultBuilder.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResultBuilder.java similarity index 96% rename from src/main/java/ru/yandex/clickhouse/response/ClickHouseResultBuilder.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResultBuilder.java index 0021f1a1e..d8173c088 100644 --- a/src/main/java/ru/yandex/clickhouse/response/ClickHouseResultBuilder.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResultBuilder.java @@ -1,11 +1,11 @@ package ru.yandex.clickhouse.response; import ru.yandex.clickhouse.settings.ClickHouseProperties; -import ru.yandex.clickhouse.util.guava.StreamUtils; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -120,7 +120,7 @@ private void appendObject(Object o, ByteArrayOutputStream baos) throws IOExcepti } else { value = o.toString(); } - ByteFragment.escape(value.getBytes(StreamUtils.UTF_8), baos); + ByteFragment.escape(value.getBytes(StandardCharsets.UTF_8), baos); } } diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResultSet.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResultSet.java new file mode 100644 index 000000000..944d06be6 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResultSet.java @@ -0,0 +1,849 @@ +package ru.yandex.clickhouse.response; + +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.sql.Array; +import java.sql.Date; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Time; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collections; +import java.util.List; +import java.util.TimeZone; +import ru.yandex.clickhouse.ClickHouseArray; +import ru.yandex.clickhouse.ClickHouseConnection; +import ru.yandex.clickhouse.ClickHouseStatement; +import ru.yandex.clickhouse.domain.ClickHouseDataType; +import ru.yandex.clickhouse.except.ClickHouseExceptionSpecifier; +import ru.yandex.clickhouse.except.ClickHouseUnknownException; +import ru.yandex.clickhouse.response.parser.ClickHouseValueParser; +import ru.yandex.clickhouse.settings.ClickHouseProperties; +import ru.yandex.clickhouse.util.ClickHouseArrayUtil; +import ru.yandex.clickhouse.util.ClickHouseBitmap; +import ru.yandex.clickhouse.util.ClickHouseValueFormatter; +import ru.yandex.clickhouse.util.Utils; + + +public class ClickHouseResultSet extends AbstractResultSet { + + private static final long[] EMPTY_LONG_ARRAY = new long[0]; + + private final TimeZone dateTimeTimeZone; + private final TimeZone dateTimeZone; + + private final StreamSplitter bis; + + private final String db; + private final String table; + + private List columns; + + private int maxRows; + + // current line + protected ByteFragment[] values; + // 1-based + private int lastReadColumn; + + // next line + protected ByteFragment nextLine; + + // total lines + private ByteFragment totalLine; + + // row counter + protected int rowNumber; + + // statement result set belongs to + private final ClickHouseStatement statement; + + private final ClickHouseProperties properties; + + private boolean usesWithTotals; + + // NOTE this can't be used for `isLast` impl because + // it does not do prefetch. It is effectively a witness + // to the fact that rs.next() returned false. + private boolean lastReached = false; + + private boolean isAfterLastReached = false; + + public ClickHouseResultSet(InputStream is, int bufferSize, String db, String table, + boolean usesWithTotals, ClickHouseStatement statement, TimeZone timeZone, + ClickHouseProperties properties) throws IOException + { + this.db = db; + this.table = table; + this.statement = statement; + this.properties = properties; + this.usesWithTotals = usesWithTotals; + this.dateTimeTimeZone = timeZone; + this.dateTimeZone = properties.isUseServerTimeZoneForDates() + ? timeZone + : TimeZone.getDefault(); // FIXME should be the timezone defined in useTimeZone? + bis = new StreamSplitter(is, (byte) 0x0A, bufferSize); /// \n + ByteFragment headerFragment = bis.next(); + if (headerFragment == null) { + throw new IllegalArgumentException("ClickHouse response without column names"); + } + String header = headerFragment.asString(true); + if (header.startsWith("Code: ") && !header.contains("\t")) { + is.close(); + throw new IOException("ClickHouse error: " + header); + } + String[] cols = toStringArray(headerFragment); + ByteFragment typesFragment = bis.next(); + if (typesFragment == null) { + throw new IllegalArgumentException("ClickHouse response without column types"); + } + String[] types = toStringArray(typesFragment); + columns = new ArrayList<>(cols.length); + TimeZone tz = null; + try { + if (statement != null && statement.getConnection() instanceof ClickHouseConnection) { + tz = ((ClickHouseConnection)statement.getConnection()).getServerTimeZone(); + } + } catch (SQLException e) { + // ignore the error + } + + if (tz == null) { + tz = timeZone; + } + + for (int i = 0; i < cols.length; i++) { + columns.add(ClickHouseColumnInfo.parse(types[i], cols[i], tz)); + } + } + + private static String[] toStringArray(ByteFragment headerFragment) { + ByteFragment[] split = headerFragment.split((byte) 0x09); + String[] c = new String[split.length]; + for (int i = 0; i < split.length; i++) { + String name = split[i].asString(true); + c[i] = name; + } + return c; + } + + /** + * Check if there is another row + * + * @return {@code true} if this result set has another row after the current + * cursor position, {@code false} else + * @throws SQLException + * if something goes wrong + * @deprecated prefer to use JDBC API methods, for example {@link #isLast()} + * or simply looping using {@code while (rs.next())} + */ + @Deprecated + public boolean hasNext() throws SQLException { + if (nextLine == null && !lastReached) { + try { + nextLine = bis.next(); + + if (nextLine == null || (maxRows != 0 && rowNumber >= maxRows) || (usesWithTotals && nextLine.length() == 0)) { + if (usesWithTotals) { + if (onTheSeparatorRow()) { + totalLine = bis.next(); + endOfStream(); + } // otherwise do not close the stream, it is single column or invalid result set case + } else { + endOfStream(); + } + } + } catch (IOException e) { + throw new SQLException(e); + } + } + return nextLine != null; + } + + @Override + public boolean isBeforeFirst() throws SQLException { + return rowNumber == 0 && hasNext(); + } + + @Override + public boolean isAfterLast() throws SQLException { + return isAfterLastReached; + } + + @Override + public boolean isFirst() throws SQLException { + return rowNumber == 1; + } + + @Override + public boolean isLast() throws SQLException { + return !hasNext(); + // && !isAfterLastReached should be probably added, + // but it may brake compatibility with the previous implementation + } + + private void endOfStream() throws IOException { + bis.close(); + lastReached = true; + nextLine = null; + } + + @Override + public boolean next() throws SQLException { + if (hasNext()) { + values = nextLine.split((byte) 0x09); + checkValues(columns, values, nextLine); + nextLine = null; + rowNumber += 1; + return true; + } + isAfterLastReached = true; + return false; + } + + private boolean onTheSeparatorRow() throws IOException { + // test bis vs "\n???\nEOF" pattern if not then rest to current position + bis.mark(); + boolean onSeparatorRow = bis.next() !=null && bis.next() == null; + bis.reset(); + + return onSeparatorRow; + } + + private static void checkValues(List columns, ByteFragment[] values, + ByteFragment fragment) throws SQLException + { + if (columns.size() != values.length) { + throw ClickHouseExceptionSpecifier.specify(fragment.asString()); + } + } + + @Override + public void close() throws SQLException { + try { + bis.close(); + } catch (IOException e) { + throw new SQLException(e); + } + } + + @Override + public boolean isClosed() throws SQLException { + try { + return bis.isClosed(); + } catch (IOException e) { + throw new SQLException(e); + } + } + + public void getTotals() throws SQLException { + if (!usesWithTotals) { + throw new IllegalStateException("Cannot get totals when totals are not being used."); + } + + nextLine = totalLine; + + this.next(); + } + + // this method is mocked in a test, do not make it final :-) + List getColumns() { + return Collections.unmodifiableList(columns); + } + + @Override + public ResultSetMetaData getMetaData() throws SQLException { + return new ClickHouseResultSetMetaData(this); + } + + @Override + public boolean wasNull() throws SQLException { + if (lastReadColumn == 0) { + throw new IllegalStateException("You should get something before check nullability"); + } + return getValue(lastReadColumn).isNull(); + } + + @Override + public int getInt(String column) throws SQLException { + return getInt(findColumn(column)); + } + + @Override + public boolean getBoolean(String column) throws SQLException { + return getBoolean(findColumn(column)); + } + + @Override + public long getLong(String column) throws SQLException { + return getLong(findColumn(column)); + } + + @Override + public String getString(String column) throws SQLException { + return getString(findColumn(column)); + } + + @Override + public byte[] getBytes(String column) throws SQLException { + return getBytes(findColumn(column)); + } + + @Override + public Timestamp getTimestamp(String column) throws SQLException { + return getTimestamp(findColumn(column)); + } + + @Override + public Timestamp getTimestamp(int columnIndex) throws SQLException { + ClickHouseColumnInfo columnInfo = getColumnInfo(columnIndex); + TimeZone tz = getEffectiveTimeZone(columnInfo); + + return ClickHouseValueParser.getParser(Timestamp.class).parse( + getValue(columnIndex), columnInfo, tz); + } + + private TimeZone getEffectiveTimeZone(ClickHouseColumnInfo columnInfo) { + TimeZone tz = null; + + if (columnInfo.getClickHouseDataType() == ClickHouseDataType.Date) { + tz = dateTimeZone; + } else { + tz = properties.isUseServerTimeZone() ? null : dateTimeTimeZone; + } + + return tz; + } + + @Override + public Timestamp getTimestamp(String column, Calendar cal) throws SQLException { + return getTimestamp(findColumn(column), cal); + } + + @Override + public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { + return getTimestamp(columnIndex); + } + + @Override + public short getShort(String column) throws SQLException { + return getShort(findColumn(column)); + } + + @Override + public byte getByte(String column) throws SQLException { + return getByte(findColumn(column)); + } + + @Override + public long[] getLongArray(String column) throws SQLException { + return getLongArray(findColumn(column)); + } + + @Override + public Array getArray(int columnIndex) throws SQLException { + ClickHouseColumnInfo colInfo = getColumnInfo(columnIndex); + if (colInfo.getClickHouseDataType() != ClickHouseDataType.Array) { + throw new SQLException("Column not an array"); + } + + final Object array; + switch (colInfo.getArrayBaseType()) { + case Date : + array = ClickHouseArrayUtil.parseArray( + getValue(columnIndex), + properties.isUseObjectsInArrays(), + dateTimeZone, + colInfo + ); + break; + default : + TimeZone timeZone = colInfo.getTimeZone() != null + ? colInfo.getTimeZone() + : dateTimeTimeZone; + array = ClickHouseArrayUtil.parseArray( + getValue(columnIndex), + properties.isUseObjectsInArrays(), + timeZone, + colInfo + ); + break; + } + return new ClickHouseArray(colInfo.getArrayBaseType(), array); + } + + @Override + public Array getArray(String column) throws SQLException { + return getArray(findColumn(column)); + } + + @Override + public double getDouble(String columnLabel) throws SQLException { + return getDouble(findColumn(columnLabel)); + } + + @Override + public float getFloat(String columnLabel) throws SQLException { + return getFloat(findColumn(columnLabel)); + } + + @Override + public Date getDate(String columnLabel) throws SQLException { + return getDate(findColumn(columnLabel)); + } + + @Override + public Time getTime(String columnLabel) throws SQLException { + return getTime(findColumn(columnLabel)); + } + + @Override + public Object getObject(String columnLabel) throws SQLException { + return getObject(findColumn(columnLabel)); + } + + @Override + public String getString(int colNum) throws SQLException { + // FIXME this won't help when datetime string is in a nested structure + ClickHouseColumnInfo columnInfo = getColumnInfo(colNum); + ByteFragment value = getValue(colNum); + ClickHouseDataType dataType = columnInfo.getClickHouseDataType(); + + // Date is time-zone netural so let's skip that. + // DateTime string returned from Server however is always formatted using server/column + // timezone. The behaviour may change when + // https://github.com/ClickHouse/ClickHouse/issues/4548 is addressed + if (!properties.isUseServerTimeZone() && ( + dataType == ClickHouseDataType.DateTime + || dataType == ClickHouseDataType.DateTime32 + || dataType == ClickHouseDataType.DateTime64)) { + TimeZone serverTimeZone = columnInfo.getTimeZone(); + if (serverTimeZone == null) { + serverTimeZone = ((ClickHouseConnection) getStatement().getConnection()).getServerTimeZone(); + } + TimeZone clientTimeZone = Utils.isNullOrEmptyString(properties.getUseTimeZone()) + ? TimeZone.getDefault() + : TimeZone.getTimeZone(properties.getUseTimeZone()); + + if (!clientTimeZone.equals(serverTimeZone)) { + Timestamp newTs = ClickHouseValueParser.getParser(Timestamp.class) + .parse(value, columnInfo, serverTimeZone); + value = ByteFragment.fromString(ClickHouseValueFormatter.formatTimestamp(newTs, clientTimeZone)); + } + } + + return ClickHouseValueParser.getParser(String.class).parse(value, columnInfo, null); + } + + @Override + public int getInt(int colNum) throws SQLException { + return ClickHouseValueParser.parseInt( + getValue(colNum), getColumnInfo(colNum)); + } + + @Override + public boolean getBoolean(int colNum) throws SQLException { + return ClickHouseValueParser.parseBoolean( + getValue(colNum), getColumnInfo(colNum)); + } + + @Override + public long getLong(int colNum) throws SQLException { + return ClickHouseValueParser.parseLong( + getValue(colNum), getColumnInfo(colNum)); + } + + @Override + public byte[] getBytes(int colNum) { + return toBytes(getValue(colNum)); + } + + /** + * Tries to parse the value as a timestamp using the connection time zone if + * applicable and return its representation as milliseconds since epoch. + * + * @param colNum + * column number + * @return timestamp value as milliseconds since epoch + * @deprecated prefer to use regular JDBC API methods, e.g. + * {@link #getTimestamp(int)} or {@link #getObject(int, Class)} + * using {@link Instant} + */ + @Deprecated + public Long getTimestampAsLong(int colNum) { + ClickHouseColumnInfo columnInfo = getColumnInfo(colNum); + TimeZone tz = getEffectiveTimeZone(columnInfo); + return getTimestampAsLong(colNum, tz); + } + + /** + * Tries to parse the value as a timestamp and return its representation as + * milliseconds since epoch + * + * @param colNum + * the column number + * @param timeZone + * time zone to use when parsing date / date time values + * @return value interpreted as timestamp as milliseconds since epoch + * @deprecated prefer to use regular JDBC API method + */ + @Deprecated + public Long getTimestampAsLong(int colNum, TimeZone timeZone) { + ByteFragment value = getValue(colNum); + if (value.isNull() || value.asString().equals("0000-00-00 00:00:00")) { + return null; + } + try { + Instant instant = ClickHouseValueParser.getParser(Instant.class) + .parse(value, getColumnInfo(colNum), timeZone); + return Long.valueOf(instant.toEpochMilli()); + } catch (SQLException sqle) { + throw new RuntimeException(sqle); + } + } + + @Override + public short getShort(int colNum) throws SQLException { + return ClickHouseValueParser.parseShort( + getValue(colNum), getColumnInfo(colNum)); + } + + @Override + public byte getByte(int colNum) { + return toByte(getValue(colNum)); + } + + /** + * Parse the value in current row at column index {@code colNum} as an array + * of long + * + * @param colNum + * column number + * @return an array of longs + * @throws SQLException + * if the value cannot be interpreted as {@code long[]} + * @deprecated prefer to use regular JDBC API + */ + @Deprecated + public long[] getLongArray(int colNum) throws SQLException { + return toLongArray(getValue(colNum), getColumnInfo(colNum)); + } + + @Override + public float getFloat(int columnIndex) throws SQLException { + return ClickHouseValueParser.parseFloat( + getValue(columnIndex), getColumnInfo(columnIndex)); + } + + @Override + public double getDouble(int columnIndex) throws SQLException { + return ClickHouseValueParser.parseDouble( + getValue(columnIndex), getColumnInfo(columnIndex)); + } + + @Override + public Statement getStatement() { + return statement; + } + + @Override + public Date getDate(int columnIndex) throws SQLException { + ClickHouseColumnInfo columnInfo = getColumnInfo(columnIndex); + TimeZone tz = getEffectiveTimeZone(columnInfo); + return ClickHouseValueParser.getParser(Date.class).parse( + getValue(columnIndex), columnInfo, tz); + } + + @Override + public Date getDate(int columnIndex, Calendar calendar) throws SQLException { + return getDate(columnIndex); + } + + @Override + public Time getTime(int columnIndex) throws SQLException { + ClickHouseColumnInfo columnInfo = getColumnInfo(columnIndex); + TimeZone tz = getEffectiveTimeZone(columnInfo); + return ClickHouseValueParser.getParser(Time.class).parse( + getValue(columnIndex), columnInfo, tz); + } + + @Override + public Time getTime(int columnIndex, Calendar calendar) throws SQLException { + return getTime(columnIndex); + } + + @Override + public Object getObject(int columnIndex) throws SQLException { + try { + if (getValue(columnIndex).isNull()) { + return null; + } + ClickHouseColumnInfo columnInfo = getColumnInfo(columnIndex); + ClickHouseDataType chType = columnInfo.getClickHouseDataType(); + switch (chType.getSqlType()) { + case Types.BIGINT: + if (chType == ClickHouseDataType.UInt64) { + return getObject(columnIndex, BigInteger.class); + } + return getObject(columnIndex, Long.class); + case Types.INTEGER: + if (!chType.isSigned()){ + return getObject(columnIndex, Long.class); + } + return getObject(columnIndex, Integer.class); + case Types.TINYINT: + case Types.SMALLINT: + return getObject(columnIndex, Integer.class); + case Types.VARCHAR: return getString(columnIndex); + case Types.REAL: return getObject(columnIndex, Float.class); + case Types.FLOAT: + case Types.DOUBLE: return getObject(columnIndex, Double.class); + case Types.DATE: return getDate(columnIndex); + case Types.TIMESTAMP: return getTimestamp(columnIndex); + case Types.BLOB: return getString(columnIndex); + case Types.ARRAY: return getArray(columnIndex); + case Types.DECIMAL: return getBigDecimal(columnIndex); + case Types.NUMERIC: return getBigInteger(columnIndex); + default: + // do not return + } + switch (chType) { + // case Array: + // case Tuple: + case AggregateFunction: + // TODO support more functions + if ("groupBitmap".equals(columnInfo.getFunctionName())) { + ClickHouseDataType innerType = columnInfo.getArrayBaseType(); + switch (innerType) { + // seems signed integers are not supported in ClickHouse + case Int8: + case Int16: + case Int32: + case Int64: + case UInt8: + case UInt16: + case UInt32: + case UInt64: + return getObject(columnIndex, ClickHouseBitmap.class); + default: + break; + } + } + return getString(columnIndex); + case Map: + case UUID : + return getObject(columnIndex, chType.getJavaClass()); + default : + return getString(columnIndex); + } + } catch (Exception e) { + throw new ClickHouseUnknownException( + "Parse exception: " + values[columnIndex - 1].toString(), + e); + } + } + + ///////////////////////////////////////////////////////// + + private static byte toByte(ByteFragment value) { + if (value.isNull()) { + return 0; + } + return Byte.parseByte(value.asString()); + } + + private static byte[] toBytes(ByteFragment value) { + if (value.isNull()) { + return null; + } + return value.unescape(); + } + + static long[] toLongArray(ByteFragment value, ClickHouseColumnInfo columnInfo) throws SQLException { + if (value.isNull()) { + return null; + } + if (value.charAt(0) != '[' || value.charAt(value.length() - 1) != ']') { + throw new IllegalArgumentException("not an array: " + value); + } + if (value.length() == 2) { + return EMPTY_LONG_ARRAY; + } + ByteFragment trim = value.subseq(1, value.length() - 2); + ByteFragment[] values = trim.split((byte) ','); + long[] result = new long[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = ClickHouseValueParser.parseLong(values[i], columnInfo); + } + return result; + } + + ////// + + @Override + public int getType() throws SQLException { + return TYPE_FORWARD_ONLY; + } + + @Override + public int getRow() throws SQLException { + return rowNumber; + } + + public String getDb() { + return db; + } + + public String getTable() { + return table; + } + + public void setMaxRows(int maxRows) { + this.maxRows = maxRows; + } + + ///// + + // 1-based index in column list + @Override + public int findColumn(String column) throws SQLException { + if (column == null || column.isEmpty()) { + throw new ClickHouseUnknownException( + "column name required", null); + } + for (int i = 0; i < columns.size(); i++) { + if (column.equalsIgnoreCase(columns.get(i).getColumnName())) { + return i+1; + } + } + throw new SQLException("no column " + column + " in columns list " + getColumnNamesString()); + } + + private ByteFragment getValue(int colNum) { + lastReadColumn = colNum; + return values[colNum - 1]; + } + + private ClickHouseColumnInfo getColumnInfo(int colNum) { + return columns.get(colNum - 1); + } + + @SuppressWarnings("unchecked") + @Override + public T getObject(int columnIndex, Class type) throws SQLException { + if (String.class.equals(type)) { + return (T) getString(columnIndex); + } + + ClickHouseColumnInfo columnInfo = getColumnInfo(columnIndex); + TimeZone tz = getEffectiveTimeZone(columnInfo); + return columnInfo.isArray() + ? (Array.class.isAssignableFrom(type) ? (T) getArray(columnIndex) : (T) getArray(columnIndex).getArray()) + : ClickHouseValueParser.getParser(type).parse(getValue(columnIndex), columnInfo, tz); + } + + @Override + public T getObject(String columnLabel, Class type) throws SQLException { + return getObject(findColumn(columnLabel), type); + } + + /** + * Retrieve the results in "raw" form. + * + * @return the results as an array of {@link ByteFragment}s + * @deprecated prefer to use regular JDBC API to retrieve the results + */ + @Deprecated + public ByteFragment[] getValues() { + return values; + } + + @Override + public BigDecimal getBigDecimal(String columnLabel) throws SQLException { + return getBigDecimal(findColumn(columnLabel)); + } + + @Override + public BigDecimal getBigDecimal(int columnIndex) throws SQLException { + return ClickHouseValueParser.getParser(BigDecimal.class) + .parse(getValue(columnIndex), getColumnInfo(columnIndex), null); + } + + @Override + public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException { + return getBigDecimal(findColumn(columnLabel), scale); + } + + @Override + public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { + BigDecimal result = ClickHouseValueParser.getParser(BigDecimal.class) + .parse(getValue(columnIndex), getColumnInfo(columnIndex), null); + return result != null + ? result.setScale(scale, RoundingMode.HALF_UP) + : null; + } + + public BigInteger getBigInteger(String columnLabel) throws SQLException { + return getBigInteger(findColumn(columnLabel)); + } + + public BigInteger getBigInteger(int columnIndex) throws SQLException { + BigDecimal dec = getBigDecimal(columnIndex); + return dec == null ? null : dec.toBigInteger(); + } + + public String[] getColumnNames() { + String[] columnNames = new String[columns.size()]; + for (int i = 0; i < columns.size(); ++i) { + columnNames[i] = columns.get(i).getColumnName(); + } + return columnNames; + } + + @Override + public void setFetchDirection(int direction) throws SQLException { + // ignore perfomance hint + } + + @Override + public void setFetchSize(int rows) throws SQLException { + // ignore perfomance hint + } + + + @Override + public String toString() { + return "ClickHouseResultSet{" + + "dateTimeTimeZone=" + dateTimeTimeZone.toString() + + ", dateTimeZone=" + dateTimeZone.toString() + + ", bis=" + bis + + ", db='" + db + '\'' + + ", table='" + table + '\'' + + ", columns=" + getColumnNamesString() + + ", maxRows=" + maxRows + + ", values=" + Arrays.toString(values) + + ", lastReadColumn=" + lastReadColumn + + ", nextLine=" + nextLine + + ", rowNumber=" + rowNumber + + ", statement=" + statement + + '}'; + } + + private String getColumnNamesString() { + StringBuilder sb = new StringBuilder(); + for (ClickHouseColumnInfo info : columns) { + sb.append(info.getColumnName()).append(' '); + } + return sb.substring(0, sb.length() - 1); + } + +} diff --git a/src/main/java/ru/yandex/clickhouse/response/ClickHouseResultSetMetaData.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResultSetMetaData.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/response/ClickHouseResultSetMetaData.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseResultSetMetaData.java diff --git a/src/main/java/ru/yandex/clickhouse/response/ClickHouseScrollableResultSet.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseScrollableResultSet.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/response/ClickHouseScrollableResultSet.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseScrollableResultSet.java diff --git a/src/main/java/ru/yandex/clickhouse/response/FastByteArrayInputStream.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/FastByteArrayInputStream.java similarity index 100% rename from src/main/java/ru/yandex/clickhouse/response/FastByteArrayInputStream.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/FastByteArrayInputStream.java diff --git a/src/main/java/ru/yandex/clickhouse/response/FastByteArrayOutputStream.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/FastByteArrayOutputStream.java similarity index 96% rename from src/main/java/ru/yandex/clickhouse/response/FastByteArrayOutputStream.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/FastByteArrayOutputStream.java index c890334f3..b8e7321f8 100644 --- a/src/main/java/ru/yandex/clickhouse/response/FastByteArrayOutputStream.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/FastByteArrayOutputStream.java @@ -117,9 +117,9 @@ public int size() { /** - * Closing a ByteArrayOutputStream has no effect. The methods in + * Closing a {@code ByteArrayOutputStream} has no effect. The methods in * this class can be called after the stream has been closed without - * generating an IOException. + * generating an {@code IOException}. */ @Override public void close() throws IOException { @@ -156,7 +156,7 @@ public void copyTo(DataOutput dest) throws IOException { * @return a input stream contained all bytes recorded in a current stream */ public FastByteArrayInputStream convertToInputStream() { - return new FastByteArrayInputStream(buf, count); + return new FastByteArrayInputStream(buf, count); } public ByteBuffer toByteBuffer() { diff --git a/src/main/java/ru/yandex/clickhouse/response/StreamSplitter.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/StreamSplitter.java similarity index 73% rename from src/main/java/ru/yandex/clickhouse/response/StreamSplitter.java rename to clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/StreamSplitter.java index 2480e41fa..b6d857f49 100644 --- a/src/main/java/ru/yandex/clickhouse/response/StreamSplitter.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/StreamSplitter.java @@ -1,15 +1,15 @@ package ru.yandex.clickhouse.response; -import ru.yandex.clickhouse.util.guava.StreamUtils; - import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; /** * We have a stream of bytes and a separator as an input. * We split the stream by the separator and pass the byte arrays to output. */ public class StreamSplitter { + private static final int MAX_ARRAY_LENGTH = Integer.MAX_VALUE - 8; private static final int buflen = 65536; // initial parameters @@ -29,7 +29,6 @@ public class StreamSplitter { private boolean closed; - public StreamSplitter(ByteFragment bf, byte sep) { this.delegate = bf.asStream(); this.sep = sep; @@ -44,7 +43,7 @@ public StreamSplitter(InputStream delegate, byte sep, int buflen) { } public StreamSplitter(InputStream delegate, byte sep) { - this(delegate,sep, buflen); + this(delegate, sep, buflen); } public ByteFragment next() throws IOException { @@ -52,17 +51,17 @@ public ByteFragment next() throws IOException { if (posNext >= posRead) { // need to read more from the stream int readBytes = readFromStream(); - if(readBytes <= 0) { + if (readBytes <= 0) { // if everything was sent out and there is nothing left in the stream return null; } } // looking for the separator int positionSep; - while((positionSep = indexOf(buf, sep, posNext, posRead)) < posNext) { + while ((positionSep = indexOf(buf, sep, posNext, posRead)) < posNext) { // read from stream till we find the separator int readBytes = readFromStream(); - if(readBytes <= 0) { + if (readBytes <= 0) { // if there is nothing to read, return everything left as a result positionSep = posRead; break; @@ -70,8 +69,8 @@ public ByteFragment next() throws IOException { } // if the separator is found, return the fragment int fragmentStart = posNext; - posNext = positionSep+1; - return new ByteFragment(buf, fragmentStart, positionSep-fragmentStart); + posNext = positionSep + 1; + return new ByteFragment(buf, fragmentStart, positionSep - fragmentStart); } // if there is no separator in read but not sent fragment - read more data @@ -81,8 +80,9 @@ protected int readFromStream() throws IOException { return -1; } else { int read = delegate.read(buf, posRead, buf.length - posRead); - if(read > 0) + if (read > 0) { posRead += read; + } return read; } } else { @@ -90,37 +90,43 @@ protected int readFromStream() throws IOException { shiftOrResize(); } int read = delegate.read(buf, posRead, buf.length - posRead); - if(read > 0) + if (read > 0) { posRead += read; + } return read; } } - // if we have read till the end of buffer, we have to create a new buffer and move data by posNext (already send data position) + // if we have read till the end of buffer, we have to create a new buffer + // and move data by posNext (already send data position) // if there is no sent data and buffer is still full - expand the buffer private void shiftOrResize() { - if(posNext > 0) { + if (posNext > 0) { byte[] oldBuf = buf; buf = new byte[buf.length]; - System.arraycopy(oldBuf, posNext, buf, 0, oldBuf.length-posNext); + System.arraycopy(oldBuf, posNext, buf, 0, oldBuf.length - posNext); posRead -= posNext; posNext = 0; } else { byte[] oldBuf = buf; - buf = new byte[buf.length*2]; + int len = buf.length * 2; + if (len > MAX_ARRAY_LENGTH) { + len = MAX_ARRAY_LENGTH; + } + buf = new byte[len]; System.arraycopy(oldBuf, 0, buf, 0, oldBuf.length); } } private static int indexOf(byte[] array, byte target, int start, int end) { - for (int i = start; i < end; i++) { - if (array[i] == target) { - return i; - } - } - return -1; - } + for (int i = start; i < end; i++) { + if (array[i] == target) { + return i; + } + } + return -1; + } public void close() throws IOException { closed = true; @@ -133,25 +139,25 @@ public boolean isClosed() throws IOException { @Override public String toString() { - String bufStr = new String(buf, StreamUtils.UTF_8).trim(); - - return "StreamSplitter{" + - "delegate=" + delegate + - ", sep=" + sep + - ", buf=" + bufStr + - ", posRead=" + posRead + - ", posNext=" + posNext + - ", readOnce=" + readOnce + - '}'; + String bufStr = new String(buf, StandardCharsets.UTF_8).trim(); + + return "StreamSplitter{" + + "delegate=" + delegate + + ", sep=" + sep + + ", buf=" + bufStr + + ", posRead=" + posRead + + ", posNext=" + posNext + + ", readOnce=" + readOnce + + '}'; } - public void mark() { + public void mark() { markedRead = posRead; markedNext = posNext; - } + } - public void reset() { + public void reset() { posRead = markedRead; posNext = markedNext; - } + } } diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseArrayParser.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseArrayParser.java new file mode 100644 index 000000000..20b3afed4 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseArrayParser.java @@ -0,0 +1,59 @@ +package ru.yandex.clickhouse.response.parser; + +import java.sql.Array; +import java.sql.SQLException; +import java.util.TimeZone; + +import ru.yandex.clickhouse.ClickHouseArray; +import ru.yandex.clickhouse.domain.ClickHouseDataType; +import ru.yandex.clickhouse.response.ByteFragment; +import ru.yandex.clickhouse.response.ClickHouseColumnInfo; +import ru.yandex.clickhouse.util.ClickHouseArrayUtil; + +final class ClickHouseArrayParser extends ClickHouseValueParser { + + private static ClickHouseArrayParser instance; + + static ClickHouseArrayParser getInstance() { + if (instance == null) { + instance = new ClickHouseArrayParser(); + } + return instance; + } + + private ClickHouseArrayParser() { + // prevent instantiation + } + + @Override + public Array parse(ByteFragment value, ClickHouseColumnInfo columnInfo, TimeZone resultTimeZone) + throws SQLException { + if (columnInfo.getClickHouseDataType() != ClickHouseDataType.Array) { + throw new SQLException("Column not an array"); + } + + if (value.isNull()) { + return null; + } + + final Object array; + switch (columnInfo.getArrayBaseType()) { + case Date: + // FIXME: properties.isUseObjectsInArrays() + array = ClickHouseArrayUtil.parseArray(value, false, resultTimeZone, columnInfo); + break; + default: + // properties.isUseObjectsInArrays() + TimeZone timeZone = columnInfo.getTimeZone() != null ? columnInfo.getTimeZone() : resultTimeZone; + array = ClickHouseArrayUtil.parseArray(value, false, timeZone, columnInfo); + break; + } + + return new ClickHouseArray(columnInfo.getArrayBaseType(), array); + } + + @Override + protected Array getDefaultValue() { + return null; + } +} diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseBitmapParser.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseBitmapParser.java new file mode 100644 index 000000000..a58f40ed3 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseBitmapParser.java @@ -0,0 +1,53 @@ +package ru.yandex.clickhouse.response.parser; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.TimeZone; +import ru.yandex.clickhouse.response.ByteFragment; +import ru.yandex.clickhouse.response.ClickHouseColumnInfo; +import ru.yandex.clickhouse.util.ClickHouseBitmap; + +final class ClickHouseBitmapParser extends ClickHouseValueParser { + private static ClickHouseBitmapParser instance; + + static ClickHouseBitmapParser getInstance() { + if (instance == null) { + instance = new ClickHouseBitmapParser(); + } + return instance; + } + + private ClickHouseBitmapParser() { + // prevent instantiation + } + + @Override + public ClickHouseBitmap parse(ByteFragment value, ClickHouseColumnInfo columnInfo, TimeZone resultTimeZone) + throws SQLException { + if (value.isNull()) { + return null; + } + + // https://github.com/ClickHouse/ClickHouse/blob/master/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h#L100 + ClickHouseBitmap rb = ClickHouseBitmap.wrap(); + + // FIXME use DataInput/DataOutput for stream after switching to RowBinary + byte[] bytes = value.unescape(); + if (bytes.length == 0) { + return rb; + } + + try { + rb = ClickHouseBitmap.deserialize(bytes, columnInfo.getArrayBaseType()); + } catch (IOException e) { + throw new SQLException("Failed to deserialize ClickHouseBitmap", e); + } + + return rb; + } + + @Override + protected ClickHouseBitmap getDefaultValue() { + return ClickHouseBitmap.wrap(); + } +} diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseDateValueParser.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseDateValueParser.java new file mode 100644 index 000000000..6f46b39b1 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseDateValueParser.java @@ -0,0 +1,240 @@ +package ru.yandex.clickhouse.response.parser; + +import java.math.BigInteger; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.format.ResolverStyle; +import java.util.Objects; +import java.util.TimeZone; +import java.util.regex.Pattern; +import ru.yandex.clickhouse.except.ClickHouseException; +import ru.yandex.clickhouse.except.ClickHouseUnknownException; +import ru.yandex.clickhouse.response.ByteFragment; +import ru.yandex.clickhouse.response.ClickHouseColumnInfo; + +abstract class ClickHouseDateValueParser extends ClickHouseValueParser { + private static final Pattern PATTERN_EMPTY_DATE = + Pattern.compile("^(0000-00-00|0000-00-00 00:00:00|0)$"); + + private static final DateTimeFormatter DATE_FORMATTER = + DateTimeFormatter.ofPattern("yyyy[-]MM[-]dd"); + private static final DateTimeFormatter DATE_TIME_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd['T'][ ]HH:mm:ss"); + private static final DateTimeFormatter TIME_FORMATTER_NUMBERS = + DateTimeFormatter.ofPattern("HH[mm][ss]") + .withResolverStyle(ResolverStyle.STRICT); + + private final Class clazz; + + protected ClickHouseDateValueParser(Class clazz) { + this.clazz = Objects.requireNonNull(clazz); + } + + protected LocalDateTime dateToLocalDate(String value, ClickHouseColumnInfo columnInfo, TimeZone timeZone) { + return parseAsLocalDate(value).atStartOfDay(); + } + + protected LocalDateTime dateTimeToLocalDateTime(String value, ClickHouseColumnInfo columnInfo, TimeZone timeZone) { + TimeZone serverTimeZone = columnInfo.getTimeZone(); + LocalDateTime localDateTime = parseAsLocalDateTime(value); + if (serverTimeZone != null + && (serverTimeZone.useDaylightTime() || serverTimeZone.getRawOffset() > 0)) { // non-UTC + localDateTime = localDateTime.atZone(columnInfo.getTimeZone().toZoneId()) + .withZoneSameInstant(java.time.ZoneId.of("UTC")).toLocalDateTime(); + } + + return localDateTime; + } + + protected ZonedDateTime dateToZonedDateTime(String value, ClickHouseColumnInfo columnInfo, TimeZone timeZone) { + LocalDate localDate = parseAsLocalDate(value); + return localDate.atStartOfDay(timeZone != null ? timeZone.toZoneId() : ZoneId.systemDefault()); + } + + protected ZonedDateTime dateTimeToZonedDateTime(String value, ClickHouseColumnInfo columnInfo, TimeZone timeZone) { + LocalDateTime localDateTime = parseAsLocalDateTime(value); + return timeZone != null && !timeZone.equals(columnInfo.getTimeZone()) + ? localDateTime.atZone(columnInfo.getTimeZone().toZoneId()).withZoneSameInstant(timeZone.toZoneId()) + : localDateTime.atZone(columnInfo.getTimeZone().toZoneId()); + } + + @Override + public T parse(ByteFragment value, ClickHouseColumnInfo columnInfo, TimeZone timeZone) throws ClickHouseException { + + if (value.isNull()) { + return null; + } + + String s = value.asString(); + + /* + * filter default values for relevant data types, + * even if the column has nullable flag set. + */ + if (PATTERN_EMPTY_DATE.matcher(s).matches()) { + return null; + } + + switch (columnInfo.getEffectiveClickHouseDataType()) { + case Date: + try { + return parseDate(s, columnInfo, timeZone); + } catch (Exception e) { + throw new ClickHouseUnknownException( + "Error parsing '" + s + "' of data type '" + + columnInfo.getOriginalTypeName() + + "' as " + clazz.getName(), + e); + } + case DateTime: + case DateTime32: + case DateTime64: + try { + return parseDateTime(s, columnInfo, timeZone); + } catch (Exception e) { + throw new ClickHouseUnknownException( + "Error parsing '" + s + "' of data type '" + + columnInfo.getOriginalTypeName() + + "' as " + clazz.getName(), + e); + } + case Int8: + case Int16: + case Int32: + case Int64: + case UInt8: + case UInt16: + case UInt32: + try { + long l = Long.parseLong(s); + return parseNumber(l, columnInfo, timeZone); + } catch (Exception e) { + throw new ClickHouseUnknownException( + "Error parsing '" + s + "' of data type '" + + columnInfo.getOriginalTypeName() + + "' as " + clazz.getName(), + e); + } + case UInt64: + // If we have a large nanos value, we trim to millis + try { + BigInteger bi = new BigInteger(s); + if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { + return parseNumber( + bi.divide(BigInteger.valueOf(1000_000L)).longValue(), + columnInfo, + timeZone); + } + return parseNumber(bi.longValue(), columnInfo, timeZone); + } catch (Exception e) { + throw new ClickHouseUnknownException( + "Error parsing '" + s + "' of data type '" + + columnInfo.getOriginalTypeName() + + "' as " + clazz.getName(), + e); + } + case String: + case Unknown: + try { + return parseOther(s, columnInfo, timeZone); + } catch (Exception e) { + throw new ClickHouseUnknownException( + "Error parsing '" + s + "' as " + clazz.getName(), e); + } + default: + throw new ClickHouseUnknownException( + "Error parsing '" + s + "' of data type '" + + columnInfo.getOriginalTypeName() + + "' as " + clazz.getName(), + null); + } + + } + + abstract T parseDate(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone); + + abstract T parseDateTime(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone); + + abstract T parseNumber(long value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone); + + abstract T parseOther(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone); + + protected final ZoneId effectiveTimeZone(ClickHouseColumnInfo columnInfo, + TimeZone timeZone) { + return timeZone != null + ? timeZone.toZoneId() + : columnInfo.getTimeZone() != null + ? columnInfo.getTimeZone().toZoneId() + : ZoneId.systemDefault(); + } + + protected final LocalDate parseAsLocalDate(String value) { + return LocalDate.parse(value, DATE_FORMATTER); + } + + protected final LocalDateTime parseAsLocalDateTime(String value) { + int index = Objects.requireNonNull(value).indexOf('.'); + if (index > 0) { + int endIndex = -1; + for (int i = index + 1, len = value.length(); i < len; i++) { + char ch = value.charAt(i); + if (!Character.isDigit(ch)) { + endIndex = i; + break; + } + } + String part1 = value.substring(0, index); + if (endIndex > index) { + part1 += value.substring(endIndex); + } + String part2 = endIndex > index ? value.substring(index, endIndex) : value.substring(index); + + LocalDateTime ts = LocalDateTime.parse(part1, DATE_TIME_FORMATTER); + int nanoSeconds = (int) Math.round(Double.parseDouble(part2) * 1000000000); + return LocalDateTime.of(ts.getYear(), ts.getMonth(), ts.getDayOfMonth(), + ts.getHour(), ts.getMinute(), ts.getSecond(), nanoSeconds); + } + + return LocalDateTime.parse(value, DATE_TIME_FORMATTER); + } + + protected final OffsetDateTime parseAsOffsetDateTime(String value) { + return OffsetDateTime.parse(value, DateTimeFormatter.ISO_DATE_TIME); + } + + protected final Instant parseAsInstant(String value) { + try { + long l = Long.parseLong(value); + return parseAsInstant(l); + } catch (NumberFormatException nfe) { + throw new DateTimeParseException("unparsable as long", value, -1, nfe); + } + } + + protected final Instant parseAsInstant(long value) { + return value > Integer.MAX_VALUE + ? Instant.ofEpochMilli(value) + : Instant.ofEpochSecond(value); + } + + protected final LocalTime parseAsLocalTime(String value) { + return LocalTime.parse( + value.length() % 2 == 0 ? value : "0" + value, + TIME_FORMATTER_NUMBERS); + } + + protected final LocalTime parseAsLocalTime(long value) { + return parseAsLocalTime(String.valueOf(value)); + } +} diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseDoubleParser.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseDoubleParser.java new file mode 100644 index 000000000..86c691b5c --- /dev/null +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseDoubleParser.java @@ -0,0 +1,58 @@ +package ru.yandex.clickhouse.response.parser; + +import java.sql.SQLException; +import java.util.TimeZone; + +import ru.yandex.clickhouse.except.ClickHouseUnknownException; +import ru.yandex.clickhouse.response.ByteFragment; +import ru.yandex.clickhouse.response.ClickHouseColumnInfo; + +final class ClickHouseDoubleParser extends ClickHouseValueParser { + + private static ClickHouseDoubleParser instance; + + static ClickHouseDoubleParser getInstance() { + if (instance == null) { + instance = new ClickHouseDoubleParser(); + } + return instance; + } + + private ClickHouseDoubleParser() { + // prevent instantiation + } + + @Override + public Double parse(ByteFragment value, ClickHouseColumnInfo columnInfo, + TimeZone resultTimeZone) throws SQLException + { + if (value.isNull()) { + return null; + } + if (value.isNaN()) { + return Double.valueOf(Double.NaN); + } + String s = value.asString(); + switch (s) { + case "+inf": + case "inf": + return Double.valueOf(Double.POSITIVE_INFINITY); + case "-inf": + return Double.valueOf(Double.NEGATIVE_INFINITY); + default: + try { + return Double.valueOf(s); + } catch (NumberFormatException nfe) { + throw new ClickHouseUnknownException( + "Error parsing '" + s + "' as Double", + nfe); + } + } + } + + @Override + protected Double getDefaultValue() { + return Double.valueOf(0); + } + +} diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseInstantParser.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseInstantParser.java new file mode 100644 index 000000000..a770a0c5e --- /dev/null +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseInstantParser.java @@ -0,0 +1,74 @@ +package ru.yandex.clickhouse.response.parser; + +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeParseException; +import java.util.TimeZone; + +import ru.yandex.clickhouse.response.ClickHouseColumnInfo; + +final class ClickHouseInstantParser extends ClickHouseDateValueParser { + + private static ClickHouseInstantParser instance; + + static ClickHouseInstantParser getInstance() { + if (instance == null) { + instance = new ClickHouseInstantParser(); + } + return instance; + } + + private ClickHouseInstantParser() { + super(Instant.class); + } + + @Override + Instant parseDate(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return parseAsLocalDate(value).atStartOfDay().toInstant(ZoneOffset.UTC); + } + + @Override + Instant parseDateTime(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return dateTimeToZonedDateTime(value, columnInfo, timeZone).toInstant(); + } + + @Override + Instant parseNumber(long value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return value > Integer.MAX_VALUE + ? Instant.ofEpochMilli(value) + : Instant.ofEpochSecond(value); + } + + @Override + Instant parseOther(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + try { + return parseAsLocalDate(value) + .atStartOfDay(effectiveTimeZone(columnInfo, timeZone)) + .toInstant(); + } catch (DateTimeParseException dtpe) { + // better luck next time + } + try { + return parseAsLocalDateTime(value) + .atZone(effectiveTimeZone(columnInfo, timeZone)) + .toInstant(); + } catch (DateTimeParseException dtpe) { + // better luck next time + } + try { + return parseAsOffsetDateTime(value) + .toInstant(); + } catch (DateTimeParseException dtpe) { + // better luck next time + } + return parseAsInstant(value); + } +} diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseLocalDateParser.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseLocalDateParser.java new file mode 100644 index 000000000..34d3ad3b3 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseLocalDateParser.java @@ -0,0 +1,64 @@ +package ru.yandex.clickhouse.response.parser; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.format.DateTimeParseException; +import java.util.TimeZone; + +import ru.yandex.clickhouse.response.ClickHouseColumnInfo; + +final class ClickHouseLocalDateParser extends ClickHouseDateValueParser { + + private static ClickHouseLocalDateParser instance; + + static ClickHouseLocalDateParser getInstance() { + if (instance == null) { + instance = new ClickHouseLocalDateParser(); + } + return instance; + } + + private ClickHouseLocalDateParser() { + super(LocalDate.class); + } + + @Override + LocalDate parseDate(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return dateToLocalDate(value, columnInfo, timeZone).toLocalDate(); + } + + @Override + LocalDate parseDateTime(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return dateTimeToLocalDateTime(value, columnInfo, timeZone).toLocalDate(); + } + + @Override + LocalDate parseNumber(long value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return parseAsInstant(value).atZone(timeZone.toZoneId()).toLocalDate(); + } + + @Override + LocalDate parseOther(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + try { + return parseAsLocalDate(value); + } catch (DateTimeParseException dtpe) { + // not parseable as date + } + try { + return parseAsLocalDateTime(value).toLocalDate(); + } catch (DateTimeParseException dtpe) { + // not parseable as datetime + } + Instant i = parseAsInstant(value); + return i.atZone(timeZone.toZoneId()).toLocalDate(); + } + +} diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseLocalDateTimeParser.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseLocalDateTimeParser.java new file mode 100644 index 000000000..34080f73f --- /dev/null +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseLocalDateTimeParser.java @@ -0,0 +1,66 @@ +package ru.yandex.clickhouse.response.parser; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.format.DateTimeParseException; +import java.util.TimeZone; + +import ru.yandex.clickhouse.response.ClickHouseColumnInfo; + +final class ClickHouseLocalDateTimeParser extends ClickHouseDateValueParser { + + private static ClickHouseLocalDateTimeParser instance; + + static ClickHouseLocalDateTimeParser getInstance() { + if (instance == null) { + instance = new ClickHouseLocalDateTimeParser(); + } + return instance; + } + + private ClickHouseLocalDateTimeParser() { + super(LocalDateTime.class); + } + + @Override + LocalDateTime parseDate(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return parseAsLocalDate(value).atStartOfDay(); + } + + @Override + LocalDateTime parseDateTime(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return dateTimeToLocalDateTime(value, columnInfo, timeZone); + } + + @Override + LocalDateTime parseNumber(long value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return parseAsInstant(value) + .atZone(timeZone.toZoneId()) + .toLocalDateTime(); + } + + @Override + LocalDateTime parseOther(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + try { + return parseAsLocalDate(value).atStartOfDay(); + } catch (DateTimeParseException dtpe) { + // not parseable as date + } + try { + return parseAsLocalDateTime(value); + } catch (DateTimeParseException dtpe) { + // not parseable as datetime + } + Instant i = parseAsInstant(value); + return i.atZone(timeZone.toZoneId()).toLocalDateTime(); + } + +} diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseLocalTimeParser.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseLocalTimeParser.java new file mode 100644 index 000000000..3e9381919 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseLocalTimeParser.java @@ -0,0 +1,63 @@ +package ru.yandex.clickhouse.response.parser; + +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.TimeZone; + +import ru.yandex.clickhouse.response.ClickHouseColumnInfo; + +final class ClickHouseLocalTimeParser extends ClickHouseDateValueParser { + + private static ClickHouseLocalTimeParser instance; + + static ClickHouseLocalTimeParser getInstance() { + if (instance == null) { + instance = new ClickHouseLocalTimeParser(); + } + return instance; + } + + private ClickHouseLocalTimeParser() { + super(LocalTime.class); + } + + @Override + LocalTime parseDate(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return LocalTime.MIDNIGHT; + } + + @Override + LocalTime parseDateTime(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return dateTimeToLocalDateTime(value, columnInfo, timeZone).toLocalTime(); + } + + @Override + LocalTime parseNumber(long value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return parseAsLocalTime(value); + } + + @Override + LocalTime parseOther(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + try { + return LocalTime.parse(value, DateTimeFormatter.ISO_LOCAL_TIME); + } catch (DateTimeParseException dtpe) { + // try different pattern + } + try { + return LocalTime.parse(value, DateTimeFormatter.ISO_OFFSET_TIME); + } catch (DateTimeParseException dtpe) { + // try different pattern + } + return parseAsLocalTime(value); + } + +} diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseMapParser.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseMapParser.java new file mode 100644 index 000000000..c0b7d44e3 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseMapParser.java @@ -0,0 +1,183 @@ +package ru.yandex.clickhouse.response.parser; + +import java.sql.Array; +import java.sql.SQLException; +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.Deque; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.TimeZone; + +import ru.yandex.clickhouse.domain.ClickHouseDataType; +import ru.yandex.clickhouse.response.ByteFragment; +import ru.yandex.clickhouse.response.ClickHouseColumnInfo; + +@SuppressWarnings("rawtypes") +final class ClickHouseMapParser extends ClickHouseValueParser { + + private static ClickHouseMapParser instance; + + static ClickHouseMapParser getInstance() { + if (instance == null) { + instance = new ClickHouseMapParser(); + } + return instance; + } + + private ClickHouseMapParser() { + // prevent instantiation + } + + int readPart(ClickHouseDataType type, String str, int startPosition, int len, StringBuilder sb, char stopChar) { + Deque stack = new ArrayDeque<>(); + stack.push('\0'); + char lastChar = '\0'; + for (int i = startPosition; i < len; i++) { + char ch = str.charAt(startPosition = i); + + if (lastChar == '\0') { + if (Character.isWhitespace(ch)) { + continue; + } + + if (ch == stopChar) { + break; + } + + switch (ch) { + case '\'': + if (lastChar != '\0') { + stack.push(lastChar); + } + lastChar = ch; + if (type != ClickHouseDataType.String) { + sb.append(ch); + } + break; + case '{': + if (lastChar != '\0') { + stack.push(lastChar); + } + lastChar = '}'; + sb.append(ch); + break; + case '(': + if (lastChar != '\0') { + stack.push(lastChar); + } + lastChar = ')'; + sb.append(ch); + break; + case '[': + if (lastChar != '\0') { + stack.push(lastChar); + } + lastChar = ']'; + sb.append(ch); + break; + case '}': + return i + 1; + default: + sb.append(ch); + break; + } + } else if (lastChar == '\'') { // quoted + if (ch != '\'' || type != ClickHouseDataType.String) { + sb.append(ch); + } + if (i + 1 < len) { + char nextChar = str.charAt(i + 1); + if (ch == '\\') { + sb.append(nextChar); + i++; + } else if (ch == '\'' && nextChar == ch) { + sb.append(ch).append(nextChar); + i++; + } else if (ch == '\'') { + lastChar = stack.pop(); + } + } + } else if (lastChar == '}' || lastChar == ')' || lastChar == ']') { + if (ch == lastChar) { + lastChar = stack.pop(); + } + sb.append(ch); + } + } + + return startPosition; + } + + @Override + public Map parse(ByteFragment value, ClickHouseColumnInfo columnInfo, TimeZone resultTimeZone) throws SQLException { + if (value.isNull()) { + return null; + } + + ClickHouseColumnInfo keyInfo = Objects.requireNonNull(columnInfo.getKeyInfo()); + ClickHouseColumnInfo valueInfo = Objects.requireNonNull(columnInfo.getValueInfo()); + + ClickHouseValueParser keyParser = ClickHouseValueParser + .getParser(keyInfo.getClickHouseDataType().getJavaClass()); + ClickHouseValueParser valueParser = ClickHouseValueParser + .getParser(valueInfo.getClickHouseDataType().getJavaClass()); + + String str = value.asString(); + int len = str == null ? 0 : str.length(); + if (len < 2) { + return Collections.emptyMap(); + } + + Map map = new LinkedHashMap<>(); + + int part = -1; // -1 - uncertain, 0 - key, 1 - value + StringBuilder sb = new StringBuilder(); + Object k = null; + Object v = null; + for (int i = 0; i < len; i++) { + char ch = str.charAt(i); + + if (Character.isWhitespace(ch)) { // skip whitespaces + continue; + } + + if (part == -1) { + if (ch == '{') { + part = 0; + continue; + } else { + throw new IllegalArgumentException("Invalid map. Expect '{' but we got '" + ch + "' at " + i); + } + } + + if (ch == '}') { + // TODO check if there's any pending characters + break; + } + + if (part == 0) { // reading key(String or Integer) + i = readPart(keyInfo.getClickHouseDataType(), str, i, len, sb, ':'); + k = keyParser.parse(ByteFragment.fromString(sb.toString()), keyInfo, resultTimeZone); + + part = 1; + sb.setLength(0); + } else { // reading value(String, Integer or Array) + i = readPart(valueInfo.getClickHouseDataType(), str, i, len, sb, ','); + v = valueParser.parse(ByteFragment.fromString(sb.toString()), valueInfo, resultTimeZone); + map.put(k, valueInfo.isArray() && v != null ? ((Array) v).getArray() : v); + + part = 0; + sb.setLength(0); + } + } + + return Collections.unmodifiableMap(map); + } + + @Override + protected Map getDefaultValue() { + return Collections.emptyMap(); + } +} diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseOffsetDateTimeParser.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseOffsetDateTimeParser.java new file mode 100644 index 000000000..a74b45866 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseOffsetDateTimeParser.java @@ -0,0 +1,69 @@ +package ru.yandex.clickhouse.response.parser; + +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.TimeZone; + +import ru.yandex.clickhouse.response.ClickHouseColumnInfo; + +final class ClickHouseOffsetDateTimeParser extends ClickHouseDateValueParser { + + private static ClickHouseOffsetDateTimeParser instance; + + static ClickHouseOffsetDateTimeParser getInstance() { + if (instance == null) { + instance = new ClickHouseOffsetDateTimeParser(); + } + return instance; + } + + private ClickHouseOffsetDateTimeParser() { + super(OffsetDateTime.class); + } + + @Override + OffsetDateTime parseDate(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return dateToZonedDateTime(value, columnInfo, timeZone).toOffsetDateTime(); + } + + @Override + OffsetDateTime parseDateTime(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return dateTimeToZonedDateTime(value, columnInfo, timeZone).toOffsetDateTime(); + } + + @Override + OffsetDateTime parseNumber(long value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return parseAsInstant(value) + .atZone(effectiveTimeZone(columnInfo, timeZone)) + .toOffsetDateTime(); + } + + @Override + OffsetDateTime parseOther(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + try { + return parseAsLocalDateTime(value) + .atZone(effectiveTimeZone(columnInfo, timeZone)) + .toOffsetDateTime(); + } catch (DateTimeParseException dtpe) { + // try another way + } + try { + return OffsetDateTime.parse(value, DateTimeFormatter.ISO_OFFSET_DATE_TIME); + } catch (DateTimeParseException dtpe) { + // try another way + } + return parseAsInstant(value) + .atZone(effectiveTimeZone(columnInfo, timeZone)) + .toOffsetDateTime(); + } + +} diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseOffsetTimeParser.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseOffsetTimeParser.java new file mode 100644 index 000000000..5feb348c1 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseOffsetTimeParser.java @@ -0,0 +1,72 @@ +package ru.yandex.clickhouse.response.parser; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.TimeZone; + +import ru.yandex.clickhouse.response.ClickHouseColumnInfo; + +final class ClickHouseOffsetTimeParser extends ClickHouseDateValueParser { + + private static ClickHouseOffsetTimeParser instance; + + static ClickHouseOffsetTimeParser getInstance() { + if (instance == null) { + instance = new ClickHouseOffsetTimeParser(); + } + return instance; + } + + private ClickHouseOffsetTimeParser() { + super(OffsetTime.class); + } + + @Override + OffsetTime parseDate(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return dateToZonedDateTime(value, columnInfo, timeZone).toOffsetDateTime().toOffsetTime(); + } + + @Override + OffsetTime parseDateTime(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return dateTimeToZonedDateTime(value, columnInfo, timeZone).toOffsetDateTime().toOffsetTime(); + } + + @Override + OffsetTime parseNumber(long value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return OffsetTime.of( + parseAsLocalTime(value), + effectiveTimeZone(columnInfo, timeZone).getRules().getOffset(Instant.now())); + } + + @Override + OffsetTime parseOther(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + try { + return OffsetTime.parse(value, DateTimeFormatter.ISO_OFFSET_TIME); + } catch (DateTimeParseException dtpe) { + // try next pattern + } + try { + return OffsetTime.of( + LocalTime.parse(value, DateTimeFormatter.ISO_LOCAL_TIME), + effectiveTimeZone(columnInfo, timeZone).getRules().getOffset(Instant.now())); + } catch (DateTimeParseException dtpe) { + // try next pattern + } + return OffsetTime.of( + parseAsLocalTime(value), + effectiveTimeZone(columnInfo, timeZone).getRules().getOffset(Instant.now())); + } + +} diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseSQLDateParser.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseSQLDateParser.java new file mode 100644 index 000000000..77c9732a3 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseSQLDateParser.java @@ -0,0 +1,117 @@ +package ru.yandex.clickhouse.response.parser; + +import java.sql.Date; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoUnit; +import java.util.TimeZone; + +import ru.yandex.clickhouse.response.ClickHouseColumnInfo; + +final class ClickHouseSQLDateParser extends ClickHouseDateValueParser { + + private static ClickHouseSQLDateParser instance; + + static ClickHouseSQLDateParser getInstance() { + if (instance == null) { + instance = new ClickHouseSQLDateParser(); + } + return instance; + } + + private ClickHouseSQLDateParser() { + super(Date.class); + } + + @Override + Date parseDate(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return new Date(dateToZonedDateTime(value, columnInfo, timeZone).truncatedTo(ChronoUnit.DAYS).toInstant().toEpochMilli()); + } + + @Override + Date parseDateTime(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + if (timeZone == null) { + timeZone = TimeZone.getDefault(); + } + return new Date(dateTimeToZonedDateTime(value, columnInfo, timeZone).truncatedTo(ChronoUnit.DAYS).toInstant().toEpochMilli()); + } + + @Override + Date parseNumber(long value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return new Date(parseAsInstant(value) + .atZone(getResultTimeZone(timeZone)) + .truncatedTo(ChronoUnit.DAYS) + .toInstant() + .toEpochMilli()); + } + + @Override + Date parseOther(String value, ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + try { + return new Date(parseAsInstant(value) + .atZone(getResultTimeZone(timeZone)) + .truncatedTo(ChronoUnit.DAYS) + .toInstant() + .toEpochMilli()); + } catch (DateTimeParseException dtpe) { + // try next candidate + } + + try { + return new Date(parseAsOffsetDateTime(value) + .toInstant() + .atZone(getResultTimeZone(timeZone)) + .truncatedTo(ChronoUnit.DAYS) + .toInstant() + .toEpochMilli()); + } catch (DateTimeParseException dtpe) { + // try next candidate + } + + try { + return new Date(parseAsLocalDateTime(value) + .atZone(getParsingTimeZone(columnInfo, timeZone)) + .withZoneSameInstant(getResultTimeZone(timeZone)) + .truncatedTo(ChronoUnit.DAYS) + .toInstant() + .toEpochMilli()); + } catch (DateTimeParseException dtpe) { + // try next candidate + } + + return new Date(LocalDateTime + .of( + parseAsLocalDate(value), + LocalTime.MIDNIGHT) + .atZone(getResultTimeZone(timeZone)) + .toInstant() + .toEpochMilli()); + } + + private static ZoneId getParsingTimeZone(ClickHouseColumnInfo columnInfo, + TimeZone timeZone) + { + return columnInfo.getTimeZone() != null + ? columnInfo.getTimeZone().toZoneId() + : timeZone != null + ? timeZone.toZoneId() + : ZoneId.systemDefault(); + } + + private static ZoneId getResultTimeZone(TimeZone timeZone) { + return timeZone != null + ? timeZone.toZoneId() + : ZoneId.systemDefault(); + } + +} diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseSQLTimeParser.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseSQLTimeParser.java new file mode 100644 index 000000000..357729fec --- /dev/null +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/parser/ClickHouseSQLTimeParser.java @@ -0,0 +1,79 @@ +package ru.yandex.clickhouse.response.parser; + +import java.sql.Time; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.TimeZone; + +import ru.yandex.clickhouse.response.ClickHouseColumnInfo; + +final class ClickHouseSQLTimeParser extends ClickHouseDateValueParser