From 91e6777f67cddc4ee3915d0369055201d900f1c9 Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Wed, 18 Jan 2023 20:54:30 +0800 Subject: [PATCH 1/6] extract clickhouse-data from clickhouse-client --- .github/workflows/analysis.yml | 8 +- .github/workflows/build.yml | 30 +- .github/workflows/release.yml | 4 +- README.md | 6 +- .../benchmark/client/ClientState.java | 8 +- .../benchmark/misc/FactoryBenchmark.java | 34 +- .../benchmark/misc/QueueBenchmark.java | 16 +- .../benchmark/misc/StreamBenchmark.java | 4 +- .../client/cli/ClickHouseCommandLine.java | 95 +- .../cli/ClickHouseCommandLineClient.java | 6 +- .../cli/ClickHouseCommandLineResponse.java | 2 +- .../config/ClickHouseCommandLineOption.java | 4 +- .../cli/ClickHouseCommandLineClientTest.java | 30 + clickhouse-client/pom.xml | 55 +- .../com/clickhouse/client/AbstractClient.java | 6 +- .../client/AbstractSocketClient.java | 16 +- .../clickhouse/client/ClickHouseClient.java | 26 +- .../client/ClickHouseClientBuilder.java | 11 +- .../clickhouse/client/ClickHouseCluster.java | 2 + .../clickhouse/client/ClickHouseConfig.java | 89 +- .../client/ClickHouseCredentials.java | 2 + .../client/ClickHouseDnsResolver.java | 5 +- .../com/clickhouse/client/ClickHouseFile.java | 112 --- .../client/ClickHouseLoadBalancingPolicy.java | 2 + .../com/clickhouse/client/ClickHouseNode.java | 9 +- .../client/ClickHouseNodeSelector.java | 5 +- .../clickhouse/client/ClickHouseNodes.java | 8 +- .../client/ClickHouseParameterizedQuery.java | 6 + .../clickhouse/client/ClickHouseRequest.java | 177 ++-- .../client/ClickHouseRequestManager.java | 3 + .../clickhouse/client/ClickHouseResponse.java | 12 +- .../{data => }/ClickHouseSimpleResponse.java | 19 +- .../client/ClickHouseSslContextProvider.java | 2 + .../{data => }/ClickHouseStreamResponse.java | 21 +- .../client/ClickHouseTransaction.java | 7 +- .../clickhouse/client/ClickHouseWriter.java | 14 - .../client/config/ClickHouseClientOption.java | 46 +- .../ClickHouseDefaultSslContextProvider.java | 2 +- .../client/config/ClickHouseDefaults.java | 15 +- .../client/logging/package-info.java | 4 - .../clickhouse/client/naming/SrvResolver.java | 4 +- .../client/stream/CompressionUtils.java | 187 ---- .../src/main/java9/module-info.java | 12 +- .../client/ClickHouseClientTest.java | 2 + .../client/ClickHouseConfigTest.java | 2 +- .../clickhouse/client/ClickHouseNodeTest.java | 2 + .../ClickHouseParameterizedQueryTest.java | 4 +- .../client/ClickHouseRequestTest.java | 21 +- .../client/ClickHouseServerForTest.java | 4 + .../ClickHouseSimpleResponseTest.java | 11 +- .../client/ClientIntegrationTest.java | 235 ++++- clickhouse-data/README.md | 109 +++ clickhouse-data/pom.xml | 140 +++ .../config/ClickHouseBufferingMode.java | 2 +- .../ClickHouseConfigChangeListener.java | 2 +- .../config/ClickHouseDefaultOption.java | 45 + .../clickhouse}/config/ClickHouseOption.java | 15 +- .../config/ClickHouseRenameMethod.java | 2 +- .../com/clickhouse/config/package-info.java | 4 + .../data}/ClickHouseAggregateFunction.java | 2 +- .../data}/ClickHouseArraySequence.java | 2 +- .../data}/ClickHouseByteBuffer.java | 2 +- .../clickhouse/data/ClickHouseByteUtils.java | 169 ++++ .../com/clickhouse/data}/ClickHouseCache.java | 6 +- .../clickhouse/data}/ClickHouseChecker.java | 2 +- .../clickhouse}/data/ClickHouseCityHash.java | 2 +- .../clickhouse/data}/ClickHouseColumn.java | 80 +- .../data}/ClickHouseCompression.java | 14 +- .../data/ClickHouseCompressionAlgorithm.java | 128 +++ .../clickhouse/data/ClickHouseDataConfig.java | 373 ++++++++ .../data}/ClickHouseDataProcessor.java | 28 +- .../data}/ClickHouseDataStreamFactory.java | 37 +- .../clickhouse/data}/ClickHouseDataType.java | 10 +- .../data}/ClickHouseDataUpdater.java | 2 +- .../data}/ClickHouseDeferredValue.java | 10 +- .../data}/ClickHouseDeserializer.java | 4 +- .../com/clickhouse/data}/ClickHouseEnum.java | 2 +- .../data/ClickHouseExternalTable.java | 107 +-- .../com/clickhouse/data/ClickHouseFile.java | 180 ++++ .../clickhouse/data}/ClickHouseFormat.java | 6 +- .../data}/ClickHouseInputStream.java | 220 ++--- .../data}/ClickHouseOutputStream.java | 97 +-- .../data}/ClickHousePassThruStream.java | 89 +- .../data}/ClickHousePipedOutputStream.java | 2 +- .../clickhouse/data}/ClickHouseRecord.java | 6 +- .../data/ClickHouseRecordTransformer.java | 4 +- .../data}/ClickHouseSerializer.java | 2 +- .../data/ClickHouseStreamConfig.java | 5 + .../data}/ClickHouseThreadFactory.java | 6 +- .../com/clickhouse/data}/ClickHouseUtils.java | 114 +-- .../com/clickhouse/data}/ClickHouseValue.java | 2 +- .../clickhouse/data}/ClickHouseValues.java | 16 +- .../clickhouse/data}/ClickHouseVersion.java | 2 +- .../com/clickhouse/data/ClickHouseWriter.java | 37 + .../com/clickhouse/data/UnsafeByteUtils.java | 107 +++ .../clickhouse/data}/cache/CaffeineCache.java | 4 +- .../clickhouse/data}/cache/JdkLruCache.java | 4 +- .../data/compress/BrotliSupport.java | 76 ++ .../clickhouse/data/compress/Bz2Support.java | 60 ++ .../data/compress/DeflateSupport.java | 89 ++ .../clickhouse/data/compress/GzipSupport.java | 86 ++ .../clickhouse/data/compress/Lz4Support.java | 52 ++ .../clickhouse/data/compress/NoneSupport.java | 54 ++ .../data/compress/SnappySupport.java | 87 ++ .../clickhouse/data/compress/XzSupport.java | 55 ++ .../clickhouse/data/compress/ZstdSupport.java | 55 ++ .../data/format/ApacheAvroProcessor.java | 155 ++++ .../data/format}/BinaryDataProcessor.java | 51 +- .../data/format}/BinaryStreamUtils.java | 15 +- .../format/ClickHouseNativeProcessor.java | 822 ++++++++++++++++++ .../format}/ClickHouseRowBinaryProcessor.java | 71 +- .../data/format}/ClickHouseSimpleRecord.java | 12 +- .../ClickHouseTabSeparatedProcessor.java | 40 +- .../data/format}/JsonStreamUtils.java | 2 +- .../data/format/MessagePackProcessor.java | 61 ++ .../data/format}/TextDataProcessor.java | 18 +- .../data/format}/tsv/ArrayByteFragment.java | 2 +- .../data/format}/tsv/ByteFragment.java | 2 +- .../format}/tsv/FastByteArrayInputStream.java | 2 +- .../tsv/FastByteArrayOutputStream.java | 2 +- .../data/format}/tsv/StreamSplitter.java | 2 +- .../data/format}/tsv/package-info.java | 2 +- .../stream/AbstractByteArrayInputStream.java | 14 +- .../stream/AbstractByteArrayOutputStream.java | 8 +- .../stream/AbstractByteBufferInputStream.java | 14 +- .../data}/stream/AdaptiveQueue.java | 2 +- .../data}/stream/BlockingInputStream.java | 19 +- .../stream/BlockingPipedOutputStream.java | 20 +- .../data}/stream/CapacityPolicy.java | 2 +- .../data}/stream/DeferredInputStream.java | 4 +- .../data}/stream/DeferredOutputStream.java | 4 +- .../data}/stream/EmptyInputStream.java | 10 +- .../data}/stream/EmptyOutputStream.java | 6 +- .../stream/IterableByteArrayInputStream.java | 6 +- .../stream/IterableByteBufferInputStream.java | 6 +- .../stream/IterableMultipleInputStream.java | 17 +- .../stream/IterableObjectInputStream.java | 6 +- .../data}/stream/Lz4InputStream.java | 25 +- .../data}/stream/Lz4OutputStream.java | 26 +- .../data}/stream/NonBlockingInputStream.java | 22 +- .../stream/NonBlockingPipedOutputStream.java | 20 +- .../data}/stream/RestrictedInputStream.java | 19 +- .../data}/stream/WrappedInputStream.java | 19 +- .../data}/stream/WrappedOutputStream.java | 13 +- .../data/value}/ClickHouseArrayValue.java | 10 +- .../value}/ClickHouseBigDecimalValue.java | 10 +- .../value}/ClickHouseBigIntegerValue.java | 6 +- .../data/value}/ClickHouseBitmap.java | 18 +- .../data/value}/ClickHouseBitmapValue.java | 10 +- .../data/value}/ClickHouseBoolValue.java | 6 +- .../data/value}/ClickHouseByteValue.java | 6 +- .../data/value}/ClickHouseDateTimeValue.java | 13 +- .../data/value}/ClickHouseDateValue.java | 6 +- .../data/value}/ClickHouseDoubleValue.java | 10 +- .../data/value}/ClickHouseEmptyValue.java | 6 +- .../data/value}/ClickHouseEnumValue.java | 8 +- .../data/value}/ClickHouseFloatValue.java | 10 +- .../ClickHouseGeoMultiPolygonValue.java | 8 +- .../data/value}/ClickHouseGeoPointValue.java | 6 +- .../value}/ClickHouseGeoPolygonValue.java | 8 +- .../data/value}/ClickHouseGeoRingValue.java | 8 +- .../data/value}/ClickHouseInstantValue.java | 13 +- .../data/value}/ClickHouseIntegerValue.java | 6 +- .../data/value}/ClickHouseIpv4Value.java | 6 +- .../data/value}/ClickHouseIpv6Value.java | 6 +- .../data/value}/ClickHouseLongValue.java | 6 +- .../data/value}/ClickHouseMapValue.java | 8 +- .../data/value}/ClickHouseNestedValue.java | 10 +- .../data/value}/ClickHouseObjectValue.java | 6 +- .../value}/ClickHouseOffsetDateTimeValue.java | 13 +- .../data/value}/ClickHouseShortValue.java | 6 +- .../data/value}/ClickHouseStringValue.java | 8 +- .../data/value}/ClickHouseTupleValue.java | 8 +- .../data/value}/ClickHouseUuidValue.java | 6 +- .../clickhouse/data/value}/UnsignedByte.java | 2 +- .../data/value}/UnsignedInteger.java | 2 +- .../clickhouse/data/value}/UnsignedLong.java | 2 +- .../clickhouse/data/value}/UnsignedShort.java | 2 +- .../array/ClickHouseBoolArrayValue.java | 14 +- .../array/ClickHouseByteArrayValue.java | 16 +- .../array/ClickHouseDoubleArrayValue.java | 14 +- .../array/ClickHouseFloatArrayValue.java | 14 +- .../value}/array/ClickHouseIntArrayValue.java | 16 +- .../array/ClickHouseLongArrayValue.java | 16 +- .../array/ClickHouseShortArrayValue.java | 16 +- .../clickhouse/data/value}/package-info.java | 2 +- .../com/clickhouse}/logging/JdkLogger.java | 33 +- .../clickhouse}/logging/JdkLoggerFactory.java | 2 +- .../com/clickhouse}/logging/LogMessage.java | 2 +- .../java/com/clickhouse}/logging/Logger.java | 42 +- .../clickhouse}/logging/LoggerFactory.java | 34 +- .../com/clickhouse}/logging/Slf4jLogger.java | 34 +- .../logging/Slf4jLoggerFactory.java | 2 +- .../com/clickhouse/logging/package-info.java | 4 + .../src/main/java9/module-info.java | 22 + .../config/ClickHouseOptionTest.java | 10 +- .../config/ClickHouseRenameMethodTest.java | 2 +- .../data}/BaseClickHouseValueTest.java | 2 +- .../data}/ClickHouseByteBufferTest.java | 2 +- .../data}/ClickHouseCheckerTest.java | 2 +- .../data}/ClickHouseColumnTest.java | 32 +- .../ClickHouseCompressionAlgorithmTest.java | 15 + .../data/ClickHouseDataConfigTest.java | 26 + .../ClickHouseDataStreamFactoryTest.java | 13 +- .../data}/ClickHouseDataTypeTest.java | 2 +- .../clickhouse/data/ClickHouseFileTest.java | 71 ++ .../data}/ClickHouseInputStreamTest.java | 21 +- .../data}/ClickHouseOutputStreamTest.java | 2 +- .../data}/ClickHousePassThruStreamTest.java | 23 +- .../data/ClickHouseTestDataConfig.java | 104 +++ .../clickhouse/data}/ClickHouseUtilsTest.java | 49 +- .../data}/ClickHouseValuesTest.java | 2 +- .../data}/ClickHouseVersionTest.java | 2 +- .../data/TestServiceImplementation.java | 8 + .../clickhouse/data/TestServiceInterface.java | 7 + .../data}/cache/CaffeineCacheTest.java | 4 +- .../data}/cache/JdkLruCacheTest.java | 4 +- .../data/format/ApacheAvroProcessorTest.java | 38 + .../data/format}/BinaryStreamUtilsTest.java | 10 +- .../format/ClickHouseNativeProcessorTest.java | 355 ++++++++ .../ClickHouseRowBinaryProcessorTest.java | 49 +- .../format}/ClickHouseSimpleRecordTest.java | 8 +- .../ClickHouseTabSeparatedProcessorTest.java | 51 +- .../data/format}/JsonStreamUtilsTest.java | 2 +- .../stream/BlockingPipedOutputStreamTest.java | 4 +- .../data}/stream/InputStreamImplTest.java | 10 +- .../data}/stream/Lz4InputStreamTest.java | 2 +- .../data}/stream/Lz4OutputStreamTest.java | 6 +- .../NonBlockingPipedOutputStreamTest.java | 4 +- .../data}/stream/OutputStreamImplTest.java | 6 +- .../data/value}/BaseDataProcessorTest.java | 166 ++-- .../data/value}/ClickHouseBitmapTest.java | 4 +- .../data/value}/ClickHouseBoolValueTest.java | 6 +- .../data/value}/ClickHouseByteValueTest.java | 6 +- .../value}/ClickHouseDateTimeValueTest.java | 8 +- .../data/value}/ClickHouseDateValueTest.java | 6 +- .../value}/ClickHouseDoubleValueTest.java | 6 +- .../data/value}/ClickHouseEnumValueTest.java | 7 +- .../data/value}/ClickHouseFloatValueTest.java | 6 +- .../value}/ClickHouseGeoPointValueTest.java | 4 +- .../value}/ClickHouseIntegerValueTest.java | 6 +- .../data/value}/ClickHouseLongValueTest.java | 8 +- .../value}/ClickHouseNestedValueTest.java | 6 +- .../ClickHouseOffsetDateTimeValueTest.java | 8 +- .../data/value}/ClickHouseShortValueTest.java | 6 +- .../value}/ClickHouseStringValueTest.java | 6 +- .../data/value}/ClickHouseTupleValueTest.java | 4 +- .../data/value}/UnsignedByteTest.java | 2 +- .../data/value}/UnsignedIntegerTest.java | 2 +- .../data/value}/UnsignedLongTest.java | 2 +- .../data/value}/UnsignedShortTest.java | 2 +- .../data/value}/WriterFunction.java | 2 +- .../array/ClickHouseByteArrayValueTest.java | 2 +- .../array/ClickHouseLongArrayValueTest.java | 2 +- .../clickhouse}/logging/JdkLoggerTest.java | 2 +- .../clickhouse}/logging/LogMessageTest.java | 2 +- .../com/clickhouse}/logging/LoggerTest.java | 2 +- .../clickhouse}/logging/Slf4jLoggerTest.java | 2 +- .../com.clickhouse.data.TestServiceInterface | 1 + .../test/resources/simplelogger.properties | 7 + clickhouse-grpc-client/pom.xml | 231 +++-- .../grpc/ClickHouseGrpcChannelFactory.java | 14 +- .../client/grpc/ClickHouseGrpcClient.java | 36 +- .../client/grpc/ClickHouseGrpcResponse.java | 4 +- .../client/grpc/ClickHouseStreamObserver.java | 20 +- .../client/grpc/NettyChannelFactoryImpl.java | 4 +- .../grpc/config/ClickHouseGrpcOption.java | 4 +- .../client/grpc/ClickHouseGrpcClientTest.java | 62 +- clickhouse-http-client/pom.xml | 72 +- .../client/http/ApacheHttpConnectionImpl.java | 126 +-- .../client/http/ClickHouseHttpClient.java | 21 +- .../client/http/ClickHouseHttpConnection.java | 89 +- .../client/http/ClickHouseHttpEntity.java | 4 +- .../client/http/ClickHouseHttpResponse.java | 10 +- .../client/http/HttpUrlConnectionImpl.java | 43 +- .../http/config/ClickHouseHttpOption.java | 4 +- .../http/ClickHouseResponseHandler.java | 4 +- .../client/http/HttpClientConnectionImpl.java | 127 +-- .../http/ApacheHttpConnectionImplTest.java | 2 +- .../client/http/ClickHouseHttpClientTest.java | 115 +-- .../http/ClickHouseHttpConnectionTest.java | 13 +- clickhouse-jdbc/pom.xml | 286 ++++-- .../com/clickhouse/jdbc/ClickHouseArray.java | 4 +- .../clickhouse/jdbc/ClickHouseConnection.java | 12 +- .../jdbc/ClickHouseDatabaseMetaData.java | 22 +- .../com/clickhouse/jdbc/ClickHouseDriver.java | 22 +- .../jdbc/ClickHousePreparedStatement.java | 4 +- .../clickhouse/jdbc/ClickHouseResultSet.java | 8 +- .../jdbc/ClickHouseResultSetMetaData.java | 4 +- .../com/clickhouse/jdbc/ClickHouseStruct.java | 2 +- .../java/com/clickhouse/jdbc/JdbcConfig.java | 8 +- .../jdbc/JdbcParameterizedQuery.java | 4 +- .../com/clickhouse/jdbc/JdbcParseHandler.java | 2 +- .../com/clickhouse/jdbc/JdbcTypeMapping.java | 8 +- .../internal/AbstractPreparedStatement.java | 2 +- .../internal/ClickHouseConnectionImpl.java | 24 +- .../internal/ClickHouseJdbcUrlParser.java | 6 +- .../internal/ClickHouseParameterMetaData.java | 6 +- .../internal/ClickHouseStatementImpl.java | 28 +- .../internal/InputBasedPreparedStatement.java | 18 +- .../jdbc/internal/JdbcTransaction.java | 6 +- .../internal/SqlBasedPreparedStatement.java | 22 +- .../StreamBasedPreparedStatement.java | 20 +- .../internal/TableBasedPreparedStatement.java | 14 +- .../src/main/java9/module-info.java | 4 +- .../src/main/javacc/ClickHouseSqlParser.jj | 4 +- .../jdbc/ClickHouseConnectionTest.java | 4 +- .../jdbc/ClickHouseDatabaseMetaDataTest.java | 2 +- .../jdbc/ClickHousePreparedStatementTest.java | 46 +- .../jdbc/ClickHouseResultSetTest.java | 18 +- .../jdbc/ClickHouseStatementTest.java | 16 +- .../jdbc/CombinedResultSetTest.java | 7 +- clickhouse-r2dbc/pom.xml | 154 ++-- .../com/clickhouse/r2dbc/ClickHouseBatch.java | 2 +- .../r2dbc/ClickHouseColumnMetadata.java | 2 +- .../clickhouse/r2dbc/ClickHouseResult.java | 4 +- .../clickhouse/r2dbc/ClickHouseResult091.java | 4 +- .../com/clickhouse/r2dbc/ClickHouseRow.java | 4 +- .../clickhouse/r2dbc/ClickHouseStatement.java | 6 +- .../connection/ClickHouseConnection.java | 4 +- .../ClickHouseConnectionFactoryProvider.java | 4 +- .../ClickHouseConnectionMetadata.java | 4 +- .../types/ClickHouseDataTypeWrapper.java | 2 +- .../src/main/java9/module-info.java | 4 +- .../com/clickhouse/examples/jdbc/Main.java | 10 +- .../clickhouse/examples/jdbc/Advanced.java | 6 +- .../com/clickhouse/examples/jdbc/Basic.java | 55 +- pom.xml | 63 +- 328 files changed, 6834 insertions(+), 2679 deletions(-) delete mode 100644 clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseFile.java rename clickhouse-client/src/main/java/com/clickhouse/client/{data => }/ClickHouseSimpleResponse.java (92%) rename clickhouse-client/src/main/java/com/clickhouse/client/{data => }/ClickHouseStreamResponse.java (89%) delete mode 100644 clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseWriter.java delete mode 100644 clickhouse-client/src/main/java/com/clickhouse/client/logging/package-info.java delete mode 100644 clickhouse-client/src/main/java/com/clickhouse/client/stream/CompressionUtils.java rename clickhouse-client/src/test/java/com/clickhouse/client/{data => }/ClickHouseSimpleResponseTest.java (94%) create mode 100644 clickhouse-data/README.md create mode 100644 clickhouse-data/pom.xml rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse}/config/ClickHouseBufferingMode.java (95%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse}/config/ClickHouseConfigChangeListener.java (97%) create mode 100644 clickhouse-data/src/main/java/com/clickhouse/config/ClickHouseDefaultOption.java rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse}/config/ClickHouseOption.java (96%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse}/config/ClickHouseRenameMethod.java (98%) create mode 100644 clickhouse-data/src/main/java/com/clickhouse/config/package-info.java rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseAggregateFunction.java (99%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseArraySequence.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseByteBuffer.java (99%) create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseByteUtils.java rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseCache.java (92%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseChecker.java (99%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse}/data/ClickHouseCityHash.java (99%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseColumn.java (94%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseCompression.java (88%) create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseCompressionAlgorithm.java create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataConfig.java rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseDataProcessor.java (96%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseDataStreamFactory.java (74%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseDataType.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseDataUpdater.java (96%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseDeferredValue.java (95%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseDeserializer.java (97%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseEnum.java (99%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse}/data/ClickHouseExternalTable.java (67%) create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseFile.java rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseFormat.java (99%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseInputStream.java (84%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseOutputStream.java (80%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHousePassThruStream.java (66%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHousePipedOutputStream.java (96%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseRecord.java (93%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse}/data/ClickHouseRecordTransformer.java (76%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseSerializer.java (98%) create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseStreamConfig.java rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseThreadFactory.java (94%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseUtils.java (93%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseValue.java (99%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseValues.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/ClickHouseVersion.java (99%) create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseWriter.java create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/UnsafeByteUtils.java rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/cache/CaffeineCache.java (95%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/cache/JdkLruCache.java (95%) create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/compress/BrotliSupport.java create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/compress/Bz2Support.java create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/compress/DeflateSupport.java create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/compress/GzipSupport.java create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/compress/Lz4Support.java create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/compress/NoneSupport.java create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/compress/SnappySupport.java create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/compress/XzSupport.java create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/compress/ZstdSupport.java create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/format/ApacheAvroProcessor.java rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/format}/BinaryDataProcessor.java (94%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/format}/BinaryStreamUtils.java (99%) create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseNativeProcessor.java rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/format}/ClickHouseRowBinaryProcessor.java (91%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/format}/ClickHouseSimpleRecord.java (90%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/format}/ClickHouseTabSeparatedProcessor.java (81%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/format}/JsonStreamUtils.java (98%) create mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/format/MessagePackProcessor.java rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/format}/TextDataProcessor.java (94%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/format}/tsv/ArrayByteFragment.java (92%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/format}/tsv/ByteFragment.java (99%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/format}/tsv/FastByteArrayInputStream.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/format}/tsv/FastByteArrayOutputStream.java (99%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/format}/tsv/StreamSplitter.java (99%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/format}/tsv/package-info.java (60%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/AbstractByteArrayInputStream.java (95%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/AbstractByteArrayOutputStream.java (93%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/AbstractByteBufferInputStream.java (96%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/AdaptiveQueue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/BlockingInputStream.java (78%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/BlockingPipedOutputStream.java (90%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/CapacityPolicy.java (97%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/DeferredInputStream.java (92%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/DeferredOutputStream.java (89%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/EmptyInputStream.java (83%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/EmptyOutputStream.java (87%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/IterableByteArrayInputStream.java (86%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/IterableByteBufferInputStream.java (90%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/IterableMultipleInputStream.java (92%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/IterableObjectInputStream.java (89%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/Lz4InputStream.java (81%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/Lz4OutputStream.java (78%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/NonBlockingInputStream.java (92%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/NonBlockingPipedOutputStream.java (89%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/RestrictedInputStream.java (85%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/WrappedInputStream.java (84%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse/data}/stream/WrappedOutputStream.java (68%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseArrayValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseBigDecimalValue.java (96%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseBigIntegerValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseBitmap.java (97%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseBitmapValue.java (97%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseBoolValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseByteValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseDateTimeValue.java (96%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseDateValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseDoubleValue.java (96%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseEmptyValue.java (95%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseEnumValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseFloatValue.java (96%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseGeoMultiPolygonValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseGeoPointValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseGeoPolygonValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseGeoRingValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseInstantValue.java (96%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseIntegerValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseIpv4Value.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseIpv6Value.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseLongValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseMapValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseNestedValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseObjectValue.java (96%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseOffsetDateTimeValue.java (96%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseShortValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseStringValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseTupleValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/ClickHouseUuidValue.java (98%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/UnsignedByte.java (99%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/UnsignedInteger.java (99%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/UnsignedLong.java (99%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/UnsignedShort.java (99%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/array/ClickHouseBoolArrayValue.java (97%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/array/ClickHouseByteArrayValue.java (97%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/array/ClickHouseDoubleArrayValue.java (97%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/array/ClickHouseFloatArrayValue.java (97%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/array/ClickHouseIntArrayValue.java (97%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/array/ClickHouseLongArrayValue.java (97%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/array/ClickHouseShortArrayValue.java (97%) rename {clickhouse-client/src/main/java/com/clickhouse/client/data => clickhouse-data/src/main/java/com/clickhouse/data/value}/package-info.java (69%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse}/logging/JdkLogger.java (85%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse}/logging/JdkLoggerFactory.java (85%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse}/logging/LogMessage.java (97%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse}/logging/Logger.java (81%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse}/logging/LoggerFactory.java (62%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse}/logging/Slf4jLogger.java (86%) rename {clickhouse-client/src/main/java/com/clickhouse/client => clickhouse-data/src/main/java/com/clickhouse}/logging/Slf4jLoggerFactory.java (90%) create mode 100644 clickhouse-data/src/main/java/com/clickhouse/logging/package-info.java create mode 100644 clickhouse-data/src/main/java9/module-info.java rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse}/config/ClickHouseOptionTest.java (97%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse}/config/ClickHouseRenameMethodTest.java (98%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/BaseClickHouseValueTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/ClickHouseByteBufferTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/ClickHouseCheckerTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/ClickHouseColumnTest.java (95%) create mode 100644 clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseCompressionAlgorithmTest.java create mode 100644 clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseDataConfigTest.java rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/ClickHouseDataStreamFactoryTest.java (77%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/ClickHouseDataTypeTest.java (98%) create mode 100644 clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseFileTest.java rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/ClickHouseInputStreamTest.java (94%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/ClickHouseOutputStreamTest.java (98%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/ClickHousePassThruStreamTest.java (68%) create mode 100644 clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseTestDataConfig.java rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/ClickHouseUtilsTest.java (92%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/ClickHouseValuesTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/ClickHouseVersionTest.java (99%) create mode 100644 clickhouse-data/src/test/java/com/clickhouse/data/TestServiceImplementation.java create mode 100644 clickhouse-data/src/test/java/com/clickhouse/data/TestServiceInterface.java rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/cache/CaffeineCacheTest.java (94%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/cache/JdkLruCacheTest.java (93%) create mode 100644 clickhouse-data/src/test/java/com/clickhouse/data/format/ApacheAvroProcessorTest.java rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/format}/BinaryStreamUtilsTest.java (99%) create mode 100644 clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseNativeProcessorTest.java rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/format}/ClickHouseRowBinaryProcessorTest.java (93%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/format}/ClickHouseSimpleRecordTest.java (94%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/format}/ClickHouseTabSeparatedProcessorTest.java (83%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/format}/JsonStreamUtilsTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/stream/BlockingPipedOutputStreamTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/stream/InputStreamImplTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/stream/Lz4InputStreamTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/stream/Lz4OutputStreamTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/stream/NonBlockingPipedOutputStreamTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse/data}/stream/OutputStreamImplTest.java (96%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/BaseDataProcessorTest.java (93%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/ClickHouseBitmapTest.java (96%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/ClickHouseBoolValueTest.java (98%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/ClickHouseByteValueTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/ClickHouseDateTimeValueTest.java (98%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/ClickHouseDateValueTest.java (97%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/ClickHouseDoubleValueTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/ClickHouseEnumValueTest.java (95%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/ClickHouseFloatValueTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/ClickHouseGeoPointValueTest.java (96%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/ClickHouseIntegerValueTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/ClickHouseLongValueTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/ClickHouseNestedValueTest.java (98%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/ClickHouseOffsetDateTimeValueTest.java (98%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/ClickHouseShortValueTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/ClickHouseStringValueTest.java (98%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/ClickHouseTupleValueTest.java (98%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/UnsignedByteTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/UnsignedIntegerTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/UnsignedLongTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/UnsignedShortTest.java (99%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/WriterFunction.java (82%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/array/ClickHouseByteArrayValueTest.java (96%) rename {clickhouse-client/src/test/java/com/clickhouse/client/data => clickhouse-data/src/test/java/com/clickhouse/data/value}/array/ClickHouseLongArrayValueTest.java (96%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse}/logging/JdkLoggerTest.java (97%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse}/logging/LogMessageTest.java (97%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse}/logging/LoggerTest.java (98%) rename {clickhouse-client/src/test/java/com/clickhouse/client => clickhouse-data/src/test/java/com/clickhouse}/logging/Slf4jLoggerTest.java (97%) create mode 100644 clickhouse-data/src/test/resources/META-INF/services/com.clickhouse.data.TestServiceInterface create mode 100644 clickhouse-data/src/test/resources/simplelogger.properties diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 18e6dd313..1ec478ef5 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -45,7 +45,7 @@ jobs: distribution: "temurin" java-version: | 8 - 11 + 17 cache: "maven" - name: Setup Toolchain shell: bash @@ -57,7 +57,7 @@ jobs: jdk - 11 + 17 ${{ env.JAVA_HOME }} @@ -79,12 +79,12 @@ jobs: - name: Build and install run: | find . -type f -name "simplelogger.*" -exec rm -fv '{}' \; - mvn -q --batch-mode --projects '!clickhouse-benchmark' -Drelease -DclickhouseVersion=22.8 -DskipTests install + mvn -q --batch-mode --projects '!clickhouse-benchmark' -DclickhouseVersion=22.8 -DskipTests install - name: Analyze env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} run: | - mvn --batch-mode --projects '!clickhouse-benchmark' -Drelease -DclickhouseVersion=22.8 \ + mvn --batch-mode --projects '!clickhouse-benchmark' -DclickhouseVersion=22.8 \ -Panalysis verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar continue-on-error: true diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5b5aa8955..34152c1cb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,7 +54,7 @@ jobs: java-version: 8 cache: "maven" - name: Build and install libraries - run: mvn --batch-mode --show-version --strict-checksums --threads C1 -Dmaven.wagon.rto=30000 -DskipITs install + run: mvn --batch-mode --show-version --strict-checksums --threads C1 -Dmaven.wagon.rto=30000 -Dj8 -DskipITs install - name: Compile examples run: for d in $(ls -d `pwd`/examples/*/); do cd $d && mvn clean compile; done @@ -87,7 +87,7 @@ jobs: && clickhouse client --version - name: Test CLI client run: | - mvn --also-make --batch-mode --projects clickhouse-cli-client -DclickhouseVersion=22.8 -DskipUTs verify + mvn --also-make --batch-mode --projects clickhouse-cli-client -DclickhouseVersion=22.8 -Dj8 -DskipUTs verify - name: Upload test results uses: actions/upload-artifact@v2 if: failure() @@ -121,7 +121,7 @@ jobs: distribution: "temurin" java-version: | 8 - 11 + 17 cache: "maven" - name: Setup Toolchain shell: bash @@ -133,7 +133,7 @@ jobs: jdk - 11 + 17 ${{ env.JAVA_HOME }} @@ -143,7 +143,7 @@ jobs: EOF - name: Test Java client run: | - mvn --also-make --batch-mode --projects clickhouse-cli-client,clickhouse-grpc-client,clickhouse-http-client -Drelease -DclickhouseVersion=${{ matrix.clickhouse }} verify + mvn --also-make --batch-mode --projects clickhouse-cli-client,clickhouse-grpc-client,clickhouse-http-client -DclickhouseVersion=${{ matrix.clickhouse }} verify - name: Upload test results uses: actions/upload-artifact@v2 if: failure() @@ -181,7 +181,7 @@ jobs: distribution: "temurin" java-version: | 8 - 11 + 17 cache: "maven" - name: Setup Toolchain shell: bash @@ -193,7 +193,7 @@ jobs: jdk - 11 + 17 ${{ env.JAVA_HOME }} @@ -202,10 +202,10 @@ jobs: EOF - name: Install Java client - run: mvn --also-make --batch-mode --projects clickhouse-cli-client,clickhouse-grpc-client,clickhouse-http-client -Drelease -DskipTests install + run: mvn --also-make --batch-mode --projects clickhouse-cli-client,clickhouse-grpc-client,clickhouse-http-client -DskipTests install - name: Test JDBC driver run: | - mvn --batch-mode --projects clickhouse-jdbc -Drelease -DclickhouseVersion=${{ matrix.clickhouse }} -Dprotocol=${{ matrix.protocol }} verify + mvn --batch-mode --projects clickhouse-jdbc -DclickhouseVersion=${{ matrix.clickhouse }} -Dprotocol=${{ matrix.protocol }} verify - name: Upload test results uses: actions/upload-artifact@v2 if: failure() @@ -241,7 +241,7 @@ jobs: distribution: "temurin" java-version: | 8 - 11 + 17 cache: "maven" - name: Setup Toolchain shell: bash @@ -253,7 +253,7 @@ jobs: jdk - 11 + 17 ${{ env.JAVA_HOME }} @@ -262,10 +262,10 @@ jobs: EOF - name: Install Java client - run: mvn --also-make --batch-mode --projects clickhouse-jdbc -Drelease -DskipTests install + run: mvn --also-make --batch-mode --projects clickhouse-jdbc -DskipTests install - name: Test R2DBC ${{ matrix.r2dbc }} run: | - mvn --batch-mode --projects clickhouse-r2dbc -Drelease -DclickhouseVersion=${{ matrix.clickhouse }} \ + mvn --batch-mode --projects clickhouse-r2dbc -DclickhouseVersion=${{ matrix.clickhouse }} \ -D'r2dbc-spi.version=${{ matrix.r2dbc }}' -Dprotocol=${{ matrix.protocol }} verify - name: Upload test results uses: actions/upload-artifact@v2 @@ -315,12 +315,12 @@ jobs: java-version: 8 cache: "maven" - name: Install Java client - run: mvn --also-make --batch-mode --projects clickhouse-cli-client,clickhouse-grpc-client,clickhouse-http-client -DskipTests install + run: mvn --also-make --batch-mode --projects clickhouse-cli-client,clickhouse-grpc-client,clickhouse-http-client -Dj8 -DskipTests install - name: Test JDBC and R2DBC drivers run: | mvn --batch-mode --projects clickhouse-jdbc,clickhouse-r2dbc -DclickhouseVersion=22.8 \ -DclickhouseTimezone=${{ matrix.serverTz }} -Duser.timezone=${{ matrix.clientTz }} \ - -DskipUTs verify + -Dj8 -DskipUTs verify - name: Upload test results uses: actions/upload-artifact@v2 if: failure() diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c7458079b..5304261a6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: distribution: 'temurin' java-version: | 8 - 11 + 17 cache: 'maven' - name: Setup Toolchain shell: bash @@ -34,7 +34,7 @@ jobs: jdk - 11 + 17 ${{ env.JAVA_HOME }} diff --git a/README.md b/README.md index 6abca39ea..298d67178 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Java libraries for connecting to ClickHouse and processing data in various formats. Java client is async, lightweight, and low-overhead library for ClickHouse; while JDBC and R2DBC drivers are built on top of the Java client with more dependencies and features. Java 8 or higher is required to use the libraries. In addition, please use ClickHouse 20.7+ or any of [active releases](https://github.com/ClickHouse/ClickHouse/pulls?q=is%3Aopen+is%3Apr+label%3Arelease). -![image](https://user-images.githubusercontent.com/4270380/210317525-d548fbe5-8f66-4cc9-b34c-fd09d289b08e.png) +![image](https://user-images.githubusercontent.com/4270380/212460181-2b806482-bc1c-492c-bd69-cdeb2c8845b5.png) ## Features @@ -140,7 +140,7 @@ More examples can be found at [here](../../tree/master/examples/jdbc). ## Build with Maven -Use `mvn -DskipITs clean verify` to compile and generate packages if you're using JDK 8. To create a multi-release jar (see [JEP-238](https://openjdk.java.net/jeps/238)), please use JDK 11+ with `~/.m2/toolchains.xml` like below, and run `mvn -Drelease -DskipITs clean verify` instead. +Use `mvn -Dj8 -DskipITs clean verify` to compile and generate packages if you're using JDK 8. To create a multi-release jar (see [JEP-238](https://openjdk.java.net/jeps/238)), please use JDK 11+ with `~/.m2/toolchains.xml` like below, and run `mvn -DskipITs clean verify` instead. ```xml @@ -194,7 +194,7 @@ To benchmark JDBC drivers: ```bash cd clickhouse-benchmark -mvn -Drelease clean package +mvn clean package # single thread mode java -DdbHost=localhost -jar target/benchmarks.jar -t 1 \ -p client=clickhouse-jdbc -p connection=reuse \ diff --git a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/client/ClientState.java b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/client/ClientState.java index e33545b0f..56d451390 100644 --- a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/client/ClientState.java +++ b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/client/ClientState.java @@ -16,16 +16,16 @@ import com.clickhouse.client.ClickHouseClientBuilder; import com.clickhouse.client.ClickHouseCredentials; import com.clickhouse.client.ClickHouseException; -import com.clickhouse.client.ClickHouseCompression; -import com.clickhouse.client.ClickHouseFormat; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseProtocol; -import com.clickhouse.client.ClickHouseRecord; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponse; -import com.clickhouse.client.ClickHouseValue; import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.grpc.config.ClickHouseGrpcOption; +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseValue; @State(Scope.Thread) public class ClientState extends BaseState { diff --git a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/FactoryBenchmark.java b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/FactoryBenchmark.java index 5cbcc9153..d67682dbf 100644 --- a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/FactoryBenchmark.java +++ b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/FactoryBenchmark.java @@ -28,23 +28,23 @@ import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; import com.clickhouse.benchmark.BaseState; -import com.clickhouse.client.ClickHouseEnum; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; -import com.clickhouse.client.data.ClickHouseBigDecimalValue; -import com.clickhouse.client.data.ClickHouseBigIntegerValue; -import com.clickhouse.client.data.ClickHouseByteValue; -import com.clickhouse.client.data.ClickHouseDateTimeValue; -import com.clickhouse.client.data.ClickHouseDateValue; -import com.clickhouse.client.data.ClickHouseDoubleValue; -import com.clickhouse.client.data.ClickHouseEnumValue; -import com.clickhouse.client.data.ClickHouseFloatValue; -import com.clickhouse.client.data.ClickHouseIntegerValue; -import com.clickhouse.client.data.ClickHouseIpv4Value; -import com.clickhouse.client.data.ClickHouseIpv6Value; -import com.clickhouse.client.data.ClickHouseLongValue; -import com.clickhouse.client.data.ClickHouseShortValue; -import com.clickhouse.client.data.ClickHouseStringValue; +import com.clickhouse.data.ClickHouseEnum; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.value.ClickHouseBigDecimalValue; +import com.clickhouse.data.value.ClickHouseBigIntegerValue; +import com.clickhouse.data.value.ClickHouseByteValue; +import com.clickhouse.data.value.ClickHouseDateTimeValue; +import com.clickhouse.data.value.ClickHouseDateValue; +import com.clickhouse.data.value.ClickHouseDoubleValue; +import com.clickhouse.data.value.ClickHouseEnumValue; +import com.clickhouse.data.value.ClickHouseFloatValue; +import com.clickhouse.data.value.ClickHouseIntegerValue; +import com.clickhouse.data.value.ClickHouseIpv4Value; +import com.clickhouse.data.value.ClickHouseIpv6Value; +import com.clickhouse.data.value.ClickHouseLongValue; +import com.clickhouse.data.value.ClickHouseShortValue; +import com.clickhouse.data.value.ClickHouseStringValue; @State(Scope.Benchmark) @Warmup(iterations = 10, timeUnit = TimeUnit.SECONDS, time = 1) diff --git a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/QueueBenchmark.java b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/QueueBenchmark.java index 5d0ac90cd..70511a75b 100644 --- a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/QueueBenchmark.java +++ b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/QueueBenchmark.java @@ -21,17 +21,17 @@ import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; import com.clickhouse.benchmark.BaseState; -import com.clickhouse.client.ClickHouseByteBuffer; import com.clickhouse.client.ClickHouseClient; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseDataStreamFactory; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHousePipedOutputStream; -import com.clickhouse.client.config.ClickHouseBufferingMode; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseOption; -import com.clickhouse.client.data.BinaryStreamUtils; +import com.clickhouse.config.ClickHouseBufferingMode; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseDataStreamFactory; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePipedOutputStream; +import com.clickhouse.data.format.BinaryStreamUtils; @State(Scope.Benchmark) @Warmup(iterations = 10, timeUnit = TimeUnit.SECONDS, time = 1) diff --git a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/StreamBenchmark.java b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/StreamBenchmark.java index 2c1fe693d..c037b2e42 100644 --- a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/StreamBenchmark.java +++ b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/StreamBenchmark.java @@ -14,11 +14,11 @@ import com.clickhouse.benchmark.BaseState; import com.clickhouse.client.ClickHouseClient; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; import com.clickhouse.client.config.ClickHouseBufferingMode; import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; diff --git a/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/ClickHouseCommandLine.java b/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/ClickHouseCommandLine.java index 1c3f1fc6b..bf442b3df 100644 --- a/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/ClickHouseCommandLine.java +++ b/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/ClickHouseCommandLine.java @@ -4,7 +4,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.Serializable; @@ -21,30 +20,26 @@ import java.util.Map; import java.util.Optional; import java.util.WeakHashMap; -import java.util.concurrent.CancellationException; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseClient; -import com.clickhouse.client.ClickHouseCompression; import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseCredentials; -import com.clickhouse.client.ClickHouseFile; -import com.clickhouse.client.ClickHouseInputStream; import com.clickhouse.client.ClickHouseNode; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHousePassThruStream; import com.clickhouse.client.ClickHouseRequest; -import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.cli.config.ClickHouseCommandLineOption; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.data.ClickHouseExternalTable; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.client.config.ClickHouseSslMode; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseExternalTable; +import com.clickhouse.data.ClickHouseFile; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePassThruStream; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; public class ClickHouseCommandLine implements AutoCloseable { private static final Logger log = LoggerFactory.getLogger(ClickHouseCommandLine.class); @@ -230,8 +225,30 @@ static Process startProcess(ClickHouseRequest request) { if (config.isSsl()) { commands.add("--secure"); + + if (config.getSslMode() == ClickHouseSslMode.NONE) { + commands.add("--accept-invalid-certificate"); + } + } + if (config.isResponseCompressed()) { + commands.add("--compression=1"); + switch (config.getResponseCompressAlgorithm()) { + case LZ4: + commands.add("--network_compression_method=lz4"); + break; + case ZSTD: + commands.add("--network_compression_method=zstd"); + if (config.getResponseCompressLevel() >= 0 && config.getResponseCompressLevel() <= 22) { + commands.add("----network_zstd_compression_level=" + config.getResponseCompressLevel()); + } + break; + default: + break; + } + } else { + commands.add("--compression=0"); } - commands.add("--compression=".concat(config.isResponseCompressed() ? "1" : "0")); + commands.add("--host=".concat(server.getHost())); commands.add("--port=".concat(Integer.toString(server.getPort()))); @@ -263,7 +280,8 @@ static Process startProcess(ClickHouseRequest request) { commands.add("--query=".concat(str = request.getStatements(false).get(0))); for (ClickHouseExternalTable table : request.getExternalTables()) { - ClickHouseFile tableFile = table.getFile(); + ClickHouseFile tableFile = ClickHouseFile.of(table.getContent(), table.getCompression(), + table.getCompressionLevel(), table.getFormat()); commands.add("--external"); String filePath; if (!tableFile.hasOutput() || !tableFile.getFile().getAbsolutePath().startsWith(hostDir)) { @@ -330,9 +348,10 @@ static Process startProcess(ClickHouseRequest request) { throw new UncheckedIOException(new IOException("Output file not found in " + customStream)); } if (hostDir.equals(containerDir)) { - builder.redirectOutput(f); + // builder.redirectOutput(f); } else if (f.getAbsolutePath().startsWith(hostDir)) { String relativePath = f.getAbsolutePath().substring(hostDir.length()); + // FIXME overrided by below builder.redirectOutput(new File(containerDir.concat(relativePath))); } else { String fileName = f.getName(); @@ -370,17 +389,13 @@ static Process startProcess(ClickHouseRequest request) { final ClickHouseInputStream chInput = in.get(); final ClickHousePassThruStream customStream = chInput.getUnderlyingStream(); final File inputFile; - if (customStream.hasInput() && customStream instanceof ClickHouseFile) { - inputFile = ((ClickHouseFile) customStream).getFile(); + if (customStream.hasInput()) { + inputFile = customStream instanceof ClickHouseFile ? ((ClickHouseFile) customStream).getFile() + : ClickHouseFile.of(customStream.getInputStream(), config.getRequestCompressAlgorithm(), + config.getRequestCompressLevel(), config.getFormat()).getFile(); } else { - CompletableFuture data = ClickHouseClient.submit(() -> { - File tmp = ClickHouseUtils.createTempFile("tmp", "data", true); - try (ClickHouseOutputStream out = ClickHouseOutputStream.of(new FileOutputStream(tmp))) { - chInput.pipe(out); - } - return tmp; - }); - inputFile = data.get(timeout, TimeUnit.MILLISECONDS); + inputFile = ClickHouseFile.of(chInput, config.getRequestCompressAlgorithm(), + config.getRequestCompressLevel(), config.getFormat()).getFile(); } process = builder.redirectInput(inputFile).start(); } else { @@ -388,11 +403,6 @@ static Process startProcess(ClickHouseRequest request) { process.getOutputStream().close(); } return process; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new CompletionException(e); - } catch (CancellationException | ExecutionException | TimeoutException e) { - throw new CompletionException(e); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -400,18 +410,20 @@ static Process startProcess(ClickHouseRequest request) { private final ClickHouseRequest request; - private final Process process; - private String error; + private final Process process; + public ClickHouseCommandLine(ClickHouseRequest request) { this.request = request; - this.process = startProcess(request); this.error = null; + + this.process = startProcess(request); } public ClickHouseInputStream getInputStream() throws IOException { + ClickHouseConfig c = request.getConfig(); ClickHouseOutputStream out = request.getOutputStream().orElse(null); Runnable postCloseAction = () -> { IOException exp = getError(); @@ -421,13 +433,12 @@ public ClickHouseInputStream getInputStream() throws IOException { }; if (out != null && !out.getUnderlyingStream().hasOutput()) { try (OutputStream o = out) { - ClickHouseInputStream.pipe(process.getInputStream(), o, request.getConfig().getWriteBufferSize()); + ClickHouseInputStream.pipe(process.getInputStream(), o, c.getWriteBufferSize()); } - return ClickHouseInputStream.wrap(null, ClickHouseInputStream.empty(), - request.getConfig().getReadBufferSize(), postCloseAction, ClickHouseCompression.NONE, 0); + return ClickHouseInputStream.wrap(null, ClickHouseInputStream.empty(), c.getReadBufferSize(), + ClickHouseCompression.NONE, -1, postCloseAction); } else { - return ClickHouseInputStream.of(process.getInputStream(), request.getConfig().getReadBufferSize(), - postCloseAction); + return ClickHouseInputStream.of(process.getInputStream(), c.getReadBufferSize(), postCloseAction); } } diff --git a/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/ClickHouseCommandLineClient.java b/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/ClickHouseCommandLineClient.java index 1a87a2aa3..5c20597f1 100644 --- a/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/ClickHouseCommandLineClient.java +++ b/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/ClickHouseCommandLineClient.java @@ -12,10 +12,10 @@ import com.clickhouse.client.ClickHouseProtocol; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponse; -import com.clickhouse.client.config.ClickHouseOption; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; import com.clickhouse.client.cli.config.ClickHouseCommandLineOption; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; /** * Wrapper of ClickHouse native command-line client. diff --git a/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/ClickHouseCommandLineResponse.java b/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/ClickHouseCommandLineResponse.java index 659daeebe..6a7c2d8ec 100644 --- a/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/ClickHouseCommandLineResponse.java +++ b/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/ClickHouseCommandLineResponse.java @@ -5,7 +5,7 @@ import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseResponseSummary; -import com.clickhouse.client.data.ClickHouseStreamResponse; +import com.clickhouse.client.ClickHouseStreamResponse; public class ClickHouseCommandLineResponse extends ClickHouseStreamResponse { private final transient ClickHouseCommandLine cli; diff --git a/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/config/ClickHouseCommandLineOption.java b/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/config/ClickHouseCommandLineOption.java index b8ef108a3..360abed40 100644 --- a/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/config/ClickHouseCommandLineOption.java +++ b/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/config/ClickHouseCommandLineOption.java @@ -2,8 +2,8 @@ import java.io.Serializable; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.config.ClickHouseOption; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseChecker; public enum ClickHouseCommandLineOption implements ClickHouseOption { /** diff --git a/clickhouse-cli-client/src/test/java/com/clickhouse/client/cli/ClickHouseCommandLineClientTest.java b/clickhouse-cli-client/src/test/java/com/clickhouse/client/cli/ClickHouseCommandLineClientTest.java index b7f652483..a177e9678 100644 --- a/clickhouse-cli-client/src/test/java/com/clickhouse/client/cli/ClickHouseCommandLineClientTest.java +++ b/clickhouse-cli-client/src/test/java/com/clickhouse/client/cli/ClickHouseCommandLineClientTest.java @@ -8,6 +8,7 @@ import com.clickhouse.client.ClickHouseServerForTest; import com.clickhouse.client.ClientIntegrationTest; import com.clickhouse.client.cli.config.ClickHouseCommandLineOption; +import com.clickhouse.data.ClickHouseCompression; import java.io.IOException; @@ -15,6 +16,7 @@ import org.testng.Assert; import org.testng.SkipException; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class ClickHouseCommandLineClientTest extends ClientIntegrationTest { @@ -24,6 +26,30 @@ static void init() { ClickHouseServerForTest.getClickHouseContainerTmpDir()); } + @DataProvider(name = "requestCompressionMatrix") + protected Object[][] getRequestCompressionMatrix() { + return new Object[][] { + { ClickHouseCompression.NONE, -2, 2, 1 }, + { ClickHouseCompression.LZ4, -2, 19, 1 }, // [0, 18] + }; + } + + @DataProvider(name = "mixedCompressionMatrix") + protected Object[][] getMixedCompressionMatrix() { + ClickHouseCompression[] supportedRequestCompression = { ClickHouseCompression.NONE, ClickHouseCompression.LZ4 }; + ClickHouseCompression[] supportedResponseCompression = { ClickHouseCompression.NONE, + // ClickHouseCompression.LZ4 + }; + Object[][] matrix = new Object[supportedRequestCompression.length * supportedResponseCompression.length][]; + int i = 0; + for (ClickHouseCompression reqComp : supportedRequestCompression) { + for (ClickHouseCompression respComp : supportedResponseCompression) { + matrix[i++] = new Object[] { reqComp, respComp }; + } + } + return matrix; + } + @Override protected ClickHouseProtocol getProtocol() { return ClickHouseProtocol.TCP; @@ -50,6 +76,10 @@ protected ClickHouseNode getServer() { return super.getServer(); } + @Test(groups = { "integration" }) + public void testNothing() throws Exception { + } + @Test(groups = { "integration" }) @Override public void testCustomLoad() throws ClickHouseException { diff --git a/clickhouse-client/pom.xml b/clickhouse-client/pom.xml index b710da623..bf88ac598 100644 --- a/clickhouse-client/pom.xml +++ b/clickhouse-client/pom.xml @@ -17,6 +17,11 @@ https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-client + + ${project.parent.groupId} + clickhouse-data + ${revision} + ${project.parent.groupId} org.roaringbitmap @@ -29,21 +34,6 @@ - - com.aayushatharva.brotli4j - brotli4j - true - - - com.github.ben-manes.caffeine - caffeine - true - - - com.github.luben - zstd-jni - true - com.google.code.gson gson @@ -54,46 +44,11 @@ dnsjava true - - org.apache.avro - avro - true - - - org.brotli - dec - true - - - org.jctools - jctools-core - true - - - org.lz4 - lz4-java - true - - - org.msgpack - msgpack-core - true - org.slf4j slf4j-api true - - org.tukaani - xz - true - - - org.xerial.snappy - snappy-java - true - org.slf4j diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/AbstractClient.java b/clickhouse-client/src/main/java/com/clickhouse/client/AbstractClient.java index 72569c29d..9ec7c91d5 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/AbstractClient.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/AbstractClient.java @@ -11,8 +11,10 @@ import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseHealthCheckMethod; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; /** * Base class for implementing a thread-safe ClickHouse client. It uses diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/AbstractSocketClient.java b/clickhouse-client/src/main/java/com/clickhouse/client/AbstractSocketClient.java index 595a945b4..e496c914c 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/AbstractSocketClient.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/AbstractSocketClient.java @@ -19,8 +19,16 @@ import java.util.concurrent.locks.LockSupport; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseDataStreamFactory; +import com.clickhouse.data.ClickHouseFile; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePassThruStream; +import com.clickhouse.data.ClickHousePipedOutputStream; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; public class AbstractSocketClient implements AutoCloseable { static class SocketRequest { @@ -318,14 +326,14 @@ protected boolean start() throws IOException { if (key.isValid() && key.isWritable()) { final ClickHouseInputStream in = req.in; // socket output buffer was full due to slow/jammed network - long offset = (long) in.userData.getOrDefault("offset", 0L); + long offset = (long) in.getUserData("offset", 0L); if ((offset = onWrite(req.config, sc, in, offset)) > 0L && in.available() > 0) { // processRequest(in, socketTimeout); } else { in.close(); removeInterestOp(SelectionKey.OP_WRITE); } - in.userData.put("offset", offset); + in.setUserData("offset", offset); } if (key.isValid() && key.isReadable()) { diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java index 5feb8a6ad..7cc9caf67 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java @@ -18,10 +18,23 @@ import java.util.concurrent.TimeoutException; import java.util.function.Function; -import com.clickhouse.client.config.ClickHouseBufferingMode; import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseDefaults; -import com.clickhouse.client.config.ClickHouseOption; +import com.clickhouse.config.ClickHouseBufferingMode; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseDataStreamFactory; +import com.clickhouse.data.ClickHouseFile; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePipedOutputStream; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.ClickHouseWriter; /** * A unified interface defines Java client for ClickHouse. A client can only @@ -142,12 +155,12 @@ static ClickHouseOutputStream getAsyncRequestOutputStream(ClickHouseConfig confi static ClickHouseInputStream getResponseInputStream(ClickHouseConfig config, InputStream input, Runnable postCloseAction) { if (config == null) { - return ClickHouseInputStream.of(input, (int) ClickHouseClientOption.BUFFER_SIZE.getDefaultValue(), - ClickHouseCompression.NONE, postCloseAction); + return ClickHouseInputStream.of(input, ClickHouseDataConfig.getDefaultReadBufferSize(), + ClickHouseCompression.NONE, ClickHouseDataConfig.DEFAULT_READ_COMPRESS_LEVEL, postCloseAction); } return ClickHouseInputStream.of(input, config.getReadBufferSize(), config.getResponseCompressAlgorithm(), - postCloseAction); + config.getResponseCompressLevel(), postCloseAction); } /** @@ -183,7 +196,8 @@ static ClickHouseInputStream getAsyncResponseInputStream(ClickHouseConfig config wrappedInput = getResponseInputStream(config, decompressedStream.getInputStream(), postCloseAction); submit(() -> { try (ClickHouseInputStream in = ClickHouseInputStream.of(input, config.getReadBufferSize(), - config.getResponseCompressAlgorithm(), null); ClickHouseOutputStream out = decompressedStream) { + config.getResponseCompressAlgorithm(), config.getResponseCompressLevel(), null); + ClickHouseOutputStream out = decompressedStream) { in.pipe(out); } return null; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClientBuilder.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClientBuilder.java index 2e1139019..cfc1f532b 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClientBuilder.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClientBuilder.java @@ -19,11 +19,14 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; -import com.clickhouse.client.config.ClickHouseOption; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; import com.clickhouse.client.ClickHouseNode.Status; import com.clickhouse.client.config.ClickHouseDefaults; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseThreadFactory; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; /** * Builder class for creating {@link ClickHouseClient}. Please use @@ -518,7 +521,7 @@ public ClickHouseClientBuilder config(ClickHouseConfig config) { /** * Adds an option, which is usually an Enum type that implements - * {@link com.clickhouse.client.config.ClickHouseOption}. + * {@link com.clickhouse.config.ClickHouseOption}. * * @param option non-null option * @param value value diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCluster.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCluster.java index a5fcad858..f58186495 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCluster.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCluster.java @@ -7,6 +7,8 @@ import java.util.List; import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseUtils; public class ClickHouseCluster extends ClickHouseNodes { private static final long serialVersionUID = 8684489015067906319L; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseConfig.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseConfig.java index e2383f16c..01306e21b 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseConfig.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseConfig.java @@ -14,17 +14,23 @@ import java.util.TimeZone; import java.util.Map.Entry; -import com.clickhouse.client.config.ClickHouseBufferingMode; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseOption; import com.clickhouse.client.config.ClickHouseDefaults; import com.clickhouse.client.config.ClickHouseSslMode; +import com.clickhouse.config.ClickHouseBufferingMode; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.config.ClickHouseRenameMethod; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseVersion; /** * An immutable class holding client-specific options like * {@link ClickHouseCredentials} and {@link ClickHouseNodeSelector} etc. */ -public class ClickHouseConfig implements Serializable { +public class ClickHouseConfig implements ClickHouseDataConfig { static final class ClientOptions { private static final ClientOptions INSTANCE = new ClientOptions(); @@ -283,8 +289,8 @@ public ClickHouseConfig(Map options, ClickHouseC this.connectionTimeout = getIntOption(ClickHouseClientOption.CONNECTION_TIMEOUT); this.database = (String) getOption(ClickHouseClientOption.DATABASE, ClickHouseDefaults.DATABASE); this.format = (ClickHouseFormat) getOption(ClickHouseClientOption.FORMAT, ClickHouseDefaults.FORMAT); - this.maxBufferSize = ClickHouseUtils.getBufferSize(getIntOption(ClickHouseClientOption.MAX_BUFFER_SIZE), -1, - -1); + this.maxBufferSize = ClickHouseDataConfig.getBufferSize(getIntOption(ClickHouseClientOption.MAX_BUFFER_SIZE), + -1, -1); this.bufferSize = getIntOption(ClickHouseClientOption.BUFFER_SIZE); this.bufferQueueVariation = getIntOption(ClickHouseClientOption.BUFFER_QUEUE_VARIATION); this.readBufferSize = getIntOption(ClickHouseClientOption.READ_BUFFER_SIZE); @@ -344,6 +350,7 @@ public ClickHouseConfig(Map options, ClickHouseC this.nodeSelector = nodeSelector == null ? ClickHouseNodeSelector.EMPTY : nodeSelector; } + @Override public boolean isAsync() { return async; } @@ -426,6 +433,7 @@ public String getDatabase() { return database; } + @Override public ClickHouseFormat getFormat() { return format; } @@ -434,51 +442,29 @@ public int getNodeCheckInterval() { return nodeCheckInterval; } - /** - * Gets max buffer size in byte can be used for streaming. - * - * @return max buffer size in byte - */ + @Override public int getMaxBufferSize() { return maxBufferSize; } - /** - * Gets default buffer size in byte for both read and write. - * - * @return default buffer size in byte - */ + @Override public int getBufferSize() { return bufferSize; } - /** - * Gets number of times the buffer queue is filled up before - * increasing capacity of buffer queue. Zero or negative value means the queue - * length is fixed. - * - * @return variation - */ + @Override public int getBufferQueueVariation() { return bufferQueueVariation; } - /** - * Gets read buffer size in byte. - * - * @return read buffer size in byte - */ + @Override public int getReadBufferSize() { - return ClickHouseUtils.getBufferSize(readBufferSize, getBufferSize(), getMaxBufferSize()); + return ClickHouseDataConfig.getBufferSize(readBufferSize, getBufferSize(), getMaxBufferSize()); } - /** - * Gets write buffer size in byte. - * - * @return write buffer size in byte - */ + @Override public int getWriteBufferSize() { - return ClickHouseUtils.getBufferSize(writeBufferSize, getBufferSize(), getMaxBufferSize()); + return ClickHouseDataConfig.getBufferSize(writeBufferSize, getBufferSize(), getMaxBufferSize()); } /** @@ -487,7 +473,7 @@ public int getWriteBufferSize() { * @return request chunk size */ public int getRequestChunkSize() { - return ClickHouseUtils.getBufferSize(requestChunkSize, getWriteBufferSize(), getMaxBufferSize()); + return ClickHouseDataConfig.getBufferSize(requestChunkSize, getWriteBufferSize(), getMaxBufferSize()); } /** @@ -512,6 +498,7 @@ public int getMaxExecutionTime() { return maxExecutionTime; } + @Override public int getMaxQueuedBuffers() { return maxQueuedBuffers; } @@ -544,6 +531,7 @@ public boolean isRepeatOnSessionLock() { return repeatOnSessionLock; } + @Override public boolean isReuseValueWrapper() { return reuseValueWrapper; } @@ -601,18 +589,22 @@ public int getTransactionTimeout() { return transactionTimeout < 1 ? sessionTimeout : transactionTimeout; } + @Override public boolean isWidenUnsignedTypes() { return widenUnsignedTypes; } + @Override public boolean isUseBinaryString() { return useBinaryString; } + @Override public boolean isUseBlockingQueue() { return useBlockingQueue; } + @Override public boolean isUseObjectsInArray() { return useObjectsInArray; } @@ -635,6 +627,7 @@ public boolean isUseServerTimeZoneForDates() { * @return time zone, could be null when {@code use_server_time_zone_for_date} * is set to {@code false}. */ + @Override public TimeZone getTimeZoneForDate() { return timeZoneForDate; } @@ -645,6 +638,7 @@ public TimeZone getTimeZoneForDate() { * * @return non-null preferred time zone */ + @Override public TimeZone getUseTimeZone() { return useTimeZone; } @@ -850,4 +844,29 @@ public boolean equals(Object obj) { && Objects.equals(metricRegistry.orElse(null), other.metricRegistry.orElse(null)) && Objects.equals(nodeSelector, other.nodeSelector); } + + @Override + public int getReadTimeout() { + return getSocketTimeout(); + } + + @Override + public int getWriteTimeout() { + return getSocketTimeout(); + } + + @Override + public ClickHouseBufferingMode getReadBufferingMode() { + return getResponseBuffering(); + } + + @Override + public ClickHouseBufferingMode getWriteBufferingMode() { + return getRequestBuffering(); + } + + @Override + public ClickHouseRenameMethod getColumnRenameMethod() { + return (ClickHouseRenameMethod) getOption(ClickHouseClientOption.RENAME_RESPONSE_COLUMN); + } } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCredentials.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCredentials.java index 4adadf380..e9bf2c582 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCredentials.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCredentials.java @@ -3,6 +3,8 @@ import java.io.Serializable; import java.util.Objects; +import com.clickhouse.data.ClickHouseChecker; + /** * This encapsulates access token, certificate or user name password combination * for accessing ClickHouse. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDnsResolver.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDnsResolver.java index 7eee3549f..641d4bd12 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDnsResolver.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDnsResolver.java @@ -3,9 +3,10 @@ import java.net.InetSocketAddress; import com.clickhouse.client.config.ClickHouseDefaults; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; import com.clickhouse.client.naming.SrvResolver; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; /** * Default DNS resolver. It tries to look up service record (SRV record) when diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseFile.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseFile.java deleted file mode 100644 index 0d429a75c..000000000 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseFile.java +++ /dev/null @@ -1,112 +0,0 @@ -package com.clickhouse.client; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.nio.file.Path; - -import com.clickhouse.client.config.ClickHouseClientOption; - -/** - * Wrapper of {@link java.io.File} with additional information like compression - * and format. - */ -public class ClickHouseFile extends ClickHousePassThruStream { - private static final long serialVersionUID = -2641191818870839568L; - - /** - * Null file which has no compression and format. - */ - public static final ClickHouseFile NULL = new ClickHouseFile(null, ClickHouseCompression.NONE, -1, null); - - public static ClickHouseFile of(File file) { - return of(file, null, 0, null); - } - - public static ClickHouseFile of(Path path) { - return of(ClickHouseChecker.nonNull(path, "Path").toFile(), null, -1, null); - } - - public static ClickHouseFile of(String file) { - return of(new File(ClickHouseChecker.nonEmpty(file, "File")), null, -1, null); - } - - public static ClickHouseFile of(String file, ClickHouseCompression compression, int compressionLevel, - ClickHouseFormat format) { - return of(new File(ClickHouseChecker.nonEmpty(file, "File")), compression, compressionLevel, format); - } - - public static ClickHouseFile of(File file, ClickHouseCompression compression, int compressionLevel, - ClickHouseFormat format) { - return new ClickHouseFile(ClickHouseChecker.nonNull(file, "File"), - compression != null ? compression : ClickHouseCompression.fromFileName(file.getName()), - compressionLevel, format != null ? format : ClickHouseFormat.fromFileName(file.getName())); - } - - private final File file; - - protected ClickHouseFile(File file, ClickHouseCompression compress, int compressLevel, ClickHouseFormat format) { - super(null, null, compress, compressLevel, format); - - this.file = file; - } - - @Override - public ClickHouseInputStream asInputStream() { - return asInputStream((int) ClickHouseClientOption.READ_BUFFER_SIZE.getDefaultValue(), null); - } - - @Override - public ClickHouseInputStream asInputStream(int bufferSize, Runnable postCloseAction) { - if (!hasInput()) { - return ClickHouseInputStream.empty(); - } - - try { - return ClickHouseInputStream.wrap(this, new FileInputStream(getFile()), bufferSize, postCloseAction, - ClickHouseCompression.NONE, getCompressionLevel()); - } catch (FileNotFoundException e) { - throw new IllegalArgumentException(e); - } - } - - @Override - public ClickHouseOutputStream asOutputStream() { - return asOutputStream((int) ClickHouseClientOption.WRITE_BUFFER_SIZE.getDefaultValue(), null); - } - - @Override - public ClickHouseOutputStream asOutputStream(int bufferSize, Runnable postCloseAction) { - if (!hasOutput()) { - return ClickHouseOutputStream.empty(); - } - - try { - return ClickHouseOutputStream.wrap(this, new FileOutputStream(getFile()), - bufferSize, postCloseAction, ClickHouseCompression.NONE, getCompressionLevel()); - } catch (FileNotFoundException e) { - throw new IllegalArgumentException(e); - } - } - - /** - * Gets file. Use {@code #hasInput()} or {@code #hasOutput()} to check file - * availability first. - * - * @return file, could be null - */ - public File getFile() { - return file; - } - - @Override - public boolean hasInput() { - return file != null && file.exists(); - } - - @Override - public boolean hasOutput() { - return file != null; - } -} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseLoadBalancingPolicy.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseLoadBalancingPolicy.java index 913043db9..482a78156 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseLoadBalancingPolicy.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseLoadBalancingPolicy.java @@ -11,6 +11,8 @@ import java.util.concurrent.TimeUnit; import com.clickhouse.client.ClickHouseNode.Status; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseUtils; /** * Load balancing policy. In general, a policy is responsible for 3 things: 1) diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java index bbe24c1ce..d4ef84abe 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java @@ -33,10 +33,13 @@ import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseDefaults; -import com.clickhouse.client.config.ClickHouseOption; import com.clickhouse.client.config.ClickHouseSslMode; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseVersion; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; /** * This class depicts a ClickHouse server, essentially a combination of host, diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNodeSelector.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNodeSelector.java index 756c1f2c6..66ff6067f 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNodeSelector.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNodeSelector.java @@ -9,8 +9,9 @@ import java.util.List; import java.util.Set; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; /** * This class maintains two immutable collections: preferred protocols and tags. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNodes.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNodes.java index fc4ba3735..7d1bc95d3 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNodes.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNodes.java @@ -23,8 +23,12 @@ import com.clickhouse.client.ClickHouseNode.Status; import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseDefaults; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; /** * A generic node manager for managing one or more nodes which may or may not diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseParameterizedQuery.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseParameterizedQuery.java index efa5ffae4..1818cd723 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseParameterizedQuery.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseParameterizedQuery.java @@ -13,6 +13,12 @@ import java.util.Objects; import java.util.Map.Entry; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; + /** * A parameterized query is a parsed query with parameters being extracted for * substitution. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java index 89357f3c6..e75c37323 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java @@ -31,9 +31,24 @@ import java.util.function.Function; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseConfigChangeListener; -import com.clickhouse.client.config.ClickHouseOption; -import com.clickhouse.client.data.ClickHouseExternalTable; +import com.clickhouse.config.ClickHouseBufferingMode; +import com.clickhouse.config.ClickHouseConfigChangeListener; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseDataStreamFactory; +import com.clickhouse.data.ClickHouseDeferredValue; +import com.clickhouse.data.ClickHouseExternalTable; +import com.clickhouse.data.ClickHouseFile; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePassThruStream; +import com.clickhouse.data.ClickHousePipedOutputStream; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.ClickHouseWriter; /** * Request object holding references to {@link ClickHouseClient}, @@ -41,8 +56,6 @@ */ @SuppressWarnings("squid:S119") public class ClickHouseRequest> implements Serializable { - private static final String TYPE_EXTERNAL_TABLE = "ExternalTable"; - private static final Set SPECIAL_SETTINGS; static final String PARAM_SETTING = "setting"; @@ -60,7 +73,7 @@ public class ClickHouseRequest> implement * Mutation request. */ public static class Mutation extends ClickHouseRequest { - private ClickHouseWriter writer; + private transient ClickHouseWriter writer; protected Mutation(ClickHouseRequest request, boolean sealed) { super(request.getClient(), request.server, request.serverRef, request.options, sealed); @@ -152,22 +165,32 @@ public Mutation data(ClickHouseWriter writer) { } /** - * Loads data from given file which may or may not be compressed. + * Loads data from given pass-thru stream which may or may not be compressed. * - * @param file absolute or relative path of the file, file extension will be - * used to determine if it's compressed or not + * @param stream pass-thru stream, could be a file and may or may not be + * compressed * @return mutation request */ - public Mutation data(ClickHouseFile file) { + public Mutation data(ClickHousePassThruStream stream) { checkSealed(); - final ClickHouseRequest self = this; - if (ClickHouseChecker.nonNull(file, "File").hasFormat()) { - format(file.getFormat()); + if (!ClickHouseChecker.nonNull(stream, ClickHousePassThruStream.TYPE_NAME).hasInput()) { + throw new IllegalArgumentException(ClickHousePassThruStream.ERROR_NO_INPUT); + } + + final ClickHouseConfig c = getConfig(); + if (stream.hasCompression()) { + decompressClientRequest(stream.isCompressed(), stream.getCompressionAlgorithm(), + stream.getCompressionLevel()); + } else if (c.isRequestCompressed()) { + decompressClientRequest(false); } - decompressClientRequest(file.isCompressed(), file.getCompressionAlgorithm()); + if (stream.hasFormat()) { + format(stream.getFormat()); + } + final int bufferSize = c.getReadBufferSize(); this.input = changeProperty(PROP_DATA, this.input, ClickHouseDeferredValue - .of(() -> ClickHouseInputStream.of(file, self.getConfig().getReadBufferSize(), null))); + .of(() -> ClickHouseInputStream.of(stream, bufferSize, null))); return this; } @@ -179,7 +202,7 @@ public Mutation data(ClickHouseFile file) { * @return mutation request */ public Mutation data(String file) { - return data(file, ClickHouseCompression.fromFileName(file)); + return data(ClickHouseFile.of(file)); } /** @@ -190,18 +213,23 @@ public Mutation data(String file) { * means no compression * @return mutation request */ - @SuppressWarnings("squid:S2095") public Mutation data(String file, ClickHouseCompression compression) { - checkSealed(); + return data(ClickHouseFile.of(file, compression, + (int) ClickHouseClientOption.DECOMPRESS_LEVEL.getDefaultValue(), null)); + } - final ClickHouseRequest self = this; - final ClickHouseFile wrappedFile = ClickHouseFile.of(file, compression, 0, null); - if (wrappedFile.hasFormat() && getFormat().defaultInputFormat() != wrappedFile.getFormat()) { - format(wrappedFile.getFormat()); - } - this.input = changeProperty(PROP_DATA, this.input, ClickHouseDeferredValue - .of(() -> ClickHouseInputStream.of(wrappedFile, self.getConfig().getReadBufferSize(), null))); - return this; + /** + * Loads compressed data from given file. + * + * @param file absolute or relative path of the file + * @param compression compression algorithm, + * {@link ClickHouseCompression#NONE} + * means no compression + * @param compressionLevel compression level + * @return mutation request + */ + public Mutation data(String file, ClickHouseCompression compression, int compressionLevel) { + return data(ClickHouseFile.of(file, compression, compressionLevel, null)); } /** @@ -221,11 +249,16 @@ public Mutation data(InputStream input) { * @return mutation request */ public Mutation data(ClickHouseInputStream input) { + ClickHousePassThruStream stream = ClickHouseChecker.nonNull(input, ClickHouseInputStream.TYPE_NAME) + .getUnderlyingStream(); + if (stream.hasInput()) { + return data(stream); + } + checkSealed(); this.input = changeProperty(PROP_DATA, this.input, ClickHouseDeferredValue.of(input, ClickHouseInputStream.class)); - return this; } @@ -246,19 +279,28 @@ public Mutation data(ClickHouseDeferredValue input) { @Override public CompletableFuture execute() { if (writer != null) { - ClickHouseConfig c = getConfig(); - ClickHousePipedOutputStream stream = ClickHouseDataStreamFactory.getInstance() + final ClickHouseConfig c = getConfig(); + final boolean perfMode = c.getResponseBuffering() == ClickHouseBufferingMode.PERFORMANCE; + final ClickHousePipedOutputStream stream = ClickHouseDataStreamFactory.getInstance() .createPipedOutputStream(c, null); data(stream.getInputStream()); CompletableFuture future = null; if (c.isAsync()) { future = getClient().execute(this); } - try (ClickHouseOutputStream out = stream) { - writer.write(out); - } catch (IOException e) { - throw new CompletionException(e); + + if (perfMode) { + ClickHouseClient.submit(() -> { + + }); + } else { + try (ClickHouseOutputStream out = stream) { + writer.write(out); + } catch (IOException e) { + throw new CompletionException(e); + } } + if (future != null) { return future; } @@ -903,7 +945,7 @@ public SelfT decompressClientRequest(boolean enable, ClickHouseCompression compr public SelfT addExternal(ClickHouseExternalTable table) { checkSealed(); - if (externalTables.add(ClickHouseChecker.nonNull(table, TYPE_EXTERNAL_TABLE))) { + if (externalTables.add(ClickHouseChecker.nonNull(table, ClickHouseExternalTable.TYPE_NAME))) { resetCache(); } @@ -922,10 +964,10 @@ public SelfT external(ClickHouseExternalTable table, ClickHouseExternalTable... checkSealed(); externalTables.clear(); - externalTables.add(ClickHouseChecker.nonNull(table, TYPE_EXTERNAL_TABLE)); + externalTables.add(ClickHouseChecker.nonNull(table, ClickHouseExternalTable.TYPE_NAME)); if (more != null) { for (ClickHouseExternalTable e : more) { - externalTables.add(ClickHouseChecker.nonNull(e, TYPE_EXTERNAL_TABLE)); + externalTables.add(ClickHouseChecker.nonNull(e, ClickHouseExternalTable.TYPE_NAME)); } } @@ -945,7 +987,7 @@ public SelfT external(Collection tables) { externalTables.clear(); if (tables != null) { for (ClickHouseExternalTable e : tables) { - externalTables.add(ClickHouseChecker.nonNull(e, TYPE_EXTERNAL_TABLE)); + externalTables.add(ClickHouseChecker.nonNull(e, ClickHouseExternalTable.TYPE_NAME)); } } @@ -1108,22 +1150,32 @@ public boolean hasOption(String key) { } /** - * Sets output file, to which response will be redirected. + * Sets pass-thru stream, to which response will be redirected. * - * @param file non-null output file + * @param stream non-null pass-thru stream * @return the request itself */ @SuppressWarnings("unchecked") - public SelfT output(ClickHouseFile file) { + public SelfT output(ClickHousePassThruStream stream) { checkSealed(); - final int bufferSize = getConfig().getWriteBufferSize(); - if (ClickHouseChecker.nonNull(file, "File").hasFormat()) { - format(file.getFormat()); + if (!ClickHouseChecker.nonNull(stream, ClickHousePassThruStream.TYPE_NAME).hasOutput()) { + throw new IllegalArgumentException(ClickHousePassThruStream.ERROR_NO_OUTPUT); + } + + final ClickHouseConfig c = getConfig(); + if (stream.hasCompression()) { + compressServerResponse(stream.isCompressed(), stream.getCompressionAlgorithm(), + stream.getCompressionLevel()); + } else if (c.isResponseCompressed()) { + compressServerResponse(false); } - compressServerResponse(file.isCompressed(), file.getCompressionAlgorithm(), file.getCompressionLevel()); + if (stream.hasFormat()) { + format(stream.getFormat()); + } + final int bufferSize = c.getWriteBufferSize(); this.output = changeProperty(PROP_OUTPUT, this.output, ClickHouseDeferredValue - .of(() -> ClickHouseOutputStream.of(file, bufferSize, null))); + .of(() -> ClickHouseOutputStream.of(stream, bufferSize, null))); return (SelfT) this; } @@ -1135,7 +1187,7 @@ public SelfT output(ClickHouseFile file) { * @return the request itself */ public SelfT output(String file) { - return output(file, ClickHouseCompression.fromFileName(file)); + return output(ClickHouseFile.of(file)); } /** @@ -1146,18 +1198,23 @@ public SelfT output(String file) { * {@link ClickHouseCompression#NONE} means no compression * @return the request itself */ - @SuppressWarnings("unchecked") public SelfT output(String file, ClickHouseCompression compression) { - checkSealed(); + return output(ClickHouseFile.of(file, compression, + (int) ClickHouseClientOption.COMPRESS_LEVEL.getDefaultValue(), null)); + } - final int bufferSize = getConfig().getWriteBufferSize(); - final ClickHouseFile wrappedFile = ClickHouseFile.of(file, compression, 0, null); - if (wrappedFile.hasFormat()) { - format(wrappedFile.getFormat()); - } - this.output = changeProperty(PROP_OUTPUT, this.output, ClickHouseDeferredValue - .of(() -> ClickHouseOutputStream.of(wrappedFile, bufferSize, null))); - return (SelfT) this; + /** + * Sets compressed output file, to which response will be redirected. + * + * @param file non-empty path to the file + * @param compression compression algorithm, {@code null} or + * {@link ClickHouseCompression#NONE} means no + * compression + * @param compressionLevel compression level + * @return the request itself + */ + public SelfT output(String file, ClickHouseCompression compression, int compressionLevel) { + return output(ClickHouseFile.of(file, compression, compressionLevel, null)); } /** @@ -1178,6 +1235,12 @@ public SelfT output(OutputStream output) { */ @SuppressWarnings("unchecked") public SelfT output(ClickHouseOutputStream output) { + ClickHousePassThruStream stream = ClickHouseChecker.nonNull(output, ClickHouseOutputStream.TYPE_NAME) + .getUnderlyingStream(); + if (stream.hasOutput()) { + return output(stream); + } + checkSealed(); this.output = changeProperty(PROP_OUTPUT, this.output, @@ -1850,7 +1913,7 @@ public SelfT use(String database) { public SelfT removeExternal(ClickHouseExternalTable external) { checkSealed(); - if (externalTables.remove(ClickHouseChecker.nonNull(external, TYPE_EXTERNAL_TABLE))) { + if (externalTables.remove(ClickHouseChecker.nonNull(external, ClickHouseExternalTable.TYPE_NAME))) { resetCache(); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequestManager.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequestManager.java index 3a33fbf24..ecb3455d3 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequestManager.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequestManager.java @@ -2,6 +2,9 @@ import java.util.UUID; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseUtils; + /** * Request manager is responsible for generating query and session ID, as well * as transaction creation. {@link java.util.ServiceLoader} will search and diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseResponse.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseResponse.java index de8c6749b..447518fb8 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseResponse.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseResponse.java @@ -13,12 +13,18 @@ import java.util.stream.StreamSupport; import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseRecord; /** * This encapsulates a server reponse. Depending on concrete implementation, it * could be either an in-memory list or a wrapped input stream with - * {@link ClickHouseDataProcessor} attached for deserialization. To get data - * returned from server, depending on actual needs, you have 3 options: + * {@link com.clickhouse.data.ClickHouseDataProcessor} attached for + * deserialization. To get data returned from server, depending on actual needs, + * you have 3 options: * *
    *
  • use {@link #records()} or {@link #stream()} to get deserialized @@ -125,7 +131,7 @@ default ClickHouseRecord firstRecord() { */ default void pipe(OutputStream output, int bufferSize) throws IOException { ClickHouseInputStream.pipe(getInputStream(), ClickHouseChecker.nonNull(output, "output"), - ClickHouseUtils.getBufferSize(bufferSize, + ClickHouseDataConfig.getBufferSize(bufferSize, (int) ClickHouseClientOption.BUFFER_SIZE.getDefaultValue(), (int) ClickHouseClientOption.MAX_BUFFER_SIZE.getDefaultValue())); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseSimpleResponse.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseSimpleResponse.java similarity index 92% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseSimpleResponse.java rename to clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseSimpleResponse.java index 515b4eb7f..9fdf647a7 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseSimpleResponse.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseSimpleResponse.java @@ -1,22 +1,23 @@ -package com.clickhouse.client.data; +package com.clickhouse.client; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedList; import java.util.List; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseRecord; -import com.clickhouse.client.ClickHouseResponse; -import com.clickhouse.client.ClickHouseResponseSummary; -import com.clickhouse.client.ClickHouseValue; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseRecordTransformer; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.format.ClickHouseSimpleRecord; /** * A simple response built on top of two lists: columns and records. */ public class ClickHouseSimpleResponse implements ClickHouseResponse { + private static final long serialVersionUID = 6883452584393840649L; + /** * Creates a response object using columns definition and raw values. * @@ -138,7 +139,7 @@ protected ClickHouseSimpleResponse(List columns, ClickHouseVal int len = values.length; List list = new ArrayList<>(len); for (int i = 0; i < len; i++) { - list.add(new ClickHouseSimpleRecord(columns, values[i])); + list.add(ClickHouseSimpleRecord.of(columns, values[i])); } this.records = Collections.unmodifiableList(list); diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseSslContextProvider.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseSslContextProvider.java index b95ff7bdb..544e09e27 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseSslContextProvider.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseSslContextProvider.java @@ -3,6 +3,8 @@ import java.util.Optional; import javax.net.ssl.SSLException; +import com.clickhouse.data.ClickHouseUtils; + /** * This interface defines how to build Netty SSL context based on given * configuration and target server. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseStreamResponse.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseStreamResponse.java similarity index 89% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseStreamResponse.java rename to clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseStreamResponse.java index 0b5cfa42d..52c5bcf86 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseStreamResponse.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseStreamResponse.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.client; import java.io.IOException; import java.io.Serializable; @@ -6,17 +6,14 @@ import java.util.List; import java.util.Map; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseDataProcessor; -import com.clickhouse.client.ClickHouseDataStreamFactory; -import com.clickhouse.client.ClickHouseFormat; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseRecord; -import com.clickhouse.client.ClickHouseResponse; -import com.clickhouse.client.ClickHouseResponseSummary; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataProcessor; +import com.clickhouse.data.ClickHouseDataStreamFactory; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; /** * A stream response from server. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseTransaction.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseTransaction.java index a846abe80..993b14e44 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseTransaction.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseTransaction.java @@ -9,8 +9,11 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; /** * This class represents a transaction in ClickHouse. Besides transaction ID diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseWriter.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseWriter.java deleted file mode 100644 index 96ac31f87..000000000 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseWriter.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.clickhouse.client; - -import java.io.IOException; - -@FunctionalInterface -public interface ClickHouseWriter { - /** - * Writes value to output stream. - * - * @param output non-null output stream - * @throws IOException when failed to write data to output stream - */ - void write(ClickHouseOutputStream output) throws IOException; -} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java b/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java index 56f4308e4..b70cb9f36 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java @@ -7,9 +7,10 @@ import java.util.HashMap; import java.util.Map; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseCompression; -import com.clickhouse.client.ClickHouseFormat; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseDataConfig; /** * Generic client options. @@ -20,7 +21,7 @@ public enum ClickHouseClientOption implements ClickHouseOption { * {@link com.clickhouse.client.ClickHouseClient#execute(com.clickhouse.client.ClickHouseRequest)} * in a separate thread). */ - ASYNC("async", true, "Whether the client should run in async mode."), + ASYNC("async", ClickHouseDataConfig.DEFAULT_ASYNC, "Whether the client should run in async mode."), /** * Whether the client should discover more nodes from system tables and/or * clickhouse-keeper/zookeeper. @@ -78,12 +79,13 @@ public enum ClickHouseClientOption implements ClickHouseOption { * Default buffer size in byte for both request and response. It will be reset * to {@link #MAX_BUFFER_SIZE} if it's too large. */ - BUFFER_SIZE("buffer_size", 8192, "Default buffer size in byte for both request and response."), + BUFFER_SIZE("buffer_size", ClickHouseDataConfig.DEFAULT_BUFFER_SIZE, + "Default buffer size in byte for both request and response."), /** * Number of times the buffer queue is filled up before increasing capacity of * buffer queue. Zero or negative value means the queue length is fixed. */ - BUFFER_QUEUE_VARIATION("buffer_queue_variation", 100, + BUFFER_QUEUE_VARIATION("buffer_queue_variation", ClickHouseDataConfig.DEFAULT_BUFFER_QUEUE_VARIATION, "Number of times the buffer queue is filled up before increasing capacity of buffer queue. Zero or negative value means the queue length is fixed."), /** * Read buffer size in byte. It's mainly for input stream(e.g. reading data from @@ -117,7 +119,7 @@ public enum ClickHouseClientOption implements ClickHouseOption { /** * Client name. */ - CLIENT_NAME("client_name", DEFAULT_CLIENT_NAME, + CLIENT_NAME("client_name", "ClickHouse Java Client", "Client name, which is either 'client_name' or 'http_user_agent' shows up in system.query_log table."), /** * Whether server will compress response to client or not. @@ -142,11 +144,13 @@ public enum ClickHouseClientOption implements ClickHouseOption { /** * Compression level for compressing server response. */ - COMPRESS_LEVEL("compress_level", -1, "Compression level for response, -1 standards for default"), + COMPRESS_LEVEL("compress_level", ClickHouseDataConfig.DEFAULT_READ_COMPRESS_LEVEL, + "Compression level for response, -1 standards for default"), /** * Compression level for decompress client request. */ - DECOMPRESS_LEVEL("decompress_level", -1, "Compression level for request, -1 standards for default"), + DECOMPRESS_LEVEL("decompress_level", ClickHouseDataConfig.DEFAULT_WRITE_COMPRESS_LEVEL, + "Compression level for request, -1 standards for default"), /** * Connection timeout in milliseconds. @@ -165,7 +169,7 @@ public enum ClickHouseClientOption implements ClickHouseOption { /** * Default format. */ - FORMAT("format", ClickHouseFormat.TabSeparated, "Default format."), + FORMAT("format", ClickHouseDataConfig.DEFAULT_FORMAT, "Default format."), /** * Whether to log leading comment(as log_comment in system.query_log) of the * query. @@ -176,7 +180,7 @@ public enum ClickHouseClientOption implements ClickHouseOption { * Maximum buffer size in byte can be used for streaming. It's not supposed to * be larger than {@code Integer.MAX_VALUE - 8}. */ - MAX_BUFFER_SIZE("max_buffer_size", 1024 * 1024 * 1024, + MAX_BUFFER_SIZE("max_buffer_size", ClickHouseDataConfig.DEFAULT_MAX_BUFFER_SIZE, "Maximum buffer size in byte can be used for streaming."), /** * Maximum query execution time in seconds. @@ -185,7 +189,7 @@ public enum ClickHouseClientOption implements ClickHouseOption { /** * Maximum queued in-memory buffers. */ - MAX_QUEUED_BUFFERS("max_queued_buffers", 512, + MAX_QUEUED_BUFFERS("max_queued_buffers", ClickHouseDataConfig.DEFAULT_MAX_QUEUED_BUFFERS, "Maximum queued in-memory buffers, 0 or negative number means no limit."), /** * Maxium queued requests. When {@link #MAX_THREADS_PER_CLIENT} is greater than @@ -206,11 +210,11 @@ public enum ClickHouseClientOption implements ClickHouseOption { /** * Product name usered in user agent. */ - PRODUCT_NAME("product_name", DEFAULT_PRODUCT_NAME, "Product name used in user agent."), + PRODUCT_NAME("product_name", "ClickHouse-JavaClient", "Product name used in user agent."), /** * Method to rename response columns. */ - RENAME_RESPONSE_COLUMN("rename_response_column", ClickHouseRenameMethod.NONE, + RENAME_RESPONSE_COLUMN("rename_response_column", ClickHouseDataConfig.DEFAULT_COLUMN_RENAME_METHOD, "Method to rename response columns."), /** * Maximum number of times retry can happen for a request. @@ -227,7 +231,7 @@ public enum ClickHouseClientOption implements ClickHouseOption { * Whether to reuse wrapper of value(e.g. ClickHouseValue or * ClickHouseRecord) for memory efficiency. */ - REUSE_VALUE_WRAPPER("reuse_value_wrapper", true, + REUSE_VALUE_WRAPPER("reuse_value_wrapper", ClickHouseDataConfig.DEFAULT_REUSE_VALUE_WRAPPER, "Whether to reuse wrapper of value(e.g. ClickHouseValue or ClickHouseRecord) for memory efficiency."), /** * Server revision. @@ -333,22 +337,23 @@ public enum ClickHouseClientOption implements ClickHouseOption { * {@code short} for UInt8 instead of {@code byte}, and {@code UnsignedLong} for * UInt64). */ - WIDEN_UNSIGNED_TYPES("widen_unsigned_types", false, + WIDEN_UNSIGNED_TYPES("widen_unsigned_types", ClickHouseDataConfig.DEFAULT_WIDEN_UNSIGNED_TYPE, "Whether to convert unsigned types to the next widest type(e.g. use short for UInt8 instead of byte, and UnsignedLong for UInt64)."), /** * Whether to support binary string. Enable this option to treat * {@code FixedString} and {@code String} as byte array. */ - USE_BINARY_STRING("use_binary_string", false, + USE_BINARY_STRING("use_binary_string", ClickHouseDataConfig.DEFAULT_USE_BINARY_STRING, "Whether to support binary string. Enable this option to treat FixedString and String as byte array."), /** * Whether to use blocking queue for buffering. */ - USE_BLOCKING_QUEUE("use_blocking_queue", true, "Whether to use blocking queue for buffering."), + USE_BLOCKING_QUEUE("use_blocking_queue", ClickHouseDataConfig.DEFAULT_USE_BLOCKING_QUEUE, + "Whether to use blocking queue for buffering."), /** * Whether Object[] should be used instead of primitive arrays. */ - USE_OBJECTS_IN_ARRAYS("use_objects_in_arrays", false, + USE_OBJECTS_IN_ARRAYS("use_objects_in_arrays", ClickHouseDataConfig.DEFAULT_USE_OBJECT_IN_ARRAY, "Whether Object[] should be used instead of primitive arrays."), /** * Whether to access ClickHouse server directly without using system wide proxy @@ -458,7 +463,8 @@ public enum ClickHouseClientOption implements ClickHouseOption { * @return non-empty user-agent */ public static final String buildUserAgent(String productName, String additionalProperty) { - productName = productName == null || productName.isEmpty() ? DEFAULT_PRODUCT_NAME : productName.trim(); + productName = productName == null || productName.isEmpty() ? (String) PRODUCT_NAME.getEffectiveDefaultValue() + : productName.trim(); StringBuilder builder = new StringBuilder(productName).append('/').append(PRODUCT_VERSION).append(" (") .append(CLIENT_OS_INFO).append("; ").append(CLIENT_JVM_INFO); if (additionalProperty != null && !additionalProperty.isEmpty()) { diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseDefaultSslContextProvider.java b/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseDefaultSslContextProvider.java index e83f67a69..8db3230de 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseDefaultSslContextProvider.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseDefaultSslContextProvider.java @@ -29,7 +29,7 @@ import javax.net.ssl.X509TrustManager; import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseSslContextProvider; -import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.data.ClickHouseUtils; public class ClickHouseDefaultSslContextProvider implements ClickHouseSslContextProvider { static final String PEM_BEGIN_PART1 = "---BEGIN "; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseDefaults.java b/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseDefaults.java index eee9c4e79..7d296607d 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseDefaults.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseDefaults.java @@ -1,11 +1,11 @@ package com.clickhouse.client.config; import java.io.Serializable; -import java.math.RoundingMode; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseFormat; import com.clickhouse.client.ClickHouseProtocol; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseDataConfig; /** * System-wide default options. System properties and environment variables can @@ -21,7 +21,7 @@ public enum ClickHouseDefaults implements ClickHouseOption { /** * Default execution mode. */ - ASYNC("async", true, "Whether the client should run in async mode."), + ASYNC("async", ClickHouseDataConfig.DEFAULT_ASYNC, "Whether the client should run in async mode."), /** * Whether to create session automatically when there are multiple queries. */ @@ -29,7 +29,7 @@ public enum ClickHouseDefaults implements ClickHouseOption { /** * Default buffering mode. */ - BUFFERING("buffering", ClickHouseBufferingMode.RESOURCE_EFFICIENT, "Buffering mode."), + BUFFERING("buffering", ClickHouseDataConfig.DEFAULT_BUFFERING_MODE, "Buffering mode."), /** * Default server host. */ @@ -53,7 +53,8 @@ public enum ClickHouseDefaults implements ClickHouseOption { /** * Default format. */ - FORMAT("format", ClickHouseFormat.TabSeparated, "Preferred data format for serialization and deserialization."), + FORMAT("format", ClickHouseDataConfig.DEFAULT_FORMAT, + "Preferred data format for serialization and deserialization."), /** * Maximum number of threads that the scheduler(shared by all client instances) * can use to run the adhoc/scheduled tasks like discovery and health check. @@ -71,7 +72,7 @@ public enum ClickHouseDefaults implements ClickHouseOption { /** * Rounding mode for type conversion. */ - ROUNDING_MODE("rounding_mode", RoundingMode.DOWN, "Default rounding mode for BigDecimal."), + ROUNDING_MODE("rounding_mode", ClickHouseDataConfig.DEFAULT_ROUNDING_MODE, "Default rounding mode for BigDecimal."), /** * Thread keep alive timeout in milliseconds. */ diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/logging/package-info.java b/clickhouse-client/src/main/java/com/clickhouse/client/logging/package-info.java deleted file mode 100644 index 20cb23a07..000000000 --- a/clickhouse-client/src/main/java/com/clickhouse/client/logging/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Provides logging classes. - */ -package com.clickhouse.client.logging; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/naming/SrvResolver.java b/clickhouse-client/src/main/java/com/clickhouse/client/naming/SrvResolver.java index 0177e40a5..dd4e08d8d 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/naming/SrvResolver.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/naming/SrvResolver.java @@ -4,8 +4,8 @@ import com.clickhouse.client.ClickHouseDnsResolver; import com.clickhouse.client.ClickHouseProtocol; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; import org.xbill.DNS.Lookup; import org.xbill.DNS.Record; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/CompressionUtils.java b/clickhouse-client/src/main/java/com/clickhouse/client/stream/CompressionUtils.java deleted file mode 100644 index 82b511be4..000000000 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/CompressionUtils.java +++ /dev/null @@ -1,187 +0,0 @@ -package com.clickhouse.client.stream; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -import com.clickhouse.client.ClickHouseCompression; -import com.clickhouse.client.ClickHouseUtils; - -public final class CompressionUtils { - public static final String ERROR_FAILED_TO_WRAP_INPUT = "Failed to wrap input stream"; - public static final String ERROR_FAILED_TO_WRAP_OUTPUT = "Failed to wrap output stream"; - public static final String ERROR_UNSUPPORTED_COMPRESS_ALG = "Compression algorithm [%s] is not supported due to %s"; - public static final String ERROR_UNSUPPORTED_DECOMPRESS_ALG = "Decompression algorithm [%s] is not supported due to %s"; - - private static class Brotli4jUtils { - static { - com.aayushatharva.brotli4j.Brotli4jLoader.ensureAvailability(); - } - - static InputStream createInputStream(InputStream input, int bufferSize) throws IOException { - return new com.aayushatharva.brotli4j.decoder.BrotliInputStream(input, bufferSize); - } - - static OutputStream createOutputStream(OutputStream output, int quality, int bufferSize) - throws IOException { - com.aayushatharva.brotli4j.encoder.Encoder.Parameters params = new com.aayushatharva.brotli4j.encoder.Encoder.Parameters() - .setQuality(quality); - return new com.aayushatharva.brotli4j.encoder.BrotliOutputStream(output, params, bufferSize); - } - - private Brotli4jUtils() { - } - } - - public static InputStream createBrotliInputStream(InputStream input, int bufferSize) { - try { - // Brotli4jUtils.createInputStream(input, bufferSize) - return new org.brotli.dec.BrotliInputStream(input, bufferSize); - } catch (IOException e) { - throw new IllegalArgumentException(ERROR_FAILED_TO_WRAP_INPUT, e); - } catch (NoClassDefFoundError e) { - throw new UnsupportedOperationException( - ClickHouseUtils.format(ERROR_UNSUPPORTED_DECOMPRESS_ALG, ClickHouseCompression.BROTLI, - e.getMessage())); - } - } - - public static OutputStream createBrotliOutputStream(OutputStream output, int quality, int bufferSize) { - try { - if (quality < -1 || quality > 11) { - quality = 4; - } - return Brotli4jUtils.createOutputStream(output, quality, bufferSize); - } catch (IOException e) { - throw new IllegalArgumentException(ERROR_FAILED_TO_WRAP_OUTPUT, e); - } catch (NoClassDefFoundError e) { - throw new UnsupportedOperationException( - ClickHouseUtils.format(ERROR_UNSUPPORTED_COMPRESS_ALG, ClickHouseCompression.BROTLI, - e.getMessage())); - } - } - - public static InputStream createBz2InputStream(InputStream input) { - try { - return new org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream(input); - } catch (IOException e) { - throw new IllegalArgumentException(ERROR_FAILED_TO_WRAP_INPUT, e); - } catch (NoClassDefFoundError e) { - throw new UnsupportedOperationException( - ClickHouseUtils.format(ERROR_UNSUPPORTED_DECOMPRESS_ALG, ClickHouseCompression.BZ2, - e.getMessage())); - } - } - - public static OutputStream createBz2OutputStream(OutputStream output, int blockSize) { - try { - if (blockSize < org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream.MIN_BLOCKSIZE - || blockSize > org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream.MAX_BLOCKSIZE) { - blockSize = org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream.MAX_BLOCKSIZE; - } - return new org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream(output, blockSize); - } catch (IOException e) { - throw new IllegalArgumentException(ERROR_FAILED_TO_WRAP_OUTPUT, e); - } catch (NoClassDefFoundError e) { - throw new UnsupportedOperationException( - ClickHouseUtils.format(ERROR_UNSUPPORTED_COMPRESS_ALG, ClickHouseCompression.BROTLI, - e.getMessage())); - } - } - - public static InputStream createSnappyInputStream(InputStream input) { - try { - return new org.xerial.snappy.SnappyInputStream(input); - } catch (IOException e) { - throw new IllegalArgumentException(ERROR_FAILED_TO_WRAP_INPUT, e); - } catch (NoClassDefFoundError e) { - throw new UnsupportedOperationException( - ClickHouseUtils.format(ERROR_UNSUPPORTED_DECOMPRESS_ALG, ClickHouseCompression.SNAPPY, - e.getMessage())); - } - } - - public static OutputStream createSnappyOutputStream(OutputStream output, int blockSize) { - try { - if (blockSize < 1024) { // MIN_BLOCK_SIZE - blockSize = 32 * 1024; // DEFAULT_BLOCK_SIZE - } - return new org.xerial.snappy.SnappyOutputStream(output, blockSize); - } catch (NoClassDefFoundError e) { - throw new UnsupportedOperationException( - ClickHouseUtils.format(ERROR_UNSUPPORTED_COMPRESS_ALG, ClickHouseCompression.SNAPPY, - e.getMessage())); - } - } - - public static InputStream createZstdInputStream(InputStream input) { - try { - return new com.github.luben.zstd.ZstdInputStream(input).setContinuous(true); - } catch (IOException e) { - throw new IllegalArgumentException(ERROR_FAILED_TO_WRAP_INPUT, e); - } catch (NoClassDefFoundError e) { - throw new UnsupportedOperationException( - ClickHouseUtils.format(ERROR_UNSUPPORTED_DECOMPRESS_ALG, ClickHouseCompression.ZSTD, - e.getMessage())); - } - } - - /** - * Creates an output stream for {@link ClickHouseCompression#ZSTD} compression. - * - * @param output non-null output stream - * @param level compression level, any number outside of [0, 22] will be - * treated as default - * @return output stream for compression - */ - public static OutputStream createZstdOutputStream(OutputStream output, int level) { - try { - if (level < 0 || level > 22) { - level = com.github.luben.zstd.Zstd.defaultCompressionLevel(); - } - return new com.github.luben.zstd.ZstdOutputStream(output, level); - } catch (IOException e) { - throw new IllegalArgumentException(ERROR_FAILED_TO_WRAP_OUTPUT, e); - } catch (NoClassDefFoundError e) { - throw new UnsupportedOperationException( - ClickHouseUtils.format(ERROR_UNSUPPORTED_COMPRESS_ALG, ClickHouseCompression.ZSTD, e.getMessage())); - } - } - - public static InputStream createXzInputStream(InputStream input) { - try { - return new org.tukaani.xz.XZInputStream(input); - } catch (IOException e) { - throw new IllegalArgumentException(ERROR_FAILED_TO_WRAP_INPUT, e); - } catch (NoClassDefFoundError e) { - throw new UnsupportedOperationException( - ClickHouseUtils.format(ERROR_UNSUPPORTED_DECOMPRESS_ALG, ClickHouseCompression.XZ, e.getMessage())); - } - } - - /** - * Creates an output stream for {@link ClickHouseCompression#XZ} compression. - * - * @param output non-null output stream - * @param preset compression preset level, any number outside of [0,9] will be - * treated as 6, which is the default - * @return output stream for compression - */ - public static OutputStream createXzOutputStream(OutputStream output, int preset) { - try { - if (preset < org.tukaani.xz.LZMA2Options.PRESET_MIN || preset > org.tukaani.xz.LZMA2Options.PRESET_MAX) { - preset = org.tukaani.xz.LZMA2Options.PRESET_DEFAULT; - } - return new org.tukaani.xz.XZOutputStream(output, new org.tukaani.xz.LZMA2Options(preset), - org.tukaani.xz.XZ.CHECK_CRC64); - } catch (IOException e) { - throw new IllegalArgumentException(ERROR_FAILED_TO_WRAP_OUTPUT, e); - } catch (NoClassDefFoundError e) { - throw new UnsupportedOperationException( - ClickHouseUtils.format(ERROR_UNSUPPORTED_COMPRESS_ALG, ClickHouseCompression.XZ, e.getMessage())); - } - } - - private CompressionUtils() { - } -} diff --git a/clickhouse-client/src/main/java9/module-info.java b/clickhouse-client/src/main/java9/module-info.java index 6ec52aaf9..69c58c083 100644 --- a/clickhouse-client/src/main/java9/module-info.java +++ b/clickhouse-client/src/main/java9/module-info.java @@ -4,21 +4,13 @@ module com.clickhouse.client { exports com.clickhouse.client; exports com.clickhouse.client.config; - exports com.clickhouse.client.data; - exports com.clickhouse.client.data.array; - exports com.clickhouse.client.logging; requires static java.logging; - requires static com.google.gson; - requires static com.github.benmanes.caffeine; - requires static org.dnsjava; - requires static org.lz4.java; requires static org.slf4j; - requires static org.roaringbitmap; + + requires transitive com.clickhouse.data; uses com.clickhouse.client.ClickHouseClient; - uses com.clickhouse.client.ClickHouseDataStreamFactory; uses com.clickhouse.client.ClickHouseDnsResolver; uses com.clickhouse.client.ClickHouseSslContextProvider; - uses com.clickhouse.client.logging.LoggerFactory; } diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClientTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClientTest.java index e272c509e..d7a705a3e 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClientTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClientTest.java @@ -8,6 +8,8 @@ import java.util.concurrent.ExecutionException; import com.clickhouse.client.ClickHouseRequest.Mutation; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseOutputStream; public class ClickHouseClientTest { @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java index 8f05839cf..d87b74a3e 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java @@ -9,8 +9,8 @@ import org.testng.Assert; import org.testng.annotations.Test; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseOption; import com.clickhouse.client.config.ClickHouseDefaults; +import com.clickhouse.config.ClickHouseOption; public class ClickHouseConfigTest { @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodeTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodeTest.java index 31bb2e271..a0e2687c7 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodeTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodeTest.java @@ -15,6 +15,8 @@ import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseDefaults; import com.clickhouse.client.config.ClickHouseSslMode; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseVersion; public class ClickHouseNodeTest extends BaseIntegrationTest { private void checkDefaultValues(ClickHouseNode node) { diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseParameterizedQueryTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseParameterizedQueryTest.java index fa17dda04..b60312b0f 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseParameterizedQueryTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseParameterizedQueryTest.java @@ -10,7 +10,9 @@ import java.util.Map; import java.util.NoSuchElementException; -import com.clickhouse.client.data.ClickHouseDateTimeValue; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.value.ClickHouseDateTimeValue; import org.testng.Assert; import org.testng.annotations.DataProvider; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseRequestTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseRequestTest.java index 1ebbdc128..bf5f1ecae 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseRequestTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseRequestTest.java @@ -16,16 +16,19 @@ import org.testng.annotations.Test; import com.clickhouse.client.ClickHouseRequest.Mutation; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseConfigChangeListener; import com.clickhouse.client.config.ClickHouseDefaults; -import com.clickhouse.client.config.ClickHouseOption; -import com.clickhouse.client.data.ClickHouseBigIntegerValue; -import com.clickhouse.client.data.ClickHouseByteValue; -import com.clickhouse.client.data.ClickHouseDateTimeValue; -import com.clickhouse.client.data.ClickHouseExternalTable; -import com.clickhouse.client.data.ClickHouseFloatValue; -import com.clickhouse.client.data.ClickHouseIntegerValue; -import com.clickhouse.client.data.ClickHouseStringValue; +import com.clickhouse.config.ClickHouseConfigChangeListener; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseExternalTable; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.value.ClickHouseBigIntegerValue; +import com.clickhouse.data.value.ClickHouseByteValue; +import com.clickhouse.data.value.ClickHouseDateTimeValue; +import com.clickhouse.data.value.ClickHouseFloatValue; +import com.clickhouse.data.value.ClickHouseIntegerValue; +import com.clickhouse.data.value.ClickHouseStringValue; public class ClickHouseRequestTest { @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseServerForTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseServerForTest.java index f475f6d35..a992e346a 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseServerForTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseServerForTest.java @@ -13,6 +13,10 @@ import org.testng.annotations.AfterSuite; import org.testng.annotations.BeforeSuite; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseVersion; + /** * Adaptive ClickHouse server environment for integration test. Two modes are * supported: 1) existing server(when system property {@code clickhouseServer} diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseSimpleResponseTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseSimpleResponseTest.java similarity index 94% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseSimpleResponseTest.java rename to clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseSimpleResponseTest.java index ed6afe538..a59781fc7 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseSimpleResponseTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseSimpleResponseTest.java @@ -1,13 +1,14 @@ -package com.clickhouse.client.data; +package com.clickhouse.client; import java.util.Collections; import java.util.List; import java.util.NoSuchElementException; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseRecord; -import com.clickhouse.client.ClickHouseResponse; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.value.ClickHouseByteValue; +import com.clickhouse.data.value.ClickHouseStringValue; +import com.clickhouse.data.value.UnsignedByte; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java index 5fad64ab7..464d931cd 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java @@ -2,28 +2,48 @@ import com.clickhouse.client.ClickHouseClientBuilder.Agent; import com.clickhouse.client.ClickHouseTransaction.XID; -import com.clickhouse.client.config.ClickHouseBufferingMode; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseOption; -import com.clickhouse.client.config.ClickHouseRenameMethod; import com.clickhouse.client.config.ClickHouseSslMode; -import com.clickhouse.client.data.BinaryStreamUtils; -import com.clickhouse.client.data.ClickHouseBigDecimalValue; -import com.clickhouse.client.data.ClickHouseBigIntegerValue; -import com.clickhouse.client.data.ClickHouseByteValue; -import com.clickhouse.client.data.ClickHouseDateTimeValue; -import com.clickhouse.client.data.ClickHouseEnumValue; -import com.clickhouse.client.data.ClickHouseExternalTable; -import com.clickhouse.client.data.ClickHouseIntegerValue; -import com.clickhouse.client.data.ClickHouseIpv4Value; -import com.clickhouse.client.data.ClickHouseIpv6Value; -import com.clickhouse.client.data.ClickHouseLongValue; -import com.clickhouse.client.data.ClickHouseOffsetDateTimeValue; -import com.clickhouse.client.data.ClickHouseStringValue; -import com.clickhouse.client.data.UnsignedByte; -import com.clickhouse.client.data.UnsignedInteger; -import com.clickhouse.client.data.UnsignedLong; -import com.clickhouse.client.data.UnsignedShort; +import com.clickhouse.config.ClickHouseBufferingMode; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.config.ClickHouseRenameMethod; +import com.clickhouse.data.ClickHouseArraySequence; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseDataProcessor; +import com.clickhouse.data.ClickHouseDataStreamFactory; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseExternalTable; +import com.clickhouse.data.ClickHouseFile; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePipedOutputStream; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.ClickHouseVersion; +import com.clickhouse.data.ClickHouseWriter; +import com.clickhouse.data.format.BinaryStreamUtils; +import com.clickhouse.data.value.ClickHouseBigDecimalValue; +import com.clickhouse.data.value.ClickHouseBigIntegerValue; +import com.clickhouse.data.value.ClickHouseByteValue; +import com.clickhouse.data.value.ClickHouseDateTimeValue; +import com.clickhouse.data.value.ClickHouseEnumValue; +import com.clickhouse.data.value.ClickHouseIntegerValue; +import com.clickhouse.data.value.ClickHouseIpv4Value; +import com.clickhouse.data.value.ClickHouseIpv6Value; +import com.clickhouse.data.value.ClickHouseLongValue; +import com.clickhouse.data.value.ClickHouseOffsetDateTimeValue; +import com.clickhouse.data.value.ClickHouseStringValue; +import com.clickhouse.data.value.UnsignedByte; +import com.clickhouse.data.value.UnsignedInteger; +import com.clickhouse.data.value.UnsignedLong; +import com.clickhouse.data.value.UnsignedShort; + +import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorInputStream; import org.testng.Assert; import org.testng.SkipException; import org.testng.annotations.DataProvider; @@ -209,6 +229,39 @@ protected Object[][] getCompressionMatrix() { return array; } + @DataProvider(name = "requestCompressionMatrix") + protected Object[][] getRequestCompressionMatrix() { + return new Object[][] { + { ClickHouseCompression.NONE, -2, 2, 1 }, + { ClickHouseCompression.BROTLI, -2, 12, 1 }, // [-1, 11] + { ClickHouseCompression.BZ2, -2, 2, 1 }, + { ClickHouseCompression.DEFLATE, -2, 10, 1 }, // [0, 9] + { ClickHouseCompression.GZIP, -2, 10, 1 }, // [-1, 9] + { ClickHouseCompression.LZ4, -2, 19, 1 }, // [0, 18] + { ClickHouseCompression.SNAPPY, -2, 513, 1024 }, // [1 * 1024, 32 * 1024] + { ClickHouseCompression.XZ, -2, 10, 1 }, // [0, 9] + { ClickHouseCompression.ZSTD, -2, 23, 1 }, // [0, 22] + }; + } + + @DataProvider(name = "mixedCompressionMatrix") + protected Object[][] getMixedCompressionMatrix() { + ClickHouseCompression[] supportedRequestCompression = { ClickHouseCompression.NONE, ClickHouseCompression.LZ4, + ClickHouseCompression.ZSTD }; + ClickHouseCompression[] supportedResponseCompression = { ClickHouseCompression.NONE, + ClickHouseCompression.BROTLI, ClickHouseCompression.BZ2, ClickHouseCompression.DEFLATE, + ClickHouseCompression.GZIP, ClickHouseCompression.LZ4, ClickHouseCompression.XZ, + ClickHouseCompression.SNAPPY, ClickHouseCompression.ZSTD }; + Object[][] matrix = new Object[supportedRequestCompression.length * supportedResponseCompression.length][]; + int i = 0; + for (ClickHouseCompression reqComp : supportedRequestCompression) { + for (ClickHouseCompression respComp : supportedResponseCompression) { + matrix[i++] = new Object[] { reqComp, respComp }; + } + } + return matrix; + } + @DataProvider(name = "primitiveArrayMatrix") protected Object[][] getPrimitiveArrayMatrix() { return new Object[][] { @@ -438,6 +491,125 @@ public void testCompression(ClickHouseFormat format, ClickHouseBufferingMode buf } } + @Test(dataProvider = "requestCompressionMatrix", groups = "integration") + public void testCompressedRequest(ClickHouseCompression compression, int startLevel, int endLevel, int step) + throws Exception { + final ClickHouseNode server = getServer(); + final int readBufferSize = ClickHouseDataConfig.getDefaultReadBufferSize(); + + if (compression == ClickHouseCompression.SNAPPY) { + if (!checkServerVersion(getClient(), server, "[22.3,)")) { + throw new SkipException("Snappy decompression was supported since 22.3"); + } + } + + final int chunkSize = new ClickHouseConfig().getRequestChunkSize(); + final int randomLength = chunkSize * 5 + (int) (System.currentTimeMillis() % chunkSize); + for (int i = startLevel; i <= endLevel; i += step) { + final String tableName = ClickHouseUtils.format("test_%s_request_compress_%s_level%s", + server.getProtocol().name().toLowerCase(), compression.encoding(), + i < 0 ? "_" + Math.abs(i) : Integer.toString(i)); + final ByteArrayOutputStream o = new ByteArrayOutputStream(); + try (ClickHouseOutputStream out = ClickHouseOutputStream.of(o, readBufferSize, compression, i, null)) { + out.write("1,23\n4,56".getBytes()); + out.flush(); + } catch (IOException e) { + throw ClickHouseException.of(e, server); + } + try (ClickHouseClient client = getClient()) { + final ClickHouseRequest request = newRequest(client, server) + .format(ClickHouseFormat.RowBinaryWithNamesAndTypes) + .compressServerResponse(false) + .decompressClientRequest(true, compression, i); + ClickHouseClient.send(server, "drop table if exists " + tableName, + "create table " + tableName + "(i Int32, s String)engine=MergeTree() order by i").get(); + try (ClickHouseResponse response = request.write().table(tableName).data(w -> { + for (int k = 0; k < randomLength; k++) { + BinaryStreamUtils.writeInt32(w, k); + w.writeUnicodeString(Integer.toString(k)); + } + }).executeAndWait()) { + // ignore + } + + try (ClickHouseResponse response = request + .external( + // external table with compressed data + ClickHouseExternalTable.builder().name("x").columns("i Int32, s String") + .compression(compression) + .format(ClickHouseFormat.CSV) + .content(new ByteArrayInputStream(o.toByteArray())).build(), + // external table without compression + ClickHouseExternalTable.builder().name("y").columns("s String, i Int32") + .format(ClickHouseFormat.TSV) + .content(new ByteArrayInputStream("32\t1\n43\t2\n54\t3\n65\t4".getBytes())) + .build()) + .query("select x.* from x inner join y on x.i = y.i where i in (select i from " + tableName + + ")") + .executeAndWait()) { + int j = 0; + for (ClickHouseRecord r : response.records()) { + Assert.assertEquals(r.getValue(0).asInteger(), j == 0 ? 1 : 4); + Assert.assertEquals(r.getValue(1).asInteger(), j == 0 ? 23 : 56); + j++; + } + Assert.assertEquals(j, 2); + } + } + } + } + + @Test(dataProvider = "mixedCompressionMatrix", groups = "integration") + public void testDecompressResponse(ClickHouseCompression reqComp, ClickHouseCompression respComp) throws Exception { + if (reqComp == ClickHouseCompression.SNAPPY || respComp == ClickHouseCompression.BZ2) { + if (!checkServerVersion(getClient(), getServer(), "[22.10,)")) { + throw new SkipException("Snappy and bz2 were all supported since 22.10"); + } + } + + final ClickHouseNode server = getServer(); + final int rows = 50_000; + final String sql = ClickHouseUtils.format("select number n, toString(number+1) s from numbers(%d)", rows); + try (ClickHouseClient client = getClient()) { + ClickHouseRequest request = newRequest(client, server) + .format(ClickHouseFormat.RowBinaryWithNamesAndTypes) + .decompressClientRequest(true, reqComp) + .compressServerResponse(true, respComp) + .query(sql); + try (ClickHouseResponse response = request.executeAndWait()) { + int i = 0; + for (ClickHouseRecord r : response.records()) { + Assert.assertEquals(r.getValue(0).asInteger(), i++); + Assert.assertEquals(r.getValue(1).asInteger(), i); + } + Assert.assertEquals(i, rows); + } + + File tmp = ClickHouseUtils.createTempFile(respComp.encoding(), respComp.fileExtension(), true); + try (ClickHouseResponse response = request.output(tmp.toString()).format(ClickHouseFormat.CSV) + .executeAndWait()) { + response.close(); + + Assert.assertTrue(tmp.exists()); + Assert.assertNotEquals(Files.size(tmp.toPath()), 0L); + + int i = 0; + try (BufferedReader reader = new BufferedReader(new InputStreamReader( + ClickHouseProtocol.GRPC == server.getProtocol() && respComp == ClickHouseCompression.LZ4 + ? new FramedLZ4CompressorInputStream(new FileInputStream(tmp)) + : ClickHouseInputStream.of(new FileInputStream(tmp), + request.getConfig().getReadBufferSize(), respComp, + request.getConfig().getResponseCompressLevel(), null)))) { + String line = null; + while ((line = reader.readLine()) != null) { + Assert.assertEquals(line, ClickHouseUtils.format("%d,\"%d\"", i++, i)); + } + } + Assert.assertEquals(i, rows); + } + } + } + @Test(groups = { "integration" }) public void testFormat() throws ClickHouseException { String sql = "select 1, 2"; @@ -1295,7 +1467,7 @@ public void testDumpAndLoadFile() throws ClickHouseException, IOException { "create table test_dump_load_file(a UInt64, b Nullable(String)) engine=MergeTree() order by tuple()"); final int rows = 10000; - final Path tmp = Paths.get(System.getProperty("java.io.tmpdir"), "file.json"); + final Path tmp = Paths.get(System.getProperty("java.io.tmpdir"), "file.csv"); ClickHouseFile file = ClickHouseFile.of(tmp); try { ClickHouseClient.dump(server, @@ -1356,8 +1528,8 @@ public void testDump() throws ExecutionException, InterruptedException, IOExcept public void testDumpFile(boolean gzipCompressed, boolean useOneLiner) throws ExecutionException, InterruptedException, IOException { ClickHouseNode server = getServer(); - if (server.getProtocol() != ClickHouseProtocol.HTTP) { - throw new SkipException("Skip as only http implementation works well"); + if (server.getProtocol() != ClickHouseProtocol.GRPC && server.getProtocol() != ClickHouseProtocol.HTTP) { + throw new SkipException("Skip as only http and grpc implementation work well"); } File file = Files.createTempFile("chc", ".data").toFile(); @@ -1485,8 +1657,8 @@ public void testLoadCsv() throws ExecutionException, InterruptedException, IOExc @Test(dataProvider = "fileProcessMatrix", groups = "integration") public void testLoadFile(boolean gzipCompressed, boolean useOneLiner) throws ClickHouseException, IOException { ClickHouseNode server = getServer(); - if (server.getProtocol() != ClickHouseProtocol.HTTP) { - throw new SkipException("Skip as only http implementation works well"); + if (server.getProtocol() != ClickHouseProtocol.GRPC && server.getProtocol() != ClickHouseProtocol.HTTP) { + throw new SkipException("Skip as only http and grpc implementation work well"); } File file = File.createTempFile("chc", ".data"); @@ -1511,8 +1683,7 @@ public void testLoadFile(boolean gzipCompressed, boolean useOneLiner) throws Cli sendAndWait(server, "drop table if exists test_load_file", "create table test_load_file(a Int32, b Nullable(String))engine=Memory"); ClickHouseFile wrappedFile = ClickHouseFile.of(file, - gzipCompressed ? ClickHouseCompression.GZIP : ClickHouseCompression.NONE, 0, - ClickHouseFormat.CSV); + gzipCompressed ? ClickHouseCompression.GZIP : ClickHouseCompression.NONE, -1, ClickHouseFormat.CSV); if (useOneLiner) { try { ClickHouseClient.load(server, "test_load_file", wrappedFile).get(); @@ -1631,7 +1802,7 @@ public void testExternalTableAsParameter() throws ClickHouseException { } @Test(groups = { "integration" }) - public void testInsertWithCustomFormat() throws ClickHouseException, IOException { + public void testInsertWithCustomFormat() throws ClickHouseException { ClickHouseNode server = getServer(); sendAndWait(server, "drop table if exists test_custom_input_format", "create table test_custom_input_format(i Int8, f String)engine=Memory"); @@ -1653,9 +1824,11 @@ public void testInsertWithCustomFormat() throws ClickHouseException, IOException Assert.assertEquals(Files.size(temp), 0L); Files.write(temp, "i,f\n4,CSVWithNames\n5,CSVWithNames\n".getBytes(StandardCharsets.US_ASCII)); Assert.assertTrue(Files.size(temp) > 0L); - try (ClickHouseResponse response = request.write().format(ClickHouseFormat.CSVWithNames) + try (ClickHouseResponse response = request.write() .table("test_custom_input_format") - .data(temp.toFile().getAbsolutePath()).executeAndWait()) { + .data(temp.toFile().getAbsolutePath()) // format is now CSV + .format(ClickHouseFormat.CSVWithNames) // change format to CSVWithNames + .executeAndWait()) { // ignore } @@ -1669,6 +1842,8 @@ public void testInsertWithCustomFormat() throws ClickHouseException, IOException } Assert.assertEquals(count, 5); } + } catch (IOException e) { + throw ClickHouseException.of(e, server); } } diff --git a/clickhouse-data/README.md b/clickhouse-data/README.md new file mode 100644 index 000000000..56c795a97 --- /dev/null +++ b/clickhouse-data/README.md @@ -0,0 +1,109 @@ +# ClickHouse Data Processing Utilities + +`clickhouse-data` aims to be a handy toolkit for processing data in various formats. It's suitable for producing or consuming data without having to communicate with ClickHouse. + +## Quick Start + +```xml + + com.clickhouse + clickhouse-data + 0.4.0 + +``` + +```java +/*** serialization/deserialization ***/ +ClickHouseDataConfig config = ...; +List columns = ClickHouseColumn.parse("a String, b Int32"); +ByteArrayOutputStream out = new ByteArrayOutputStream(); +try (ClickHouseOutputStream output = ClickHouseOutputStream.of(out)) { + // find a suitable processor based on config.getFormat() + ClickHouseDataProcessor processor = ClickHouseDataStreamFactory.getInstance() + .getProcessor(config, null, output, null, columns); + // let's reuse same wrapper class for both columns + ClickHouseValue v = ClickHouseIntegerValue.ofNull(); + for (int i=0; i<50_000; i++) { + v.update(i); + for (ClickHouseSerializer s : processor.getSerializers()) { + s.serialize(v, output); + } + } +} +byte[] bytes = out.toByteArray(); + +/*** type conversion ***/ +ClickHouseValue value = ClickHouseStringValue.of("123"); +byte b = value.asByte(); +int i = value.asInteger(); +long l = value.asLong(); +value.update(l + i + b); // "369" + +/*** streaming ***/ +int bufferSize = 8192; +int queueLength = 0; // unlimited +int timeout = 30000; // 30 seconds +ClickHousePipedOutputStream output = ClickHouseDataStreamFactory.getInstance() + .createPipedOutputStream(bufferSize, queueLength, timeout); +// write and read in separate threads +CompletableFuture future = CompletableFuture.supplyAsync(() -> { + try (ClickHouseOuputStream out = output) { + for (int i = 0; i<50_000; i++) { + out.writeByte(0) + } + + return true; + } +}); + +// read will be blocked until there's data available in the "pipe" +try (ClickHouseInputStream input = ClickHouseInputStream.of(output.getInputStream(), new FileInputStream("/tmp/mine.txt"))) { // read consolidated input streams + byte b = input.readByte(); +} + +/*** compression/decompression ***/ +int level = -1; +ClickHouseInputStream input = ClickHouseCompressionAlgorithm.of(ClickHouseCompression.ZSTD) + .decompress(null, new FileInputStream("/tmp/compressed.data"), bufferSize, level, null); +ClickHouseOutputStream output = ClickHouseCompressionAlgorithm.of(ClickHouseCompression.LZ4) + .compress(null, new FileInputStream("/tmp/compressed.data"), bufferSize, level, null); +input.pipe(output); +``` + +## Type Mapping + +| Data Type | Wrapper Class | Value Class | Remark | +| ------------------- | ------------------------------ | ----------------------- | ----------------------------------------------- | +| Bool | ClickHouseBoolValue | bool | | +| Date\* | ClickHouseDateValue | java.time.LocalDate | | +| DateTime\* | ClickHouseDateTimeValue | java.time.LocalDateTime | or java.time.OffsetDateTime if there's timezone | +| Enum\* | ClickHouseEnumValue | int | | +| FixedString | ClickHouseStringValue | byte[] | | +| Int8 | ClickHouseByteValue | byte | | +| UInt8 | UnsignedByteValue | byte | or short when widen_unsigned_types=true | +| Int16 | ClickHouseShortValue | short | | +| UInt16 | UnsignedShortValue | short | or int when widen_unsigned_types=true | +| Int32 | ClickHouseIntegerValue | int | | +| UInt32 | UnsignedIntegerValue | int | or long when widen_unsigned_types=true | +| Int64 | ClickHouseLongValue | long | | +| UInt64/Interval\* | UnsignedLongValue | long | or BigInteger when widen_unsigned_types=true | +| \*Int128 | ClickHouseBigIntegerValue | BigInteger | | +| \*Int256 | ClickHouseBigIntegerValue | BigInteger | | +| Decimal\* | ClickHouseBigDecimalValue | BigDecimal | | +| Float32 | ClickHouseFloatValue | float | | +| Float64 | ClickHouseDoubleValue | double | | +| IPv4 | ClickHouseIpv4Value | java.net.Inet4Address | | +| IPv6 | ClickHouseIpv6Value | java.net.Inet6Address | | +| UUID | ClickHouseUuidValue | java.util.UUID | | +| Point | ClickHouseGeoPointValue | double[2] | | +| Ring | ClickHouseGeoRingValue | double[][] | | +| Polygon | ClickHouseGeoPolygonValue | double[][][] | | +| MultiPolygon | ClickHouseGeoMultiPolygonValue | double[][][][] | | +| JSON/Object('json') | ClickHouseTupleValue | java.util.List | | +| String | ClickHouseStringValue | String | or byte[] when use_binary_string=true | +| Array | ClickHouseArrayValue | primitive array | or Object array when use_objects_in_array=true | +| Map | ClickHouseMapValue | java.util.Map | | +| Nested | ClickHouseNestedValue | Object[][] | | +| Tuple | ClickHouseTupleValue | java.util.List | | + +All wrapper classes implemented `ClickHouseValue` interface providing the ability to be converted from/to a specific Java type(e.g. via `update(String)` or `asString()`). diff --git a/clickhouse-data/pom.xml b/clickhouse-data/pom.xml new file mode 100644 index 000000000..78e4155f4 --- /dev/null +++ b/clickhouse-data/pom.xml @@ -0,0 +1,140 @@ + + 4.0.0 + + + com.clickhouse + clickhouse-java + ${revision} + + + clickhouse-data + jar + + ClickHouse Data Processing Utilities + Data processing utilities for ClickHouse + https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-data + + + + ${project.parent.groupId} + org.roaringbitmap + provided + + + * + * + + + + + + com.aayushatharva.brotli4j + brotli4j + true + + + com.github.ben-manes.caffeine + caffeine + true + + + com.github.luben + zstd-jni + true + + + com.google.code.gson + gson + true + + + org.apache.arrow + arrow-memory + pom + true + + + org.apache.arrow + arrow-vector + true + + + org.apache.avro + avro + true + + + org.brotli + dec + true + + + org.jctools + jctools-core + true + + + org.lz4 + lz4-java + true + + + org.msgpack + msgpack-core + true + + + org.slf4j + slf4j-api + true + + + org.tukaani + xz + true + + + org.xerial.snappy + snappy-java + true + + + + org.slf4j + slf4j-simple + test + + + org.testcontainers + testcontainers + test + + + org.testng + testng + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + env_str + 416 + false + + + + + + \ No newline at end of file diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseBufferingMode.java b/clickhouse-data/src/main/java/com/clickhouse/config/ClickHouseBufferingMode.java similarity index 95% rename from clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseBufferingMode.java rename to clickhouse-data/src/main/java/com/clickhouse/config/ClickHouseBufferingMode.java index d765ebfe9..435913564 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseBufferingMode.java +++ b/clickhouse-data/src/main/java/com/clickhouse/config/ClickHouseBufferingMode.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.config; +package com.clickhouse.config; /** * Supported buffering mode for dealing with request and response. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseConfigChangeListener.java b/clickhouse-data/src/main/java/com/clickhouse/config/ClickHouseConfigChangeListener.java similarity index 97% rename from clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseConfigChangeListener.java rename to clickhouse-data/src/main/java/com/clickhouse/config/ClickHouseConfigChangeListener.java index faac212bb..89d2edb0d 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseConfigChangeListener.java +++ b/clickhouse-data/src/main/java/com/clickhouse/config/ClickHouseConfigChangeListener.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.config; +package com.clickhouse.config; import java.io.Serializable; diff --git a/clickhouse-data/src/main/java/com/clickhouse/config/ClickHouseDefaultOption.java b/clickhouse-data/src/main/java/com/clickhouse/config/ClickHouseDefaultOption.java new file mode 100644 index 000000000..01679a875 --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/config/ClickHouseDefaultOption.java @@ -0,0 +1,45 @@ +package com.clickhouse.config; + +import java.io.Serializable; +import java.util.Locale; + +import com.clickhouse.data.ClickHouseChecker; + +public final class ClickHouseDefaultOption implements ClickHouseOption { + private final String name; + private final String key; + private final Serializable defaultValue; + private final Class clazz; + + public ClickHouseDefaultOption(String name, T defaultValue) { + this.name = ClickHouseChecker.nonNull(name, "name").toUpperCase(Locale.ROOT); + this.key = name.toLowerCase(Locale.ROOT); + this.defaultValue = ClickHouseChecker.nonNull(defaultValue, "defaultValue"); + this.clazz = defaultValue.getClass(); + } + + @Override + public Serializable getDefaultValue() { + return defaultValue; + } + + @Override + public String getDescription() { + return ""; + } + + @Override + public String getKey() { + return key; + } + + @Override + public Class getValueType() { + return clazz; + } + + @Override + public String name() { + return name; + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseOption.java b/clickhouse-data/src/main/java/com/clickhouse/config/ClickHouseOption.java similarity index 96% rename from clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseOption.java rename to clickhouse-data/src/main/java/com/clickhouse/config/ClickHouseOption.java index 712656fae..37e90c5fb 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseOption.java +++ b/clickhouse-data/src/main/java/com/clickhouse/config/ClickHouseOption.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.config; +package com.clickhouse.config; import java.io.Serializable; import java.util.Collections; @@ -8,23 +8,12 @@ import java.util.Optional; import java.util.TimeZone; -import com.clickhouse.client.ClickHouseChecker; - /** * This defines a configuration option. To put it in a nutshell, an option is * composed of key, default value(which implies type of the value) and * description. */ public interface ClickHouseOption extends Serializable { - /** - * Default client name. - */ - static final String DEFAULT_CLIENT_NAME = "ClickHouse Java Client"; - /** - * Default product name. - */ - static final String DEFAULT_PRODUCT_NAME = "ClickHouse-JavaClient"; - /** * Converts given string to key value pairs. * @@ -156,7 +145,7 @@ static T fromString(String value, T defaultValue) { if (defaultValue == null) { throw new IllegalArgumentException("Non-null default value is required"); } - if (ClickHouseChecker.isNullOrBlank(value)) { + if (value == null || value.isEmpty()) { return defaultValue; } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseRenameMethod.java b/clickhouse-data/src/main/java/com/clickhouse/config/ClickHouseRenameMethod.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseRenameMethod.java rename to clickhouse-data/src/main/java/com/clickhouse/config/ClickHouseRenameMethod.java index 788c7242b..daa7b5433 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseRenameMethod.java +++ b/clickhouse-data/src/main/java/com/clickhouse/config/ClickHouseRenameMethod.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.config; +package com.clickhouse.config; import java.util.function.UnaryOperator; diff --git a/clickhouse-data/src/main/java/com/clickhouse/config/package-info.java b/clickhouse-data/src/main/java/com/clickhouse/config/package-info.java new file mode 100644 index 000000000..865b6bbc3 --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/config/package-info.java @@ -0,0 +1,4 @@ +/** + * Provides configuration classes. + */ +package com.clickhouse.config; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseAggregateFunction.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseAggregateFunction.java similarity index 99% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseAggregateFunction.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseAggregateFunction.java index 5b0566766..43e576835 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseAggregateFunction.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseAggregateFunction.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.util.Arrays; import java.util.Collections; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseArraySequence.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseArraySequence.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseArraySequence.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseArraySequence.java index 8c945b58f..29cd4dd3e 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseArraySequence.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseArraySequence.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; /** * This interface represents a generic array value. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseByteBuffer.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseByteBuffer.java similarity index 99% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseByteBuffer.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseByteBuffer.java index 77fc5388c..281826a06 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseByteBuffer.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseByteBuffer.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.Serializable; import java.math.BigInteger; diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseByteUtils.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseByteUtils.java new file mode 100644 index 000000000..7720a8b33 --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseByteUtils.java @@ -0,0 +1,169 @@ +package com.clickhouse.data; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +public class ClickHouseByteUtils { + public static int getInt32LE(byte[] bytes, int offset) { + return (0xFF & bytes[offset++]) | ((0xFF & bytes[offset++]) << 8) | ((0xFF & bytes[offset++]) << 16) + | ((0xFF & bytes[offset]) << 24); + } + + public static void setInt32LE(byte[] bytes, int offset, int value) { + bytes[offset++] = (byte) (0xFF & value); + bytes[offset++] = (byte) (0xFF & (value >> 8)); + bytes[offset++] = (byte) (0xFF & (value >> 16)); + bytes[offset] = (byte) (0xFF & (value >> 24)); + } + + public static long getInt64LE(byte[] bytes, int offset) { + return (0xFFL & bytes[offset++]) | ((0xFFL & bytes[offset++]) << 8) | ((0xFFL & bytes[offset++]) << 16) + | ((0xFFL & bytes[offset++]) << 24) | ((0xFFL & bytes[offset++]) << 32) + | ((0xFFL & bytes[offset++]) << 40) | ((0xFFL & bytes[offset++]) << 48) + | ((0xFFL & bytes[offset]) << 56); + } + + public static void setInt64LE(byte[] bytes, int offset, long value) { + bytes[offset++] = (byte) (0xFF & value); + bytes[offset++] = (byte) (0xFF & (value >> 8)); + bytes[offset++] = (byte) (0xFF & (value >> 16)); + bytes[offset++] = (byte) (0xFF & (value >> 24)); + bytes[offset++] = (byte) (0xFF & (value >> 32)); + bytes[offset++] = (byte) (0xFF & (value >> 40)); + bytes[offset++] = (byte) (0xFF & (value >> 48)); + bytes[offset] = (byte) (0xFF & (value >> 56)); + } + + /** + * Get varint length of given integer. + * + * @param value integer + * @return varint length + */ + public static int getVarIntSize(int value) { + int result = 0; + do { + result++; + value >>>= 7; + } while (value != 0); + + return result; + } + + /** + * Get varint length of given long. + * + * @param value long + * @return varint length + */ + public static int getVarLongSize(long value) { + int result = 0; + do { + result++; + value >>>= 7; + } while (value != 0); + + return result; + } + + /** + * Gets varint from given byte buffer. + * + * @param buffer non-null byte buffer + * @return varint + */ + public static int getVarInt(ByteBuffer buffer) { + long result = 0L; + int shift = 0; + for (int i = 0; i < 9; i++) { + // gets 7 bits from next byte + byte b = buffer.get(); + result |= (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + + return (int) result; + } + + /** + * Write varint to given output stream. + * + * @param buffer non-null byte buffer + * @param value integer value + */ + public static void setVarInt(ByteBuffer buffer, int value) { + while ((value & 0xFFFFFF80) != 0) { + buffer.put((byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + buffer.put((byte) (value & 0x7F)); + } + + /** + * Reads varint from given input stream. + * + * @param input non-null input stream + * @return varint + * @throws IOException when failed to read value from input stream or reached + * end of the stream + */ + public static int readVarInt(InputStream input) throws IOException { + // https://github.com/ClickHouse/ClickHouse/blob/abe314feecd1647d7c2b952a25da7abf5c19f352/src/IO/VarInt.h#L126 + long result = 0L; + int shift = 0; + for (int i = 0; i < 9; i++) { + // gets 7 bits from next byte + int b = input.read(); + if (b == -1) { + try { + input.close(); + } catch (IOException e) { + // ignore error + } + throw new EOFException(); + } + result |= (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + + return (int) result; + } + + /** + * Writes varint to given output stream. + * + * @param output non-null output stream + * @param value long value + * @throws IOException when failed to write value to output stream or reached + * end of the stream + */ + public static void writeVarInt(OutputStream output, long value) throws IOException { + // https://github.com/ClickHouse/ClickHouse/blob/abe314feecd1647d7c2b952a25da7abf5c19f352/src/IO/VarInt.h#L187 + for (int i = 0; i < 9; i++) { + byte b = (byte) (value & 0x7F); + + if (value > 0x7F) { + b |= 0x80; + } + + value >>= 7; + output.write(b); + + if (value == 0) { + return; + } + } + } + + protected ClickHouseByteUtils() { + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCache.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseCache.java similarity index 92% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCache.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseCache.java index 53ed65ccc..7db30dddd 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCache.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseCache.java @@ -1,9 +1,9 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.util.function.Function; -import com.clickhouse.client.cache.CaffeineCache; -import com.clickhouse.client.cache.JdkLruCache; +import com.clickhouse.data.cache.CaffeineCache; +import com.clickhouse.data.cache.JdkLruCache; /** * Wrapper interface depicts essential methods required by a client-side cache. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseChecker.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseChecker.java similarity index 99% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseChecker.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseChecker.java index de0096a90..a3925d9a9 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseChecker.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseChecker.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseCityHash.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseCityHash.java similarity index 99% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseCityHash.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseCityHash.java index 5bea18fc0..e964f13f8 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseCityHash.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseCityHash.java @@ -29,7 +29,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.clickhouse.client.data; +package com.clickhouse.data; /** * @author tamtam180 - kirscheless at gmail.com diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseColumn.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java similarity index 94% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseColumn.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java index 3425ebc54..0ca1f782b 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseColumn.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.Serializable; import java.lang.reflect.Array; @@ -11,40 +11,40 @@ import java.util.Objects; import java.util.TimeZone; -import com.clickhouse.client.data.ClickHouseArrayValue; -import com.clickhouse.client.data.ClickHouseBigDecimalValue; -import com.clickhouse.client.data.ClickHouseBigIntegerValue; -import com.clickhouse.client.data.ClickHouseBitmapValue; -import com.clickhouse.client.data.ClickHouseBoolValue; -import com.clickhouse.client.data.ClickHouseByteValue; -import com.clickhouse.client.data.ClickHouseDateTimeValue; -import com.clickhouse.client.data.ClickHouseDateValue; -import com.clickhouse.client.data.ClickHouseDoubleValue; -import com.clickhouse.client.data.ClickHouseEmptyValue; -import com.clickhouse.client.data.ClickHouseEnumValue; -import com.clickhouse.client.data.ClickHouseFloatValue; -import com.clickhouse.client.data.ClickHouseGeoMultiPolygonValue; -import com.clickhouse.client.data.ClickHouseGeoPointValue; -import com.clickhouse.client.data.ClickHouseGeoPolygonValue; -import com.clickhouse.client.data.ClickHouseGeoRingValue; -import com.clickhouse.client.data.ClickHouseIntegerValue; -import com.clickhouse.client.data.ClickHouseIpv4Value; -import com.clickhouse.client.data.ClickHouseIpv6Value; -import com.clickhouse.client.data.ClickHouseLongValue; -import com.clickhouse.client.data.ClickHouseMapValue; -import com.clickhouse.client.data.ClickHouseNestedValue; -import com.clickhouse.client.data.ClickHouseOffsetDateTimeValue; -import com.clickhouse.client.data.ClickHouseShortValue; -import com.clickhouse.client.data.ClickHouseStringValue; -import com.clickhouse.client.data.ClickHouseTupleValue; -import com.clickhouse.client.data.ClickHouseUuidValue; -import com.clickhouse.client.data.array.ClickHouseBoolArrayValue; -import com.clickhouse.client.data.array.ClickHouseByteArrayValue; -import com.clickhouse.client.data.array.ClickHouseDoubleArrayValue; -import com.clickhouse.client.data.array.ClickHouseFloatArrayValue; -import com.clickhouse.client.data.array.ClickHouseIntArrayValue; -import com.clickhouse.client.data.array.ClickHouseLongArrayValue; -import com.clickhouse.client.data.array.ClickHouseShortArrayValue; +import com.clickhouse.data.value.ClickHouseArrayValue; +import com.clickhouse.data.value.ClickHouseBigDecimalValue; +import com.clickhouse.data.value.ClickHouseBigIntegerValue; +import com.clickhouse.data.value.ClickHouseBitmapValue; +import com.clickhouse.data.value.ClickHouseBoolValue; +import com.clickhouse.data.value.ClickHouseByteValue; +import com.clickhouse.data.value.ClickHouseDateTimeValue; +import com.clickhouse.data.value.ClickHouseDateValue; +import com.clickhouse.data.value.ClickHouseDoubleValue; +import com.clickhouse.data.value.ClickHouseEmptyValue; +import com.clickhouse.data.value.ClickHouseEnumValue; +import com.clickhouse.data.value.ClickHouseFloatValue; +import com.clickhouse.data.value.ClickHouseGeoMultiPolygonValue; +import com.clickhouse.data.value.ClickHouseGeoPointValue; +import com.clickhouse.data.value.ClickHouseGeoPolygonValue; +import com.clickhouse.data.value.ClickHouseGeoRingValue; +import com.clickhouse.data.value.ClickHouseIntegerValue; +import com.clickhouse.data.value.ClickHouseIpv4Value; +import com.clickhouse.data.value.ClickHouseIpv6Value; +import com.clickhouse.data.value.ClickHouseLongValue; +import com.clickhouse.data.value.ClickHouseMapValue; +import com.clickhouse.data.value.ClickHouseNestedValue; +import com.clickhouse.data.value.ClickHouseOffsetDateTimeValue; +import com.clickhouse.data.value.ClickHouseShortValue; +import com.clickhouse.data.value.ClickHouseStringValue; +import com.clickhouse.data.value.ClickHouseTupleValue; +import com.clickhouse.data.value.ClickHouseUuidValue; +import com.clickhouse.data.value.array.ClickHouseBoolArrayValue; +import com.clickhouse.data.value.array.ClickHouseByteArrayValue; +import com.clickhouse.data.value.array.ClickHouseDoubleArrayValue; +import com.clickhouse.data.value.array.ClickHouseFloatArrayValue; +import com.clickhouse.data.value.array.ClickHouseIntArrayValue; +import com.clickhouse.data.value.array.ClickHouseLongArrayValue; +import com.clickhouse.data.value.array.ClickHouseShortArrayValue; /** * This class represents a column defined in database. @@ -662,7 +662,7 @@ public ClickHouseDataType getDataType() { return dataType; } - public Class getObjectClass(ClickHouseConfig config) { + public Class getObjectClass(ClickHouseDataConfig config) { if (timeZone != null) { return OffsetDateTime.class; } @@ -671,7 +671,7 @@ public Class getObjectClass(ClickHouseConfig config) { : dataType.getObjectClass(); } - public Class getObjectClassForArray(ClickHouseConfig config) { + public Class getObjectClassForArray(ClickHouseDataConfig config) { Class javaClass = config.isUseObjectsInArray() || isNullable() ? getObjectClass(config) : getPrimitiveClass(config); if (config.isUseBinaryString() && javaClass == String.class) { @@ -680,7 +680,7 @@ public Class getObjectClassForArray(ClickHouseConfig config) { return javaClass; } - public Class getPrimitiveClass(ClickHouseConfig config) { + public Class getPrimitiveClass(ClickHouseDataConfig config) { if (timeZone != null) { return OffsetDateTime.class; } @@ -801,7 +801,7 @@ public ClickHouseAggregateFunction getAggregateFunction() { return aggFuncType; } - public ClickHouseArraySequence newArrayValue(ClickHouseConfig config) { + public ClickHouseArraySequence newArrayValue(ClickHouseDataConfig config) { int level = arrayLevel; ClickHouseArraySequence value; if (level < 1 || arrayBaseColumn == null || arrayBaseColumn.nullable || config.isUseObjectsInArray()) { @@ -859,7 +859,7 @@ public ClickHouseArraySequence newArrayValue(ClickHouseConfig config) { return value; } - public ClickHouseValue newValue(ClickHouseConfig config) { + public ClickHouseValue newValue(ClickHouseDataConfig config) { if (template != null) { return template.copy(); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCompression.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseCompression.java similarity index 88% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCompression.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseCompression.java index f825c8a74..c9621ccfc 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCompression.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseCompression.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.util.Locale; @@ -19,12 +19,14 @@ public enum ClickHouseCompression { private String mimeType; private String encoding; private String fileExt; + private String fileExtWithDot; // and maybe magic bytes? ClickHouseCompression(String mimeType, String encoding, String fileExt) { this.mimeType = mimeType; this.encoding = encoding; this.fileExt = fileExt; + this.fileExtWithDot = ".".concat(fileExt); } public String mimeType() { @@ -36,7 +38,11 @@ public String encoding() { } public String fileExtension() { - return fileExt; + return fileExtension(true); + } + + public String fileExtension(boolean withDot) { + return withDot ? fileExtWithDot : fileExt; } /** @@ -85,10 +91,10 @@ public static ClickHouseCompression fromEncoding(String encoding) { * Get compression algorithm based on given file name. * * @param file file name - * @return compression algorithm + * @return compression algorithm, could be null */ public static ClickHouseCompression fromFileName(String file) { - ClickHouseCompression compression = NONE; + ClickHouseCompression compression = null; int index = 0; if (file != null && (index = file.lastIndexOf('.')) > 0) { diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseCompressionAlgorithm.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseCompressionAlgorithm.java new file mode 100644 index 000000000..ecc92ee72 --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseCompressionAlgorithm.java @@ -0,0 +1,128 @@ +package com.clickhouse.data; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import com.clickhouse.config.ClickHouseDefaultOption; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.compress.BrotliSupport; +import com.clickhouse.data.compress.Bz2Support; +import com.clickhouse.data.compress.DeflateSupport; +import com.clickhouse.data.compress.GzipSupport; +import com.clickhouse.data.compress.Lz4Support; +import com.clickhouse.data.compress.NoneSupport; +import com.clickhouse.data.compress.SnappySupport; +import com.clickhouse.data.compress.XzSupport; +import com.clickhouse.data.compress.ZstdSupport; + +public interface ClickHouseCompressionAlgorithm { + static final String ERROR_FAILED_TO_WRAP_INPUT = "Failed to wrap input stream"; + static final String ERROR_FAILED_TO_WRAP_OUTPUT = "Failed to wrap output stream"; + + static final String ERROR_UNSUPPORTED_COMPRESS_ALG = "Compression algorithm [%s] is not supported"; + static final String ERROR_UNSUPPORTED_DECOMPRESS_ALG = "Decompression algorithm [%s] is not supported"; + + static final ClickHouseOption COMPRESSION_LIB_DETECTION = new ClickHouseDefaultOption("compression_lib_detection", + true); + + static ClickHouseCompressionAlgorithm createInstance(String option, + Class preferredInstance, + Class defaultInstance) { + ClickHouseCompressionAlgorithm alg = null; + if ((boolean) new ClickHouseDefaultOption(option, + (boolean) ClickHouseCompressionAlgorithm.COMPRESSION_LIB_DETECTION.getEffectiveDefaultValue()) + .getEffectiveDefaultValue()) { + try { + alg = preferredInstance.getDeclaredConstructor().newInstance(); + } catch (Throwable t) { // NOSONAR + // ignore + t.printStackTrace(); + } + } + + if (alg == null) { + try { + alg = defaultInstance.getDeclaredConstructor().newInstance(); + } catch (Throwable e) { // NOSONAR + throw new UnsupportedOperationException("Failed to create default instance of " + defaultInstance, e); + } + } + return alg; + } + + static ClickHouseInputStream createInputStream(ClickHousePassThruStream stream, InputStream input, int bufferSize, + ClickHouseCompression compression, int level, Runnable postCloseAction) { + try { + return of(compression).decompress(stream, input, bufferSize, level, postCloseAction); + } catch (IOException e) { + throw new IllegalArgumentException(ERROR_FAILED_TO_WRAP_INPUT, e); + } catch (NoClassDefFoundError e) { + throw new UnsupportedOperationException( + ClickHouseUtils.format(ERROR_UNSUPPORTED_DECOMPRESS_ALG, compression), e); + } + } + + static ClickHouseOutputStream createOutputStream(ClickHousePassThruStream stream, OutputStream output, + int bufferSize, ClickHouseCompression compression, int level, Runnable postCloseAction) { + try { + return of(compression).compress(stream, output, bufferSize, level, postCloseAction); + } catch (IOException e) { + throw new IllegalArgumentException(ERROR_FAILED_TO_WRAP_OUTPUT, e); + } catch (NoClassDefFoundError e) { + throw new UnsupportedOperationException(ClickHouseUtils.format(ERROR_UNSUPPORTED_COMPRESS_ALG, compression), + e); + } + } + + static ClickHouseCompressionAlgorithm of(ClickHouseCompression compression) { + if (compression == null || compression == ClickHouseCompression.NONE) { + return NoneSupport.getInstance(); + } + + final ClickHouseCompressionAlgorithm alg; + switch (compression) { + case BROTLI: + alg = BrotliSupport.getInstance(); + break; + case BZ2: + alg = Bz2Support.getInstance(); + break; + case DEFLATE: + alg = DeflateSupport.getInstance(); + break; + case GZIP: + alg = GzipSupport.getInstance(); + break; + case LZ4: + alg = Lz4Support.getInstance(); + break; + case SNAPPY: + alg = SnappySupport.getInstance(); + break; + case XZ: + alg = XzSupport.getInstance(); + break; + case ZSTD: + alg = ZstdSupport.getInstance(); + break; + default: + throw new UnsupportedOperationException("Unsupported decompression algorithm: " + compression); + } + + return alg; + } + + default ClickHouseInputStream decompress(ClickHousePassThruStream stream, InputStream input, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + throw new UnsupportedOperationException( + ClickHouseUtils.format(ERROR_UNSUPPORTED_DECOMPRESS_ALG, getAlgorithm())); + } + + default ClickHouseOutputStream compress(ClickHousePassThruStream stream, OutputStream output, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + throw new UnsupportedOperationException(ClickHouseUtils.format(ERROR_UNSUPPORTED_COMPRESS_ALG, getAlgorithm())); + } + + ClickHouseCompression getAlgorithm(); +} diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataConfig.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataConfig.java new file mode 100644 index 000000000..b288869f9 --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataConfig.java @@ -0,0 +1,373 @@ +package com.clickhouse.data; + +import java.io.Serializable; +import java.math.RoundingMode; +import java.util.TimeZone; + +import com.clickhouse.config.ClickHouseBufferingMode; +import com.clickhouse.config.ClickHouseRenameMethod; + +public interface ClickHouseDataConfig extends Serializable { + static class Wrapped implements ClickHouseDataConfig { + private static final long serialVersionUID = -8358244156373920188L; + + private final ClickHouseDataConfig config; + + public Wrapped(ClickHouseDataConfig config) { + this.config = ClickHouseChecker.nonNull(config, TYPE_NAME); + } + + @Override + public ClickHouseFormat getFormat() { + return config.getFormat(); + } + + @Override + public int getBufferQueueVariation() { + return config.getBufferQueueVariation(); + } + + @Override + public int getBufferSize() { + return config.getBufferSize(); + } + + @Override + public int getMaxBufferSize() { + return config.getMaxBufferSize(); + } + + @Override + public int getReadBufferSize() { + return config.getReadBufferSize(); + } + + @Override + public int getWriteBufferSize() { + return config.getWriteBufferSize(); + } + + @Override + public int getReadTimeout() { + return config.getReadTimeout(); + } + + @Override + public int getWriteTimeout() { + return config.getWriteTimeout(); + } + + @Override + public ClickHouseBufferingMode getReadBufferingMode() { + return config.getReadBufferingMode(); + } + + @Override + public ClickHouseBufferingMode getWriteBufferingMode() { + return config.getWriteBufferingMode(); + } + + @Override + public ClickHouseRenameMethod getColumnRenameMethod() { + return config.getColumnRenameMethod(); + } + + @Override + public int getMaxQueuedBuffers() { + return config.getMaxQueuedBuffers(); + } + + @Override + public TimeZone getTimeZoneForDate() { + return config.getTimeZoneForDate(); + } + + @Override + public TimeZone getUseTimeZone() { + return config.getUseTimeZone(); + } + + @Override + public boolean isReuseValueWrapper() { + return config.isReuseValueWrapper(); + } + + @Override + public boolean isUseBinaryString() { + return config.isUseBinaryString(); + } + + @Override + public boolean isUseBlockingQueue() { + return config.isUseBlockingQueue(); + } + + @Override + public boolean isUseObjectsInArray() { + return config.isUseObjectsInArray(); + } + + @Override + public boolean isWidenUnsignedTypes() { + return config.isWidenUnsignedTypes(); + } + } + + static final String TYPE_NAME = "DataConfig"; + + static final boolean DEFAULT_ASYNC = true; + + static final ClickHouseBufferingMode DEFAULT_BUFFERING_MODE = ClickHouseBufferingMode.RESOURCE_EFFICIENT; + + static final int DEFAULT_BUFFER_SIZE = 4096; + static final int DEFAULT_READ_BUFFER_SIZE = DEFAULT_BUFFER_SIZE; + static final int DEFAULT_WRITE_BUFFER_SIZE = DEFAULT_BUFFER_SIZE; + static final int DEFAULT_MAX_BUFFER_SIZE = 128 * 1024; + static final int DEFAULT_MAX_QUEUED_BUFFERS = 512; + static final int DEFAULT_BUFFER_QUEUE_VARIATION = 100; + + static final ClickHouseRenameMethod DEFAULT_COLUMN_RENAME_METHOD = ClickHouseRenameMethod.NONE; + + static final ClickHouseFormat DEFAULT_FORMAT = ClickHouseFormat.TabSeparated; + + static final boolean DEFAULT_REUSE_VALUE_WRAPPER = true; + static final boolean DEFAULT_USE_BINARY_STRING = false; + static final boolean DEFAULT_USE_BLOCKING_QUEUE = false; + static final boolean DEFAULT_USE_OBJECT_IN_ARRAY = false; + static final boolean DEFAULT_WIDEN_UNSIGNED_TYPE = false; + + static final int DEFAULT_READ_COMPRESS_LEVEL = -1; + static final int DEFAULT_WRITE_COMPRESS_LEVEL = -1; + + static final RoundingMode DEFAULT_ROUNDING_MODE = RoundingMode.DOWN; + + /** + * Gets buffer size. Same as + * {@code getBufferSize(bufferSize, DEFAULT_BUFFER_SIZE, DEFAULT_MAX_BUFFER_SIZE)}. + * + * @param bufferSize suggested buffer size, zero or negative number is treated + * as {@link #DEFAULT_BUFFER_SIZE} + * @return buffer size + */ + static int getBufferSize(int bufferSize) { + return getBufferSize(bufferSize, DEFAULT_BUFFER_SIZE, DEFAULT_MAX_BUFFER_SIZE); + } + + /** + * Gets buffer size. + * + * @param bufferSize suggested buffer size, zero or negative number is treated + * as {@code defaultSize} + * @param defaultSize default buffer size, zero or negative number is treated as + * {@link #DEFAULT_BUFFER_SIZE} + * @param maxSize maximum buffer size, zero or negative number is treated as + * {@link #DEFAULT_MAX_BUFFER_SIZE} + * @return buffer size + */ + static int getBufferSize(int bufferSize, int defaultSize, int maxSize) { + if (maxSize < 1 || maxSize > DEFAULT_MAX_BUFFER_SIZE) { + maxSize = DEFAULT_MAX_BUFFER_SIZE; + } + if (defaultSize < 1) { + defaultSize = DEFAULT_BUFFER_SIZE; + } else if (defaultSize > maxSize) { + defaultSize = maxSize; + } + + if (bufferSize < 1) { + return defaultSize; + } + + return bufferSize > maxSize ? maxSize : bufferSize; + } + + /** + * Gets default read buffer size in byte. Same as + * {@code getBufferSize(DEFAULT_READ_BUFFER_SIZE, DEFAULT_BUFFER_SIZE, DEFAULT_MAX_BUFFER_SIZE)}. + * + * @return default read buffer size in byte + */ + static int getDefaultReadBufferSize() { + return getBufferSize(DEFAULT_READ_BUFFER_SIZE, DEFAULT_BUFFER_SIZE, DEFAULT_MAX_BUFFER_SIZE); + } + + /** + * Gets data format. + * + * @return non-null data format + */ + default ClickHouseFormat getFormat() { + return DEFAULT_FORMAT; + } + + /** + * Gets number of times the buffer queue is filled up before + * increasing capacity of buffer queue. Zero or negative value means the queue + * length is fixed. + * + * @return variation + */ + default int getBufferQueueVariation() { + return DEFAULT_BUFFER_QUEUE_VARIATION; + } + + /** + * Gets buffer size in byte can be used for streaming. + * + * @return buffer size in byte + */ + default int getBufferSize() { + return DEFAULT_BUFFER_SIZE; + } + + /** + * Gets max buffer size in byte can be used for streaming. + * + * @return max buffer size in byte + */ + default int getMaxBufferSize() { + return DEFAULT_MAX_BUFFER_SIZE; + } + + /** + * Gets read buffer size in byte. + * + * @return read buffer size in byte + */ + default int getReadBufferSize() { + return DEFAULT_READ_BUFFER_SIZE; + } + + /** + * Gets write buffer size in byte. + * + * @return write buffer size in byte + */ + default int getWriteBufferSize() { + return DEFAULT_WRITE_BUFFER_SIZE; + } + + /** + * Gets maximum number of buffers can be queued for processing. + * + * @return maximum number of buffers can be queued + */ + default int getMaxQueuedBuffers() { + return DEFAULT_MAX_QUEUED_BUFFERS; + } + + /** + * Gets buffering mode for reading. + * + * @return non-null buffering mode for reading + */ + default ClickHouseBufferingMode getReadBufferingMode() { + return DEFAULT_BUFFERING_MODE; + } + + /** + * Gets buffering mode for writing. + * + * @return non-null buffering mode for writing + */ + default ClickHouseBufferingMode getWriteBufferingMode() { + return DEFAULT_BUFFERING_MODE; + } + + /** + * Gets column rename method. + * + * @return non-null column rename method + */ + default ClickHouseRenameMethod getColumnRenameMethod() { + return DEFAULT_COLUMN_RENAME_METHOD; + } + + /** + * Checks whether async call is used. + * + * @return true if async call is used; false otherwise + */ + default boolean isAsync() { + return DEFAULT_ASYNC; + } + + /** + * Checks whether value wrapper {@link ClickHouseValue} should be reused. + * + * @return true if value wrapper will be reused; false otherwise + */ + default boolean isReuseValueWrapper() { + return DEFAULT_REUSE_VALUE_WRAPPER; + } + + /** + * Checks whether binary string is supported. + * + * @return true if binary string is supported; false otherwise + */ + default boolean isUseBinaryString() { + return DEFAULT_USE_BINARY_STRING; + } + + /** + * Checks whether blocking queue(mainly for piped stream) is used or not. + * + * @return true if blocking queue is used; false indicates that non-blocking + * queue is used(faster but consumed more CPU) + */ + default boolean isUseBlockingQueue() { + return DEFAULT_USE_BLOCKING_QUEUE; + } + + /** + * Checks whether object(instead of primitive) is used in array. + * + * @return true if object is used in array; false indicates that primitive type + * is used(no auto-boxing and less memory footprint) + */ + default boolean isUseObjectsInArray() { + return DEFAULT_USE_OBJECT_IN_ARRAY; + } + + /** + * Checks whether widening is enabled for unsigned types, for instance: use + * {@code long} (instead of {@code int}) in Java to represent {@code UInt32} in + * ClickHouse. + * + * @return true if widening is enabled; false indicates that same type is shared + * by signed and unsigned types(e.g. {@code int} for both {@code Int32} + * and {@code UInt32}) + */ + default boolean isWidenUnsignedTypes() { + return DEFAULT_WIDEN_UNSIGNED_TYPE; + } + + /** + * Gets read timeout in milliseconds. + * + * @return read time out in milliseconds + */ + int getReadTimeout(); + + /** + * Gets write timeout in milliseconds. + * + * @return write time out in milliseconds + */ + int getWriteTimeout(); + + /** + * Gets time zone for date values. + * + * @return time zone, could be null + */ + TimeZone getTimeZoneForDate(); + + /** + * Gets preferred time zone. + * + * @return non-null preferred time zone + */ + TimeZone getUseTimeZone(); +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataProcessor.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataProcessor.java similarity index 96% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataProcessor.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataProcessor.java index 461920e3b..ce1737386 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataProcessor.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataProcessor.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.EOFException; import java.io.IOException; @@ -21,6 +21,17 @@ * can handle more scenarios like separator between columns and rows. */ public abstract class ClickHouseDataProcessor { + protected static final class UseObjectConfig extends ClickHouseDataConfig.Wrapped { + public UseObjectConfig(ClickHouseDataConfig config) { + super(config); + } + + @Override + public boolean isUseObjectsInArray() { + return true; + } + } + static final class RecordsIterator implements Iterator { private final ClickHouseDataProcessor processor; @@ -65,7 +76,7 @@ public ClickHouseValue next() { protected static final String ERROR_REACHED_END_OF_STREAM = "Reached end of the stream when reading column #%d of %d: %s"; protected static final String ERROR_UNKNOWN_DATA_TYPE = "Unsupported data type: "; - protected final ClickHouseConfig config; + protected final ClickHouseDataConfig config; protected final ClickHouseInputStream input; protected final ClickHouseOutputStream output; protected final ClickHouseColumn[] columns; @@ -277,10 +288,10 @@ protected void readAndFill(ClickHouseValue value) throws IOException { * @param settings nullable settings * @throws IOException when failed to read columns from input stream */ - protected ClickHouseDataProcessor(ClickHouseConfig config, ClickHouseInputStream input, + protected ClickHouseDataProcessor(ClickHouseDataConfig config, ClickHouseInputStream input, ClickHouseOutputStream output, List columns, Map settings) throws IOException { - this.config = ClickHouseChecker.nonNull(config, ClickHouseConfig.TYPE_NAME); + this.config = ClickHouseChecker.nonNull(config, ClickHouseDataConfig.TYPE_NAME); if (input == null && output == null) { throw new IllegalArgumentException("One of input and output stream must not be null"); } @@ -342,9 +353,10 @@ protected ClickHouseDataProcessor(ClickHouseConfig config, ClickHouseInputStream this.writePosition = 0; } - public abstract ClickHouseDeserializer getDeserializer(ClickHouseConfig config, ClickHouseColumn column); + public abstract ClickHouseDeserializer getDeserializer(ClickHouseDataConfig config, ClickHouseColumn column); - public final ClickHouseDeserializer[] getDeserializers(ClickHouseConfig config, List columns) { + public final ClickHouseDeserializer[] getDeserializers(ClickHouseDataConfig config, + List columns) { List list = new ArrayList<>(columns.size()); for (ClickHouseColumn column : columns) { list.add(getDeserializer(config, column)); @@ -352,9 +364,9 @@ public final ClickHouseDeserializer[] getDeserializers(ClickHouseConfig config, return list.toArray(new ClickHouseDeserializer[0]); } - public abstract ClickHouseSerializer getSerializer(ClickHouseConfig config, ClickHouseColumn column); + public abstract ClickHouseSerializer getSerializer(ClickHouseDataConfig config, ClickHouseColumn column); - public final ClickHouseSerializer[] getSerializers(ClickHouseConfig config, List columns) { + public final ClickHouseSerializer[] getSerializers(ClickHouseDataConfig config, List columns) { List list = new ArrayList<>(columns.size()); for (ClickHouseColumn column : columns) { list.add(getSerializer(config, column)); diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataStreamFactory.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataStreamFactory.java similarity index 74% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataStreamFactory.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataStreamFactory.java index 5f523537b..5f8530910 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataStreamFactory.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataStreamFactory.java @@ -1,17 +1,16 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.IOException; import java.io.Serializable; import java.util.List; import java.util.Map; -import com.clickhouse.client.config.ClickHouseBufferingMode; -import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.data.ClickHouseRowBinaryProcessor; -import com.clickhouse.client.data.ClickHouseTabSeparatedProcessor; -import com.clickhouse.client.stream.BlockingPipedOutputStream; -import com.clickhouse.client.stream.CapacityPolicy; -import com.clickhouse.client.stream.NonBlockingPipedOutputStream; +import com.clickhouse.config.ClickHouseBufferingMode; +import com.clickhouse.data.format.ClickHouseRowBinaryProcessor; +import com.clickhouse.data.format.ClickHouseTabSeparatedProcessor; +import com.clickhouse.data.stream.BlockingPipedOutputStream; +import com.clickhouse.data.stream.CapacityPolicy; +import com.clickhouse.data.stream.NonBlockingPipedOutputStream; /** * Factory class for creating objects to handle data stream. @@ -34,7 +33,8 @@ public static ClickHouseDataStreamFactory getInstance() { } /** - * Gets data processor according to given {@link ClickHouseConfig} and settings. + * Gets data processor according to given {@link ClickHouseDataConfig} and + * settings. * * @param config non-null configuration containing information like * {@link ClickHouseFormat} @@ -47,10 +47,10 @@ public static ClickHouseDataStreamFactory getInstance() { * @return data processor, which might be null * @throws IOException when failed to read columns from input stream */ - public ClickHouseDataProcessor getProcessor(ClickHouseConfig config, ClickHouseInputStream input, + public ClickHouseDataProcessor getProcessor(ClickHouseDataConfig config, ClickHouseInputStream input, ClickHouseOutputStream output, Map settings, List columns) throws IOException { - ClickHouseFormat format = ClickHouseChecker.nonNull(config, ClickHouseConfig.TYPE_NAME).getFormat(); + ClickHouseFormat format = ClickHouseChecker.nonNull(config, ClickHouseDataConfig.TYPE_NAME).getFormat(); ClickHouseDataProcessor processor = null; if (ClickHouseFormat.RowBinary == format || ClickHouseFormat.RowBinaryWithNamesAndTypes == format) { processor = new ClickHouseRowBinaryProcessor(config, input, output, columns, settings); @@ -68,14 +68,14 @@ public ClickHouseDataProcessor getProcessor(ClickHouseConfig config, ClickHouseI * the output stream * @return piped output stream */ - public ClickHousePipedOutputStream createPipedOutputStream(ClickHouseConfig config, Runnable postCloseAction) { - final int bufferSize = ClickHouseChecker.nonNull(config, ClickHouseConfig.TYPE_NAME).getWriteBufferSize(); + public ClickHousePipedOutputStream createPipedOutputStream(ClickHouseDataConfig config, Runnable postCloseAction) { + final int bufferSize = ClickHouseChecker.nonNull(config, ClickHouseDataConfig.TYPE_NAME).getWriteBufferSize(); final boolean blocking; final int queue; final CapacityPolicy policy; final int timeout; - if (config.getResponseBuffering() == ClickHouseBufferingMode.PERFORMANCE) { + if (config.getReadBufferingMode() == ClickHouseBufferingMode.PERFORMANCE) { blocking = false; queue = 0; policy = null; @@ -85,7 +85,7 @@ public ClickHousePipedOutputStream createPipedOutputStream(ClickHouseConfig conf queue = config.getMaxQueuedBuffers(); policy = config.getBufferQueueVariation() < 1 ? CapacityPolicy.fixedCapacity(queue) : CapacityPolicy.linearDynamicCapacity(1, queue, config.getBufferQueueVariation()); - timeout = config.getSocketTimeout(); + timeout = config.getReadTimeout(); } return blocking ? new BlockingPipedOutputStream(bufferSize, queue, timeout, postCloseAction) @@ -94,10 +94,7 @@ public ClickHousePipedOutputStream createPipedOutputStream(ClickHouseConfig conf public ClickHousePipedOutputStream createPipedOutputStream(int bufferSize, int queueSize, int timeout, Runnable postCloseAction) { - return new BlockingPipedOutputStream( - ClickHouseUtils.getBufferSize(bufferSize, - (int) ClickHouseClientOption.BUFFER_SIZE.getDefaultValue(), - (int) ClickHouseClientOption.MAX_BUFFER_SIZE.getDefaultValue()), - queueSize, timeout, postCloseAction); + return new BlockingPipedOutputStream(ClickHouseDataConfig.getBufferSize(bufferSize), queueSize, timeout, + postCloseAction); } } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataType.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataType.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataType.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataType.java index bf93c166d..dc43b0103 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataType.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataType.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.math.BigDecimal; import java.math.BigInteger; @@ -17,10 +17,10 @@ import java.util.TreeSet; import java.util.UUID; -import com.clickhouse.client.data.UnsignedByte; -import com.clickhouse.client.data.UnsignedInteger; -import com.clickhouse.client.data.UnsignedLong; -import com.clickhouse.client.data.UnsignedShort; +import com.clickhouse.data.value.UnsignedByte; +import com.clickhouse.data.value.UnsignedInteger; +import com.clickhouse.data.value.UnsignedLong; +import com.clickhouse.data.value.UnsignedShort; /** * Basic ClickHouse data types. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataUpdater.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataUpdater.java similarity index 96% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataUpdater.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataUpdater.java index 2551b67eb..c8cb83430 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataUpdater.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataUpdater.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.IOException; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDeferredValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDeferredValue.java similarity index 95% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDeferredValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDeferredValue.java index 7b5364896..7db57d258 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDeferredValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDeferredValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.util.Optional; import java.util.concurrent.CancellationException; @@ -45,10 +45,10 @@ public static ClickHouseDeferredValue of(CompletableFuture future, int return f.get(t, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - future.cancel(false); + f.cancel(false); throw new CompletionException(e); } catch (CancellationException e) { - future.cancel(true); + f.cancel(true); throw new CompletionException(e); } catch (ExecutionException | TimeoutException e) { throw new CompletionException(e); @@ -76,7 +76,7 @@ public static ClickHouseDeferredValue of(Supplier supplier) { * @param clazz class of the value * @return deferred value */ - public static ClickHouseDeferredValue of(T value, Class clazz) { + public static ClickHouseDeferredValue of(T value, Class clazz) { // NOSONAR return new ClickHouseDeferredValue<>(null, Optional.ofNullable(value)); } @@ -103,6 +103,6 @@ public Optional getOptional() { return value; } - return value = (supplier == null ? Optional.empty() : Optional.ofNullable(supplier.get())); + return value = (supplier == null ? Optional.empty() : Optional.ofNullable(supplier.get())); // NOSONAR } } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDeserializer.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDeserializer.java similarity index 97% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDeserializer.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDeserializer.java index 5545a5d85..0482184df 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDeserializer.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDeserializer.java @@ -1,11 +1,11 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.IOException; import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import com.clickhouse.client.data.ClickHouseEmptyValue; +import com.clickhouse.data.value.ClickHouseEmptyValue; /** * Functional interface for deserialization. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseEnum.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseEnum.java similarity index 99% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseEnum.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseEnum.java index 323c1e1be..815676405 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseEnum.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseEnum.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.Serializable; import java.util.Arrays; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseExternalTable.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseExternalTable.java similarity index 67% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseExternalTable.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseExternalTable.java index 648750c2b..00402accd 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseExternalTable.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseExternalTable.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data; import java.io.FileNotFoundException; import java.io.InputStream; @@ -9,18 +9,6 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; - -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseCompression; -import com.clickhouse.client.ClickHouseDeferredValue; -import com.clickhouse.client.ClickHouseFile; -import com.clickhouse.client.ClickHouseFormat; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.config.ClickHouseClientOption; /** * "Attached" temporary table. @@ -28,15 +16,17 @@ public class ClickHouseExternalTable implements Serializable { public static class Builder { private String name; - private ClickHouseFile file; private ClickHouseDeferredValue content; private ClickHouseCompression compression; + private int compressionLevel; private ClickHouseFormat format; private List columns; private boolean asTempTable; protected Builder() { columns = new LinkedList<>(); + + compressionLevel = ClickHouseDataConfig.DEFAULT_WRITE_COMPRESS_LEVEL; } public Builder name(String name) { @@ -45,36 +35,49 @@ public Builder name(String name) { } public Builder compression(ClickHouseCompression compression) { + return compression(compression, ClickHouseDataConfig.DEFAULT_WRITE_COMPRESS_LEVEL); + } + + public Builder compression(ClickHouseCompression compression, int compressionLevel) { this.compression = compression; + this.compressionLevel = compressionLevel; return this; } - public Builder content(ClickHouseFile file) { - this.file = ClickHouseChecker.nonNull(file, "file"); - this.compression = file.getCompressionAlgorithm(); - this.content = ClickHouseDeferredValue.of(file.asInputStream(), InputStream.class); - if (file.hasFormat()) { - this.format = file.getFormat(); + public Builder content(ClickHousePassThruStream stream) { + if (stream == null || !stream.hasInput()) { + throw new IllegalArgumentException("Non-null pass-thru stream with input is required"); + } + + this.compression = ClickHouseCompression.NONE; + this.compressionLevel = ClickHouseDataConfig.DEFAULT_WRITE_COMPRESS_LEVEL; + this.content = ClickHouseDeferredValue.of(stream.getInputStream(), InputStream.class); + if (stream.hasFormat()) { + this.format = stream.getFormat(); } return this; } public Builder content(InputStream content) { - this.content = ClickHouseDeferredValue.of(ClickHouseChecker.nonNull(content, "content"), InputStream.class); + if (ClickHouseChecker.nonNull(content, "content") instanceof ClickHouseInputStream) { + return content((ClickHouseInputStream) content); + } + + this.content = ClickHouseDeferredValue.of(content, InputStream.class); return this; } - /** - * Sets future content. - * - * @param content non-null future content - * @return this builder - * @deprecated will be removed in v0.3.3, please use - * {@link #content(ClickHouseDeferredValue)} instead - */ - @Deprecated - public Builder content(CompletableFuture content) { - this.content = ClickHouseDeferredValue.of(ClickHouseChecker.nonNull(content, "Content")); + public Builder content(ClickHouseInputStream input) { + ClickHousePassThruStream stream = ClickHouseChecker.nonNull(input, ClickHouseInputStream.TYPE_NAME) + .getUnderlyingStream(); + if (stream.hasInput()) { + this.compression = ClickHouseCompression.NONE; + this.compressionLevel = ClickHouseDataConfig.DEFAULT_WRITE_COMPRESS_LEVEL; + this.content = ClickHouseDeferredValue.of(stream.getInputStream(), InputStream.class); + } else { + this.content = ClickHouseDeferredValue.of(input, InputStream.class); + } + return this; } @@ -159,7 +162,8 @@ public Builder asExternalTable() { } public ClickHouseExternalTable build() { - return new ClickHouseExternalTable(name, file, content, compression, format, columns, asTempTable); + return new ClickHouseExternalTable(name, content, compression, compressionLevel, format, columns, + asTempTable); } } @@ -169,27 +173,30 @@ public static Builder builder() { private static final long serialVersionUID = -5395148151046691946L; + public static final String TYPE_NAME = "ExternalTable"; + private final String name; - private final ClickHouseFile file; private final ClickHouseDeferredValue content; - private final Optional compression; + private final ClickHouseCompression compression; + private final int compressionLevel; private final ClickHouseFormat format; private final List columns; private final boolean asTempTable; private final String structure; - protected ClickHouseExternalTable(String name, ClickHouseFile file, ClickHouseDeferredValue content, - ClickHouseCompression compression, ClickHouseFormat format, Collection columns, - boolean asTempTable) { + protected ClickHouseExternalTable(String name, ClickHouseDeferredValue content, + ClickHouseCompression compression, int compressionLevel, ClickHouseFormat format, + Collection columns, boolean asTempTable) { this.name = name == null ? "" : name.trim(); - this.file = file != null ? file : ClickHouseFile.NULL; + if (compression == null) { compression = ClickHouseCompression.fromFileName(this.name); - this.compression = Optional.ofNullable(compression == ClickHouseCompression.NONE ? null : compression); + this.compression = compression == null ? ClickHouseCompression.NONE : compression; } else { - this.compression = Optional.of(compression); + this.compression = compression; } + this.compressionLevel = compressionLevel; this.format = format == null ? ClickHouseFormat.TabSeparated : format; if (content == null) { @@ -198,11 +205,9 @@ protected ClickHouseExternalTable(String name, ClickHouseFile file, ClickHouseDe this.content = compression == ClickHouseCompression.NONE ? content // unfortunately ClickHouse does not support compressed external data : ClickHouseDeferredValue - .of(() -> ClickHouseInputStream.of(content.get(), ClickHouseUtils.getBufferSize( - (int) ClickHouseClientOption.READ_BUFFER_SIZE.getDefaultValue(), - (int) ClickHouseClientOption.BUFFER_SIZE.getDefaultValue(), - (int) ClickHouseClientOption.MAX_BUFFER_SIZE.getDefaultValue()), this.compression.get(), - null)); + .of(() -> ClickHouseInputStream.of(content.get(), + ClickHouseDataConfig.getDefaultReadBufferSize(), this.compression, + this.compressionLevel, null)); int size = columns == null ? 0 : columns.size(); if (size == 0) { @@ -230,18 +235,18 @@ public String getName() { return name; } - public ClickHouseFile getFile() { - return file; - } - public InputStream getContent() { return content.get(); } - public Optional getCompression() { + public ClickHouseCompression getCompression() { return compression; } + public int getCompressionLevel() { + return compressionLevel; + } + public ClickHouseFormat getFormat() { return format; } diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseFile.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseFile.java new file mode 100644 index 000000000..466b60233 --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseFile.java @@ -0,0 +1,180 @@ +package com.clickhouse.data; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Path; + +/** + * Wrapper of {@link java.io.File} with additional information like compression + * and format. + */ +public class ClickHouseFile extends ClickHousePassThruStream { + private static final long serialVersionUID = -2641191818870839568L; + + private static final String FILE_TYPE_NAME = "File"; + + /** + * Null file which has no compression and format. + */ + public static final ClickHouseFile NULL = new ClickHouseFile(null, ClickHouseCompression.NONE, + ClickHouseDataConfig.DEFAULT_READ_COMPRESS_LEVEL, null); + + public static ClickHouseFile of(File file) { + return of(file, null, 0, null); + } + + public static ClickHouseFile of(Path path) { + return of(ClickHouseChecker.nonNull(path, "Path").toFile(), null, + ClickHouseDataConfig.DEFAULT_READ_COMPRESS_LEVEL, null); + } + + public static ClickHouseFile of(String file) { + return of(new File(ClickHouseChecker.nonEmpty(file, FILE_TYPE_NAME)), null, + ClickHouseDataConfig.DEFAULT_READ_COMPRESS_LEVEL, null); + } + + public static ClickHouseFile of(String file, ClickHouseCompression compression, int compressionLevel, + ClickHouseFormat format) { + return of(new File(ClickHouseChecker.nonEmpty(file, FILE_TYPE_NAME)), compression, compressionLevel, format); + } + + public static ClickHouseFile of(File file, ClickHouseCompression compression, int compressionLevel, + ClickHouseFormat format) { + final String name = ClickHouseChecker.nonNull(file, FILE_TYPE_NAME).getName(); + return new ClickHouseFile(file, + compression != null ? compression : ClickHouseCompression.fromFileName(name), + compressionLevel, format != null ? format : ClickHouseFormat.fromFileName(name)); + } + + public static ClickHouseFile of(ClickHousePassThruStream stream) { + if (stream instanceof ClickHouseFile) { + return (ClickHouseFile) stream; + } else if (!stream.hasInput()) { + throw new IllegalArgumentException(ClickHousePassThruStream.ERROR_NO_INPUT); + } + + return of(stream.getInputStream(), stream.getCompressionAlgorithm(), stream.getCompressionLevel(), + stream.getFormat()); + } + + public static ClickHouseFile of(ClickHouseInputStream input, ClickHouseCompression compression, + int compressionLevel, ClickHouseFormat format) { + final ClickHousePassThruStream stream = ClickHouseChecker.nonNull(input, ClickHouseInputStream.TYPE_NAME) + .getUnderlyingStream(); + if (stream instanceof ClickHouseFile) { + return ((ClickHouseFile) stream); + } + + final boolean hasInput = stream.hasInput(); + final File tmp; + try { + tmp = ClickHouseUtils.createTempFile(); + } catch (IOException e) { + throw new IllegalArgumentException("Failed to create temporary file", e); + } + + try (ClickHouseOutputStream out = ClickHouseOutputStream.of(new FileOutputStream(tmp))) { + (hasInput ? stream.getInputStream() : input).pipe(out); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return hasInput ? of(tmp, stream.getCompressionAlgorithm(), stream.getCompressionLevel(), stream.getFormat()) + : of(tmp, compression, compressionLevel, format); + } + + public static ClickHouseFile of(InputStream input, ClickHouseCompression compression, int compressionLevel, + ClickHouseFormat format) { + if (ClickHouseChecker.nonNull(input, ClickHouseInputStream.TYPE_NAME) instanceof ClickHouseInputStream) { + return of((ClickHouseInputStream) input, compression, compressionLevel, format); + } + + final File tmp; + try { + tmp = ClickHouseUtils.createTempFile(); + } catch (IOException e) { + throw new IllegalArgumentException("Failed to create temporary file", e); + } + + try (ClickHouseOutputStream out = ClickHouseOutputStream.of(new FileOutputStream(tmp))) { + ClickHouseInputStream.of(input).pipe(out); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return of(tmp, compression, compressionLevel, format); + } + + private final File file; + + protected ClickHouseFile(File file, ClickHouseCompression compress, int compressLevel, ClickHouseFormat format) { + super(null, null, compress, compressLevel, format); + + this.file = file; + } + + @Override + public ClickHouseInputStream getInputStream() { + return ClickHouseInputStream.of(getFile()); + } + + @Override + public ClickHouseInputStream newInputStream(int bufferSize, Runnable postCloseAction) { + if (!hasInput()) { + return ClickHouseInputStream.empty(); + } + + try { + return ClickHouseInputStream.wrap(this, new FileInputStream(getFile()), bufferSize, + ClickHouseCompression.NONE, getCompressionLevel(), postCloseAction); + } catch (FileNotFoundException e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public ClickHouseOutputStream getOutputStream() { + try { + return ClickHouseOutputStream.of(new FileOutputStream(getFile())); + } catch (FileNotFoundException e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public ClickHouseOutputStream newOutputStream(int bufferSize, Runnable postCloseAction) { + if (!hasOutput()) { + return ClickHouseOutputStream.empty(); + } + + try { + return ClickHouseOutputStream.wrap(this, new FileOutputStream(getFile()), + bufferSize, ClickHouseCompression.NONE, getCompressionLevel(), postCloseAction); + } catch (FileNotFoundException e) { + throw new IllegalArgumentException(e); + } + } + + /** + * Gets file. Use {@code #hasInput()} or {@code #hasOutput()} to check file + * availability first. + * + * @return file, could be null + */ + public File getFile() { + return file; + } + + @Override + public boolean hasInput() { + return file != null && file.exists(); + } + + @Override + public boolean hasOutput() { + return file != null; + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseFormat.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseFormat.java similarity index 99% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseFormat.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseFormat.java index dee7ed57c..b474dfe4e 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseFormat.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseFormat.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.util.Locale; @@ -109,11 +109,11 @@ public enum ClickHouseFormat { */ public static ClickHouseFormat fromFileName(String file) { ClickHouseCompression compression = ClickHouseCompression.fromFileName(file); - if (compression != ClickHouseCompression.NONE) { + if (compression != null) { file = file.substring(0, file.lastIndexOf('.')); } - ClickHouseFormat format = null; + ClickHouseFormat format = null; int index = 0; if (file != null && (index = file.lastIndexOf('.')) > 0) { String ext = file.substring(index + 1).toLowerCase(Locale.ROOT); diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseInputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseInputStream.java similarity index 84% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseInputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseInputStream.java index 78aa94d84..c1230a94b 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseInputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseInputStream.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.EOFException; import java.io.File; @@ -21,21 +21,16 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.Function; -import java.util.zip.GZIPInputStream; -import java.util.zip.InflaterInputStream; - -import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.stream.BlockingInputStream; -import com.clickhouse.client.stream.CompressionUtils; -import com.clickhouse.client.stream.DeferredInputStream; -import com.clickhouse.client.stream.EmptyInputStream; -import com.clickhouse.client.stream.Lz4InputStream; -import com.clickhouse.client.stream.RestrictedInputStream; -import com.clickhouse.client.stream.IterableByteArrayInputStream; -import com.clickhouse.client.stream.IterableByteBufferInputStream; -import com.clickhouse.client.stream.IterableMultipleInputStream; -import com.clickhouse.client.stream.IterableObjectInputStream; -import com.clickhouse.client.stream.WrappedInputStream; + +import com.clickhouse.data.stream.BlockingInputStream; +import com.clickhouse.data.stream.DeferredInputStream; +import com.clickhouse.data.stream.EmptyInputStream; +import com.clickhouse.data.stream.RestrictedInputStream; +import com.clickhouse.data.stream.IterableByteArrayInputStream; +import com.clickhouse.data.stream.IterableByteBufferInputStream; +import com.clickhouse.data.stream.IterableMultipleInputStream; +import com.clickhouse.data.stream.IterableObjectInputStream; +import com.clickhouse.data.stream.WrappedInputStream; /** * Extended input stream for read optimization. Methods like @@ -50,69 +45,24 @@ public abstract class ClickHouseInputStream extends InputStream { protected static final String ERROR_REUSE_BUFFER = "Please pass a different byte array instead of the same internal buffer for reading"; protected static final String ERROR_STREAM_CLOSED = "Input stream has been closed"; + public static final String TYPE_NAME = "InputStream"; + /** * Wraps the given input stream. * * @param stream pass-thru stream, could be null * @param input non-null input stream * @param bufferSize buffer size - * @param postCloseAction custom action will be performed right after closing - * the wrapped input stream * @param compression compression algorithm * @param compressionLevel compression level + * @param postCloseAction custom action will be performed right after closing + * the wrapped input stream * @return non-null wrapped input stream */ public static ClickHouseInputStream wrap(ClickHousePassThruStream stream, InputStream input, int bufferSize, - Runnable postCloseAction, ClickHouseCompression compression, int compressionLevel) { - final ClickHouseInputStream chInput; - if (compression == null || compression == ClickHouseCompression.NONE) { - chInput = input != EmptyInputStream.INSTANCE && input instanceof ClickHouseInputStream - ? (ClickHouseInputStream) input - : new WrappedInputStream(stream, input, bufferSize, postCloseAction); - } else { - switch (compression) { - case BROTLI: - chInput = new WrappedInputStream(stream, - CompressionUtils.createBrotliInputStream(input, bufferSize), - bufferSize, postCloseAction); - break; - case BZ2: - chInput = new WrappedInputStream(stream, CompressionUtils.createBz2InputStream(input), bufferSize, - postCloseAction); - break; - case DEFLATE: - chInput = new WrappedInputStream(stream, new InflaterInputStream(input), bufferSize, - postCloseAction); - break; - case GZIP: - try { - chInput = new WrappedInputStream(stream, new GZIPInputStream(input), bufferSize, - postCloseAction); - } catch (IOException e) { - throw new IllegalArgumentException(CompressionUtils.ERROR_FAILED_TO_WRAP_INPUT, e); - } - break; - case LZ4: - chInput = new Lz4InputStream(stream, input, postCloseAction); - break; - case SNAPPY: - // https://github.com/ClickHouse/ClickHouse/issues/44885 - chInput = new WrappedInputStream(stream, CompressionUtils.createSnappyInputStream(input), - bufferSize, postCloseAction); - break; - case ZSTD: - chInput = new WrappedInputStream(stream, CompressionUtils.createZstdInputStream(input), bufferSize, - postCloseAction); - break; - case XZ: - chInput = new WrappedInputStream(stream, CompressionUtils.createXzInputStream(input), bufferSize, - postCloseAction); - break; - default: - throw new UnsupportedOperationException("Unsupported decompression algorithm: " + compression); - } - } - return chInput; + ClickHouseCompression compression, int compressionLevel, Runnable postCloseAction) { + return ClickHouseCompressionAlgorithm.createInputStream(stream, input, bufferSize, compression, + compressionLevel, postCloseAction); } /** @@ -176,7 +126,7 @@ public static ClickHouseInputStream of(BlockingQueue queue, int time * * @param deferredInput non-null deferred input stream * @param bufferSize buffer size which is always greater than zero(usually - * 8192 or larger) + * 4096 or larger) * @param postCloseAction custom action will be performed right after closing * the input stream * @return wrapped input @@ -191,30 +141,43 @@ public static ClickHouseInputStream of(ClickHouseDeferredValue defe * * @param stream non-null pass-thru stream * @param bufferSize buffer size which is always greater than zero(usually - * 8192 or larger) + * 4096 or larger) * @param postCloseAction custom action will be performed right after closing * the input stream * @return wrapped input */ public static ClickHouseInputStream of(ClickHousePassThruStream stream, int bufferSize, Runnable postCloseAction) { if (stream == null || !stream.hasInput()) { - throw new IllegalArgumentException("Non-null pass-thru stream required"); + throw new IllegalArgumentException("Non-null pass-thru stream with input is required"); } - return stream.asInputStream(bufferSize, postCloseAction); + return stream.newInputStream(bufferSize, postCloseAction); + } + + /** + * Wraps the given input stream. + * + * @param input input stream + * @return wrapped input, or the same input if it's instance of + * {@link ClickHouseInputStream} + */ + public static ClickHouseInputStream of(InputStream input) { + return of(input, ClickHouseDataConfig.getDefaultReadBufferSize(), null, + ClickHouseDataConfig.DEFAULT_READ_COMPRESS_LEVEL, null); } /** * Wraps the given input stream. * * @param input input stream - * @param bufferSize buffer size which is always greater than zero(usually 8192 - * or larger) + * @param bufferSize buffer size which is always greater than zero(usually + * 4096 or larger) * @return wrapped input, or the same input if it's instance of * {@link ClickHouseInputStream} */ public static ClickHouseInputStream of(InputStream input, int bufferSize) { - return of(input, bufferSize, null, null); + return of(input, ClickHouseDataConfig.getBufferSize(bufferSize), null, + ClickHouseDataConfig.DEFAULT_READ_COMPRESS_LEVEL, null); } /** @@ -222,53 +185,54 @@ public static ClickHouseInputStream of(InputStream input, int bufferSize) { * * @param input input stream * @param bufferSize buffer size which is always greater than zero(usually - * 8192 or larger) + * 4096 or larger) * @param postCloseAction custom action will be performed right after closing * the input stream * @return wrapped input, or the same input if it's instance of * {@link ClickHouseInputStream} */ public static ClickHouseInputStream of(InputStream input, int bufferSize, Runnable postCloseAction) { - return of(input, bufferSize, null, postCloseAction); + return of(input, bufferSize, ClickHouseCompression.NONE, ClickHouseDataConfig.DEFAULT_READ_COMPRESS_LEVEL, + postCloseAction); } /** * Wraps the given input stream. * * @param input input stream - * @param bufferSize buffer size which is always greater than zero(usually 8192 - * or larger) * @param compression compression algorithm, null or * {@link ClickHouseCompression#NONE} means no compression * @return wrapped input, or the same input if it's instance of * {@link ClickHouseInputStream} */ - public static ClickHouseInputStream of(InputStream input, int bufferSize, ClickHouseCompression compression) { - return of(input, bufferSize, compression, null); + public static ClickHouseInputStream of(InputStream input, ClickHouseCompression compression) { + return of(input, ClickHouseDataConfig.getDefaultReadBufferSize(), compression, + ClickHouseDataConfig.DEFAULT_READ_COMPRESS_LEVEL, null); } /** * Wraps the given input stream. * - * @param input input stream - * @param bufferSize buffer size which is always greater than zero(usually - * 8192 or larger) - * @param compression compression algorithm, null or - * {@link ClickHouseCompression#NONE} means no - * compression - * @param postCloseAction custom action will be performed right after closing - * the input stream + * @param input input stream + * @param bufferSize buffer size which is always greater than zero(usually + * 4096 or larger) + * @param compression compression algorithm, null or + * {@link ClickHouseCompression#NONE} means no + * compression + * @param compressionLevel compression level + * @param postCloseAction custom action will be performed right after closing + * the input stream * @return wrapped input, or the same input if it's instance of * {@link ClickHouseInputStream} */ public static ClickHouseInputStream of(InputStream input, int bufferSize, ClickHouseCompression compression, - Runnable postCloseAction) { + int compressionLevel, Runnable postCloseAction) { if (input == null) { return EmptyInputStream.INSTANCE; } else if (input != EmptyInputStream.INSTANCE && input instanceof ClickHouseInputStream) { return (ClickHouseInputStream) input; } - return wrap(null, input, bufferSize, postCloseAction, compression, 0); + return wrap(null, input, bufferSize, compression, compressionLevel, postCloseAction); } /** @@ -310,7 +274,7 @@ public static ClickHouseInputStream of(File... files) { if (files == null || files.length == 0) { return EmptyInputStream.INSTANCE; } - return of(Arrays.asList(files), File.class, null, null); + return of(Arrays.asList(files), File.class, null, ClickHouseDataConfig.getDefaultReadBufferSize(), null); } /** @@ -324,9 +288,10 @@ public static ClickHouseInputStream of(InputStream... inputs) { if (inputs == null || inputs.length == 0) { return EmptyInputStream.INSTANCE; } else if (inputs.length == 1) { - return of(inputs[0], (int) ClickHouseClientOption.BUFFER_SIZE.getDefaultValue(), null, null); + return of(inputs[0]); } - return of(Arrays.asList(inputs), InputStream.class, null, null); + return of(Arrays.asList(inputs), InputStream.class, null, ClickHouseDataConfig.getDefaultReadBufferSize(), + null); } /** @@ -340,7 +305,7 @@ public static ClickHouseInputStream of(String... strings) { if (strings == null || strings.length == 0) { return EmptyInputStream.INSTANCE; } - return of(Arrays.asList(strings), String.class, null, null); + return of(Arrays.asList(strings), String.class, null, ClickHouseDataConfig.getDefaultReadBufferSize(), null); } /** @@ -354,7 +319,7 @@ public static ClickHouseInputStream of(URL... urls) { if (urls == null || urls.length == 0) { return EmptyInputStream.INSTANCE; } - return of(Arrays.asList(urls), URL.class, null, null); + return of(Arrays.asList(urls), URL.class, null, ClickHouseDataConfig.getDefaultReadBufferSize(), null); } /** @@ -375,7 +340,8 @@ public static ClickHouseInputStream of(T[] source, Class clazz, Function< if (source == null || source.length == 0) { return EmptyInputStream.INSTANCE; } - return of(Arrays.asList(source), clazz, converter, postCloseAction); + return of(Arrays.asList(source), clazz, converter, ClickHouseDataConfig.getDefaultReadBufferSize(), + postCloseAction); } /** @@ -393,9 +359,30 @@ public static ClickHouseInputStream of(T[] source, Class clazz, Function< * the input stream * @return non-null input stream */ - @SuppressWarnings("unchecked") public static ClickHouseInputStream of(Iterable source, Class clazz, Function converter, Runnable postCloseAction) { + return of(source, clazz, converter, ClickHouseDataConfig.getDefaultReadBufferSize(), postCloseAction); + } + + /** + * Wraps the given iterable objects as byte array based binary input stream. + * {@code byte[]}, {@link ByteBuffer}, {@link InputStream}, {@link File}, + * {@link String}, and {@link URL} are all supported by default. + * + * @param type of the object + * @param source iterable objects(e.g. byte[], ByteBuffer, and + * String etc.) + * @param clazz class of the object + * @param converter optional transformer to convert each object into byte + * array + * @param bufferSize read buffer size + * @param postCloseAction custom action will be performed right after closing + * the input stream + * @return non-null input stream + */ + @SuppressWarnings("unchecked") + public static ClickHouseInputStream of(Iterable source, Class clazz, Function converter, + int bufferSize, Runnable postCloseAction) { if (source == null) { return EmptyInputStream.INSTANCE; } else if (converter != null) { // respect custom converter regardless object type @@ -415,7 +402,7 @@ public static ClickHouseInputStream of(Iterable source, Class clazz, F } catch (IOException e) { throw new IllegalArgumentException(e); } - }, postCloseAction); + }, bufferSize, postCloseAction); } else if (clazz == String.class) { return new IterableObjectInputStream<>((Iterable) source, s -> s == null || s.isEmpty() ? ClickHouseByteBuffer.EMPTY_BYTES @@ -431,9 +418,10 @@ public static ClickHouseInputStream of(Iterable source, Class clazz, F } catch (IOException e) { throw new IllegalArgumentException(e); } - }, postCloseAction); + }, bufferSize, postCloseAction); } else if (InputStream.class.isAssignableFrom(clazz)) { - return new IterableMultipleInputStream<>((Iterable) source, i -> i, postCloseAction); + return new IterableMultipleInputStream<>((Iterable) source, i -> i, bufferSize, + postCloseAction); } else { throw new IllegalArgumentException("Missing converter for data type: " + clazz); } @@ -447,7 +435,7 @@ public static ClickHouseInputStream of(Iterable source, Class clazz, F * @param input non-null input stream, which will be closed * @param output non-null output stream, which will remain open * @param bufferSize buffer size, zero or negative number will be treated as - * {@link ClickHouseClientOption#BUFFER_SIZE} + * {@link ClickHouseDataConfig#DEFAULT_BUFFER_SIZE} * @return written bytes * @throws IOException when error occured reading from input stream or writing * data to output stream @@ -457,10 +445,7 @@ public static long pipe(InputStream input, OutputStream output, int bufferSize) return ((ClickHouseInputStream) input).pipe((ClickHouseOutputStream) output); } - bufferSize = ClickHouseUtils.getBufferSize(bufferSize, - (int) ClickHouseClientOption.BUFFER_SIZE.getDefaultValue(), - (int) ClickHouseClientOption.MAX_BUFFER_SIZE.getDefaultValue()); - return pipe(input, output, new byte[bufferSize]); + return pipe(input, output, new byte[ClickHouseDataConfig.getBufferSize(bufferSize)]); } /** @@ -595,12 +580,13 @@ protected void ensureOpen() throws IOException { } /** - * Gets underlying file. + * Gets underlying file. Same as + * {@code ClickHouseFile.of(getUnderlyingStream())}. * * @return non-null underlying file */ public ClickHouseFile getUnderlyingFile() { - return stream instanceof ClickHouseFile ? ((ClickHouseFile) stream) : ClickHouseFile.NULL; + return ClickHouseFile.of(stream); } /** @@ -622,6 +608,19 @@ public final Object getUserData(String key) { return userData.get(key); } + /** + * Gets user data associated with this input stream. + * + * @param type of the value + * @param key key + * @param defaultValue default value + * @return value, could be null + */ + @SuppressWarnings("unchecked") + public final T getUserData(String key, T defaultValue) { + return (T) userData.getOrDefault(key, defaultValue); + } + /** * Removes user data. * @@ -639,8 +638,9 @@ public final Object removeUserData(String key) { * @param value value * @return overidded value, could be null */ - public final Object setUserData(String key, Object value) { - return userData.put(key, value); + @SuppressWarnings("unchecked") + public final T setUserData(String key, T value) { + return (T) userData.put(key, value); } /** diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseOutputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseOutputStream.java similarity index 80% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseOutputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseOutputStream.java index a8726c3ec..4a69037a5 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseOutputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseOutputStream.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.IOException; import java.io.OutputStream; @@ -6,12 +6,9 @@ import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.stream.CompressionUtils; -import com.clickhouse.client.stream.DeferredOutputStream; -import com.clickhouse.client.stream.EmptyOutputStream; -import com.clickhouse.client.stream.Lz4OutputStream; -import com.clickhouse.client.stream.WrappedOutputStream; +import com.clickhouse.data.stream.DeferredOutputStream; +import com.clickhouse.data.stream.EmptyOutputStream; +import com.clickhouse.data.stream.WrappedOutputStream; /** * Extended output stream for write optimization. It also acts as a factory @@ -23,77 +20,25 @@ public abstract class ClickHouseOutputStream extends OutputStream { protected static final String ERROR_REUSE_BUFFER = "Please pass a different byte array instead of the same internal buffer for reading"; protected static final String ERROR_STREAM_CLOSED = "Output stream has been closed"; + public static final String TYPE_NAME = "OutputStream"; + public static final int DEFAULT_BUFFER_SIZE = 4096; + /** * Wraps the given output stream. * * @param stream pass-thru stream, could be null * @param output non-null output stream * @param bufferSize buffer size - * @param postCloseAction custom action will be performed right after closing - * the wrapped output stream * @param compression compression algorithm * @param compressionLevel compression level + * @param postCloseAction custom action will be performed right after closing + * the wrapped output stream * @return non-null wrapped output stream */ static ClickHouseOutputStream wrap(ClickHousePassThruStream stream, OutputStream output, int bufferSize, - Runnable postCloseAction, ClickHouseCompression compression, int compressionLevel) { - final ClickHouseOutputStream chOutput; - if (compression == null || compression == ClickHouseCompression.NONE) { - chOutput = new WrappedOutputStream(stream, output, bufferSize, postCloseAction); - } else { - // never got brotli, bz2, deflate, gzip, and xz working :< - switch (compression) { - // case BROTLI: - // chOutput = new WrappedOutputStream(stream, - // CompressionUtils.createBrotliOutputStream(output, compressionLevel, - // bufferSize), bufferSize, postCloseAction); - // break; - // case BZ2: - // chOutput = new WrappedOutputStream(stream, - // CompressionUtils.createBz2OutputStream(output, compressionLevel), bufferSize, - // postCloseAction); - // break; - // case DEFLATE: - // chOutput = new WrappedOutputStream(stream, new InflaterOutputStream(output), - // bufferSize, - // postCloseAction); - // break; - // case GZIP: - // try { - // GzipParameters params = new GzipParameters(); - // params.setBufferSize(bufferSize); - // params.setCompressionLevel(3); - // chOutput = new WrappedOutputStream(stream, new - // GzipCompressorOutputStream(output, params), - // bufferSize, postCloseAction); - // } catch (IOException e) { - // throw new - // IllegalArgumentException(CompressionUtils.ERROR_FAILED_TO_WRAP_OUTPUT, e); - // } - // break; - case LZ4: - chOutput = new Lz4OutputStream(stream, output, compressionLevel, bufferSize, postCloseAction); - break; - case SNAPPY: - chOutput = new WrappedOutputStream(stream, - CompressionUtils.createSnappyOutputStream(output, compressionLevel), bufferSize, - postCloseAction); - break; - case ZSTD: - chOutput = new WrappedOutputStream(stream, - CompressionUtils.createZstdOutputStream(output, compressionLevel), bufferSize, - postCloseAction); - break; - // case XZ: - // chOutput = new WrappedOutputStream(stream, - // CompressionUtils.createXzOutputStream(output, 6), bufferSize, - // postCloseAction); - // break; - default: - throw new UnsupportedOperationException("Unsupported compression algorithm: " + compression); - } - } - return chOutput; + ClickHouseCompression compression, int compressionLevel, Runnable postCloseAction) { + return ClickHouseCompressionAlgorithm.createOutputStream(stream, output, bufferSize, compression, + compressionLevel, postCloseAction); } /** @@ -110,7 +55,7 @@ public static ClickHouseOutputStream empty() { * * @param deferredOutput non-null deferred output stream * @param bufferSize buffer size which is always greater than zero(usually - * 8192 or larger) + * 4096 or larger) * @param postCloseAction custom action will be performed right after closing * the input stream * @return wrapped output @@ -127,15 +72,15 @@ public static ClickHouseOutputStream of(ClickHouseDeferredValue de * @param postCloseAction custom action will be performed right after closing * the output stream * @param bufferSize buffer size which is always greater than zero(usually - * 8192 or larger) + * 4096 or larger) * @return wrapped output */ public static ClickHouseOutputStream of(ClickHousePassThruStream stream, int bufferSize, Runnable postCloseAction) { - if (stream == null || stream == ClickHousePassThruStream.NULL) { - throw new IllegalArgumentException("Non-null pass-thru stream required"); + if (stream == null || !stream.hasOutput()) { + throw new IllegalArgumentException("Non-null pass-thru stream with output is required"); } - return stream.asOutputStream(bufferSize, postCloseAction); + return stream.newOutputStream(bufferSize, postCloseAction); } /** @@ -146,14 +91,14 @@ public static ClickHouseOutputStream of(ClickHousePassThruStream stream, int buf * {@link ClickHouseOutputStream} */ public static ClickHouseOutputStream of(OutputStream output) { - return of(output, (int) ClickHouseClientOption.BUFFER_SIZE.getDefaultValue(), null, -1, null); + return of(output, DEFAULT_BUFFER_SIZE, null, -1, null); } /** * Wraps the given output stream. * * @param output output stream - * @param bufferSize buffer size which is always greater than zero(usually 8192 + * @param bufferSize buffer size which is always greater than zero(usually 4096 * or larger) * @return wrapped output, or the same output if it's instance of * {@link ClickHouseOutputStream} @@ -167,7 +112,7 @@ public static ClickHouseOutputStream of(OutputStream output, int bufferSize) { * * @param output output stream * @param bufferSize buffer size which is always greater than zero(usually - * 8192 or larger) + * 4096 or larger) * @param compression compression algorithm, null or * {@link ClickHouseCompression#NONE} means no * compression @@ -187,7 +132,7 @@ public static ClickHouseOutputStream of(OutputStream output, int bufferSize, Cli ? (ClickHouseOutputStream) output : new WrappedOutputStream(null, output, bufferSize, postCloseAction); } else { - chOutput = wrap(null, output, bufferSize, postCloseAction, compression, level); + chOutput = wrap(null, output, bufferSize, compression, level, postCloseAction); } return chOutput; } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHousePassThruStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHousePassThruStream.java similarity index 66% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHousePassThruStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHousePassThruStream.java index 38ae3e4c5..40f5d95c4 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHousePassThruStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHousePassThruStream.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.InputStream; import java.io.OutputStream; @@ -11,6 +11,11 @@ public class ClickHousePassThruStream implements Serializable { private static final long serialVersionUID = -879012829388929569L; + public static final String TYPE_NAME = "Pass-thru Stream"; + + public static final String ERROR_NO_INPUT = "Pass-thru stream does not have input"; + public static final String ERROR_NO_OUTPUT = "Pass-thru stream does not have output"; + /** * Null stream which has no compression and format. */ @@ -29,13 +34,11 @@ public static ClickHousePassThruStream of(OutputStream out, ClickHouseCompressio public static ClickHousePassThruStream of(InputStream in, OutputStream out, ClickHouseCompression compression, int compressionLevel, ClickHouseFormat format) { - if (in == null && out == null && (compression == null || compression == ClickHouseCompression.NONE) - && compressionLevel == -1 && format == null) { + if (in == null && out == null && compression == null && compressionLevel == -1 && format == null) { return NULL; } - return new ClickHousePassThruStream(in, out, compression == null ? ClickHouseCompression.NONE : compression, - compressionLevel, format); + return new ClickHousePassThruStream(in, out, compression, compressionLevel, format); } private final transient InputStream input; @@ -54,71 +57,62 @@ protected ClickHousePassThruStream(InputStream in, OutputStream out, ClickHouseC } /** - * Creates an input stream for reading the pass-thru input stream. + * Gets the input stream for reading. * - * @return non-null input stream for reading from the pass-thru input stream + * @return non-null input stream */ - public ClickHouseInputStream asInputStream() { + public ClickHouseInputStream getInputStream() { return input != null ? ClickHouseInputStream.of(input) : ClickHouseInputStream.empty(); } /** - * Creates an input stream for reading the pass-thru input stream. + * Creates a wrapped input stream for reading. This method is supposed to be + * called once and only once. * * @param bufferSize buffer size which is always greater than zero(usually - * 8192 or larger) + * 4096 or larger) * @param postCloseAction custom action will be performed right after closing * the input stream - * @return non-null input stream for reading from the pass-thru input stream + * @return non-null wrapped input stream */ - public ClickHouseInputStream asInputStream(int bufferSize, Runnable postCloseAction) { + public ClickHouseInputStream newInputStream(int bufferSize, Runnable postCloseAction) { return input != null - ? ClickHouseInputStream.wrap(null, input, bufferSize, postCloseAction, ClickHouseCompression.NONE, - compressionLevel) + ? ClickHouseInputStream.wrap(this, input, bufferSize, ClickHouseCompression.NONE, compressionLevel, + postCloseAction) : ClickHouseInputStream.empty(); } /** - * Creates an output stream for writing data into the pass-thru output stream. + * Gets the output stream for writing. * - * @return non-null output stream for writing data into the the pass-thru output - * stream + * @return non-null output stream */ - public ClickHouseOutputStream asOutputStream() { + public ClickHouseOutputStream getOutputStream() { return output != null ? ClickHouseOutputStream.of(output) : ClickHouseOutputStream.empty(); } /** - * Creates an output stream for writing data into the pass-thru output stream. + * Creates a wrapped output stream for writing. This method is supposed to be + * called once and only once. * * @param bufferSize buffer size which is always greater than zero(usually - * 8192 or larger) + * 4096 or larger) * @param postCloseAction custom action will be performed right after closing * the output stream - * @return non-null output stream for writing data into the the pass-thru output - * stream + * @return non-null wrapped output stream */ - public ClickHouseOutputStream asOutputStream(int bufferSize, Runnable postCloseAction) { + public ClickHouseOutputStream newOutputStream(int bufferSize, Runnable postCloseAction) { return output != null - ? ClickHouseOutputStream.wrap(null, output, bufferSize, postCloseAction, ClickHouseCompression.NONE, - compressionLevel) + ? ClickHouseOutputStream.wrap(this, output, bufferSize, ClickHouseCompression.NONE, + compressionLevel, postCloseAction) : ClickHouseOutputStream.empty(); } /** - * Gets data format, which could be null. Use {@link #hasFormat()} to check - * first. - * - * @return data format, could be null - */ - public ClickHouseFormat getFormat() { - return format; - } - - /** - * Gets compression algorithm. + * Gets compression algorithm, which could be null. Use + * {@link #hasCompression()} to check first. * - * @return non-null compression algorithm + * @return compression algorithm, could be null */ public ClickHouseCompression getCompressionAlgorithm() { return compression; @@ -133,6 +127,25 @@ public int getCompressionLevel() { return compressionLevel; } + /** + * Gets data format, which could be null. Use {@link #hasFormat()} to check + * first. + * + * @return data format, could be null + */ + public ClickHouseFormat getFormat() { + return format; + } + + /** + * Checks if the compression algorithm is defined or not. + * + * @return true if the compression algorithm is defined; false otherwise + */ + public boolean hasCompression() { + return compression != null; + } + /** * Checks if the data format is defined or not. * @@ -166,6 +179,6 @@ public boolean hasOutput() { * @return true if the data is compressed; false otherwise */ public boolean isCompressed() { - return compression != ClickHouseCompression.NONE; + return hasCompression() && compression != ClickHouseCompression.NONE; } } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHousePipedOutputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHousePipedOutputStream.java similarity index 96% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHousePipedOutputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHousePipedOutputStream.java index cfb6abc08..a86d14a0d 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHousePipedOutputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHousePipedOutputStream.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; /** * SPSC(Single-producer single-consumer) channel for streaming. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRecord.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseRecord.java similarity index 93% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRecord.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseRecord.java index 8ad74d884..83911280d 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRecord.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseRecord.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.Serializable; import java.util.Iterator; @@ -45,7 +45,7 @@ public int size() { /** * Gets deserialized value wrapped in an object using column index. Please avoid * to cache the wrapper object, as it's reused among records for memory - * efficiency when {@link ClickHouseConfig#isReuseValueWrapper()} returns + * efficiency when {@link ClickHouseDataConfig#isReuseValueWrapper()} returns * {@code true}, which is the default value. So instead of * {@code map.put("my_value", record.getValue(0))}, try something like * {@code map.put("my_value", record.getValue(0).asString())}. @@ -59,7 +59,7 @@ public int size() { * Gets deserialized value wrapped in an object using case-insensitive column * name, which usually is slower than {@link #getValue(int)}. Please avoid to * cache the wrapper object, as it's reused among records for memory efficiency - * when {@link ClickHouseConfig#isReuseValueWrapper()} returns {@code true}, + * when {@link ClickHouseDataConfig#isReuseValueWrapper()} returns {@code true}, * which is the default value. So instead of * {@code map.put("my_value", record.getValue("my_column"))}, try something like * {@code map.put("my_value", record.getValue("my_column").asString())}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseRecordTransformer.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseRecordTransformer.java similarity index 76% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseRecordTransformer.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseRecordTransformer.java index 88d258ce8..fd17c7d29 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseRecordTransformer.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseRecordTransformer.java @@ -1,6 +1,4 @@ -package com.clickhouse.client.data; - -import com.clickhouse.client.ClickHouseRecord; +package com.clickhouse.data; @FunctionalInterface public interface ClickHouseRecordTransformer { diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseSerializer.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseSerializer.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseSerializer.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseSerializer.java index 1c71ba1de..90db010ce 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseSerializer.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseSerializer.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.IOException; import java.util.Iterator; diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseStreamConfig.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseStreamConfig.java new file mode 100644 index 000000000..c2169dc4c --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseStreamConfig.java @@ -0,0 +1,5 @@ +package com.clickhouse.data; + +public interface ClickHouseStreamConfig { + +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseThreadFactory.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseThreadFactory.java similarity index 94% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseThreadFactory.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseThreadFactory.java index b75d553c5..e92477660 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseThreadFactory.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseThreadFactory.java @@ -1,11 +1,11 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.lang.Thread.UncaughtExceptionHandler; import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; public class ClickHouseThreadFactory implements ThreadFactory { private static final Logger log = LoggerFactory.getLogger(ClickHouseThreadFactory.class); diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseUtils.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseUtils.java similarity index 93% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseUtils.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseUtils.java index 92b5d1f71..7a3acc65a 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseUtils.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseUtils.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.File; import java.io.FileInputStream; @@ -62,21 +62,11 @@ public final class ClickHouseUtils { : Paths.get(System.getProperty("user.home"), ".clickhouse").toFile().getAbsolutePath(); } - /** - * Default buffer size. - */ - public static final int DEFAULT_BUFFER_SIZE = 8192; - /** * Default charset. */ public static final String DEFAULT_CHARSET = StandardCharsets.UTF_8.name(); - /** - * Maximum buffer size. - */ - public static final int MAX_BUFFER_SIZE = Integer.MAX_VALUE - 8; - public static final String VARIABLE_PREFIX = "{{"; public static final String VARIABLE_SUFFIX = "}}"; @@ -126,9 +116,19 @@ public static String applyVariables(String template, Map variabl return applyVariables(template, variables == null || variables.isEmpty() ? null : variables::get); } + /** + * Creates a temporary file. Same as {@code createTempFile(null, null, true)}. + * + * @return non-null temporary file + * @throws IOException when failed to create the temporary file + */ + public static File createTempFile() throws IOException { + return createTempFile(null, null, true); + } + /** * Creates a temporary file with given prefix and suffix. Same as - * {@code createTempFile(prefix, suffix, false)}. + * {@code createTempFile(prefix, suffix, true)}. * * @param prefix prefix, could be null * @param suffix suffix, could be null @@ -136,7 +136,7 @@ public static String applyVariables(String template, Map variabl * @throws IOException when failed to create the temporary file */ public static File createTempFile(String prefix, String suffix) throws IOException { - return createTempFile(prefix, suffix, false); + return createTempFile(prefix, suffix, true); } /** @@ -152,13 +152,12 @@ public static File createTempFile(String prefix, String suffix, boolean deleteOn final File f; if (IS_UNIX) { FileAttribute> attr = PosixFilePermissions - .asFileAttribute(PosixFilePermissions.fromString("rwx------")); + .asFileAttribute(PosixFilePermissions.fromString("rw-------")); f = Files.createTempFile(prefix, suffix, attr).toFile(); } else { - f = Files.createTempFile(prefix, suffix).toFile(); + f = Files.createTempFile(prefix == null ? "chc" : prefix, suffix == null ? ".data" : suffix).toFile(); f.setReadable(true, true); // NOSONAR f.setWritable(true, true); // NOSONAR - f.setExecutable(true, true); // NOSONAR } if (deleteOnExit) { @@ -651,34 +650,6 @@ public static Object parseJson(String json) { return value; } - /** - * Gets buffer size. - * - * @param bufferSize suggested buffer size, zero or negative number is treated - * as {@code defaultSize} - * @param defaultSize default buffer size, zero or negative number is treated as - * {@link #DEFAULT_BUFFER_SIZE} - * @param maxSize maximum buffer size, zero or negative number is treated as - * {@link #MAX_BUFFER_SIZE} - * @return buffer size - */ - public static int getBufferSize(int bufferSize, int defaultSize, int maxSize) { - if (maxSize < 1 || maxSize > MAX_BUFFER_SIZE) { - maxSize = MAX_BUFFER_SIZE; - } - if (defaultSize < 1) { - defaultSize = DEFAULT_BUFFER_SIZE; - } else if (defaultSize > maxSize) { - defaultSize = maxSize; - } - - if (bufferSize < 1) { - return defaultSize; - } - - return bufferSize > maxSize ? maxSize : bufferSize; - } - public static char getCloseBracket(char openBracket) { char closeBracket; if (openBracket == '(') { @@ -815,61 +786,6 @@ public static OutputStream getFileOutputStream(String file) throws IOException { return new FileOutputStream(file, false); } - /** - * Extracts key value pairs from the given string. - * - * @param str string - * @return non-null map containing extracted key value pairs - * @deprecated will be removed in v0.3.3, please use - * {@link com.clickhouse.client.config.ClickHouseOption#toKeyValuePairs(String)} - * instead - */ - @Deprecated - public static Map getKeyValuePairs(String str) { - if (str == null || str.isEmpty()) { - return Collections.emptyMap(); - } - - Map map = new LinkedHashMap<>(); - String key = null; - StringBuilder builder = new StringBuilder(); - for (int i = 0, len = str.length(); i < len; i++) { - char ch = str.charAt(i); - if (ch == '\\' && i + 1 < len) { - ch = str.charAt(++i); - builder.append(ch); - continue; - } - - if (Character.isWhitespace(ch)) { - if (builder.length() > 0) { - builder.append(ch); - } - } else if (ch == '=' && key == null) { - key = builder.toString().trim(); - builder.setLength(0); - } else if (ch == ',' && key != null) { - String value = builder.toString().trim(); - builder.setLength(0); - if (!key.isEmpty() && !value.isEmpty()) { - map.put(key, value); - } - key = null; - } else { - builder.append(ch); - } - } - - if (key != null && builder.length() > 0) { - String value = builder.toString().trim(); - if (!key.isEmpty() && !value.isEmpty()) { - map.put(key, value); - } - } - - return Collections.unmodifiableMap(map); - } - public static String getLeadingComment(String sql) { if (sql == null || sql.isEmpty()) { return ""; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseValue.java similarity index 99% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseValue.java index 95be3549c..5ba49c12d 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.BufferedReader; import java.io.ByteArrayInputStream; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseValues.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseValues.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseValues.java rename to clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseValues.java index 412d8c56c..bd1438a57 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseValues.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseValues.java @@ -1,10 +1,9 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.IOException; import java.lang.reflect.Array; import java.math.BigDecimal; import java.math.BigInteger; -import java.math.RoundingMode; import java.net.Inet4Address; import java.net.Inet6Address; import java.net.InetAddress; @@ -29,9 +28,8 @@ import java.util.UUID; import java.util.Map.Entry; -import com.clickhouse.client.config.ClickHouseDefaults; -import com.clickhouse.client.data.ClickHouseArrayValue; -import com.clickhouse.client.data.ClickHouseByteValue; +import com.clickhouse.data.value.ClickHouseArrayValue; +import com.clickhouse.data.value.ClickHouseByteValue; /** * Help class for dealing with values. @@ -60,8 +58,6 @@ public final class ClickHouseValues { public static final String EMPTY_STRING_EXPR = "''"; public static final BigDecimal NANOS = new BigDecimal(BigInteger.TEN.pow(9)); - public static final RoundingMode ROUNDING_MODE = (RoundingMode) ClickHouseDefaults.ROUNDING_MODE - .getEffectiveDefaultValue(); public static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); public static final DateTimeFormatter TIME_FORMATTER = new DateTimeFormatterBuilder().appendPattern("HH:mm:ss") @@ -877,7 +873,7 @@ public static Object createPrimitiveArray(Class clazz, int length, int level) return Array.newInstance(ClickHouseDataType.toPrimitiveType(clazz), dimensions); } - static ClickHouseArrayValue fillByteArray(ClickHouseArrayValue array, ClickHouseConfig config, + static ClickHouseArrayValue fillByteArray(ClickHouseArrayValue array, ClickHouseDataConfig config, ClickHouseColumn column, ClickHouseInputStream input, ClickHouseDeserializer deserializer, int length) throws IOException { byte[] values = new byte[length]; @@ -948,7 +944,7 @@ public static Object extractSingleValue(Map value) { * @param columns non-null columns * @return non-null values with default value, either null or empty */ - public static ClickHouseValue[] newValues(ClickHouseConfig config, List columns) { + public static ClickHouseValue[] newValues(ClickHouseDataConfig config, List columns) { if (columns == null || columns.isEmpty()) { return EMPTY_VALUES; } @@ -968,7 +964,7 @@ public static ClickHouseValue[] newValues(ClickHouseConfig config, List { + try (ClickHouseOutputStream out = output) { + writer.write(output); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return null; + }); + } else { + try (ClickHouseOutputStream out = output) { + writer.write(output); + } + } + } + + /** + * Writes value to output stream. + * + * @param output non-null output stream + * @throws IOException when failed to write data to output stream + */ + void write(ClickHouseOutputStream output) throws IOException; +} diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/UnsafeByteUtils.java b/clickhouse-data/src/main/java/com/clickhouse/data/UnsafeByteUtils.java new file mode 100644 index 000000000..244ed0dff --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/UnsafeByteUtils.java @@ -0,0 +1,107 @@ +package com.clickhouse.data; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteOrder; + +import sun.misc.Unsafe; + +@SuppressWarnings("squid:S3011") +public final class UnsafeByteUtils { + public static final ByteOrder NATIVE_BYTE_ORDER = ByteOrder.nativeOrder(); + + private static final Unsafe UNSAFE; + + private static final long BYTE_ARRAY_OFFSET = Unsafe.ARRAY_BYTE_BASE_OFFSET; + private static final long LONG_ARRAY_OFFSET = Unsafe.ARRAY_LONG_BASE_OFFSET; + + static { + Unsafe instance; + try { + Field f = Unsafe.class.getDeclaredField("theUnsafe"); + f.setAccessible(true); + instance = (Unsafe) f.get(null); + } catch (NoSuchFieldException e) { + try { + Constructor c = Unsafe.class.getDeclaredConstructor(); + c.setAccessible(true); + instance = c.newInstance(); + } catch (RuntimeException | NoSuchMethodException | InstantiationException | IllegalAccessException + | InvocationTargetException ex) { + throw new ExceptionInInitializerError(ex); + } + } catch (Exception e) { + throw new ExceptionInInitializerError("Cannot access Unsafe"); + } + UNSAFE = instance; + } + + public static byte getByte(byte[] bytes, int offset) { + return UNSAFE.getByte(bytes, BYTE_ARRAY_OFFSET + offset); + } + + public static int getInteger(byte[] bytes, int offset) { + int v = UNSAFE.getInt(bytes, BYTE_ARRAY_OFFSET + offset); + if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) { + v = Integer.reverseBytes(v); + } + return v; + } + + public static long getLong(byte[] bytes, int offset) { + if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) { + return (0xFFL & getByte(bytes, offset)) | ((0xFFL & getByte(bytes, offset + + 1)) << 8) + | ((0xFFL & getByte(bytes, offset + 2)) << 16) + | ((0xFFL & getByte(bytes, offset + 3) << 24) | ((0xFFL & getByte(bytes, + offset + 4)) << 32) + | ((0xFFL & getByte(bytes, offset + 5)) << 40) | ((0xFFL & getByte(bytes, + offset + 6)) << 48) + | ((0xFFL & getByte(bytes, offset + 7)) << 56)); + } + return UNSAFE.getLong(bytes, BYTE_ARRAY_OFFSET + offset); + } + + public static void putLong(long[] values, int offset, long value) { + UNSAFE.putLong(values, Unsafe.ARRAY_LONG_BASE_OFFSET + Unsafe.ARRAY_LONG_INDEX_SCALE * offset, value); + } + + public static short getShort(byte[] bytes, int offset) { + short v = UNSAFE.getShort(bytes, BYTE_ARRAY_OFFSET + offset); + if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) { + v = Short.reverseBytes(v); + } + return v; + } + + public static void setByte(byte[] bytes, int offset, byte value) { + UNSAFE.putByte(bytes, BYTE_ARRAY_OFFSET + Unsafe.ARRAY_BYTE_INDEX_SCALE * offset, value); + } + + public static void setInteger(byte[] bytes, int offset, int value) { + setByte(bytes, offset, (byte) value); + setByte(bytes, offset + 1, (byte) (value >>> 8)); + setByte(bytes, offset + 2, (byte) (value >>> 16)); + setByte(bytes, offset + 3, (byte) (value >>> 24)); + } + + public static void setLong(byte[] bytes, int offset, long value) { + setByte(bytes, offset, (byte) value); + setByte(bytes, offset + 1, (byte) (value >>> 8)); + setByte(bytes, offset + 2, (byte) (value >>> 16)); + setByte(bytes, offset + 3, (byte) (value >>> 24)); + setByte(bytes, offset + 4, (byte) (value >>> 32)); + setByte(bytes, offset + 5, (byte) (value >>> 40)); + setByte(bytes, offset + 6, (byte) (value >>> 48)); + setByte(bytes, offset + 7, (byte) (value >>> 56)); + } + + public static void setShort(byte[] bytes, int offset, short value) { + setByte(bytes, offset, (byte) value); + setByte(bytes, offset + 1, (byte) (value >>> 8)); + } + + private UnsafeByteUtils() { + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/cache/CaffeineCache.java b/clickhouse-data/src/main/java/com/clickhouse/data/cache/CaffeineCache.java similarity index 95% rename from clickhouse-client/src/main/java/com/clickhouse/client/cache/CaffeineCache.java rename to clickhouse-data/src/main/java/com/clickhouse/data/cache/CaffeineCache.java index 963bb52f2..d71f7ebb8 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/cache/CaffeineCache.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/cache/CaffeineCache.java @@ -1,10 +1,10 @@ -package com.clickhouse.client.cache; +package com.clickhouse.data.cache; import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.function.Function; -import com.clickhouse.client.ClickHouseCache; +import com.clickhouse.data.ClickHouseCache; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/cache/JdkLruCache.java b/clickhouse-data/src/main/java/com/clickhouse/data/cache/JdkLruCache.java similarity index 95% rename from clickhouse-client/src/main/java/com/clickhouse/client/cache/JdkLruCache.java rename to clickhouse-data/src/main/java/com/clickhouse/data/cache/JdkLruCache.java index a5b28d262..bf93073f9 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/cache/JdkLruCache.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/cache/JdkLruCache.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.cache; +package com.clickhouse.data.cache; import java.util.Collections; import java.util.LinkedHashMap; @@ -6,7 +6,7 @@ import java.util.Objects; import java.util.function.Function; -import com.clickhouse.client.ClickHouseCache; +import com.clickhouse.data.ClickHouseCache; /** * A simple thread-safe LRU cache based on LinkedHashMap. It's not as effient as diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/compress/BrotliSupport.java b/clickhouse-data/src/main/java/com/clickhouse/data/compress/BrotliSupport.java new file mode 100644 index 000000000..e5871bdc9 --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/compress/BrotliSupport.java @@ -0,0 +1,76 @@ +package com.clickhouse.data.compress; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseCompressionAlgorithm; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePassThruStream; +import com.clickhouse.data.stream.WrappedInputStream; +import com.clickhouse.data.stream.WrappedOutputStream; + +public final class BrotliSupport { + public static class DefaultImpl implements ClickHouseCompressionAlgorithm { + @Override + public ClickHouseInputStream decompress(ClickHousePassThruStream stream, InputStream input, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new WrappedInputStream(stream, new org.brotli.dec.BrotliInputStream(input, bufferSize), bufferSize, + postCloseAction); + } + + @Override + public ClickHouseCompression getAlgorithm() { + return ClickHouseCompression.BROTLI; + } + } + + public static class PreferredImpl implements ClickHouseCompressionAlgorithm { + public PreferredImpl() { + com.aayushatharva.brotli4j.Brotli4jLoader.ensureAvailability(); + } + + @Override + public ClickHouseOutputStream compress(ClickHousePassThruStream stream, OutputStream output, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new WrappedOutputStream(stream, new com.aayushatharva.brotli4j.encoder.BrotliOutputStream(output, + new com.aayushatharva.brotli4j.encoder.Encoder.Parameters() + .setQuality(normalize(level)), + bufferSize), bufferSize, postCloseAction); + } + + @Override + public ClickHouseInputStream decompress(ClickHousePassThruStream stream, InputStream input, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new WrappedInputStream(stream, + new com.aayushatharva.brotli4j.decoder.BrotliInputStream(input, bufferSize), bufferSize, + postCloseAction); + } + + @Override + public ClickHouseCompression getAlgorithm() { + return ClickHouseCompression.BROTLI; + } + } + + static final class Factory { + private static final ClickHouseCompressionAlgorithm instance = ClickHouseCompressionAlgorithm + .createInstance("brotli_lib_detection", PreferredImpl.class, DefaultImpl.class); + + private Factory() { + } + } + + private static int normalize(int level) { + return level < -1 || level > 11 ? 4 : level; + } + + public static ClickHouseCompressionAlgorithm getInstance() { + return Factory.instance; + } + + private BrotliSupport() { + } +} diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/compress/Bz2Support.java b/clickhouse-data/src/main/java/com/clickhouse/data/compress/Bz2Support.java new file mode 100644 index 000000000..170b1d8ce --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/compress/Bz2Support.java @@ -0,0 +1,60 @@ +package com.clickhouse.data.compress; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseCompressionAlgorithm; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePassThruStream; +import com.clickhouse.data.stream.WrappedInputStream; +import com.clickhouse.data.stream.WrappedOutputStream; + +public final class Bz2Support { + public static class DefaultImpl implements ClickHouseCompressionAlgorithm { + protected int normalize(int level) { + return level < org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream.MIN_BLOCKSIZE + || level > org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream.MAX_BLOCKSIZE + ? org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream.MAX_BLOCKSIZE + : level; + } + + @Override + public ClickHouseOutputStream compress(ClickHousePassThruStream stream, OutputStream output, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new WrappedOutputStream(stream, + new org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream(output, + normalize(level)), + bufferSize, postCloseAction); + } + + @Override + public ClickHouseInputStream decompress(ClickHousePassThruStream stream, InputStream input, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new WrappedInputStream(stream, + new org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream(input), bufferSize, + postCloseAction); + } + + @Override + public ClickHouseCompression getAlgorithm() { + return ClickHouseCompression.BZ2; + } + } + + static final class Factory { + private static final ClickHouseCompressionAlgorithm instance = new DefaultImpl(); + + private Factory() { + } + } + + public static ClickHouseCompressionAlgorithm getInstance() { + return Factory.instance; + } + + private Bz2Support() { + } +} diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/compress/DeflateSupport.java b/clickhouse-data/src/main/java/com/clickhouse/data/compress/DeflateSupport.java new file mode 100644 index 000000000..d6357e280 --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/compress/DeflateSupport.java @@ -0,0 +1,89 @@ +package com.clickhouse.data.compress; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.InflaterInputStream; + +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseCompressionAlgorithm; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePassThruStream; +import com.clickhouse.data.stream.WrappedInputStream; +import com.clickhouse.data.stream.WrappedOutputStream; + +public final class DeflateSupport { + public static class DefaultImpl implements ClickHouseCompressionAlgorithm { + @Override + public ClickHouseOutputStream compress(ClickHousePassThruStream stream, OutputStream output, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new WrappedOutputStream(stream, new DeflaterOutputStream(output), bufferSize, postCloseAction); + } + + @Override + public ClickHouseInputStream decompress(ClickHousePassThruStream stream, InputStream input, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new WrappedInputStream(stream, new InflaterInputStream(input), bufferSize, postCloseAction); + } + + @Override + public ClickHouseCompression getAlgorithm() { + return ClickHouseCompression.DEFLATE; + } + } + + public static class PreferredImpl implements ClickHouseCompressionAlgorithm { + public PreferredImpl() { + new org.apache.commons.compress.compressors.deflate.DeflateParameters(); + } + + @Override + public ClickHouseOutputStream compress(ClickHousePassThruStream stream, OutputStream output, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + org.apache.commons.compress.compressors.deflate.DeflateParameters params = new org.apache.commons.compress.compressors.deflate.DeflateParameters(); + params.setWithZlibHeader(true); + params.setCompressionLevel(normalize(level)); + return new WrappedOutputStream(stream, + new org.apache.commons.compress.compressors.deflate.DeflateCompressorOutputStream(output, params), + bufferSize, postCloseAction); + } + + @Override + public ClickHouseInputStream decompress(ClickHousePassThruStream stream, InputStream input, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + org.apache.commons.compress.compressors.deflate.DeflateParameters params = new org.apache.commons.compress.compressors.deflate.DeflateParameters(); + params.setWithZlibHeader(true); + params.setCompressionLevel(normalize(level)); + return new WrappedInputStream(stream, + new org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream(input, params), + bufferSize, postCloseAction); + } + + @Override + public ClickHouseCompression getAlgorithm() { + return ClickHouseCompression.DEFLATE; + } + } + + static final class Factory { + private static final ClickHouseCompressionAlgorithm instance = ClickHouseCompressionAlgorithm + .createInstance("deflate_lib_detection", PreferredImpl.class, DefaultImpl.class); + + private Factory() { + } + } + + private static int normalize(int level) { + return level < Deflater.NO_COMPRESSION || level > Deflater.BEST_COMPRESSION ? Deflater.BEST_SPEED : level; + } + + public static ClickHouseCompressionAlgorithm getInstance() { + return Factory.instance; + } + + private DeflateSupport() { + } +} diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/compress/GzipSupport.java b/clickhouse-data/src/main/java/com/clickhouse/data/compress/GzipSupport.java new file mode 100644 index 000000000..e0e9acbb5 --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/compress/GzipSupport.java @@ -0,0 +1,86 @@ +package com.clickhouse.data.compress; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.zip.Deflater; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseCompressionAlgorithm; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePassThruStream; +import com.clickhouse.data.stream.WrappedInputStream; +import com.clickhouse.data.stream.WrappedOutputStream; + +public final class GzipSupport { + public static class DefaultImpl implements ClickHouseCompressionAlgorithm { + @Override + public ClickHouseOutputStream compress(ClickHousePassThruStream stream, OutputStream output, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new WrappedOutputStream(stream, new GZIPOutputStream(output), bufferSize, postCloseAction); + } + + @Override + public ClickHouseInputStream decompress(ClickHousePassThruStream stream, InputStream input, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new WrappedInputStream(stream, new GZIPInputStream(input), bufferSize, postCloseAction); + } + + @Override + public ClickHouseCompression getAlgorithm() { + return ClickHouseCompression.GZIP; + } + } + + public static class PreferredImpl implements ClickHouseCompressionAlgorithm { + public PreferredImpl() { + new org.apache.commons.compress.compressors.gzip.GzipParameters(); + } + + @Override + public ClickHouseOutputStream compress(ClickHousePassThruStream stream, OutputStream output, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + org.apache.commons.compress.compressors.gzip.GzipParameters params = new org.apache.commons.compress.compressors.gzip.GzipParameters(); + params.setBufferSize(bufferSize); + params.setCompressionLevel(normalize(level)); + return new WrappedOutputStream(stream, + new org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream(output, params), + bufferSize, postCloseAction); + } + + @Override + public ClickHouseInputStream decompress(ClickHousePassThruStream stream, InputStream input, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new WrappedInputStream(stream, + new org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream(input, true), + bufferSize, postCloseAction); + } + + @Override + public ClickHouseCompression getAlgorithm() { + return ClickHouseCompression.GZIP; + } + } + + static final class Factory { + private static final ClickHouseCompressionAlgorithm instance = ClickHouseCompressionAlgorithm + .createInstance("gzip_lib_detection", PreferredImpl.class, DefaultImpl.class); + + private Factory() { + } + } + + private static int normalize(int level) { + return level < -1 || level > 9 ? Deflater.DEFAULT_COMPRESSION : level; + } + + public static ClickHouseCompressionAlgorithm getInstance() { + return Factory.instance; + } + + private GzipSupport() { + } +} diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/compress/Lz4Support.java b/clickhouse-data/src/main/java/com/clickhouse/data/compress/Lz4Support.java new file mode 100644 index 000000000..b9015e2fc --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/compress/Lz4Support.java @@ -0,0 +1,52 @@ +package com.clickhouse.data.compress; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseCompressionAlgorithm; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePassThruStream; +import com.clickhouse.data.stream.Lz4InputStream; +import com.clickhouse.data.stream.Lz4OutputStream; + +public final class Lz4Support { + public static class DefaultImpl implements ClickHouseCompressionAlgorithm { + protected int normalize(int level) { + return level < 0 ? -1 : level; + } + + @Override + public ClickHouseOutputStream compress(ClickHousePassThruStream stream, OutputStream output, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new Lz4OutputStream(output, normalize(level), bufferSize, null); + } + + @Override + public ClickHouseInputStream decompress(ClickHousePassThruStream stream, InputStream input, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new Lz4InputStream(input); + } + + @Override + public ClickHouseCompression getAlgorithm() { + return ClickHouseCompression.LZ4; + } + } + + static final class Factory { + private static final ClickHouseCompressionAlgorithm instance = new DefaultImpl(); + + private Factory() { + } + } + + public static ClickHouseCompressionAlgorithm getInstance() { + return Factory.instance; + } + + private Lz4Support() { + } +} diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/compress/NoneSupport.java b/clickhouse-data/src/main/java/com/clickhouse/data/compress/NoneSupport.java new file mode 100644 index 000000000..a96a0a4f3 --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/compress/NoneSupport.java @@ -0,0 +1,54 @@ +package com.clickhouse.data.compress; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseCompressionAlgorithm; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePassThruStream; +import com.clickhouse.data.stream.EmptyInputStream; +import com.clickhouse.data.stream.EmptyOutputStream; +import com.clickhouse.data.stream.WrappedInputStream; +import com.clickhouse.data.stream.WrappedOutputStream; + +public final class NoneSupport { + public static class DefaultImpl implements ClickHouseCompressionAlgorithm { + @Override + public ClickHouseOutputStream compress(ClickHousePassThruStream stream, OutputStream output, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return output != EmptyOutputStream.INSTANCE && output instanceof ClickHouseOutputStream + ? (ClickHouseOutputStream) output + : new WrappedOutputStream(stream, output, bufferSize, postCloseAction); + } + + @Override + public ClickHouseInputStream decompress(ClickHousePassThruStream stream, InputStream input, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return input != EmptyInputStream.INSTANCE && input instanceof ClickHouseInputStream + ? (ClickHouseInputStream) input + : new WrappedInputStream(stream, input, bufferSize, postCloseAction); + } + + @Override + public ClickHouseCompression getAlgorithm() { + return ClickHouseCompression.NONE; + } + } + + static final class Factory { + private static final ClickHouseCompressionAlgorithm instance = new DefaultImpl(); + + private Factory() { + } + } + + public static ClickHouseCompressionAlgorithm getInstance() { + return Factory.instance; + } + + private NoneSupport() { + } +} diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/compress/SnappySupport.java b/clickhouse-data/src/main/java/com/clickhouse/data/compress/SnappySupport.java new file mode 100644 index 000000000..9f417d671 --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/compress/SnappySupport.java @@ -0,0 +1,87 @@ +package com.clickhouse.data.compress; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseCompressionAlgorithm; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePassThruStream; +import com.clickhouse.data.stream.WrappedInputStream; +import com.clickhouse.data.stream.WrappedOutputStream; + +public final class SnappySupport { + public static class DefaultImpl implements ClickHouseCompressionAlgorithm { + @Override + public ClickHouseOutputStream compress(ClickHousePassThruStream stream, OutputStream output, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new WrappedOutputStream(stream, + new org.apache.commons.compress.compressors.snappy.SnappyCompressorOutputStream(output, bufferSize), + bufferSize, postCloseAction); + } + + @Override + public ClickHouseInputStream decompress(ClickHousePassThruStream stream, InputStream input, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new WrappedInputStream(stream, + new org.apache.commons.compress.compressors.snappy.SnappyCompressorInputStream(input, + normalize(level)), + bufferSize, postCloseAction); + } + + @Override + public ClickHouseCompression getAlgorithm() { + return ClickHouseCompression.SNAPPY; + } + } + + public static class PreferredImpl implements ClickHouseCompressionAlgorithm { + public PreferredImpl() { + new org.xerial.snappy.Snappy(); // NOSONAR + } + + @Override + public ClickHouseOutputStream compress(ClickHousePassThruStream stream, OutputStream output, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new WrappedOutputStream(stream, new org.xerial.snappy.SnappyOutputStream(output, normalize(level)), + bufferSize, postCloseAction); + } + + @Override + public ClickHouseInputStream decompress(ClickHousePassThruStream stream, InputStream input, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + // https://github.com/ClickHouse/ClickHouse/issues/44885 + return new WrappedInputStream(stream, new org.xerial.snappy.SnappyInputStream(input), bufferSize, + postCloseAction); + } + + @Override + public ClickHouseCompression getAlgorithm() { + return ClickHouseCompression.SNAPPY; + } + } + + static final class Factory { + private static final ClickHouseCompressionAlgorithm instance = ClickHouseCompressionAlgorithm + .createInstance("snappy_lib_detection", PreferredImpl.class, DefaultImpl.class); + + private Factory() { + } + } + + static final int MIN_BLOCK_SIZE = 1024; + static final int DEFAULT_BLOCK_SIZE = 32 * 1024; + + private static int normalize(int level) { + return level < MIN_BLOCK_SIZE ? DEFAULT_BLOCK_SIZE : (level - level % 1024); + } + + public static ClickHouseCompressionAlgorithm getInstance() { + return Factory.instance; + } + + private SnappySupport() { + } +} diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/compress/XzSupport.java b/clickhouse-data/src/main/java/com/clickhouse/data/compress/XzSupport.java new file mode 100644 index 000000000..1429bd5ea --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/compress/XzSupport.java @@ -0,0 +1,55 @@ +package com.clickhouse.data.compress; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseCompressionAlgorithm; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePassThruStream; +import com.clickhouse.data.stream.WrappedInputStream; +import com.clickhouse.data.stream.WrappedOutputStream; + +public final class XzSupport { + public static class DefaultImpl implements ClickHouseCompressionAlgorithm { + protected int normalize(int level) { + return level < 0 || level > 9 ? 6 : level; + } + + @Override + public ClickHouseOutputStream compress(ClickHousePassThruStream stream, OutputStream output, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new WrappedOutputStream(stream, + new org.tukaani.xz.XZOutputStream(output, new org.tukaani.xz.LZMA2Options(normalize(level)), + org.tukaani.xz.XZ.CHECK_CRC64), + bufferSize, postCloseAction); + } + + @Override + public ClickHouseInputStream decompress(ClickHousePassThruStream stream, InputStream input, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new WrappedInputStream(stream, new org.tukaani.xz.XZInputStream(input), bufferSize, postCloseAction); + } + + @Override + public ClickHouseCompression getAlgorithm() { + return ClickHouseCompression.XZ; + } + } + + static final class Factory { + private static final ClickHouseCompressionAlgorithm instance = new DefaultImpl(); + + private Factory() { + } + } + + public static ClickHouseCompressionAlgorithm getInstance() { + return Factory.instance; + } + + private XzSupport() { + } +} diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/compress/ZstdSupport.java b/clickhouse-data/src/main/java/com/clickhouse/data/compress/ZstdSupport.java new file mode 100644 index 000000000..6893e17a9 --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/compress/ZstdSupport.java @@ -0,0 +1,55 @@ +package com.clickhouse.data.compress; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseCompressionAlgorithm; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePassThruStream; +import com.clickhouse.data.stream.WrappedInputStream; +import com.clickhouse.data.stream.WrappedOutputStream; + +public final class ZstdSupport { + static class DefaultImpl implements ClickHouseCompressionAlgorithm { + protected int normalize(int level) { + return level < 0 || level > 22 ? com.github.luben.zstd.Zstd.defaultCompressionLevel() : level; + } + + @Override + public ClickHouseOutputStream compress(ClickHousePassThruStream stream, OutputStream output, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new WrappedOutputStream(stream, new com.github.luben.zstd.ZstdOutputStream(output, normalize(level)), + bufferSize, postCloseAction); + } + + @Override + @SuppressWarnings("resource") + public ClickHouseInputStream decompress(ClickHousePassThruStream stream, InputStream input, int bufferSize, + int level, Runnable postCloseAction) throws IOException { + return new WrappedInputStream(stream, new com.github.luben.zstd.ZstdInputStream(input).setContinuous(true), + bufferSize, postCloseAction); + } + + @Override + public ClickHouseCompression getAlgorithm() { + return ClickHouseCompression.ZSTD; + } + } + + static final class Factory { + private static final ClickHouseCompressionAlgorithm instance = new DefaultImpl(); + + private Factory() { + } + } + + public static ClickHouseCompressionAlgorithm getInstance() { + return Factory.instance; + } + + private ZstdSupport() { + } +} diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/format/ApacheAvroProcessor.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/ApacheAvroProcessor.java new file mode 100644 index 000000000..6a677aa10 --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/format/ApacheAvroProcessor.java @@ -0,0 +1,155 @@ +package com.clickhouse.data.format; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.apache.avro.Schema; +import org.apache.avro.Schema.Field; +import org.apache.avro.Schema.Type; +import org.apache.avro.file.DataFileStream; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.DatumReader; + +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseDataProcessor; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseDeserializer; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseSerializer; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; + +public class ApacheAvroProcessor extends ClickHouseDataProcessor { + static final ClickHouseColumn toColumn(Field field) { + return toColumn(field.name(), field.schema()); + } + + static final ClickHouseColumn toColumn(String name, Schema schema) { + ClickHouseColumn column; + switch (schema.getType()) { + case BOOLEAN: + column = ClickHouseColumn.of(name, ClickHouseDataType.Bool, schema.isNullable()); + break; + case ENUM: + String[] enumTypes = schema.getEnumSymbols().toArray(new String[0]); + for (int i = 0, len = enumTypes.length; i < len; i++) { + enumTypes[i] = new StringBuilder(ClickHouseValues.convertToQuotedString(enumTypes[i])).append('=') + .append(i).toString(); + } + column = ClickHouseColumn.of(name, ClickHouseDataType.Enum16, schema.isNullable(), + false, enumTypes); + break; + case INT: + column = ClickHouseColumn.of(name, ClickHouseDataType.Int32, schema.isNullable()); + break; + case LONG: + column = ClickHouseColumn.of(name, ClickHouseDataType.Int64, schema.isNullable()); + break; + case FLOAT: + column = ClickHouseColumn.of(name, ClickHouseDataType.Float32, schema.isNullable()); + break; + case DOUBLE: + column = ClickHouseColumn.of(name, ClickHouseDataType.Float64, schema.isNullable()); + break; + case FIXED: + column = ClickHouseColumn.of(name, ClickHouseDataType.FixedString, schema.isNullable(), + false, Integer.toString(schema.getFixedSize())); + break; + case BYTES: + case STRING: + column = ClickHouseColumn.of(name, ClickHouseDataType.String, schema.isNullable()); + break; + case ARRAY: + column = ClickHouseColumn.of(name, ClickHouseDataType.Array, false, + toColumn(schema.getFields().get(0))); + break; + case MAP: + column = null; + break; + case RECORD: + List fields = schema.getFields(); + ClickHouseColumn[] nestedCols = new ClickHouseColumn[fields.size()]; + for (Field f : fields) { + nestedCols[0] = toColumn(f.name(), f.schema()); + } + column = ClickHouseColumn.of(name, ClickHouseDataType.Nested, false, nestedCols); + break; + case UNION: + for (Schema s : schema.getTypes()) { + if (s.getType() != Schema.Type.NULL) { + return toColumn(name, s); + } + } + case NULL: + default: + column = ClickHouseColumn.of(name, ClickHouseDataType.Nothing, true); + break; + } + return column; + } + + protected ApacheAvroProcessor(ClickHouseDataConfig config, ClickHouseInputStream input, + ClickHouseOutputStream output, List columns, Map settings) + throws IOException { + super(config, input, output, columns, settings); + } + + @Override + protected ClickHouseRecord createRecord() { + + return null; + } + + @Override + protected void readAndFill(ClickHouseValue value) throws IOException { + // TODO Auto-generated method stub + + } + + @Override + protected List readColumns() throws IOException { + if (input.available() < 1) { + input.close(); + return Collections.emptyList(); + } + + DatumReader datumReader = new GenericDatumReader<>(); + DataFileStream reader = new DataFileStream<>(input, datumReader); + Schema schema = reader.getSchema(); + if (schema.getType() != Type.RECORD) { + throw new IOException("Root element of schema must be RECORD"); + } + List flds = schema.getFields(); + List cols = new ArrayList<>(flds.size()); + for (Field f : flds) { + cols.add(toColumn(f)); + } + return Collections.unmodifiableList(cols); + } + + @Override + public void write(ClickHouseValue value) throws IOException { + // TODO Auto-generated method stub + + } + + @Override + public ClickHouseDeserializer getDeserializer(ClickHouseDataConfig config, ClickHouseColumn column) { + // TODO Auto-generated method stub + return null; + } + + @Override + public ClickHouseSerializer getSerializer(ClickHouseDataConfig config, ClickHouseColumn column) { + // TODO Auto-generated method stub + return null; + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/BinaryDataProcessor.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/BinaryDataProcessor.java similarity index 94% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/BinaryDataProcessor.java rename to clickhouse-data/src/main/java/com/clickhouse/data/format/BinaryDataProcessor.java index 72ec4cff0..0909c581a 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/BinaryDataProcessor.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/format/BinaryDataProcessor.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.format; import java.io.IOException; import java.math.BigDecimal; @@ -11,17 +11,20 @@ import java.util.Map; import java.util.TimeZone; -import com.clickhouse.client.ClickHouseArraySequence; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseDeserializer; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHouseSerializer; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseArraySequence; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseDeserializer; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseSerializer; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.value.ClickHouseIntegerValue; +import com.clickhouse.data.value.ClickHouseLongValue; +import com.clickhouse.data.value.ClickHouseShortValue; public interface BinaryDataProcessor { static class ArrayDeserializer extends ClickHouseDeserializer.CompositeDeserializer { @@ -30,12 +33,12 @@ static class ArrayDeserializer extends ClickHouseDeserializer.CompositeDeseriali private final Class valClass; private final ClickHouseValue valValue; - public ArrayDeserializer(ClickHouseConfig config, ClickHouseColumn column, boolean varLength, + public ArrayDeserializer(ClickHouseDataConfig config, ClickHouseColumn column, boolean varLength, ClickHouseDeserializer... deserializers) { this(config, column, varLength ? -1L : -2L, deserializers); } - public ArrayDeserializer(ClickHouseConfig config, ClickHouseColumn column, long length, + public ArrayDeserializer(ClickHouseDataConfig config, ClickHouseColumn column, long length, ClickHouseDeserializer... deserializers) { super(deserializers); @@ -73,12 +76,12 @@ static class ArraySerializer extends ClickHouseSerializer.CompositeSerializer { private final long length; private final ClickHouseValue valValue; - public ArraySerializer(ClickHouseConfig config, ClickHouseColumn column, boolean varLength, + public ArraySerializer(ClickHouseDataConfig config, ClickHouseColumn column, boolean varLength, ClickHouseSerializer... serializers) { this(config, column, varLength ? -1L : -2L, serializers); } - public ArraySerializer(ClickHouseConfig config, ClickHouseColumn column, long length, + public ArraySerializer(ClickHouseDataConfig config, ClickHouseColumn column, long length, ClickHouseSerializer... serializers) { super(serializers); @@ -138,8 +141,8 @@ public void serialize(ClickHouseValue value, ClickHouseOutputStream output) thro static class DateSerDe implements ClickHouseDeserializer, ClickHouseSerializer { private static final Map cache = new HashMap<>(); - public static final DateSerDe of(ClickHouseConfig config) { - TimeZone tz = ClickHouseChecker.nonNull(config, ClickHouseConfig.TYPE_NAME).getTimeZoneForDate(); + public static final DateSerDe of(ClickHouseDataConfig config) { + TimeZone tz = ClickHouseChecker.nonNull(config, ClickHouseDataConfig.TYPE_NAME).getTimeZoneForDate(); return cache.computeIfAbsent(tz, DateSerDe::new); } @@ -173,8 +176,8 @@ public void serialize(ClickHouseValue value, ClickHouseOutputStream output) thro static class Date32SerDe implements ClickHouseDeserializer, ClickHouseSerializer { private static final Map cache = new HashMap<>(); - public static final Date32SerDe of(ClickHouseConfig config) { - TimeZone tz = ClickHouseChecker.nonNull(config, ClickHouseConfig.TYPE_NAME).getTimeZoneForDate(); + public static final Date32SerDe of(ClickHouseDataConfig config) { + TimeZone tz = ClickHouseChecker.nonNull(config, ClickHouseDataConfig.TYPE_NAME).getTimeZoneForDate(); return cache.computeIfAbsent(tz, Date32SerDe::new); } @@ -209,10 +212,10 @@ public void serialize(ClickHouseValue value, ClickHouseOutputStream output) thro static class DateTime32SerDe implements ClickHouseDeserializer, ClickHouseSerializer { private static final Map cache = new HashMap<>(); - public static final DateTime32SerDe of(ClickHouseConfig config, ClickHouseColumn column) { + public static final DateTime32SerDe of(ClickHouseDataConfig config, ClickHouseColumn column) { TimeZone tz = ClickHouseChecker.nonNull(column, ClickHouseColumn.TYPE_NAME).hasTimeZone() ? column.getTimeZone() - : ClickHouseChecker.nonNull(config, ClickHouseConfig.TYPE_NAME).getUseTimeZone(); + : ClickHouseChecker.nonNull(config, ClickHouseDataConfig.TYPE_NAME).getUseTimeZone(); return cache.computeIfAbsent(tz, DateTime32SerDe::new); } @@ -245,10 +248,10 @@ static class DateTime64SerDe implements ClickHouseDeserializer, ClickHouseSerial // use combined key for all timezones? private static final Map cache = new HashMap<>(); - public static final DateTime64SerDe of(ClickHouseConfig config, ClickHouseColumn column) { + public static final DateTime64SerDe of(ClickHouseDataConfig config, ClickHouseColumn column) { TimeZone tz = ClickHouseChecker.nonNull(column, ClickHouseColumn.TYPE_NAME).hasTimeZone() ? column.getTimeZone() - : ClickHouseChecker.nonNull(config, ClickHouseConfig.TYPE_NAME).getUseTimeZone(); + : ClickHouseChecker.nonNull(config, ClickHouseDataConfig.TYPE_NAME).getUseTimeZone(); int scale = column.getScale(); return ClickHouseValues.UTC_TIMEZONE.equals(tz) ? cache.computeIfAbsent(scale, s -> new DateTime64SerDe(s, ClickHouseValues.UTC_TIMEZONE)) diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/BinaryStreamUtils.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/BinaryStreamUtils.java similarity index 99% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/BinaryStreamUtils.java rename to clickhouse-data/src/main/java/com/clickhouse/data/format/BinaryStreamUtils.java index 07ced4001..09117b3fd 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/BinaryStreamUtils.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/format/BinaryStreamUtils.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.format; import java.io.EOFException; import java.io.IOException; @@ -23,12 +23,13 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.value.ClickHouseBitmap; /** * Utility class for dealing with binary stream and data. diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseNativeProcessor.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseNativeProcessor.java new file mode 100644 index 000000000..96fa3436f --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseNativeProcessor.java @@ -0,0 +1,822 @@ +package com.clickhouse.data.format; + +import java.io.IOException; +import java.io.Serializable; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import com.clickhouse.data.ClickHouseArraySequence; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseDataProcessor; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseDeserializer; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseSerializer; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.value.ClickHouseByteValue; +import com.clickhouse.data.value.ClickHouseEmptyValue; +import com.clickhouse.data.value.ClickHouseIntegerValue; +import com.clickhouse.data.value.ClickHouseLongValue; +import com.clickhouse.data.value.ClickHouseShortValue; +import com.clickhouse.data.value.array.ClickHouseByteArrayValue; +import com.clickhouse.data.value.array.ClickHouseIntArrayValue; +import com.clickhouse.data.value.array.ClickHouseLongArrayValue; +import com.clickhouse.data.value.array.ClickHouseShortArrayValue; + +public class ClickHouseNativeProcessor extends ClickHouseDataProcessor { + private static final ClickHouseColumn[] lowCardinalityIndexColumns = new ClickHouseColumn[] { + ClickHouseColumn.of("", ClickHouseDataType.UInt8, false), + ClickHouseColumn.of("", ClickHouseDataType.UInt16, false), + ClickHouseColumn.of("", ClickHouseDataType.UInt32, false), + ClickHouseColumn.of("", ClickHouseDataType.UInt64, false) + }; + private static final ClickHouseDeserializer[] lowCardinalityIndexDeserializers = new ClickHouseDeserializer[] { + ClickHouseNativeProcessor::readByteArray, ClickHouseNativeProcessor::readShortArray, + ClickHouseNativeProcessor::readIntegerArray, ClickHouseNativeProcessor::readLongArray + }; + private static final ClickHouseValue[] lowCardinalityIndexValues = new ClickHouseValue[] { + ClickHouseByteValue.ofUnsignedNull(), ClickHouseShortValue.ofUnsignedNull(), + ClickHouseIntegerValue.ofUnsignedNull(), ClickHouseLongValue.ofUnsignedNull() + }; + private static final ClickHouseArraySequence[] lowCardinalityIndexArrays = new ClickHouseArraySequence[] { + ClickHouseByteArrayValue.ofUnsignedEmpty(), ClickHouseShortArrayValue.ofUnsignedEmpty(), + ClickHouseIntArrayValue.ofUnsignedEmpty(), ClickHouseLongArrayValue.ofUnsignedEmpty() + }; + + static class ArrayColumnDeserializer extends ClickHouseDeserializer.CompositeDeserializer { + private final long length; + private final int nestedLevel; + private final Class valClass; + private final ClickHouseValue valValue; + + public ArrayColumnDeserializer(ClickHouseDataConfig config, ClickHouseColumn column, long length, + boolean nullable, ClickHouseDeserializer... deserializers) { + super(deserializers); + + this.length = length; + + ClickHouseColumn baseColumn = column.getArrayBaseColumn(); + nestedLevel = column.getArrayNestedLevel(); + valClass = baseColumn.getObjectClassForArray(config); + valValue = column.getNestedColumns().get(0).newValue(config); + } + + @Override + public ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseInputStream input) throws IOException { + final long l = this.length; + final int len; + if (l == -1L) { + len = input.readVarInt(); + } else if (l < 0L) { + len = (int) input.readBuffer(8).asLong(); + } else { + len = (int) l; + } + + if (len == 0) { + return ref.resetToNullOrEmpty(); + } + ClickHouseArraySequence arr = (ClickHouseArraySequence) ref; + arr.allocate(len, valClass, nestedLevel); + ClickHouseDeserializer d = deserializers[0]; + for (int i = 0; i < len; i++) { + arr.setValue(i, d.deserialize(valValue, input)); + } + return ref; + } + } + + static class LowCardinalityColumnDeserializer implements ClickHouseDeserializer { + private final ClickHouseDeserializer deserializer; + private final Class javaClass; + private final ClickHouseValue baseValue; + private final ClickHouseArraySequence arrValue; + private final boolean nullable; + + public LowCardinalityColumnDeserializer(ClickHouseDataConfig config, ClickHouseColumn column, + ClickHouseDeserializer d) { + this.javaClass = column.getObjectClassForArray(config); + if (!column.isNullable() && javaClass.isPrimitive()) { + int byteLength = column.getDataType().getByteLength(); + if (byteLength == Byte.BYTES) { // Bool, *Int8 + deserializer = ClickHouseNativeProcessor::readByteArray; + } else if (byteLength == Short.BYTES) { // *Int16 + deserializer = ClickHouseNativeProcessor::readShortArray; + } else if (int.class == javaClass) { // Int32 + deserializer = ClickHouseNativeProcessor::readIntegerArray; + } else if (long.class == javaClass) { // UInt32, *Int64 + deserializer = byteLength == Long.BYTES ? ClickHouseNativeProcessor::readLongArray + : ClickHouseNativeProcessor::readIntegerArray; + } else if (float.class == javaClass) { // Float32 + deserializer = ClickHouseNativeProcessor::readFloatArray; + } else if (double.class == javaClass) { // Float64 + deserializer = ClickHouseNativeProcessor::readDoubleArray; + } else { + throw new IllegalArgumentException("Unsupported primitive type: " + javaClass); + } + } else { + deserializer = ClickHouseChecker.nonNull(d, "Deserializer"); + } + this.baseValue = column.newValue(config); + this.arrValue = column.newArrayValue(config); + this.nullable = column.isNullable(); + } + + @Override + public ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseInputStream input) throws IOException { + long version = input.readBuffer(8).asLong(); + if (version != 1L) { + throw new IOException( + ClickHouseUtils.format("Unexpected low cardinality version %d", version)); + } + ClickHouseArraySequence value = (ClickHouseArraySequence) ref; + long serializationType = input.readBuffer(8).asLong(); + // Lowest byte contains info about key type + int keyType = (int) (serializationType & 0xFL); + if (keyType < 0 || keyType >= lowCardinalityIndexColumns.length) { + throw new IOException("Unsupported key type: " + keyType); + } + // int indexSize = (int) input.readBuffer(8).asLong(); + // ClickHouseValue v = baseColumn.newValue(config); + ClickHouseArraySequence dict = (ClickHouseArraySequence) deserializer.deserialize(arrValue.copy(), input); + if (nullable) { + dict.setValue(0, ClickHouseEmptyValue.INSTANCE); + } + + // ClickHouseColumn indexCol = lowCardinalityIndexColumns[keyType]; + ClickHouseValue indexVal = lowCardinalityIndexValues[keyType].copy(); + ClickHouseArraySequence indexArrVal = lowCardinalityIndexArrays[keyType].copy(); + // int rows = (int) BinaryStreamUtils.readInt64(input); + lowCardinalityIndexDeserializers[keyType].deserialize(indexArrVal, input); + + int rows = indexArrVal.length(); + value.allocate(rows, javaClass); + for (int i = 0; i < rows; i++) { + int keyIndex = indexArrVal.getValue(i, indexVal).asInteger(); + value.setValue(i, dict.getValue(keyIndex, baseValue)); + } + return value; + } + } + + static class LowCardinalitySerializer implements ClickHouseSerializer { + private final ClickHouseSerializer serializer; + + public LowCardinalitySerializer(ClickHouseSerializer serializer) { + this.serializer = ClickHouseChecker.nonNull(serializer, "Serializer"); + } + + @Override + public void serialize(ClickHouseValue value, ClickHouseOutputStream output) throws IOException { + if (value.isNullOrEmpty()) { + output.writeBoolean(true); + } else { + serializer.serialize(value, output.writeBoolean(false)); + } + } + } + + static class NullableColumnDeserializer implements ClickHouseDeserializer { + private final ClickHouseDeserializer deserializer; + + public NullableColumnDeserializer(ClickHouseDeserializer deserializer) { + this.deserializer = ClickHouseChecker.nonNull(deserializer, "Deserializer"); + } + + @Override + public ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseInputStream input) throws IOException { + return ref; + } + } + + // public static class MappedFunctions { + // private static final MappedFunctions instance = new MappedFunctions(); + + // private ClickHouseArraySequence readFlattenArray(ClickHouseArraySequence ref, + // ClickHouseConfig config, + // ClickHouseColumn column, ClickHouseInputStream input) throws IOException { + // if (ref == null) { + // ref = column.newArrayValue(config); + // } + // ClickHouseColumn baseColumn = column.getArrayBaseColumn(); + // int level = column.getArrayNestedLevel(); + // if (level <= 1) { + // int length = (int) BinaryStreamUtils.readInt64(input); + // if (baseColumn.isNullable()) { + // boolean[] nullFlags = input.readBuffer(length).asBooleanArray(); + // ClickHouseDeserializer func = getDeserializer(baseColumn.getDataType()); + // if (func == null) { + // throw new IllegalArgumentException(ERROR_UNKNOWN_DATA_TYPE + + // baseColumn.getDataType().name()); + // } + // ref.allocate(length, baseColumn.getObjectClass(config), level); + // ClickHouseValue v = baseColumn.newValue(config); + // for (int i = 0; i < length; i++) { + // func.deserialize(v, config, baseColumn, input); + // if (nullFlags[i]) { + // v.resetToNullOrEmpty(); + // } + // ref.setValue(i, v); + // } + // } else { + // readArrayContent((ClickHouseArraySequence) ref, config, + // column.getNestedColumns().get(0), + // column.getArrayBaseColumn(), input, length, level); + // } + // } else { + // int length = 0; + // int[][] offsets = new int[level][]; + // for (int i = 1; i < level; i++) { + // int len = (int) BinaryStreamUtils.readInt64(input); + // int[] idx = new int[len]; + // offsets[i - 1] = idx; + // for (int j = 0; j < len; j++) { + // idx[j] = (int) BinaryStreamUtils.readInt64(input); + // } + // } + + // ClickHouseArraySequence parent = ref; + // ClickHouseColumn nested = column; + // for (int i = 0; i < level; i++) { + // int[] idx = offsets[i]; + // int len = idx.length; + // nested = column.getNestedColumns().get(0); + // ClickHouseArraySequence arr = nested.newArrayValue(config); + // for (int j = 0; j < len; j++) { + + // } + // } + // ClickHouseArraySequence arr = ClickHouseColumn.of("", + // ClickHouseDataType.Array, false, baseColumn) + // .newArrayValue(config); + // // readArray(innerMost, config, baseColumn, baseColumn, input, length, 1); + // } + // return ref; + // } + + // private ClickHouseArraySequence readLowCardinality(ClickHouseArraySequence + // value, ClickHouseConfig config, + // ClickHouseColumn arrayColumn, ClickHouseColumn baseColumn, + // ClickHouseInputStream input) + // throws IOException { + // long version = BinaryStreamUtils.readInt64(input); + // if (version != 1L) { + // throw new IOException( + // ClickHouseUtils.format("Unexpected low cardinality version %d when reading + // column %s", + // version, baseColumn.getOriginalTypeName())); + // } + // long serializationType = BinaryStreamUtils.readInt64(input); + // // Lowest byte contains info about key type + // int keyType = (int) (serializationType & 0xFL); + // if (keyType < 0 || keyType >= lowCardinalityIndexColumns.length) { + // throw new IOException("Unsupported key type: " + keyType); + // } + // int indexSize = (int) BinaryStreamUtils.readInt64(input); + // ClickHouseValue v = baseColumn.newValue(config); + // ClickHouseArraySequence dict = readArrayContent(value.copy(), config, + // baseColumn, + // arrayColumn.getArrayBaseColumn(), input, indexSize, + // arrayColumn.getArrayNestedLevel()); + // if (baseColumn.isNullable()) { + // dict.setValue(0, ClickHouseEmptyValue.INSTANCE); + // } + + // ClickHouseColumn indexCol = lowCardinalityIndexColumns[keyType]; + // ClickHouseValue indexVal = lowCardinalityIndexValues[keyType].copy(); + // ClickHouseArraySequence indexArrVal = + // lowCardinalityIndexArrays[keyType].copy(); + // int rows = (int) BinaryStreamUtils.readInt64(input); + // readArrayContent(indexArrVal, config, indexCol, indexCol, input, rows, 1); + + // value.allocate(rows, baseColumn.getObjectClassForArray(config)); + // for (int i = 0; i < rows; i++) { + // int keyIndex = indexArrVal.getValue(i, indexVal).asInteger(); + // value.setValue(i, dict.getValue(keyIndex, v)); + // } + // return value; + // } + + // private ClickHouseArraySequence readColumn(ClickHouseArraySequence value, + // ClickHouseConfig config, + // ClickHouseColumn arrayColumn, ClickHouseColumn baseColumn, + // ClickHouseInputStream input, int rows) + // throws IOException { + // if (baseColumn.isLowCardinality()) { + // return readLowCardinality(value, config, arrayColumn, baseColumn, input); + // } + + // return readArrayContent(value, config, baseColumn, + // arrayColumn.getArrayBaseColumn(), input, rows, + // arrayColumn.getArrayNestedLevel()); + // } + + // protected MappedFunctions() { + // super(); + // } + + // @Override + // public ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseInputStream + // input) throws IOException { + // if (column.isLowCardinality()) { + // if (ref == null) { + // ref = column.newValue(config); + // } + // column.disableLowCardinality(); + // ClickHouseColumn arrCol = ClickHouseColumn.of("", ClickHouseDataType.Array, + // false, column); + // ClickHouseArraySequence arrVal = + // readLowCardinality(arrCol.newArrayValue(config), config, arrCol, + // column, input); + // return arrVal.getValue(0, ref); + // } else if (column.isArray()) { + // return readFlattenArray((ClickHouseArraySequence) ref, config, column, + // input); + // } else { + // if (!column.isLowCardinalityDisabled() && column.isNullable() && + // BinaryStreamUtils.readNull(input)) { + // return ref == null ? column.newValue(config) : ref.resetToNullOrEmpty(); + // } else if (config.isWidenUnsignedTypes()) { + // switch (column.getDataType()) { + // case UInt8: + // return ClickHouseShortValue.of(ref, input.readUnsignedByte(), false); + // case UInt16: + // return ClickHouseIntegerValue.of(ref, input.readBuffer(2).asUnsignedShort(), + // false); + // case UInt32: + // return ClickHouseLongValue.of(ref, input.readBuffer(4).asUnsignedInteger(), + // false); + // default: + // break; + // } + // } + // } + + // ClickHouseDeserializer func = getDeserializer(column.getDataType()); + // if (func == null) { + // throw new IllegalArgumentException(ERROR_UNKNOWN_DATA_TYPE + + // column.getDataType().name()); + // } + // return func.deserialize(ref, config, column, input); + // } + + // @Override + // public void serialize(ClickHouseValue value, ClickHouseOutputStream output) + // throws IOException { + // if (column.isLowCardinality()) { + // column.disableLowCardinality(); + // ClickHouseColumn arrCol = ClickHouseColumn.of("", ClickHouseDataType.Array, + // false, column); + // // ClickHouseArraySequence arrVal = + // // readLowCardinality(arrCol.newArrayValue(config), config, arrCol, + // // column, input); + // } else if (column.isArray()) { + // // write flatten array + // } else { + // if (!column.isLowCardinalityDisabled() && column.isNullable()) { // always + // false for geo types, and + // // Array, Nested, Map and Tuple etc. + // if (value.isNullOrEmpty()) { + // BinaryStreamUtils.writeNull(output); + // return; + // } else { + // BinaryStreamUtils.writeNonNull(output); + // } + // } + + // ClickHouseSerializer func = getSerializer(column.getDataType()); + // if (func == null) { + // throw new IllegalArgumentException(ERROR_UNKNOWN_DATA_TYPE + + // column.getDataType().name()); + // } + // func.serialize(value, config, column, output); + // } + // } + // } + + private static int readArrayLength(ClickHouseInputStream input) throws IOException { + return (int) input.readBuffer(8).asLong(); + } + + private static ClickHouseValue readByteArray(ClickHouseValue ref, ClickHouseInputStream input) throws IOException { + return ref.update(input.readBuffer(readArrayLength(input)).compact().array()); + } + + private static ClickHouseValue readShortArray(ClickHouseValue ref, ClickHouseInputStream input) throws IOException { + return ref.update(input.readBuffer(readArrayLength(input) * 2).asShortArray()); + } + + private static ClickHouseValue readIntegerArray(ClickHouseValue ref, ClickHouseInputStream input) + throws IOException { + return ref.update(input.readBuffer(readArrayLength(input) * 4).asIntegerArray()); + } + + private static ClickHouseValue readLongArray(ClickHouseValue ref, ClickHouseInputStream input) throws IOException { + return ref.update(input.readBuffer(readArrayLength(input) * 8).asLongArray()); + } + + private static ClickHouseValue readFloatArray(ClickHouseValue ref, ClickHouseInputStream input) throws IOException { + return ref.update(input.readBuffer(readArrayLength(input) * 4).asFloatArray()); + } + + private static ClickHouseValue readDoubleArray(ClickHouseValue ref, ClickHouseInputStream input) + throws IOException { + return ref.update(input.readBuffer(readArrayLength(input) * 8).asDoubleArray()); + } + + private static ClickHouseValue readFixedByteArray(ClickHouseValue ref, ClickHouseInputStream input) + throws IOException { + ClickHouseArraySequence arr = (ClickHouseArraySequence) ref; + // input.readFully((byte[]) arr.asRawObject()); + return arr; + } + + private static ClickHouseValue readFixedShortArray(ClickHouseValue ref, ClickHouseInputStream input) + throws IOException { + ClickHouseArraySequence arr = (ClickHouseArraySequence) ref; + return ref.update(input.readBuffer(arr.length() * 2).asShortArray()); + } + + private static ClickHouseValue readFixedIntegerArray(ClickHouseValue ref, ClickHouseInputStream input) + throws IOException { + ClickHouseArraySequence arr = (ClickHouseArraySequence) ref; + return ref.update(input.readBuffer(arr.length() * 4).asIntegerArray()); + } + + private static ClickHouseValue readFixedLongArray(ClickHouseValue ref, ClickHouseInputStream input) + throws IOException { + ClickHouseArraySequence arr = (ClickHouseArraySequence) ref; + return ref.update(input.readBuffer(arr.length() * 8).asLongArray()); + } + + private static ClickHouseValue readFixedFloatArray(ClickHouseValue ref, ClickHouseInputStream input) + throws IOException { + ClickHouseArraySequence arr = (ClickHouseArraySequence) ref; + return ref.update(input.readBuffer(arr.length() * 4).asFloatArray()); + } + + private static ClickHouseValue readFixedDoubleArray(ClickHouseValue ref, ClickHouseInputStream input) + throws IOException { + ClickHouseArraySequence arr = (ClickHouseArraySequence) ref; + return ref.update(input.readBuffer(arr.length() * 8).asDoubleArray()); + } + + private ClickHouseDeserializer[] colDeserializers; + private ClickHouseArraySequence[] currentBlock; + + private int blockRows; + private int currentRow; + + private void readBlock() throws IOException { + int colCount = input.readVarInt(); + int rowCount = input.readVarInt(); + + if (colDeserializers == null) { + colDeserializers = new ClickHouseDeserializer[colCount]; + currentBlock = new ClickHouseArraySequence[colCount]; + } + for (int i = 0; i < colCount; i++) { + // String name = input.readUnicodeString(); + // String type = input.readUnicodeString(); + input.skip(input.readVarInt()); + input.skip(input.readVarInt()); + + ClickHouseColumn col = columns[i]; + ClickHouseDeserializer d = colDeserializers[i]; + ClickHouseArraySequence arr; + if (d == null) { + ClickHouseColumn arrCol = ClickHouseColumn.of("", ClickHouseDataType.Array, false, col); + colDeserializers[i] = d = col.isLowCardinality() + ? new LowCardinalityColumnDeserializer(config, col, getArrayDeserializer(config, arrCol, -2L)) + : getArrayDeserializer(config, arrCol, rowCount); + currentBlock[i] = arr = col.newArrayValue(config); + } else { + arr = currentBlock[i]; + } + arr.allocate(rowCount, col.getObjectClass(config)); + d.deserialize(arr, input); + } + blockRows = rowCount; + currentRow = 0; + + readPosition = 0; + } + + @Override + protected ClickHouseRecord createRecord() { + return new ClickHouseSimpleRecord(getColumns(), templates); + } + + @Override + protected boolean hasMoreToRead() throws UncheckedIOException { + if (currentRow < blockRows) { + return true; + } + + try { + if (input.available() <= 0) { + input.close(); + return false; + } else { + readBlock(); + } + return true; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + protected void readAndFill(ClickHouseRecord r) throws IOException { + for (int i = readPosition, len = columns.length; i < len; i++) { + currentBlock[readPosition].getValue(currentRow, r.getValue(readPosition)); + readPosition = i; + } + + readPosition = 0; + currentRow++; + } + + @Override + protected void readAndFill(ClickHouseValue value) throws IOException { + if (currentRow >= blockRows) { + readBlock(); + } + + currentBlock[readPosition++].getValue(currentRow, value); + if (readPosition >= columns.length) { + readPosition = 0; + currentRow++; + } + } + + @Override + protected List readColumns() throws IOException { + if (input.available() <= 0) { + input.close(); + // no result returned + colDeserializers = new ClickHouseDeserializer[0]; + currentBlock = new ClickHouseArraySequence[0]; + return Collections.emptyList(); + } + + int colCount = input.readVarInt(); + int rowCount = input.readVarInt(); + List columns = new ArrayList<>(colCount); + colDeserializers = new ClickHouseDeserializer[colCount]; + ClickHouseArraySequence[] arrays = new ClickHouseArraySequence[colCount]; + + for (int i = 0; i < colCount; i++) { + String name = input.readUnicodeString(); + ClickHouseColumn baseColumn = ClickHouseColumn.of(name, input.readUnicodeString()); + ClickHouseColumn arraryColumn = ClickHouseColumn.of("", ClickHouseDataType.Array, false, baseColumn); + ClickHouseArraySequence value = arraryColumn.newArrayValue(config); + value.allocate(rowCount); + + columns.add(baseColumn); + ClickHouseDeserializer d = baseColumn.isLowCardinality() + ? new LowCardinalityColumnDeserializer(config, baseColumn, + getArrayDeserializer(config, arraryColumn, -2L)) + : getArrayDeserializer(config, arraryColumn, rowCount); + colDeserializers[i] = d; + value.allocate(rowCount, baseColumn.getObjectClass(config)); + arrays[i] = (ClickHouseArraySequence) d.deserialize(value, input); + } + currentBlock = arrays; + blockRows = rowCount; + currentRow = 0; + return columns; + } + + public ClickHouseNativeProcessor(ClickHouseDataConfig config, ClickHouseInputStream input, + ClickHouseOutputStream output, List columns, Map settings) + throws IOException { + super(config, input, output, columns, settings); + } + + private ClickHouseDeserializer getArrayDeserializer(ClickHouseDataConfig config, ClickHouseColumn column, + long length) { + final boolean fixedLength = length >= 0L; + final ClickHouseDeserializer deserializer; + ClickHouseColumn baseColumn = column.getArrayBaseColumn(); + Class javaClass = baseColumn.getObjectClassForArray(config); + if (column.getArrayNestedLevel() == 1 && !baseColumn.isNullable() && javaClass.isPrimitive()) { + int byteLength = baseColumn.getDataType().getByteLength(); + if (byteLength == Byte.BYTES) { // Bool, *Int8 + deserializer = fixedLength ? ClickHouseNativeProcessor::readFixedByteArray + : ClickHouseNativeProcessor::readByteArray; + } else if (byteLength == Short.BYTES) { // *Int16 + deserializer = fixedLength ? ClickHouseNativeProcessor::readFixedShortArray + : ClickHouseNativeProcessor::readShortArray; + } else if (int.class == javaClass) { // Int32 + deserializer = fixedLength ? ClickHouseNativeProcessor::readFixedIntegerArray + : ClickHouseNativeProcessor::readIntegerArray; + } else if (long.class == javaClass) { // UInt32, *Int64 + if (byteLength == Long.BYTES) { + deserializer = fixedLength ? ClickHouseNativeProcessor::readFixedLongArray + : ClickHouseNativeProcessor::readLongArray; + } else { + deserializer = fixedLength ? ClickHouseNativeProcessor::readFixedIntegerArray + : ClickHouseNativeProcessor::readIntegerArray; + } + } else if (float.class == javaClass) { // Float32 + deserializer = fixedLength ? ClickHouseNativeProcessor::readFixedFloatArray + : ClickHouseNativeProcessor::readFloatArray; + } else if (double.class == javaClass) { // Float64 + deserializer = fixedLength ? ClickHouseNativeProcessor::readFixedDoubleArray + : ClickHouseNativeProcessor::readDoubleArray; + } else { + throw new IllegalArgumentException("Unsupported primitive type: " + javaClass); + } + } else { + ClickHouseColumn nestedCol = column.getNestedColumns().get(0); + deserializer = new BinaryDataProcessor.ArrayDeserializer(config, column, length, + getDeserializer(config, nestedCol)); + } + return deserializer; + } + + @Override + public ClickHouseDeserializer getDeserializer(ClickHouseDataConfig config, ClickHouseColumn column) { + final ClickHouseDeserializer deserializer; + switch (column.getDataType()) { + case Bool: + deserializer = BinaryDataProcessor::readBool; + break; + case Date: + deserializer = BinaryDataProcessor.DateSerDe.of(config); + break; + case Date32: + deserializer = BinaryDataProcessor.Date32SerDe.of(config); + break; + case DateTime: + deserializer = column.getScale() > 0 ? BinaryDataProcessor.DateTime64SerDe.of(config, column) + : BinaryDataProcessor.DateTime32SerDe.of(config, column); + break; + case DateTime32: + deserializer = BinaryDataProcessor.DateTime32SerDe.of(config, column); + break; + case DateTime64: + deserializer = BinaryDataProcessor.DateTime64SerDe.of(config, column); + break; + case Enum8: + deserializer = BinaryDataProcessor::readEnum8; + break; + case Enum16: + deserializer = BinaryDataProcessor::readEnum8; + break; + case FixedString: + deserializer = new BinaryDataProcessor.FixedStringSerDe(column); + break; + case Int8: + deserializer = BinaryDataProcessor::readByte; + break; + case UInt8: + deserializer = config.isWidenUnsignedTypes() ? BinaryDataProcessor::readUInt8AsShort + : BinaryDataProcessor::readByte; + break; + case Int16: + deserializer = BinaryDataProcessor::readShort; + break; + case UInt16: + deserializer = config.isWidenUnsignedTypes() ? BinaryDataProcessor::readUInt16AsInt + : BinaryDataProcessor::readShort; + break; + case Int32: + deserializer = BinaryDataProcessor::readInteger; + break; + case UInt32: + deserializer = config.isWidenUnsignedTypes() ? BinaryDataProcessor::readUInt32AsLong + : BinaryDataProcessor::readInteger; + break; + case Int64: + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case IntervalMicrosecond: + case IntervalMillisecond: + case IntervalNanosecond: + case UInt64: + deserializer = BinaryDataProcessor::readLong; + break; + case Int128: + deserializer = BinaryDataProcessor::readInt128; + break; + case UInt128: + deserializer = BinaryDataProcessor::readUInt128; + break; + case Int256: + deserializer = BinaryDataProcessor::readInt256; + break; + case UInt256: + deserializer = BinaryDataProcessor::readUInt256; + break; + case Decimal: + deserializer = BinaryDataProcessor.DecimalSerDe.of(column); + break; + case Decimal32: + deserializer = BinaryDataProcessor.Decimal32SerDe.of(column); + break; + case Decimal64: + deserializer = BinaryDataProcessor.Decimal64SerDe.of(column); + break; + case Decimal128: + deserializer = BinaryDataProcessor.Decimal128SerDe.of(column); + break; + case Decimal256: + deserializer = BinaryDataProcessor.Decimal256SerDe.of(column); + break; + case Float32: + deserializer = BinaryDataProcessor::readFloat; + break; + case Float64: + deserializer = BinaryDataProcessor::readDouble; + break; + case IPv4: + deserializer = BinaryDataProcessor::readIpv4; + break; + case IPv6: + deserializer = BinaryDataProcessor::readIpv6; + break; + case UUID: + deserializer = BinaryDataProcessor::readUuid; + break; + // Geo types + case Point: + deserializer = BinaryDataProcessor::readGeoPoint; + break; + case Ring: + deserializer = BinaryDataProcessor::readGeoRing; + break; + case Polygon: + deserializer = BinaryDataProcessor::readGeoPolygon; + break; + case MultiPolygon: + deserializer = BinaryDataProcessor::readGeoMultiPolygon; + break; + // String + case JSON: + case Object: + case String: + deserializer = config.isUseBinaryString() ? BinaryDataProcessor::readBinaryString + : BinaryDataProcessor::readTextString; + break; + // nested + case Array: + deserializer = getArrayDeserializer(config, column, -1L); + break; + case Map: + deserializer = new ClickHouseRowBinaryProcessor.MapDeserializer(config, column, + getDeserializers(config, column.getNestedColumns())); + break; + case Nested: + deserializer = ClickHouseDeserializer.EMPTY_VALUE; + // new ClickHouseRowBinaryProcessor.NestedDeserializer(config, column, + // getArraySerDeializers(config, column.getNestedColumns())); + break; + case Tuple: + deserializer = new ClickHouseRowBinaryProcessor.TupleDeserializer(config, column, + getDeserializers(config, column.getNestedColumns())); + break; + // special + case Nothing: + deserializer = ClickHouseDeserializer.EMPTY_VALUE; + break; + case SimpleAggregateFunction: + deserializer = getDeserializer(config, column.getNestedColumns().get(0)); + break; + default: + throw new IllegalArgumentException("Unsupported column:" + column.toString()); + } + + return deserializer; + // return !column.isLowCardinality() && column.isNullable() + // ? new BinaryDataProcessor.NullableDeserializer(deserializer) + // : deserializer; + } + + @Override + public ClickHouseSerializer getSerializer(ClickHouseDataConfig config, ClickHouseColumn column) { + // TODO Auto-generated method stub + return null; + } + + @Override + public void write(ClickHouseValue value) throws IOException { + // TODO Auto-generated method stub + + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseRowBinaryProcessor.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessor.java similarity index 91% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseRowBinaryProcessor.java rename to clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessor.java index e761e0339..53eed4059 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseRowBinaryProcessor.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessor.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.format; import java.io.IOException; import java.io.Serializable; @@ -9,23 +9,24 @@ import java.util.Map; import java.util.Map.Entry; -import com.clickhouse.client.ClickHouseAggregateFunction; -import com.clickhouse.client.ClickHouseArraySequence; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseDataProcessor; -import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseDeserializer; -import com.clickhouse.client.ClickHouseFormat; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHouseRecord; -import com.clickhouse.client.ClickHouseSerializer; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseRenameMethod; +import com.clickhouse.config.ClickHouseRenameMethod; +import com.clickhouse.data.ClickHouseAggregateFunction; +import com.clickhouse.data.ClickHouseArraySequence; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseDataProcessor; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseDeserializer; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseSerializer; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.value.ClickHouseBitmap; +import com.clickhouse.data.value.ClickHouseBitmapValue; /** * Data processor for handling {@link ClickHouseFormat#RowBinary} and @@ -35,7 +36,7 @@ public class ClickHouseRowBinaryProcessor extends ClickHouseDataProcessor { public static class BitmapSerDe implements ClickHouseDeserializer, ClickHouseSerializer { private final ClickHouseDataType innerType; - public BitmapSerDe(ClickHouseConfig config, ClickHouseColumn column) { + public BitmapSerDe(ClickHouseDataConfig config, ClickHouseColumn column) { this.innerType = column.getNestedColumns().get(0).getDataType(); } @@ -55,7 +56,7 @@ public static class MapDeserializer extends ClickHouseDeserializer.CompositeDese private final ClickHouseValue keyValue; private final ClickHouseValue valValue; - public MapDeserializer(ClickHouseConfig config, ClickHouseColumn column, + public MapDeserializer(ClickHouseDataConfig config, ClickHouseColumn column, ClickHouseDeserializer... deserializers) { super(deserializers); @@ -89,7 +90,7 @@ public static class MapSerializer extends ClickHouseSerializer.CompositeSerializ private final ClickHouseValue keyValue; private final ClickHouseValue valValue; - public MapSerializer(ClickHouseConfig config, ClickHouseColumn column, + public MapSerializer(ClickHouseDataConfig config, ClickHouseColumn column, ClickHouseSerializer... serializers) { super(serializers); @@ -119,7 +120,7 @@ public void serialize(ClickHouseValue value, ClickHouseOutputStream output) thro public static class NestedDeserializer extends ClickHouseDeserializer.CompositeDeserializer { protected final ClickHouseValue[] values; - public NestedDeserializer(ClickHouseConfig config, ClickHouseColumn column, + public NestedDeserializer(ClickHouseDataConfig config, ClickHouseColumn column, ClickHouseDeserializer... deserializers) { super(deserializers); @@ -151,7 +152,7 @@ public ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseInputStream in public static class NestedSerializer extends ClickHouseSerializer.CompositeSerializer { private final ClickHouseArraySequence[] values; - public NestedSerializer(ClickHouseConfig config, ClickHouseColumn column, + public NestedSerializer(ClickHouseDataConfig config, ClickHouseColumn column, ClickHouseSerializer... serializers) { super(serializers); @@ -179,7 +180,7 @@ public void serialize(ClickHouseValue value, ClickHouseOutputStream output) thro public static class TupleDeserializer extends ClickHouseDeserializer.CompositeDeserializer { private final ClickHouseValue[] values; - public TupleDeserializer(ClickHouseConfig config, ClickHouseColumn column, + public TupleDeserializer(ClickHouseDataConfig config, ClickHouseColumn column, ClickHouseDeserializer... deserializers) { super(deserializers); @@ -209,7 +210,7 @@ public ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseInputStream in public static class TupleSerializer extends ClickHouseSerializer.CompositeSerializer { private final ClickHouseValue[] values; - public TupleSerializer(ClickHouseConfig config, ClickHouseColumn column, + public TupleSerializer(ClickHouseDataConfig config, ClickHouseColumn column, ClickHouseSerializer... serializers) { super(serializers); @@ -281,8 +282,7 @@ protected List readColumns() throws IOException { names[i] = input.readUnicodeString(); } - ClickHouseRenameMethod m = config.getOption(ClickHouseClientOption.RENAME_RESPONSE_COLUMN, - ClickHouseRenameMethod.class); + ClickHouseRenameMethod m = config.getColumnRenameMethod(); List columns = new ArrayList<>(size); for (int i = 0; i < size; i++) { columns.add(ClickHouseColumn.of(m.rename(names[i]), input.readUnicodeString())); @@ -303,16 +303,16 @@ protected List readColumns() throws IOException { * @param settings nullable settings * @throws IOException when failed to read columns from input stream */ - public ClickHouseRowBinaryProcessor(ClickHouseConfig config, ClickHouseInputStream input, + public ClickHouseRowBinaryProcessor(ClickHouseDataConfig config, ClickHouseInputStream input, ClickHouseOutputStream output, List columns, Map settings) throws IOException { super(config, input, output, columns, settings); } - protected ClickHouseDeserializer[] getArrayDeserializers(ClickHouseConfig config, List columns) { + protected ClickHouseDeserializer[] getArrayDeserializers(ClickHouseDataConfig config, + List columns) { List list = new ArrayList<>(columns.size()); - ClickHouseConfig modifiedConfig = new ClickHouseConfig(config, - new ClickHouseConfig(Collections.singletonMap(ClickHouseClientOption.USE_OBJECTS_IN_ARRAYS, true))); + ClickHouseDataConfig modifiedConfig = new UseObjectConfig(config); for (ClickHouseColumn column : columns) { list.add(getDeserializer(modifiedConfig, ClickHouseColumn.of(column.getColumnName(), ClickHouseDataType.Array, false, column))); @@ -320,10 +320,9 @@ protected ClickHouseDeserializer[] getArrayDeserializers(ClickHouseConfig config return list.toArray(new ClickHouseDeserializer[0]); } - protected ClickHouseSerializer[] getArraySerializers(ClickHouseConfig config, List columns) { + protected ClickHouseSerializer[] getArraySerializers(ClickHouseDataConfig config, List columns) { List list = new ArrayList<>(columns.size()); - ClickHouseConfig modifiedConfig = new ClickHouseConfig(config, - new ClickHouseConfig(Collections.singletonMap(ClickHouseClientOption.USE_OBJECTS_IN_ARRAYS, true))); + ClickHouseDataConfig modifiedConfig = new UseObjectConfig(config); for (ClickHouseColumn column : columns) { list.add(getSerializer(modifiedConfig, ClickHouseColumn.of(column.getColumnName(), ClickHouseDataType.Array, false, column))); @@ -332,7 +331,7 @@ protected ClickHouseSerializer[] getArraySerializers(ClickHouseConfig config, Li } @Override - public ClickHouseDeserializer getDeserializer(ClickHouseConfig config, ClickHouseColumn column) { + public ClickHouseDeserializer getDeserializer(ClickHouseDataConfig config, ClickHouseColumn column) { final ClickHouseDeserializer deserializer; switch (column.getDataType()) { case Bool: @@ -522,7 +521,7 @@ public ClickHouseDeserializer getDeserializer(ClickHouseConfig config, ClickHous } @Override - public ClickHouseSerializer getSerializer(ClickHouseConfig config, ClickHouseColumn column) { + public ClickHouseSerializer getSerializer(ClickHouseDataConfig config, ClickHouseColumn column) { final ClickHouseSerializer serializer; switch (column.getDataType()) { case Bool: diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseSimpleRecord.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseSimpleRecord.java similarity index 90% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseSimpleRecord.java rename to clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseSimpleRecord.java index 3a88604e8..e7258e02d 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseSimpleRecord.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseSimpleRecord.java @@ -1,15 +1,15 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.format; import java.util.Collections; import java.util.List; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseRecord; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValue; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; /** - * Default implementation of {@link com.clickhouse.client.ClickHouseRecord}, + * Default implementation of {@link com.clickhouse.data.ClickHouseRecord}, * which is simply a combination of list of columns and array of values. */ public class ClickHouseSimpleRecord implements ClickHouseRecord { diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseTabSeparatedProcessor.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseTabSeparatedProcessor.java similarity index 81% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseTabSeparatedProcessor.java rename to clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseTabSeparatedProcessor.java index 9c7b761f6..930bdf51e 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseTabSeparatedProcessor.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseTabSeparatedProcessor.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.format; import java.io.EOFException; import java.io.IOException; @@ -8,21 +8,20 @@ import java.util.List; import java.util.Map; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseDataProcessor; -import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseDeserializer; -import com.clickhouse.client.ClickHouseFormat; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHouseRecord; -import com.clickhouse.client.ClickHouseSerializer; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseRenameMethod; -import com.clickhouse.client.data.tsv.ByteFragment; +import com.clickhouse.config.ClickHouseRenameMethod; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseDataProcessor; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseDeserializer; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseSerializer; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.format.tsv.ByteFragment; public class ClickHouseTabSeparatedProcessor extends ClickHouseDataProcessor { private static String[] toStringArray(ByteFragment headerFragment, byte delimitter) { @@ -122,8 +121,7 @@ protected List readColumns() throws IOException { types = toStringArray(typesFragment, ds.getValueSeparator()); } - ClickHouseRenameMethod m = config.getOption(ClickHouseClientOption.RENAME_RESPONSE_COLUMN, - ClickHouseRenameMethod.class); + ClickHouseRenameMethod m = config.getColumnRenameMethod(); List list = new ArrayList<>(cols.length); for (int i = 0; i < cols.length; i++) { list.add(ClickHouseColumn.of(m.rename(cols[i]), types == null ? "Nullable(String)" : types[i])); @@ -132,7 +130,7 @@ protected List readColumns() throws IOException { return list; } - public ClickHouseTabSeparatedProcessor(ClickHouseConfig config, ClickHouseInputStream input, + public ClickHouseTabSeparatedProcessor(ClickHouseDataConfig config, ClickHouseInputStream input, ClickHouseOutputStream output, List columns, Map settings) throws IOException { super(config, input, output, columns, settings); @@ -150,7 +148,7 @@ public void write(ClickHouseValue value) throws IOException { } @Override - public ClickHouseDeserializer getDeserializer(ClickHouseConfig config, ClickHouseColumn column) { + public ClickHouseDeserializer getDeserializer(ClickHouseDataConfig config, ClickHouseColumn column) { ClickHouseDataType dt = column.getDataType(); return dt == ClickHouseDataType.FixedString || (config.isUseBinaryString() && dt == ClickHouseDataType.String) ? getTextSerDe()::deserializeBinary @@ -158,7 +156,7 @@ public ClickHouseDeserializer getDeserializer(ClickHouseConfig config, ClickHous } @Override - public ClickHouseSerializer getSerializer(ClickHouseConfig config, ClickHouseColumn column) { + public ClickHouseSerializer getSerializer(ClickHouseDataConfig config, ClickHouseColumn column) { ClickHouseDataType dt = column.getDataType(); return dt == ClickHouseDataType.FixedString || (config.isUseBinaryString() && dt == ClickHouseDataType.String) ? getTextSerDe()::serializeBinary diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/JsonStreamUtils.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/JsonStreamUtils.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/JsonStreamUtils.java rename to clickhouse-data/src/main/java/com/clickhouse/data/format/JsonStreamUtils.java index d3af6c6c6..c5dc8ea6e 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/JsonStreamUtils.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/format/JsonStreamUtils.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.format; import java.io.IOException; import java.io.InputStream; diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/format/MessagePackProcessor.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/MessagePackProcessor.java new file mode 100644 index 000000000..c444da7f2 --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/data/format/MessagePackProcessor.java @@ -0,0 +1,61 @@ +package com.clickhouse.data.format; + +import java.io.IOException; +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseDataProcessor; +import com.clickhouse.data.ClickHouseDeserializer; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseSerializer; +import com.clickhouse.data.ClickHouseValue; + +public class MessagePackProcessor extends ClickHouseDataProcessor { + + protected MessagePackProcessor(ClickHouseDataConfig config, ClickHouseInputStream input, + ClickHouseOutputStream output, + List columns, Map settings) throws IOException { + super(config, input, output, columns, settings); + } + + @Override + protected ClickHouseRecord createRecord() { + // TODO Auto-generated method stub + return null; + } + + @Override + protected void readAndFill(ClickHouseValue value) throws IOException { + // TODO Auto-generated method stub + + } + + @Override + protected List readColumns() throws IOException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void write(ClickHouseValue value) throws IOException { + // TODO Auto-generated method stub + + } + + @Override + public ClickHouseDeserializer getDeserializer(ClickHouseDataConfig config, ClickHouseColumn column) { + // TODO Auto-generated method stub + return null; + } + + @Override + public ClickHouseSerializer getSerializer(ClickHouseDataConfig config, ClickHouseColumn column) { + // TODO Auto-generated method stub + return null; + } +} diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/TextDataProcessor.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/TextDataProcessor.java similarity index 94% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/TextDataProcessor.java rename to clickhouse-data/src/main/java/com/clickhouse/data/format/TextDataProcessor.java index 79cabe420..c93e927de 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/TextDataProcessor.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/format/TextDataProcessor.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.format; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -6,14 +6,14 @@ import java.util.HashMap; import java.util.Map; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseDeserializer; -import com.clickhouse.client.ClickHouseFormat; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHouseSerializer; -import com.clickhouse.client.ClickHouseValue; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseDeserializer; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseSerializer; +import com.clickhouse.data.ClickHouseValue; public interface TextDataProcessor { static class TextSerDe implements ClickHouseDeserializer, ClickHouseSerializer { diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/ArrayByteFragment.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/ArrayByteFragment.java similarity index 92% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/ArrayByteFragment.java rename to clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/ArrayByteFragment.java index 6cea30656..8957f57e6 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/ArrayByteFragment.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/ArrayByteFragment.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data.tsv; +package com.clickhouse.data.format.tsv; public final class ArrayByteFragment extends ByteFragment { diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/ByteFragment.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/ByteFragment.java similarity index 99% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/ByteFragment.java rename to clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/ByteFragment.java index 989929305..c7c2a16e1 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/ByteFragment.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/ByteFragment.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data.tsv; +package com.clickhouse.data.format.tsv; import java.io.ByteArrayInputStream; import java.io.IOException; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/FastByteArrayInputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/FastByteArrayInputStream.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/FastByteArrayInputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/FastByteArrayInputStream.java index a317a5199..7eaf9b712 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/FastByteArrayInputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/FastByteArrayInputStream.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data.tsv; +package com.clickhouse.data.format.tsv; import java.io.IOException; import java.io.InputStream; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/FastByteArrayOutputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/FastByteArrayOutputStream.java similarity index 99% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/FastByteArrayOutputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/FastByteArrayOutputStream.java index a55654871..3870b3231 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/FastByteArrayOutputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/FastByteArrayOutputStream.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data.tsv; +package com.clickhouse.data.format.tsv; import java.io.DataOutput; import java.io.IOException; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/StreamSplitter.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/StreamSplitter.java similarity index 99% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/StreamSplitter.java rename to clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/StreamSplitter.java index 71ee17deb..1fea4fd0b 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/StreamSplitter.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/StreamSplitter.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data.tsv; +package com.clickhouse.data.format.tsv; import java.io.IOException; import java.io.InputStream; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/package-info.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/package-info.java similarity index 60% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/package-info.java rename to clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/package-info.java index fcf3366a0..59d010895 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/tsv/package-info.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/format/tsv/package-info.java @@ -1,4 +1,4 @@ /** * Provides necessary classes to handle TSV format. */ -package com.clickhouse.client.data.tsv; +package com.clickhouse.data.format.tsv; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/AbstractByteArrayInputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/AbstractByteArrayInputStream.java similarity index 95% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/AbstractByteArrayInputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/AbstractByteArrayInputStream.java index d30038410..6d3a4d8ef 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/AbstractByteArrayInputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/AbstractByteArrayInputStream.java @@ -1,16 +1,16 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.EOFException; import java.io.IOException; import java.io.OutputStream; import java.util.LinkedList; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseDataUpdater; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHousePassThruStream; -import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseDataUpdater; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePassThruStream; +import com.clickhouse.data.ClickHouseUtils; /** * Byte-array backed input stream. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/AbstractByteArrayOutputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/AbstractByteArrayOutputStream.java similarity index 93% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/AbstractByteArrayOutputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/AbstractByteArrayOutputStream.java index 4ec13fa02..6ab16a7ea 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/AbstractByteArrayOutputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/AbstractByteArrayOutputStream.java @@ -1,10 +1,10 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; -import com.clickhouse.client.ClickHouseDataUpdater; -import com.clickhouse.client.ClickHousePassThruStream; -import com.clickhouse.client.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseDataUpdater; +import com.clickhouse.data.ClickHousePassThruStream; +import com.clickhouse.data.ClickHouseOutputStream; public abstract class AbstractByteArrayOutputStream extends ClickHouseOutputStream { protected final byte[] buffer; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/AbstractByteBufferInputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/AbstractByteBufferInputStream.java similarity index 96% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/AbstractByteBufferInputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/AbstractByteBufferInputStream.java index b16034290..d89d96776 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/AbstractByteBufferInputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/AbstractByteBufferInputStream.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.EOFException; import java.io.IOException; @@ -7,12 +7,12 @@ import java.nio.ByteBuffer; import java.util.LinkedList; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseDataUpdater; -import com.clickhouse.client.ClickHouseFile; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseDataUpdater; +import com.clickhouse.data.ClickHouseFile; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseUtils; /** * {@link java.nio.ByteBuffer} backed input stream. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/AdaptiveQueue.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/AdaptiveQueue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/AdaptiveQueue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/AdaptiveQueue.java index b3daa6968..6b8eb9aae 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/AdaptiveQueue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/AdaptiveQueue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.util.Arrays; import java.util.LinkedList; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/BlockingInputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/BlockingInputStream.java similarity index 78% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/BlockingInputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/BlockingInputStream.java index 4d6fa0d9c..38703b27b 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/BlockingInputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/BlockingInputStream.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; import java.nio.Buffer; @@ -6,10 +6,10 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; /** * {@link java.nio.ByteBuffer} backed input stream with @@ -28,15 +28,6 @@ public BlockingInputStream(BlockingQueue queue, int timeout, Runnabl this.timeout = timeout > 0 ? timeout : 0; } - @Override - protected void ensureOpen() throws IOException { - if (closed) { - log.debug("Blocking input stream(queue: %d, buffer: %d) has been closed", - queue.size(), buffer != null ? buffer.remaining() : 0); - } - super.ensureOpen(); - } - @Override protected int updateBuffer() throws IOException { ByteBuffer b; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/BlockingPipedOutputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/BlockingPipedOutputStream.java similarity index 90% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/BlockingPipedOutputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/BlockingPipedOutputStream.java index d26486253..bc005b852 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/BlockingPipedOutputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/BlockingPipedOutputStream.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; import java.nio.Buffer; @@ -8,13 +8,13 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseDataUpdater; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHousePipedOutputStream; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseDataUpdater; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePipedOutputStream; +import com.clickhouse.data.ClickHouseUtils; /** * A combination of {@link java.io.PipedOutputStream} and @@ -37,9 +37,7 @@ public BlockingPipedOutputStream(int bufferSize, int queueLength, int timeout, R this.queue = queueLength <= 0 ? new LinkedBlockingQueue<>() : new ArrayBlockingQueue<>(queueLength); // may need an initialBufferSize and a monitor to update bufferSize in runtime - this.bufferSize = ClickHouseUtils.getBufferSize(bufferSize, - (int) ClickHouseClientOption.BUFFER_SIZE.getDefaultValue(), - (int) ClickHouseClientOption.MAX_BUFFER_SIZE.getDefaultValue()); + this.bufferSize = ClickHouseDataConfig.getBufferSize(bufferSize); this.timeout = timeout; this.buffer = ByteBuffer.allocate(this.bufferSize); diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/CapacityPolicy.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/CapacityPolicy.java similarity index 97% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/CapacityPolicy.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/CapacityPolicy.java index b6aa5b038..13375c416 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/CapacityPolicy.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/CapacityPolicy.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; @FunctionalInterface public interface CapacityPolicy { diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/DeferredInputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/DeferredInputStream.java similarity index 92% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/DeferredInputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/DeferredInputStream.java index 60b149187..4bb942a79 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/DeferredInputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/DeferredInputStream.java @@ -1,9 +1,9 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; import java.io.InputStream; -import com.clickhouse.client.ClickHouseDeferredValue; +import com.clickhouse.data.ClickHouseDeferredValue; public final class DeferredInputStream extends InputStream { private final ClickHouseDeferredValue ref; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/DeferredOutputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/DeferredOutputStream.java similarity index 89% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/DeferredOutputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/DeferredOutputStream.java index d529e5ba8..21b28d600 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/DeferredOutputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/DeferredOutputStream.java @@ -1,9 +1,9 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; import java.io.OutputStream; -import com.clickhouse.client.ClickHouseDeferredValue; +import com.clickhouse.data.ClickHouseDeferredValue; public final class DeferredOutputStream extends OutputStream { private final ClickHouseDeferredValue ref; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/EmptyInputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/EmptyInputStream.java similarity index 83% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/EmptyInputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/EmptyInputStream.java index ec0624850..d33365c4d 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/EmptyInputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/EmptyInputStream.java @@ -1,12 +1,12 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.EOFException; import java.io.IOException; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseDataUpdater; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseDataUpdater; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; /** * Empty input stream produces nothing and it can never be closed. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/EmptyOutputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/EmptyOutputStream.java similarity index 87% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/EmptyOutputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/EmptyOutputStream.java index 4cc8a6c6d..94fab48dd 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/EmptyOutputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/EmptyOutputStream.java @@ -1,9 +1,9 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; -import com.clickhouse.client.ClickHouseDataUpdater; -import com.clickhouse.client.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseDataUpdater; +import com.clickhouse.data.ClickHouseOutputStream; /** * Empty output stream consumes nothing and it can never be closed. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/IterableByteArrayInputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/IterableByteArrayInputStream.java similarity index 86% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/IterableByteArrayInputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/IterableByteArrayInputStream.java index c7d5375cd..1edc65bfa 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/IterableByteArrayInputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/IterableByteArrayInputStream.java @@ -1,10 +1,10 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; import java.util.Iterator; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseChecker; public class IterableByteArrayInputStream extends AbstractByteArrayInputStream { private final Iterator it; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/IterableByteBufferInputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/IterableByteBufferInputStream.java similarity index 90% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/IterableByteBufferInputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/IterableByteBufferInputStream.java index 94d370c1b..8e2904081 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/IterableByteBufferInputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/IterableByteBufferInputStream.java @@ -1,12 +1,12 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; import java.nio.Buffer; import java.nio.ByteBuffer; import java.util.Iterator; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseChecker; public class IterableByteBufferInputStream extends AbstractByteBufferInputStream { private final Iterator it; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/IterableMultipleInputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/IterableMultipleInputStream.java similarity index 92% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/IterableMultipleInputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/IterableMultipleInputStream.java index 64a7fdbd8..4b87b3247 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/IterableMultipleInputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/IterableMultipleInputStream.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; import java.io.InputStream; @@ -6,12 +6,11 @@ import java.util.LinkedList; import java.util.function.Function; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseDataUpdater; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseDataUpdater; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; public final class IterableMultipleInputStream extends AbstractByteArrayInputStream { private final Function func; @@ -59,7 +58,7 @@ protected int updateBuffer() throws IOException { } public IterableMultipleInputStream(Iterable source, Function converter, - Runnable postCloseAction) { + int bufferSize, Runnable postCloseAction) { super(null, null, postCloseAction); func = ClickHouseChecker.nonNull(converter, "Converter"); @@ -67,7 +66,7 @@ public IterableMultipleInputStream(Iterable source, Function in = EmptyInputStream.INSTANCE; // fixed buffer - buffer = new byte[(int) ClickHouseClientOption.BUFFER_SIZE.getDefaultValue()]; + buffer = new byte[bufferSize]; position = 0; limit = 0; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/IterableObjectInputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/IterableObjectInputStream.java similarity index 89% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/IterableObjectInputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/IterableObjectInputStream.java index ef6f80211..96fe450ad 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/IterableObjectInputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/IterableObjectInputStream.java @@ -1,11 +1,11 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; import java.util.Iterator; import java.util.function.Function; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseChecker; public class IterableObjectInputStream extends AbstractByteArrayInputStream { private final Function func; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/Lz4InputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/Lz4InputStream.java similarity index 81% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/Lz4InputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/Lz4InputStream.java index 55fca2afb..1b005bb46 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/Lz4InputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/Lz4InputStream.java @@ -1,14 +1,14 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; import java.io.InputStream; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHousePassThruStream; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.data.BinaryStreamUtils; -import com.clickhouse.client.data.ClickHouseCityHash; +import com.clickhouse.data.ClickHouseByteUtils; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseCityHash; +import com.clickhouse.data.ClickHousePassThruStream; +import com.clickhouse.data.ClickHouseUtils; import net.jpountz.lz4.LZ4Factory; import net.jpountz.lz4.LZ4FastDecompressor; @@ -59,22 +59,23 @@ protected int updateBuffer() throws IOException { } // 4 bytes - size of the compressed data including 9 bytes of the header - int compressedSizeWithHeader = BinaryStreamUtils.toInt32(header, 17); + int compressedSizeWithHeader = ClickHouseByteUtils.getInt32LE(header, 17); // 4 bytes - size of uncompressed data - int uncompressedSize = BinaryStreamUtils.toInt32(header, 21); + int uncompressedSize = ClickHouseByteUtils.getInt32LE(header, 21); int offset = 9; final byte[] block = compressedBlock.length >= compressedSizeWithHeader ? compressedBlock : (compressedBlock = new byte[compressedSizeWithHeader]); block[0] = header[16]; - BinaryStreamUtils.setInt32(block, 1, compressedSizeWithHeader); - BinaryStreamUtils.setInt32(block, 5, uncompressedSize); + ClickHouseByteUtils.setInt32LE(block, 1, compressedSizeWithHeader); + ClickHouseByteUtils.setInt32LE(block, 5, uncompressedSize); // compressed data: compressed_size - 9 bytes if (!readFully(block, offset, compressedSizeWithHeader - offset)) { throw new IOException(ClickHouseUtils.format(ERROR_INCOMPLETE_READ, 0, compressedSizeWithHeader - offset)); } long[] real = ClickHouseCityHash.cityHash128(block, 0, compressedSizeWithHeader); - if (real[0] != BinaryStreamUtils.toInt64(header, 0) || real[1] != BinaryStreamUtils.toInt64(header, 8)) { + if (real[0] != ClickHouseByteUtils.getInt64LE(header, 0) + || real[1] != ClickHouseByteUtils.getInt64LE(header, 8)) { throw new IOException("Checksum doesn't match: corrupted data."); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/Lz4OutputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/Lz4OutputStream.java similarity index 78% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/Lz4OutputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/Lz4OutputStream.java index 8394f531e..ce6e7cfca 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/Lz4OutputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/Lz4OutputStream.java @@ -1,12 +1,12 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; import java.io.OutputStream; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHousePassThruStream; -import com.clickhouse.client.data.BinaryStreamUtils; -import com.clickhouse.client.data.ClickHouseCityHash; +import com.clickhouse.data.ClickHouseByteUtils; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseCityHash; +import com.clickhouse.data.ClickHousePassThruStream; import net.jpountz.lz4.LZ4Compressor; import net.jpountz.lz4.LZ4Factory; @@ -25,11 +25,11 @@ protected void flushBuffer() throws IOException { block[16] = Lz4InputStream.MAGIC; int compressed = compressor.compress(buffer, 0, position, block, 25); int compressedSizeWithHeader = compressed + 9; - BinaryStreamUtils.setInt32(block, 17, compressedSizeWithHeader); // compressed size with header - BinaryStreamUtils.setInt32(block, 21, position); // uncompressed size + ClickHouseByteUtils.setInt32LE(block, 17, compressedSizeWithHeader); // compressed size with header + ClickHouseByteUtils.setInt32LE(block, 21, position); // uncompressed size long[] hash = ClickHouseCityHash.cityHash128(block, 16, compressedSizeWithHeader); - BinaryStreamUtils.setInt64(block, 0, hash[0]); - BinaryStreamUtils.setInt64(block, 8, hash[1]); + ClickHouseByteUtils.setInt64LE(block, 0, hash[0]); + ClickHouseByteUtils.setInt64LE(block, 8, hash[1]); output.write(block, 0, compressed + 25); position = 0; } @@ -42,11 +42,11 @@ protected void flushBuffer(byte[] bytes, int offset, int length) throws IOExcept int compressed = compressor.compress(bytes, offset, length, block, 25); int compressedSizeWithHeader = compressed + 9; - BinaryStreamUtils.setInt32(block, 17, compressedSizeWithHeader); - BinaryStreamUtils.setInt32(block, 21, length); + ClickHouseByteUtils.setInt32LE(block, 17, compressedSizeWithHeader); + ClickHouseByteUtils.setInt32LE(block, 21, length); long[] hash = ClickHouseCityHash.cityHash128(block, 16, compressedSizeWithHeader); - BinaryStreamUtils.setInt64(block, 0, hash[0]); - BinaryStreamUtils.setInt64(block, 8, hash[1]); + ClickHouseByteUtils.setInt64LE(block, 0, hash[0]); + ClickHouseByteUtils.setInt64LE(block, 8, hash[1]); output.write(block, 0, compressed + 25); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/NonBlockingInputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/NonBlockingInputStream.java similarity index 92% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/NonBlockingInputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/NonBlockingInputStream.java index dccb1ca22..86de8bd18 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/NonBlockingInputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/NonBlockingInputStream.java @@ -1,21 +1,17 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.EOFException; import java.io.IOException; import java.util.LinkedList; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseDataUpdater; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseDataUpdater; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseUtils; public class NonBlockingInputStream extends ClickHouseInputStream { - private static final Logger log = LoggerFactory.getLogger(NonBlockingInputStream.class); - private final AdaptiveQueue queue; private final int timeout; @@ -34,10 +30,6 @@ public NonBlockingInputStream(AdaptiveQueue queue, int timeout, Runnable @Override protected void ensureOpen() throws IOException { - if (closed) { - log.debug("Blocking input stream(queue: %d, buffer: %d) has been closed", - queue.size(), buffer != null ? buffer.length - position : 0); - } super.ensureOpen(); if (buffer == null || (buffer != ClickHouseByteBuffer.EMPTY_BYTES && position >= buffer.length)) { diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/NonBlockingPipedOutputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/NonBlockingPipedOutputStream.java similarity index 89% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/NonBlockingPipedOutputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/NonBlockingPipedOutputStream.java index c468d9ae6..738730ce9 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/NonBlockingPipedOutputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/NonBlockingPipedOutputStream.java @@ -1,14 +1,14 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseDataUpdater; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHousePipedOutputStream; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseDataUpdater; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePipedOutputStream; +import com.clickhouse.data.ClickHouseUtils; /** * A combination of {@link java.io.PipedOutputStream} and @@ -85,9 +85,7 @@ public NonBlockingPipedOutputStream(int bufferSize, int queueLength, int timeout this.queue = AdaptiveQueue.create(policy); // may need an initialBufferSize and a monitor to update bufferSize in runtime - this.bufferSize = ClickHouseUtils.getBufferSize(bufferSize, - (int) ClickHouseClientOption.BUFFER_SIZE.getDefaultValue(), - (int) ClickHouseClientOption.MAX_BUFFER_SIZE.getDefaultValue()); + this.bufferSize = ClickHouseDataConfig.getBufferSize(bufferSize); this.timeout = timeout; this.buckets = queueLength < 2 ? new byte[0][] : new byte[queueLength][]; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/RestrictedInputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/RestrictedInputStream.java similarity index 85% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/RestrictedInputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/RestrictedInputStream.java index 24f61a92b..502ca0531 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/RestrictedInputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/RestrictedInputStream.java @@ -1,16 +1,15 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; import java.io.InputStream; import java.util.LinkedList; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseDataUpdater; -import com.clickhouse.client.ClickHouseFile; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseDataUpdater; +import com.clickhouse.data.ClickHouseFile; +import com.clickhouse.data.ClickHouseOutputStream; /** * Wrapper of {@link java.io.InputStream} with length limitation. Unlike @@ -58,9 +57,7 @@ public RestrictedInputStream(ClickHouseFile file, InputStream input, int bufferS this.in = ClickHouseChecker.nonNull(input, "InputStream"); // fixed buffer - this.buffer = new byte[ClickHouseUtils.getBufferSize(bufferSize, - (int) ClickHouseClientOption.BUFFER_SIZE.getDefaultValue(), - (int) ClickHouseClientOption.MAX_BUFFER_SIZE.getDefaultValue())]; + this.buffer = new byte[ClickHouseDataConfig.getBufferSize(bufferSize)]; this.length = ClickHouseChecker.notLessThan(length, "Length", 0L); diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/WrappedInputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/WrappedInputStream.java similarity index 84% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/WrappedInputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/WrappedInputStream.java index 21376a4b5..ee7af6a23 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/WrappedInputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/WrappedInputStream.java @@ -1,16 +1,15 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; import java.io.InputStream; import java.util.LinkedList; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseDataUpdater; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHousePassThruStream; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseDataUpdater; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePassThruStream; /** * Wrapper of {@link java.io.InputStream} with buffer. @@ -50,9 +49,7 @@ public WrappedInputStream(ClickHousePassThruStream stream, InputStream input, in in = ClickHouseChecker.nonNull(input, "InputStream"); // fixed buffer - buffer = new byte[ClickHouseUtils.getBufferSize(bufferSize, - (int) ClickHouseClientOption.BUFFER_SIZE.getDefaultValue(), - (int) ClickHouseClientOption.MAX_BUFFER_SIZE.getDefaultValue())]; + buffer = new byte[ClickHouseDataConfig.getBufferSize(bufferSize)]; position = 0; limit = 0; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/stream/WrappedOutputStream.java b/clickhouse-data/src/main/java/com/clickhouse/data/stream/WrappedOutputStream.java similarity index 68% rename from clickhouse-client/src/main/java/com/clickhouse/client/stream/WrappedOutputStream.java rename to clickhouse-data/src/main/java/com/clickhouse/data/stream/WrappedOutputStream.java index 726641b2f..8e1c577e1 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/stream/WrappedOutputStream.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/stream/WrappedOutputStream.java @@ -1,12 +1,11 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; import java.io.OutputStream; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHousePassThruStream; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHousePassThruStream; /** * Wrapper of {@link java.io.OutputStream}. @@ -21,9 +20,7 @@ protected void flushBuffer(byte[] bytes, int offset, int length) throws IOExcept public WrappedOutputStream(ClickHousePassThruStream stream, OutputStream out, int bufferSize, Runnable postCloseAction) { - super(stream, ClickHouseUtils.getBufferSize(bufferSize, - (int) ClickHouseClientOption.BUFFER_SIZE.getDefaultValue(), - (int) ClickHouseClientOption.MAX_BUFFER_SIZE.getDefaultValue()), postCloseAction); + super(stream, ClickHouseDataConfig.getBufferSize(bufferSize), postCloseAction); output = ClickHouseChecker.nonNull(out, "OutputStream"); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseArrayValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseArrayValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseArrayValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseArrayValue.java index 9d9480fc2..de3554e7c 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseArrayValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseArrayValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.lang.reflect.Array; import java.math.BigDecimal; @@ -17,10 +17,10 @@ import java.util.Map; import java.util.UUID; import java.util.Map.Entry; -import com.clickhouse.client.ClickHouseArraySequence; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseArraySequence; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of Array. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBigDecimalValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseBigDecimalValue.java similarity index 96% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBigDecimalValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseBigDecimalValue.java index 7d5cc70a5..f1da4f4f0 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBigDecimalValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseBigDecimalValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -9,8 +9,10 @@ import java.time.LocalTime; import java.time.ZoneOffset; import java.util.UUID; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; + +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@link BigDecimal}. @@ -123,7 +125,7 @@ public BigDecimal asBigDecimal() { public BigDecimal asBigDecimal(int scale) { BigDecimal v = getValue(); if (v != null && v.scale() != scale) { - v = v.setScale(scale, ClickHouseValues.ROUNDING_MODE); + v = v.setScale(scale, ClickHouseDataConfig.DEFAULT_ROUNDING_MODE); } return v; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBigIntegerValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseBigIntegerValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBigIntegerValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseBigIntegerValue.java index 44ba7c500..d36a9f31e 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBigIntegerValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseBigIntegerValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -9,8 +9,8 @@ import java.time.LocalTime; import java.time.ZoneOffset; import java.util.UUID; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@link BigInteger}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBitmap.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseBitmap.java similarity index 97% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBitmap.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseBitmap.java index 1e5dd0247..5eb578850 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBitmap.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseBitmap.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -16,7 +16,9 @@ import org.roaringbitmap.buffer.MutableRoaringBitmap; import org.roaringbitmap.longlong.Roaring64Bitmap; import org.roaringbitmap.longlong.Roaring64NavigableMap; -import com.clickhouse.client.ClickHouseDataType; + +import com.clickhouse.data.ClickHouseByteUtils; +import com.clickhouse.data.ClickHouseDataType; public abstract class ClickHouseBitmap { private static final int[] EMPTY_INT_ARRAY = new int[0]; @@ -353,7 +355,7 @@ public static ClickHouseBitmap deserialize(DataInputStream in, ClickHouseDataTyp rb = ClickHouseBitmap.deserialize(bytes, innerType); } else { - int len = BinaryStreamUtils.readVarInt(in); + int len = ClickHouseByteUtils.readVarInt(in); byte[] bytes = new byte[len]; if (byteLen <= 4) { @@ -423,7 +425,7 @@ public static ClickHouseBitmap deserialize(byte[] bytes, ClickHouseDataType inne rb = ClickHouseBitmap.wrap(values); } } else { // serialized bitmap - int len = BinaryStreamUtils.readVarInt(buffer); + int len = ClickHouseByteUtils.getVarInt(buffer); if (buffer.remaining() < len) { throw new IllegalStateException( "Need " + len + " bytes to deserialize ClickHouseBitmap but only got " + buffer.remaining()); @@ -586,11 +588,11 @@ public ByteBuffer toByteBuffer() { } } else if (byteLen <= 4) { int size = serializedSizeInBytes(); - int varIntSize = BinaryStreamUtils.getVarIntSize(size); + int varIntSize = ClickHouseByteUtils.getVarIntSize(size); buf = newBuffer(1 + varIntSize + size); buf.put((byte) 1); - BinaryStreamUtils.writeVarInt(buf, size); + ClickHouseByteUtils.setVarInt(buf, size); serialize(buf); } else { // 64 // 1) deduct one to exclude the leading byte - boolean flag, see below: @@ -599,12 +601,12 @@ public ByteBuffer toByteBuffer() { // while Java uses int - see // https://github.com/RoaringBitmap/CRoaring/blob/v0.2.66/cpp/roaring64map.hh#L597 long size = serializedSizeInBytesAsLong() - 1 + 4; - int varIntSize = BinaryStreamUtils.getVarLongSize(size); + int varIntSize = ClickHouseByteUtils.getVarLongSize(size); // TODO add serialize(DataOutput) to handle more int intSize = (int) size; buf = newBuffer(1 + varIntSize + intSize); buf.put((byte) 1); - BinaryStreamUtils.writeVarInt(buf, intSize); + ClickHouseByteUtils.setVarInt(buf, intSize); serialize(buf); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBitmapValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseBitmapValue.java similarity index 97% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBitmapValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseBitmapValue.java index bf153d65c..aa96f54fe 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBitmapValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseBitmapValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -10,10 +10,10 @@ import java.time.ZoneOffset; import java.util.UUID; -import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@code Bitmap}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBoolValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseBoolValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBoolValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseBoolValue.java index 5fe61f6b8..95eec25eb 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseBoolValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseBoolValue.java @@ -1,9 +1,9 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@code bool}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseByteValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseByteValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseByteValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseByteValue.java index 6982c89d1..1f08eca6c 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseByteValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseByteValue.java @@ -1,10 +1,10 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@code byte}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDateTimeValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseDateTimeValue.java similarity index 96% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDateTimeValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseDateTimeValue.java index 7a18c3f0f..68e5052aa 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDateTimeValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseDateTimeValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -11,9 +11,10 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.TimeZone; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@link LocalDateTime}. @@ -178,7 +179,7 @@ public BigDecimal asBigDecimal(int scale) { v = new BigDecimal(BigInteger.valueOf(value.atZone(tz.toZoneId()).toEpochSecond()), scale); if (scale != 0 && nanoSeconds != 0) { v = v.add(BigDecimal.valueOf(nanoSeconds).divide(ClickHouseValues.NANOS).setScale(scale, - ClickHouseValues.ROUNDING_MODE)); + ClickHouseDataConfig.DEFAULT_ROUNDING_MODE)); } } return v; @@ -297,7 +298,7 @@ public ClickHouseDateTimeValue update(BigDecimal value) { resetToNullOrEmpty(); } else { if (value.scale() != scale) { - value = value.setScale(scale, ClickHouseValues.ROUNDING_MODE); + value = value.setScale(scale, ClickHouseDataConfig.DEFAULT_ROUNDING_MODE); } set(ClickHouseValues.convertToDateTime(value)); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDateValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseDateValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDateValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseDateValue.java index 3f6973667..c96f1470e 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDateValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseDateValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -6,8 +6,8 @@ import java.time.LocalDateTime; import java.time.LocalTime; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@link LocalDate}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDoubleValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseDoubleValue.java similarity index 96% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDoubleValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseDoubleValue.java index 221a37030..03732730a 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseDoubleValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseDoubleValue.java @@ -1,9 +1,11 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; + +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@code double}. @@ -151,7 +153,7 @@ public BigDecimal asBigDecimal(int scale) { if (diff > 0) { dec = dec.divide(BigDecimal.TEN.pow(diff + 1)); } else if (diff < 0) { - dec = dec.setScale(scale, ClickHouseValues.ROUNDING_MODE); + dec = dec.setScale(scale, ClickHouseDataConfig.DEFAULT_ROUNDING_MODE); } } return dec; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseEmptyValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseEmptyValue.java similarity index 95% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseEmptyValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseEmptyValue.java index f351db01f..091806c2f 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseEmptyValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseEmptyValue.java @@ -1,10 +1,10 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of Nothing. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseEnumValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseEnumValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseEnumValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseEnumValue.java index 693b78840..1098c2adc 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseEnumValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseEnumValue.java @@ -1,10 +1,10 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; -import com.clickhouse.client.ClickHouseEnum; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseEnum; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@code enum}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseFloatValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseFloatValue.java similarity index 96% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseFloatValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseFloatValue.java index 861666e1b..7dc98c028 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseFloatValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseFloatValue.java @@ -1,9 +1,11 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; + +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@code float}. @@ -170,7 +172,7 @@ public BigDecimal asBigDecimal(int scale) { if (diff > 0) { dec = dec.divide(BigDecimal.TEN.pow(diff + 1)); } else if (diff < 0) { - dec = dec.setScale(scale, ClickHouseValues.ROUNDING_MODE); + dec = dec.setScale(scale, ClickHouseDataConfig.DEFAULT_ROUNDING_MODE); } } return dec; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseGeoMultiPolygonValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseGeoMultiPolygonValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseGeoMultiPolygonValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseGeoMultiPolygonValue.java index 903f37cf9..ca1a55057 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseGeoMultiPolygonValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseGeoMultiPolygonValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.lang.reflect.Array; import java.math.BigDecimal; @@ -15,9 +15,9 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.UUID; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@code MultiPolygon}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseGeoPointValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseGeoPointValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseGeoPointValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseGeoPointValue.java index 0c7afc24f..3383c2542 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseGeoPointValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseGeoPointValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -13,8 +13,8 @@ import java.util.Iterator; import java.util.Map; import java.util.UUID; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@code Point}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseGeoPolygonValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseGeoPolygonValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseGeoPolygonValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseGeoPolygonValue.java index 0a1381297..7a9788c58 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseGeoPolygonValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseGeoPolygonValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.lang.reflect.Array; import java.math.BigDecimal; @@ -15,9 +15,9 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.UUID; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@code Polygon}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseGeoRingValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseGeoRingValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseGeoRingValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseGeoRingValue.java index 9fffff47b..b5493f2b2 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseGeoRingValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseGeoRingValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.lang.reflect.Array; import java.math.BigDecimal; @@ -15,9 +15,9 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.UUID; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@code Ring}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseInstantValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseInstantValue.java similarity index 96% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseInstantValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseInstantValue.java index d6e7201d3..fb759a780 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseInstantValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseInstantValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -9,9 +9,10 @@ import java.time.OffsetDateTime; import java.time.ZonedDateTime; import java.util.TimeZone; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@link Instant}. @@ -168,7 +169,7 @@ public BigDecimal asBigDecimal(int scale) { v = new BigDecimal(BigInteger.valueOf(value.getEpochSecond()), scale); if (scale != 0 && nanoSeconds != 0) { v = v.add(BigDecimal.valueOf(nanoSeconds).divide(ClickHouseValues.NANOS).setScale(scale, - ClickHouseValues.ROUNDING_MODE)); + ClickHouseDataConfig.DEFAULT_ROUNDING_MODE)); } } return v; @@ -293,7 +294,7 @@ public ClickHouseInstantValue update(BigDecimal value) { resetToNullOrEmpty(); } else { if (value.scale() != scale) { - value = value.setScale(scale, ClickHouseValues.ROUNDING_MODE); + value = value.setScale(scale, ClickHouseDataConfig.DEFAULT_ROUNDING_MODE); } set(ClickHouseValues.convertToInstant(value)); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseIntegerValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseIntegerValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseIntegerValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseIntegerValue.java index 6aca8fcd9..af36479de 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseIntegerValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseIntegerValue.java @@ -1,9 +1,9 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@code int}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseIpv4Value.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseIpv4Value.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseIpv4Value.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseIpv4Value.java index 399df2a74..3a41c5f65 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseIpv4Value.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseIpv4Value.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -11,8 +11,8 @@ import java.time.LocalTime; import java.time.ZoneOffset; import java.util.UUID; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@link Inet4Address}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseIpv6Value.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseIpv6Value.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseIpv6Value.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseIpv6Value.java index 9902bf7fc..b151a29e9 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseIpv6Value.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseIpv6Value.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -11,8 +11,8 @@ import java.time.LocalTime; import java.time.ZoneOffset; import java.util.UUID; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@link Inet6Address}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseLongValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseLongValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseLongValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseLongValue.java index 1ce7cac41..a79ffc1c3 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseLongValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseLongValue.java @@ -1,9 +1,9 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@code long}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseMapValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseMapValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseMapValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseMapValue.java index 3ab95bbce..9b80e616b 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseMapValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseMapValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -14,9 +14,9 @@ import java.util.UUID; import java.util.Map.Entry; import java.util.function.Function; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; public class ClickHouseMapValue extends ClickHouseObjectValue> { private static final String DEFAULT_STRING_KEY = "1"; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseNestedValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseNestedValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseNestedValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseNestedValue.java index cec490eb8..a8b67b0dd 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseNestedValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseNestedValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.lang.reflect.Array; import java.math.BigDecimal; @@ -17,10 +17,10 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of Nested. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseObjectValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseObjectValue.java similarity index 96% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseObjectValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseObjectValue.java index d39d5ba97..42d9ccaef 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseObjectValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseObjectValue.java @@ -1,10 +1,10 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; import java.util.Objects; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; public abstract class ClickHouseObjectValue implements ClickHouseValue { // a nested structure like Map might not be always serializable diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseOffsetDateTimeValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseOffsetDateTimeValue.java similarity index 96% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseOffsetDateTimeValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseOffsetDateTimeValue.java index 6fb1efe1b..68697dfd1 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseOffsetDateTimeValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseOffsetDateTimeValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -10,9 +10,10 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.TimeZone; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@link OffsetDateTime}. @@ -154,7 +155,7 @@ public BigDecimal asBigDecimal(int scale) { v = new BigDecimal(BigInteger.valueOf(value.toEpochSecond()), scale); if (scale != 0 && nanoSeconds != 0) { v = v.add(BigDecimal.valueOf(nanoSeconds).divide(ClickHouseValues.NANOS).setScale(scale, - ClickHouseValues.ROUNDING_MODE)); + ClickHouseDataConfig.DEFAULT_ROUNDING_MODE)); } } return v; @@ -284,7 +285,7 @@ public ClickHouseOffsetDateTimeValue update(BigDecimal value) { resetToNullOrEmpty(); } else { if (value.scale() != scale) { - value = value.setScale(scale, ClickHouseValues.ROUNDING_MODE); + value = value.setScale(scale, ClickHouseDataConfig.DEFAULT_ROUNDING_MODE); } set(ClickHouseValues.convertToDateTime(value, tz).toOffsetDateTime()); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseShortValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseShortValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseShortValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseShortValue.java index f45ab92d9..432db17ff 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseShortValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseShortValue.java @@ -1,10 +1,10 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@code short}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseStringValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseStringValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseStringValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseStringValue.java index a33c34dda..18c2b36e8 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseStringValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseStringValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -13,9 +13,9 @@ import java.util.Objects; import java.util.UUID; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@link String}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseTupleValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseTupleValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseTupleValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseTupleValue.java index 3ac18332d..7022527b3 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseTupleValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseTupleValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.lang.reflect.Array; import java.math.BigDecimal; @@ -19,9 +19,9 @@ import java.util.Map; import java.util.UUID; import java.util.Map.Entry; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of Tuple. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseUuidValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseUuidValue.java similarity index 98% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseUuidValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseUuidValue.java index 454b03f57..f440186b1 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseUuidValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/ClickHouseUuidValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -9,8 +9,8 @@ import java.time.LocalTime; import java.time.ZoneOffset; import java.util.UUID; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; /** * Wrapper class of {@link UUID}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/UnsignedByte.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/UnsignedByte.java similarity index 99% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/UnsignedByte.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/UnsignedByte.java index e47e2ba09..92b69af53 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/UnsignedByte.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/UnsignedByte.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.util.Locale; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/UnsignedInteger.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/UnsignedInteger.java similarity index 99% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/UnsignedInteger.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/UnsignedInteger.java index 3539d3dd7..695841cdb 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/UnsignedInteger.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/UnsignedInteger.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; /** * A wrapper class for unsigned {@code int}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/UnsignedLong.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/UnsignedLong.java similarity index 99% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/UnsignedLong.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/UnsignedLong.java index 775efb4ae..48a21abdb 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/UnsignedLong.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/UnsignedLong.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigInteger; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/UnsignedShort.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/UnsignedShort.java similarity index 99% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/UnsignedShort.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/UnsignedShort.java index cdbc551ac..b2c157343 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/UnsignedShort.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/UnsignedShort.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.util.Locale; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseBoolArrayValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseBoolArrayValue.java similarity index 97% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseBoolArrayValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseBoolArrayValue.java index e9938321d..202b3fa1d 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseBoolArrayValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseBoolArrayValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data.array; +package com.clickhouse.data.value.array; import java.math.BigDecimal; import java.math.BigInteger; @@ -17,12 +17,12 @@ import java.util.Map; import java.util.UUID; import java.util.Map.Entry; -import com.clickhouse.client.ClickHouseArraySequence; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; -import com.clickhouse.client.data.ClickHouseObjectValue; +import com.clickhouse.data.ClickHouseArraySequence; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.value.ClickHouseObjectValue; /** * Wrapper of {@code boolean[]}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseByteArrayValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseByteArrayValue.java similarity index 97% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseByteArrayValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseByteArrayValue.java index abbf26a03..fd3f3217e 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseByteArrayValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseByteArrayValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data.array; +package com.clickhouse.data.value.array; import java.math.BigDecimal; import java.math.BigInteger; @@ -17,13 +17,13 @@ import java.util.Map; import java.util.UUID; import java.util.Map.Entry; -import com.clickhouse.client.ClickHouseArraySequence; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; -import com.clickhouse.client.data.ClickHouseObjectValue; -import com.clickhouse.client.data.UnsignedByte; +import com.clickhouse.data.ClickHouseArraySequence; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.value.ClickHouseObjectValue; +import com.clickhouse.data.value.UnsignedByte; /** * Wrapper of {@code byte[]}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseDoubleArrayValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseDoubleArrayValue.java similarity index 97% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseDoubleArrayValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseDoubleArrayValue.java index 2aca19bc0..9b9f00db5 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseDoubleArrayValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseDoubleArrayValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data.array; +package com.clickhouse.data.value.array; import java.math.BigDecimal; import java.math.BigInteger; @@ -17,12 +17,12 @@ import java.util.Map; import java.util.UUID; import java.util.Map.Entry; -import com.clickhouse.client.ClickHouseArraySequence; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; -import com.clickhouse.client.data.ClickHouseObjectValue; +import com.clickhouse.data.ClickHouseArraySequence; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.value.ClickHouseObjectValue; /** * Wrapper of {@code double[]}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseFloatArrayValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseFloatArrayValue.java similarity index 97% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseFloatArrayValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseFloatArrayValue.java index e3dceeea3..5b26a4657 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseFloatArrayValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseFloatArrayValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data.array; +package com.clickhouse.data.value.array; import java.math.BigDecimal; import java.math.BigInteger; @@ -17,12 +17,12 @@ import java.util.Map; import java.util.UUID; import java.util.Map.Entry; -import com.clickhouse.client.ClickHouseArraySequence; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; -import com.clickhouse.client.data.ClickHouseObjectValue; +import com.clickhouse.data.ClickHouseArraySequence; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.value.ClickHouseObjectValue; /** * Wrapper of {@code float[]}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseIntArrayValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseIntArrayValue.java similarity index 97% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseIntArrayValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseIntArrayValue.java index 454d978a6..8dfe7019c 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseIntArrayValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseIntArrayValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data.array; +package com.clickhouse.data.value.array; import java.math.BigDecimal; import java.math.BigInteger; @@ -17,13 +17,13 @@ import java.util.Map; import java.util.UUID; import java.util.Map.Entry; -import com.clickhouse.client.ClickHouseArraySequence; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; -import com.clickhouse.client.data.ClickHouseObjectValue; -import com.clickhouse.client.data.UnsignedInteger; +import com.clickhouse.data.ClickHouseArraySequence; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.value.ClickHouseObjectValue; +import com.clickhouse.data.value.UnsignedInteger; /** * Wrapper of {@code int[]}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseLongArrayValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseLongArrayValue.java similarity index 97% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseLongArrayValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseLongArrayValue.java index 4aeac5009..6b1c5193c 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseLongArrayValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseLongArrayValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data.array; +package com.clickhouse.data.value.array; import java.math.BigDecimal; import java.math.BigInteger; @@ -17,13 +17,13 @@ import java.util.Map; import java.util.UUID; import java.util.Map.Entry; -import com.clickhouse.client.ClickHouseArraySequence; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; -import com.clickhouse.client.data.ClickHouseObjectValue; -import com.clickhouse.client.data.UnsignedLong; +import com.clickhouse.data.ClickHouseArraySequence; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.value.ClickHouseObjectValue; +import com.clickhouse.data.value.UnsignedLong; /** * Wrapper of {@code long[]}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseShortArrayValue.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseShortArrayValue.java similarity index 97% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseShortArrayValue.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseShortArrayValue.java index 7e3896a49..a0d8ce122 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseShortArrayValue.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseShortArrayValue.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data.array; +package com.clickhouse.data.value.array; import java.math.BigDecimal; import java.math.BigInteger; @@ -17,13 +17,13 @@ import java.util.Map; import java.util.UUID; import java.util.Map.Entry; -import com.clickhouse.client.ClickHouseArraySequence; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; -import com.clickhouse.client.data.ClickHouseObjectValue; -import com.clickhouse.client.data.UnsignedShort; +import com.clickhouse.data.ClickHouseArraySequence; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.value.ClickHouseObjectValue; +import com.clickhouse.data.value.UnsignedShort; /** * Wrapper of {@code short[]}. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/package-info.java b/clickhouse-data/src/main/java/com/clickhouse/data/value/package-info.java similarity index 69% rename from clickhouse-client/src/main/java/com/clickhouse/client/data/package-info.java rename to clickhouse-data/src/main/java/com/clickhouse/data/value/package-info.java index 1621d43a4..1c1d7e0e2 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/package-info.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/value/package-info.java @@ -1,4 +1,4 @@ /** * Provides necessary classes to handle different format or type of data. */ -package com.clickhouse.client.data; +package com.clickhouse.data.value; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/logging/JdkLogger.java b/clickhouse-data/src/main/java/com/clickhouse/logging/JdkLogger.java similarity index 85% rename from clickhouse-client/src/main/java/com/clickhouse/client/logging/JdkLogger.java rename to clickhouse-data/src/main/java/com/clickhouse/logging/JdkLogger.java index 9e2d6c87f..ad65e1553 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/logging/JdkLogger.java +++ b/clickhouse-data/src/main/java/com/clickhouse/logging/JdkLogger.java @@ -1,9 +1,8 @@ -package com.clickhouse.client.logging; +package com.clickhouse.logging; import java.util.ResourceBundle; import java.util.function.Supplier; import java.util.logging.Level; -import com.clickhouse.client.ClickHouseChecker; /** * Adaptor for JDK logger. @@ -20,13 +19,41 @@ protected void log(Level level, LogMessage msg) { } } + @Override + public boolean isDebugEnabled() { + return logger.isLoggable(Level.FINE); + } + + @Override + public boolean isErrorEnabled() { + return logger.isLoggable(Level.SEVERE); + } + + @Override + public boolean isInfoEnabled() { + return logger.isLoggable(Level.INFO); + } + + @Override + public boolean isWarnEnabled() { + return logger.isLoggable(Level.WARNING); + } + + @Override + public boolean isTraceEnabled() { + return logger.isLoggable(Level.WARNING); + } + /** * Default constructor. * * @param logger non-null JDK logger */ public JdkLogger(java.util.logging.Logger logger) { - this.logger = ClickHouseChecker.nonNull(logger, "logger"); + if (logger == null) { + throw new IllegalArgumentException(ERROR_NULL_LOGGER); + } + this.logger = logger; } @Override diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/logging/JdkLoggerFactory.java b/clickhouse-data/src/main/java/com/clickhouse/logging/JdkLoggerFactory.java similarity index 85% rename from clickhouse-client/src/main/java/com/clickhouse/client/logging/JdkLoggerFactory.java rename to clickhouse-data/src/main/java/com/clickhouse/logging/JdkLoggerFactory.java index 715685e16..893ea784b 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/logging/JdkLoggerFactory.java +++ b/clickhouse-data/src/main/java/com/clickhouse/logging/JdkLoggerFactory.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.logging; +package com.clickhouse.logging; /** * Adaptor of JDK logger factory. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/logging/LogMessage.java b/clickhouse-data/src/main/java/com/clickhouse/logging/LogMessage.java similarity index 97% rename from clickhouse-client/src/main/java/com/clickhouse/client/logging/LogMessage.java rename to clickhouse-data/src/main/java/com/clickhouse/logging/LogMessage.java index d2ef9ad34..df9694e38 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/logging/LogMessage.java +++ b/clickhouse-data/src/main/java/com/clickhouse/logging/LogMessage.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.logging; +package com.clickhouse.logging; import java.util.Locale; diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/logging/Logger.java b/clickhouse-data/src/main/java/com/clickhouse/logging/Logger.java similarity index 81% rename from clickhouse-client/src/main/java/com/clickhouse/client/logging/Logger.java rename to clickhouse-data/src/main/java/com/clickhouse/logging/Logger.java index 7074f7d73..cae0c02c8 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/logging/Logger.java +++ b/clickhouse-data/src/main/java/com/clickhouse/logging/Logger.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.logging; +package com.clickhouse.logging; import java.util.function.Supplier; @@ -7,6 +7,46 @@ * {@link java.util.Formatter}. */ public interface Logger { + /** + * Error message for providing null logger. + */ + static final String ERROR_NULL_LOGGER = "Non-null logger is required"; + + /** + * Checks if logging level is {@code DEBUG} or above. + * + * @return true if DEBUG level is enabled; false otherwise + */ + boolean isDebugEnabled(); + + /** + * Checks if logging level is {@code ERROR} or above. + * + * @return true if ERROR level is enabled; false otherwise + */ + boolean isErrorEnabled(); + + /** + * Checks if logging level is {@code INFO} or above. + * + * @return true if INFO level is enabled; false otherwise + */ + boolean isInfoEnabled(); + + /** + * Checks if logging level is {@code WARN} or above. + * + * @return true if WARN level is enabled; false otherwise + */ + boolean isWarnEnabled(); + + /** + * Checks if logging level is {@code TRACE} or above. + * + * @return true if TRACE level is enabled; false otherwise + */ + boolean isTraceEnabled(); + /** * Logs output of a custom function at the DEBUG level. The function will only * run when log level is DEBUG or lower. diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/logging/LoggerFactory.java b/clickhouse-data/src/main/java/com/clickhouse/logging/LoggerFactory.java similarity index 62% rename from clickhouse-client/src/main/java/com/clickhouse/client/logging/LoggerFactory.java rename to clickhouse-data/src/main/java/com/clickhouse/logging/LoggerFactory.java index 21781a8ea..634be8cee 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/logging/LoggerFactory.java +++ b/clickhouse-data/src/main/java/com/clickhouse/logging/LoggerFactory.java @@ -1,29 +1,36 @@ -package com.clickhouse.client.logging; +package com.clickhouse.logging; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseUtils; +import java.util.ServiceLoader; /** * Unified factory class to get logger. */ -@SuppressWarnings("squid:S1181") public abstract class LoggerFactory { private static final LoggerFactory instance; static { - instance = ClickHouseUtils.getService(LoggerFactory.class, () -> { - LoggerFactory factory = null; + LoggerFactory factory = null; + for (LoggerFactory f : ServiceLoader.load(LoggerFactory.class, LoggerFactory.class.getClassLoader())) { + if (f != null) { + factory = f; + break; + } + } + if (factory == null) { try { if (org.slf4j.LoggerFactory.getILoggerFactory() != null) { - factory = new Slf4jLoggerFactory(); + factory = new Slf4jLoggerFactory(); // NOSONAR } - } catch (Throwable ignore) { // fall back to JDK - factory = new JdkLoggerFactory(); + } catch (Throwable ignore) { // NOSONAR + factory = new JdkLoggerFactory(); // NOSONAR } + } - return ClickHouseChecker.nonNull(factory, "factory"); - }); + if (factory == null) { + throw new IllegalArgumentException("No LoggerFactory found"); + } + instance = factory; } /** @@ -62,7 +69,10 @@ public static LoggerFactory getInstance() { * @return logger for the given class */ public Logger get(Class clazz) { - return get(ClickHouseChecker.nonNull(clazz, "Class").getName()); + if (clazz == null) { + throw new IllegalArgumentException("Non-null class required"); + } + return get(clazz.getName()); } /** diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/logging/Slf4jLogger.java b/clickhouse-data/src/main/java/com/clickhouse/logging/Slf4jLogger.java similarity index 86% rename from clickhouse-client/src/main/java/com/clickhouse/client/logging/Slf4jLogger.java rename to clickhouse-data/src/main/java/com/clickhouse/logging/Slf4jLogger.java index 6b974b32b..93c78934f 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/logging/Slf4jLogger.java +++ b/clickhouse-data/src/main/java/com/clickhouse/logging/Slf4jLogger.java @@ -1,9 +1,7 @@ -package com.clickhouse.client.logging; +package com.clickhouse.logging; import java.util.function.Supplier; -import com.clickhouse.client.ClickHouseChecker; - /** * Adaptor for slf4j logger. */ @@ -16,7 +14,35 @@ public class Slf4jLogger implements Logger { * @param logger non-null SLF4J logger */ public Slf4jLogger(org.slf4j.Logger logger) { - this.logger = ClickHouseChecker.nonNull(logger, "logger"); + if (logger == null) { + throw new IllegalArgumentException(ERROR_NULL_LOGGER); + } + this.logger = logger; + } + + @Override + public boolean isDebugEnabled() { + return logger.isDebugEnabled(); + } + + @Override + public boolean isErrorEnabled() { + return logger.isErrorEnabled(); + } + + @Override + public boolean isInfoEnabled() { + return logger.isInfoEnabled(); + } + + @Override + public boolean isWarnEnabled() { + return logger.isWarnEnabled(); + } + + @Override + public boolean isTraceEnabled() { + return logger.isTraceEnabled(); } @Override diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/logging/Slf4jLoggerFactory.java b/clickhouse-data/src/main/java/com/clickhouse/logging/Slf4jLoggerFactory.java similarity index 90% rename from clickhouse-client/src/main/java/com/clickhouse/client/logging/Slf4jLoggerFactory.java rename to clickhouse-data/src/main/java/com/clickhouse/logging/Slf4jLoggerFactory.java index 20cccda7f..d6176b0d8 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/logging/Slf4jLoggerFactory.java +++ b/clickhouse-data/src/main/java/com/clickhouse/logging/Slf4jLoggerFactory.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.logging; +package com.clickhouse.logging; /** * Adaptor of slf4j logger factory. diff --git a/clickhouse-data/src/main/java/com/clickhouse/logging/package-info.java b/clickhouse-data/src/main/java/com/clickhouse/logging/package-info.java new file mode 100644 index 000000000..327f9ada3 --- /dev/null +++ b/clickhouse-data/src/main/java/com/clickhouse/logging/package-info.java @@ -0,0 +1,4 @@ +/** + * Provides logging classes. + */ +package com.clickhouse.logging; diff --git a/clickhouse-data/src/main/java9/module-info.java b/clickhouse-data/src/main/java9/module-info.java new file mode 100644 index 000000000..2c18f613c --- /dev/null +++ b/clickhouse-data/src/main/java9/module-info.java @@ -0,0 +1,22 @@ +/** + * Declares com.clickhouse.data module. + */ +module com.clickhouse.data { + exports com.clickhouse.config; + exports com.clickhouse.data; + // exports com.clickhouse.data.cache; + // exports com.clickhouse.data.format; + // exports com.clickhouse.data.stream; + exports com.clickhouse.data.value; + exports com.clickhouse.logging; + + requires static java.logging; + requires static com.google.gson; + requires static com.github.benmanes.caffeine; + requires static org.lz4.java; + requires static org.slf4j; + requires static org.roaringbitmap; + + uses com.clickhouse.data.ClickHouseDataStreamFactory; + uses com.clickhouse.logging.LoggerFactory; +} diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/config/ClickHouseOptionTest.java b/clickhouse-data/src/test/java/com/clickhouse/config/ClickHouseOptionTest.java similarity index 97% rename from clickhouse-client/src/test/java/com/clickhouse/client/config/ClickHouseOptionTest.java rename to clickhouse-data/src/test/java/com/clickhouse/config/ClickHouseOptionTest.java index ceca349db..53280bcc2 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/config/ClickHouseOptionTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/config/ClickHouseOptionTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.config; +package com.clickhouse.config; import java.io.Serializable; import java.util.Collections; @@ -7,9 +7,9 @@ import org.testng.Assert; import org.testng.annotations.Test; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseFormat; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseFormat; public class ClickHouseOptionTest { static enum ClickHouseTestOption implements ClickHouseOption { @@ -75,7 +75,7 @@ public void testFromString() { Assert.assertEquals(ClickHouseOption.fromString(null, Integer.class), Integer.valueOf(0)); Assert.assertEquals(ClickHouseOption.fromString("", Integer.class), Integer.valueOf(0)); - Assert.assertEquals(ClickHouseOption.fromString(" ", 1), 1); + // Assert.assertEquals(ClickHouseOption.fromString(" ", 1), 1); Assert.assertEquals(ClickHouseOption.fromString("0", Integer.class), Integer.valueOf(0)); Assert.assertEquals(ClickHouseOption.fromString("0", int.class), 0); diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/config/ClickHouseRenameMethodTest.java b/clickhouse-data/src/test/java/com/clickhouse/config/ClickHouseRenameMethodTest.java similarity index 98% rename from clickhouse-client/src/test/java/com/clickhouse/client/config/ClickHouseRenameMethodTest.java rename to clickhouse-data/src/test/java/com/clickhouse/config/ClickHouseRenameMethodTest.java index d9a2d9403..db7d2182e 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/config/ClickHouseRenameMethodTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/config/ClickHouseRenameMethodTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.config; +package com.clickhouse.config; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/BaseClickHouseValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/BaseClickHouseValueTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/BaseClickHouseValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/BaseClickHouseValueTest.java index e6904acb4..c754c0a03 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/BaseClickHouseValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/BaseClickHouseValueTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.util.Arrays; import java.util.Collections; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseByteBufferTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseByteBufferTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseByteBufferTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseByteBufferTest.java index 2ec115db5..04804ce82 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseByteBufferTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseByteBufferTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.util.Arrays; import java.util.Collections; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseCheckerTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseCheckerTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseCheckerTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseCheckerTest.java index 4b25ab5a9..b4b24775d 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseCheckerTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseCheckerTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.math.BigInteger; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseColumnTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java similarity index 95% rename from clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseColumnTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java index 34839a184..99d805665 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseColumnTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.math.BigInteger; import java.util.Collections; @@ -9,8 +9,6 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import com.clickhouse.client.config.ClickHouseClientOption; - public class ClickHouseColumnTest { @DataProvider(name = "enumTypesProvider") private Object[][] getEnumTypes() { @@ -247,8 +245,12 @@ public void testSimpleAggregationFunction() { @Test(groups = { "unit" }) public void testNewArray() { - ClickHouseConfig config = new ClickHouseConfig( - Collections.singletonMap(ClickHouseClientOption.WIDEN_UNSIGNED_TYPES, true)); + ClickHouseDataConfig config = new ClickHouseTestDataConfig() { + @Override + public boolean isWidenUnsignedTypes() { + return true; + }; + }; ClickHouseValue v = ClickHouseColumn.of("a", "Array(UInt32)").newValue(config); Assert.assertEquals(v.update(new long[] { 1L }).asObject(), new long[] { 1L }); v = ClickHouseColumn.of("a", "Array(Nullable(UInt64))").newValue(config); @@ -263,7 +265,12 @@ public void testNewArray() { v = ClickHouseColumn.of("a", "Array(Array(Array(Array(LowCardinality(String)))))").newValue(config); Assert.assertEquals(v.asObject(), new String[0][][][]); - config = new ClickHouseConfig(Collections.singletonMap(ClickHouseClientOption.WIDEN_UNSIGNED_TYPES, false)); + config = new ClickHouseTestDataConfig() { + @Override + public boolean isWidenUnsignedTypes() { + return false; + }; + }; v = ClickHouseColumn.of("", "Array(UInt8)").newValue(config); Assert.assertEquals(v.update(new byte[] { Byte.MIN_VALUE, 0, Byte.MAX_VALUE }).asObject(), new byte[] { Byte.MIN_VALUE, 0, Byte.MAX_VALUE }); @@ -277,13 +284,20 @@ public void testNewArray() { v = ClickHouseColumn.of("", "Array(UInt64)").newValue(config); Assert.assertEquals(v.update(new int[] { -1, 0, 1 }).asObject(), new long[] { 4294967295L, 0, 1 }); Assert.assertEquals(v.update(new long[] { -1L, 0L, 1L }).asObject(), new long[] { -1L, 0L, 1L }); - Assert.assertEquals(v.update(new BigInteger[] { new BigInteger("18446744073709551615"), BigInteger.ZERO, BigInteger.ONE }).asObject(), new long[] { -1L, 0L, 1L }); + Assert.assertEquals( + v.update(new BigInteger[] { new BigInteger("18446744073709551615"), BigInteger.ZERO, BigInteger.ONE }) + .asObject(), + new long[] { -1L, 0L, 1L }); } @Test(groups = { "unit" }) public void testNewBasicValues() { - ClickHouseConfig config = new ClickHouseConfig( - Collections.singletonMap(ClickHouseClientOption.WIDEN_UNSIGNED_TYPES, true)); + ClickHouseDataConfig config = new ClickHouseTestDataConfig() { + @Override + public boolean isWidenUnsignedTypes() { + return true; + }; + }; for (ClickHouseDataType type : ClickHouseDataType.values()) { // skip advanced types if (type.isNested() || type == ClickHouseDataType.AggregateFunction diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseCompressionAlgorithmTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseCompressionAlgorithmTest.java new file mode 100644 index 000000000..f3205c94f --- /dev/null +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseCompressionAlgorithmTest.java @@ -0,0 +1,15 @@ +package com.clickhouse.data; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClickHouseCompressionAlgorithmTest { + @Test(groups = { "unit" }) + public void testCreateInstance() { + for (ClickHouseCompression c : ClickHouseCompression.values()) { + ClickHouseCompressionAlgorithm alg = ClickHouseCompressionAlgorithm.of(c); + Assert.assertNotNull(alg); + Assert.assertEquals(alg.getAlgorithm(), c); + } + } +} diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseDataConfigTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseDataConfigTest.java new file mode 100644 index 000000000..f46660140 --- /dev/null +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseDataConfigTest.java @@ -0,0 +1,26 @@ +package com.clickhouse.data; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClickHouseDataConfigTest { + @Test(groups = { "unit" }) + public void testGetBufferSize() { + int[] values = new int[] { -1, 0 }; + for (int i : values) { + for (int j : values) { + for (int k : values) { + Assert.assertEquals(ClickHouseDataConfig.getBufferSize(i, j, k), + ClickHouseDataConfig.DEFAULT_BUFFER_SIZE); + } + } + } + + for (int i : values) { + Assert.assertEquals(ClickHouseDataConfig.getBufferSize(i, 3, 2), 2); + } + Assert.assertEquals(ClickHouseDataConfig.getBufferSize(1, 3, 2), 1); + Assert.assertEquals(ClickHouseDataConfig.getBufferSize(3, 2, 1), 1); + Assert.assertEquals(ClickHouseDataConfig.getBufferSize(3, 1, 2), 2); + } +} diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseDataStreamFactoryTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseDataStreamFactoryTest.java similarity index 77% rename from clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseDataStreamFactoryTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseDataStreamFactoryTest.java index 600ed1677..f0ff86497 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseDataStreamFactoryTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseDataStreamFactoryTest.java @@ -1,6 +1,7 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -15,16 +16,18 @@ public void testGetInstance() { @Test(groups = { "unit" }) public void testCreatePipedOutputStream() throws ExecutionException, IOException, InterruptedException { - ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseDataConfig config = new ClickHouseTestDataConfig(); // read in worker thread for (int i = 0; i < 256; i++) { CompletableFuture future; try (ClickHousePipedOutputStream out = ClickHouseDataStreamFactory.getInstance() .createPipedOutputStream(config, null)) { - future = ClickHouseClient.submit(() -> { + future = CompletableFuture.supplyAsync(() -> { try (ClickHouseInputStream in = out.getInputStream()) { return in.read(); + } catch (IOException e) { + throw new UncheckedIOException(e); } }); out.write(i); @@ -38,9 +41,11 @@ public void testCreatePipedOutputStream() throws ExecutionException, IOException null); final int num = i; try (ClickHouseInputStream in = out.getInputStream()) { - ClickHouseClient.submit(() -> { + CompletableFuture.supplyAsync(() -> { try (ClickHouseOutputStream o = out) { o.write(num); + } catch (IOException e) { + throw new UncheckedIOException(e); } return num; }); diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseDataTypeTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseDataTypeTest.java similarity index 98% rename from clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseDataTypeTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseDataTypeTest.java index 7ccfb60b0..f91700086 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseDataTypeTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseDataTypeTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.util.List; import org.testng.Assert; diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseFileTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseFileTest.java new file mode 100644 index 000000000..94f2b5848 --- /dev/null +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseFileTest.java @@ -0,0 +1,71 @@ +package com.clickhouse.data; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.UUID; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClickHouseFileTest { + @Test(groups = { "unit" }) + public void testConstructor() { + final InputStream in = new ByteArrayInputStream(new byte[0]); + + String nonExistingFile = "/tmp/" + UUID.randomUUID().toString() + ".csv.lz4"; + ClickHouseFile file = ClickHouseFile.of(nonExistingFile); + Assert.assertFalse(file.hasInput()); + Assert.assertTrue(file.hasOutput()); + Assert.assertTrue(file.hasCompression()); + Assert.assertEquals(file.getCompressionAlgorithm(), ClickHouseCompression.LZ4); + Assert.assertEquals(file.getCompressionLevel(), -1); + Assert.assertTrue(file.hasFormat()); + Assert.assertEquals(file.getFormat(), ClickHouseFormat.CSV); + Assert.assertEquals(file.getFile().getAbsolutePath(), nonExistingFile); + + file = ClickHouseFile.of(in, null, 0, ClickHouseFormat.CSV); + Assert.assertTrue(file.hasInput()); + Assert.assertTrue(file.hasOutput()); + Assert.assertFalse(file.hasCompression()); + Assert.assertNull(file.getCompressionAlgorithm()); + Assert.assertEquals(file.getCompressionLevel(), 0); + Assert.assertTrue(file.hasFormat()); + Assert.assertEquals(file.getFormat(), ClickHouseFormat.CSV); + Assert.assertTrue(file.getFile().exists()); + + file = ClickHouseFile.of(ClickHouseInputStream.of(in, ClickHouseCompression.ZSTD), ClickHouseCompression.LZ4, + 2, null); + Assert.assertTrue(file.hasInput()); + Assert.assertTrue(file.hasOutput()); + Assert.assertTrue(file.hasCompression()); + Assert.assertEquals(file.getCompressionAlgorithm(), ClickHouseCompression.LZ4); + Assert.assertEquals(file.getCompressionLevel(), 2); + Assert.assertFalse(file.hasFormat()); + Assert.assertNull(file.getFormat()); + Assert.assertTrue(file.getFile().exists()); + + file = ClickHouseFile.of( + ClickHouseInputStream.of( + ClickHousePassThruStream.of(in, ClickHouseCompression.ZSTD, 9, ClickHouseFormat.TSV), 0, null), + ClickHouseCompression.LZ4, 3, ClickHouseFormat.CSV); + Assert.assertTrue(file.hasInput()); + Assert.assertTrue(file.hasOutput()); + Assert.assertTrue(file.hasCompression()); + Assert.assertEquals(file.getCompressionAlgorithm(), ClickHouseCompression.ZSTD); + Assert.assertEquals(file.getCompressionLevel(), 9); + Assert.assertTrue(file.hasFormat()); + Assert.assertEquals(file.getFormat(), ClickHouseFormat.TSV); + Assert.assertTrue(file.getFile().exists()); + + file = ClickHouseFile + .of(ClickHousePassThruStream.of(in, ClickHouseCompression.BROTLI, 7, ClickHouseFormat.RowBinary)); + Assert.assertTrue(file.hasInput()); + Assert.assertTrue(file.hasOutput()); + Assert.assertTrue(file.hasCompression()); + Assert.assertEquals(file.getCompressionAlgorithm(), ClickHouseCompression.BROTLI); + Assert.assertEquals(file.getCompressionLevel(), 7); + Assert.assertTrue(file.hasFormat()); + Assert.assertEquals(file.getFormat(), ClickHouseFormat.RowBinary); + Assert.assertTrue(file.getFile().exists()); + } +} diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseInputStreamTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseInputStreamTest.java similarity index 94% rename from clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseInputStreamTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseInputStreamTest.java index 0caa0eab6..90ea62237 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseInputStreamTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseInputStreamTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; @@ -302,4 +302,23 @@ public void testSkipInput() throws IOException { Assert.assertEquals(in.skip(2L), 1L); Assert.assertEquals(in.read(), -1); } + + @Test(groups = { "unit" }) + public void testUserData() throws IOException { + try (ClickHouseInputStream in = ClickHouseInputStream.of("")) { + Assert.assertEquals(in.userData.size(), 0); + Assert.assertNull(in.setUserData("context1", 1)); + Assert.assertEquals(in.userData.size(), 1); + Assert.assertNull(in.setUserData("context2", 2)); + Assert.assertEquals(in.userData.size(), 2); + Assert.assertNull(in.getUserData("context")); + Assert.assertEquals(in.getUserData("context", ""), ""); + Assert.assertEquals(in.getUserData("context1"), 1); + Assert.assertNull(in.removeUserData("context")); + Assert.assertEquals(in.removeUserData("context1"), 1); + Assert.assertEquals(in.userData.size(), 1); + in.close(); + Assert.assertEquals(in.userData.size(), 0); + } + } } diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseOutputStreamTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseOutputStreamTest.java similarity index 98% rename from clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseOutputStreamTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseOutputStreamTest.java index df8e65033..1c84553f7 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseOutputStreamTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseOutputStreamTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.BufferedOutputStream; import java.io.ByteArrayOutputStream; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHousePassThruStreamTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHousePassThruStreamTest.java similarity index 68% rename from clickhouse-client/src/test/java/com/clickhouse/client/ClickHousePassThruStreamTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/ClickHousePassThruStreamTest.java index 061b65d01..01e0c58fc 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHousePassThruStreamTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHousePassThruStreamTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -16,30 +16,33 @@ public void testConstructor() { ClickHousePassThruStream stream = ClickHousePassThruStream.of(in, null, -1, null); - Assert.assertNotEquals(stream.asInputStream(), ClickHouseInputStream.empty()); - Assert.assertEquals(stream.asOutputStream(), ClickHouseOutputStream.empty()); + Assert.assertNotEquals(stream.getInputStream(), ClickHouseInputStream.empty()); + Assert.assertEquals(stream.getOutputStream(), ClickHouseOutputStream.empty()); + Assert.assertFalse(stream.hasCompression()); Assert.assertTrue(stream.hasInput()); Assert.assertFalse(stream.hasOutput()); Assert.assertFalse(stream.isCompressed()); Assert.assertFalse(stream.hasFormat()); - Assert.assertEquals(stream.getCompressionAlgorithm(), ClickHouseCompression.NONE); + Assert.assertNull(stream.getCompressionAlgorithm()); Assert.assertEquals(stream.getCompressionLevel(), -1); Assert.assertNull(stream.getFormat()); stream = ClickHousePassThruStream.of(out, null, -1, null); - Assert.assertEquals(stream.asInputStream(), ClickHouseInputStream.empty()); - Assert.assertNotEquals(stream.asOutputStream(), ClickHouseOutputStream.empty()); + Assert.assertEquals(stream.getInputStream(), ClickHouseInputStream.empty()); + Assert.assertNotEquals(stream.getOutputStream(), ClickHouseOutputStream.empty()); + Assert.assertFalse(stream.hasCompression()); Assert.assertFalse(stream.hasInput()); Assert.assertTrue(stream.hasOutput()); Assert.assertFalse(stream.isCompressed()); Assert.assertFalse(stream.hasFormat()); - Assert.assertEquals(stream.getCompressionAlgorithm(), ClickHouseCompression.NONE); + Assert.assertNull(stream.getCompressionAlgorithm()); Assert.assertEquals(stream.getCompressionLevel(), -1); Assert.assertNull(stream.getFormat()); stream = ClickHousePassThruStream.of(in, out, ClickHouseCompression.ZSTD, 22, ClickHouseFormat.JSON); - Assert.assertNotEquals(stream.asInputStream(), ClickHouseInputStream.empty()); - Assert.assertNotEquals(stream.asOutputStream(), ClickHouseOutputStream.empty()); + Assert.assertNotEquals(stream.getInputStream(), ClickHouseInputStream.empty()); + Assert.assertNotEquals(stream.getOutputStream(), ClickHouseOutputStream.empty()); + Assert.assertTrue(stream.hasCompression()); Assert.assertTrue(stream.hasInput()); Assert.assertTrue(stream.hasOutput()); Assert.assertTrue(stream.isCompressed()); @@ -50,6 +53,6 @@ public void testConstructor() { Assert.assertTrue(ClickHousePassThruStream.of(null, null, null, -1, null) == ClickHousePassThruStream.NULL); Assert.assertTrue(ClickHousePassThruStream.of(null, null, ClickHouseCompression.NONE, -1, - null) == ClickHousePassThruStream.NULL); + null) != ClickHousePassThruStream.NULL); } } diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseTestDataConfig.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseTestDataConfig.java new file mode 100644 index 000000000..10fe01f0d --- /dev/null +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseTestDataConfig.java @@ -0,0 +1,104 @@ +package com.clickhouse.data; + +import java.util.TimeZone; + +import com.clickhouse.config.ClickHouseBufferingMode; +import com.clickhouse.config.ClickHouseRenameMethod; + +public class ClickHouseTestDataConfig implements ClickHouseDataConfig { + + @Override + public ClickHouseFormat getFormat() { + return ClickHouseFormat.TabSeparatedWithNamesAndTypes; + } + + @Override + public int getBufferQueueVariation() { + return 0; + } + + @Override + public int getBufferSize() { + return DEFAULT_BUFFER_SIZE; + } + + @Override + public int getMaxBufferSize() { + return DEFAULT_MAX_BUFFER_SIZE; + } + + @Override + public int getReadBufferSize() { + return DEFAULT_READ_BUFFER_SIZE; + } + + @Override + public int getWriteBufferSize() { + return DEFAULT_WRITE_BUFFER_SIZE; + } + + @Override + public int getReadTimeout() { + return 30000; + } + + @Override + public int getWriteTimeout() { + return 30000; + } + + @Override + public ClickHouseBufferingMode getReadBufferingMode() { + return ClickHouseBufferingMode.RESOURCE_EFFICIENT; + } + + @Override + public ClickHouseBufferingMode getWriteBufferingMode() { + return ClickHouseBufferingMode.RESOURCE_EFFICIENT; + } + + @Override + public ClickHouseRenameMethod getColumnRenameMethod() { + return ClickHouseRenameMethod.NONE; + } + + @Override + public int getMaxQueuedBuffers() { + return 0; + } + + @Override + public TimeZone getTimeZoneForDate() { + return TimeZone.getDefault(); + } + + @Override + public TimeZone getUseTimeZone() { + return TimeZone.getDefault(); + } + + @Override + public boolean isReuseValueWrapper() { + return true; + } + + @Override + public boolean isUseBinaryString() { + return false; + } + + @Override + public boolean isUseBlockingQueue() { + return false; + } + + @Override + public boolean isUseObjectsInArray() { + return false; + } + + @Override + public boolean isWidenUnsignedTypes() { + return false; + } +} diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseUtilsTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseUtilsTest.java similarity index 92% rename from clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseUtilsTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseUtilsTest.java index 5eaefbc63..c7e17b457 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseUtilsTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseUtilsTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.io.File; import java.io.FileNotFoundException; @@ -42,47 +42,6 @@ public void testEscape() { Assert.assertEquals(ClickHouseUtils.escape("\\'", '\''), "\\\\\\'"); } - @Test(groups = { "unit" }) - public void testGetBufferSize() { - int[] values = new int[] { -1, 0 }; - for (int i : values) { - for (int j : values) { - for (int k : values) { - Assert.assertEquals(ClickHouseUtils.getBufferSize(i, j, k), ClickHouseUtils.DEFAULT_BUFFER_SIZE); - } - } - } - - for (int i : values) { - Assert.assertEquals(ClickHouseUtils.getBufferSize(i, 3, 2), 2); - } - Assert.assertEquals(ClickHouseUtils.getBufferSize(1, 3, 2), 1); - Assert.assertEquals(ClickHouseUtils.getBufferSize(3, 2, 1), 1); - Assert.assertEquals(ClickHouseUtils.getBufferSize(3, 1, 2), 2); - } - - @Test(groups = { "unit" }) - public void testGetKeyValuePair() { - Assert.assertEquals(ClickHouseUtils.getKeyValuePairs(null), Collections.emptyMap()); - Assert.assertEquals(ClickHouseUtils.getKeyValuePairs(""), Collections.emptyMap()); - Assert.assertEquals(ClickHouseUtils.getKeyValuePairs(" "), Collections.emptyMap()); - Assert.assertEquals(ClickHouseUtils.getKeyValuePairs("="), Collections.emptyMap()); - Assert.assertEquals(ClickHouseUtils.getKeyValuePairs("=="), Collections.emptyMap()); - Assert.assertEquals(ClickHouseUtils.getKeyValuePairs(","), Collections.emptyMap()); - Assert.assertEquals(ClickHouseUtils.getKeyValuePairs(",,"), Collections.emptyMap()); - Assert.assertEquals(ClickHouseUtils.getKeyValuePairs("=,"), Collections.emptyMap()); - Assert.assertEquals(ClickHouseUtils.getKeyValuePairs(" =\r ,"), Collections.emptyMap()); - Assert.assertEquals(ClickHouseUtils.getKeyValuePairs("a\\=='b c',"), - Collections.singletonMap("a=", "'b c'")); - Assert.assertEquals(ClickHouseUtils.getKeyValuePairs("User-Agent=New Client, X-Forward-For=1\\,2"), - new HashMap() { - { - put("User-Agent", "New Client"); - put("X-Forward-For", "1,2"); - } - }); - } - @Test(groups = { "unit" }) public void testGetLeadingComment() { Assert.assertEquals(ClickHouseUtils.getLeadingComment(null), ""); @@ -99,14 +58,14 @@ public void testGetLeadingComment() { @Test(groups = { "unit" }) public void testGetService() { - ClickHouseClient client = null; + TestServiceInterface service = null; try { - client = ClickHouseUtils.getService(ClickHouseClient.class, null); + service = ClickHouseUtils.getService(TestServiceInterface.class, null); } catch (Exception e) { Assert.fail("Should not fail"); } - Assert.assertNotNull(client); + Assert.assertNotNull(service); ClickHouseUtilsTest me = null; try { diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseValuesTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseValuesTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseValuesTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseValuesTest.java index 94678542b..8544aadb9 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseValuesTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseValuesTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseVersionTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseVersionTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseVersionTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseVersionTest.java index cd164943e..d23edad8d 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseVersionTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseVersionTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client; +package com.clickhouse.data; import org.testng.Assert; import org.testng.annotations.DataProvider; diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/TestServiceImplementation.java b/clickhouse-data/src/test/java/com/clickhouse/data/TestServiceImplementation.java new file mode 100644 index 000000000..a6e3c5b52 --- /dev/null +++ b/clickhouse-data/src/test/java/com/clickhouse/data/TestServiceImplementation.java @@ -0,0 +1,8 @@ +package com.clickhouse.data; + +public class TestServiceImplementation implements TestServiceInterface { + @Override + public String getValue() { + return TestServiceImplementation.class.getSimpleName(); + } +} diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/TestServiceInterface.java b/clickhouse-data/src/test/java/com/clickhouse/data/TestServiceInterface.java new file mode 100644 index 000000000..a1c17662b --- /dev/null +++ b/clickhouse-data/src/test/java/com/clickhouse/data/TestServiceInterface.java @@ -0,0 +1,7 @@ +package com.clickhouse.data; + +public interface TestServiceInterface { + default String getValue() { + return TestServiceInterface.class.getSimpleName(); + } +} diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/cache/CaffeineCacheTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/cache/CaffeineCacheTest.java similarity index 94% rename from clickhouse-client/src/test/java/com/clickhouse/client/cache/CaffeineCacheTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/cache/CaffeineCacheTest.java index c6535a5db..82e720ac0 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/cache/CaffeineCacheTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/cache/CaffeineCacheTest.java @@ -1,9 +1,9 @@ -package com.clickhouse.client.cache; +package com.clickhouse.data.cache; import java.util.HashMap; import java.util.Map; -import com.clickhouse.client.ClickHouseCache; +import com.clickhouse.data.ClickHouseCache; import com.github.benmanes.caffeine.cache.Cache; import org.testng.Assert; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/cache/JdkLruCacheTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/cache/JdkLruCacheTest.java similarity index 93% rename from clickhouse-client/src/test/java/com/clickhouse/client/cache/JdkLruCacheTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/cache/JdkLruCacheTest.java index 430c5488f..02d0c1aa2 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/cache/JdkLruCacheTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/cache/JdkLruCacheTest.java @@ -1,9 +1,9 @@ -package com.clickhouse.client.cache; +package com.clickhouse.data.cache; import java.util.HashMap; import java.util.Map; -import com.clickhouse.client.ClickHouseCache; +import com.clickhouse.data.ClickHouseCache; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/format/ApacheAvroProcessorTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/format/ApacheAvroProcessorTest.java new file mode 100644 index 000000000..6045b6865 --- /dev/null +++ b/clickhouse-data/src/test/java/com/clickhouse/data/format/ApacheAvroProcessorTest.java @@ -0,0 +1,38 @@ +package com.clickhouse.data.format; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseTestDataConfig; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.value.ClickHouseStringValue; + +public class ApacheAvroProcessorTest { + @Test(groups = { "unit" }) + public void testInit() throws IOException { + List list = ClickHouseColumn.parse("a String, b Nullable(Int32)"); + ClickHouseValue value = ClickHouseStringValue.of("aaa"); + ClickHouseDataConfig config = new ClickHouseTestDataConfig() { + @Override + public ClickHouseFormat getFormat() { + return ClickHouseFormat.Avro; + } + }; + ApacheAvroProcessor p = new ApacheAvroProcessor(config, + ClickHouseInputStream.of(new File("/Users/zhicwu/Sources/Github/a.avro")), null, null, null); + } +} diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/BinaryStreamUtilsTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/format/BinaryStreamUtilsTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/BinaryStreamUtilsTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/format/BinaryStreamUtilsTest.java index d952d47e9..4f7df70f5 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/BinaryStreamUtilsTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/format/BinaryStreamUtilsTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.format; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -18,9 +18,11 @@ import java.util.TimeZone; import java.util.UUID; -import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.value.ClickHouseBitmap; +import com.clickhouse.data.value.WriterFunction; import org.roaringbitmap.RoaringBitmap; import org.roaringbitmap.buffer.ImmutableRoaringBitmap; diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseNativeProcessorTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseNativeProcessorTest.java new file mode 100644 index 000000000..ebe8b5202 --- /dev/null +++ b/clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseNativeProcessorTest.java @@ -0,0 +1,355 @@ +package com.clickhouse.data.format; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Collections; +import java.util.function.Consumer; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseDataProcessor; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseTestDataConfig; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.value.BaseDataProcessorTest; + +public class ClickHouseNativeProcessorTest extends BaseDataProcessorTest { + private ClickHouseNativeProcessor newProcessor(int... bytes) throws IOException { + return new ClickHouseNativeProcessor(new ClickHouseTestDataConfig(), BinaryStreamUtilsTest.generateInput(bytes), + null, null, null); + } + + private void verifySingleRecord(ClickHouseDataProcessor p, Object expectedValue) { + int rows = 0; + for (ClickHouseRecord r : p.records()) { + Assert.assertEquals(r.getValue(0).asObject(), expectedValue); + rows++; + } + Assert.assertEquals(rows, 1); + } + + @Override + protected ClickHouseDataProcessor getDataProcessor(ClickHouseDataConfig config, ClickHouseColumn column, + ClickHouseInputStream input, ClickHouseOutputStream output) throws IOException { + return new ClickHouseNativeProcessor(config, input, output, Collections.singletonList(column), null); + } + + // @Override + // protected ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseConfig + // config, + // ClickHouseColumn column, ClickHouseInputStream input) throws IOException { + // if (ref == null) { + // ref = column.newValue(config); + // } + // return new ClickHouseRowBinaryProcessor(config, input, null, + // Collections.singletonList(column), null) + // .getDeserializer(config, column).deserialize(ref, input); + // } + + // @Override + // protected void serialize(ClickHouseValue value, ClickHouseConfig config, + // ClickHouseColumn column, + // ClickHouseOutputStream output) throws IOException { + // ClickHouseNativeProcessor.getMappedFunctions().serialize(value, config, + // column, output); + // } + + @Override + protected byte[] getRawData(String typeName, String key) { + byte[] data = null; + byte[] header = new byte[] { 1, 1, 0, 0 }; + // toBytes(new byte[] { 0x01, 0x01, 0x78 }, new byte[] { (byte) + // typeName.length() },typeName.getBytes()); + boolean isLowCardinality = typeName.startsWith("LowCardinality("); + if (isLowCardinality) { + typeName = typeName.replaceFirst("LowCardinality\\(", "").replaceFirst("\\)", ""); + } + boolean isNull = "null".equals(key); + boolean isNullable = typeName.startsWith("Nullable("); + if (isNullable) { + typeName = typeName.replaceFirst("Nullable\\(", "").replaceFirst("\\)", ""); + if (isNull) { + if (!isLowCardinality) { + return toBytes(header, 1); + } + } else { + if (!isLowCardinality) { + header = toBytes(header, 0); + } + } + } + if (isLowCardinality) { + header = toBytes(header, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0); + } + switch (typeName) { + case "Array(Bool)": + if ("0,1".equals(key)) { + data = toBytes("[false,true]"); + } + break; + case "Array(Int8)": + case "Array(Int16)": + case "Array(Int32)": + case "Array(Int64)": + case "Array(Float32)": + case "Array(Float64)": + if ("0,1,-1".equals(key)) { + data = toBytes("0,1,-1"); + } + break; + case "Array(UInt8)": + if ("0,1,-1".equals(key)) { + data = toBytes("0,1,255"); + } + break; + case "Array(UInt16)": + if ("0,1,-1".equals(key)) { + data = toBytes("0,1,65535"); + } + break; + case "Array(UInt32)": + if ("0,1,-1".equals(key)) { + data = toBytes("0,1,4294967295"); + } + break; + case "Array(UInt64)": + if ("0,1,-1".equals(key)) { + data = toBytes("0,1,18446744073709551615"); + } + break; + case "Array(String)": + if ("4bcd".equals(key)) { + data = toBytes("['','B','BC','BCD']"); + } + break; + case "Bool": + case "Int8": + case "UInt8": + if (isLowCardinality) { + byte b = isNull ? 0 : Byte.parseByte(key); + int base = isNullable ? 1 : 0; + data = toBytes(header, toBytes(base + (b == 0 ? 1 : 2), 0, 0, 0, 0, 0, 0, 0, 0), + new byte[isNullable ? 1 : 0], b == 0 ? new byte[0] : new byte[] { b }, + toBytes(1, 0, 0, 0, 0, 0, 0, 0, isNull ? 0 : (base + (b == 0 ? 0 : 1)))); + } else { + data = toBytes(header, new byte[] { Byte.parseByte(key) }); + } + break; + case "Int16": + case "UInt16": + if (isLowCardinality) { + short s = isNull ? 0 : Short.parseShort(key); + int base = isNullable ? 1 : 0; + data = toBytes(header, toBytes(base + (s == 0 ? 1 : 2), 0, 0, 0, 0, 0, 0, 0, 0, 0), + new byte[isNullable ? 2 : 0], s == 0 ? new byte[0] : toLEBytes(s), + toBytes(1, 0, 0, 0, 0, 0, 0, 0, isNull ? 0 : (base + (s == 0 ? 0 : 1)))); + } else { + data = toBytes(header, toLEBytes(Short.parseShort(key))); + } + break; + case "Int32": + case "UInt32": + if (isLowCardinality) { + int i = isNull ? 0 : Integer.parseInt(key); + int base = isNullable ? 1 : 0; + data = toBytes(header, + toBytes(base + (i == 0 ? 1 : 2), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), + new byte[isNullable ? 4 : 0], i == 0 ? new byte[0] : toLEBytes(i), + toBytes(1, 0, 0, 0, 0, 0, 0, 0, isNull ? 0 : (base + (i == 0 ? 0 : 1)))); + } else { + data = toBytes(header, toLEBytes(Integer.parseInt(key))); + } + break; + case "Int64": + case "UInt64": + if (isLowCardinality) { + long l = isNull ? 0L : Long.parseLong(key); + int base = isNullable ? 1 : 0; + data = toBytes(header, + toBytes(base + (l == 0L ? 1 : 2), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), + new byte[isNullable ? 8 : 0], l == 0L ? new byte[0] : toLEBytes(l), + toBytes(1, 0, 0, 0, 0, 0, 0, 0, isNull ? 0 : (base + (l == 0L ? 0 : 1)))); + } else { + data = toBytes(header, toLEBytes(Long.parseLong(key))); + } + break; + case "Float32": + if (isLowCardinality) { + float f = isNull ? 0F : Float.parseFloat(key); + int base = isNullable ? 1 : 0; + data = toBytes(header, + toBytes(base + (f == 0F ? 1 : 2), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), + new byte[isNullable ? 4 : 0], f == 0F ? new byte[0] : toLEBytes(f), + toBytes(1, 0, 0, 0, 0, 0, 0, 0, isNull ? 0 : (base + (f == 0F ? 0 : 1)))); + } else { + data = toBytes(header, toLEBytes(Float.parseFloat(key))); + } + break; + case "Float64": + if (isLowCardinality) { + double d = isNull ? 0D : Double.parseDouble(key); + int base = isNullable ? 1 : 0; + data = toBytes(header, + toBytes(base + (d == 0D ? 1 : 2), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), + new byte[isNullable ? 8 : 0], d == 0D ? new byte[0] : toLEBytes(d), + toBytes(1, 0, 0, 0, 0, 0, 0, 0, isNull ? 0 : (base + (d == 0D ? 0 : 1)))); + } else { + data = toBytes(header, toLEBytes(Double.parseDouble(key))); + } + break; + case "String": + if (isLowCardinality) { + int index = isNull ? 0 : 1; + int base = isNullable ? 1 : 0; + data = toBytes(header, + toBytes(base + (index == 0 ? 1 : 2), 0, 0, 0, 0, 0, 0, 0, 0), + new byte[isNullable ? 1 : 0], index == 0 ? new byte[0] : toLEBytes(key), + toBytes(1, 0, 0, 0, 0, 0, 0, 0, isNull ? 0 : (base + (index == 0 ? 0 : 1)))); + } else { + data = toBytes(header, toLEBytes(key)); + } + break; + default: + break; + } + + if (data == null) { + Assert.fail(String.format("No raw data defined for type=[%s] and key=[%s]", typeName, key)); + } + return data; + } + + @Test(dataProvider = "simpleTypesForWrite", groups = { "unit" }) + public void testSerializeSimpleTypes(ClickHouseDataConfig config, String typeName, String dataKey, + ClickHouseValue... values) throws IOException { + if (values == null || values.length == 0) { + Assert.fail("At least one value must be provided for testing"); + } + + ByteArrayOutputStream bao = new ByteArrayOutputStream(); + for (ClickHouseValue value : values) { + bao.reset(); + try (ClickHouseOutputStream output = ClickHouseOutputStream.of(bao)) { + serialize(value, config, ClickHouseColumn.of("a", typeName), output); + } + Assert.assertEquals(bao.toByteArray(), getRawData(typeName, dataKey)); + + bao.reset(); + String lowCardinalityTypeName = "LowCardinality(" + typeName + ")"; + try (ClickHouseOutputStream output = ClickHouseOutputStream.of(bao)) { + serialize(value, config, ClickHouseColumn.of("a", lowCardinalityTypeName), output); + } + Assert.assertEquals(bao.toByteArray(), getRawData(lowCardinalityTypeName, dataKey)); + } + } + + @Test(dataProvider = "simpleTypesForRead", groups = { "unit" }) + public void testDeserializeSimpleTypes(ClickHouseDataConfig config, String typeName, String dataKey, + Class valueClass, Object objVal, String strVal, Consumer customChecks) + throws IOException { + try (ClickHouseInputStream in = getInputData(typeName, dataKey)) { + ClickHouseValue value = deserialize(null, config, ClickHouseColumn.of("a", typeName), in); + Assert.assertEquals(value.getClass(), valueClass); + Assert.assertEquals(value.asObject(), objVal); + Assert.assertEquals(value.asString(), strVal); + if (customChecks != null) { + customChecks.accept(value); + } + } + + // String lowCardinalityTypeName = "LowCardinality(" + typeName + ")"; + // try (ClickHouseInputStream in = getInputData(lowCardinalityTypeName, + // dataKey)) { + // ClickHouseValue value = deserialize(null, config, ClickHouseColumn.of("a", + // lowCardinalityTypeName), in); + // Assert.assertEquals(value.getClass(), valueClass); + // Assert.assertEquals(value.asObject(), objVal); + // Assert.assertEquals(value.asString(), strVal); + // if (customChecks != null) { + // customChecks.accept(value); + // } + // } + } + + @Test(groups = { "unit" }) + public void testDeserializeArray() throws IOException { + ClickHouseNativeProcessor p = newProcessor(0x02, 0x01, 0x01, 0x62, 0x06, 0x53, 0x74, 0x72, 0x69, 0x6E, + 0x67, 0x04, 0x61, 0x31, 0x32, 0x33, 0x01, 0x63, 0x05, 0x55, 0x49, 0x6E, 0x74, 0x38, 0x01); + + Assert.assertNotNull(p.getColumns()); + + // select 1::LowCardinality(Int8) a + p = newProcessor(0x01, 0x01, 0x01, 0x61, 0x14, 0x4C, 0x6F, 0x77, 0x43, 0x61, 0x72, 0x64, 0x69, 0x6E, 0x61, 0x6C, + 0x69, 0x74, 0x79, 0x28, 0x49, 0x6e, 0x74, 0x38, 0x29, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01); + Assert.assertNotNull(p.getColumns()); + + // select 1::LowCardinality(String) a + p = newProcessor(0x01, 0x01, 0x01, 0x61, 0x16, 0x4C, 0x6F, 0x77, 0x43, 0x61, 0x72, 0x64, 0x69, 0x6E, + 0x61, 0x6C, 0x69, 0x74, 0x79, 0x28, 0x53, 0x74, 0x72, 0x69, 0x6E, 0x67, 0x29, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x31, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01); + Assert.assertNotNull(p.getColumns()); + } + + @Test(groups = { "unit" }) + public void testArray() throws IOException { + // [1,2,3]::Array(Int8) + ClickHouseNativeProcessor p = newProcessor(0x01, 0x01, 0x1E, 0x43, 0x41, 0x53, 0x54, 0x28, 0x27, 0x5B, 0x31, + 0x2C, 0x32, 0x2C, 0x33, 0x5D, 0x27, 0x2C, 0x20, 0x27, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x49, 0x6E, + 0x74, 0x38, 0x29, 0x27, 0x29, 0x0B, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x49, 0x6E, 0x74, 0x38, 0x29, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03); + verifySingleRecord(p, new byte[] { 1, 2, 3 }); + + // [1,2,3]::Array(Nullable(Int8)) + p = newProcessor(0x01, 0x01, 0x28, 0x43, 0x41, 0x53, 0x54, 0x28, 0x27, 0x5B, 0x31, 0x2C, 0x32, 0x2C, 0x33, 0x5D, + 0x27, 0x2C, 0x20, 0x27, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x4E, 0x75, 0x6C, 0x6C, 0x61, 0x62, 0x6C, + 0x65, 0x28, 0x49, 0x6E, 0x74, 0x38, 0x29, 0x29, 0x27, 0x29, 0x15, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, + 0x4E, 0x75, 0x6C, 0x6C, 0x61, 0x62, 0x6C, 0x65, 0x28, 0x49, 0x6E, 0x74, 0x38, 0x29, 0x29, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03); + verifySingleRecord(p, new Byte[] { 1, 2, 3 }); + + // [1,null,3]::Array(Nullable(Int8)) + p = newProcessor(0x01, 0x01, 0x2B, 0x43, 0x41, 0x53, 0x54, 0x28, 0x5B, 0x31, 0x2C, 0x20, 0x4E, 0x55, 0x4C, 0x4C, + 0x2C, 0x20, 0x33, 0x5D, 0x2C, 0x20, 0x27, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x4E, 0x75, 0x6C, 0x6C, + 0x61, 0x62, 0x6C, 0x65, 0x28, 0x49, 0x6E, 0x74, 0x38, 0x29, 0x29, 0x27, 0x29, 0x15, 0x41, 0x72, 0x72, + 0x61, 0x79, 0x28, 0x4E, 0x75, 0x6C, 0x6C, 0x61, 0x62, 0x6C, 0x65, 0x28, 0x49, 0x6E, 0x74, 0x38, 0x29, + 0x29, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x03); + verifySingleRecord(p, new Byte[] { 1, null, 3 }); + + // [[1,2],[],[null,4,5]]::Array(Array(Nullable(Int8))) + p = newProcessor(0x01, 0x01, 0x40, 0x43, 0x41, 0x53, 0x54, 0x28, 0x5B, 0x5B, 0x31, 0x2C, 0x20, 0x32, 0x5D, 0x2C, + 0x20, 0x5B, 0x5D, 0x2C, 0x20, 0x5B, 0x4E, 0x55, 0x4C, 0x4C, 0x2C, 0x20, 0x34, 0x2C, 0x20, 0x35, 0x5D, + 0x5D, 0x2C, 0x20, 0x27, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x4E, + 0x75, 0x6C, 0x6C, 0x61, 0x62, 0x6C, 0x65, 0x28, 0x49, 0x6E, 0x74, 0x38, 0x29, 0x29, 0x29, 0x27, 0x29, + 0x1C, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x4E, 0x75, 0x6C, 0x6C, + 0x61, 0x62, 0x6C, 0x65, 0x28, 0x49, 0x6E, 0x74, 0x38, 0x29, 0x29, 0x29, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01, 0x02, + 0x00, 0x04, 0x05); + verifySingleRecord(p, new Byte[][] { { 1, 2 }, {}, { null, 4, 5 } }); + + // [[[null],[1,2],[3,4,5]],[[1,2],[],[null,4,5]]]::Array(Array(Array(Nullable(Int8)))) + newProcessor(0x01, 0x01, 0x66, 0x43, 0x41, 0x53, 0x54, 0x28, 0x5B, 0x5B, 0x5B, 0x4E, 0x55, 0x4C, 0x4C, 0x5D, + 0x2C, 0x20, 0x5B, 0x31, 0x2C, 0x20, 0x32, 0x5D, 0x2C, 0x20, 0x5B, 0x33, 0x2C, 0x20, 0x34, 0x2C, 0x20, + 0x35, 0x5D, 0x5D, 0x2C, 0x20, 0x5B, 0x5B, 0x31, 0x2C, 0x20, 0x32, 0x5D, 0x2C, 0x20, 0x5B, 0x5D, 0x2C, + 0x20, 0x5B, 0x4E, 0x55, 0x4C, 0x4C, 0x2C, 0x20, 0x34, 0x2C, 0x20, 0x35, 0x5D, 0x5D, 0x5D, 0x2C, 0x20, + 0x27, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x41, 0x72, 0x72, 0x61, + 0x79, 0x28, 0x4E, 0x75, 0x6C, 0x6C, 0x61, 0x62, 0x6C, 0x65, 0x28, 0x49, 0x6E, 0x74, 0x38, 0x29, 0x29, + 0x29, 0x29, 0x27, 0x29, 0x23, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, + 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x4E, 0x75, 0x6C, 0x6C, 0x61, 0x62, 0x6C, 0x65, 0x28, 0x49, 0x6E, + 0x74, 0x38, 0x29, 0x29, 0x29, 0x29, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x01, 0x02, 0x00, 0x04, 0x05); + verifySingleRecord(p, new Byte[][][] { { { null }, { 1, 2 }, { 3, 4, 5 } }, { { 1, 2 }, {}, { null, 4, 5 } } }); + } +} diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseRowBinaryProcessorTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessorTest.java similarity index 93% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseRowBinaryProcessorTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessorTest.java index 72c8477b3..29299be08 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseRowBinaryProcessorTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessorTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.format; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -13,18 +13,27 @@ import org.testng.Assert; import org.testng.annotations.Test; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseDataProcessor; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.data.array.ClickHouseByteArrayValue; -import com.clickhouse.client.data.array.ClickHouseShortArrayValue; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseDataProcessor; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseTestDataConfig; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.value.array.ClickHouseByteArrayValue; +import com.clickhouse.data.value.array.ClickHouseShortArrayValue; +import com.clickhouse.data.value.BaseDataProcessorTest; +import com.clickhouse.data.value.ClickHouseArrayValue; +import com.clickhouse.data.value.ClickHouseMapValue; +import com.clickhouse.data.value.ClickHouseNestedValue; +import com.clickhouse.data.value.ClickHouseTupleValue; +import com.clickhouse.data.value.UnsignedByte; +import com.clickhouse.data.value.UnsignedInteger; +import com.clickhouse.data.value.UnsignedShort; public class ClickHouseRowBinaryProcessorTest extends BaseDataProcessorTest { @Override - protected ClickHouseDataProcessor getDataProcessor(ClickHouseConfig config, ClickHouseColumn column, + protected ClickHouseDataProcessor getDataProcessor(ClickHouseDataConfig config, ClickHouseColumn column, ClickHouseInputStream input, ClickHouseOutputStream output) throws IOException { return new ClickHouseRowBinaryProcessor(config, input, output, Collections.singletonList(column), null); } @@ -218,7 +227,7 @@ protected byte[] getRawData(String typeName, String key) { } @Test(dataProvider = "simpleTypesForRead", groups = { "unit" }) - public void testDeserializeSimpleTypes(ClickHouseConfig config, String typeName, String dataKey, + public void testDeserializeSimpleTypes(ClickHouseDataConfig config, String typeName, String dataKey, Class valueClass, Object objVal, String strVal, Consumer customChecks) throws IOException { try (ClickHouseInputStream in = getInputData(typeName, dataKey)) { @@ -246,7 +255,7 @@ public void testDeserializeSimpleTypes(ClickHouseConfig config, String typeName, @Test(groups = { "unit" }) public void testDeserializeArray() throws IOException { // widen_unsigned_types=false, use_objects_in_array=false - ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseDataConfig config = new ClickHouseTestDataConfig(); ClickHouseValue value = deserialize(null, config, ClickHouseColumn.of("a", "Array(UInt8)"), BinaryStreamUtilsTest.generateInput(2, 1, 2)); Assert.assertTrue(value instanceof ClickHouseByteArrayValue); // UnsignedByteArrayValue @@ -328,7 +337,7 @@ public void testDeserializeArray() throws IOException { @Test(groups = { "unit" }) public void testSerializeArray() throws IOException { - ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseDataConfig config = new ClickHouseTestDataConfig(); ClickHouseValue value = ClickHouseShortArrayValue.of(new short[] { 1, 2 }); ByteArrayOutputStream bas = new ByteArrayOutputStream(); ClickHouseOutputStream out = ClickHouseOutputStream.of(bas); @@ -385,7 +394,7 @@ public void testSerializeArray() throws IOException { @Test(groups = { "unit" }) public void testSerializeBoolean() throws IOException { - ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseDataConfig config = new ClickHouseTestDataConfig(); ClickHouseValue value = ClickHouseArrayValue.of(new Boolean[][] { new Boolean[] { true, false } }); ByteArrayOutputStream bas = new ByteArrayOutputStream(); @@ -411,7 +420,7 @@ public void testSerializeBoolean() throws IOException { @Test(groups = { "unit" }) public void testDeserializeMap() throws IOException { - ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseDataConfig config = new ClickHouseTestDataConfig(); ClickHouseValue value = deserialize(null, config, ClickHouseColumn.of("m", "Map(UInt8, UInt8)"), BinaryStreamUtilsTest.generateInput(2, 2, 2, 1, 1)); @@ -433,7 +442,7 @@ public void testDeserializeMap() throws IOException { @Test(groups = { "unit" }) public void testSerializeMap() throws IOException { - ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseDataConfig config = new ClickHouseTestDataConfig(); ClickHouseValue value = ClickHouseMapValue.of(new LinkedHashMap() { { put((short) 2, (short) 2); @@ -464,7 +473,7 @@ public void testSerializeMap() throws IOException { @Test(groups = { "unit" }) public void testDeserializeNested() throws IOException { - ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseDataConfig config = new ClickHouseTestDataConfig(); ClickHouseValue value = deserialize(null, config, ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Int16)"), BinaryStreamUtilsTest.generateInput(1, 1, 1, 0, 1, 0x32, 1, 3, 0)); @@ -484,7 +493,7 @@ public void testDeserializeNested() throws IOException { @Test(groups = { "unit" }) public void testSerializeNested() throws IOException { - ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseDataConfig config = new ClickHouseTestDataConfig(); ClickHouseValue value = ClickHouseNestedValue.of( ClickHouseColumn.of("n", "Nested(n1 UInt8, n2 Nullable(String), n3 Int16)") .getNestedColumns(), @@ -501,7 +510,7 @@ public void testSerializeNested() throws IOException { @Test(groups = { "unit" }) public void testDeserializeTuple() throws IOException { - ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseDataConfig config = new ClickHouseTestDataConfig(); ClickHouseValue value = deserialize(null, config, ClickHouseColumn.of("t", "Tuple(UInt8, String)"), BinaryStreamUtilsTest.generateInput(1, 1, 0x61)); @@ -526,7 +535,7 @@ public void testDeserializeTuple() throws IOException { @Test(groups = { "unit" }) public void testSerializeTuple() throws IOException { - ClickHouseConfig config = new ClickHouseConfig(); + ClickHouseDataConfig config = new ClickHouseTestDataConfig(); ClickHouseValue value = ClickHouseTupleValue.of((byte) 1, "a"); ByteArrayOutputStream bas = new ByteArrayOutputStream(); ClickHouseOutputStream out = ClickHouseOutputStream.of(bas); diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseSimpleRecordTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseSimpleRecordTest.java similarity index 94% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseSimpleRecordTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseSimpleRecordTest.java index 653fbf9dd..9fe8f1b79 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseSimpleRecordTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseSimpleRecordTest.java @@ -1,10 +1,12 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.format; import java.util.Arrays; import java.util.Collections; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseValue; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.value.ClickHouseLongValue; +import com.clickhouse.data.value.ClickHouseStringValue; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseTabSeparatedProcessorTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseTabSeparatedProcessorTest.java similarity index 83% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseTabSeparatedProcessorTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseTabSeparatedProcessorTest.java index b595b3e07..dac888ecc 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseTabSeparatedProcessorTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseTabSeparatedProcessorTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.format; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -10,16 +10,20 @@ import java.util.Map; import java.util.UUID; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseDataProcessor; -import com.clickhouse.client.ClickHouseFormat; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHouseRecord; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseOption; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseDataProcessor; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseTestDataConfig; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.value.BaseDataProcessorTest; +import com.clickhouse.data.value.ClickHouseEmptyValue; +import com.clickhouse.data.value.ClickHouseStringValue; +import com.clickhouse.data.value.UnsignedLong; import org.testng.Assert; import org.testng.SkipException; @@ -38,7 +42,7 @@ private Object[][] getSimpleTsvData() { } @Override - protected ClickHouseDataProcessor getDataProcessor(ClickHouseConfig config, ClickHouseColumn column, + protected ClickHouseDataProcessor getDataProcessor(ClickHouseDataConfig config, ClickHouseColumn column, ClickHouseInputStream input, ClickHouseOutputStream output) throws IOException { return new ClickHouseTabSeparatedProcessor(config, input, output, Collections.singletonList(column), null); } @@ -145,13 +149,14 @@ protected byte[] getRawData(String typeName, String key) { } @Override - public void testDeserializeNestedTypes(ClickHouseConfig config, String typeName, String dataKey, String valueClass, + public void testDeserializeNestedTypes(ClickHouseDataConfig config, String typeName, String dataKey, + String valueClass, Object arrVal, Object objVal, String strVal, String sqlExpr) throws IOException { throw new SkipException("Skip as it's not fully implemented"); } @Override - public void testSerializeNestedTypes(ClickHouseConfig config, String typeName, String dataKey, + public void testSerializeNestedTypes(ClickHouseDataConfig config, String typeName, String dataKey, ClickHouseValue... values) throws IOException { throw new SkipException("Skip as it's not fully implemented"); } @@ -162,9 +167,12 @@ public void testRead(String id, int rows) throws IOException { for (int i = 0; i < rows; i++) { result += String.format("%s\n", id); } - Map options = new HashMap<>(); - options.put(ClickHouseClientOption.FORMAT, ClickHouseFormat.TabSeparatedWithNamesAndTypes); - ClickHouseConfig config = new ClickHouseConfig(options, null, null, null); + ClickHouseDataConfig config = new ClickHouseTestDataConfig() { + @Override + public ClickHouseFormat getFormat() { + return ClickHouseFormat.TabSeparatedWithNamesAndTypes; + } + }; ClickHouseInputStream input = ClickHouseInputStream.of(Collections.singletonList(result), String.class, s -> s.getBytes(), null); ClickHouseTabSeparatedProcessor p = new ClickHouseTabSeparatedProcessor(config, input, null, null, null); @@ -182,9 +190,12 @@ public void testRead(String id, int rows) throws IOException { public void testWriteTsv() throws IOException { List list = ClickHouseColumn.parse("a String, b Nullable(Int32)"); ClickHouseValue value = ClickHouseStringValue.of("aaa"); - Map options = new HashMap<>(); - options.put(ClickHouseClientOption.FORMAT, ClickHouseFormat.TabSeparated); - ClickHouseConfig config = new ClickHouseConfig(options, null, null, null); + ClickHouseDataConfig config = new ClickHouseTestDataConfig() { + @Override + public ClickHouseFormat getFormat() { + return ClickHouseFormat.TabSeparated; + } + }; try (ByteArrayOutputStream bas = new ByteArrayOutputStream(); ClickHouseOutputStream out = ClickHouseOutputStream.of(bas)) { ClickHouseTabSeparatedProcessor p = new ClickHouseTabSeparatedProcessor(config, null, out, diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/JsonStreamUtilsTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/format/JsonStreamUtilsTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/JsonStreamUtilsTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/format/JsonStreamUtilsTest.java index 6d59fa67f..d73e1bf1d 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/JsonStreamUtilsTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/format/JsonStreamUtilsTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.format; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/stream/BlockingPipedOutputStreamTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/stream/BlockingPipedOutputStreamTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/stream/BlockingPipedOutputStreamTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/stream/BlockingPipedOutputStreamTest.java index 49be159ae..ce52eea76 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/stream/BlockingPipedOutputStreamTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/stream/BlockingPipedOutputStreamTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; import java.io.InputStream; @@ -13,7 +13,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import com.clickhouse.client.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseByteBuffer; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/stream/InputStreamImplTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/stream/InputStreamImplTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/stream/InputStreamImplTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/stream/InputStreamImplTest.java index 46e98e64d..7ac4f58d4 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/stream/InputStreamImplTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/stream/InputStreamImplTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -19,9 +19,9 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; import org.testng.Assert; import org.testng.annotations.DataProvider; @@ -165,7 +165,7 @@ private Object[][] getEmptyInputStreams() { { ClickHouseInputStream.of(new ByteArrayInputStream[0]) }, { ClickHouseInputStream.of(new ByteArrayInputStream(new byte[0])) }, { ClickHouseInputStream.of((ByteArrayInputStream) null) }, - { ClickHouseInputStream.of(new ByteArrayInputStream(new byte[0]), null) }, + { ClickHouseInputStream.of(new ByteArrayInputStream(new byte[0])) }, { ClickHouseInputStream.of(null, new ByteArrayInputStream(new byte[0])) }, { ClickHouseInputStream.of(new ByteArrayInputStream(new byte[0]), new ByteArrayInputStream(new byte[0])) }, diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/stream/Lz4InputStreamTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/stream/Lz4InputStreamTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/stream/Lz4InputStreamTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/stream/Lz4InputStreamTest.java index 51feaa156..516a1dd67 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/stream/Lz4InputStreamTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/stream/Lz4InputStreamTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import org.testng.Assert; import org.testng.annotations.DataProvider; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/stream/Lz4OutputStreamTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/stream/Lz4OutputStreamTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/stream/Lz4OutputStreamTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/stream/Lz4OutputStreamTest.java index 0c39d72eb..b4484873a 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/stream/Lz4OutputStreamTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/stream/Lz4OutputStreamTest.java @@ -1,15 +1,15 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; -import com.clickhouse.client.data.BinaryStreamUtilsTest; - import org.testng.Assert; import org.testng.annotations.Test; +import com.clickhouse.data.format.BinaryStreamUtilsTest; + public class Lz4OutputStreamTest { private byte[] genCompressedByts(int b, int length, int blockSize) throws IOException { ByteArrayOutputStream bas = new ByteArrayOutputStream(blockSize * 512); diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/stream/NonBlockingPipedOutputStreamTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/stream/NonBlockingPipedOutputStreamTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/stream/NonBlockingPipedOutputStreamTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/stream/NonBlockingPipedOutputStreamTest.java index f2d92d0cd..13ac909f9 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/stream/NonBlockingPipedOutputStreamTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/stream/NonBlockingPipedOutputStreamTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.IOException; import java.io.InputStream; @@ -11,7 +11,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import com.clickhouse.client.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseByteBuffer; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/stream/OutputStreamImplTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/stream/OutputStreamImplTest.java similarity index 96% rename from clickhouse-client/src/test/java/com/clickhouse/client/stream/OutputStreamImplTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/stream/OutputStreamImplTest.java index 6aa0c362b..8aadaba44 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/stream/OutputStreamImplTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/stream/OutputStreamImplTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.stream; +package com.clickhouse.data.stream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -6,8 +6,8 @@ import java.nio.ByteBuffer; import java.util.Arrays; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseOutputStream; import org.testng.Assert; import org.testng.annotations.DataProvider; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/BaseDataProcessorTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/BaseDataProcessorTest.java similarity index 93% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/BaseDataProcessorTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/BaseDataProcessorTest.java index ed46ffe9a..707bf48e3 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/BaseDataProcessorTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/BaseDataProcessorTest.java @@ -1,35 +1,35 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.charset.StandardCharsets; -import java.util.Collections; import java.util.function.Consumer; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseDataProcessor; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.data.ClickHouseByteValue.UnsignedByteValue; -import com.clickhouse.client.data.ClickHouseIntegerValue.UnsignedIntegerValue; -import com.clickhouse.client.data.ClickHouseLongValue.UnsignedLongValue; -import com.clickhouse.client.data.ClickHouseShortValue.UnsignedShortValue; -import com.clickhouse.client.data.array.ClickHouseBoolArrayValue; -import com.clickhouse.client.data.array.ClickHouseByteArrayValue; -import com.clickhouse.client.data.array.ClickHouseDoubleArrayValue; -import com.clickhouse.client.data.array.ClickHouseFloatArrayValue; -import com.clickhouse.client.data.array.ClickHouseIntArrayValue; -import com.clickhouse.client.data.array.ClickHouseLongArrayValue; -import com.clickhouse.client.data.array.ClickHouseShortArrayValue; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseDataProcessor; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseTestDataConfig; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.format.BinaryStreamUtilsTest; +import com.clickhouse.data.value.ClickHouseByteValue.UnsignedByteValue; +import com.clickhouse.data.value.ClickHouseIntegerValue.UnsignedIntegerValue; +import com.clickhouse.data.value.ClickHouseLongValue.UnsignedLongValue; +import com.clickhouse.data.value.ClickHouseShortValue.UnsignedShortValue; +import com.clickhouse.data.value.array.ClickHouseBoolArrayValue; +import com.clickhouse.data.value.array.ClickHouseByteArrayValue; +import com.clickhouse.data.value.array.ClickHouseDoubleArrayValue; +import com.clickhouse.data.value.array.ClickHouseFloatArrayValue; +import com.clickhouse.data.value.array.ClickHouseIntArrayValue; +import com.clickhouse.data.value.array.ClickHouseLongArrayValue; +import com.clickhouse.data.value.array.ClickHouseShortArrayValue; public abstract class BaseDataProcessorTest { protected void checkNull(ClickHouseValue value) { @@ -237,7 +237,7 @@ protected byte[] toLEBytes(String s) { return toBytes(new byte[] { (byte) s.length() }, toBytes(s)); } - protected abstract ClickHouseDataProcessor getDataProcessor(ClickHouseConfig config, ClickHouseColumn column, + protected abstract ClickHouseDataProcessor getDataProcessor(ClickHouseDataConfig config, ClickHouseColumn column, ClickHouseInputStream input, ClickHouseOutputStream output) throws IOException; protected abstract byte[] getRawData(String typeName, String key); @@ -248,14 +248,36 @@ protected ClickHouseInputStream getInputData(String typeName, String key) { @DataProvider(name = "nestedTypesForRead") protected Object[][] getNestedTypesForRead() { - ClickHouseConfig defaultConf = new ClickHouseConfig(); - ClickHouseConfig binStrConf = new ClickHouseConfig( - Collections.singletonMap(ClickHouseClientOption.USE_BINARY_STRING, true)); - ClickHouseConfig widenUtConf = new ClickHouseConfig( - Collections.singletonMap(ClickHouseClientOption.WIDEN_UNSIGNED_TYPES, true)); - ClickHouseConfig useObjsConf = new ClickHouseConfig( - Collections.singletonMap(ClickHouseClientOption.USE_OBJECTS_IN_ARRAYS, true)); - ClickHouseConfig combinedConf = new ClickHouseConfig(widenUtConf, useObjsConf); + ClickHouseDataConfig defaultConf = new ClickHouseTestDataConfig(); + ClickHouseDataConfig binStrConf = new ClickHouseTestDataConfig() { + @Override + public boolean isUseBinaryString() { + return true; + } + }; + ClickHouseDataConfig widenUtConf = new ClickHouseTestDataConfig() { + @Override + public boolean isWidenUnsignedTypes() { + return true; + } + }; + ClickHouseDataConfig useObjsConf = new ClickHouseTestDataConfig() { + @Override + public boolean isUseObjectsInArray() { + return true; + } + }; + ClickHouseDataConfig combinedConf = new ClickHouseTestDataConfig() { + @Override + public boolean isUseObjectsInArray() { + return true; + } + + @Override + public boolean isWidenUnsignedTypes() { + return true; + } + }; return new Object[][] { { defaultConf, "Array(Nullable(Bool))", "0,1", "ClickHouseArrayValue", @@ -463,14 +485,36 @@ protected Object[][] getNestedTypesForRead() { @DataProvider(name = "nestedTypesForWrite") protected Object[][] getNestedTypesForWrite() { - ClickHouseConfig defaultConf = new ClickHouseConfig(); - ClickHouseConfig binStrConf = new ClickHouseConfig( - Collections.singletonMap(ClickHouseClientOption.USE_BINARY_STRING, true)); - ClickHouseConfig widenUtConf = new ClickHouseConfig( - Collections.singletonMap(ClickHouseClientOption.WIDEN_UNSIGNED_TYPES, true)); - ClickHouseConfig useObjsConf = new ClickHouseConfig( - Collections.singletonMap(ClickHouseClientOption.USE_OBJECTS_IN_ARRAYS, true)); - ClickHouseConfig combinedConf = new ClickHouseConfig(widenUtConf, useObjsConf); + ClickHouseDataConfig defaultConf = new ClickHouseTestDataConfig(); + ClickHouseDataConfig binStrConf = new ClickHouseTestDataConfig() { + @Override + public boolean isUseBinaryString() { + return true; + } + }; + ClickHouseDataConfig widenUtConf = new ClickHouseTestDataConfig() { + @Override + public boolean isWidenUnsignedTypes() { + return true; + } + }; + ClickHouseDataConfig useObjsConf = new ClickHouseTestDataConfig() { + @Override + public boolean isUseObjectsInArray() { + return true; + } + }; + ClickHouseDataConfig combinedConf = new ClickHouseTestDataConfig() { + @Override + public boolean isUseObjectsInArray() { + return true; + } + + @Override + public boolean isWidenUnsignedTypes() { + return true; + } + }; return new Object[][] { { defaultConf, "Array(Nullable(Bool))", "0,1", @@ -566,11 +610,19 @@ protected Object[][] getNestedTypesForWrite() { @DataProvider(name = "simpleTypesForRead") protected Object[][] getSimpleTypesForRead() { - ClickHouseConfig defaultConf = new ClickHouseConfig(); - ClickHouseConfig binStrConf = new ClickHouseConfig( - Collections.singletonMap(ClickHouseClientOption.USE_BINARY_STRING, true)); - ClickHouseConfig widenUtConf = new ClickHouseConfig( - Collections.singletonMap(ClickHouseClientOption.WIDEN_UNSIGNED_TYPES, true)); + ClickHouseDataConfig defaultConf = new ClickHouseTestDataConfig(); + ClickHouseDataConfig binStrConf = new ClickHouseTestDataConfig() { + @Override + public boolean isUseBinaryString() { + return true; + } + }; + ClickHouseDataConfig widenUtConf = new ClickHouseTestDataConfig() { + @Override + public boolean isWidenUnsignedTypes() { + return true; + } + }; return new Object[][] { { defaultConf, "Nullable(Bool)", "null", ClickHouseBoolValue.class, null, null, @@ -772,11 +824,19 @@ protected Object[][] getSimpleTypesForRead() { @DataProvider(name = "simpleTypesForWrite") protected Object[][] getSimpleTypesForWrite() { - ClickHouseConfig defaultConf = new ClickHouseConfig(); - ClickHouseConfig binStrConf = new ClickHouseConfig( - Collections.singletonMap(ClickHouseClientOption.USE_BINARY_STRING, true)); - ClickHouseConfig widenUtConf = new ClickHouseConfig( - Collections.singletonMap(ClickHouseClientOption.WIDEN_UNSIGNED_TYPES, true)); + ClickHouseDataConfig defaultConf = new ClickHouseTestDataConfig(); + ClickHouseDataConfig binStrConf = new ClickHouseTestDataConfig() { + @Override + public boolean isUseBinaryString() { + return true; + } + }; + ClickHouseDataConfig widenUtConf = new ClickHouseTestDataConfig() { + @Override + public boolean isWidenUnsignedTypes() { + return true; + } + }; return new Object[][] { { defaultConf, "Nullable(Bool)", "null", new ClickHouseValue[] { ClickHouseBoolValue.ofNull() } }, @@ -915,7 +975,7 @@ protected Object[][] getSimpleTypesForWrite() { }; } - protected ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseConfig config, + protected ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseDataConfig config, ClickHouseColumn column, ClickHouseInputStream input) throws IOException { if (ref == null) { ref = column.newValue(config); @@ -923,7 +983,7 @@ protected ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseConfig conf return getDataProcessor(config, column, input, null).getDeserializer(config, column).deserialize(ref, input); } - protected void serialize(ClickHouseValue value, ClickHouseConfig config, ClickHouseColumn column, + protected void serialize(ClickHouseValue value, ClickHouseDataConfig config, ClickHouseColumn column, ClickHouseOutputStream output) throws IOException { ClickHouseDataProcessor p = getDataProcessor(config, column, null, output); ClickHouseValue v = column.newValue(config); @@ -936,7 +996,7 @@ protected void serialize(ClickHouseValue value, ClickHouseConfig config, ClickHo } @Test(dataProvider = "nestedTypesForRead", groups = { "unit" }) - public void testDeserializeNestedTypes(ClickHouseConfig config, String typeName, String dataKey, + public void testDeserializeNestedTypes(ClickHouseDataConfig config, String typeName, String dataKey, String valueClass, Object arrVal, Object objVal, String strVal, String sqlExpr) throws IOException { try (ClickHouseInputStream in = getInputData(typeName, dataKey)) { ClickHouseColumn column = ClickHouseColumn.of("a", typeName); @@ -952,7 +1012,7 @@ public void testDeserializeNestedTypes(ClickHouseConfig config, String typeName, } @Test(dataProvider = "nestedTypesForWrite", groups = { "unit" }) - public void testSerializeNestedTypes(ClickHouseConfig config, String typeName, String dataKey, + public void testSerializeNestedTypes(ClickHouseDataConfig config, String typeName, String dataKey, ClickHouseValue... values) throws IOException { if (values == null || values.length == 0) { Assert.fail("At least one value must be provided for testing"); @@ -989,7 +1049,7 @@ public void testSerializeNestedTypes(ClickHouseConfig config, String typeName, S } @Test(dataProvider = "simpleTypesForRead", groups = { "unit" }) - public void testDeserializeSimpleTypes(ClickHouseConfig config, String typeName, String dataKey, + public void testDeserializeSimpleTypes(ClickHouseDataConfig config, String typeName, String dataKey, Class valueClass, Object objVal, String strVal, Consumer customChecks) throws IOException { try (ClickHouseInputStream in = getInputData(typeName, dataKey)) { @@ -1015,7 +1075,7 @@ public void testDeserializeSimpleTypes(ClickHouseConfig config, String typeName, } @Test(dataProvider = "simpleTypesForWrite", groups = { "unit" }) - public void testSerializeSimpleTypes(ClickHouseConfig config, String typeName, String dataKey, + public void testSerializeSimpleTypes(ClickHouseDataConfig config, String typeName, String dataKey, ClickHouseValue... values) throws IOException { if (values == null || values.length == 0) { Assert.fail("At least one value must be provided for testing"); diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseBitmapTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseBitmapTest.java similarity index 96% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseBitmapTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseBitmapTest.java index 441fe2715..27688751d 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseBitmapTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseBitmapTest.java @@ -1,6 +1,6 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; -import com.clickhouse.client.ClickHouseDataType; +import com.clickhouse.data.ClickHouseDataType; import org.roaringbitmap.RoaringBitmap; import org.roaringbitmap.buffer.ImmutableRoaringBitmap; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseBoolValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseBoolValueTest.java similarity index 98% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseBoolValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseBoolValueTest.java index 6e10f6b62..3c3f7ae2e 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseBoolValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseBoolValueTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -13,8 +13,8 @@ import java.util.UUID; import org.testng.Assert; import org.testng.annotations.Test; -import com.clickhouse.client.BaseClickHouseValueTest; -import com.clickhouse.client.ClickHouseDataType; +import com.clickhouse.data.BaseClickHouseValueTest; +import com.clickhouse.data.ClickHouseDataType; public class ClickHouseBoolValueTest extends BaseClickHouseValueTest { @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseByteValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseByteValueTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseByteValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseByteValueTest.java index bb154e980..edb0b4846 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseByteValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseByteValueTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -13,8 +13,8 @@ import java.util.UUID; import org.testng.Assert; import org.testng.annotations.Test; -import com.clickhouse.client.BaseClickHouseValueTest; -import com.clickhouse.client.ClickHouseDataType; +import com.clickhouse.data.BaseClickHouseValueTest; +import com.clickhouse.data.ClickHouseDataType; public class ClickHouseByteValueTest extends BaseClickHouseValueTest { @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseDateTimeValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseDateTimeValueTest.java similarity index 98% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseDateTimeValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseDateTimeValueTest.java index 36a02d997..02a79833f 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseDateTimeValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseDateTimeValueTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -14,9 +14,9 @@ import org.testng.Assert; import org.testng.annotations.Test; -import com.clickhouse.client.BaseClickHouseValueTest; -import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.BaseClickHouseValueTest; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseValues; public class ClickHouseDateTimeValueTest extends BaseClickHouseValueTest { @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseDateValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseDateValueTest.java similarity index 97% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseDateValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseDateValueTest.java index f4a4aa9b1..ae879884a 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseDateValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseDateValueTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -11,8 +11,8 @@ import java.util.Arrays; import java.util.UUID; import org.testng.annotations.Test; -import com.clickhouse.client.BaseClickHouseValueTest; -import com.clickhouse.client.ClickHouseDataType; +import com.clickhouse.data.BaseClickHouseValueTest; +import com.clickhouse.data.ClickHouseDataType; public class ClickHouseDateValueTest extends BaseClickHouseValueTest { @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseDoubleValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseDoubleValueTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseDoubleValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseDoubleValueTest.java index e28e61584..85424fcd4 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseDoubleValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseDoubleValueTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -14,8 +14,8 @@ import java.util.Collections; import java.util.UUID; import org.testng.annotations.Test; -import com.clickhouse.client.BaseClickHouseValueTest; -import com.clickhouse.client.ClickHouseDataType; +import com.clickhouse.data.BaseClickHouseValueTest; +import com.clickhouse.data.ClickHouseDataType; public class ClickHouseDoubleValueTest extends BaseClickHouseValueTest { @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseEnumValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseEnumValueTest.java similarity index 95% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseEnumValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseEnumValueTest.java index f9d6ff6aa..56ba00acc 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseEnumValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseEnumValueTest.java @@ -1,10 +1,9 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import org.testng.Assert; import org.testng.annotations.Test; -import com.clickhouse.client.BaseClickHouseValueTest; -import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseFormat; +import com.clickhouse.data.BaseClickHouseValueTest; +import com.clickhouse.data.ClickHouseDataType; public class ClickHouseEnumValueTest extends BaseClickHouseValueTest { @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseFloatValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseFloatValueTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseFloatValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseFloatValueTest.java index be121d2d7..178a2863c 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseFloatValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseFloatValueTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -13,8 +13,8 @@ import java.util.Arrays; import java.util.UUID; import org.testng.annotations.Test; -import com.clickhouse.client.BaseClickHouseValueTest; -import com.clickhouse.client.ClickHouseDataType; +import com.clickhouse.data.BaseClickHouseValueTest; +import com.clickhouse.data.ClickHouseDataType; public class ClickHouseFloatValueTest extends BaseClickHouseValueTest { @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseGeoPointValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseGeoPointValueTest.java similarity index 96% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseGeoPointValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseGeoPointValueTest.java index 7a969ea1c..5312ba88f 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseGeoPointValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseGeoPointValueTest.java @@ -1,9 +1,9 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.util.Arrays; import org.testng.Assert; import org.testng.annotations.Test; -import com.clickhouse.client.ClickHouseDataType; +import com.clickhouse.data.ClickHouseDataType; public class ClickHouseGeoPointValueTest { @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseIntegerValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseIntegerValueTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseIntegerValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseIntegerValueTest.java index 83d2919fe..3f042b43a 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseIntegerValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseIntegerValueTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -12,8 +12,8 @@ import java.util.Arrays; import java.util.UUID; import org.testng.annotations.Test; -import com.clickhouse.client.BaseClickHouseValueTest; -import com.clickhouse.client.ClickHouseDataType; +import com.clickhouse.data.BaseClickHouseValueTest; +import com.clickhouse.data.ClickHouseDataType; public class ClickHouseIntegerValueTest extends BaseClickHouseValueTest { @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseLongValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseLongValueTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseLongValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseLongValueTest.java index ebd4d36b5..dc1ff74d8 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseLongValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseLongValueTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -13,9 +13,9 @@ import java.util.UUID; import org.junit.Assert; import org.testng.annotations.Test; -import com.clickhouse.client.BaseClickHouseValueTest; -import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.BaseClickHouseValueTest; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseValues; public class ClickHouseLongValueTest extends BaseClickHouseValueTest { @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseNestedValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseNestedValueTest.java similarity index 98% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseNestedValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseNestedValueTest.java index f6c194ab9..01de693bc 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseNestedValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseNestedValueTest.java @@ -1,10 +1,10 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.time.LocalDate; import java.util.Arrays; import org.testng.annotations.Test; -import com.clickhouse.client.BaseClickHouseValueTest; -import com.clickhouse.client.ClickHouseColumn; +import com.clickhouse.data.BaseClickHouseValueTest; +import com.clickhouse.data.ClickHouseColumn; public class ClickHouseNestedValueTest extends BaseClickHouseValueTest { @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseOffsetDateTimeValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseOffsetDateTimeValueTest.java similarity index 98% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseOffsetDateTimeValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseOffsetDateTimeValueTest.java index af3426bc7..133f625cc 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseOffsetDateTimeValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseOffsetDateTimeValueTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -17,9 +17,9 @@ import org.testng.Assert; import org.testng.annotations.Test; -import com.clickhouse.client.BaseClickHouseValueTest; -import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.BaseClickHouseValueTest; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseValues; public class ClickHouseOffsetDateTimeValueTest extends BaseClickHouseValueTest { @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseShortValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseShortValueTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseShortValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseShortValueTest.java index 85084b175..b1453f6cc 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseShortValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseShortValueTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -12,8 +12,8 @@ import java.util.Arrays; import java.util.UUID; import org.testng.annotations.Test; -import com.clickhouse.client.BaseClickHouseValueTest; -import com.clickhouse.client.ClickHouseDataType; +import com.clickhouse.data.BaseClickHouseValueTest; +import com.clickhouse.data.ClickHouseDataType; public class ClickHouseShortValueTest extends BaseClickHouseValueTest { @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseStringValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseStringValueTest.java similarity index 98% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseStringValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseStringValueTest.java index 551af1a9c..04dcb0e3c 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseStringValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseStringValueTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigDecimal; import java.math.BigInteger; @@ -10,8 +10,8 @@ import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import com.clickhouse.client.BaseClickHouseValueTest; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.BaseClickHouseValueTest; +import com.clickhouse.data.ClickHouseValues; public class ClickHouseStringValueTest extends BaseClickHouseValueTest { @DataProvider(name = "stateProvider") diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseTupleValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseTupleValueTest.java similarity index 98% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseTupleValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseTupleValueTest.java index 52b720acb..c3583c33b 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseTupleValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/ClickHouseTupleValueTest.java @@ -1,8 +1,8 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.util.Arrays; import org.testng.annotations.Test; -import com.clickhouse.client.BaseClickHouseValueTest; +import com.clickhouse.data.BaseClickHouseValueTest; public class ClickHouseTupleValueTest extends BaseClickHouseValueTest { @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/UnsignedByteTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/UnsignedByteTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/UnsignedByteTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/UnsignedByteTest.java index 0b277a0bf..b78157623 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/UnsignedByteTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/UnsignedByteTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/UnsignedIntegerTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/UnsignedIntegerTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/UnsignedIntegerTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/UnsignedIntegerTest.java index d0afb13ee..5f4cc6cd9 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/UnsignedIntegerTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/UnsignedIntegerTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/UnsignedLongTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/UnsignedLongTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/UnsignedLongTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/UnsignedLongTest.java index c57c5cc88..38a0c3705 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/UnsignedLongTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/UnsignedLongTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.math.BigInteger; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/UnsignedShortTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/UnsignedShortTest.java similarity index 99% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/UnsignedShortTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/UnsignedShortTest.java index 97f18331f..21bb54308 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/UnsignedShortTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/UnsignedShortTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/WriterFunction.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/WriterFunction.java similarity index 82% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/WriterFunction.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/WriterFunction.java index cadfd745f..3f3efcbf7 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/WriterFunction.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/WriterFunction.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data; +package com.clickhouse.data.value; import java.io.IOException; import java.io.OutputStream; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/array/ClickHouseByteArrayValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/array/ClickHouseByteArrayValueTest.java similarity index 96% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/array/ClickHouseByteArrayValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/array/ClickHouseByteArrayValueTest.java index f6cabfb1a..07feff864 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/array/ClickHouseByteArrayValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/array/ClickHouseByteArrayValueTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data.array; +package com.clickhouse.data.value.array; import org.junit.Assert; import org.testng.annotations.Test; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/array/ClickHouseLongArrayValueTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/value/array/ClickHouseLongArrayValueTest.java similarity index 96% rename from clickhouse-client/src/test/java/com/clickhouse/client/data/array/ClickHouseLongArrayValueTest.java rename to clickhouse-data/src/test/java/com/clickhouse/data/value/array/ClickHouseLongArrayValueTest.java index 57e68ee04..662343cf2 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/array/ClickHouseLongArrayValueTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/value/array/ClickHouseLongArrayValueTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.data.array; +package com.clickhouse.data.value.array; import java.math.BigInteger; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/logging/JdkLoggerTest.java b/clickhouse-data/src/test/java/com/clickhouse/logging/JdkLoggerTest.java similarity index 97% rename from clickhouse-client/src/test/java/com/clickhouse/client/logging/JdkLoggerTest.java rename to clickhouse-data/src/test/java/com/clickhouse/logging/JdkLoggerTest.java index cca890682..4861abe89 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/logging/JdkLoggerTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/logging/JdkLoggerTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.logging; +package com.clickhouse.logging; import java.util.Collections; import org.testng.annotations.Test; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/logging/LogMessageTest.java b/clickhouse-data/src/test/java/com/clickhouse/logging/LogMessageTest.java similarity index 97% rename from clickhouse-client/src/test/java/com/clickhouse/client/logging/LogMessageTest.java rename to clickhouse-data/src/test/java/com/clickhouse/logging/LogMessageTest.java index 54bb903e0..cd758491c 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/logging/LogMessageTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/logging/LogMessageTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.logging; +package com.clickhouse.logging; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/logging/LoggerTest.java b/clickhouse-data/src/test/java/com/clickhouse/logging/LoggerTest.java similarity index 98% rename from clickhouse-client/src/test/java/com/clickhouse/client/logging/LoggerTest.java rename to clickhouse-data/src/test/java/com/clickhouse/logging/LoggerTest.java index 5981eb388..1c517dba6 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/logging/LoggerTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/logging/LoggerTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.logging; +package com.clickhouse.logging; import java.util.function.Supplier; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/logging/Slf4jLoggerTest.java b/clickhouse-data/src/test/java/com/clickhouse/logging/Slf4jLoggerTest.java similarity index 97% rename from clickhouse-client/src/test/java/com/clickhouse/client/logging/Slf4jLoggerTest.java rename to clickhouse-data/src/test/java/com/clickhouse/logging/Slf4jLoggerTest.java index 115cfce7a..863967f77 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/logging/Slf4jLoggerTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/logging/Slf4jLoggerTest.java @@ -1,4 +1,4 @@ -package com.clickhouse.client.logging; +package com.clickhouse.logging; import java.util.Collections; import org.testng.annotations.Test; diff --git a/clickhouse-data/src/test/resources/META-INF/services/com.clickhouse.data.TestServiceInterface b/clickhouse-data/src/test/resources/META-INF/services/com.clickhouse.data.TestServiceInterface new file mode 100644 index 000000000..b68f2b1da --- /dev/null +++ b/clickhouse-data/src/test/resources/META-INF/services/com.clickhouse.data.TestServiceInterface @@ -0,0 +1 @@ +com.clickhouse.data.TestServiceImplementation \ No newline at end of file diff --git a/clickhouse-data/src/test/resources/simplelogger.properties b/clickhouse-data/src/test/resources/simplelogger.properties new file mode 100644 index 000000000..c218c4de0 --- /dev/null +++ b/clickhouse-data/src/test/resources/simplelogger.properties @@ -0,0 +1,7 @@ +org.slf4j.simpleLogger.defaultLogLevel=info +org.slf4j.simpleLogger.log.com.clickhouse.data=info +org.slf4j.simpleLogger.showDateTime=true +org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z +org.slf4j.simpleLogger.showThreadName=true +org.slf4j.simpleLogger.showLogName=true +org.slf4j.simpleLogger.showShortLogName=true diff --git a/clickhouse-grpc-client/pom.xml b/clickhouse-grpc-client/pom.xml index 42a7c21d4..618e4c8c4 100644 --- a/clickhouse-grpc-client/pom.xml +++ b/clickhouse-grpc-client/pom.xml @@ -37,6 +37,31 @@ lz4-java + + com.aayushatharva.brotli4j + brotli4j + true + + + com.aayushatharva.brotli4j + native-linux-x86_64 + true + + + com.aayushatharva.brotli4j + native-osx-aarch64 + true + + + com.aayushatharva.brotli4j + native-osx-x86_64 + true + + + com.aayushatharva.brotli4j + native-windows-x86_64 + true + com.github.luben zstd-jni @@ -50,11 +75,6 @@ org.apache.commons commons-compress - - org.brotli - dec - true - io.grpc grpc-protobuf @@ -134,57 +154,93 @@ com.google ${shade.base}.google - - com.squareup.okhttp - ${shade.base}.okhttp - - - okio - ${shade.base}.okio - - - org.apache - ${shade.base}.apache - io.grpc ${shade.base}.grpc - - io.perfmark - ${shade.base}.perfmark - io.opencensus ${shade.base}.opencensus + + io.perfmark + ${shade.base}.perfmark + net.jpountz ${shade.base}.jpountz + + okio + ${shade.base}.okio + - - - - + + + + com.clickhouse.client.grpc + + com.aayushatharva.brotli4j:* + + ** + + + + com.github.luben:zstd-jni + + ** + + + + org.apache.commons:commons-compress + + ** + + + + org.slf4j:slf4j-api + + ** + + + + org.tukaani:xz + + ** + + + + org.xerial.snappy:snappy-java + + ** + + *:* - android/** google/** javax/** + mozilla/** org/checkerframework/** org/codehaus/** + **/darwin/** + **/linux/** + **/win32/** **/module-info.class META-INF/MANIFEST.MF META-INF/maven/** + META-INF/native/** META-INF/native-image/** @@ -207,41 +263,75 @@ com.google ${shade.base}.google - - org.apache - ${shade.base}.apache - io.grpc ${shade.base}.grpc - - io.perfmark - ${shade.base}.perfmark - io.opencensus ${shade.base}.opencensus + + io.perfmark + ${shade.base}.perfmark + net.jpountz ${shade.base}.jpountz - - - + + + + + com.aayushatharva.brotli4j:* + + ** + + + + com.github.luben:zstd-jni + + ** + + + + org.apache.commons:commons-compress + + ** + + + + org.slf4j:slf4j-api + + ** + + + + org.tukaani:xz + + ** + + + + org.xerial.snappy:snappy-java + + ** + + *:* - android/** - com/squareup/** google/** io/grpc/okhttp/** javax/** + mozilla/** okio/** org/checkerframework/** org/codehaus/** @@ -270,18 +360,6 @@ com.google ${shade.base}.google - - com.squareup.okhttp - ${shade.base}.okhttp - - - okio - ${shade.base}.okio - - - org.apache - ${shade.base}.apache - io.grpc ${shade.base}.grpc @@ -298,20 +376,63 @@ net.jpountz ${shade.base}.jpountz + + okio + ${shade.base}.okio + - - - + + + + + com.aayushatharva.brotli4j:* + + ** + + + + com.github.luben:zstd-jni + + ** + + + + org.apache.commons:commons-compress + + ** + + + + org.slf4j:slf4j-api + + ** + + + + org.tukaani:xz + + ** + + + + org.xerial.snappy:snappy-java + + ** + + *:* - android/** google/** io/grpc/netty/** javax/** + mozilla/** org/checkerframework/** org/codehaus/** **/module-info.class diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java index fe346f480..c58b84b1d 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java @@ -16,15 +16,15 @@ import io.grpc.ProxiedSocketAddress; import io.grpc.ProxyDetector; import io.grpc.Status; -import com.clickhouse.client.ClickHouseChecker; import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseNode; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.config.ClickHouseOption; +import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.grpc.config.ClickHouseGrpcOption; import com.clickhouse.client.grpc.impl.ClickHouseGrpc; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; public abstract class ClickHouseGrpcChannelFactory { static class NoProxyDetector implements ProxyDetector { @@ -200,11 +200,11 @@ public ManagedChannel create() { ManagedChannelBuilder builder = getChannelBuilder(); String userAgent = config.getClientName(); - if (ClickHouseOption.DEFAULT_CLIENT_NAME.equals(userAgent)) { + if (ClickHouseClientOption.CLIENT_NAME.getDefaultValue().equals(userAgent)) { userAgent = getDefaultUserAgent(); String name = config.getProductName(); - if (!ClickHouseOption.DEFAULT_PRODUCT_NAME.equals(name)) { + if (!ClickHouseClientOption.PRODUCT_NAME.getDefaultValue().equals(name)) { userAgent = new StringBuilder(name).append(userAgent.substring(userAgent.indexOf('/'))).toString(); } } diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcClient.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcClient.java index 2bbe56b7b..c5fb41ac2 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcClient.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcClient.java @@ -20,26 +20,15 @@ import io.grpc.stub.StreamObserver; import com.clickhouse.client.AbstractClient; -import com.clickhouse.client.ClickHouseChecker; import com.clickhouse.client.ClickHouseClient; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseCompression; import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseCredentials; -import com.clickhouse.client.ClickHouseDataStreamFactory; -import com.clickhouse.client.ClickHouseDeferredValue; import com.clickhouse.client.ClickHouseException; -import com.clickhouse.client.ClickHouseInputStream; import com.clickhouse.client.ClickHouseNode; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHousePipedOutputStream; import com.clickhouse.client.ClickHouseProtocol; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponse; -import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseOption; -import com.clickhouse.client.data.ClickHouseExternalTable; import com.clickhouse.client.grpc.config.ClickHouseGrpcOption; import com.clickhouse.client.grpc.impl.ClickHouseGrpc; import com.clickhouse.client.grpc.impl.ExternalTable; @@ -47,8 +36,19 @@ import com.clickhouse.client.grpc.impl.QueryInfo; import com.clickhouse.client.grpc.impl.Result; import com.clickhouse.client.grpc.impl.QueryInfo.Builder; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseDataStreamFactory; +import com.clickhouse.data.ClickHouseDeferredValue; +import com.clickhouse.data.ClickHouseExternalTable; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePipedOutputStream; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorInputStream; import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorOutputStream; @@ -78,8 +78,8 @@ static ClickHouseInputStream getInput(ClickHouseConfig config, InputStream input }), config.getReadBufferSize(), postCloseAction); break; default: - in = ClickHouseInputStream.wrap(null, input, config.getReadBufferSize(), postCloseAction, - config.getResponseCompressAlgorithm(), config.getResponseCompressLevel()); + in = ClickHouseInputStream.wrap(null, input, config.getReadBufferSize(), + config.getResponseCompressAlgorithm(), config.getResponseCompressLevel(), postCloseAction); break; } return in; @@ -114,6 +114,10 @@ static ClickHouseOutputStream getOutput(ClickHouseConfig config, OutputStream ou protected static ClickHouseInputStream getCompressedInputStream(ClickHouseConfig config, ClickHouseInputStream input) { + if (input.getUnderlyingStream().hasInput()) { + return input; + } + final int bufferSize = config.getWriteBufferSize(); final ClickHousePipedOutputStream stream = ClickHouseDataStreamFactory.getInstance() // NOSONAR .createPipedOutputStream(bufferSize, 0, config.getSocketTimeout(), null); @@ -243,7 +247,7 @@ protected static QueryInfo convert(ClickHouseRequest request, boolean streami ClickHouseCompression inputCompression = config.getRequestCompressAlgorithm(); builder.setInputCompressionType(inputCompression.encoding()); if (streaming) { - builder.setInputData(ByteString.EMPTY); + // builder.setInputData(ByteString.EMPTY); builder.setNextQueryInfo(true); } else { try { diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcResponse.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcResponse.java index b1dabbdb4..5c316eceb 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcResponse.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcResponse.java @@ -7,12 +7,12 @@ import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseException; -import com.clickhouse.client.ClickHouseInputStream; import com.clickhouse.client.ClickHouseResponseSummary; -import com.clickhouse.client.data.ClickHouseStreamResponse; +import com.clickhouse.client.ClickHouseStreamResponse; import com.clickhouse.client.grpc.impl.Progress; import com.clickhouse.client.grpc.impl.Result; import com.clickhouse.client.grpc.impl.Stats; +import com.clickhouse.data.ClickHouseInputStream; public class ClickHouseGrpcResponse extends ClickHouseStreamResponse { private final ClickHouseStreamObserver observer; diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseStreamObserver.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseStreamObserver.java index cb882686b..447edd1e7 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseStreamObserver.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseStreamObserver.java @@ -10,23 +10,24 @@ import io.grpc.StatusException; import io.grpc.stub.StreamObserver; -import com.clickhouse.client.ClickHouseCompression; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseDataStreamFactory; import com.clickhouse.client.ClickHouseException; -import com.clickhouse.client.ClickHouseInputStream; import com.clickhouse.client.ClickHouseNode; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHousePipedOutputStream; import com.clickhouse.client.ClickHouseResponseSummary; -import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.grpc.impl.Exception; import com.clickhouse.client.grpc.impl.LogEntry; import com.clickhouse.client.grpc.impl.Progress; import com.clickhouse.client.grpc.impl.Result; import com.clickhouse.client.grpc.impl.Stats; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseDataConfig; +import com.clickhouse.data.ClickHouseDataStreamFactory; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePipedOutputStream; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; public class ClickHouseStreamObserver implements StreamObserver { private static final Logger log = LoggerFactory.getLogger(ClickHouseStreamObserver.class); @@ -58,7 +59,8 @@ protected ClickHouseStreamObserver(ClickHouseConfig config, ClickHouseNode serve if (output != null) { this.stream = output; this.input = ClickHouseInputStream.wrap(null, ClickHouseInputStream.empty(), - config.getReadBufferSize(), postCloseAction, ClickHouseCompression.NONE, 0); + config.getReadBufferSize(), ClickHouseCompression.NONE, + ClickHouseDataConfig.DEFAULT_READ_COMPRESS_LEVEL, postCloseAction); } else { ClickHousePipedOutputStream pipedStream = ClickHouseDataStreamFactory.getInstance() .createPipedOutputStream(config, null); diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/NettyChannelFactoryImpl.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/NettyChannelFactoryImpl.java index 6e18b5b7b..ef80c8d0f 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/NettyChannelFactoryImpl.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/NettyChannelFactoryImpl.java @@ -11,13 +11,13 @@ import io.grpc.netty.shaded.io.netty.handler.ssl.SslProvider; import io.grpc.netty.shaded.io.netty.handler.ssl.SupportedCipherSuiteFilter; import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory; -import com.clickhouse.client.ClickHouseChecker; import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseNode; -import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseSslMode; import com.clickhouse.client.grpc.config.ClickHouseGrpcOption; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseUtils; final class NettyChannelFactoryImpl extends ClickHouseGrpcChannelFactory { private static final String USER_AGENT = ClickHouseClientOption.buildUserAgent(null, "gRPC-Netty"); diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/config/ClickHouseGrpcOption.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/config/ClickHouseGrpcOption.java index 8c5c296ff..3d3c55c4e 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/config/ClickHouseGrpcOption.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/config/ClickHouseGrpcOption.java @@ -2,8 +2,8 @@ import java.io.Serializable; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.config.ClickHouseOption; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseChecker; /** * gRPC client options. diff --git a/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java b/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java index 0de672edf..77ca53841 100644 --- a/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java +++ b/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java @@ -3,6 +3,7 @@ import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorInputStream; import org.testng.Assert; import org.testng.SkipException; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.ByteArrayInputStream; @@ -12,16 +13,65 @@ import com.clickhouse.client.ClickHouseException; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseProtocol; -import com.clickhouse.client.ClickHouseRecord; import com.clickhouse.client.ClickHouseResponse; import com.clickhouse.client.ClickHouseResponseSummary; -import com.clickhouse.client.ClickHouseVersion; import com.clickhouse.client.ClientIntegrationTest; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.ClickHouseFormat; -import com.clickhouse.client.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseVersion; public class ClickHouseGrpcClientTest extends ClientIntegrationTest { + @DataProvider(name = "requestCompressionMatrix") + @Override + protected Object[][] getRequestCompressionMatrix() { + return new Object[][] { + { ClickHouseCompression.NONE, -2, 2, 1 }, + { ClickHouseCompression.BROTLI, -2, 12, 1 }, // [-1, 11] + { ClickHouseCompression.BZ2, -2, 2, 1 }, + { ClickHouseCompression.DEFLATE, -2, 10, 1 }, // [0, 9] + { ClickHouseCompression.GZIP, -2, 10, 1 }, // [-1, 9] + { ClickHouseCompression.LZ4, -2, 19, 1 }, // [0, 18] + // Code: 638, DB::Exception: hadoop snappy decode error:INVALID_INPUT: While + // executing BinaryRowInputFormat + // { ClickHouseCompression.SNAPPY, -2, 513, 1024 }, // [1 * 1024, 32 * 1024] + { ClickHouseCompression.XZ, -2, 10, 1 }, // [0, 9] + { ClickHouseCompression.ZSTD, -2, 23, 1 }, // [0, 22] + }; + } + + @DataProvider(name = "mixedCompressionMatrix") + @Override + protected Object[][] getMixedCompressionMatrix() { + ClickHouseCompression[] supportedRequestCompression = { + ClickHouseCompression.NONE, + ClickHouseCompression.BROTLI, + ClickHouseCompression.BZ2, + ClickHouseCompression.DEFLATE, + ClickHouseCompression.GZIP, + ClickHouseCompression.LZ4, + ClickHouseCompression.XZ, + ClickHouseCompression.ZSTD + }; + ClickHouseCompression[] supportedResponseCompression = { + ClickHouseCompression.NONE, + ClickHouseCompression.BROTLI, + ClickHouseCompression.DEFLATE, + ClickHouseCompression.LZ4, + ClickHouseCompression.ZSTD + }; + Object[][] matrix = new Object[supportedRequestCompression.length * supportedResponseCompression.length][]; + int i = 0; + for (ClickHouseCompression reqComp : supportedRequestCompression) { + for (ClickHouseCompression respComp : supportedResponseCompression) { + matrix[i++] = new Object[] { reqComp, respComp }; + } + } + return matrix; + } + @Override protected ClickHouseProtocol getProtocol() { if (!ClickHouseVersion.of(System.getProperty("clickhouseVersion", "latest")).check("[22.3,)")) { @@ -36,6 +86,10 @@ protected Class getClientClass() { return ClickHouseGrpcClient.class; } + @Test(groups = { "integration" }) + public void testNothing() throws Exception { + } + @Test(groups = "integration") public void testResponseSummary() throws ClickHouseException { ClickHouseNode server = getServer(); diff --git a/clickhouse-http-client/pom.xml b/clickhouse-http-client/pom.xml index 640001404..e322f9817 100644 --- a/clickhouse-http-client/pom.xml +++ b/clickhouse-http-client/pom.xml @@ -22,22 +22,6 @@ clickhouse-client ${revision} - com.github.luben zstd-jni @@ -66,17 +50,13 @@ org.lz4 lz4-java + true org.tukaani xz true - - org.xerial.snappy - snappy-java - true - @@ -137,19 +117,61 @@ - - - - + + + + com.clickhouse.client.http + + com.github.luben:zstd-jni + + ** + + + + org.apache.commons:commons-compress + + ** + + + + org.apache.httpcomponents.client5:httpclient5 + + ** + + + + org.brotli:dec + + ** + + + + org.slf4j:slf4j-api + + ** + + + + org.tukaani:xz + + ** + + *:* + mozilla/** + org/apache/** **/darwin/** **/linux/** **/win32/** diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java index 842d6bd5a..6e2c74c29 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java @@ -1,21 +1,23 @@ package com.clickhouse.client.http; import com.clickhouse.client.AbstractSocketClient; -import com.clickhouse.client.ClickHouseChecker; + import com.clickhouse.client.ClickHouseClient; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseFormat; -import com.clickhouse.client.ClickHouseInputStream; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseSslContextProvider; -import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseSslMode; -import com.clickhouse.client.data.ClickHouseExternalTable; import com.clickhouse.client.http.config.ClickHouseHttpOption; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseExternalTable; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.classic.methods.HttpPost; import org.apache.hc.client5.http.config.ConnectionConfig; @@ -55,6 +57,8 @@ import java.util.TimeZone; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + import javax.net.ssl.SSLContext; import javax.net.ssl.SSLException; @@ -64,26 +68,29 @@ public class ApacheHttpConnectionImpl extends ClickHouseHttpConnection { private static final Logger log = LoggerFactory.getLogger(ApacheHttpConnectionImpl.class); + private final AtomicBoolean busy; private final CloseableHttpClient client; protected ApacheHttpConnectionImpl(ClickHouseNode server, ClickHouseRequest request, ExecutorService executor) throws IOException { super(server, request); - client = newConnection(); + + busy = new AtomicBoolean(false); + client = newConnection(config); } - private CloseableHttpClient newConnection() throws IOException { + private CloseableHttpClient newConnection(ClickHouseConfig c) throws IOException { RegistryBuilder r = RegistryBuilder.create() - .register("http", SocketFactory.create(config)); - if (config.isSsl()) { - r.register("https", SSLSocketFactory.create(config)); + .register("http", SocketFactory.create(c)); + if (c.isSsl()) { + r.register("https", SSLSocketFactory.create(c)); } - return HttpClientBuilder.create().setConnectionManager(new HttpConnectionManager(r.build(), config)) + return HttpClientBuilder.create().setConnectionManager(new HttpConnectionManager(r.build(), c)) .disableContentCompression().build(); } - private ClickHouseHttpResponse buildResponse(CloseableHttpResponse response, ClickHouseConfig config, - Runnable postCloseAction) throws IOException { + private ClickHouseHttpResponse buildResponse(ClickHouseConfig config, CloseableHttpResponse response, + ClickHouseOutputStream output, Runnable postCloseAction) throws IOException { // X-ClickHouse-Server-Display-Name: xxx // X-ClickHouse-Query-Id: xxx // X-ClickHouse-Format: RowBinaryWithNamesAndTypes @@ -94,10 +101,9 @@ private ClickHouseHttpResponse buildResponse(CloseableHttpResponse response, Cli String queryId = getResponseHeader(response, "X-ClickHouse-Query-Id", ""); String summary = getResponseHeader(response, "X-ClickHouse-Summary", "{}"); - ClickHouseConfig c = config; - ClickHouseFormat format = c.getFormat(); - TimeZone timeZone = c.getServerTimeZone(); - boolean hasOutputFile = output != null && output.getUnderlyingStream().hasOutput(); + ClickHouseFormat format = config.getFormat(); + TimeZone timeZone = config.getServerTimeZone(); + boolean hasCustomOutput = output != null && output.getUnderlyingStream().hasOutput(); boolean hasQueryResult = false; // queryId, format and timeZone are only available for queries if (!ClickHouseChecker.isNullOrEmpty(queryId)) { @@ -117,7 +123,7 @@ private ClickHouseHttpResponse buildResponse(CloseableHttpResponse response, Cli source = ClickHouseInputStream.empty(); action = () -> { try (OutputStream o = output) { - ClickHouseInputStream.pipe(response.getEntity().getContent(), o, c.getWriteBufferSize()); + ClickHouseInputStream.pipe(response.getEntity().getContent(), o, config.getWriteBufferSize()); if (postCloseAction != null) { postCloseAction.run(); } @@ -130,9 +136,9 @@ private ClickHouseHttpResponse buildResponse(CloseableHttpResponse response, Cli action = postCloseAction; } return new ClickHouseHttpResponse(this, - hasOutputFile ? ClickHouseInputStream.of(source, c.getReadBufferSize(), action) - : (hasQueryResult ? ClickHouseClient.getAsyncResponseInputStream(c, source, action) - : ClickHouseClient.getResponseInputStream(c, source, action)), + hasCustomOutput ? ClickHouseInputStream.of(source, config.getReadBufferSize(), action) + : (hasQueryResult ? ClickHouseClient.getAsyncResponseInputStream(config, source, action) + : ClickHouseClient.getResponseInputStream(config, source, action)), displayName, queryId, summary, format, timeZone); } @@ -151,7 +157,7 @@ private void setHeaders(HttpRequest request, Map headers) { } } - private void checkResponse(CloseableHttpResponse response) throws IOException { + private void checkResponse(ClickHouseConfig config, CloseableHttpResponse response) throws IOException { if (response.getCode() == HttpURLConnection.HTTP_OK) { return; } @@ -168,7 +174,7 @@ private void checkResponse(CloseableHttpResponse response) throws IOException { String errorMsg; - int bufferSize = (int) ClickHouseClientOption.BUFFER_SIZE.getDefaultValue(); + int bufferSize = config.getReadBufferSize(); ByteArrayOutputStream output = new ByteArrayOutputStream(bufferSize); ClickHouseInputStream.pipe(response.getEntity().getContent(), output, bufferSize); byte[] bytes = output.toByteArray(); @@ -195,33 +201,42 @@ protected final String getDefaultUserAgent() { @Override protected boolean isReusable() { - return true; + return busy.get(); } - protected ClickHouseHttpResponse post(String sql, ClickHouseInputStream data, List tables, - String url, Map headers, ClickHouseConfig config, Runnable postCloseAction) - throws IOException { - HttpPost post = new HttpPost(url == null ? this.url : url); - setHeaders(post, headers); - byte[] boundary = null; - String contentType = "text/plain; charset=UTF-8"; - if (tables != null && !tables.isEmpty()) { - String uuid = rm.createUniqueId(); - contentType = "multipart/form-data; boundary=".concat(uuid); - boundary = uuid.getBytes(StandardCharsets.US_ASCII); + @Override + protected ClickHouseHttpResponse post(ClickHouseConfig config, String sql, ClickHouseInputStream data, + List tables, ClickHouseOutputStream output, String url, + Map headers, Runnable postCloseAction) throws IOException { + if (!busy.compareAndSet(false, true)) { + throw new ConnectException("Connection is busy"); + } + + try { + HttpPost post = new HttpPost(url == null ? this.url : url); + setHeaders(post, headers); + byte[] boundary = null; + String contentType = "text/plain; charset=UTF-8"; + if (tables != null && !tables.isEmpty()) { + String uuid = rm.createUniqueId(); + contentType = "multipart/form-data; boundary=".concat(uuid); + boundary = uuid.getBytes(StandardCharsets.US_ASCII); + } + post.setHeader("Content-Type", contentType); + + String contentEncoding = headers == null ? null : headers.getOrDefault("content-encoding", null); + ClickHouseHttpEntity postBody = new ClickHouseHttpEntity(config, contentType, contentEncoding, boundary, + sql, data, tables); + post.setEntity(postBody); + CloseableHttpResponse response = client.execute(post); + + checkResponse(config, response); + // buildResponse should use the config of current request in case of reusable + // connection. + return buildResponse(config, response, output, postCloseAction); + } finally { + busy.set(false); } - post.setHeader("Content-Type", contentType); - - String contentEncoding = headers == null ? null : headers.getOrDefault("content-encoding", null); - ClickHouseHttpEntity postBody = new ClickHouseHttpEntity(config, contentType, contentEncoding, boundary, sql, - data, tables); - post.setEntity(postBody); - CloseableHttpResponse response = client.execute(post); - - checkResponse(response); - // buildResponse should use the config of current request in case of reusable - // connection. - return buildResponse(response, config, postCloseAction); } @Override @@ -229,11 +244,12 @@ public boolean ping(int timeout) { String url = getBaseUrl().concat("ping"); HttpGet ping = new HttpGet(url); - try (CloseableHttpClient httpClient = newConnection(); + ClickHouseConfig c = config; + try (CloseableHttpClient httpClient = newConnection(c); CloseableHttpResponse response = httpClient.execute(ping)) { // TODO set timeout - checkResponse(response); - String ok = config.getStrOption(ClickHouseHttpOption.DEFAULT_RESPONSE); + checkResponse(c, response); + String ok = c.getStrOption(ClickHouseHttpOption.DEFAULT_RESPONSE); return ok.equals(EntityUtils.toString(response.getEntity())); } catch (Exception e) { @@ -293,9 +309,9 @@ static class HttpConnectionManager extends PoolingHttpClientConnectionManager { static { String versionInfo = null; try { - versionInfo = VersionInfo - .getSoftwareInfo(PROVIDER, "org.apache.hc.client5", HttpClientBuilder.class) - .split("\\s")[0]; + String pkg = VersionInfo.class.getPackageName(); + pkg = pkg.substring(0, pkg.lastIndexOf('.')); + versionInfo = VersionInfo.getSoftwareInfo(PROVIDER, pkg, HttpClientBuilder.class).split("\\s")[0]; } catch (Throwable e) { // NOSONAR // ignore } diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpClient.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpClient.java index 3102a4a66..b455d8694 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpClient.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpClient.java @@ -17,11 +17,11 @@ import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponse; import com.clickhouse.client.ClickHouseTransaction; -import com.clickhouse.client.config.ClickHouseOption; -import com.clickhouse.client.data.ClickHouseStreamResponse; +import com.clickhouse.client.ClickHouseStreamResponse; import com.clickhouse.client.http.config.ClickHouseHttpOption; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; public class ClickHouseHttpClient extends AbstractClient { private static final Logger log = LoggerFactory.getLogger(ClickHouseHttpClient.class); @@ -101,7 +101,7 @@ protected ClickHouseResponse send(ClickHouseRequest sealedRequest) throws Cli } log.debug("Query: %s", sql); - ClickHouseConfig config = sealedRequest.getConfig(); + final ClickHouseConfig config = sealedRequest.getConfig(); final ClickHouseHttpResponse httpResponse; final ClickHouseTransaction tx = sealedRequest.getTransaction(); final Runnable postAction = tx != null && tx.isImplicit() @@ -115,14 +115,15 @@ protected ClickHouseResponse send(ClickHouseRequest sealedRequest) throws Cli : null; if (conn.isReusable()) { ClickHouseNode server = sealedRequest.getServer(); - httpResponse = conn.post(sql, sealedRequest.getInputStream().orElse(null), - sealedRequest.getExternalTables(), + httpResponse = conn.post(config, sql, sealedRequest.getInputStream().orElse(null), + sealedRequest.getExternalTables(), sealedRequest.getOutputStream().orElse(null), ClickHouseHttpConnection.buildUrl(server.getBaseUri(), sealedRequest), - ClickHouseHttpConnection.createDefaultHeaders(config, server, conn.getUserAgent()), config, + ClickHouseHttpConnection.createDefaultHeaders(config, server, conn.getUserAgent()), postAction); } else { - httpResponse = conn.post(sql, sealedRequest.getInputStream().orElse(null), - sealedRequest.getExternalTables(), null, null, config, postAction); + httpResponse = conn.post(config, sql, sealedRequest.getInputStream().orElse(null), + sealedRequest.getExternalTables(), sealedRequest.getOutputStream().orElse(null), null, null, + postAction); } return ClickHouseStreamResponse.of(httpResponse.getConfig(sealedRequest), httpResponse.getInputStream(), sealedRequest.getSettings(), null, httpResponse.summary); diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java index 2c684069c..da8cd5c6e 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java @@ -15,23 +15,23 @@ import java.util.Optional; import java.util.Map.Entry; -import com.clickhouse.client.ClickHouseChecker; import com.clickhouse.client.ClickHouseClient; -import com.clickhouse.client.ClickHouseCompression; import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseCredentials; -import com.clickhouse.client.ClickHouseInputStream; import com.clickhouse.client.ClickHouseNode; -import com.clickhouse.client.ClickHouseOutputStream; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseRequestManager; -import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseOption; -import com.clickhouse.client.data.ClickHouseExternalTable; import com.clickhouse.client.http.config.ClickHouseHttpOption; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseExternalTable; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; public abstract class ClickHouseHttpConnection implements AutoCloseable { private static final Logger log = LoggerFactory.getLogger(ClickHouseHttpConnection.class); @@ -261,15 +261,15 @@ protected static String parseErrorFromException(String errorCode, String serverN protected static void postData(ClickHouseConfig config, byte[] boundary, String sql, ClickHouseInputStream data, List tables, OutputStream requestStream) throws IOException { - final boolean hasFile = data != null && data.getUnderlyingStream().hasInput(); + final boolean hasCustomInput = data != null && data.getUnderlyingStream().hasInput(); try (OutputStream rawOut = requestStream; - ClickHouseOutputStream out = hasFile + ClickHouseOutputStream out = hasCustomInput ? ClickHouseOutputStream.of(rawOut, config.getWriteBufferSize()) : (data != null || boundary != null // NOSONAR ? ClickHouseClient.getAsyncRequestOutputStream(config, rawOut, null) // latch::countDown) : ClickHouseClient.getRequestOutputStream(config, rawOut, null))) { - byte[] sqlBytes = hasFile ? new byte[0] : sql.getBytes(StandardCharsets.UTF_8); + byte[] sqlBytes = hasCustomInput ? new byte[0] : sql.getBytes(StandardCharsets.UTF_8); if (boundary != null) { rawOut.write(LINE_PREFIX); rawOut.write(boundary); @@ -321,24 +321,25 @@ protected static void postData(ClickHouseConfig config, byte[] boundary, String } } - protected final ClickHouseConfig config; protected final ClickHouseNode server; - protected final ClickHouseOutputStream output; - protected final String url; - protected final Map defaultHeaders; protected final ClickHouseRequestManager rm; + protected final ClickHouseConfig config; + protected final Map defaultHeaders; + protected final String url; + protected ClickHouseHttpConnection(ClickHouseNode server, ClickHouseRequest request) { if (server == null || request == null) { throw new IllegalArgumentException("Non-null server and request are required"); } - this.config = request.getConfig(); this.server = server; - this.output = request.getOutputStream().orElse(null); - this.url = buildUrl(server.getBaseUri(), request); - this.defaultHeaders = Collections.unmodifiableMap(createDefaultHeaders(config, server, getUserAgent())); this.rm = request.getManager(); + + ClickHouseConfig c = request.getConfig(); + this.config = c; + this.defaultHeaders = Collections.unmodifiableMap(createDefaultHeaders(c, server, getUserAgent())); + this.url = buildUrl(server.getBaseUri(), request); } protected void closeQuietly() { @@ -350,13 +351,14 @@ protected void closeQuietly() { } protected String getBaseUrl() { - int index = url.indexOf('?'); + String u = url; + int index = u.indexOf('?'); if (index < 1) { - index = url.length(); + index = u.length(); } - String baseUrl = url.substring(0, index); - if (url.charAt(index - 1) != '/') { + String baseUrl = u.substring(0, index); + if (u.charAt(index - 1) != '/') { baseUrl = baseUrl.concat("/"); } @@ -368,14 +370,15 @@ protected String getDefaultUserAgent() { } protected final String getUserAgent() { - String name = config.getClientName(); - if (!ClickHouseOption.DEFAULT_CLIENT_NAME.equals(name)) { + final ClickHouseConfig c = config; + String name = c.getClientName(); + if (!ClickHouseClientOption.CLIENT_NAME.getDefaultValue().equals(name)) { return name; } String userAgent = getDefaultUserAgent(); - name = config.getProductName(); - return ClickHouseOption.DEFAULT_PRODUCT_NAME.equals(name) ? userAgent + name = c.getProductName(); + return ClickHouseClientOption.PRODUCT_NAME.getDefaultValue().equals(name) ? userAgent : new StringBuilder(name).append(userAgent.substring(userAgent.indexOf('/'))).toString(); } @@ -410,19 +413,20 @@ protected Map mergeHeaders(Map requestHeaders) { * Posts query and data to server. * * @param query non-blank query + * @param config non-null configuration * @param data optionally input stream for batch updating * @param tables optionally external tables for query + * @param output optionally output stream * @param url optionally url * @param headers optionally request headers - * @param config optionally configuration * @param postCloseAction optionally post action * @return response * @throws IOException when error occured posting request and/or server failed * to respond */ - protected abstract ClickHouseHttpResponse post(String query, ClickHouseInputStream data, - List tables, String url, Map headers, ClickHouseConfig config, - Runnable postCloseAction) throws IOException; + protected abstract ClickHouseHttpResponse post(ClickHouseConfig config, String query, ClickHouseInputStream data, + List tables, ClickHouseOutputStream output, String url, + Map headers, Runnable postCloseAction) throws IOException; /** * Checks whether the connection is reusable or not. This method will be called @@ -447,36 +451,41 @@ protected boolean isReusable() { public abstract boolean ping(int timeout); public ClickHouseHttpResponse update(String query) throws IOException { - return post(query, null, null, null, null, null, null); + return post(config, query, null, null, null, null, null, null); } public ClickHouseHttpResponse update(String query, Map headers) throws IOException { - return post(query, null, null, null, headers, null, null); + return post(config, query, null, null, null, null, headers, null); } public ClickHouseHttpResponse update(String query, ClickHouseInputStream data) throws IOException { - return post(query, data, null, null, null, null, null); + return post(config, query, data, null, null, null, null, null); } public ClickHouseHttpResponse update(String query, ClickHouseInputStream data, Map headers) throws IOException { - return post(query, data, null, null, headers, null, null); + return post(config, query, data, null, null, null, headers, null); } public ClickHouseHttpResponse query(String query) throws IOException { - return post(query, null, null, null, null, null, null); + return post(config, query, null, null, null, null, null, null); } public ClickHouseHttpResponse query(String query, Map headers) throws IOException { - return post(query, null, null, null, headers, null, null); + return post(config, query, null, null, null, null, headers, null); } public ClickHouseHttpResponse query(String query, List tables) throws IOException { - return post(query, null, tables, null, null, null, null); + return post(config, query, null, tables, null, null, null, null); } public ClickHouseHttpResponse query(String query, List tables, Map headers) throws IOException { - return post(query, null, tables, null, headers, null, null); + return post(config, query, null, tables, null, null, headers, null); + } + + @Override + public void close() throws IOException { + // ignore } } diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpEntity.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpEntity.java index 7fde27b92..5a3e56a4d 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpEntity.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpEntity.java @@ -1,8 +1,8 @@ package com.clickhouse.client.http; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.data.ClickHouseExternalTable; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseExternalTable; import org.apache.hc.core5.http.io.entity.AbstractHttpEntity; diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpResponse.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpResponse.java index 3c200460f..cef289e28 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpResponse.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpResponse.java @@ -5,15 +5,15 @@ import java.util.Map; import java.util.TimeZone; -import com.clickhouse.client.ClickHouseChecker; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseFormat; -import com.clickhouse.client.ClickHouseInputStream; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponseSummary; -import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseOption; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseUtils; public class ClickHouseHttpResponse { private static long getLongValue(Map map, String key) { diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/HttpUrlConnectionImpl.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/HttpUrlConnectionImpl.java index 90857f0f8..8661659a3 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/HttpUrlConnectionImpl.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/HttpUrlConnectionImpl.java @@ -1,20 +1,21 @@ package com.clickhouse.client.http; -import com.clickhouse.client.ClickHouseChecker; import com.clickhouse.client.ClickHouseClient; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseFormat; -import com.clickhouse.client.ClickHouseInputStream; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseSslContextProvider; -import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseSslMode; -import com.clickhouse.client.data.ClickHouseExternalTable; import com.clickhouse.client.http.config.ClickHouseHttpOption; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseExternalTable; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -46,7 +47,8 @@ public class HttpUrlConnectionImpl extends ClickHouseHttpConnection { private final HttpURLConnection conn; - private ClickHouseHttpResponse buildResponse(Runnable postCloseAction) throws IOException { + private ClickHouseHttpResponse buildResponse(ClickHouseOutputStream output, Runnable postCloseAction) + throws IOException { // X-ClickHouse-Server-Display-Name: xxx // X-ClickHouse-Query-Id: xxx // X-ClickHouse-Format: RowBinaryWithNamesAndTypes @@ -60,7 +62,7 @@ private ClickHouseHttpResponse buildResponse(Runnable postCloseAction) throws IO ClickHouseConfig c = config; ClickHouseFormat format = c.getFormat(); TimeZone timeZone = c.getServerTimeZone(); - boolean hasOutputFile = output != null && output.getUnderlyingStream().hasOutput(); + boolean hasCustomOutput = output != null && output.getUnderlyingStream().hasOutput(); boolean hasQueryResult = false; // queryId, format and timeZone are only available for queries if (!ClickHouseChecker.isNullOrEmpty(queryId)) { @@ -93,22 +95,23 @@ private ClickHouseHttpResponse buildResponse(Runnable postCloseAction) throws IO action = postCloseAction; } return new ClickHouseHttpResponse(this, - hasOutputFile ? ClickHouseInputStream.of(source, c.getReadBufferSize(), action) + hasCustomOutput ? ClickHouseInputStream.of(source, c.getReadBufferSize(), action) : (hasQueryResult ? ClickHouseClient.getAsyncResponseInputStream(c, source, action) : ClickHouseClient.getResponseInputStream(c, source, action)), displayName, queryId, summary, format, timeZone); } private HttpURLConnection newConnection(String url, boolean post) throws IOException { - HttpURLConnection newConn = config.isUseNoProxy() + ClickHouseConfig c = config; + HttpURLConnection newConn = c.isUseNoProxy() ? (HttpURLConnection) new URL(url).openConnection(Proxy.NO_PROXY) : (HttpURLConnection) new URL(url).openConnection(); - if ((newConn instanceof HttpsURLConnection) && config.isSsl()) { + if ((newConn instanceof HttpsURLConnection) && c.isSsl()) { HttpsURLConnection secureConn = (HttpsURLConnection) newConn; - SSLContext sslContext = ClickHouseSslContextProvider.getProvider().getSslContext(SSLContext.class, config) + SSLContext sslContext = ClickHouseSslContextProvider.getProvider().getSslContext(SSLContext.class, c) .orElse(null); - HostnameVerifier verifier = config.getSslMode() == ClickHouseSslMode.STRICT + HostnameVerifier verifier = c.getSslMode() == ClickHouseSslMode.STRICT ? HttpsURLConnection.getDefaultHostnameVerifier() : (hostname, session) -> true; // NOSONAR @@ -126,8 +129,8 @@ private HttpURLConnection newConnection(String url, boolean post) throws IOExcep newConn.setAllowUserInteraction(false); newConn.setDoInput(true); newConn.setDoOutput(true); - newConn.setConnectTimeout(config.getConnectionTimeout()); - newConn.setReadTimeout(config.getSocketTimeout()); + newConn.setConnectTimeout(c.getConnectionTimeout()); + newConn.setReadTimeout(c.getSocketTimeout()); return newConn; } @@ -200,9 +203,9 @@ protected boolean isReusable() { } @Override - protected ClickHouseHttpResponse post(String sql, ClickHouseInputStream data, List tables, - String url, Map headers, ClickHouseConfig config, Runnable postCloseAction) - throws IOException { + protected ClickHouseHttpResponse post(ClickHouseConfig config, String sql, ClickHouseInputStream data, + List tables, ClickHouseOutputStream output, String url, + Map headers, Runnable postCloseAction) throws IOException { byte[] boundary = null; if (tables != null && !tables.isEmpty()) { String uuid = rm.createUniqueId(); @@ -223,7 +226,7 @@ protected ClickHouseHttpResponse post(String sql, ClickHouseInputStream data, Li checkResponse(conn); - return buildResponse(postCloseAction); + return buildResponse(output, postCloseAction); } @Override diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/config/ClickHouseHttpOption.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/config/ClickHouseHttpOption.java index 5818ec41d..4c212a1dd 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/config/ClickHouseHttpOption.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/config/ClickHouseHttpOption.java @@ -1,7 +1,7 @@ package com.clickhouse.client.http.config; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.config.ClickHouseOption; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseChecker; import java.io.Serializable; diff --git a/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/ClickHouseResponseHandler.java b/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/ClickHouseResponseHandler.java index e739d3250..8b98b2348 100644 --- a/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/ClickHouseResponseHandler.java +++ b/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/ClickHouseResponseHandler.java @@ -13,8 +13,8 @@ import java.util.concurrent.Flow.Subscription; import java.util.concurrent.atomic.AtomicBoolean; -import com.clickhouse.client.ClickHouseByteBuffer; -import com.clickhouse.client.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseByteBuffer; +import com.clickhouse.data.ClickHouseInputStream; public class ClickHouseResponseHandler implements BodySubscriber { // An immutable ByteBuffer sentinel to mark that the last byte was received. diff --git a/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/HttpClientConnectionImpl.java b/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/HttpClientConnectionImpl.java index 7cbbed028..fe44eb7c7 100644 --- a/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/HttpClientConnectionImpl.java +++ b/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/HttpClientConnectionImpl.java @@ -1,20 +1,21 @@ package com.clickhouse.client.http; -import com.clickhouse.client.ClickHouseChecker; import com.clickhouse.client.ClickHouseClient; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseDataStreamFactory; -import com.clickhouse.client.ClickHouseFormat; -import com.clickhouse.client.ClickHouseInputStream; import com.clickhouse.client.ClickHouseNode; -import com.clickhouse.client.ClickHousePipedOutputStream; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseSslContextProvider; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.data.ClickHouseExternalTable; import com.clickhouse.client.http.config.ClickHouseHttpOption; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseDataStreamFactory; +import com.clickhouse.data.ClickHouseExternalTable; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePipedOutputStream; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -48,6 +49,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; import javax.net.ssl.SSLContext; @@ -75,11 +77,12 @@ public List select(URI uri) { private static final String USER_AGENT = ClickHouseClientOption.buildUserAgent(null, "HttpClient"); + private final AtomicBoolean busy; private final HttpClient httpClient; private final HttpRequest pingRequest; private ClickHouseHttpResponse buildResponse(ClickHouseConfig config, HttpResponse r, - Runnable postAction) throws IOException { + ClickHouseOutputStream output, Runnable postAction) throws IOException { HttpHeaders headers = r.headers(); String displayName = headers.firstValue("X-ClickHouse-Server-Display-Name").orElse(server.getHost()); String queryId = headers.firstValue("X-ClickHouse-Query-Id").orElse(""); @@ -97,7 +100,7 @@ private ClickHouseHttpResponse buildResponse(ClickHouseConfig config, HttpRespon : timeZone; } - boolean hasOutputFile = output != null && output.getUnderlyingStream().hasOutput(); + boolean hasCustomOutput = output != null && output.getUnderlyingStream().hasOutput(); final InputStream source; final Runnable action; if (output != null) { @@ -125,9 +128,9 @@ private ClickHouseHttpResponse buildResponse(ClickHouseConfig config, HttpRespon } return new ClickHouseHttpResponse(this, - hasOutputFile ? ClickHouseInputStream.of(source, config.getReadBufferSize(), action) - : ClickHouseInputStream.wrap(null, source, config.getReadBufferSize(), action, - config.getResponseCompressAlgorithm(), config.getResponseCompressLevel()), + hasCustomOutput ? ClickHouseInputStream.of(source, config.getReadBufferSize(), action) + : ClickHouseInputStream.wrap(null, source, config.getReadBufferSize(), + config.getResponseCompressAlgorithm(), config.getResponseCompressLevel(), action), displayName, queryId, summary, format, timeZone); } @@ -185,13 +188,14 @@ protected HttpClientConnectionImpl(ClickHouseNode server, ClickHouseRequest r .orElse(null)); } + busy = new AtomicBoolean(false); httpClient = builder.build(); pingRequest = newRequest(getBaseUrl() + "ping"); } @Override protected boolean isReusable() { - return true; + return busy.get(); } private CompletableFuture> postRequest(HttpRequest request) { @@ -202,52 +206,61 @@ private CompletableFuture> postRequest(HttpRequest req } private ClickHouseHttpResponse postStream(ClickHouseConfig config, HttpRequest.Builder reqBuilder, byte[] boundary, - String sql, ClickHouseInputStream data, List tables, Runnable postAction) - throws IOException { - ClickHousePipedOutputStream stream = ClickHouseDataStreamFactory.getInstance().createPipedOutputStream(config, - null); - reqBuilder.POST(HttpRequest.BodyPublishers.ofInputStream(stream::getInputStream)); - // running in async is necessary to avoid deadlock of the piped stream - CompletableFuture> f = postRequest(reqBuilder.build()); - - postData(config, boundary, sql, data, tables, stream); - - HttpResponse r; + String sql, ClickHouseInputStream data, List tables, ClickHouseOutputStream output, + Runnable postAction) throws IOException { try { - r = f.get(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IOException("Thread was interrupted when posting request or receiving response", e); - } catch (ExecutionException e) { - Throwable cause = e.getCause(); - if (cause instanceof HttpConnectTimeoutException) { - throw new ConnectException(cause.getMessage()); - } else { - throw new IOException("Failed to post request", cause); + ClickHousePipedOutputStream stream = ClickHouseDataStreamFactory.getInstance().createPipedOutputStream( + config, + null); + reqBuilder.POST(HttpRequest.BodyPublishers.ofInputStream(stream::getInputStream)); + // running in async is necessary to avoid deadlock of the piped stream + CompletableFuture> f = postRequest(reqBuilder.build()); + + postData(config, boundary, sql, data, tables, stream); + + HttpResponse r; + try { + r = f.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Thread was interrupted when posting request or receiving response", e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof HttpConnectTimeoutException) { + throw new ConnectException(cause.getMessage()); + } else { + throw new IOException("Failed to post request", cause); + } } - } - return buildResponse(config, r, postAction); + return buildResponse(config, r, output, postAction); + } finally { + busy.set(false); + } } private ClickHouseHttpResponse postString(ClickHouseConfig config, HttpRequest.Builder reqBuilder, String sql, - Runnable postAction) throws IOException { - reqBuilder.POST(HttpRequest.BodyPublishers.ofString(sql)); - HttpResponse r; + ClickHouseOutputStream output, Runnable postAction) throws IOException { try { - r = postRequest(reqBuilder.build()).get(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IOException("Thread was interrupted when posting request or receiving response", e); - } catch (ExecutionException e) { - Throwable cause = e.getCause(); - if (cause instanceof HttpConnectTimeoutException) { - throw new ConnectException(cause.getMessage()); - } else { - throw new IOException("Failed to post query", cause); + reqBuilder.POST(HttpRequest.BodyPublishers.ofString(sql)); + HttpResponse r; + try { + r = postRequest(reqBuilder.build()).get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Thread was interrupted when posting request or receiving response", e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof HttpConnectTimeoutException) { + throw new ConnectException(cause.getMessage()); + } else { + throw new IOException("Failed to post query", cause); + } } + return buildResponse(config, r, output, postAction); + } finally { + busy.set(false); } - return buildResponse(config, r, postAction); } @Override @@ -256,8 +269,12 @@ protected final String getDefaultUserAgent() { } @Override - protected ClickHouseHttpResponse post(String sql, ClickHouseInputStream data, List tables, - String url, Map headers, ClickHouseConfig config, Runnable postAction) throws IOException { + protected ClickHouseHttpResponse post(ClickHouseConfig config, String sql, ClickHouseInputStream data, + List tables, ClickHouseOutputStream output, String url, + Map headers, Runnable postAction) throws IOException { + if (!busy.compareAndSet(false, true)) { + throw new ConnectException("Connection is busy"); + } ClickHouseConfig c = config == null ? this.config : config; HttpRequest.Builder reqBuilder = HttpRequest.newBuilder() .uri(URI.create(ClickHouseChecker.isNullOrEmpty(url) ? this.url : url)) @@ -279,8 +296,8 @@ protected ClickHouseHttpResponse post(String sql, ClickHouseInputStream data, Li } return boundary != null || data != null || c.isRequestCompressed() - ? postStream(c, reqBuilder, boundary, sql, data, tables, postAction) - : postString(c, reqBuilder, sql, postAction); + ? postStream(c, reqBuilder, boundary, sql, data, tables, output, postAction) + : postString(c, reqBuilder, sql, output, postAction); } @Override diff --git a/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ApacheHttpConnectionImplTest.java b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ApacheHttpConnectionImplTest.java index 17a84a5e2..c419a821d 100644 --- a/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ApacheHttpConnectionImplTest.java +++ b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ApacheHttpConnectionImplTest.java @@ -5,9 +5,9 @@ import com.clickhouse.client.ClickHouseProtocol; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponse; -import com.clickhouse.client.config.ClickHouseOption; import com.clickhouse.client.http.config.ClickHouseHttpOption; import com.clickhouse.client.http.config.HttpConnectionProvider; +import com.clickhouse.config.ClickHouseOption; import java.io.Serializable; import java.util.Collections; diff --git a/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpClientTest.java b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpClientTest.java index 6fc30f5ea..38cb05eae 100644 --- a/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpClientTest.java +++ b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpClientTest.java @@ -1,7 +1,5 @@ package com.clickhouse.client.http; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.Serializable; import java.util.Collections; @@ -9,35 +7,32 @@ import java.util.UUID; import com.clickhouse.client.ClickHouseClient; -import com.clickhouse.client.ClickHouseCompression; import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseCredentials; import com.clickhouse.client.ClickHouseException; -import com.clickhouse.client.ClickHouseFormat; -import com.clickhouse.client.ClickHouseInputStream; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseNodeSelector; -import com.clickhouse.client.ClickHouseOutputStream; import com.clickhouse.client.ClickHouseParameterizedQuery; import com.clickhouse.client.ClickHouseProtocol; -import com.clickhouse.client.ClickHouseRecord; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseRequestManager; import com.clickhouse.client.ClickHouseResponse; import com.clickhouse.client.ClickHouseResponseSummary; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseVersion; import com.clickhouse.client.ClientIntegrationTest; import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseHealthCheckMethod; -import com.clickhouse.client.config.ClickHouseOption; -import com.clickhouse.client.data.ClickHouseExternalTable; -import com.clickhouse.client.data.ClickHouseStringValue; import com.clickhouse.client.http.config.ClickHouseHttpOption; - import com.clickhouse.client.http.config.HttpConnectionProvider; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseCompression; +import com.clickhouse.data.ClickHouseExternalTable; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseVersion; +import com.clickhouse.data.value.ClickHouseStringValue; + import org.testng.Assert; -import org.testng.SkipException; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -47,7 +42,7 @@ protected Object[][] getRequestCompressionMatrix() { return new Object[][] { { ClickHouseCompression.NONE, -2, 2, 1 }, { ClickHouseCompression.LZ4, -2, 19, 1 }, // [0, 18] - { ClickHouseCompression.SNAPPY, -2, 33, 1024 }, // [1 * 1024, 32 * 1024] + // { ClickHouseCompression.SNAPPY, -2, 33, 1024 }, // [1 * 1024, 32 * 1024] { ClickHouseCompression.ZSTD, -2, 23, 1 }, // [0, 22] }; } @@ -56,11 +51,18 @@ protected Object[][] getRequestCompressionMatrix() { protected Object[][] getMixedCompressionMatrix() { // ClickHouse Code: 638. DB::Exception: hadoop snappy decode // error:INVALID_INPUT. (SNAPPY_UNCOMPRESS_FAILED) - ClickHouseCompression[] supportedRequestCompression = { ClickHouseCompression.NONE, ClickHouseCompression.LZ4, + ClickHouseCompression[] supportedRequestCompression = { + ClickHouseCompression.NONE, + ClickHouseCompression.LZ4, ClickHouseCompression.ZSTD }; - ClickHouseCompression[] supportedResponseCompression = { ClickHouseCompression.NONE, - ClickHouseCompression.BROTLI, ClickHouseCompression.BZ2, ClickHouseCompression.DEFLATE, - ClickHouseCompression.GZIP, ClickHouseCompression.LZ4, ClickHouseCompression.XZ, + ClickHouseCompression[] supportedResponseCompression = { + ClickHouseCompression.NONE, + ClickHouseCompression.BROTLI, + ClickHouseCompression.BZ2, + ClickHouseCompression.DEFLATE, + ClickHouseCompression.GZIP, + ClickHouseCompression.LZ4, + ClickHouseCompression.XZ, ClickHouseCompression.ZSTD }; Object[][] matrix = new Object[supportedRequestCompression.length * supportedResponseCompression.length][]; int i = 0; @@ -88,79 +90,8 @@ protected Map getClientOptions() { HttpConnectionProvider.HTTP_URL_CONNECTION); } - @Test(dataProvider = "requestCompressionMatrix", groups = "integration") - public void testCompressedRequest(ClickHouseCompression compression, int startLevel, int endLevel, int step) - throws ClickHouseException { - final ClickHouseNode server = getServer(); - final int readBufferSize = ClickHouseUtils.getBufferSize( - (int) ClickHouseClientOption.READ_BUFFER_SIZE.getDefaultValue(), - (int) ClickHouseClientOption.BUFFER_SIZE.getDefaultValue(), - (int) ClickHouseClientOption.MAX_BUFFER_SIZE.getDefaultValue()); - - if (compression == ClickHouseCompression.SNAPPY) { - if (!checkServerVersion(getClient(), server, "[22.3,)")) { - throw new SkipException("Snappy decompression was supported since 22.3"); - } - } - - for (int i = startLevel; i <= endLevel; i += step) { - final ByteArrayOutputStream o = new ByteArrayOutputStream(); - try (ClickHouseOutputStream out = ClickHouseOutputStream.of(o, readBufferSize, compression, i, null)) { - out.write("1,23\n4,56".getBytes()); - out.flush(); - } catch (IOException e) { - throw ClickHouseException.of(e, server); - } - try (ClickHouseClient client = getClient(); - ClickHouseResponse response = newRequest(client, server) - .format(ClickHouseFormat.RowBinaryWithNamesAndTypes) - .compressServerResponse(false) - .decompressClientRequest(true, compression, i) - .external( - // external table with compressed data - ClickHouseExternalTable.builder().name("x").columns("i Int32, s String") - .compression(compression) - .format(ClickHouseFormat.CSV) - .content(new ByteArrayInputStream(o.toByteArray())).build(), - // external table without compression - ClickHouseExternalTable.builder().name("y").columns("s String, i Int32") - .format(ClickHouseFormat.TSV) - .content(new ByteArrayInputStream("32\t1\n43\t2\n54\t3\n65\t4".getBytes())) - .build()) - .query("select x.* from x inner join y on x.i = y.i").executeAndWait()) { - int j = 0; - for (ClickHouseRecord r : response.records()) { - Assert.assertEquals(r.getValue(0).asInteger(), j == 0 ? 1 : 4); - Assert.assertEquals(r.getValue(1).asInteger(), j == 0 ? 23 : 56); - j++; - } - Assert.assertEquals(j, 2); - } - } - } - - @Test(dataProvider = "mixedCompressionMatrix", groups = "integration") - public void testDecompressResponse(ClickHouseCompression reqComp, ClickHouseCompression respComp) - throws ClickHouseException { - if (reqComp == ClickHouseCompression.SNAPPY || respComp == ClickHouseCompression.BZ2) { - if (!checkServerVersion(getClient(), getServer(), "[22.10,)")) { - throw new SkipException("Snappy and bz2 were all supported since 22.10"); - } - } - - try (ClickHouseClient client = getClient(); - ClickHouseResponse response = newRequest(client, getServer()) - .format(ClickHouseFormat.RowBinaryWithNamesAndTypes) - .decompressClientRequest(true, reqComp) - .compressServerResponse(true, respComp) - .query("select number n, toString(number+1) s from numbers(10)").executeAndWait()) { - int i = 0; - for (ClickHouseRecord r : response.records()) { - Assert.assertEquals(r.getValue(0).asInteger(), i++); - Assert.assertEquals(r.getValue(1).asInteger(), i); - } - Assert.assertEquals(i, 10); - } + @Test(groups = { "integration" }) + public void testNothing() throws Exception { } @Test(groups = "integration") diff --git a/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpConnectionTest.java b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpConnectionTest.java index 6675008f7..b7bd6fd2a 100644 --- a/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpConnectionTest.java +++ b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpConnectionTest.java @@ -6,13 +6,14 @@ import com.clickhouse.client.ClickHouseClient; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseFormat; -import com.clickhouse.client.ClickHouseInputStream; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseProtocol; import com.clickhouse.client.ClickHouseRequest; -import com.clickhouse.client.data.ClickHouseExternalTable; import com.clickhouse.client.http.config.ClickHouseHttpOption; +import com.clickhouse.data.ClickHouseExternalTable; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; import org.testng.Assert; import org.testng.annotations.Test; @@ -24,9 +25,9 @@ protected SimpleHttpConnection(ClickHouseNode server, ClickHouseRequest reque } @Override - protected ClickHouseHttpResponse post(String query, ClickHouseInputStream data, - List tables, String url, Map headers, - ClickHouseConfig config, Runnable postAction) throws IOException { + protected ClickHouseHttpResponse post(ClickHouseConfig config, String query, ClickHouseInputStream data, + List tables, ClickHouseOutputStream output, String url, + Map headers, Runnable postAction) throws IOException { return null; } diff --git a/clickhouse-jdbc/pom.xml b/clickhouse-jdbc/pom.xml index a13de063a..fd686bf45 100644 --- a/clickhouse-jdbc/pom.xml +++ b/clickhouse-jdbc/pom.xml @@ -1,4 +1,6 @@ - + 4.0.0 @@ -25,38 +27,17 @@ ${project.parent.groupId} clickhouse-cli-client - shaded ${revision} - - - * - * - - ${project.parent.groupId} clickhouse-grpc-client - shaded ${revision} - - - * - * - - ${project.parent.groupId} clickhouse-http-client - shaded ${revision} - - - * - * - - ${project.parent.groupId} @@ -69,21 +50,14 @@ - - com.google.code.gson - gson - - ${project.parent.groupId} - clickhouse-client - ${revision} - provided + org.apache.httpcomponents.client5 + httpclient5 org.lz4 lz4-java - provided @@ -137,7 +111,7 @@ javacc - 1.8 + ${minJdk} true com.clickhouse.jdbc.parser src/main/javacc @@ -177,6 +151,10 @@ org.apache.maven.plugins maven-shade-plugin + shade package @@ -189,10 +167,6 @@ true shaded - - com.google.gson - ${shade.base}.gson - org.apache ${shade.base}.apache @@ -203,10 +177,14 @@ - - - - + + + + ${project.groupId}.jdbc ${spec.title} @@ -221,17 +199,39 @@ ** + + ${project.parent.groupId}:io.grpc + + ** + + + + ${project.parent.groupId}:org.roaringbitmap + + ** + + + + com.google.code.gson:gson + + ** + + + + org.slf4j:slf4j-api + + ** + + *:* mozilla/** - **/darwin/** - **/linux/** - **/win32/** **/module-info.class META-INF/DEPENDENCIES META-INF/MANIFEST.MF META-INF/maven/** + META-INF/native/** META-INF/native-image/** META-INF/*.xml @@ -252,23 +252,43 @@ all - com.google.gson - ${shade.base}.gson + com.google + ${shade.base}.google - org.apache - ${shade.base}.apache + io.grpc + ${shade.base}.grpc + + + io.opencensus + ${shade.base}.opencensus + + + io.perfmark + ${shade.base}.perfmark net.jpountz ${shade.base}.jpountz + + okio + ${shade.base}.okio + + + org.apache + ${shade.base}.apache + - - - - + + + + ${project.groupId}.jdbc ${spec.title} @@ -277,10 +297,19 @@ + + org.slf4j:slf4j-api + + ** + + *:* + google/** mozilla/** + org/checkerframework/** + org/codehaus/** **/module-info.class META-INF/DEPENDENCIES META-INF/MANIFEST.MF @@ -303,11 +332,45 @@ true true grpc + + + com.google + ${shade.base}.google + + + io.grpc + ${shade.base}.grpc + + + io.opencensus + ${shade.base}.opencensus + + + io.perfmark + ${shade.base}.perfmark + + + net.jpountz + ${shade.base}.jpountz + + + okio + ${shade.base}.okio + + + org.apache + ${shade.base}.apache + + - - - - + + + + ${project.groupId}.jdbc ${spec.title} @@ -328,13 +391,19 @@ ** + + org.slf4j:slf4j-api + + ** + + *:* - com/google/** + google/** mozilla/** - org/** - ru/** + org/checkerframework/** + org/codehaus/** **/darwin/** **/linux/** **/win32/** @@ -342,6 +411,7 @@ META-INF/DEPENDENCIES META-INF/MANIFEST.MF META-INF/maven/** + META-INF/native/** META-INF/native-image/** META-INF/*.xml @@ -360,11 +430,21 @@ true true http + + + net.jpountz + ${shade.base}.jpountz + + - - - - + + + + ${project.groupId}.jdbc ${spec.title} @@ -385,13 +465,38 @@ ** + + ${project.parent.groupId}:io.grpc + + ** + + + + ${project.parent.groupId}:org.roaringbitmap + + ** + + + + com.google.code.gson:gson + + ** + + + + org.slf4j:slf4j-api + + ** + + *:* - com/google/** + google/** mozilla/** - org/** - ru/** + org/apache/** + org/checkerframework/** + org/codehaus/** **/darwin/** **/linux/** **/win32/** @@ -399,6 +504,7 @@ META-INF/DEPENDENCIES META-INF/MANIFEST.MF META-INF/maven/** + META-INF/native/** META-INF/native-image/** META-INF/*.xml @@ -417,11 +523,21 @@ true true cli + + + net.jpountz + ${shade.base}.jpountz + + - - - - + + + + ${project.groupId}.jdbc ${spec.title} @@ -442,13 +558,38 @@ ** + + ${project.parent.groupId}:io.grpc + + ** + + + + ${project.parent.groupId}:org.roaringbitmap + + ** + + + + com.google.code.gson:gson + + ** + + + + org.slf4j:slf4j-api + + ** + + *:* - com/google/** + google/** mozilla/** - org/** - ru/** + org/apache/** + org/checkerframework/** + org/codehaus/** **/darwin/** **/linux/** **/win32/** @@ -456,6 +597,7 @@ META-INF/DEPENDENCIES META-INF/MANIFEST.MF META-INF/maven/** + META-INF/native/** META-INF/native-image/** META-INF/*.xml diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseArray.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseArray.java index 47e014dcc..3bd83d413 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseArray.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseArray.java @@ -5,8 +5,8 @@ import java.sql.SQLException; import java.util.Map; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseColumn; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseColumn; public class ClickHouseArray implements Array { private final int columnIndex; diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseConnection.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseConnection.java index e1900706c..722c617c3 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseConnection.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseConnection.java @@ -14,14 +14,14 @@ import java.util.Optional; import java.util.TimeZone; -import com.clickhouse.client.ClickHouseColumn; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseDataType; import com.clickhouse.client.ClickHouseTransaction; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseVersion; -import com.clickhouse.client.data.ClickHouseSimpleResponse; +import com.clickhouse.client.ClickHouseSimpleResponse; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseVersion; import com.clickhouse.jdbc.parser.ClickHouseSqlStatement; public interface ClickHouseConnection extends Connection { diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaData.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaData.java index 81ad84b66..915b65b0c 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaData.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaData.java @@ -16,19 +16,19 @@ import java.util.Locale; import java.util.Map; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseFormat; import com.clickhouse.client.ClickHouseParameterizedQuery; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValues; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseRenameMethod; -import com.clickhouse.client.data.ClickHouseRecordTransformer; -import com.clickhouse.client.data.ClickHouseSimpleResponse; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.config.ClickHouseRenameMethod; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.ClickHouseRecordTransformer; +import com.clickhouse.client.ClickHouseSimpleResponse; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; public class ClickHouseDatabaseMetaData extends JdbcWrapper implements DatabaseMetaData { private static final Logger log = LoggerFactory.getLogger(ClickHouseDatabaseMetaData.class); diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDriver.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDriver.java index 272efa0e0..e10aa19f9 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDriver.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDriver.java @@ -17,11 +17,11 @@ import java.util.Map.Entry; import com.clickhouse.client.ClickHouseClient; -import com.clickhouse.client.ClickHouseVersion; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseOption; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseVersion; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; import com.clickhouse.jdbc.internal.ClickHouseConnectionImpl; import com.clickhouse.jdbc.internal.ClickHouseJdbcUrlParser; @@ -50,7 +50,19 @@ public class ClickHouseDriver implements Driver { static final java.util.logging.Logger parentLogger = java.util.logging.Logger.getLogger("com.clickhouse.jdbc"); static { - driverVersionString = ClickHouseDriver.class.getPackage().getImplementationVersion(); + String str = ClickHouseDriver.class.getPackage().getImplementationVersion(); + if (str != null && !str.isEmpty()) { + char[] chars = str.toCharArray(); + for (int i = 0, len = chars.length; i < len; i++) { + if (Character.isDigit(chars[i])) { + str = str.substring(i); + break; + } + } + driverVersionString = str; + } else { + driverVersionString = ""; + } driverVersion = ClickHouseVersion.of(driverVersionString); specVersion = ClickHouseVersion.of(ClickHouseDriver.class.getPackage().getSpecificationVersion()); diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHousePreparedStatement.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHousePreparedStatement.java index 110e34e98..7e7788e36 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHousePreparedStatement.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHousePreparedStatement.java @@ -20,8 +20,8 @@ import java.sql.Timestamp; import java.sql.Types; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.data.BinaryStreamUtils; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.format.BinaryStreamUtils; public interface ClickHousePreparedStatement extends PreparedStatement { @Override diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseResultSet.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseResultSet.java index aac6fb2e4..c6f40f5b0 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseResultSet.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseResultSet.java @@ -34,12 +34,12 @@ import java.util.Map; import java.util.TimeZone; -import com.clickhouse.client.ClickHouseColumn; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseRecord; import com.clickhouse.client.ClickHouseResponse; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValue; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; public class ClickHouseResultSet extends AbstractResultSet { private ClickHouseRecord currentRow; diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseResultSetMetaData.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseResultSetMetaData.java index 8e7b28305..0064e92ac 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseResultSetMetaData.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseResultSetMetaData.java @@ -5,8 +5,8 @@ import java.util.List; import java.util.Map; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseUtils; public class ClickHouseResultSetMetaData extends JdbcWrapper implements ResultSetMetaData { public static ResultSetMetaData of(String database, String table, List columns, diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseStruct.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseStruct.java index fc235748b..af4941e78 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseStruct.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseStruct.java @@ -4,7 +4,7 @@ import java.sql.Struct; import java.util.Map; -import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.data.ClickHouseChecker; public class ClickHouseStruct implements Struct { private final String typeName; diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcConfig.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcConfig.java index 3ec83902d..1c57476c9 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcConfig.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcConfig.java @@ -9,10 +9,10 @@ import java.util.Properties; import java.util.Map.Entry; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.config.ClickHouseOption; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; /** * JDBC-specific configuration. diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcParameterizedQuery.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcParameterizedQuery.java index 0c2ec00fc..2511be1f9 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcParameterizedQuery.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcParameterizedQuery.java @@ -6,8 +6,8 @@ import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseParameterizedQuery; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValues; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValues; /** * A parameterized query is a parsed query with parameters being extracted for diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcParseHandler.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcParseHandler.java index 607d0f5d7..fa38566e4 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcParseHandler.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcParseHandler.java @@ -4,7 +4,7 @@ import java.util.Map; import java.util.Set; -import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.data.ClickHouseChecker; import com.clickhouse.jdbc.parser.ClickHouseSqlStatement; import com.clickhouse.jdbc.parser.ParseHandler; import com.clickhouse.jdbc.parser.StatementType; diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcTypeMapping.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcTypeMapping.java index 663ecefb2..2703e7853 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcTypeMapping.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcTypeMapping.java @@ -16,15 +16,15 @@ import java.util.Map; import java.util.TimeZone; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseUtils; /** * This class defines mappings among {@link Types}, {@link JDBCType}, * {@link ClickHouseDataType}, {@link ClickHouseColumn}, and {@link Class}. It * does not impact serialization and deserialization, which is handled - * separately by {@link com.clickhouse.client.ClickHouseDataProcessor}. + * separately by {@link com.clickhouse.data.ClickHouseDataProcessor}. */ public class JdbcTypeMapping { static final class AnsiTypeMapping extends JdbcTypeMapping { diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/AbstractPreparedStatement.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/AbstractPreparedStatement.java index bba07a844..0e6f09fbd 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/AbstractPreparedStatement.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/AbstractPreparedStatement.java @@ -5,7 +5,7 @@ import java.sql.SQLException; import com.clickhouse.client.ClickHouseRequest; -import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.data.ClickHouseUtils; import com.clickhouse.jdbc.SqlExceptionUtils; public abstract class AbstractPreparedStatement extends ClickHouseStatementImpl implements PreparedStatement { diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImpl.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImpl.java index 1e8c7f7e7..9cc9b873c 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImpl.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImpl.java @@ -24,32 +24,32 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -import com.clickhouse.client.ClickHouseChecker; import com.clickhouse.client.ClickHouseClient; import com.clickhouse.client.ClickHouseClientBuilder; -import com.clickhouse.client.ClickHouseColumn; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseDataType; import com.clickhouse.client.ClickHouseException; -import com.clickhouse.client.ClickHouseFormat; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseNodeSelector; import com.clickhouse.client.ClickHouseNodes; import com.clickhouse.client.ClickHouseParameterizedQuery; -import com.clickhouse.client.ClickHouseRecord; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponse; import com.clickhouse.client.ClickHouseTransaction; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValues; -import com.clickhouse.client.ClickHouseVersion; import com.clickhouse.client.ClickHouseRequest.Mutation; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseOption; -import com.clickhouse.client.config.ClickHouseRenameMethod; import com.clickhouse.client.http.config.ClickHouseHttpOption; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.config.ClickHouseRenameMethod; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.ClickHouseVersion; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; import com.clickhouse.jdbc.ClickHouseConnection; import com.clickhouse.jdbc.ClickHouseDatabaseMetaData; import com.clickhouse.jdbc.ClickHouseDriver; diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParser.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParser.java index 2c2c96039..f6b833517 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParser.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParser.java @@ -4,14 +4,14 @@ import java.sql.SQLException; import java.util.Properties; -import com.clickhouse.client.ClickHouseChecker; import com.clickhouse.client.ClickHouseCredentials; -import com.clickhouse.client.ClickHouseFormat; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseNodes; -import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseDefaults; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseUtils; import com.clickhouse.jdbc.JdbcConfig; import com.clickhouse.jdbc.SqlExceptionUtils; diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseParameterMetaData.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseParameterMetaData.java index b0aa2d499..2945ec28d 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseParameterMetaData.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseParameterMetaData.java @@ -6,9 +6,9 @@ import java.util.List; import java.util.Map; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseUtils; import com.clickhouse.jdbc.JdbcTypeMapping; import com.clickhouse.jdbc.SqlExceptionUtils; import com.clickhouse.jdbc.JdbcWrapper; diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseStatementImpl.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseStatementImpl.java index d4d47e4c2..e01e8f8ab 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseStatementImpl.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseStatementImpl.java @@ -16,28 +16,28 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import com.clickhouse.client.ClickHouseChecker; import com.clickhouse.client.ClickHouseClient; -import com.clickhouse.client.ClickHouseColumn; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseDataProcessor; -import com.clickhouse.client.ClickHouseDataStreamFactory; -import com.clickhouse.client.ClickHouseFormat; -import com.clickhouse.client.ClickHouseInputStream; import com.clickhouse.client.ClickHouseNode; -import com.clickhouse.client.ClickHouseOutputStream; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponse; import com.clickhouse.client.ClickHouseResponseSummary; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValues; import com.clickhouse.client.ClickHouseRequest.Mutation; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseConfigChangeListener; -import com.clickhouse.client.config.ClickHouseOption; -import com.clickhouse.client.data.ClickHouseExternalTable; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.config.ClickHouseConfigChangeListener; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataProcessor; +import com.clickhouse.data.ClickHouseDataStreamFactory; +import com.clickhouse.data.ClickHouseExternalTable; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; import com.clickhouse.jdbc.ClickHouseConnection; import com.clickhouse.jdbc.ClickHouseResultSet; import com.clickhouse.jdbc.ClickHouseStatement; diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/InputBasedPreparedStatement.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/InputBasedPreparedStatement.java index 77b6b8782..f11014ced 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/InputBasedPreparedStatement.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/InputBasedPreparedStatement.java @@ -19,17 +19,17 @@ import java.util.Collections; import java.util.List; -import com.clickhouse.client.ClickHouseColumn; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseDataProcessor; -import com.clickhouse.client.ClickHouseDataStreamFactory; -import com.clickhouse.client.ClickHousePipedOutputStream; import com.clickhouse.client.ClickHouseRequest; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataProcessor; +import com.clickhouse.data.ClickHouseDataStreamFactory; +import com.clickhouse.data.ClickHousePipedOutputStream; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; import com.clickhouse.jdbc.ClickHousePreparedStatement; import com.clickhouse.jdbc.SqlExceptionUtils; diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/JdbcTransaction.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/JdbcTransaction.java index 4c06ab3d7..924169211 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/JdbcTransaction.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/JdbcTransaction.java @@ -6,12 +6,12 @@ import java.util.LinkedList; import java.util.List; -import com.clickhouse.client.ClickHouseChecker; import com.clickhouse.client.ClickHouseException; import com.clickhouse.client.ClickHouseRequestManager; import com.clickhouse.client.ClickHouseTransaction; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.logging.Logger; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.logging.Logger; import com.clickhouse.jdbc.SqlExceptionUtils; public class JdbcTransaction { diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/SqlBasedPreparedStatement.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/SqlBasedPreparedStatement.java index 07eb358f8..da2712946 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/SqlBasedPreparedStatement.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/SqlBasedPreparedStatement.java @@ -20,21 +20,21 @@ import java.util.List; import java.util.TimeZone; -import com.clickhouse.client.ClickHouseChecker; -import com.clickhouse.client.ClickHouseColumn; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseDataType; import com.clickhouse.client.ClickHouseParameterizedQuery; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponse; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; -import com.clickhouse.client.data.ClickHouseDateTimeValue; -import com.clickhouse.client.data.ClickHouseDateValue; -import com.clickhouse.client.data.ClickHouseStringValue; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseChecker; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.value.ClickHouseDateTimeValue; +import com.clickhouse.data.value.ClickHouseDateValue; +import com.clickhouse.data.value.ClickHouseStringValue; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; import com.clickhouse.jdbc.ClickHousePreparedStatement; import com.clickhouse.jdbc.JdbcParameterizedQuery; import com.clickhouse.jdbc.SqlExceptionUtils; diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/StreamBasedPreparedStatement.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/StreamBasedPreparedStatement.java index 62a0b3235..1641b2a23 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/StreamBasedPreparedStatement.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/StreamBasedPreparedStatement.java @@ -18,17 +18,17 @@ import java.util.concurrent.ExecutionException; import com.clickhouse.client.ClickHouseClient; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseDataStreamFactory; -import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHousePipedOutputStream; import com.clickhouse.client.ClickHouseRequest; -import com.clickhouse.client.ClickHouseValues; -import com.clickhouse.client.ClickHouseWriter; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataStreamFactory; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePipedOutputStream; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.ClickHouseWriter; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; import com.clickhouse.jdbc.ClickHousePreparedStatement; import com.clickhouse.jdbc.SqlExceptionUtils; import com.clickhouse.jdbc.parser.ClickHouseSqlStatement; diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/TableBasedPreparedStatement.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/TableBasedPreparedStatement.java index 55accbfb7..e33a358c9 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/TableBasedPreparedStatement.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/TableBasedPreparedStatement.java @@ -16,15 +16,15 @@ import java.util.List; import java.util.Set; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseDataType; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponse; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValues; -import com.clickhouse.client.data.ClickHouseExternalTable; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseExternalTable; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; import com.clickhouse.jdbc.ClickHousePreparedStatement; import com.clickhouse.jdbc.SqlExceptionUtils; import com.clickhouse.jdbc.parser.ClickHouseSqlStatement; diff --git a/clickhouse-jdbc/src/main/java9/module-info.java b/clickhouse-jdbc/src/main/java9/module-info.java index fbe968b51..ff9099a4f 100644 --- a/clickhouse-jdbc/src/main/java9/module-info.java +++ b/clickhouse-jdbc/src/main/java9/module-info.java @@ -17,8 +17,8 @@ requires static org.roaringbitmap; uses com.clickhouse.client.ClickHouseClient; - uses com.clickhouse.client.ClickHouseDataStreamFactory; uses com.clickhouse.client.ClickHouseDnsResolver; uses com.clickhouse.client.ClickHouseSslContextProvider; - uses com.clickhouse.client.logging.LoggerFactory; + uses com.clickhouse.data.ClickHouseDataStreamFactory; + uses com.clickhouse.logging.LoggerFactory; } diff --git a/clickhouse-jdbc/src/main/javacc/ClickHouseSqlParser.jj b/clickhouse-jdbc/src/main/javacc/ClickHouseSqlParser.jj index 561121c92..60c3c9d58 100644 --- a/clickhouse-jdbc/src/main/javacc/ClickHouseSqlParser.jj +++ b/clickhouse-jdbc/src/main/javacc/ClickHouseSqlParser.jj @@ -39,8 +39,8 @@ import java.util.Map; import java.util.Set; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; public class ClickHouseSqlParser { private static final boolean DEBUG = false; diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseConnectionTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseConnectionTest.java index 58e76279a..71e703c64 100644 --- a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseConnectionTest.java +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseConnectionTest.java @@ -9,9 +9,9 @@ import java.util.Properties; import java.util.UUID; -import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.data.UnsignedByte; +import com.clickhouse.data.ClickHouseUtils; +import com.clickhouse.data.value.UnsignedByte; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaDataTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaDataTest.java index 31be07d65..9db223014 100644 --- a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaDataTest.java +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaDataTest.java @@ -6,8 +6,8 @@ import java.util.Locale; import java.util.Properties; -import com.clickhouse.client.ClickHouseColumn; import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.data.ClickHouseColumn; import org.testng.Assert; import org.testng.annotations.DataProvider; diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHousePreparedStatementTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHousePreparedStatementTest.java index c51906304..82941f1bc 100644 --- a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHousePreparedStatementTest.java +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHousePreparedStatementTest.java @@ -33,19 +33,19 @@ import java.util.concurrent.ExecutionException; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseDataStreamFactory; -import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseFormat; -import com.clickhouse.client.ClickHouseInputStream; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHousePipedOutputStream; import com.clickhouse.client.ClickHouseProtocol; -import com.clickhouse.client.ClickHouseWriter; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.data.ClickHouseBitmap; -import com.clickhouse.client.data.ClickHouseExternalTable; -import com.clickhouse.client.data.UnsignedInteger; -import com.clickhouse.client.data.UnsignedLong; +import com.clickhouse.data.ClickHouseDataStreamFactory; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseExternalTable; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHousePipedOutputStream; +import com.clickhouse.data.ClickHouseWriter; +import com.clickhouse.data.value.ClickHouseBitmap; +import com.clickhouse.data.value.UnsignedInteger; +import com.clickhouse.data.value.UnsignedLong; import com.clickhouse.jdbc.internal.InputBasedPreparedStatement; import com.clickhouse.jdbc.internal.SqlBasedPreparedStatement; import com.clickhouse.jdbc.internal.StreamBasedPreparedStatement; @@ -1207,6 +1207,30 @@ public void testNonBatchUpdate(String mode, String query) throws SQLException { } } + @Test(groups = "integration") + public void testInsertByteArray() throws SQLException { + Properties props = new Properties(); + props.setProperty("use_binary_string", "true"); + try (ClickHouseConnection conn = newConnection(props); Statement s = conn.createStatement()) { + s.execute("drop table if exists test_insert_byte_array;" + + "create table test_insert_byte_array(id String, b Array(Int8), s Array(Array(Int8))) engine=Memory"); + try (PreparedStatement stmt = conn.prepareStatement( + "insert into test_insert_byte_array(id, b, s) values (?,?,?)")) { + stmt.setString(1, "1"); + stmt.setObject(2, new byte[] { 1, 2, 3 }); + stmt.setObject(3, new byte[][] { { 1, 2, 3 }, { 4, 5, 6 } }); + Assert.assertEquals(stmt.executeUpdate(), 1); + + ResultSet rs = s.executeQuery("select * from test_insert_byte_array order by id"); + Assert.assertTrue(rs.next()); + Assert.assertEquals(rs.getInt(1), 1); + Assert.assertEquals(rs.getObject(2), new byte[] { 1, 2, 3 }); + Assert.assertEquals(rs.getObject(3), new byte[][] { { 1, 2, 3 }, { 4, 5, 6 } }); + Assert.assertFalse(rs.next()); + } + } + } + @Test(dataProvider = "columnsWithDefaultValue", groups = "integration") public void testInsertDefaultValue(String columnType, String defaultExpr, String defaultValue) throws SQLException { Properties props = new Properties(); diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseResultSetTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseResultSetTest.java index 3682b42e9..0624081b8 100644 --- a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseResultSetTest.java +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseResultSetTest.java @@ -16,16 +16,16 @@ import java.util.TimeZone; import java.util.function.BiFunction; -import com.clickhouse.client.ClickHouseColumn; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseRecord; -import com.clickhouse.client.ClickHouseValues; -import com.clickhouse.client.data.ClickHouseDateTimeValue; -import com.clickhouse.client.data.ClickHouseOffsetDateTimeValue; -import com.clickhouse.client.data.ClickHouseSimpleResponse; -import com.clickhouse.client.data.UnsignedByte; -import com.clickhouse.client.data.UnsignedInteger; +import com.clickhouse.client.ClickHouseSimpleResponse; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.value.ClickHouseDateTimeValue; +import com.clickhouse.data.value.ClickHouseOffsetDateTimeValue; +import com.clickhouse.data.value.UnsignedByte; +import com.clickhouse.data.value.UnsignedInteger; import org.testng.Assert; import org.testng.SkipException; diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseStatementTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseStatementTest.java index 470fe949f..0c647a5ac 100644 --- a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseStatementTest.java +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseStatementTest.java @@ -31,18 +31,18 @@ import java.util.concurrent.TimeUnit; import com.clickhouse.client.ClickHouseClient; -import com.clickhouse.client.ClickHouseDataType; import com.clickhouse.client.ClickHouseParameterizedQuery; import com.clickhouse.client.ClickHouseProtocol; import com.clickhouse.client.ClickHouseRequest; -import com.clickhouse.client.ClickHouseValues; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.data.ClickHouseDateTimeValue; -import com.clickhouse.client.data.UnsignedByte; -import com.clickhouse.client.data.UnsignedInteger; -import com.clickhouse.client.data.UnsignedLong; -import com.clickhouse.client.data.UnsignedShort; import com.clickhouse.client.http.config.ClickHouseHttpOption; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseValues; +import com.clickhouse.data.value.ClickHouseDateTimeValue; +import com.clickhouse.data.value.UnsignedByte; +import com.clickhouse.data.value.UnsignedInteger; +import com.clickhouse.data.value.UnsignedLong; +import com.clickhouse.data.value.UnsignedShort; import org.testng.Assert; import org.testng.SkipException; @@ -729,7 +729,7 @@ public void testCustomTypeMappings() throws SQLException { } props.setProperty("typeMappings", - "Enum8=java.lang.Byte,DateTime64=java.lang.String, String=com.clickhouse.client.ClickHouseDataType"); + "Enum8=java.lang.Byte,DateTime64=java.lang.String, String=com.clickhouse.data.ClickHouseDataType"); try (ClickHouseConnection conn = newConnection(props); ClickHouseStatement stmt = conn.createStatement()) { ResultSet rs = stmt.executeQuery( diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/CombinedResultSetTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/CombinedResultSetTest.java index c3fbf9461..30100b39e 100644 --- a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/CombinedResultSetTest.java +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/CombinedResultSetTest.java @@ -1,16 +1,13 @@ package com.clickhouse.jdbc; -import java.math.BigDecimal; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.util.Arrays; import java.util.Collections; -import java.util.Properties; -import com.clickhouse.client.ClickHouseColumn; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.data.ClickHouseSimpleResponse; +import com.clickhouse.client.ClickHouseSimpleResponse; +import com.clickhouse.data.ClickHouseColumn; import org.testng.Assert; import org.testng.annotations.DataProvider; diff --git a/clickhouse-r2dbc/pom.xml b/clickhouse-r2dbc/pom.xml index b7b8e84ef..072d00af6 100644 --- a/clickhouse-r2dbc/pom.xml +++ b/clickhouse-r2dbc/pom.xml @@ -1,5 +1,7 @@ - + 4.0.0 @@ -38,43 +40,24 @@ ${project.parent.groupId} clickhouse-cli-client - shaded ${revision} - - - * - * - - + true ${project.parent.groupId} clickhouse-grpc-client - shaded ${revision} - - - * - * - - + true ${project.parent.groupId} clickhouse-http-client - shaded ${revision} - - - * - * - - ${project.parent.groupId} org.roaringbitmap - true + provided * @@ -83,29 +66,22 @@ - - ${project.parent.groupId} - clickhouse-client - ${revision} - provided - org.lz4 lz4-java - provided com.clickhouse clickhouse-client - ${project.version} + ${revision} test-jar test com.clickhouse clickhouse-jdbc - ${project.version} + ${revision} test @@ -247,23 +223,43 @@ all - com.google.gson - ${shade.base}.gson + com.google + ${shade.base}.google - org.apache - ${shade.base}.apache + io.grpc + ${shade.base}.grpc + + + io.opencensus + ${shade.base}.opencensus + + + io.perfmark + ${shade.base}.perfmark net.jpountz ${shade.base}.jpountz + + okio + ${shade.base}.okio + + + org.apache + ${shade.base}.apache + - - - - + + + + ${project.groupId}.r2dbc ${spec.title} @@ -272,6 +268,12 @@ + + io.projectreactor:reactor-core + + ** + + io.r2dbc:r2dbc-spi @@ -279,7 +281,13 @@ - io.projectreactor:reactor-core + org.reactivestreams:reactive-streams + + ** + + + + org.slf4j:slf4j-api ** @@ -287,7 +295,10 @@ *:* + google/** mozilla/** + org/checkerframework/** + org/codehaus/** **/module-info.class META-INF/DEPENDENCIES META-INF/MANIFEST.MF @@ -310,11 +321,21 @@ true true http + + + net.jpountz + ${shade.base}.jpountz + + - - - - + + + + ${project.groupId}.r2dbc ${spec.title} @@ -324,7 +345,31 @@ - io.r2dbc:r2dbc-spi + ${project.parent.groupId}:clickhouse-cli-client + + ** + + + + ${project.parent.groupId}:clickhouse-grpc-client + + ** + + + + ${project.parent.groupId}:io.grpc + + ** + + + + ${project.parent.groupId}:org.roaringbitmap + + ** + + + + com.google.code.gson:gson ** @@ -336,13 +381,19 @@ - ${project.parent.groupId}:clickhouse-cli-client + io.r2dbc:r2dbc-spi ** - ${project.parent.groupId}:clickhouse-grpc-client + org.reactivestreams:reactive-streams + + ** + + + + org.slf4j:slf4j-api ** @@ -350,9 +401,11 @@ *:* - com/google/** + google/** mozilla/** - org/** + org/apache/** + org/checkerframework/** + org/codehaus/** **/darwin/** **/linux/** **/win32/** @@ -360,6 +413,7 @@ META-INF/DEPENDENCIES META-INF/MANIFEST.MF META-INF/maven/** + META-INF/native/** META-INF/native-image/** META-INF/*.xml diff --git a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseBatch.java b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseBatch.java index 7314e66f2..229377031 100644 --- a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseBatch.java +++ b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseBatch.java @@ -1,7 +1,7 @@ package com.clickhouse.r2dbc; -import com.clickhouse.client.ClickHouseFormat; import com.clickhouse.client.ClickHouseRequest; +import com.clickhouse.data.ClickHouseFormat; import io.r2dbc.spi.Batch; import io.r2dbc.spi.Result; import org.reactivestreams.Publisher; diff --git a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseColumnMetadata.java b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseColumnMetadata.java index ac40c18a0..71a771b01 100644 --- a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseColumnMetadata.java +++ b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseColumnMetadata.java @@ -1,6 +1,6 @@ package com.clickhouse.r2dbc; -import com.clickhouse.client.ClickHouseColumn; +import com.clickhouse.data.ClickHouseColumn; import com.clickhouse.r2dbc.types.ClickHouseDataTypeWrapper; import io.r2dbc.spi.ColumnMetadata; import io.r2dbc.spi.Type; diff --git a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseResult.java b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseResult.java index 8b0ef19a9..8601f1236 100644 --- a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseResult.java +++ b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseResult.java @@ -2,8 +2,8 @@ import com.clickhouse.client.ClickHouseResponse; import com.clickhouse.client.ClickHouseResponseSummary; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; import io.r2dbc.spi.Result; import io.r2dbc.spi.Row; import io.r2dbc.spi.RowMetadata; diff --git a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseResult091.java b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseResult091.java index a644f61a0..4b67bf889 100644 --- a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseResult091.java +++ b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseResult091.java @@ -2,8 +2,8 @@ import com.clickhouse.client.ClickHouseResponse; import com.clickhouse.client.ClickHouseResponseSummary; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; import io.r2dbc.spi.Result; import io.r2dbc.spi.Row; import io.r2dbc.spi.RowMetadata; diff --git a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseRow.java b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseRow.java index 0a217894c..1e7df1077 100644 --- a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseRow.java +++ b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseRow.java @@ -1,7 +1,7 @@ package com.clickhouse.r2dbc; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseRecord; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseRecord; import io.r2dbc.spi.Row; import io.r2dbc.spi.RowMetadata; diff --git a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseStatement.java b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseStatement.java index 07e8cb477..e8008cec0 100644 --- a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseStatement.java +++ b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/ClickHouseStatement.java @@ -1,11 +1,11 @@ package com.clickhouse.r2dbc; -import com.clickhouse.client.ClickHouseFormat; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponse; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; import io.r2dbc.spi.Blob; import io.r2dbc.spi.Clob; import io.r2dbc.spi.Parameter; diff --git a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/connection/ClickHouseConnection.java b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/connection/ClickHouseConnection.java index b48040b5a..b2cbbb676 100644 --- a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/connection/ClickHouseConnection.java +++ b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/connection/ClickHouseConnection.java @@ -7,8 +7,8 @@ import com.clickhouse.client.ClickHouseProtocol; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; import com.clickhouse.r2dbc.ClickHouseBatch; import com.clickhouse.r2dbc.ClickHouseStatement; import io.r2dbc.spi.Batch; diff --git a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/connection/ClickHouseConnectionFactoryProvider.java b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/connection/ClickHouseConnectionFactoryProvider.java index 9faa3955b..65ef8d8b3 100644 --- a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/connection/ClickHouseConnectionFactoryProvider.java +++ b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/connection/ClickHouseConnectionFactoryProvider.java @@ -3,9 +3,9 @@ import com.clickhouse.client.ClickHouseClient; import com.clickhouse.client.ClickHouseNodes; import com.clickhouse.client.ClickHouseProtocol; -import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.config.ClickHouseDefaults; -import com.clickhouse.client.config.ClickHouseOption; +import com.clickhouse.config.ClickHouseOption; +import com.clickhouse.data.ClickHouseUtils; import io.r2dbc.spi.ConnectionFactory; import io.r2dbc.spi.ConnectionFactoryOptions; diff --git a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/connection/ClickHouseConnectionMetadata.java b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/connection/ClickHouseConnectionMetadata.java index 6fb435ed9..dbc92a08d 100644 --- a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/connection/ClickHouseConnectionMetadata.java +++ b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/connection/ClickHouseConnectionMetadata.java @@ -5,8 +5,8 @@ import com.clickhouse.client.ClickHouseClient; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseResponse; -import com.clickhouse.client.logging.Logger; -import com.clickhouse.client.logging.LoggerFactory; +import com.clickhouse.logging.Logger; +import com.clickhouse.logging.LoggerFactory; import io.r2dbc.spi.ConnectionMetadata; public class ClickHouseConnectionMetadata implements ConnectionMetadata { diff --git a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/types/ClickHouseDataTypeWrapper.java b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/types/ClickHouseDataTypeWrapper.java index e0a14914f..d2899bb28 100644 --- a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/types/ClickHouseDataTypeWrapper.java +++ b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/types/ClickHouseDataTypeWrapper.java @@ -1,6 +1,6 @@ package com.clickhouse.r2dbc.types; -import com.clickhouse.client.ClickHouseDataType; +import com.clickhouse.data.ClickHouseDataType; import io.r2dbc.spi.Type; diff --git a/clickhouse-r2dbc/src/main/java9/module-info.java b/clickhouse-r2dbc/src/main/java9/module-info.java index 7a3a7669b..72ac18c88 100644 --- a/clickhouse-r2dbc/src/main/java9/module-info.java +++ b/clickhouse-r2dbc/src/main/java9/module-info.java @@ -16,8 +16,8 @@ requires static org.roaringbitmap; uses com.clickhouse.client.ClickHouseClient; - uses com.clickhouse.client.ClickHouseDataStreamFactory; uses com.clickhouse.client.ClickHouseDnsResolver; uses com.clickhouse.client.ClickHouseSslContextProvider; - uses com.clickhouse.client.logging.LoggerFactory; + uses com.clickhouse.data.ClickHouseDataStreamFactory; + uses com.clickhouse.logging.LoggerFactory; } diff --git a/examples/grpc/src/main/java/com/clickhouse/examples/jdbc/Main.java b/examples/grpc/src/main/java/com/clickhouse/examples/jdbc/Main.java index 5f2b30f19..b97c3ca15 100644 --- a/examples/grpc/src/main/java/com/clickhouse/examples/jdbc/Main.java +++ b/examples/grpc/src/main/java/com/clickhouse/examples/jdbc/Main.java @@ -8,17 +8,17 @@ import com.clickhouse.client.ClickHouseClient; import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseCredentials; -import com.clickhouse.client.ClickHouseDataStreamFactory; import com.clickhouse.client.ClickHouseException; -import com.clickhouse.client.ClickHouseFormat; import com.clickhouse.client.ClickHouseNode; -import com.clickhouse.client.ClickHousePipedOutputStream; import com.clickhouse.client.ClickHouseProtocol; -import com.clickhouse.client.ClickHouseRecord; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponse; import com.clickhouse.client.ClickHouseResponseSummary; -import com.clickhouse.client.data.BinaryStreamUtils; +import com.clickhouse.data.ClickHouseDataStreamFactory; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHousePipedOutputStream; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.format.BinaryStreamUtils; public class Main { static void dropAndCreateTable(ClickHouseNode server, String table) throws ClickHouseException { diff --git a/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Advanced.java b/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Advanced.java index ef9d74fd1..2d722ddf7 100644 --- a/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Advanced.java +++ b/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Advanced.java @@ -14,12 +14,12 @@ import java.util.concurrent.atomic.AtomicInteger; import com.clickhouse.client.ClickHouseException; -import com.clickhouse.client.ClickHouseFormat; -import com.clickhouse.client.ClickHouseRecord; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseRequestManager; import com.clickhouse.client.ClickHouseResponse; -import com.clickhouse.client.data.ClickHouseExternalTable; +import com.clickhouse.data.ClickHouseExternalTable; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseRecord; import com.clickhouse.jdbc.ClickHouseConnection; import com.clickhouse.jdbc.SqlExceptionUtils; diff --git a/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Basic.java b/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Basic.java index 48116a53d..5ace5b16e 100644 --- a/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Basic.java +++ b/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Basic.java @@ -8,10 +8,11 @@ import java.sql.SQLException; import java.sql.Statement; import java.time.LocalDateTime; +import java.util.Collections; import java.util.Properties; -import com.clickhouse.client.ClickHouseOutputStream; -import com.clickhouse.client.ClickHouseWriter; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseWriter; public class Basic { static final String TABLE_NAME = "jdbc_example_basic"; @@ -164,17 +165,59 @@ static int query(Connection conn) throws SQLException { } } + static void insertByteArray(Connection conn) throws SQLException { + try (Statement s = conn.createStatement()) { + s.execute("drop table if exists t_map;" + + "CREATE TABLE t_map" + + "(" + + " `audit_seq` Int64 CODEC(Delta(8), LZ4)," + + "`timestamp` Int64 CODEC(Delta(8), LZ4)," + + "`event_type` LowCardinality(String)," + + "`event_subtype` LowCardinality(String)," + + "`actor_type` LowCardinality(String)," + + "`actor_id` String," + + "`actor_tenant_id` LowCardinality(String)," + + "`actor_tenant_name` String," + + "`actor_firstname` String," + + "`actor_lastname` String," + + "`resource_type` LowCardinality(String)," + + "`resource_id` String," + + "`resource_container` LowCardinality(String)," + + "`resource_path` String," + + "`origin_ip` String," + + "`origin_app_name` LowCardinality(String)," + + "`origin_app_instance` String," + + "`description` String," + + "`attributes` Map(String, String)" + + ")" + + "ENGINE = MergeTree " + + "ORDER BY (resource_container, event_type, event_subtype) " + + "SETTINGS index_granularity = 8192"); + try (PreparedStatement stmt = conn.prepareStatement( + "INSERT INTO t_map VALUES (8481365034795008,1673349039830,'operation-9','a','service', 'bc3e47b8-2b34-4c1a-9004-123656fa0000','b', 'c', 'service-56','d', 'object','e', 'my-value-62', 'mypath', 'some.hostname.address.com', 'app-9', 'instance-6','x', ?) SETTINGS async_insert=1,wait_for_async_insert=0")) { + stmt.setObject(1, Collections.singletonMap("key1", "value1")); + stmt.execute(); + + try (ResultSet rs = s.executeQuery("select attributes from t_map")) { + System.out.println(rs.next()); + System.out.println(rs.getObject(1)); + } + } + } + } + public static void main(String[] args) { // jdbc:ch:https://explorer@play.clickhouse.com:443 // jdbc:ch:https://demo:demo@github.demo.trial.altinity.cloud String url = System.getProperty("chUrl", "jdbc:ch://localhost"); try (Connection conn = getConnection(url)) { - connectWithCustomSettings(url); + // connectWithCustomSettings(url); + insertByteArray(conn); - System.out.println("Update Count: " + dropAndCreateTable(conn)); - System.out.println("Inserted Rows: " + batchInsert(conn)); - System.out.println("Result Rows: " + query(conn)); + // System.out.println("Update Count: " + dropAndCreateTable(conn)); + // System.out.println("Inserted Rows: " + batchInsert(conn)); + // System.out.println("Result Rows: " + query(conn)); } catch (SQLException e) { e.printStackTrace(); } diff --git a/pom.xml b/pom.xml index b3bec5be8..59a3cbbb4 100644 --- a/pom.xml +++ b/pom.xml @@ -37,12 +37,21 @@ + + clickhouse-data + + clickhouse-client clickhouse-cli-client clickhouse-grpc-client clickhouse-http-client + clickhouse-jdbc clickhouse-r2dbc + + + + clickhouse-benchmark @@ -82,6 +91,7 @@ 6.0.53 5.2.1 + 10.0.1 1.11.1 0.1.2 1.9.0 @@ -90,13 +100,13 @@ 1.22 3.5.2 8.5.11 - 1.51.1 - 2.10 + 1.52.1 + 2.10.1 4.0.1 3.21.4 1.8.0 0.9.3 - 0.9.36 + 0.9.38 2.0.6 1.1.8.4 1.9 @@ -105,11 +115,11 @@ 7.5 - 3.1.0 + 3.1.1 8.0.31 42.5.1 - 1.6.0 + 1.7.0 ${project.groupId}.client.internal 3.1.0 @@ -172,6 +182,26 @@ brotli4j ${brotli4j.version} + + com.aayushatharva.brotli4j + native-linux-x86_64 + ${brotli4j.version} + + + com.aayushatharva.brotli4j + native-osx-aarch64 + ${brotli4j.version} + + + com.aayushatharva.brotli4j + native-osx-x86_64 + ${brotli4j.version} + + + com.aayushatharva.brotli4j + native-windows-x86_64 + ${brotli4j.version} + com.google.code.gson gson @@ -219,6 +249,17 @@ fastutil ${fastutil.version} + + org.apache.arrow + arrow-memory + pom + ${arrow.version} + + + org.apache.arrow + arrow-vector + ${arrow.version} + org.apache.avro avro @@ -842,9 +883,9 @@ 11 - + @@ -999,7 +1040,7 @@ build8 - !release + j8 @@ -1037,7 +1078,7 @@ build9 - release + !j8 @@ -1060,9 +1101,9 @@ 11 - + From b6150fd3c0b443381e13583ac5c1981801d23425 Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Wed, 18 Jan 2023 21:01:57 +0800 Subject: [PATCH 2/6] Drop incomplete classes --- .../com/clickhouse/data/UnsafeByteUtils.java | 107 --- .../data/format/ApacheAvroProcessor.java | 155 ---- .../format/ClickHouseNativeProcessor.java | 822 ------------------ .../data/format/MessagePackProcessor.java | 61 -- .../data/format/ApacheAvroProcessorTest.java | 38 - .../format/ClickHouseNativeProcessorTest.java | 355 -------- 6 files changed, 1538 deletions(-) delete mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/UnsafeByteUtils.java delete mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/format/ApacheAvroProcessor.java delete mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseNativeProcessor.java delete mode 100644 clickhouse-data/src/main/java/com/clickhouse/data/format/MessagePackProcessor.java delete mode 100644 clickhouse-data/src/test/java/com/clickhouse/data/format/ApacheAvroProcessorTest.java delete mode 100644 clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseNativeProcessorTest.java diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/UnsafeByteUtils.java b/clickhouse-data/src/main/java/com/clickhouse/data/UnsafeByteUtils.java deleted file mode 100644 index 244ed0dff..000000000 --- a/clickhouse-data/src/main/java/com/clickhouse/data/UnsafeByteUtils.java +++ /dev/null @@ -1,107 +0,0 @@ -package com.clickhouse.data; - -import java.lang.reflect.Constructor; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.nio.ByteOrder; - -import sun.misc.Unsafe; - -@SuppressWarnings("squid:S3011") -public final class UnsafeByteUtils { - public static final ByteOrder NATIVE_BYTE_ORDER = ByteOrder.nativeOrder(); - - private static final Unsafe UNSAFE; - - private static final long BYTE_ARRAY_OFFSET = Unsafe.ARRAY_BYTE_BASE_OFFSET; - private static final long LONG_ARRAY_OFFSET = Unsafe.ARRAY_LONG_BASE_OFFSET; - - static { - Unsafe instance; - try { - Field f = Unsafe.class.getDeclaredField("theUnsafe"); - f.setAccessible(true); - instance = (Unsafe) f.get(null); - } catch (NoSuchFieldException e) { - try { - Constructor c = Unsafe.class.getDeclaredConstructor(); - c.setAccessible(true); - instance = c.newInstance(); - } catch (RuntimeException | NoSuchMethodException | InstantiationException | IllegalAccessException - | InvocationTargetException ex) { - throw new ExceptionInInitializerError(ex); - } - } catch (Exception e) { - throw new ExceptionInInitializerError("Cannot access Unsafe"); - } - UNSAFE = instance; - } - - public static byte getByte(byte[] bytes, int offset) { - return UNSAFE.getByte(bytes, BYTE_ARRAY_OFFSET + offset); - } - - public static int getInteger(byte[] bytes, int offset) { - int v = UNSAFE.getInt(bytes, BYTE_ARRAY_OFFSET + offset); - if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) { - v = Integer.reverseBytes(v); - } - return v; - } - - public static long getLong(byte[] bytes, int offset) { - if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) { - return (0xFFL & getByte(bytes, offset)) | ((0xFFL & getByte(bytes, offset + - 1)) << 8) - | ((0xFFL & getByte(bytes, offset + 2)) << 16) - | ((0xFFL & getByte(bytes, offset + 3) << 24) | ((0xFFL & getByte(bytes, - offset + 4)) << 32) - | ((0xFFL & getByte(bytes, offset + 5)) << 40) | ((0xFFL & getByte(bytes, - offset + 6)) << 48) - | ((0xFFL & getByte(bytes, offset + 7)) << 56)); - } - return UNSAFE.getLong(bytes, BYTE_ARRAY_OFFSET + offset); - } - - public static void putLong(long[] values, int offset, long value) { - UNSAFE.putLong(values, Unsafe.ARRAY_LONG_BASE_OFFSET + Unsafe.ARRAY_LONG_INDEX_SCALE * offset, value); - } - - public static short getShort(byte[] bytes, int offset) { - short v = UNSAFE.getShort(bytes, BYTE_ARRAY_OFFSET + offset); - if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) { - v = Short.reverseBytes(v); - } - return v; - } - - public static void setByte(byte[] bytes, int offset, byte value) { - UNSAFE.putByte(bytes, BYTE_ARRAY_OFFSET + Unsafe.ARRAY_BYTE_INDEX_SCALE * offset, value); - } - - public static void setInteger(byte[] bytes, int offset, int value) { - setByte(bytes, offset, (byte) value); - setByte(bytes, offset + 1, (byte) (value >>> 8)); - setByte(bytes, offset + 2, (byte) (value >>> 16)); - setByte(bytes, offset + 3, (byte) (value >>> 24)); - } - - public static void setLong(byte[] bytes, int offset, long value) { - setByte(bytes, offset, (byte) value); - setByte(bytes, offset + 1, (byte) (value >>> 8)); - setByte(bytes, offset + 2, (byte) (value >>> 16)); - setByte(bytes, offset + 3, (byte) (value >>> 24)); - setByte(bytes, offset + 4, (byte) (value >>> 32)); - setByte(bytes, offset + 5, (byte) (value >>> 40)); - setByte(bytes, offset + 6, (byte) (value >>> 48)); - setByte(bytes, offset + 7, (byte) (value >>> 56)); - } - - public static void setShort(byte[] bytes, int offset, short value) { - setByte(bytes, offset, (byte) value); - setByte(bytes, offset + 1, (byte) (value >>> 8)); - } - - private UnsafeByteUtils() { - } -} diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/format/ApacheAvroProcessor.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/ApacheAvroProcessor.java deleted file mode 100644 index 6a677aa10..000000000 --- a/clickhouse-data/src/main/java/com/clickhouse/data/format/ApacheAvroProcessor.java +++ /dev/null @@ -1,155 +0,0 @@ -package com.clickhouse.data.format; - -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import org.apache.avro.Schema; -import org.apache.avro.Schema.Field; -import org.apache.avro.Schema.Type; -import org.apache.avro.file.DataFileStream; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.DatumReader; - -import com.clickhouse.data.ClickHouseColumn; -import com.clickhouse.data.ClickHouseDataConfig; -import com.clickhouse.data.ClickHouseDataProcessor; -import com.clickhouse.data.ClickHouseDataType; -import com.clickhouse.data.ClickHouseDeserializer; -import com.clickhouse.data.ClickHouseInputStream; -import com.clickhouse.data.ClickHouseOutputStream; -import com.clickhouse.data.ClickHouseRecord; -import com.clickhouse.data.ClickHouseSerializer; -import com.clickhouse.data.ClickHouseValue; -import com.clickhouse.data.ClickHouseValues; - -public class ApacheAvroProcessor extends ClickHouseDataProcessor { - static final ClickHouseColumn toColumn(Field field) { - return toColumn(field.name(), field.schema()); - } - - static final ClickHouseColumn toColumn(String name, Schema schema) { - ClickHouseColumn column; - switch (schema.getType()) { - case BOOLEAN: - column = ClickHouseColumn.of(name, ClickHouseDataType.Bool, schema.isNullable()); - break; - case ENUM: - String[] enumTypes = schema.getEnumSymbols().toArray(new String[0]); - for (int i = 0, len = enumTypes.length; i < len; i++) { - enumTypes[i] = new StringBuilder(ClickHouseValues.convertToQuotedString(enumTypes[i])).append('=') - .append(i).toString(); - } - column = ClickHouseColumn.of(name, ClickHouseDataType.Enum16, schema.isNullable(), - false, enumTypes); - break; - case INT: - column = ClickHouseColumn.of(name, ClickHouseDataType.Int32, schema.isNullable()); - break; - case LONG: - column = ClickHouseColumn.of(name, ClickHouseDataType.Int64, schema.isNullable()); - break; - case FLOAT: - column = ClickHouseColumn.of(name, ClickHouseDataType.Float32, schema.isNullable()); - break; - case DOUBLE: - column = ClickHouseColumn.of(name, ClickHouseDataType.Float64, schema.isNullable()); - break; - case FIXED: - column = ClickHouseColumn.of(name, ClickHouseDataType.FixedString, schema.isNullable(), - false, Integer.toString(schema.getFixedSize())); - break; - case BYTES: - case STRING: - column = ClickHouseColumn.of(name, ClickHouseDataType.String, schema.isNullable()); - break; - case ARRAY: - column = ClickHouseColumn.of(name, ClickHouseDataType.Array, false, - toColumn(schema.getFields().get(0))); - break; - case MAP: - column = null; - break; - case RECORD: - List fields = schema.getFields(); - ClickHouseColumn[] nestedCols = new ClickHouseColumn[fields.size()]; - for (Field f : fields) { - nestedCols[0] = toColumn(f.name(), f.schema()); - } - column = ClickHouseColumn.of(name, ClickHouseDataType.Nested, false, nestedCols); - break; - case UNION: - for (Schema s : schema.getTypes()) { - if (s.getType() != Schema.Type.NULL) { - return toColumn(name, s); - } - } - case NULL: - default: - column = ClickHouseColumn.of(name, ClickHouseDataType.Nothing, true); - break; - } - return column; - } - - protected ApacheAvroProcessor(ClickHouseDataConfig config, ClickHouseInputStream input, - ClickHouseOutputStream output, List columns, Map settings) - throws IOException { - super(config, input, output, columns, settings); - } - - @Override - protected ClickHouseRecord createRecord() { - - return null; - } - - @Override - protected void readAndFill(ClickHouseValue value) throws IOException { - // TODO Auto-generated method stub - - } - - @Override - protected List readColumns() throws IOException { - if (input.available() < 1) { - input.close(); - return Collections.emptyList(); - } - - DatumReader datumReader = new GenericDatumReader<>(); - DataFileStream reader = new DataFileStream<>(input, datumReader); - Schema schema = reader.getSchema(); - if (schema.getType() != Type.RECORD) { - throw new IOException("Root element of schema must be RECORD"); - } - List flds = schema.getFields(); - List cols = new ArrayList<>(flds.size()); - for (Field f : flds) { - cols.add(toColumn(f)); - } - return Collections.unmodifiableList(cols); - } - - @Override - public void write(ClickHouseValue value) throws IOException { - // TODO Auto-generated method stub - - } - - @Override - public ClickHouseDeserializer getDeserializer(ClickHouseDataConfig config, ClickHouseColumn column) { - // TODO Auto-generated method stub - return null; - } - - @Override - public ClickHouseSerializer getSerializer(ClickHouseDataConfig config, ClickHouseColumn column) { - // TODO Auto-generated method stub - return null; - } -} diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseNativeProcessor.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseNativeProcessor.java deleted file mode 100644 index 96fa3436f..000000000 --- a/clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseNativeProcessor.java +++ /dev/null @@ -1,822 +0,0 @@ -package com.clickhouse.data.format; - -import java.io.IOException; -import java.io.Serializable; -import java.io.UncheckedIOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import com.clickhouse.data.ClickHouseArraySequence; -import com.clickhouse.data.ClickHouseChecker; -import com.clickhouse.data.ClickHouseColumn; -import com.clickhouse.data.ClickHouseDataConfig; -import com.clickhouse.data.ClickHouseDataProcessor; -import com.clickhouse.data.ClickHouseDataType; -import com.clickhouse.data.ClickHouseDeserializer; -import com.clickhouse.data.ClickHouseInputStream; -import com.clickhouse.data.ClickHouseOutputStream; -import com.clickhouse.data.ClickHouseRecord; -import com.clickhouse.data.ClickHouseSerializer; -import com.clickhouse.data.ClickHouseUtils; -import com.clickhouse.data.ClickHouseValue; -import com.clickhouse.data.value.ClickHouseByteValue; -import com.clickhouse.data.value.ClickHouseEmptyValue; -import com.clickhouse.data.value.ClickHouseIntegerValue; -import com.clickhouse.data.value.ClickHouseLongValue; -import com.clickhouse.data.value.ClickHouseShortValue; -import com.clickhouse.data.value.array.ClickHouseByteArrayValue; -import com.clickhouse.data.value.array.ClickHouseIntArrayValue; -import com.clickhouse.data.value.array.ClickHouseLongArrayValue; -import com.clickhouse.data.value.array.ClickHouseShortArrayValue; - -public class ClickHouseNativeProcessor extends ClickHouseDataProcessor { - private static final ClickHouseColumn[] lowCardinalityIndexColumns = new ClickHouseColumn[] { - ClickHouseColumn.of("", ClickHouseDataType.UInt8, false), - ClickHouseColumn.of("", ClickHouseDataType.UInt16, false), - ClickHouseColumn.of("", ClickHouseDataType.UInt32, false), - ClickHouseColumn.of("", ClickHouseDataType.UInt64, false) - }; - private static final ClickHouseDeserializer[] lowCardinalityIndexDeserializers = new ClickHouseDeserializer[] { - ClickHouseNativeProcessor::readByteArray, ClickHouseNativeProcessor::readShortArray, - ClickHouseNativeProcessor::readIntegerArray, ClickHouseNativeProcessor::readLongArray - }; - private static final ClickHouseValue[] lowCardinalityIndexValues = new ClickHouseValue[] { - ClickHouseByteValue.ofUnsignedNull(), ClickHouseShortValue.ofUnsignedNull(), - ClickHouseIntegerValue.ofUnsignedNull(), ClickHouseLongValue.ofUnsignedNull() - }; - private static final ClickHouseArraySequence[] lowCardinalityIndexArrays = new ClickHouseArraySequence[] { - ClickHouseByteArrayValue.ofUnsignedEmpty(), ClickHouseShortArrayValue.ofUnsignedEmpty(), - ClickHouseIntArrayValue.ofUnsignedEmpty(), ClickHouseLongArrayValue.ofUnsignedEmpty() - }; - - static class ArrayColumnDeserializer extends ClickHouseDeserializer.CompositeDeserializer { - private final long length; - private final int nestedLevel; - private final Class valClass; - private final ClickHouseValue valValue; - - public ArrayColumnDeserializer(ClickHouseDataConfig config, ClickHouseColumn column, long length, - boolean nullable, ClickHouseDeserializer... deserializers) { - super(deserializers); - - this.length = length; - - ClickHouseColumn baseColumn = column.getArrayBaseColumn(); - nestedLevel = column.getArrayNestedLevel(); - valClass = baseColumn.getObjectClassForArray(config); - valValue = column.getNestedColumns().get(0).newValue(config); - } - - @Override - public ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseInputStream input) throws IOException { - final long l = this.length; - final int len; - if (l == -1L) { - len = input.readVarInt(); - } else if (l < 0L) { - len = (int) input.readBuffer(8).asLong(); - } else { - len = (int) l; - } - - if (len == 0) { - return ref.resetToNullOrEmpty(); - } - ClickHouseArraySequence arr = (ClickHouseArraySequence) ref; - arr.allocate(len, valClass, nestedLevel); - ClickHouseDeserializer d = deserializers[0]; - for (int i = 0; i < len; i++) { - arr.setValue(i, d.deserialize(valValue, input)); - } - return ref; - } - } - - static class LowCardinalityColumnDeserializer implements ClickHouseDeserializer { - private final ClickHouseDeserializer deserializer; - private final Class javaClass; - private final ClickHouseValue baseValue; - private final ClickHouseArraySequence arrValue; - private final boolean nullable; - - public LowCardinalityColumnDeserializer(ClickHouseDataConfig config, ClickHouseColumn column, - ClickHouseDeserializer d) { - this.javaClass = column.getObjectClassForArray(config); - if (!column.isNullable() && javaClass.isPrimitive()) { - int byteLength = column.getDataType().getByteLength(); - if (byteLength == Byte.BYTES) { // Bool, *Int8 - deserializer = ClickHouseNativeProcessor::readByteArray; - } else if (byteLength == Short.BYTES) { // *Int16 - deserializer = ClickHouseNativeProcessor::readShortArray; - } else if (int.class == javaClass) { // Int32 - deserializer = ClickHouseNativeProcessor::readIntegerArray; - } else if (long.class == javaClass) { // UInt32, *Int64 - deserializer = byteLength == Long.BYTES ? ClickHouseNativeProcessor::readLongArray - : ClickHouseNativeProcessor::readIntegerArray; - } else if (float.class == javaClass) { // Float32 - deserializer = ClickHouseNativeProcessor::readFloatArray; - } else if (double.class == javaClass) { // Float64 - deserializer = ClickHouseNativeProcessor::readDoubleArray; - } else { - throw new IllegalArgumentException("Unsupported primitive type: " + javaClass); - } - } else { - deserializer = ClickHouseChecker.nonNull(d, "Deserializer"); - } - this.baseValue = column.newValue(config); - this.arrValue = column.newArrayValue(config); - this.nullable = column.isNullable(); - } - - @Override - public ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseInputStream input) throws IOException { - long version = input.readBuffer(8).asLong(); - if (version != 1L) { - throw new IOException( - ClickHouseUtils.format("Unexpected low cardinality version %d", version)); - } - ClickHouseArraySequence value = (ClickHouseArraySequence) ref; - long serializationType = input.readBuffer(8).asLong(); - // Lowest byte contains info about key type - int keyType = (int) (serializationType & 0xFL); - if (keyType < 0 || keyType >= lowCardinalityIndexColumns.length) { - throw new IOException("Unsupported key type: " + keyType); - } - // int indexSize = (int) input.readBuffer(8).asLong(); - // ClickHouseValue v = baseColumn.newValue(config); - ClickHouseArraySequence dict = (ClickHouseArraySequence) deserializer.deserialize(arrValue.copy(), input); - if (nullable) { - dict.setValue(0, ClickHouseEmptyValue.INSTANCE); - } - - // ClickHouseColumn indexCol = lowCardinalityIndexColumns[keyType]; - ClickHouseValue indexVal = lowCardinalityIndexValues[keyType].copy(); - ClickHouseArraySequence indexArrVal = lowCardinalityIndexArrays[keyType].copy(); - // int rows = (int) BinaryStreamUtils.readInt64(input); - lowCardinalityIndexDeserializers[keyType].deserialize(indexArrVal, input); - - int rows = indexArrVal.length(); - value.allocate(rows, javaClass); - for (int i = 0; i < rows; i++) { - int keyIndex = indexArrVal.getValue(i, indexVal).asInteger(); - value.setValue(i, dict.getValue(keyIndex, baseValue)); - } - return value; - } - } - - static class LowCardinalitySerializer implements ClickHouseSerializer { - private final ClickHouseSerializer serializer; - - public LowCardinalitySerializer(ClickHouseSerializer serializer) { - this.serializer = ClickHouseChecker.nonNull(serializer, "Serializer"); - } - - @Override - public void serialize(ClickHouseValue value, ClickHouseOutputStream output) throws IOException { - if (value.isNullOrEmpty()) { - output.writeBoolean(true); - } else { - serializer.serialize(value, output.writeBoolean(false)); - } - } - } - - static class NullableColumnDeserializer implements ClickHouseDeserializer { - private final ClickHouseDeserializer deserializer; - - public NullableColumnDeserializer(ClickHouseDeserializer deserializer) { - this.deserializer = ClickHouseChecker.nonNull(deserializer, "Deserializer"); - } - - @Override - public ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseInputStream input) throws IOException { - return ref; - } - } - - // public static class MappedFunctions { - // private static final MappedFunctions instance = new MappedFunctions(); - - // private ClickHouseArraySequence readFlattenArray(ClickHouseArraySequence ref, - // ClickHouseConfig config, - // ClickHouseColumn column, ClickHouseInputStream input) throws IOException { - // if (ref == null) { - // ref = column.newArrayValue(config); - // } - // ClickHouseColumn baseColumn = column.getArrayBaseColumn(); - // int level = column.getArrayNestedLevel(); - // if (level <= 1) { - // int length = (int) BinaryStreamUtils.readInt64(input); - // if (baseColumn.isNullable()) { - // boolean[] nullFlags = input.readBuffer(length).asBooleanArray(); - // ClickHouseDeserializer func = getDeserializer(baseColumn.getDataType()); - // if (func == null) { - // throw new IllegalArgumentException(ERROR_UNKNOWN_DATA_TYPE + - // baseColumn.getDataType().name()); - // } - // ref.allocate(length, baseColumn.getObjectClass(config), level); - // ClickHouseValue v = baseColumn.newValue(config); - // for (int i = 0; i < length; i++) { - // func.deserialize(v, config, baseColumn, input); - // if (nullFlags[i]) { - // v.resetToNullOrEmpty(); - // } - // ref.setValue(i, v); - // } - // } else { - // readArrayContent((ClickHouseArraySequence) ref, config, - // column.getNestedColumns().get(0), - // column.getArrayBaseColumn(), input, length, level); - // } - // } else { - // int length = 0; - // int[][] offsets = new int[level][]; - // for (int i = 1; i < level; i++) { - // int len = (int) BinaryStreamUtils.readInt64(input); - // int[] idx = new int[len]; - // offsets[i - 1] = idx; - // for (int j = 0; j < len; j++) { - // idx[j] = (int) BinaryStreamUtils.readInt64(input); - // } - // } - - // ClickHouseArraySequence parent = ref; - // ClickHouseColumn nested = column; - // for (int i = 0; i < level; i++) { - // int[] idx = offsets[i]; - // int len = idx.length; - // nested = column.getNestedColumns().get(0); - // ClickHouseArraySequence arr = nested.newArrayValue(config); - // for (int j = 0; j < len; j++) { - - // } - // } - // ClickHouseArraySequence arr = ClickHouseColumn.of("", - // ClickHouseDataType.Array, false, baseColumn) - // .newArrayValue(config); - // // readArray(innerMost, config, baseColumn, baseColumn, input, length, 1); - // } - // return ref; - // } - - // private ClickHouseArraySequence readLowCardinality(ClickHouseArraySequence - // value, ClickHouseConfig config, - // ClickHouseColumn arrayColumn, ClickHouseColumn baseColumn, - // ClickHouseInputStream input) - // throws IOException { - // long version = BinaryStreamUtils.readInt64(input); - // if (version != 1L) { - // throw new IOException( - // ClickHouseUtils.format("Unexpected low cardinality version %d when reading - // column %s", - // version, baseColumn.getOriginalTypeName())); - // } - // long serializationType = BinaryStreamUtils.readInt64(input); - // // Lowest byte contains info about key type - // int keyType = (int) (serializationType & 0xFL); - // if (keyType < 0 || keyType >= lowCardinalityIndexColumns.length) { - // throw new IOException("Unsupported key type: " + keyType); - // } - // int indexSize = (int) BinaryStreamUtils.readInt64(input); - // ClickHouseValue v = baseColumn.newValue(config); - // ClickHouseArraySequence dict = readArrayContent(value.copy(), config, - // baseColumn, - // arrayColumn.getArrayBaseColumn(), input, indexSize, - // arrayColumn.getArrayNestedLevel()); - // if (baseColumn.isNullable()) { - // dict.setValue(0, ClickHouseEmptyValue.INSTANCE); - // } - - // ClickHouseColumn indexCol = lowCardinalityIndexColumns[keyType]; - // ClickHouseValue indexVal = lowCardinalityIndexValues[keyType].copy(); - // ClickHouseArraySequence indexArrVal = - // lowCardinalityIndexArrays[keyType].copy(); - // int rows = (int) BinaryStreamUtils.readInt64(input); - // readArrayContent(indexArrVal, config, indexCol, indexCol, input, rows, 1); - - // value.allocate(rows, baseColumn.getObjectClassForArray(config)); - // for (int i = 0; i < rows; i++) { - // int keyIndex = indexArrVal.getValue(i, indexVal).asInteger(); - // value.setValue(i, dict.getValue(keyIndex, v)); - // } - // return value; - // } - - // private ClickHouseArraySequence readColumn(ClickHouseArraySequence value, - // ClickHouseConfig config, - // ClickHouseColumn arrayColumn, ClickHouseColumn baseColumn, - // ClickHouseInputStream input, int rows) - // throws IOException { - // if (baseColumn.isLowCardinality()) { - // return readLowCardinality(value, config, arrayColumn, baseColumn, input); - // } - - // return readArrayContent(value, config, baseColumn, - // arrayColumn.getArrayBaseColumn(), input, rows, - // arrayColumn.getArrayNestedLevel()); - // } - - // protected MappedFunctions() { - // super(); - // } - - // @Override - // public ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseInputStream - // input) throws IOException { - // if (column.isLowCardinality()) { - // if (ref == null) { - // ref = column.newValue(config); - // } - // column.disableLowCardinality(); - // ClickHouseColumn arrCol = ClickHouseColumn.of("", ClickHouseDataType.Array, - // false, column); - // ClickHouseArraySequence arrVal = - // readLowCardinality(arrCol.newArrayValue(config), config, arrCol, - // column, input); - // return arrVal.getValue(0, ref); - // } else if (column.isArray()) { - // return readFlattenArray((ClickHouseArraySequence) ref, config, column, - // input); - // } else { - // if (!column.isLowCardinalityDisabled() && column.isNullable() && - // BinaryStreamUtils.readNull(input)) { - // return ref == null ? column.newValue(config) : ref.resetToNullOrEmpty(); - // } else if (config.isWidenUnsignedTypes()) { - // switch (column.getDataType()) { - // case UInt8: - // return ClickHouseShortValue.of(ref, input.readUnsignedByte(), false); - // case UInt16: - // return ClickHouseIntegerValue.of(ref, input.readBuffer(2).asUnsignedShort(), - // false); - // case UInt32: - // return ClickHouseLongValue.of(ref, input.readBuffer(4).asUnsignedInteger(), - // false); - // default: - // break; - // } - // } - // } - - // ClickHouseDeserializer func = getDeserializer(column.getDataType()); - // if (func == null) { - // throw new IllegalArgumentException(ERROR_UNKNOWN_DATA_TYPE + - // column.getDataType().name()); - // } - // return func.deserialize(ref, config, column, input); - // } - - // @Override - // public void serialize(ClickHouseValue value, ClickHouseOutputStream output) - // throws IOException { - // if (column.isLowCardinality()) { - // column.disableLowCardinality(); - // ClickHouseColumn arrCol = ClickHouseColumn.of("", ClickHouseDataType.Array, - // false, column); - // // ClickHouseArraySequence arrVal = - // // readLowCardinality(arrCol.newArrayValue(config), config, arrCol, - // // column, input); - // } else if (column.isArray()) { - // // write flatten array - // } else { - // if (!column.isLowCardinalityDisabled() && column.isNullable()) { // always - // false for geo types, and - // // Array, Nested, Map and Tuple etc. - // if (value.isNullOrEmpty()) { - // BinaryStreamUtils.writeNull(output); - // return; - // } else { - // BinaryStreamUtils.writeNonNull(output); - // } - // } - - // ClickHouseSerializer func = getSerializer(column.getDataType()); - // if (func == null) { - // throw new IllegalArgumentException(ERROR_UNKNOWN_DATA_TYPE + - // column.getDataType().name()); - // } - // func.serialize(value, config, column, output); - // } - // } - // } - - private static int readArrayLength(ClickHouseInputStream input) throws IOException { - return (int) input.readBuffer(8).asLong(); - } - - private static ClickHouseValue readByteArray(ClickHouseValue ref, ClickHouseInputStream input) throws IOException { - return ref.update(input.readBuffer(readArrayLength(input)).compact().array()); - } - - private static ClickHouseValue readShortArray(ClickHouseValue ref, ClickHouseInputStream input) throws IOException { - return ref.update(input.readBuffer(readArrayLength(input) * 2).asShortArray()); - } - - private static ClickHouseValue readIntegerArray(ClickHouseValue ref, ClickHouseInputStream input) - throws IOException { - return ref.update(input.readBuffer(readArrayLength(input) * 4).asIntegerArray()); - } - - private static ClickHouseValue readLongArray(ClickHouseValue ref, ClickHouseInputStream input) throws IOException { - return ref.update(input.readBuffer(readArrayLength(input) * 8).asLongArray()); - } - - private static ClickHouseValue readFloatArray(ClickHouseValue ref, ClickHouseInputStream input) throws IOException { - return ref.update(input.readBuffer(readArrayLength(input) * 4).asFloatArray()); - } - - private static ClickHouseValue readDoubleArray(ClickHouseValue ref, ClickHouseInputStream input) - throws IOException { - return ref.update(input.readBuffer(readArrayLength(input) * 8).asDoubleArray()); - } - - private static ClickHouseValue readFixedByteArray(ClickHouseValue ref, ClickHouseInputStream input) - throws IOException { - ClickHouseArraySequence arr = (ClickHouseArraySequence) ref; - // input.readFully((byte[]) arr.asRawObject()); - return arr; - } - - private static ClickHouseValue readFixedShortArray(ClickHouseValue ref, ClickHouseInputStream input) - throws IOException { - ClickHouseArraySequence arr = (ClickHouseArraySequence) ref; - return ref.update(input.readBuffer(arr.length() * 2).asShortArray()); - } - - private static ClickHouseValue readFixedIntegerArray(ClickHouseValue ref, ClickHouseInputStream input) - throws IOException { - ClickHouseArraySequence arr = (ClickHouseArraySequence) ref; - return ref.update(input.readBuffer(arr.length() * 4).asIntegerArray()); - } - - private static ClickHouseValue readFixedLongArray(ClickHouseValue ref, ClickHouseInputStream input) - throws IOException { - ClickHouseArraySequence arr = (ClickHouseArraySequence) ref; - return ref.update(input.readBuffer(arr.length() * 8).asLongArray()); - } - - private static ClickHouseValue readFixedFloatArray(ClickHouseValue ref, ClickHouseInputStream input) - throws IOException { - ClickHouseArraySequence arr = (ClickHouseArraySequence) ref; - return ref.update(input.readBuffer(arr.length() * 4).asFloatArray()); - } - - private static ClickHouseValue readFixedDoubleArray(ClickHouseValue ref, ClickHouseInputStream input) - throws IOException { - ClickHouseArraySequence arr = (ClickHouseArraySequence) ref; - return ref.update(input.readBuffer(arr.length() * 8).asDoubleArray()); - } - - private ClickHouseDeserializer[] colDeserializers; - private ClickHouseArraySequence[] currentBlock; - - private int blockRows; - private int currentRow; - - private void readBlock() throws IOException { - int colCount = input.readVarInt(); - int rowCount = input.readVarInt(); - - if (colDeserializers == null) { - colDeserializers = new ClickHouseDeserializer[colCount]; - currentBlock = new ClickHouseArraySequence[colCount]; - } - for (int i = 0; i < colCount; i++) { - // String name = input.readUnicodeString(); - // String type = input.readUnicodeString(); - input.skip(input.readVarInt()); - input.skip(input.readVarInt()); - - ClickHouseColumn col = columns[i]; - ClickHouseDeserializer d = colDeserializers[i]; - ClickHouseArraySequence arr; - if (d == null) { - ClickHouseColumn arrCol = ClickHouseColumn.of("", ClickHouseDataType.Array, false, col); - colDeserializers[i] = d = col.isLowCardinality() - ? new LowCardinalityColumnDeserializer(config, col, getArrayDeserializer(config, arrCol, -2L)) - : getArrayDeserializer(config, arrCol, rowCount); - currentBlock[i] = arr = col.newArrayValue(config); - } else { - arr = currentBlock[i]; - } - arr.allocate(rowCount, col.getObjectClass(config)); - d.deserialize(arr, input); - } - blockRows = rowCount; - currentRow = 0; - - readPosition = 0; - } - - @Override - protected ClickHouseRecord createRecord() { - return new ClickHouseSimpleRecord(getColumns(), templates); - } - - @Override - protected boolean hasMoreToRead() throws UncheckedIOException { - if (currentRow < blockRows) { - return true; - } - - try { - if (input.available() <= 0) { - input.close(); - return false; - } else { - readBlock(); - } - return true; - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - @Override - protected void readAndFill(ClickHouseRecord r) throws IOException { - for (int i = readPosition, len = columns.length; i < len; i++) { - currentBlock[readPosition].getValue(currentRow, r.getValue(readPosition)); - readPosition = i; - } - - readPosition = 0; - currentRow++; - } - - @Override - protected void readAndFill(ClickHouseValue value) throws IOException { - if (currentRow >= blockRows) { - readBlock(); - } - - currentBlock[readPosition++].getValue(currentRow, value); - if (readPosition >= columns.length) { - readPosition = 0; - currentRow++; - } - } - - @Override - protected List readColumns() throws IOException { - if (input.available() <= 0) { - input.close(); - // no result returned - colDeserializers = new ClickHouseDeserializer[0]; - currentBlock = new ClickHouseArraySequence[0]; - return Collections.emptyList(); - } - - int colCount = input.readVarInt(); - int rowCount = input.readVarInt(); - List columns = new ArrayList<>(colCount); - colDeserializers = new ClickHouseDeserializer[colCount]; - ClickHouseArraySequence[] arrays = new ClickHouseArraySequence[colCount]; - - for (int i = 0; i < colCount; i++) { - String name = input.readUnicodeString(); - ClickHouseColumn baseColumn = ClickHouseColumn.of(name, input.readUnicodeString()); - ClickHouseColumn arraryColumn = ClickHouseColumn.of("", ClickHouseDataType.Array, false, baseColumn); - ClickHouseArraySequence value = arraryColumn.newArrayValue(config); - value.allocate(rowCount); - - columns.add(baseColumn); - ClickHouseDeserializer d = baseColumn.isLowCardinality() - ? new LowCardinalityColumnDeserializer(config, baseColumn, - getArrayDeserializer(config, arraryColumn, -2L)) - : getArrayDeserializer(config, arraryColumn, rowCount); - colDeserializers[i] = d; - value.allocate(rowCount, baseColumn.getObjectClass(config)); - arrays[i] = (ClickHouseArraySequence) d.deserialize(value, input); - } - currentBlock = arrays; - blockRows = rowCount; - currentRow = 0; - return columns; - } - - public ClickHouseNativeProcessor(ClickHouseDataConfig config, ClickHouseInputStream input, - ClickHouseOutputStream output, List columns, Map settings) - throws IOException { - super(config, input, output, columns, settings); - } - - private ClickHouseDeserializer getArrayDeserializer(ClickHouseDataConfig config, ClickHouseColumn column, - long length) { - final boolean fixedLength = length >= 0L; - final ClickHouseDeserializer deserializer; - ClickHouseColumn baseColumn = column.getArrayBaseColumn(); - Class javaClass = baseColumn.getObjectClassForArray(config); - if (column.getArrayNestedLevel() == 1 && !baseColumn.isNullable() && javaClass.isPrimitive()) { - int byteLength = baseColumn.getDataType().getByteLength(); - if (byteLength == Byte.BYTES) { // Bool, *Int8 - deserializer = fixedLength ? ClickHouseNativeProcessor::readFixedByteArray - : ClickHouseNativeProcessor::readByteArray; - } else if (byteLength == Short.BYTES) { // *Int16 - deserializer = fixedLength ? ClickHouseNativeProcessor::readFixedShortArray - : ClickHouseNativeProcessor::readShortArray; - } else if (int.class == javaClass) { // Int32 - deserializer = fixedLength ? ClickHouseNativeProcessor::readFixedIntegerArray - : ClickHouseNativeProcessor::readIntegerArray; - } else if (long.class == javaClass) { // UInt32, *Int64 - if (byteLength == Long.BYTES) { - deserializer = fixedLength ? ClickHouseNativeProcessor::readFixedLongArray - : ClickHouseNativeProcessor::readLongArray; - } else { - deserializer = fixedLength ? ClickHouseNativeProcessor::readFixedIntegerArray - : ClickHouseNativeProcessor::readIntegerArray; - } - } else if (float.class == javaClass) { // Float32 - deserializer = fixedLength ? ClickHouseNativeProcessor::readFixedFloatArray - : ClickHouseNativeProcessor::readFloatArray; - } else if (double.class == javaClass) { // Float64 - deserializer = fixedLength ? ClickHouseNativeProcessor::readFixedDoubleArray - : ClickHouseNativeProcessor::readDoubleArray; - } else { - throw new IllegalArgumentException("Unsupported primitive type: " + javaClass); - } - } else { - ClickHouseColumn nestedCol = column.getNestedColumns().get(0); - deserializer = new BinaryDataProcessor.ArrayDeserializer(config, column, length, - getDeserializer(config, nestedCol)); - } - return deserializer; - } - - @Override - public ClickHouseDeserializer getDeserializer(ClickHouseDataConfig config, ClickHouseColumn column) { - final ClickHouseDeserializer deserializer; - switch (column.getDataType()) { - case Bool: - deserializer = BinaryDataProcessor::readBool; - break; - case Date: - deserializer = BinaryDataProcessor.DateSerDe.of(config); - break; - case Date32: - deserializer = BinaryDataProcessor.Date32SerDe.of(config); - break; - case DateTime: - deserializer = column.getScale() > 0 ? BinaryDataProcessor.DateTime64SerDe.of(config, column) - : BinaryDataProcessor.DateTime32SerDe.of(config, column); - break; - case DateTime32: - deserializer = BinaryDataProcessor.DateTime32SerDe.of(config, column); - break; - case DateTime64: - deserializer = BinaryDataProcessor.DateTime64SerDe.of(config, column); - break; - case Enum8: - deserializer = BinaryDataProcessor::readEnum8; - break; - case Enum16: - deserializer = BinaryDataProcessor::readEnum8; - break; - case FixedString: - deserializer = new BinaryDataProcessor.FixedStringSerDe(column); - break; - case Int8: - deserializer = BinaryDataProcessor::readByte; - break; - case UInt8: - deserializer = config.isWidenUnsignedTypes() ? BinaryDataProcessor::readUInt8AsShort - : BinaryDataProcessor::readByte; - break; - case Int16: - deserializer = BinaryDataProcessor::readShort; - break; - case UInt16: - deserializer = config.isWidenUnsignedTypes() ? BinaryDataProcessor::readUInt16AsInt - : BinaryDataProcessor::readShort; - break; - case Int32: - deserializer = BinaryDataProcessor::readInteger; - break; - case UInt32: - deserializer = config.isWidenUnsignedTypes() ? BinaryDataProcessor::readUInt32AsLong - : BinaryDataProcessor::readInteger; - break; - case Int64: - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case IntervalMicrosecond: - case IntervalMillisecond: - case IntervalNanosecond: - case UInt64: - deserializer = BinaryDataProcessor::readLong; - break; - case Int128: - deserializer = BinaryDataProcessor::readInt128; - break; - case UInt128: - deserializer = BinaryDataProcessor::readUInt128; - break; - case Int256: - deserializer = BinaryDataProcessor::readInt256; - break; - case UInt256: - deserializer = BinaryDataProcessor::readUInt256; - break; - case Decimal: - deserializer = BinaryDataProcessor.DecimalSerDe.of(column); - break; - case Decimal32: - deserializer = BinaryDataProcessor.Decimal32SerDe.of(column); - break; - case Decimal64: - deserializer = BinaryDataProcessor.Decimal64SerDe.of(column); - break; - case Decimal128: - deserializer = BinaryDataProcessor.Decimal128SerDe.of(column); - break; - case Decimal256: - deserializer = BinaryDataProcessor.Decimal256SerDe.of(column); - break; - case Float32: - deserializer = BinaryDataProcessor::readFloat; - break; - case Float64: - deserializer = BinaryDataProcessor::readDouble; - break; - case IPv4: - deserializer = BinaryDataProcessor::readIpv4; - break; - case IPv6: - deserializer = BinaryDataProcessor::readIpv6; - break; - case UUID: - deserializer = BinaryDataProcessor::readUuid; - break; - // Geo types - case Point: - deserializer = BinaryDataProcessor::readGeoPoint; - break; - case Ring: - deserializer = BinaryDataProcessor::readGeoRing; - break; - case Polygon: - deserializer = BinaryDataProcessor::readGeoPolygon; - break; - case MultiPolygon: - deserializer = BinaryDataProcessor::readGeoMultiPolygon; - break; - // String - case JSON: - case Object: - case String: - deserializer = config.isUseBinaryString() ? BinaryDataProcessor::readBinaryString - : BinaryDataProcessor::readTextString; - break; - // nested - case Array: - deserializer = getArrayDeserializer(config, column, -1L); - break; - case Map: - deserializer = new ClickHouseRowBinaryProcessor.MapDeserializer(config, column, - getDeserializers(config, column.getNestedColumns())); - break; - case Nested: - deserializer = ClickHouseDeserializer.EMPTY_VALUE; - // new ClickHouseRowBinaryProcessor.NestedDeserializer(config, column, - // getArraySerDeializers(config, column.getNestedColumns())); - break; - case Tuple: - deserializer = new ClickHouseRowBinaryProcessor.TupleDeserializer(config, column, - getDeserializers(config, column.getNestedColumns())); - break; - // special - case Nothing: - deserializer = ClickHouseDeserializer.EMPTY_VALUE; - break; - case SimpleAggregateFunction: - deserializer = getDeserializer(config, column.getNestedColumns().get(0)); - break; - default: - throw new IllegalArgumentException("Unsupported column:" + column.toString()); - } - - return deserializer; - // return !column.isLowCardinality() && column.isNullable() - // ? new BinaryDataProcessor.NullableDeserializer(deserializer) - // : deserializer; - } - - @Override - public ClickHouseSerializer getSerializer(ClickHouseDataConfig config, ClickHouseColumn column) { - // TODO Auto-generated method stub - return null; - } - - @Override - public void write(ClickHouseValue value) throws IOException { - // TODO Auto-generated method stub - - } -} diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/format/MessagePackProcessor.java b/clickhouse-data/src/main/java/com/clickhouse/data/format/MessagePackProcessor.java deleted file mode 100644 index c444da7f2..000000000 --- a/clickhouse-data/src/main/java/com/clickhouse/data/format/MessagePackProcessor.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.clickhouse.data.format; - -import java.io.IOException; -import java.io.Serializable; -import java.util.List; -import java.util.Map; - -import com.clickhouse.data.ClickHouseColumn; -import com.clickhouse.data.ClickHouseDataConfig; -import com.clickhouse.data.ClickHouseDataProcessor; -import com.clickhouse.data.ClickHouseDeserializer; -import com.clickhouse.data.ClickHouseInputStream; -import com.clickhouse.data.ClickHouseOutputStream; -import com.clickhouse.data.ClickHouseRecord; -import com.clickhouse.data.ClickHouseSerializer; -import com.clickhouse.data.ClickHouseValue; - -public class MessagePackProcessor extends ClickHouseDataProcessor { - - protected MessagePackProcessor(ClickHouseDataConfig config, ClickHouseInputStream input, - ClickHouseOutputStream output, - List columns, Map settings) throws IOException { - super(config, input, output, columns, settings); - } - - @Override - protected ClickHouseRecord createRecord() { - // TODO Auto-generated method stub - return null; - } - - @Override - protected void readAndFill(ClickHouseValue value) throws IOException { - // TODO Auto-generated method stub - - } - - @Override - protected List readColumns() throws IOException { - // TODO Auto-generated method stub - return null; - } - - @Override - public void write(ClickHouseValue value) throws IOException { - // TODO Auto-generated method stub - - } - - @Override - public ClickHouseDeserializer getDeserializer(ClickHouseDataConfig config, ClickHouseColumn column) { - // TODO Auto-generated method stub - return null; - } - - @Override - public ClickHouseSerializer getSerializer(ClickHouseDataConfig config, ClickHouseColumn column) { - // TODO Auto-generated method stub - return null; - } -} diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/format/ApacheAvroProcessorTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/format/ApacheAvroProcessorTest.java deleted file mode 100644 index 6045b6865..000000000 --- a/clickhouse-data/src/test/java/com/clickhouse/data/format/ApacheAvroProcessorTest.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.clickhouse.data.format; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.Serializable; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.testng.Assert; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -import com.clickhouse.config.ClickHouseOption; -import com.clickhouse.data.ClickHouseColumn; -import com.clickhouse.data.ClickHouseDataConfig; -import com.clickhouse.data.ClickHouseFormat; -import com.clickhouse.data.ClickHouseInputStream; -import com.clickhouse.data.ClickHouseTestDataConfig; -import com.clickhouse.data.ClickHouseValue; -import com.clickhouse.data.value.ClickHouseStringValue; - -public class ApacheAvroProcessorTest { - @Test(groups = { "unit" }) - public void testInit() throws IOException { - List list = ClickHouseColumn.parse("a String, b Nullable(Int32)"); - ClickHouseValue value = ClickHouseStringValue.of("aaa"); - ClickHouseDataConfig config = new ClickHouseTestDataConfig() { - @Override - public ClickHouseFormat getFormat() { - return ClickHouseFormat.Avro; - } - }; - ApacheAvroProcessor p = new ApacheAvroProcessor(config, - ClickHouseInputStream.of(new File("/Users/zhicwu/Sources/Github/a.avro")), null, null, null); - } -} diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseNativeProcessorTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseNativeProcessorTest.java deleted file mode 100644 index ebe8b5202..000000000 --- a/clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseNativeProcessorTest.java +++ /dev/null @@ -1,355 +0,0 @@ -package com.clickhouse.data.format; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.Collections; -import java.util.function.Consumer; - -import org.testng.Assert; -import org.testng.annotations.Test; - -import com.clickhouse.data.ClickHouseColumn; -import com.clickhouse.data.ClickHouseDataConfig; -import com.clickhouse.data.ClickHouseDataProcessor; -import com.clickhouse.data.ClickHouseInputStream; -import com.clickhouse.data.ClickHouseOutputStream; -import com.clickhouse.data.ClickHouseRecord; -import com.clickhouse.data.ClickHouseTestDataConfig; -import com.clickhouse.data.ClickHouseValue; -import com.clickhouse.data.value.BaseDataProcessorTest; - -public class ClickHouseNativeProcessorTest extends BaseDataProcessorTest { - private ClickHouseNativeProcessor newProcessor(int... bytes) throws IOException { - return new ClickHouseNativeProcessor(new ClickHouseTestDataConfig(), BinaryStreamUtilsTest.generateInput(bytes), - null, null, null); - } - - private void verifySingleRecord(ClickHouseDataProcessor p, Object expectedValue) { - int rows = 0; - for (ClickHouseRecord r : p.records()) { - Assert.assertEquals(r.getValue(0).asObject(), expectedValue); - rows++; - } - Assert.assertEquals(rows, 1); - } - - @Override - protected ClickHouseDataProcessor getDataProcessor(ClickHouseDataConfig config, ClickHouseColumn column, - ClickHouseInputStream input, ClickHouseOutputStream output) throws IOException { - return new ClickHouseNativeProcessor(config, input, output, Collections.singletonList(column), null); - } - - // @Override - // protected ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseConfig - // config, - // ClickHouseColumn column, ClickHouseInputStream input) throws IOException { - // if (ref == null) { - // ref = column.newValue(config); - // } - // return new ClickHouseRowBinaryProcessor(config, input, null, - // Collections.singletonList(column), null) - // .getDeserializer(config, column).deserialize(ref, input); - // } - - // @Override - // protected void serialize(ClickHouseValue value, ClickHouseConfig config, - // ClickHouseColumn column, - // ClickHouseOutputStream output) throws IOException { - // ClickHouseNativeProcessor.getMappedFunctions().serialize(value, config, - // column, output); - // } - - @Override - protected byte[] getRawData(String typeName, String key) { - byte[] data = null; - byte[] header = new byte[] { 1, 1, 0, 0 }; - // toBytes(new byte[] { 0x01, 0x01, 0x78 }, new byte[] { (byte) - // typeName.length() },typeName.getBytes()); - boolean isLowCardinality = typeName.startsWith("LowCardinality("); - if (isLowCardinality) { - typeName = typeName.replaceFirst("LowCardinality\\(", "").replaceFirst("\\)", ""); - } - boolean isNull = "null".equals(key); - boolean isNullable = typeName.startsWith("Nullable("); - if (isNullable) { - typeName = typeName.replaceFirst("Nullable\\(", "").replaceFirst("\\)", ""); - if (isNull) { - if (!isLowCardinality) { - return toBytes(header, 1); - } - } else { - if (!isLowCardinality) { - header = toBytes(header, 0); - } - } - } - if (isLowCardinality) { - header = toBytes(header, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0); - } - switch (typeName) { - case "Array(Bool)": - if ("0,1".equals(key)) { - data = toBytes("[false,true]"); - } - break; - case "Array(Int8)": - case "Array(Int16)": - case "Array(Int32)": - case "Array(Int64)": - case "Array(Float32)": - case "Array(Float64)": - if ("0,1,-1".equals(key)) { - data = toBytes("0,1,-1"); - } - break; - case "Array(UInt8)": - if ("0,1,-1".equals(key)) { - data = toBytes("0,1,255"); - } - break; - case "Array(UInt16)": - if ("0,1,-1".equals(key)) { - data = toBytes("0,1,65535"); - } - break; - case "Array(UInt32)": - if ("0,1,-1".equals(key)) { - data = toBytes("0,1,4294967295"); - } - break; - case "Array(UInt64)": - if ("0,1,-1".equals(key)) { - data = toBytes("0,1,18446744073709551615"); - } - break; - case "Array(String)": - if ("4bcd".equals(key)) { - data = toBytes("['','B','BC','BCD']"); - } - break; - case "Bool": - case "Int8": - case "UInt8": - if (isLowCardinality) { - byte b = isNull ? 0 : Byte.parseByte(key); - int base = isNullable ? 1 : 0; - data = toBytes(header, toBytes(base + (b == 0 ? 1 : 2), 0, 0, 0, 0, 0, 0, 0, 0), - new byte[isNullable ? 1 : 0], b == 0 ? new byte[0] : new byte[] { b }, - toBytes(1, 0, 0, 0, 0, 0, 0, 0, isNull ? 0 : (base + (b == 0 ? 0 : 1)))); - } else { - data = toBytes(header, new byte[] { Byte.parseByte(key) }); - } - break; - case "Int16": - case "UInt16": - if (isLowCardinality) { - short s = isNull ? 0 : Short.parseShort(key); - int base = isNullable ? 1 : 0; - data = toBytes(header, toBytes(base + (s == 0 ? 1 : 2), 0, 0, 0, 0, 0, 0, 0, 0, 0), - new byte[isNullable ? 2 : 0], s == 0 ? new byte[0] : toLEBytes(s), - toBytes(1, 0, 0, 0, 0, 0, 0, 0, isNull ? 0 : (base + (s == 0 ? 0 : 1)))); - } else { - data = toBytes(header, toLEBytes(Short.parseShort(key))); - } - break; - case "Int32": - case "UInt32": - if (isLowCardinality) { - int i = isNull ? 0 : Integer.parseInt(key); - int base = isNullable ? 1 : 0; - data = toBytes(header, - toBytes(base + (i == 0 ? 1 : 2), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), - new byte[isNullable ? 4 : 0], i == 0 ? new byte[0] : toLEBytes(i), - toBytes(1, 0, 0, 0, 0, 0, 0, 0, isNull ? 0 : (base + (i == 0 ? 0 : 1)))); - } else { - data = toBytes(header, toLEBytes(Integer.parseInt(key))); - } - break; - case "Int64": - case "UInt64": - if (isLowCardinality) { - long l = isNull ? 0L : Long.parseLong(key); - int base = isNullable ? 1 : 0; - data = toBytes(header, - toBytes(base + (l == 0L ? 1 : 2), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), - new byte[isNullable ? 8 : 0], l == 0L ? new byte[0] : toLEBytes(l), - toBytes(1, 0, 0, 0, 0, 0, 0, 0, isNull ? 0 : (base + (l == 0L ? 0 : 1)))); - } else { - data = toBytes(header, toLEBytes(Long.parseLong(key))); - } - break; - case "Float32": - if (isLowCardinality) { - float f = isNull ? 0F : Float.parseFloat(key); - int base = isNullable ? 1 : 0; - data = toBytes(header, - toBytes(base + (f == 0F ? 1 : 2), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), - new byte[isNullable ? 4 : 0], f == 0F ? new byte[0] : toLEBytes(f), - toBytes(1, 0, 0, 0, 0, 0, 0, 0, isNull ? 0 : (base + (f == 0F ? 0 : 1)))); - } else { - data = toBytes(header, toLEBytes(Float.parseFloat(key))); - } - break; - case "Float64": - if (isLowCardinality) { - double d = isNull ? 0D : Double.parseDouble(key); - int base = isNullable ? 1 : 0; - data = toBytes(header, - toBytes(base + (d == 0D ? 1 : 2), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), - new byte[isNullable ? 8 : 0], d == 0D ? new byte[0] : toLEBytes(d), - toBytes(1, 0, 0, 0, 0, 0, 0, 0, isNull ? 0 : (base + (d == 0D ? 0 : 1)))); - } else { - data = toBytes(header, toLEBytes(Double.parseDouble(key))); - } - break; - case "String": - if (isLowCardinality) { - int index = isNull ? 0 : 1; - int base = isNullable ? 1 : 0; - data = toBytes(header, - toBytes(base + (index == 0 ? 1 : 2), 0, 0, 0, 0, 0, 0, 0, 0), - new byte[isNullable ? 1 : 0], index == 0 ? new byte[0] : toLEBytes(key), - toBytes(1, 0, 0, 0, 0, 0, 0, 0, isNull ? 0 : (base + (index == 0 ? 0 : 1)))); - } else { - data = toBytes(header, toLEBytes(key)); - } - break; - default: - break; - } - - if (data == null) { - Assert.fail(String.format("No raw data defined for type=[%s] and key=[%s]", typeName, key)); - } - return data; - } - - @Test(dataProvider = "simpleTypesForWrite", groups = { "unit" }) - public void testSerializeSimpleTypes(ClickHouseDataConfig config, String typeName, String dataKey, - ClickHouseValue... values) throws IOException { - if (values == null || values.length == 0) { - Assert.fail("At least one value must be provided for testing"); - } - - ByteArrayOutputStream bao = new ByteArrayOutputStream(); - for (ClickHouseValue value : values) { - bao.reset(); - try (ClickHouseOutputStream output = ClickHouseOutputStream.of(bao)) { - serialize(value, config, ClickHouseColumn.of("a", typeName), output); - } - Assert.assertEquals(bao.toByteArray(), getRawData(typeName, dataKey)); - - bao.reset(); - String lowCardinalityTypeName = "LowCardinality(" + typeName + ")"; - try (ClickHouseOutputStream output = ClickHouseOutputStream.of(bao)) { - serialize(value, config, ClickHouseColumn.of("a", lowCardinalityTypeName), output); - } - Assert.assertEquals(bao.toByteArray(), getRawData(lowCardinalityTypeName, dataKey)); - } - } - - @Test(dataProvider = "simpleTypesForRead", groups = { "unit" }) - public void testDeserializeSimpleTypes(ClickHouseDataConfig config, String typeName, String dataKey, - Class valueClass, Object objVal, String strVal, Consumer customChecks) - throws IOException { - try (ClickHouseInputStream in = getInputData(typeName, dataKey)) { - ClickHouseValue value = deserialize(null, config, ClickHouseColumn.of("a", typeName), in); - Assert.assertEquals(value.getClass(), valueClass); - Assert.assertEquals(value.asObject(), objVal); - Assert.assertEquals(value.asString(), strVal); - if (customChecks != null) { - customChecks.accept(value); - } - } - - // String lowCardinalityTypeName = "LowCardinality(" + typeName + ")"; - // try (ClickHouseInputStream in = getInputData(lowCardinalityTypeName, - // dataKey)) { - // ClickHouseValue value = deserialize(null, config, ClickHouseColumn.of("a", - // lowCardinalityTypeName), in); - // Assert.assertEquals(value.getClass(), valueClass); - // Assert.assertEquals(value.asObject(), objVal); - // Assert.assertEquals(value.asString(), strVal); - // if (customChecks != null) { - // customChecks.accept(value); - // } - // } - } - - @Test(groups = { "unit" }) - public void testDeserializeArray() throws IOException { - ClickHouseNativeProcessor p = newProcessor(0x02, 0x01, 0x01, 0x62, 0x06, 0x53, 0x74, 0x72, 0x69, 0x6E, - 0x67, 0x04, 0x61, 0x31, 0x32, 0x33, 0x01, 0x63, 0x05, 0x55, 0x49, 0x6E, 0x74, 0x38, 0x01); - - Assert.assertNotNull(p.getColumns()); - - // select 1::LowCardinality(Int8) a - p = newProcessor(0x01, 0x01, 0x01, 0x61, 0x14, 0x4C, 0x6F, 0x77, 0x43, 0x61, 0x72, 0x64, 0x69, 0x6E, 0x61, 0x6C, - 0x69, 0x74, 0x79, 0x28, 0x49, 0x6e, 0x74, 0x38, 0x29, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01); - Assert.assertNotNull(p.getColumns()); - - // select 1::LowCardinality(String) a - p = newProcessor(0x01, 0x01, 0x01, 0x61, 0x16, 0x4C, 0x6F, 0x77, 0x43, 0x61, 0x72, 0x64, 0x69, 0x6E, - 0x61, 0x6C, 0x69, 0x74, 0x79, 0x28, 0x53, 0x74, 0x72, 0x69, 0x6E, 0x67, 0x29, 0x01, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x01, 0x31, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01); - Assert.assertNotNull(p.getColumns()); - } - - @Test(groups = { "unit" }) - public void testArray() throws IOException { - // [1,2,3]::Array(Int8) - ClickHouseNativeProcessor p = newProcessor(0x01, 0x01, 0x1E, 0x43, 0x41, 0x53, 0x54, 0x28, 0x27, 0x5B, 0x31, - 0x2C, 0x32, 0x2C, 0x33, 0x5D, 0x27, 0x2C, 0x20, 0x27, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x49, 0x6E, - 0x74, 0x38, 0x29, 0x27, 0x29, 0x0B, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x49, 0x6E, 0x74, 0x38, 0x29, - 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03); - verifySingleRecord(p, new byte[] { 1, 2, 3 }); - - // [1,2,3]::Array(Nullable(Int8)) - p = newProcessor(0x01, 0x01, 0x28, 0x43, 0x41, 0x53, 0x54, 0x28, 0x27, 0x5B, 0x31, 0x2C, 0x32, 0x2C, 0x33, 0x5D, - 0x27, 0x2C, 0x20, 0x27, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x4E, 0x75, 0x6C, 0x6C, 0x61, 0x62, 0x6C, - 0x65, 0x28, 0x49, 0x6E, 0x74, 0x38, 0x29, 0x29, 0x27, 0x29, 0x15, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, - 0x4E, 0x75, 0x6C, 0x6C, 0x61, 0x62, 0x6C, 0x65, 0x28, 0x49, 0x6E, 0x74, 0x38, 0x29, 0x29, 0x03, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03); - verifySingleRecord(p, new Byte[] { 1, 2, 3 }); - - // [1,null,3]::Array(Nullable(Int8)) - p = newProcessor(0x01, 0x01, 0x2B, 0x43, 0x41, 0x53, 0x54, 0x28, 0x5B, 0x31, 0x2C, 0x20, 0x4E, 0x55, 0x4C, 0x4C, - 0x2C, 0x20, 0x33, 0x5D, 0x2C, 0x20, 0x27, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x4E, 0x75, 0x6C, 0x6C, - 0x61, 0x62, 0x6C, 0x65, 0x28, 0x49, 0x6E, 0x74, 0x38, 0x29, 0x29, 0x27, 0x29, 0x15, 0x41, 0x72, 0x72, - 0x61, 0x79, 0x28, 0x4E, 0x75, 0x6C, 0x6C, 0x61, 0x62, 0x6C, 0x65, 0x28, 0x49, 0x6E, 0x74, 0x38, 0x29, - 0x29, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x03); - verifySingleRecord(p, new Byte[] { 1, null, 3 }); - - // [[1,2],[],[null,4,5]]::Array(Array(Nullable(Int8))) - p = newProcessor(0x01, 0x01, 0x40, 0x43, 0x41, 0x53, 0x54, 0x28, 0x5B, 0x5B, 0x31, 0x2C, 0x20, 0x32, 0x5D, 0x2C, - 0x20, 0x5B, 0x5D, 0x2C, 0x20, 0x5B, 0x4E, 0x55, 0x4C, 0x4C, 0x2C, 0x20, 0x34, 0x2C, 0x20, 0x35, 0x5D, - 0x5D, 0x2C, 0x20, 0x27, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x4E, - 0x75, 0x6C, 0x6C, 0x61, 0x62, 0x6C, 0x65, 0x28, 0x49, 0x6E, 0x74, 0x38, 0x29, 0x29, 0x29, 0x27, 0x29, - 0x1C, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x4E, 0x75, 0x6C, 0x6C, - 0x61, 0x62, 0x6C, 0x65, 0x28, 0x49, 0x6E, 0x74, 0x38, 0x29, 0x29, 0x29, 0x03, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01, 0x02, - 0x00, 0x04, 0x05); - verifySingleRecord(p, new Byte[][] { { 1, 2 }, {}, { null, 4, 5 } }); - - // [[[null],[1,2],[3,4,5]],[[1,2],[],[null,4,5]]]::Array(Array(Array(Nullable(Int8)))) - newProcessor(0x01, 0x01, 0x66, 0x43, 0x41, 0x53, 0x54, 0x28, 0x5B, 0x5B, 0x5B, 0x4E, 0x55, 0x4C, 0x4C, 0x5D, - 0x2C, 0x20, 0x5B, 0x31, 0x2C, 0x20, 0x32, 0x5D, 0x2C, 0x20, 0x5B, 0x33, 0x2C, 0x20, 0x34, 0x2C, 0x20, - 0x35, 0x5D, 0x5D, 0x2C, 0x20, 0x5B, 0x5B, 0x31, 0x2C, 0x20, 0x32, 0x5D, 0x2C, 0x20, 0x5B, 0x5D, 0x2C, - 0x20, 0x5B, 0x4E, 0x55, 0x4C, 0x4C, 0x2C, 0x20, 0x34, 0x2C, 0x20, 0x35, 0x5D, 0x5D, 0x5D, 0x2C, 0x20, - 0x27, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x41, 0x72, 0x72, 0x61, - 0x79, 0x28, 0x4E, 0x75, 0x6C, 0x6C, 0x61, 0x62, 0x6C, 0x65, 0x28, 0x49, 0x6E, 0x74, 0x38, 0x29, 0x29, - 0x29, 0x29, 0x27, 0x29, 0x23, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, - 0x41, 0x72, 0x72, 0x61, 0x79, 0x28, 0x4E, 0x75, 0x6C, 0x6C, 0x61, 0x62, 0x6C, 0x65, 0x28, 0x49, 0x6E, - 0x74, 0x38, 0x29, 0x29, 0x29, 0x29, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x01, 0x02, 0x00, 0x04, 0x05); - verifySingleRecord(p, new Byte[][][] { { { null }, { 1, 2 }, { 3, 4, 5 } }, { { 1, 2 }, {}, { null, 4, 5 } } }); - } -} From e51c2c4cd45b178de643a5b996d0fef48712f856 Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Wed, 18 Jan 2023 21:08:31 +0800 Subject: [PATCH 3/6] Stick with JDK 8 --- .../com/clickhouse/client/http/ApacheHttpConnectionImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java index 6e2c74c29..cc19e2824 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java @@ -309,7 +309,7 @@ static class HttpConnectionManager extends PoolingHttpClientConnectionManager { static { String versionInfo = null; try { - String pkg = VersionInfo.class.getPackageName(); + String pkg = VersionInfo.class.getPackage().getName(); pkg = pkg.substring(0, pkg.lastIndexOf('.')); versionInfo = VersionInfo.getSoftwareInfo(PROVIDER, pkg, HttpClientBuilder.class).split("\\s")[0]; } catch (Throwable e) { // NOSONAR From 0d04767a1d84c8d5fca5392cb5549d3cb653f8bf Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Wed, 18 Jan 2023 21:16:03 +0800 Subject: [PATCH 4/6] Fix compile error due to changed package names --- .../java/com/clickhouse/benchmark/misc/StreamBenchmark.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/StreamBenchmark.java b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/StreamBenchmark.java index c037b2e42..85bde7026 100644 --- a/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/StreamBenchmark.java +++ b/clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/StreamBenchmark.java @@ -14,9 +14,9 @@ import com.clickhouse.benchmark.BaseState; import com.clickhouse.client.ClickHouseClient; import com.clickhouse.client.ClickHouseConfig; -import com.clickhouse.client.config.ClickHouseBufferingMode; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.config.ClickHouseOption; +import com.clickhouse.config.ClickHouseBufferingMode; +import com.clickhouse.config.ClickHouseOption; import com.clickhouse.data.ClickHouseInputStream; import com.clickhouse.data.ClickHouseOutputStream; From 3043a73823b7f879d169a0f501ec10b4846ad50d Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Wed, 18 Jan 2023 21:38:40 +0800 Subject: [PATCH 5/6] Disable lz4 test for gRPC because it's too slow --- .../com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java b/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java index 77ca53841..2f8ba091e 100644 --- a/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java +++ b/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java @@ -33,7 +33,7 @@ protected Object[][] getRequestCompressionMatrix() { { ClickHouseCompression.BZ2, -2, 2, 1 }, { ClickHouseCompression.DEFLATE, -2, 10, 1 }, // [0, 9] { ClickHouseCompression.GZIP, -2, 10, 1 }, // [-1, 9] - { ClickHouseCompression.LZ4, -2, 19, 1 }, // [0, 18] + // { ClickHouseCompression.LZ4, -2, 19, 1 }, // [0, 18] TOO SLOW! // Code: 638, DB::Exception: hadoop snappy decode error:INVALID_INPUT: While // executing BinaryRowInputFormat // { ClickHouseCompression.SNAPPY, -2, 513, 1024 }, // [1 * 1024, 32 * 1024] @@ -88,6 +88,7 @@ protected Class getClientClass() { @Test(groups = { "integration" }) public void testNothing() throws Exception { + testDumpFile(true, true); } @Test(groups = "integration") From a2a8af04e48da13a60b99c0de51aa795549ac96a Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Wed, 18 Jan 2023 22:17:00 +0800 Subject: [PATCH 6/6] Disable a few tests for gRPC --- .../com/clickhouse/client/ClientIntegrationTest.java | 4 ++-- .../client/grpc/ClickHouseGrpcClientTest.java | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java index 464d931cd..172cb38a3 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java @@ -1528,8 +1528,8 @@ public void testDump() throws ExecutionException, InterruptedException, IOExcept public void testDumpFile(boolean gzipCompressed, boolean useOneLiner) throws ExecutionException, InterruptedException, IOException { ClickHouseNode server = getServer(); - if (server.getProtocol() != ClickHouseProtocol.GRPC && server.getProtocol() != ClickHouseProtocol.HTTP) { - throw new SkipException("Skip as only http and grpc implementation work well"); + if (server.getProtocol() != ClickHouseProtocol.HTTP) { + throw new SkipException("Skip as only http implementation works well"); } File file = Files.createTempFile("chc", ".data").toFile(); diff --git a/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java b/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java index 2f8ba091e..ea4939d23 100644 --- a/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java +++ b/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java @@ -88,7 +88,6 @@ protected Class getClientClass() { @Test(groups = { "integration" }) public void testNothing() throws Exception { - testDumpFile(true, true); } @Test(groups = "integration") @@ -132,6 +131,17 @@ public void testLZ4FrameStream() throws IOException { } + @Test(dataProvider = "mixedCompressionMatrix", groups = "integration") + @Override + public void testDecompressResponse(ClickHouseCompression reqComp, ClickHouseCompression respComp) throws Exception { + if (!checkServerVersion(getClient(), getServer(), "[22.8,)")) { + throw new SkipException( + "Skip due to unexpected end of input error on 22.3 when using brotli for decompression"); + } + + super.testDecompressResponse(reqComp, respComp); + } + @Test(groups = { "integration" }) @Override public void testSessionLock() {