From 13fef38608324d46470cdae9c1bb32cb190ffca6 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Wed, 17 Feb 2021 15:52:43 -0400 Subject: [PATCH 1/8] tests for ORC / Parquet --- .../clickhouse/domain/ClickHouseFormat.java | 5 +- .../clickhouse/integration/StreamSQLTest.java | 165 +++++++++++++++++- .../resources/data_samples/test_sample.orc.gz | Bin 0 -> 1575 bytes .../data_samples/test_sample.parquet.gz | Bin 0 -> 2019 bytes 4 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 src/test/resources/data_samples/test_sample.orc.gz create mode 100644 src/test/resources/data_samples/test_sample.parquet.gz diff --git a/src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java b/src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java index 6a8f9371b..ef8614cb2 100644 --- a/src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java +++ b/src/main/java/ru/yandex/clickhouse/domain/ClickHouseFormat.java @@ -37,7 +37,10 @@ public enum ClickHouseFormat { Native, Null, XML, - CapnProto; + CapnProto, + Parquet, + ORC + ; public static boolean containsFormat(String statement) { if (statement == null || statement.isEmpty()) { diff --git a/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java b/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java index 210e75f4e..088c74c82 100644 --- a/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java +++ b/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java @@ -8,8 +8,10 @@ import ru.yandex.clickhouse.ClickHouseDataSource; import ru.yandex.clickhouse.domain.ClickHouseCompression; import ru.yandex.clickhouse.domain.ClickHouseFormat; -import ru.yandex.clickhouse.settings.ClickHouseProperties; +import ru.yandex.clickhouse.util.ClickHouseVersionNumberUtil; + import java.io.*; +import java.math.BigDecimal; import java.nio.charset.Charset; import java.sql.ResultSet; import java.sql.SQLException; @@ -216,4 +218,165 @@ public void CSVInsertCompressedIntoTable() throws SQLException, IOException { Assert.assertEquals(rs.getLong("uniq"), 1); } + @Test + public void ORCInsertCompressedIntoTable() throws SQLException { + // clickhouse-client -q "select number int, toString(number) str, 1/number flt, toDecimal64( 1/(number+1) , 9) dcml, + // toDateTime('2020-01-01 00:00:00') + number time from numbers(100) format ORC"|gzip > test_sample.orc.gz + + String version = ClickHouseContainerForTest.getClickHouseVersion(); + if (version.isEmpty() || ClickHouseVersionNumberUtil.getMajorVersion(version) <= 20) { + return; + } + + connection.createStatement().execute("DROP TABLE IF EXISTS test.orc_stream_compressed"); + connection.createStatement().execute( + "CREATE TABLE test.orc_stream_compressed (int Int64, str String, flt Float64, " + + "dcml Decimal64(9), time DateTime) ENGINE = Log();" + ); + + InputStream inputStream = BufferedInputStream.class.getResourceAsStream("/data_samples/test_sample.orc.gz"); + + connection.createStatement(). + write() + .table("test.orc_stream_compressed") + .format(ClickHouseFormat.ORC) + .dataCompression(ClickHouseCompression.gzip) + .data(inputStream) + .send(); + + ResultSet rs = connection.createStatement().executeQuery( + "SELECT count() AS cnt, " + + "sum(int) sum_int, " + + "round(sum(flt),2) AS sum_flt, " + + "uniqExact(str) uniq_str, " + + "max(dcml) max_dcml, " + + "min(time) min_time, " + + "max(time) max_time " + + "FROM test.orc_stream_compressed"); + Assert.assertTrue(rs.next()); + Assert.assertEquals(rs.getInt("cnt"), 100); + Assert.assertEquals(rs.getLong("sum_int"), 4950); + Assert.assertEquals(rs.getFloat("sum_flt"), Float.POSITIVE_INFINITY); + Assert.assertEquals(rs.getLong("uniq_str"), 100); + Assert.assertEquals(rs.getBigDecimal("max_dcml"), new BigDecimal("1.000000000")); + Assert.assertEquals(rs.getString("min_time"), "2020-01-01 00:00:00"); + Assert.assertEquals(rs.getString("max_time"), "2020-01-01 00:01:39"); + } + + @Test + public void ORCInsertCompressedIntoTable1() throws SQLException { + // clickhouse-client -q "select number int, toString(number) str, 1/number flt, toDecimal64( 1/(number+1) , 9) dcml, + // toDateTime('2020-01-01 00:00:00') + number time from numbers(100) format ORC"|gzip > test_sample.orc.gz + + String version = ClickHouseContainerForTest.getClickHouseVersion(); + if (version.isEmpty() || ClickHouseVersionNumberUtil.getMajorVersion(version) <= 20) { + return; + } + + connection.createStatement().execute("DROP TABLE IF EXISTS test.orc1_stream_compressed"); + connection.createStatement().execute( + "CREATE TABLE test.orc1_stream_compressed (int Int64, str String, flt Float64, " + + "dcml Decimal64(9), time DateTime) ENGINE = Log();" + ); + + InputStream inputStream = BufferedInputStream.class.getResourceAsStream("/data_samples/test_sample.orc.gz"); + + connection.createStatement(). + write() + .sql("insert into test.orc1_stream_compressed format ORC") + .dataCompression(ClickHouseCompression.gzip) + .data(inputStream) + .send(); + + ResultSet rs = connection.createStatement().executeQuery( + "select * from test.orc1_stream_compressed where int=42"); + Assert.assertTrue(rs.next()); + Assert.assertEquals(rs.getInt("int"), 42); + Assert.assertEquals(rs.getString("str"), "42"); + Assert.assertTrue( Math.abs(rs.getFloat("flt") - 0.023809524) < 0.0001); + Assert.assertTrue( Math.abs(rs.getFloat("dcml") - 0.023255813) < 0.0001); + Assert.assertEquals(rs.getString("time"), "2020-01-01 00:00:42"); + } + + @Test + public void ParquetInsertCompressedIntoTable() throws SQLException { + // clickhouse-client -q "select number int, toString(number) str, 1/number flt, toDecimal64( 1/(number+1) , 9) dcml, + // toDateTime('2020-01-01 00:00:00') + number time from numbers(100) format Parquet"|gzip > test_sample.parquet.gz + + + String version = ClickHouseContainerForTest.getClickHouseVersion(); + if (version.isEmpty() || ClickHouseVersionNumberUtil.getMajorVersion(version) <= 20) { + return; + } + + connection.createStatement().execute("DROP TABLE IF EXISTS test.parquet_stream_compressed"); + connection.createStatement().execute( + "CREATE TABLE test.parquet_stream_compressed (int Int64, str String, flt Float64, " + + "dcml Decimal64(9), time DateTime) ENGINE = Log();" + ); + + InputStream inputStream = BufferedInputStream.class.getResourceAsStream("/data_samples/test_sample.parquet.gz"); + + connection.createStatement(). + write() + .table("test.parquet_stream_compressed") + .format(ClickHouseFormat.Parquet) + .dataCompression(ClickHouseCompression.gzip) + .data(inputStream) + .send(); + + ResultSet rs = connection.createStatement().executeQuery( + "SELECT count() AS cnt, " + + "sum(int) sum_int, " + + "round(sum(flt),2) AS sum_flt, " + + "uniqExact(str) uniq_str, " + + "max(dcml) max_dcml, " + + "min(time) min_time, " + + "max(time) max_time " + + "FROM test.parquet_stream_compressed"); + Assert.assertTrue(rs.next()); + Assert.assertEquals(rs.getInt("cnt"), 100); + Assert.assertEquals(rs.getLong("sum_int"), 4950); + Assert.assertEquals(rs.getFloat("sum_flt"), Float.POSITIVE_INFINITY); + Assert.assertEquals(rs.getLong("uniq_str"), 100); + Assert.assertEquals(rs.getBigDecimal("max_dcml"), new BigDecimal("1.000000000")); + Assert.assertEquals(rs.getString("min_time"), "2020-01-01 00:00:00"); + Assert.assertEquals(rs.getString("max_time"), "2020-01-01 00:01:39"); + } + + @Test + public void ParquetInsertCompressedIntoTable1() throws SQLException { + // clickhouse-client -q "select number int, toString(number) str, 1/number flt, toDecimal64( 1/(number+1) , 9) dcml, + // toDateTime('2020-01-01 00:00:00') + number time from numbers(100) format Parquet"|gzip > test_sample.parquet.gz + + String version = ClickHouseContainerForTest.getClickHouseVersion(); + if (version.isEmpty() || ClickHouseVersionNumberUtil.getMajorVersion(version) <= 20) { + return; + } + + connection.createStatement().execute("DROP TABLE IF EXISTS test.parquet1_stream_compressed"); + connection.createStatement().execute( + "CREATE TABLE test.parquet1_stream_compressed (int Int64, str String, flt Float64, " + + "dcml Decimal64(9), time DateTime) ENGINE = Log();" + ); + + InputStream inputStream = BufferedInputStream.class.getResourceAsStream("/data_samples/test_sample.parquet.gz"); + + connection.createStatement(). + write() + .sql("insert into test.parquet1_stream_compressed format Parquet") + .dataCompression(ClickHouseCompression.gzip) + .data(inputStream) + .send(); + + ResultSet rs = connection.createStatement().executeQuery( + "select * from test.parquet1_stream_compressed where int=42"); + Assert.assertTrue(rs.next()); + Assert.assertEquals(rs.getInt("int"), 42); + Assert.assertEquals(rs.getString("str"), "42"); + Assert.assertTrue( Math.abs(rs.getFloat("flt") - 0.023809524) < 0.0001); + Assert.assertTrue( Math.abs(rs.getFloat("dcml") - 0.023255813) < 0.0001); + Assert.assertEquals(rs.getString("time"), "2020-01-01 00:00:42"); + } + } diff --git a/src/test/resources/data_samples/test_sample.orc.gz b/src/test/resources/data_samples/test_sample.orc.gz new file mode 100644 index 0000000000000000000000000000000000000000..a387cd5914c911be5c55b0d33bfc5b9c95973c83 GIT binary patch literal 1575 zcmV+?2H5!@iwFQLV=Z6+1C>>KP*hbIzjyC0+`S9Svb-+q10*LkM!oL`{&B({YLntP zr8d?m<5pVQXi+(t(_LL)!Iig&hzN=(NZ}cGG)*%!OLH_&3$#c}w9HTp%`gnha174~jL1lg%u+1PGAzq- zEYAw8$V#ltQ5?-N9LsSW&k3B!Nu116Jk2va%X2)>3%tlnyev=xEieKrZ~`v~f+$FW zEK(vZGAa#D%+Ny&!PHu7C-< z_P}$yfxl+E;Q5_kxMt-rPZ=0pQCORBs}$xd0c1j~I<~{S+kkz9>AfQpi(&q)V0mEq zq)Xov!F#rV1>*}3IDg*^@7oNrm)l~iw-v&B3xU%z5g*|!fcF;wJnY;4dhboJj(jkz zHCi1TVLf@ErKUdce%S_CS1zy>MXMth*0&x!AoEh|@8`febHE+Lis-p5>tMa>K**x{ z;6?S>uzdtzl@2_ z&w%_gz>=`~75cmBkY_sZ9rfkh;d9a<-*hmcqc&#g(KN_A4OsNi>PUn9(?Iw=N7y|_ zD%2qrd|cJPgHBI@dZd7Jj_R-Gm9K%itO2GCSklb1Zm5qNbb1_^x7iJKS`A``^i(>g zt%iCfgSz>j%)61340THe?W|>ON?#JxF9~#BZF{tHauU?h1xjy!>Q`Onf_kc`nG5Qg z2#ypjbG)6L2=z?_OB|;6J9UXr=L8V4z!AK_kpT5h0A9hf=Q#`sQ1^K7>ud*gDkUE3 z9}j}=bzSow77w3;6Fhmz4E#rw?QPZGN>=ZAtaQz-13IN??{!phU93Y}6Yl4ueaG`% zsB$Iu>y(Oz3l(R@+=k-znWcwnV_d7sr|jF7`g-fiCg9n9Y{IEyZmIFiA6#W;4x=1A zGyeFE%CVbImqqsM**~VW@Zus`L5ns54)9y{mb&S~^wpcbCP^K{lA&=<1d@KT3bxYg*wx% z4d0s%rskQP`&OIIu6aoGWc3o=N6!%{JDQ22ssqG>zH*|osesshB7jYSZ`3-_`hq_2i38LFX<4AKxIBQ?~MvW5n7 zgo=o|U#}smA*v##&2`#br=l0(t;Tg^9jYy0>cT((BMZE=fF1_)g7ca8u;}$OdFdP|;P)G!h>z{h~ zQ_t(EXI4EcWmG-mV2rXyY2>4xAs_WO`KbRUAN6A7qn=Ct)wF4oQGBIN{h=s|K0=3> ZM!QkUYvmmbi<$9S=wB``t&!RY004JA6G8w0 literal 0 HcmV?d00001 diff --git a/src/test/resources/data_samples/test_sample.parquet.gz b/src/test/resources/data_samples/test_sample.parquet.gz new file mode 100644 index 0000000000000000000000000000000000000000..eab2be518536700db73d4a9055c29e28d1c12e69 GIT binary patch literal 2019 zcmV<92ORhxiwFQ9V=Z6+1MSy)R1;So2k@CN6DE+sh)fI^WvC$rBuWSgp=d9mieRl! zsVyohh*%V45va@2b!%Egq!d9>5hzC279aRP>H`mJ!EP(IXFb>l>!Z5|Ti0q^w^{2d z*q%N64UdYau&16~-G7!dpWn>g(!F$rodE) zhG`H3u`nH;ff?{D%!D|Ihv#4xB*6198(x4HAra=lT$l$*Fdr7cLP&-bo}09Y>itRW z+Y6L%k``L%pq_grNCm0dOzF7(W)(->|C#SdavimHZE~be(WS#k@`?oFj?3f^*jcvPg&SpNj zqkT`_^aFDbU8rCA(alxwb{@``V|;BFrxqA*6GKV8RW3`j@&t*pjaJ_FP&znmr&P-g zH8t-bOq~%V&QIgqQO;eBOgM4B1kW)F8u?(Fr3?@PMhFEHm>~=-G^+}C4qymorIZ0e zzzCsW0yBhxg_W|C(E4vNAr`v##P|z^-p7O*>ACk$%&g~tcGH6xQz#wGm`pvB8rWtY z9Ah@qQ4b`;ej!UQ#u7p&_n!Qcj=3u@aN6q%ZK>OU=?q$Gc-l3~9(dZZG~;-Mb>%1rk#Srnna$GFXS_n*PO7%k zGN6{_HS;=66Nw%%rKxTwVSlX~zLOLvqBy?-JCQj=JK=nP<7jo= z!54|OWb~(2Y;W>qVhw3O+3M9@U(HvFRm5kv6yBD)@GBGil8q z|ATtgX8wY>iPU;t4Y1a3;yZDi?c2JBxm(U(7t0Ah?&XDJ=9Tjf97kMgj$3iIjPDZ5 zh=Q?VE7Mar7Q1Y|UAB#SPPdV~blkDuP+sc%(!^47+19i&`QRIRmG}m6uXfq+!i5sO zD3*}U7MqK!M~Pl1zE0vsU4O?m^L2f&SWHf(E=&HdvRH4?6qAca#g@`8u{Bh|z5^ERN5uxjguU$v4(-&z9$h>!F z**gvukU^FHRYA$V^V^3c_u41N zXin~$J0}u5>5hyu=)(Tvin9xw5wHtgn7d`4u^hLhl|G&y>k&pmzc)h zL{7YoCxb^|G_8=+KiX%b@3$5O;^!UlET=n{YIc4qy5;OU$Ti;8Al6;_J7Ud}1jOS7dvd(?wBN%_c&Np9YT%o_NL_JX&{L}af4l$RzuouUi`2gT zpat9Ch&$bPMwI z+VT{^4S9d6=YI=-^^E!vklXe{#(000s` B=6C=A literal 0 HcmV?d00001 From e6972319174813abd87e3b4ed77029c99bbb2494 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Wed, 17 Feb 2021 16:11:37 -0400 Subject: [PATCH 2/8] fix version --- .../ru/yandex/clickhouse/integration/StreamSQLTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java b/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java index 088c74c82..c9a59d910 100644 --- a/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java +++ b/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java @@ -224,7 +224,7 @@ public void ORCInsertCompressedIntoTable() throws SQLException { // toDateTime('2020-01-01 00:00:00') + number time from numbers(100) format ORC"|gzip > test_sample.orc.gz String version = ClickHouseContainerForTest.getClickHouseVersion(); - if (version.isEmpty() || ClickHouseVersionNumberUtil.getMajorVersion(version) <= 20) { + if (version.isEmpty() || ClickHouseVersionNumberUtil.getMajorVersion(version) < 20) { return; } @@ -269,7 +269,7 @@ public void ORCInsertCompressedIntoTable1() throws SQLException { // toDateTime('2020-01-01 00:00:00') + number time from numbers(100) format ORC"|gzip > test_sample.orc.gz String version = ClickHouseContainerForTest.getClickHouseVersion(); - if (version.isEmpty() || ClickHouseVersionNumberUtil.getMajorVersion(version) <= 20) { + if (version.isEmpty() || ClickHouseVersionNumberUtil.getMajorVersion(version) < 20) { return; } @@ -305,7 +305,7 @@ public void ParquetInsertCompressedIntoTable() throws SQLException { String version = ClickHouseContainerForTest.getClickHouseVersion(); - if (version.isEmpty() || ClickHouseVersionNumberUtil.getMajorVersion(version) <= 20) { + if (version.isEmpty() || ClickHouseVersionNumberUtil.getMajorVersion(version) < 20) { return; } @@ -350,7 +350,7 @@ public void ParquetInsertCompressedIntoTable1() throws SQLException { // toDateTime('2020-01-01 00:00:00') + number time from numbers(100) format Parquet"|gzip > test_sample.parquet.gz String version = ClickHouseContainerForTest.getClickHouseVersion(); - if (version.isEmpty() || ClickHouseVersionNumberUtil.getMajorVersion(version) <= 20) { + if (version.isEmpty() || ClickHouseVersionNumberUtil.getMajorVersion(version) < 20) { return; } From 90ea6f8821e7379fa96e913d2dc0b633d86adcff Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Wed, 17 Feb 2021 20:26:04 -0400 Subject: [PATCH 3/8] install tzdata into CH-server docker to make ORC work --- .../ClickHouseContainerForTest.java | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/test/java/ru/yandex/clickhouse/ClickHouseContainerForTest.java b/src/test/java/ru/yandex/clickhouse/ClickHouseContainerForTest.java index 05c2c1b2a..a15522ef7 100644 --- a/src/test/java/ru/yandex/clickhouse/ClickHouseContainerForTest.java +++ b/src/test/java/ru/yandex/clickhouse/ClickHouseContainerForTest.java @@ -1,10 +1,10 @@ package ru.yandex.clickhouse; import java.time.Duration; - import org.testcontainers.containers.BindMode; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.builder.ImageFromDockerfile; import org.testng.annotations.AfterSuite; import org.testng.annotations.BeforeSuite; @@ -23,6 +23,7 @@ public class ClickHouseContainerForTest { static { String imageTag = System.getProperty("clickhouseVersion"); + if (imageTag == null || (imageTag = imageTag.trim()).isEmpty()) { clickhouseVersion = imageTag = ""; } else { @@ -33,7 +34,24 @@ public class ClickHouseContainerForTest { } imageTag = ":" + imageTag; } - clickhouseContainer = new GenericContainer<>("yandex/clickhouse-server" + imageTag) + + /* on java 8 better to replace with + * clickhouseContainer = new GenericContainer<>( + * new ImageFromDockerfile() + * .withDockerfileFromBuilder(builder -> + * builder + * .from("yandex/clickhouse-server" + imageTag ) + * .run("apt-get update && apt-get install tzdata") + * )) + */ + + final String fullImageName = "yandex/clickhouse-server" + imageTag; + final String runInstallTZ = "RUN apt-get update && apt-get install tzdata"; + + ImageFromDockerfile i = new ImageFromDockerfile(); + i.withFileFromString("Dockerfile", String.format("FROM %s \n %s", fullImageName, runInstallTZ)); + + clickhouseContainer = new GenericContainer<>(i) .withExposedPorts(HTTP_PORT, NATIVE_PORT, MYSQL_PORT) .withClasspathResourceMapping( "ru/yandex/clickhouse/users.d", From 3a565655dfea02b47d9ccb633ac0c6a945747cfd Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Wed, 17 Feb 2021 20:29:59 -0400 Subject: [PATCH 4/8] run ORC / Parquet tests only with CH >= 20.8.4.11 --- .../clickhouse/integration/StreamSQLTest.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java b/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java index c9a59d910..b944d4f9e 100644 --- a/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java +++ b/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java @@ -223,8 +223,8 @@ public void ORCInsertCompressedIntoTable() throws SQLException { // clickhouse-client -q "select number int, toString(number) str, 1/number flt, toDecimal64( 1/(number+1) , 9) dcml, // toDateTime('2020-01-01 00:00:00') + number time from numbers(100) format ORC"|gzip > test_sample.orc.gz - String version = ClickHouseContainerForTest.getClickHouseVersion(); - if (version.isEmpty() || ClickHouseVersionNumberUtil.getMajorVersion(version) < 20) { + String version = connection.getServerVersion(); + if (version.compareTo("20.8.4.11") < 0) { return; } @@ -268,8 +268,8 @@ public void ORCInsertCompressedIntoTable1() throws SQLException { // clickhouse-client -q "select number int, toString(number) str, 1/number flt, toDecimal64( 1/(number+1) , 9) dcml, // toDateTime('2020-01-01 00:00:00') + number time from numbers(100) format ORC"|gzip > test_sample.orc.gz - String version = ClickHouseContainerForTest.getClickHouseVersion(); - if (version.isEmpty() || ClickHouseVersionNumberUtil.getMajorVersion(version) < 20) { + String version = connection.getServerVersion(); + if (version.compareTo("20.8.4.11") < 0) { return; } @@ -303,9 +303,8 @@ public void ParquetInsertCompressedIntoTable() throws SQLException { // clickhouse-client -q "select number int, toString(number) str, 1/number flt, toDecimal64( 1/(number+1) , 9) dcml, // toDateTime('2020-01-01 00:00:00') + number time from numbers(100) format Parquet"|gzip > test_sample.parquet.gz - - String version = ClickHouseContainerForTest.getClickHouseVersion(); - if (version.isEmpty() || ClickHouseVersionNumberUtil.getMajorVersion(version) < 20) { + String version = connection.getServerVersion(); + if (version.compareTo("20.8.4.11") < 0) { return; } @@ -349,8 +348,8 @@ public void ParquetInsertCompressedIntoTable1() throws SQLException { // clickhouse-client -q "select number int, toString(number) str, 1/number flt, toDecimal64( 1/(number+1) , 9) dcml, // toDateTime('2020-01-01 00:00:00') + number time from numbers(100) format Parquet"|gzip > test_sample.parquet.gz - String version = ClickHouseContainerForTest.getClickHouseVersion(); - if (version.isEmpty() || ClickHouseVersionNumberUtil.getMajorVersion(version) < 20) { + String version = connection.getServerVersion(); + if (version.compareTo("20.8.4.11") < 0) { return; } From 6ac3a972e018b4b0e1160907fb7c2b726d06b29c Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Wed, 17 Feb 2021 20:46:00 -0400 Subject: [PATCH 5/8] fix getResourceAsStream for JDK8+ --- .../ru/yandex/clickhouse/integration/StreamSQLTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java b/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java index b944d4f9e..9c619186b 100644 --- a/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java +++ b/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java @@ -234,7 +234,7 @@ public void ORCInsertCompressedIntoTable() throws SQLException { "dcml Decimal64(9), time DateTime) ENGINE = Log();" ); - InputStream inputStream = BufferedInputStream.class.getResourceAsStream("/data_samples/test_sample.orc.gz"); + InputStream inputStream = StreamSQLTest.class.getResourceAsStream("/data_samples/test_sample.orc.gz"); connection.createStatement(). write() @@ -279,7 +279,7 @@ public void ORCInsertCompressedIntoTable1() throws SQLException { "dcml Decimal64(9), time DateTime) ENGINE = Log();" ); - InputStream inputStream = BufferedInputStream.class.getResourceAsStream("/data_samples/test_sample.orc.gz"); + InputStream inputStream = StreamSQLTest.class.getResourceAsStream("/data_samples/test_sample.orc.gz"); connection.createStatement(). write() @@ -314,7 +314,7 @@ public void ParquetInsertCompressedIntoTable() throws SQLException { "dcml Decimal64(9), time DateTime) ENGINE = Log();" ); - InputStream inputStream = BufferedInputStream.class.getResourceAsStream("/data_samples/test_sample.parquet.gz"); + InputStream inputStream = StreamSQLTest.class.getResourceAsStream("/data_samples/test_sample.parquet.gz"); connection.createStatement(). write() @@ -359,7 +359,7 @@ public void ParquetInsertCompressedIntoTable1() throws SQLException { "dcml Decimal64(9), time DateTime) ENGINE = Log();" ); - InputStream inputStream = BufferedInputStream.class.getResourceAsStream("/data_samples/test_sample.parquet.gz"); + InputStream inputStream = StreamSQLTest.class.getResourceAsStream("/data_samples/test_sample.parquet.gz"); connection.createStatement(). write() From d638ebeb3b14118ecdf05e860e065445612d893d Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Thu, 18 Feb 2021 09:17:48 -0400 Subject: [PATCH 6/8] swith to jdk 1.8 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 99d86ee8f..e85ec3844 100644 --- a/pom.xml +++ b/pom.xml @@ -61,7 +61,7 @@ 2.9.10.8 29.0-jre 2.3.1 - 1.7 + 1.8 1.15.1 6.14.3 1.10.19 From 24e2c1f7ea965e400dbc034418c3265255b92cee Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Thu, 18 Feb 2021 09:19:37 -0400 Subject: [PATCH 7/8] more cleaner way to build CH-server container --- .../ClickHouseContainerForTest.java | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/src/test/java/ru/yandex/clickhouse/ClickHouseContainerForTest.java b/src/test/java/ru/yandex/clickhouse/ClickHouseContainerForTest.java index a15522ef7..911d14843 100644 --- a/src/test/java/ru/yandex/clickhouse/ClickHouseContainerForTest.java +++ b/src/test/java/ru/yandex/clickhouse/ClickHouseContainerForTest.java @@ -35,23 +35,14 @@ public class ClickHouseContainerForTest { imageTag = ":" + imageTag; } - /* on java 8 better to replace with - * clickhouseContainer = new GenericContainer<>( - * new ImageFromDockerfile() - * .withDockerfileFromBuilder(builder -> - * builder - * .from("yandex/clickhouse-server" + imageTag ) - * .run("apt-get update && apt-get install tzdata") - * )) - */ - - final String fullImageName = "yandex/clickhouse-server" + imageTag; - final String runInstallTZ = "RUN apt-get update && apt-get install tzdata"; - - ImageFromDockerfile i = new ImageFromDockerfile(); - i.withFileFromString("Dockerfile", String.format("FROM %s \n %s", fullImageName, runInstallTZ)); - - clickhouseContainer = new GenericContainer<>(i) + final String imageNameWithTag = "yandex/clickhouse-server" + imageTag; + + clickhouseContainer = new GenericContainer<>( new ImageFromDockerfile() + .withDockerfileFromBuilder(builder -> + builder + .from( imageNameWithTag ) + .run("apt-get update && apt-get install tzdata") + )) .withExposedPorts(HTTP_PORT, NATIVE_PORT, MYSQL_PORT) .withClasspathResourceMapping( "ru/yandex/clickhouse/users.d", From afd617a5d613896c69294b062aa5ef9b8e560f46 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Thu, 18 Feb 2021 09:20:44 -0400 Subject: [PATCH 8/8] more correct ch-version check, to include 20.8 into test --- .../ru/yandex/clickhouse/integration/StreamSQLTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java b/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java index 9c619186b..c3798f35e 100644 --- a/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java +++ b/src/test/java/ru/yandex/clickhouse/integration/StreamSQLTest.java @@ -224,7 +224,7 @@ public void ORCInsertCompressedIntoTable() throws SQLException { // toDateTime('2020-01-01 00:00:00') + number time from numbers(100) format ORC"|gzip > test_sample.orc.gz String version = connection.getServerVersion(); - if (version.compareTo("20.8.4.11") < 0) { + if (version.compareTo("20.8") < 0) { return; } @@ -269,7 +269,7 @@ public void ORCInsertCompressedIntoTable1() throws SQLException { // toDateTime('2020-01-01 00:00:00') + number time from numbers(100) format ORC"|gzip > test_sample.orc.gz String version = connection.getServerVersion(); - if (version.compareTo("20.8.4.11") < 0) { + if (version.compareTo("20.8") < 0) { return; } @@ -304,7 +304,7 @@ public void ParquetInsertCompressedIntoTable() throws SQLException { // toDateTime('2020-01-01 00:00:00') + number time from numbers(100) format Parquet"|gzip > test_sample.parquet.gz String version = connection.getServerVersion(); - if (version.compareTo("20.8.4.11") < 0) { + if (version.compareTo("20.8") < 0) { return; } @@ -349,7 +349,7 @@ public void ParquetInsertCompressedIntoTable1() throws SQLException { // toDateTime('2020-01-01 00:00:00') + number time from numbers(100) format Parquet"|gzip > test_sample.parquet.gz String version = connection.getServerVersion(); - if (version.compareTo("20.8.4.11") < 0) { + if (version.compareTo("20.8") < 0) { return; }