From a66b3b09f8a493d9d313d7a4f6a13fd5a66dd714 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Fri, 11 Apr 2025 09:14:49 -0700 Subject: [PATCH 1/3] Implementation of a JPA example. A few corrects in existing example --- clickhouse-jdbc/pom.xml | 7 +- examples/demo-service/README.md | 34 ++++++++- examples/demo-service/build.gradle.kts | 22 +++--- .../demo_service/DbConfiguration.java | 12 +++- .../demo_service/JPAInsertController.java | 54 ++++++++++++++ .../demo_service/MetricsConfig.java | 14 ++++ ...etController.java => QueryController.java} | 25 +++---- .../clickhouse/demo_service/data/UIEvent.java | 33 +++++++++ .../jpa/ClickHouseStringArrayType.java | 70 +++++++++++++++++++ .../jpa/UIEventsDbRepository.java | 11 +++ .../src/main/resources/application.properties | 29 +++++++- 11 files changed, 274 insertions(+), 37 deletions(-) create mode 100644 examples/demo-service/src/main/java/com/clickhouse/demo_service/JPAInsertController.java rename examples/demo-service/src/main/java/com/clickhouse/demo_service/{DatasetController.java => QueryController.java} (90%) create mode 100644 examples/demo-service/src/main/java/com/clickhouse/demo_service/data/UIEvent.java create mode 100644 examples/demo-service/src/main/java/com/clickhouse/demo_service/jpa/ClickHouseStringArrayType.java create mode 100644 examples/demo-service/src/main/java/com/clickhouse/demo_service/jpa/UIEventsDbRepository.java diff --git a/clickhouse-jdbc/pom.xml b/clickhouse-jdbc/pom.xml index 0874e3676..9ced3bece 100644 --- a/clickhouse-jdbc/pom.xml +++ b/clickhouse-jdbc/pom.xml @@ -45,11 +45,6 @@ org.apache.commons commons-compress - - - org.lz4 - lz4-java - com.google.code.gson gson @@ -90,7 +85,7 @@ org.lz4 lz4-java - provided + ${lz4.version} com.github.luben diff --git a/examples/demo-service/README.md b/examples/demo-service/README.md index 44ecf73ec..f4c1684f8 100644 --- a/examples/demo-service/README.md +++ b/examples/demo-service/README.md @@ -12,18 +12,46 @@ Application uses `system.numbers` table to generate dataset of any size. It is v - server generates data as if it was a real table - no need to change schema or create tables -To run +Run clickhouse instance in docker: +``` +docker run --rm -e CLICKHOUSE_USER=default -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 -e CLICKHOUSE_PASSWORD=secret\ + -p 8123:8123 clickhouse/clickhouse-server + +``` + +Create table: +``` +CREATE TABLE ui_events +( + `id` UUID, + `timestamp` DateTime64, + `eventName` String +) +ENGINE = MergeTree +ORDER BY timestamp +``` + +To run -```bash +```shell ./gradlew bootRun ``` To test -```bash +```shell curl http://localhost:8080/direct/dataset/0?limit=100000 ``` +JPA Insert +```shell + curl -v -X POST -H "Content-Type: application/json" -d '{"id": "123", "timestamp": "2025-04-07T14:30:00.000Z", "eventName": "Login", "tags": ["security", "activity"]}' http://localhost:8080/events/ui_events +``` +JPA Select +```shell + curl -v -X GET http://localhost:8080/events/ui_events +``` + ## Features - [x] Client V2 New Implementation \ No newline at end of file diff --git a/examples/demo-service/build.gradle.kts b/examples/demo-service/build.gradle.kts index 2547b1214..66b1b19cd 100644 --- a/examples/demo-service/build.gradle.kts +++ b/examples/demo-service/build.gradle.kts @@ -1,7 +1,7 @@ plugins { java - id("org.springframework.boot") version "3.2.8" - id("io.spring.dependency-management") version "1.1.6" + id("org.springframework.boot") version "3.4.4" + id("io.spring.dependency-management") version "1.1.7" } group = "com.clickhouse" @@ -29,22 +29,26 @@ val ch_java_client_version: String by extra dependencies { - // -- clickhouse dependencies - // Main dependency - implementation("com.clickhouse:client-v2:${ch_java_client_version}-SNAPSHOT:all") // local or nightly build + // Add this if working with client directly (not JDBC) +// implementation("com.clickhouse:client-v2:${ch_java_client_version}-SNAPSHOT:all") // local or nightly build // implementation("com.clickhouse:client-v2:${ch_java_client_version}:all") // release version - // -- application dependencies + // OR this if working with JDBC (or both) + implementation("com.clickhouse:clickhouse-jdbc:${ch_java_client_version}-SNAPSHOT:all") // local or nightly build +// implementation("com.clickhouse:clickhouse-jdbc:${ch_java_client_version}:all") // release version + + // Other dependencies implementation("org.springframework.boot:spring-boot-starter-actuator") implementation("org.springframework.boot:spring-boot-starter-web") + + // To enable JPA + implementation("org.springframework.boot:spring-boot-starter-data-jpa") + compileOnly("org.projectlombok:lombok") annotationProcessor("org.projectlombok:lombok") implementation("io.micrometer:micrometer-core:1.14.3") - - - // -- test dependencies testImplementation("org.springframework.boot:spring-boot-starter-test") testRuntimeOnly("org.junit.platform:junit-platform-launcher") diff --git a/examples/demo-service/src/main/java/com/clickhouse/demo_service/DbConfiguration.java b/examples/demo-service/src/main/java/com/clickhouse/demo_service/DbConfiguration.java index 454f15d50..a81a7c535 100644 --- a/examples/demo-service/src/main/java/com/clickhouse/demo_service/DbConfiguration.java +++ b/examples/demo-service/src/main/java/com/clickhouse/demo_service/DbConfiguration.java @@ -2,16 +2,20 @@ import com.clickhouse.client.api.Client; -import io.micrometer.core.instrument.logging.LoggingMeterRegistry; +import io.micrometer.core.instrument.MeterRegistry; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; @Configuration +@EnableJpaRepositories +@EnableTransactionManagement public class DbConfiguration { @Bean - public Client chDirectClient(LoggingMeterRegistry loggingMeterRegistry, @Value("${db.url}") String dbUrl, @Value("${db.user}") String dbUser, + public Client chDirectClient(MeterRegistry meterRegistry, @Value("${db.url}") String dbUrl, @Value("${db.user}") String dbUser, @Value("${db.pass}") String dbPassword) { return new Client.Builder() .addEndpoint(dbUrl) @@ -27,7 +31,9 @@ public Client chDirectClient(LoggingMeterRegistry loggingMeterRegistry, @Value(" .setSocketSndbuf(500_000) .setClientNetworkBufferSize(500_000) .allowBinaryReaderToReuseBuffers(true) // using buffer pool for binary reader - .registerClientMetrics(loggingMeterRegistry, "clickhouse-client-metrics") + .registerClientMetrics(meterRegistry, "clickhouse-client-metrics") .build(); } + + // JPA configured via application.properties } diff --git a/examples/demo-service/src/main/java/com/clickhouse/demo_service/JPAInsertController.java b/examples/demo-service/src/main/java/com/clickhouse/demo_service/JPAInsertController.java new file mode 100644 index 000000000..80c0012dc --- /dev/null +++ b/examples/demo-service/src/main/java/com/clickhouse/demo_service/JPAInsertController.java @@ -0,0 +1,54 @@ +package com.clickhouse.demo_service; + +import com.clickhouse.demo_service.data.UIEvent; +import com.clickhouse.demo_service.jpa.UIEventsDbRepository; +import jakarta.annotation.PostConstruct; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import lombok.extern.java.Log; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.web.bind.annotation.*; + +import java.util.Collection; + +/** + * Class demonstrates usage of ClickHouse JDBC driver with JPA + * + */ +@RestController +@RequestMapping("/events/") +@Log +public class JPAInsertController { + + @Autowired + private UIEventsDbRepository uiEvents; + + @Autowired + private PlatformTransactionManager transactionManager; + + @Autowired + private EntityManager entityManager; + + private TransactionTemplate transactionTemplate; + + @PostConstruct + public void setUp() { + transactionTemplate = new TransactionTemplate(transactionManager); + } + + @PostMapping("/ui_events") + @Transactional(Transactional.TxType.NOT_SUPPORTED) + public void addUIEvent(@RequestBody UIEvent event) { + // do input validation and conversion + transactionTemplate.executeWithoutResult(transactionStatus -> { + entityManager.persist(event); + }); + } + + @GetMapping("/ui_events") + public Collection lastUIEvents() { + return uiEvents.findAll(); + } +} diff --git a/examples/demo-service/src/main/java/com/clickhouse/demo_service/MetricsConfig.java b/examples/demo-service/src/main/java/com/clickhouse/demo_service/MetricsConfig.java index f038db845..ba30ba589 100644 --- a/examples/demo-service/src/main/java/com/clickhouse/demo_service/MetricsConfig.java +++ b/examples/demo-service/src/main/java/com/clickhouse/demo_service/MetricsConfig.java @@ -2,7 +2,12 @@ import io.micrometer.core.instrument.Clock; import io.micrometer.core.instrument.logging.LoggingMeterRegistry; import io.micrometer.core.instrument.logging.LoggingRegistryConfig; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; import java.time.Duration; @@ -32,10 +37,19 @@ public Duration step() { // Create the LoggingMeterRegistry bean. @Bean + @ConditionalOnProperty("app.log_metrics") public LoggingMeterRegistry loggingMeterRegistry(LoggingRegistryConfig config) { LoggingMeterRegistry registry = new LoggingMeterRegistry(config, Clock.SYSTEM); // Start the registry’s internal scheduler so that metrics are published periodically. registry.start(); return registry; } + + @Bean + @ConditionalOnProperty(value = "app.log_metrics", havingValue = "false") + public SimpleMeterRegistry simpleMeterRegistry() { + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + + return registry; + } } diff --git a/examples/demo-service/src/main/java/com/clickhouse/demo_service/DatasetController.java b/examples/demo-service/src/main/java/com/clickhouse/demo_service/QueryController.java similarity index 90% rename from examples/demo-service/src/main/java/com/clickhouse/demo_service/DatasetController.java rename to examples/demo-service/src/main/java/com/clickhouse/demo_service/QueryController.java index 56ade8991..684c667a6 100644 --- a/examples/demo-service/src/main/java/com/clickhouse/demo_service/DatasetController.java +++ b/examples/demo-service/src/main/java/com/clickhouse/demo_service/QueryController.java @@ -16,11 +16,7 @@ import jakarta.annotation.PostConstruct; import jakarta.servlet.http.HttpServletResponse; import lombok.extern.java.Log; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.ResponseBody; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import java.util.ArrayList; import java.util.LinkedList; @@ -31,15 +27,14 @@ import java.util.stream.Collectors; /** - * Dataset API: - * - /direct/dataset/0/?limit=N - uses client v2 directly to fetch N rows from a virtual dataset. - * - *

Example: {@code curl -v http://localhost:8080/direct/dataset/0?limit=10}

+ * Class demonstrates using ClickHouse client directly from a service. + * It avoids JDBC overhead and much easier to use. + * Data may be streamed from database directly to the service response. */ @RestController -@RequestMapping("/") +@RequestMapping("/dataset") @Log -public class DatasetController { +public class QueryController { private final Client chDirectClient; @@ -47,7 +42,7 @@ public class DatasetController { private BasicObjectsPool> pool; - public DatasetController(Client chDirectClient) { + public QueryController(Client chDirectClient) { this.chDirectClient = chDirectClient; } @@ -95,7 +90,7 @@ VirtualDatasetRecord create() { * @param limit * @return */ - @GetMapping("/dataset/reader") + @GetMapping("/reader") public List directDatasetFetch(@RequestParam(name = "limit", required = false) Integer limit) { limit = limit == null ? 100 : limit; @@ -140,7 +135,7 @@ public List directDatasetFetch(@RequestParam(name = "limit * @param httpResp * @param limit */ - @GetMapping("/dataset/json_each_row_in_and_out") + @GetMapping("/json_each_row_in_and_out") @ResponseBody public void directDataFetchJSONEachRow(HttpServletResponse httpResp, @RequestParam(name = "limit", required = false) Integer limit) { limit = limit == null ? 100 : limit; @@ -184,7 +179,7 @@ public void directDataFetchJSONEachRow(HttpServletResponse httpResp, @RequestPar * @param limit * @return */ - @GetMapping("/dataset/read_to_pojo") + @GetMapping("/read_to_pojo") public CalculationResult directDatasetReadToPojo(@RequestParam(name = "limit", required = false) Integer limit, @RequestParam(name = "cache", required = false) Boolean cache) { limit = limit == null ? 100 : limit; diff --git a/examples/demo-service/src/main/java/com/clickhouse/demo_service/data/UIEvent.java b/examples/demo-service/src/main/java/com/clickhouse/demo_service/data/UIEvent.java new file mode 100644 index 000000000..0a2403fcf --- /dev/null +++ b/examples/demo-service/src/main/java/com/clickhouse/demo_service/data/UIEvent.java @@ -0,0 +1,33 @@ +package com.clickhouse.demo_service.data; + +import com.clickhouse.demo_service.jpa.ClickHouseStringArrayType; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Data; +import org.hibernate.annotations.Array; +import org.hibernate.annotations.JdbcType; +import org.hibernate.annotations.Type; +import org.hibernate.type.descriptor.jdbc.ArrayJdbcType; + +import java.sql.JDBCType; +import java.sql.Timestamp; +import java.util.Collection; +import java.util.List; + +@Entity +@Data +@Table(name = "ui_events") +public class UIEvent { + + @Id + private String id; + + private Timestamp timestamp; + + private String eventName; + + @JdbcType(ArrayJdbcType.class) + @Type(ClickHouseStringArrayType.class) + private Collection tags; +} diff --git a/examples/demo-service/src/main/java/com/clickhouse/demo_service/jpa/ClickHouseStringArrayType.java b/examples/demo-service/src/main/java/com/clickhouse/demo_service/jpa/ClickHouseStringArrayType.java new file mode 100644 index 000000000..b465db547 --- /dev/null +++ b/examples/demo-service/src/main/java/com/clickhouse/demo_service/jpa/ClickHouseStringArrayType.java @@ -0,0 +1,70 @@ +package com.clickhouse.demo_service.jpa; + +import org.hibernate.engine.spi.SharedSessionContractImplementor; +import org.hibernate.usertype.UserType; + +import java.io.Serializable; +import java.sql.*; +import java.util.*; + +public class ClickHouseStringArrayType implements UserType { + @Override + public int getSqlType() { + return Types.ARRAY; + } + + @Override + public Class returnedClass() { + return Collection.class; + } + + @Override + public boolean equals(Collection x, Collection y) { + return x.equals(y); + } + + @Override + public int hashCode(Collection x) { + return x.hashCode(); + } + + @Override + public Collection nullSafeGet(ResultSet rs, int position, SharedSessionContractImplementor session, Object owner) throws SQLException { + // This implementation not optimal, but portable. + Array array = rs.getArray(position); + if (array != null) { + List list = new ArrayList<>(); + for (Object i : (Object[]) array.getArray() ) { + list.add(i); + } + return list; + } + return Collections.emptyList(); + } + + @Override + public void nullSafeSet(PreparedStatement st, Collection value, int index, SharedSessionContractImplementor session) throws SQLException { + st.setObject(index, value); + } + + @Override + public Collection deepCopy(Collection value) { + return new ArrayList(value); + } + + @Override + public boolean isMutable() { + return false; + } + + @Override + public Serializable disassemble(Collection value) { + return new ArrayList<>(value); + } + + @Override + public Collection assemble(Serializable cached, Object owner) { + return (Collection) cached; + } + +} diff --git a/examples/demo-service/src/main/java/com/clickhouse/demo_service/jpa/UIEventsDbRepository.java b/examples/demo-service/src/main/java/com/clickhouse/demo_service/jpa/UIEventsDbRepository.java new file mode 100644 index 000000000..044825c75 --- /dev/null +++ b/examples/demo-service/src/main/java/com/clickhouse/demo_service/jpa/UIEventsDbRepository.java @@ -0,0 +1,11 @@ +package com.clickhouse.demo_service.jpa; + +import com.clickhouse.demo_service.data.UIEvent; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.util.UUID; + +@Repository +public interface UIEventsDbRepository extends JpaRepository { +} diff --git a/examples/demo-service/src/main/resources/application.properties b/examples/demo-service/src/main/resources/application.properties index aecd0bb18..0dfb1fba4 100644 --- a/examples/demo-service/src/main/resources/application.properties +++ b/examples/demo-service/src/main/resources/application.properties @@ -1,6 +1,33 @@ spring.application.name=demo-service +app.log_metrics=false db.url=http://localhost:8123/default db.user=default -db.pass= \ No newline at end of file +db.pass=secret + + +spring.jpa.properties.hibernate.dialect=org.hibernate.annotations.processing.GenericDialect +spring.jpa.hibernate.connection.provider_class=com.zaxxer.hikari.hibernate.HikariConnectionProvider +spring.jpa.show-sql=true +spring.jpa.hibernate.ddl-auto=validate + +spring.datasource.url=jdbc:ch:http//${CH_ADDRESS:localhost:8123}/default +spring.datasource.username=${db.user} +spring.datasource.password=${db.pass} +spring.datasource.driver-class-name=com.clickhouse.jdbc.ClickHouseDriver + +spring.datasource.hikari.connection-timeout=30000 +spring.datasource.hikari.idle-timeout=20000 +spring.datasource.hikari.max-lifetime=300000 +spring.datasource.hikari.maximum-pool-size=100 +spring.datasource.hikari.minimum-idle=1 +spring.datasource.hikari.pool-name=ChConnPool +spring.datasource.hikari.connection-test-query=select 1 + +# To not throw exception on unsupported operations +spring.datasource.hikari.dataSourceProperties.jdbc_ignore_unsupported_values=true + + +logging.level.com.zaxxer.hikari.HikariConfig=DEBUG +logging.level.com.zaxxer.hikari=TRACE \ No newline at end of file From 2d6db112b1bff2d95700b97c3e04b799a1f3a262 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Fri, 11 Apr 2025 10:46:44 -0700 Subject: [PATCH 2/3] Updated documentation --- examples/demo-service/README.md | 60 ++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/examples/demo-service/README.md b/examples/demo-service/README.md index f4c1684f8..775018898 100644 --- a/examples/demo-service/README.md +++ b/examples/demo-service/README.md @@ -1,57 +1,69 @@ # ClickHouse-Java Demo Service ## About -This is an example of a Spring Boot application using different ClickHouse-Java clients and features. +This is and example of a Spring Boot service using ClickHouse client directly and via JPA. +Example is an application that requires ClickHouse DB running externally. It can be a Docker or +ClickHouse Cloud instance. +## How to Run -## Usage +### Initialize DB +Set up a ClickHouse instance. -This example requires an instance of ClickHouse running locally on in remote server. -Application uses `system.numbers` table to generate dataset of any size. It is very convenient because: -- data is already there -- server generates data as if it was a real table -- no need to change schema or create tables +Example of running with Docker: +```shell +docker run -d --name demo-service-db -e CLICKHOUSE_USER=default -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 -e CLICKHOUSE_PASSWORD=secret\ + -p 8123:8123 clickhouse/clickhouse-server -Run clickhouse instance in docker: +docker ps +# output should be like: +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +2abfefc40a84 clickhouse/clickhouse-server "/entrypoint.sh" 4 seconds ago Up 2 seconds 9000/tcp, 0.0.0.0:8123->8123/tcp, :::8123->8123/tcp, 9009/tcp demo-service-db ``` -docker run --rm -e CLICKHOUSE_USER=default -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 -e CLICKHOUSE_PASSWORD=secret\ - -p 8123:8123 clickhouse/clickhouse-server +Example how to run client: +```shell +docker exec -it demo-service-db clickhouse-client ``` -Create table: +Create table (needed for JPA example): ``` CREATE TABLE ui_events ( - `id` UUID, + `id` String, `timestamp` DateTime64, - `eventName` String + `event_name` String, + `tags` Array(String) ) ENGINE = MergeTree ORDER BY timestamp ``` -To run +### Run Demo-Service ```shell ./gradlew bootRun ``` -To test +### Interact with API + +#### Direct Client +To read `limit` number of rows from `system.numbers`: ```shell curl http://localhost:8080/direct/dataset/0?limit=100000 ``` -JPA Insert +#### JPA + +To insert some data: ```shell - curl -v -X POST -H "Content-Type: application/json" -d '{"id": "123", "timestamp": "2025-04-07T14:30:00.000Z", "eventName": "Login", "tags": ["security", "activity"]}' http://localhost:8080/events/ui_events -``` -JPA Select -```shell - curl -v -X GET http://localhost:8080/events/ui_events +curl -v -X POST -H "Content-Type: application/json" \ + -d '{"id": "4NAD7B8HH1", "timestamp": "2025-04-07T14:30:00.000Z", "eventName": "Login", "tags": ["security", "activity"]}'\ + http://localhost:8080/events/ui_events ``` -## Features - -- [x] Client V2 New Implementation \ No newline at end of file +To fetch inserted data: +```shell +curl -v -X GET http://localhost:8080/events/ui_events +``` \ No newline at end of file From c7576e5165d6bb448dabb0c867494480aeaf6e21 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Mon, 14 Apr 2025 09:40:12 -0700 Subject: [PATCH 3/3] review comments --- examples/demo-service/README.md | 2 +- .../clickhouse/demo_service/jpa/ClickHouseStringArrayType.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/demo-service/README.md b/examples/demo-service/README.md index 775018898..c8a1608e2 100644 --- a/examples/demo-service/README.md +++ b/examples/demo-service/README.md @@ -1,7 +1,7 @@ # ClickHouse-Java Demo Service ## About -This is and example of a Spring Boot service using ClickHouse client directly and via JPA. +This is an example of a Spring Boot service using ClickHouse client directly and via JPA. Example is an application that requires ClickHouse DB running externally. It can be a Docker or ClickHouse Cloud instance. diff --git a/examples/demo-service/src/main/java/com/clickhouse/demo_service/jpa/ClickHouseStringArrayType.java b/examples/demo-service/src/main/java/com/clickhouse/demo_service/jpa/ClickHouseStringArrayType.java index b465db547..1f2b1f61a 100644 --- a/examples/demo-service/src/main/java/com/clickhouse/demo_service/jpa/ClickHouseStringArrayType.java +++ b/examples/demo-service/src/main/java/com/clickhouse/demo_service/jpa/ClickHouseStringArrayType.java @@ -54,6 +54,7 @@ public Collection deepCopy(Collection value) { @Override public boolean isMutable() { + // value should not be changed return false; }