diff --git a/README.md b/README.md index 0835f3a..2acb441 100644 --- a/README.md +++ b/README.md @@ -102,27 +102,25 @@ CeresDB is a high-performance, distributed, schema-less, cloud native time-serie io.ceresdb ceresdb-all - 0.1.0 + 1.0.0-alpha ``` ## Init CeresDB client ```java -// CeresDB options -final CeresDBOptions opts = CeresDBOptions.newBuilder("127.0.0.1", 8831) //// CeresDB default grpc port 8831 - .tenant("public", "sub_test", "test_token") // tenant info +final CeresDBOptions opts = CeresDBOptions.newBuilder("127.0.0.1", 8831, DIRECT) // CeresDB default grpc port 8831，use DIRECT RouteMode + .database("public") // use database for client, can be overridden by the RequestContext in request // maximum retry times when write fails // (only some error codes will be retried, such as the routing table failure) .writeMaxRetries(1) // maximum retry times when read fails // (only some error codes will be retried, such as the routing table failure) - .readMaxRetries(1) - .build(); + .readMaxRetries(1).build(); final CeresDBClient client = new CeresDBClient(); - if (!client.init(this.opts)) { +if (!client.init(opts)) { throw new IllegalStateException("Fail to start CeresDBClient"); - } +} ``` For more configuration options, see [configuration](docs/configuration.md) @@ -132,105 +130,87 @@ CeresDB is a Schema-less time-series database, so creating table schema ahead of The following table creation statement（using the SQL API included in SDK ）shows all field types supported by CeresDB： ```java -SqlResult result = client.management().executeSql("CREATE TABLE MY_FIRST_TABL(" + - "ts TIMESTAMP NOT NULL," + - "c1 STRING TAG NOT NULL," + - "c2 STRING TAG NOT NULL," + - "c3 DOUBLE NULL," + - "c4 STRING NULL," + - "c5 INT64 NULL," + - "c6 FLOAT NULL," + - "c7 INT32 NULL," + - "c8 INT16 NULL," + - "c9 INT8 NULL," + - "c10 BOOLEAN NULL," - "c11 UINT64 NULL," - "c12 UINT32 NULL," - "c13 UINT16 NULL," - "c14 UINT8 NULL," - "c15 TIMESTAMP NULL," - "c16 VARBINARY NULL," - "TIMESTAMP KEY(ts)) ENGINE=Analytic" -); +// Create table manually, creating table schema ahead of data ingestion is not required +String createTableSql = "CREATE TABLE IF NOT EXISTS machine_table(" + "ts TIMESTAMP NOT NULL," + // + "ts TIMESTAMP NOT NULL," + + "city STRING TAG NOT NULL," + + "ip STRING TAG NOT NULL," + + "cpu DOUBLE NULL," + + "mem DOUBLE NULL," + + "TIMESTAMP KEY(ts)" + // timestamp column must be specified + ") ENGINE=Analytic"; + +Result createResult = client.sqlQuery(new SqlQueryRequest(createTableSql)).get(); +if (!createResult.isOk()) { + throw new IllegalStateException("Fail to create table"); +} ``` -## Write data example +## How to build write data ```java -final long t0 = System.currentTimeMillis(); -final long t1 = t0 + 1000; -final long t2 = t1 + 1000; -final Rows data = Series.newBuilder("machine_metric") - .tag("city", "Singapore") - .tag("ip", "127.0.0.1") - .toRowsBuilder() - // codes below organizes 3 lines data (3 timestamps) for the `cpu` and `mem` column, this will just transport once through network. CeresDB encourage practices like this, because the SDK could use efficient compression algorithm to reduce network traffic and also be friendly to the sever side. - .field(t0, "cpu", FieldValue.withDouble(0.23)) // first row, first column - .field(t0, "mem", FieldValue.withDouble(0.55)) // first row, second column - .field(t1, "cpu", FieldValue.withDouble(0.25)) // second row, first column - .field(t1, "mem", FieldValue.withDouble(0.56)) // second row, second column - .field(t2, "cpu", FieldValue.withDouble(0.21)) // third row, first column - .field(t2, "mem", FieldValue.withDouble(0.52)) // third row, second column +final Point point = Point.newPointBuilder("machine_table") + .setTimestamp(t0) + .addTag("city", "Singapore") + .addTag("ip", "10.0.0.1") + .addField("cpu", Value.withDouble(0.23)) + .addField("mem", Value.withDouble(0.55)) .build(); +``` -final CompletableFuture> wf = client.write(data); +## Write data example +```java +final CompletableFuture> wf = client.write(new WriteRequest(pointList)); // here the `future.get` is just for demonstration, a better async programming practice would be using the CompletableFuture API -final Result wr = wf.get(); - - Assert.assertTrue(wr.isOk()); - Assert.assertEquals(3, wr.getOk().getSuccess()); - // `Result` class referenced the Rust language practice, provides rich functions (such as mapXXX, andThen) transforming the result value to improve programming efficiency. You can refer to the API docs for detail usage. - Assert.assertEquals(3, wr.mapOr(0, WriteOk::getSuccess).intValue()); - Assert.assertEquals(0, wr.getOk().getFailed()); - Assert.assertEquals(0, wr.mapOr(-1, WriteOk::getFailed).intValue()); +final Result writeResult = wf.get(); +Assert.assertTrue(writeResult.isOk()); +// `Result` class referenced the Rust language practice, provides rich functions (such as mapXXX, andThen) transforming the result value to improve programming efficiency. You can refer to the API docs for detail usage. +Assert.assertEquals(3, writeResult.getOk().getSuccess()); +Assert.assertEquals(3, writeResult.mapOr(0, WriteOk::getSuccess).intValue()); +Assert.assertEquals(0, writeResult.mapOr(-1, WriteOk::getFailed).intValue()); ``` See [write](docs/write.md) ## Query data example ```java -final QueryRequest queryRequest = QueryRequest.newBuilder() - .forMetrics("machine_metric") // table name is optional. If not provided, SQL parser will parse the `ql` to get the table name and do the routing automaticly - .ql("select timestamp, cpu, mem from machine_metric") // +final SqlQueryRequest queryRequest = SqlQueryRequest.newBuilder() + .forTables("machine_table") // table name is optional. If not provided, SQL parser will parse the `sql` to get the table name and do the routing automaticly + .sql("select * from machine_table where ts = %d", t0) // .build(); -final CompletableFuture> qf = client.query(queryRequest); +final CompletableFuture> qf = client.sqlQuery(queryRequest); // here the `future.get` is just for demonstration, a better async programming practice would be using the CompletableFuture API -final Result qr = qf.get(); +final Result queryResult = qf.get(); + +Assert.assertTrue(queryResult.isOk()); - Assert.assertTrue(qr.isOk()); +final SqlQueryOk queryOk = queryResult.getOk(); +Assert.assertEquals(1, queryOk.getRowCount()); -final QueryOk queryOk = qr.getOk(); +// get rows as list +final List rows = queryOk.getRowList(); -final List records = queryOk.mapToRecord().collect(Collectors.toList()) +// get rows as stream +final Stream rowStream = queryOk.stream(); +rowStream.forEach(row -> System.out.println(row.toString())); ``` See [read](docs/read.md) ## stream write/read Example CeresDB support streaming writing and reading，suitable for large-scale data reading and writing。 ```java -final Calendar time = Calendar.getInstance(); -final StreamWriteBuf writeBuf = client.streamWrite("machine_metric"); - for (int i = 0; i < 1000; i++) { - time.add(Calendar.MILLISECOND, 1); - Collection rows = new ArrayList<>(); - final long t0 = System.currentTimeMillis(); - final long t1 = t0 + 1000; - final long t2 = t1 + 1000; - final Rows data = Series.newBuilder("machine_metric").tag("city", "Singapore").tag("ip", "127.0.0.1") - .toRowsBuilder() - .field(t0, "cpu", FieldValue.withDouble(0.23)) - .field(t0, "mem", FieldValue.withDouble(0.55)) - .field(t1, "cpu", FieldValue.withDouble(0.25)) - .field(t1, "mem", FieldValue.withDouble(0.56)) - .field(t2, "cpu", FieldValue.withDouble(0.21)) - .field(t2, "mem", FieldValue.withDouble(0.52)) +final StreamWriteBuf writeBuf = client.streamWrite("machine_table"); +for (int i = 0; i < 1000; i++) { + final Point point = Point.newPointBuilder("machine_table") + .setTimestamp(timestamp) + .addTag("city", "Beijing") + .addTag("ip", "10.0.0.3") + .addField("cpu", Value.withDouble(0.42)) + .addField("mem", Value.withDouble(0.67)) .build(); - rows.add(data); - writeBuf.writeAndFlush(data); - } -final CompletableFuture writeOk = writeBuf.completed(); - Assert.assertEquals(1000, writeOk.join().getSuccess()); + writeBuf.writeAndFlush(Arrays.asList(point)); + timestamp = timestamp+1; +} -final QueryRequest req = QueryRequest.newBuilder().ql("select * from %s", "machine_metric").build(); -final Iterator it = client.blockingStreamQuery(req, 3, TimeUnit.SECONDS); +final CompletableFuture writeOk = writeBuf.completed(); ``` See [streaming](docs/streaming.md) diff --git a/README_CN.md b/README_CN.md index f145f9a..38724a1 100644 --- a/README_CN.md +++ b/README_CN.md @@ -99,27 +99,23 @@ CeresDBClient 是 CeresDB 的高性能 Java 版客户端。CeresDB 是定位为 io.ceresdb ceresdb-all - 0.1.0-RC + 1.0.0-alpha ``` ## 初始化 CeresDB Client ```java // CeresDB options -final CeresDBOptions opts = CeresDBOptions.newBuilder("127.0.0.1", 8831) // - .tenant("test", "sub_test", "test_token") // tenant info - // maximum retry times when write fails - // (only some error codes will be retried, such as the routing table failure) - .writeMaxRetries(1) - // maximum retry times when read fails - // (only some error codes will be retried, such as the routing table failure) - .readMaxRetries(1) +final CeresDBOptions opts = CeresDBOptions.newBuilder("127.0.0.1", 8831, DIRECT) // 默认 gprc 端口号，DIRECT 模式 + .database("public") // Client所使用的database，可被RequestContext的database覆盖 + .writeMaxRetries(1) // 写入失败重试次数上限（只有部分错误 code 才会重试，比如路由表失效） + .readMaxRetries(1) // 查询失败重试次数上限（只有部分错误 code 才会重试，比如路由表失效） .build(); final CeresDBClient client = new CeresDBClient(); - if (!client.init(this.opts)) { +if (!client.init(opts)) { throw new IllegalStateException("Fail to start CeresDBClient"); - } +} ``` 配置详情见 [configuration](docs/configuration.md) @@ -131,118 +127,88 @@ CeresDB 是一个 Schema-less 的时序数据引擎，你可以不必创建 sche 下面的建表语句（使用 SDK 的 SQL API）包含了 CeresDB 支持的所有字段类型： ```java -final CeresDBOptions opts = CeresDBOptions.newBuilder("127.0.0.1", 8831) // 默认 gprc 端口号 - .managementAddress("127.0.0.1", 5440) // 注意，直接使用 sql 需要连接 CeresDB 的 http 端口 - .tenant("public", "sub_test", "test_token") // 租户信息 - .writeMaxRetries(1) // 写入失败重试次数上限（只有部分错误 code 才会重试，比如路由表失效） - .readMaxRetries(1) // 查询失败重试次数上限（只有部分错误 code 才会重试，比如路由表失效） - .build(); - -SqlResult result = client.management().executeSql("CREATE TABLE MY_FIRST_TABL(" + - "ts TIMESTAMP NOT NULL," + - "c1 STRING TAG NOT NULL," + - "c2 STRING TAG NOT NULL," + - "c3 DOUBLE NULL," + - "c4 STRING NULL," + - "c5 INT64 NULL," + - "c6 FLOAT NULL," + - "c7 INT32 NULL," + - "c8 INT16 NULL," + - "c9 INT8 NULL," + - "c10 BOOLEAN NULL," - "c11 UINT64 NULL," - "c12 UINT32 NULL," - "c13 UINT16 NULL," - "c14 UINT8 NULL," - "c15 TIMESTAMP NULL," - "c16 VARBINARY NULL," - "TIMESTAMP KEY(ts)) ENGINE=Analytic" -); +String createTableSql = "CREATE TABLE IF NOT EXISTS machine_table(" + "ts TIMESTAMP NOT NULL," + // + "ts TIMESTAMP NOT NULL," + + "city STRING TAG NOT NULL," + + "ip STRING TAG NOT NULL," + + "cpu DOUBLE NULL," + + "mem DOUBLE NULL," + + "TIMESTAMP KEY(ts)" + // 建表时必须指定时间戳序列 + ") ENGINE=Analytic"; + +Result createResult = client.sqlQuery(new SqlQueryRequest(createTableSql)).get(); +if (!createResult.isOk()) { + throw new IllegalStateException("Fail to create table"); +} ``` 详情见 [table](docs/table.md) +## 构建写入数据 +```java +final Point point = Point.newPointBuilder("machine_table") + .setTimestamp(t0) + .addTag("city", "Singapore") + .addTag("ip", "10.0.0.1") + .addField("cpu", Value.withDouble(0.23)) + .addField("mem", Value.withDouble(0.55)) + .build(); +``` ## 写入 Example ```java - -final long t0 = System.currentTimeMillis(); -final long t1 = t0 + 1000; -final long t2 = t1 + 1000; -final Rows data = Series.newBuilder("machine_metric") - .tag("city", "Singapore") - .tag("ip", "127.0.0.1") - .toRowsBuilder() - // 下面针对 cpu、mem 两列，一次写入了三行数据（3 个时间戳），CeresDB 鼓励这种实践，SDK 可以通过高效的压缩来减少网络传输，并且对 server 端写入非常友好 - .field(t0, "cpu", FieldValue.withDouble(0.23)) // 第 1 行第 1 列 - .field(t0, "mem", FieldValue.withDouble(0.55)) // 第 1 行第 2 列 - .field(t1, "cpu", FieldValue.withDouble(0.25)) // 第 2 行第 1 列 - .field(t1, "mem", FieldValue.withDouble(0.56)) // 第 2 行第 2 列 - .field(t2, "cpu", FieldValue.withDouble(0.21)) // 第 3 行第 1 列 - .field(t2, "mem", FieldValue.withDouble(0.52)) // 第 3 行第 2 列 - .build(); - -final CompletableFuture> wf = client.write(data); +final CompletableFuture> wf = client.write(new WriteRequest(pointList)); // 这里用 `future.get` 只是方便演示，推荐借助 CompletableFuture 强大的 API 实现异步编程 -final Result wr = wf.get(); +final Result writeResult = wf.get(); -Assert.assertTrue(wr.isOk()); -Assert.assertEquals(3, wr.getOk().getSuccess()); +Assert.assertTrue(writeResult.isOk()); +Assert.assertEquals(3, writeResult.getOk().getSuccess()); // `Result` 类参考了 Rust 语言，提供了丰富的 mapXXX、andThen 类 function 方便对结果值进行转换，提高编程效率，欢迎参考 API 文档使用 -Assert.assertEquals(3, wr.mapOr(0, WriteOk::getSuccess).intValue()); -Assert.assertEquals(0, wr.getOk().getFailed()); -Assert.assertEquals(0, wr.mapOr(-1, WriteOk::getFailed).intValue()); - +Assert.assertEquals(3, writeResult.mapOr(0, WriteOk::getSuccess).intValue()); +Assert.assertEquals(0, writeResult.mapOr(-1, WriteOk::getFailed).intValue()); ``` 详情见 [write](docs/write.md) ## 查询 Example ```java -final QueryRequest queryRequest = QueryRequest.newBuilder() - .forMetrics("machine_metric") // 表名可选填，不填的话 SQL Parser 会自动解析 ql 涉及到的表名并完成自动路由 - .ql("select timestamp, cpu, mem from machine_metric") // +final SqlQueryRequest queryRequest = SqlQueryRequest.newBuilder() + .forTables("machine_table") // 这里表名是可选的，如果未提供，SDK将自动解析SQL填充表名并自动路由 + .sql("select * from machine_table where ts = %d", t0) // .build(); -final CompletableFuture> qf = client.query(queryRequest); +final CompletableFuture> qf = client.sqlQuery(queryRequest); // 这里用 `future.get` 只是方便演示，推荐借助 CompletableFuture 强大的 API 实现异步编程 -final Result qr = qf.get(); +final Result queryResult = qf.get(); -Assert.assertTrue(qr.isOk()); +Assert.assertTrue(queryResult.isOk()); -final QueryOk queryOk = qr.getOk(); +final SqlQueryOk queryOk = queryResult.getOk(); +Assert.assertEquals(1, queryOk.getRowCount()); -final List records = queryOk.mapToRecord().collect(Collectors.toList()) -final Stream users = queryOk.map(bytes -> parseUser(bytes)); +// 直接获取结果数组 +final List rows = queryOk.getRowList(); +// 获取结果流 +final Stream rowStream = queryOk.stream(); +rowStream.forEach(row -> System.out.println(row.toString())); ``` 详情见 [read](docs/read.md) ## 流式读写 Example CeresDB 支持流式读写，适用于大规模数据读写。 ```java -final Calendar time = Calendar.getInstance(); -final StreamWriteBuf writeBuf = client.streamWrite("machine_metric"); - for (int i = 0; i < 1000; i++) { - time.add(Calendar.MILLISECOND, 1); - Collection rows = new ArrayList<>(); - final long t0 = System.currentTimeMillis(); - final long t1 = t0 + 1000; - final long t2 = t1 + 1000; - final Rows data = Series.newBuilder("machine_metric").tag("city", "Singapore").tag("ip", "127.0.0.1") - .toRowsBuilder() - .field(t0, "cpu", FieldValue.withDouble(0.23)) - .field(t0, "mem", FieldValue.withDouble(0.55)) - .field(t1, "cpu", FieldValue.withDouble(0.25)) - .field(t1, "mem", FieldValue.withDouble(0.56)) - .field(t2, "cpu", FieldValue.withDouble(0.21)) - .field(t2, "mem", FieldValue.withDouble(0.52)) +final StreamWriteBuf writeBuf = client.streamWrite("machine_table"); +for (int i = 0; i < 1000; i++) { + final Point point = Point.newPointBuilder("machine_table") + .setTimestamp(timestamp) + .addTag("city", "Beijing") + .addTag("ip", "10.0.0.3") + .addField("cpu", Value.withDouble(0.42)) + .addField("mem", Value.withDouble(0.67)) .build(); - rows.add(data); - writeBuf.writeAndFlush(data); - } -final CompletableFuture writeOk = writeBuf.completed(); - Assert.assertEquals(1000, writeOk.join().getSuccess()); + writeBuf.writeAndFlush(Arrays.asList(point)); + timestamp = timestamp+1; +} -final QueryRequest req = QueryRequest.newBuilder().ql("select * from %s", "machine_metric").build(); -final Iterator it = client.blockingStreamQuery(req, 3, TimeUnit.SECONDS); +final CompletableFuture writeOk = writeBuf.completed(); ``` 详情见 [streaming](docs/streaming.md) diff --git a/ceresdb-all/pom.xml b/ceresdb-all/pom.xml index 55019da..1f9427a 100644 --- a/ceresdb-all/pom.xml +++ b/ceresdb-all/pom.xml @@ -20,11 +20,6 @@ ceresdb-grpc ${project.version} - - ${project.groupId} - ceresdb-http - ${project.version} - ${project.groupId} ceresdb-protocol diff --git a/ceresdb-common/src/main/java/com/google/protobuf/ByteStringHelper.java b/ceresdb-common/src/main/java/com/google/protobuf/ByteStringHelper.java index db3aa05..e81c919 100644 --- a/ceresdb-common/src/main/java/com/google/protobuf/ByteStringHelper.java +++ b/ceresdb-common/src/main/java/com/google/protobuf/ByteStringHelper.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package com.google.protobuf; @@ -23,7 +10,6 @@ /** * A {@code ByteString} helper. * - * @author jiachun.fjc */ public class ByteStringHelper { diff --git a/ceresdb-common/src/main/java/com/google/protobuf/BytesStealer.java b/ceresdb-common/src/main/java/com/google/protobuf/BytesStealer.java index 01deb74..b1630f3 100644 --- a/ceresdb-common/src/main/java/com/google/protobuf/BytesStealer.java +++ b/ceresdb-common/src/main/java/com/google/protobuf/BytesStealer.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package com.google.protobuf; @@ -22,7 +9,6 @@ /** * Try to gets the bytes form ByteString with no copy. * - * @author jiachun.fjc */ public class BytesStealer extends ByteOutput { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/Copiable.java b/ceresdb-common/src/main/java/io/ceresdb/common/Copiable.java index 0f5ad18..866041d 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/Copiable.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/Copiable.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common; diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/Display.java b/ceresdb-common/src/main/java/io/ceresdb/common/Display.java index 1ee2c33..efaa44b 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/Display.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/Display.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common; @@ -23,7 +10,6 @@ * their own state and output state information via the {@code display} * method. * - * @author jiachun.fjc */ public interface Display { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/Endpoint.java b/ceresdb-common/src/main/java/io/ceresdb/common/Endpoint.java index 1f8b8a8..8fc5c13 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/Endpoint.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/Endpoint.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common; @@ -23,7 +10,6 @@ /** * A IP address with port. * - * @author jiachun.fjc */ public class Endpoint implements Serializable { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/InFlightLimiter.java b/ceresdb-common/src/main/java/io/ceresdb/common/InFlightLimiter.java index a13935e..6a1f32c 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/InFlightLimiter.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/InFlightLimiter.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common; @@ -28,7 +15,6 @@ /** * In-flight limiter. * - * @author jiachun.fjc */ public class InFlightLimiter implements Limiter { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/Lifecycle.java b/ceresdb-common/src/main/java/io/ceresdb/common/Lifecycle.java index 9113bae..1e5508a 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/Lifecycle.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/Lifecycle.java @@ -1,25 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common; /** * Service life cycle mark interface. * - * @author jiachun.fjc */ public interface Lifecycle { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/Limiter.java b/ceresdb-common/src/main/java/io/ceresdb/common/Limiter.java index 3a6843c..8d7e8a9 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/Limiter.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/Limiter.java @@ -1,27 +1,10 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common; import java.util.concurrent.TimeUnit; -/** - * - * @author jiachun.fjc - */ public interface Limiter { /** diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/OptKeys.java b/ceresdb-common/src/main/java/io/ceresdb/common/OptKeys.java index 9a0fd51..236ca26 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/OptKeys.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/OptKeys.java @@ -1,25 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common; /** * System properties option keys * - * @author jiachun.fjc */ public final class OptKeys { @@ -29,11 +15,8 @@ public final class OptKeys { public static final String USE_OS_SIGNAL = "CeresDB.client.use_os_signal"; public static final String REPORT_PERIOD = "CeresDB.reporter.period_minutes"; public static final String SIG_OUT_DIR = "CeresDB.signal.out_dir"; - public static final String HTTP_READ_TIMEOUT_MS = "CeresDB.http.read_timeout_ms"; - public static final String HTTP_WRITE_TIMEOUT_MS = "CeresDB.http.write_timeout_ms"; public static final String GRPC_CONN_RESET_THRESHOLD = "CeresDB.grpc.conn.failures.reset_threshold"; public static final String AVAILABLE_CPUS = "CeresDB.available_cpus"; - public static final String NAME_VALIDATE = "CeresDB.avro.name_validate"; public static final String WRITE_LIMIT_PERCENT = "CeresDB.rpc.write.limit_percent"; private OptKeys() { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/SPI.java b/ceresdb-common/src/main/java/io/ceresdb/common/SPI.java index 9f42e0e..f7036d3 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/SPI.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/SPI.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common; @@ -25,7 +12,6 @@ /** * Service provide interface annotation. * - * @author jiachun.fjc */ @Documented @Retention(RetentionPolicy.RUNTIME) diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/Streamable.java b/ceresdb-common/src/main/java/io/ceresdb/common/Streamable.java index 5d436ff..de5d326 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/Streamable.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/Streamable.java @@ -1,27 +1,10 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common; import java.util.stream.Stream; -/** - * - * @author jiachun.fjc - */ public interface Streamable { /** diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/Tenant.java b/ceresdb-common/src/main/java/io/ceresdb/common/Tenant.java deleted file mode 100644 index 514248e..0000000 --- a/ceresdb-common/src/main/java/io/ceresdb/common/Tenant.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.common; - -/** - * A tenant for CeresDB. - * - * @author jiachun.fjc - */ -public final class Tenant implements Copiable { - - private final String tenant; - // Default subtenant, which is used if you do not re-specify a - // subtenant each time you make a call. - private final String childTenant; - // Don't tell the secret to anyone, heaven knows and earth knows, - // you know and I know. ＼（＾▽＾）／ - private final String token; - - private Tenant(String tenant, String childTenant, String token) { - this.tenant = tenant; - this.childTenant = childTenant; - this.token = token; - } - - public static Tenant of(final String tenant, final String childTenant, final String token) { - return new Tenant(tenant, childTenant, token); - } - - public String getTenant() { - return tenant; - } - - public String getChildTenant() { - return childTenant; - } - - public String getToken() { - return token; - } - - @Override - public Tenant copy() { - return of(this.tenant, this.childTenant, this.token); - } - - @Override - public String toString() { - return "Tenant{" + // - "tenant='" + tenant + '\'' + // - ", childTenant='" + childTenant + '\'' + // - '}'; - } -} diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/VisibleForTest.java b/ceresdb-common/src/main/java/io/ceresdb/common/VisibleForTest.java index 38f7b03..7714954 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/VisibleForTest.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/VisibleForTest.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common; diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/MetricParser.java b/ceresdb-common/src/main/java/io/ceresdb/common/parser/SqlParser.java similarity index 51% rename from ceresdb-protocol/src/main/java/io/ceresdb/MetricParser.java rename to ceresdb-common/src/main/java/io/ceresdb/common/parser/SqlParser.java index 4721128..5f82eda 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/MetricParser.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/parser/SqlParser.java @@ -1,32 +1,18 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ -package io.ceresdb; +package io.ceresdb.common.parser; import java.util.Collections; import java.util.List; /** - * A metric QL parser that can extract metric names from the given QL. + * A SQL parser that can extract table names from the given SQL. * - * @author jiachun.fjc */ -public interface MetricParser { +public interface SqlParser { - MetricParser DEFAULT = new NoopMetricParser(); + SqlParser DEFAULT = new NoopSqlParser(); StatementType statementType(); @@ -35,7 +21,7 @@ public interface MetricParser { * * @return metric names */ - List metricNames(); + List tableNames(); /** * Extract the column names and types from the given create table QL. @@ -69,7 +55,7 @@ default String valueType() { } } - class NoopMetricParser implements MetricParser { + class NoopSqlParser implements SqlParser { @Override public StatementType statementType() { @@ -77,7 +63,7 @@ public StatementType statementType() { } @Override - public List metricNames() { + public List tableNames() { return Collections.emptyList(); } diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/parser/SqlParserFactory.java b/ceresdb-common/src/main/java/io/ceresdb/common/parser/SqlParserFactory.java new file mode 100644 index 0000000..f6c7501 --- /dev/null +++ b/ceresdb-common/src/main/java/io/ceresdb/common/parser/SqlParserFactory.java @@ -0,0 +1,19 @@ +/* + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + */ +package io.ceresdb.common.parser; + +public interface SqlParserFactory { + + SqlParserFactory DEFAULT = new NoopFactory(); + + SqlParser getParser(final String sql); + + class NoopFactory implements SqlParserFactory { + + @Override + public SqlParser getParser(final String sql) { + return SqlParser.DEFAULT; + } + } +} diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/parser/SqlParserFactoryProvider.java b/ceresdb-common/src/main/java/io/ceresdb/common/parser/SqlParserFactoryProvider.java new file mode 100644 index 0000000..a39ec30 --- /dev/null +++ b/ceresdb-common/src/main/java/io/ceresdb/common/parser/SqlParserFactoryProvider.java @@ -0,0 +1,17 @@ +/* + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + */ +package io.ceresdb.common.parser; + +import io.ceresdb.common.util.ServiceLoader; + +public class SqlParserFactoryProvider { + + private static final SqlParserFactory METRIC_PARSER_FACTORY = ServiceLoader // + .load(SqlParserFactory.class) // + .firstOrDefault(() -> SqlParserFactory.DEFAULT); + + public static SqlParserFactory getSqlParserFactory() { + return METRIC_PARSER_FACTORY; + } +} diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/signal/FileOutputHelper.java b/ceresdb-common/src/main/java/io/ceresdb/common/signal/FileOutputHelper.java index 58c5058..01b8f2d 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/signal/FileOutputHelper.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/signal/FileOutputHelper.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.signal; @@ -26,10 +13,6 @@ import io.ceresdb.common.util.Files; import io.ceresdb.common.util.SystemPropertyUtil; -/** - * - * @author jiachun.fjc - */ public final class FileOutputHelper { private static final String OUT_DIR = SystemPropertyUtil.get(OptKeys.SIG_OUT_DIR, ""); diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/signal/FileSignal.java b/ceresdb-common/src/main/java/io/ceresdb/common/signal/FileSignal.java index cdf416a..a56f006 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/signal/FileSignal.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/signal/FileSignal.java @@ -1,26 +1,10 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.signal; import java.util.Optional; -/** - * @author jiachun.fjc - */ public enum FileSignal { ClearCache("clear_cache.sig", "How to clear route cache"), // RwLogging("rw_logging.sig", "How to open or close read/write log(The second execution means close)"), // diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/signal/FileSignals.java b/ceresdb-common/src/main/java/io/ceresdb/common/signal/FileSignals.java index e356f2b..175d530 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/signal/FileSignals.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/signal/FileSignals.java @@ -1,27 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.signal; import java.io.File; import java.nio.file.Paths; -/** - * @author jiachun.fjc - */ public class FileSignals { private static final String[] EMPTY_ARRAY = new String[0]; diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/signal/Signal.java b/ceresdb-common/src/main/java/io/ceresdb/common/signal/Signal.java index fbd7a51..5abfb27 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/signal/Signal.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/signal/Signal.java @@ -1,25 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.signal; /** * Signal types. * - * @author jiachun.fjc */ public enum Signal { SIG_USR2("USR2"); diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/signal/SignalHandler.java b/ceresdb-common/src/main/java/io/ceresdb/common/signal/SignalHandler.java index 19f8383..9faae9b 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/signal/SignalHandler.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/signal/SignalHandler.java @@ -1,25 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.signal; /** * This is the signal handler interface. * - * @author jiachun.fjc */ public interface SignalHandler { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/signal/SignalHandlersLoader.java b/ceresdb-common/src/main/java/io/ceresdb/common/signal/SignalHandlersLoader.java index 94dd276..68e48f0 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/signal/SignalHandlersLoader.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/signal/SignalHandlersLoader.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.signal; @@ -33,7 +20,6 @@ * * Do not support windows. * - * @author jiachun.fjc */ public class SignalHandlersLoader { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/signal/SignalHelper.java b/ceresdb-common/src/main/java/io/ceresdb/common/signal/SignalHelper.java index cd89a2a..4d7d0a8 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/signal/SignalHelper.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/signal/SignalHelper.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.signal; @@ -26,7 +13,6 @@ /** * A signal helper, provides ANSI/ISO C signal support. * - * @author jiachun.fjc */ public final class SignalHelper { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/AuthUtil.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/AuthUtil.java deleted file mode 100644 index d56341b..0000000 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/AuthUtil.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.common.util; - -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import io.ceresdb.common.Tenant; - -/** - * A tool class for generating auth headers. - * - * @author jiachun.fjc - */ -public class AuthUtil { - - public static final String HEAD_TIMESTAMP = "x-ceresdb-timestamp"; - public static final String HEAD_ACCESS_TENANT = "x-ceresdb-access-tenant"; - public static final String HEAD_ACCESS_TOKEN = "x-ceresdb-access-token"; - public static final String HEAD_ACCESS_CHILD_TENANT = "x-ceresdb-access-child-tenant"; - - public static Map authHeaders(final Tenant tenant) { - if (tenant == null) { - return Collections.emptyMap(); - } - - final String timestamp = String.valueOf(Clock.defaultClock().getTick()); - final String token = tenant.getToken(); - final String tenantName = tenant.getTenant(); - final String childTenantName = tenant.getChildTenant(); - - final Map headers = new HashMap<>(); - headers.put(HEAD_TIMESTAMP, timestamp); - if (tenantName != null) { - headers.put(HEAD_ACCESS_TENANT, tenantName); - } - if (token != null) { - headers.put(HEAD_ACCESS_TOKEN, SHA256(token + timestamp)); - } - if (childTenantName != null) { - headers.put(HEAD_ACCESS_CHILD_TENANT, childTenantName); - } - return headers; - } - - public static void replaceChildTenant(final Map headers, final String childTenant) { - if (childTenant != null) { - headers.put(HEAD_ACCESS_CHILD_TENANT, childTenant); - } - } - - private static String SHA256(final String str) { - try { - final MessageDigest md = MessageDigest.getInstance("SHA-256"); - final byte[] bytes = md.digest(str.getBytes(StandardCharsets.UTF_8)); - return toHex(bytes); - } catch (final Exception e) { - throw new RuntimeException(e); - } - } - - private static String toHex(final byte[] bytes) { - final char[] HEX_DIGITS = "0123456789abcdef".toCharArray(); - final StringBuilder buf = new StringBuilder(bytes.length * 2); - for (final byte b : bytes) { - buf.append(HEX_DIGITS[(b >> 4) & 0x0f]); - buf.append(HEX_DIGITS[b & 0x0f]); - } - return buf.toString(); - } -} diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/Clock.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/Clock.java index 73dbc25..872dd49 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/Clock.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/Clock.java @@ -1,25 +1,8 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; -/** - * - * @author jiachun.fjc - */ public abstract class Clock { /** diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/Cpus.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/Cpus.java index 4888bac..a7f99d8 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/Cpus.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/Cpus.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -21,7 +8,6 @@ /** * Utility for cpu. * - * @author jiachun.fjc */ public class Cpus { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/DirectExecutor.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/DirectExecutor.java index fb8b37d..e33f712 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/DirectExecutor.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/DirectExecutor.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -23,7 +10,6 @@ /** * An executor that run task directly. * - * @author jiachun.fjc */ public class DirectExecutor implements Executor { private final String name; diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/ExecutorServiceHelper.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/ExecutorServiceHelper.java index 73ca24a..c9eada6 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/ExecutorServiceHelper.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/ExecutorServiceHelper.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -25,7 +12,6 @@ /** * Executor service shutdown helper. * - * @author jiachun.fjc */ public final class ExecutorServiceHelper { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/Files.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/Files.java index dfd3fd8..71e569d 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/Files.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/Files.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -22,10 +9,6 @@ import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -/** - * - * @author jiachun.fjc - */ public class Files { /** diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/LogScheduledThreadPoolExecutor.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/LogScheduledThreadPoolExecutor.java index 623ff4f..31f4378 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/LogScheduledThreadPoolExecutor.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/LogScheduledThreadPoolExecutor.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -31,7 +18,6 @@ * schedule tasks to run after a given delay with a logger witch can print * error message for failed execution. * - * @author jiachun.fjc */ public class LogScheduledThreadPoolExecutor extends ScheduledThreadPoolExecutor { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/LogThreadPoolExecutor.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/LogThreadPoolExecutor.java index bffe9fb..5ca7968 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/LogThreadPoolExecutor.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/LogThreadPoolExecutor.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -32,7 +19,6 @@ * A {@link java.util.concurrent.ExecutorService} that witch can print * error message for failed execution. * - * @author jiachun.fjc */ public class LogThreadPoolExecutor extends ThreadPoolExecutor { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricExecutor.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricExecutor.java index 5c6c4c8..5112b6e 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricExecutor.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricExecutor.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -24,7 +11,6 @@ * A {@link java.util.concurrent.Executor} that with a timer metric * which aggregates timing durations and provides duration statistics. * - * @author jiachun.fjc */ public class MetricExecutor implements Executor { private final Executor pool; diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricReporter.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricReporter.java index b0233fd..b261e21 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricReporter.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricReporter.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricScheduledThreadPoolExecutor.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricScheduledThreadPoolExecutor.java index ab66acd..f4b38c4 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricScheduledThreadPoolExecutor.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricScheduledThreadPoolExecutor.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -25,7 +12,6 @@ * schedule tasks to run after a given delay with a timer metric * which aggregates timing durations and provides duration statistics. * - * @author jiachun.fjc */ public class MetricScheduledThreadPoolExecutor extends LogScheduledThreadPoolExecutor { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricThreadPoolExecutor.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricThreadPoolExecutor.java index ac83f0a..558caab 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricThreadPoolExecutor.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricThreadPoolExecutor.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -25,7 +12,6 @@ * A {@link java.util.concurrent.ExecutorService} that with a timer metric * which aggregates timing durations and provides duration statistics. * - * @author jiachun.fjc */ public class MetricThreadPoolExecutor extends LogThreadPoolExecutor { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricsUtil.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricsUtil.java index 2eaaee6..fad76fb 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricsUtil.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/MetricsUtil.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -34,7 +21,6 @@ * In CeresDB client, metrics are required. As for whether to output (log) metrics * results, you decide. * - * @author jiachun.fjc */ public final class MetricsUtil { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/NamedThreadFactory.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/NamedThreadFactory.java index ab5ad2a..83ea4e3 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/NamedThreadFactory.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/NamedThreadFactory.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -25,7 +12,6 @@ /** * Named thread factory. * - * @author jiachun.fjc */ public class NamedThreadFactory implements ThreadFactory { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/ObjectPool.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/ObjectPool.java index 99ef2c0..a1c31cb 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/ObjectPool.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/ObjectPool.java @@ -1,25 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; /** * An object pool. * - * @author jiachun.fjc */ public interface ObjectPool { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/Platform.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/Platform.java index bcafc2a..9c65d3c 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/Platform.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/Platform.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -23,10 +10,6 @@ import io.ceresdb.common.OptKeys; -/** - * - * @author jiachun.fjc - */ public class Platform { private static final Logger LOG = LoggerFactory.getLogger(Platform.class); diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/RcObjectPool.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/RcObjectPool.java index d7fa638..d8ba9d6 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/RcObjectPool.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/RcObjectPool.java @@ -1,25 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; /** * An shared object pool with ref count. * - * @author jiachun.fjc */ public class RcObjectPool implements ObjectPool { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/RcResourceHolder.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/RcResourceHolder.java index bc33a9a..d8f10f0 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/RcResourceHolder.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/RcResourceHolder.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -22,10 +9,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * - * @author jiachun.fjc - */ public class RcResourceHolder { private static final Logger LOG = LoggerFactory.getLogger(RcResourceHolder.class); diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/RefCell.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/RefCell.java index 546c540..8e11724 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/RefCell.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/RefCell.java @@ -1,24 +1,8 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; -/** - * @author jiachun.fjc - */ public class RefCell { private T value; diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/Requires.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/Requires.java index e132ef6..a6125e1 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/Requires.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/Requires.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -20,7 +7,6 @@ * Simple static methods to be called at the start of your own methods to verify * correct arguments and state. * - * @author jiachun.fjc */ @SuppressWarnings("PMD") public final class Requires { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/SerializingExecutor.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/SerializingExecutor.java index 0568751..612be8c 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/SerializingExecutor.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/SerializingExecutor.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/ServiceLoader.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/ServiceLoader.java index d8d6353..f8e2bd6 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/ServiceLoader.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/ServiceLoader.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -40,7 +27,6 @@ /** * A simple service-provider loading facility (SPI). * - * @author jiachun.fjc */ @SuppressWarnings("PMD") public final class ServiceLoader implements Iterable { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/SharedScheduledPool.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/SharedScheduledPool.java index ef63e76..194f62a 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/SharedScheduledPool.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/SharedScheduledPool.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -21,7 +8,6 @@ /** * Like rust: pub type SharedScheduledPool = RcObjectPool * - * @author jiachun.fjc */ public class SharedScheduledPool extends RcObjectPool { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/SharedThreadPool.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/SharedThreadPool.java index 5406c4f..1589ace 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/SharedThreadPool.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/SharedThreadPool.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -21,7 +8,6 @@ /** * Like rust: pub type SharedThreadPool = RcObjectPool * - * @author jiachun.fjc */ public class SharedThreadPool extends RcObjectPool { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/Spines.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/Spines.java index 627c257..67d4c98 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/Spines.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/Spines.java @@ -1,35 +1,18 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; import java.util.ArrayList; -import java.util.Collection; +import java.util.List; -/** - * - * @author jiachun.fjc - */ public class Spines { - public static Collection newBuf() { + public static List newBuf() { return new ArrayList<>(); } - public static Collection newBuf(final int initialCapacity) { + public static List newBuf(final int initialCapacity) { return new ArrayList<>(initialCapacity); } } diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/StringBuilderHelper.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/StringBuilderHelper.java index 67d80a8..f5daadd 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/StringBuilderHelper.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/StringBuilderHelper.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -21,7 +8,6 @@ * * Be careful that do not to nest in the same thread. * - * @author jiachun.fjc */ public class StringBuilderHelper { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/Strings.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/Strings.java index 44ba843..10daca2 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/Strings.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/Strings.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -23,7 +10,6 @@ * Static utility methods pertaining to {@code String} or {@code CharSequence} * instances. * - * @author jiachun.fjc */ public final class Strings { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/SystemPropertyUtil.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/SystemPropertyUtil.java index 692fbba..dd97f0c 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/SystemPropertyUtil.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/SystemPropertyUtil.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/ThreadPoolMetricRegistry.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/ThreadPoolMetricRegistry.java index 0a93966..baa9447 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/ThreadPoolMetricRegistry.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/ThreadPoolMetricRegistry.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -21,7 +8,6 @@ /** * A global timer metric registry for thread pool, use threadLocal to pass timer context. * - * @author jiachun.fjc */ public class ThreadPoolMetricRegistry { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/ThreadPoolUtil.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/ThreadPoolUtil.java index a01f97d..a366b7b 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/ThreadPoolUtil.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/ThreadPoolUtil.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -27,7 +14,6 @@ * Factory and utility methods for creating {@link ThreadPoolExecutor} and * {@link ScheduledThreadPoolExecutor}. * - * @author jiachun.fjc */ public final class ThreadPoolUtil { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/TopKSelector.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/TopKSelector.java index 87b40b2..a27af8e 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/TopKSelector.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/TopKSelector.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -23,7 +10,6 @@ /** * TopK selector based on a priority heap. * - * @author jiachun.fjc */ public class TopKSelector { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/UnsignedUtil.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/UnsignedUtil.java index 067eec0..e4b4ea0 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/UnsignedUtil.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/UnsignedUtil.java @@ -1,27 +1,10 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; import java.math.BigInteger; -/** - * - * @author jiachun.fjc - */ public final class UnsignedUtil { /** diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/IntegerFieldUpdater.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/IntegerFieldUpdater.java index 78cbbb1..312de92 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/IntegerFieldUpdater.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/IntegerFieldUpdater.java @@ -1,24 +1,8 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util.internal; -/** - * @author jiachun.fjc - */ public interface IntegerFieldUpdater { void set(final U obj, final int newValue); diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/LongFieldUpdater.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/LongFieldUpdater.java index c47be3e..d498057 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/LongFieldUpdater.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/LongFieldUpdater.java @@ -1,24 +1,8 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util.internal; -/** - * @author jiachun.fjc - */ public interface LongFieldUpdater { void set(final U obj, final long newValue); diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ReferenceFieldUpdater.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ReferenceFieldUpdater.java index c646c87..4def0b1 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ReferenceFieldUpdater.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ReferenceFieldUpdater.java @@ -1,24 +1,8 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util.internal; -/** - * @author jiachun.fjc - */ public interface ReferenceFieldUpdater { void set(final U obj, final W newValue); diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ReflectionIntegerFieldUpdater.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ReflectionIntegerFieldUpdater.java index 1d3633f..69b562a 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ReflectionIntegerFieldUpdater.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ReflectionIntegerFieldUpdater.java @@ -1,27 +1,10 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util.internal; import java.lang.reflect.Field; -/** - * - * @author jiachun.fjc - */ final class ReflectionIntegerFieldUpdater implements IntegerFieldUpdater { private final Field field; diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ReflectionLongFieldUpdater.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ReflectionLongFieldUpdater.java index c6e3698..638dd30 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ReflectionLongFieldUpdater.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ReflectionLongFieldUpdater.java @@ -1,27 +1,10 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util.internal; import java.lang.reflect.Field; -/** - * - * @author jiachun.fjc - */ final class ReflectionLongFieldUpdater implements LongFieldUpdater { private final Field field; diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ReflectionReferenceFieldUpdater.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ReflectionReferenceFieldUpdater.java index c45cadd..e37c65f 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ReflectionReferenceFieldUpdater.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ReflectionReferenceFieldUpdater.java @@ -1,27 +1,10 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util.internal; import java.lang.reflect.Field; -/** - * - * @author jiachun.fjc - */ @SuppressWarnings("unchecked") final class ReflectionReferenceFieldUpdater implements ReferenceFieldUpdater { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ThrowUtil.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ThrowUtil.java index 14da6c6..c3fe329 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ThrowUtil.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/ThrowUtil.java @@ -1,25 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util.internal; /** * Throwing tool. * - * @author jiachun.fjc */ public final class ThrowUtil { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/UnsafeIntegerFieldUpdater.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/UnsafeIntegerFieldUpdater.java index 463b2ab..9514310 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/UnsafeIntegerFieldUpdater.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/UnsafeIntegerFieldUpdater.java @@ -1,27 +1,10 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util.internal; import java.lang.reflect.Field; -/** - * - * @author jiachun.fjc - */ @SuppressWarnings("PMD") final class UnsafeIntegerFieldUpdater implements IntegerFieldUpdater { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/UnsafeLongFieldUpdater.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/UnsafeLongFieldUpdater.java index 96a70aa..c2e0a64 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/UnsafeLongFieldUpdater.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/UnsafeLongFieldUpdater.java @@ -1,27 +1,10 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util.internal; import java.lang.reflect.Field; -/** - * - * @author jiachun.fjc - */ @SuppressWarnings("PMD") final class UnsafeLongFieldUpdater implements LongFieldUpdater { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/UnsafeReferenceFieldUpdater.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/UnsafeReferenceFieldUpdater.java index 528c28e..6b0ffae 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/UnsafeReferenceFieldUpdater.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/UnsafeReferenceFieldUpdater.java @@ -1,27 +1,10 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util.internal; import java.lang.reflect.Field; -/** - * - * @author jiachun.fjc - */ @SuppressWarnings({ "unchecked", "PMD" }) final class UnsafeReferenceFieldUpdater implements ReferenceFieldUpdater { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/UnsafeUtil.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/UnsafeUtil.java index 0646b17..5d0e13c 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/UnsafeUtil.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/UnsafeUtil.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util.internal; @@ -28,7 +15,6 @@ /** * For the {@link sun.misc.Unsafe} access. * - * @author jiachun.fjc */ @SuppressWarnings({ "ConstantConditions", "PMD" }) public final class UnsafeUtil { diff --git a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/Updaters.java b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/Updaters.java index bcd4668..d7fe18e 100644 --- a/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/Updaters.java +++ b/ceresdb-common/src/main/java/io/ceresdb/common/util/internal/Updaters.java @@ -1,25 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util.internal; /** * Sometime instead of reflection, better performance. * - * @author jiachun.fjc */ public class Updaters { diff --git a/ceresdb-common/src/test/java/io/ceresdb/common/EndpointTest.java b/ceresdb-common/src/test/java/io/ceresdb/common/EndpointTest.java index 3ce13fc..3c0d539 100644 --- a/ceresdb-common/src/test/java/io/ceresdb/common/EndpointTest.java +++ b/ceresdb-common/src/test/java/io/ceresdb/common/EndpointTest.java @@ -1,27 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common; import org.junit.Assert; import org.junit.Test; -/** - * @author jiachun.fjc - */ public class EndpointTest { @Test diff --git a/ceresdb-common/src/test/java/io/ceresdb/common/util/DirectExecutorTest.java b/ceresdb-common/src/test/java/io/ceresdb/common/util/DirectExecutorTest.java index d9894ef..26600b6 100644 --- a/ceresdb-common/src/test/java/io/ceresdb/common/util/DirectExecutorTest.java +++ b/ceresdb-common/src/test/java/io/ceresdb/common/util/DirectExecutorTest.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -21,9 +8,6 @@ import org.junit.Assert; import org.junit.Test; -/** - * @author jiachun.fjc - */ public class DirectExecutorTest { @Test diff --git a/ceresdb-common/src/test/java/io/ceresdb/common/util/ExecutorServiceHelperTest.java b/ceresdb-common/src/test/java/io/ceresdb/common/util/ExecutorServiceHelperTest.java index 837c796..a75775d 100644 --- a/ceresdb-common/src/test/java/io/ceresdb/common/util/ExecutorServiceHelperTest.java +++ b/ceresdb-common/src/test/java/io/ceresdb/common/util/ExecutorServiceHelperTest.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -22,9 +9,6 @@ import org.junit.Assert; import org.junit.Test; -/** - * @author jiachun.fjc - */ public class ExecutorServiceHelperTest { @Test diff --git a/ceresdb-common/src/test/java/io/ceresdb/common/util/RcObjectPoolTest.java b/ceresdb-common/src/test/java/io/ceresdb/common/util/RcObjectPoolTest.java index 68acf7c..3e0656b 100644 --- a/ceresdb-common/src/test/java/io/ceresdb/common/util/RcObjectPoolTest.java +++ b/ceresdb-common/src/test/java/io/ceresdb/common/util/RcObjectPoolTest.java @@ -1,28 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; import org.junit.Assert; import org.junit.Test; -/** - * - * @author jiachun.fjc - */ public class RcObjectPoolTest { static final ObjectPool.Resource RESOURCE = new ObjectPool.Resource() { diff --git a/ceresdb-common/src/test/java/io/ceresdb/common/util/SpinedBufTest.java b/ceresdb-common/src/test/java/io/ceresdb/common/util/SpinedBufTest.java index b1e70ec..83e479f 100644 --- a/ceresdb-common/src/test/java/io/ceresdb/common/util/SpinedBufTest.java +++ b/ceresdb-common/src/test/java/io/ceresdb/common/util/SpinedBufTest.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -23,10 +10,6 @@ import org.junit.Assert; import org.junit.Test; -/** - * - * @author jiachun.fjc - */ public class SpinedBufTest { @Test diff --git a/ceresdb-common/src/test/java/io/ceresdb/common/util/StringsTest.java b/ceresdb-common/src/test/java/io/ceresdb/common/util/StringsTest.java index a76a4fe..f7984c3 100644 --- a/ceresdb-common/src/test/java/io/ceresdb/common/util/StringsTest.java +++ b/ceresdb-common/src/test/java/io/ceresdb/common/util/StringsTest.java @@ -1,27 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; import org.junit.Assert; import org.junit.Test; -/** - * @author jiachun.fjc - */ public class StringsTest { @Test diff --git a/ceresdb-common/src/test/java/io/ceresdb/common/util/TopKSelectorTest.java b/ceresdb-common/src/test/java/io/ceresdb/common/util/TopKSelectorTest.java index 9d16b54..256dd53 100644 --- a/ceresdb-common/src/test/java/io/ceresdb/common/util/TopKSelectorTest.java +++ b/ceresdb-common/src/test/java/io/ceresdb/common/util/TopKSelectorTest.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -25,9 +12,6 @@ import org.junit.Assert; import org.junit.Test; -/** - * @author jiachun.fjc - */ public class TopKSelectorTest { @Test diff --git a/ceresdb-common/src/test/java/io/ceresdb/common/util/UnsignedUtilTest.java b/ceresdb-common/src/test/java/io/ceresdb/common/util/UnsignedUtilTest.java index 5acd130..fcf13f5 100644 --- a/ceresdb-common/src/test/java/io/ceresdb/common/util/UnsignedUtilTest.java +++ b/ceresdb-common/src/test/java/io/ceresdb/common/util/UnsignedUtilTest.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.common.util; @@ -21,9 +8,6 @@ import org.junit.Assert; import org.junit.Test; -/** - * @author jiachun.fjc - */ public class UnsignedUtilTest { @SuppressWarnings("all") diff --git a/ceresdb-example/src/test/java/io/ceresdb/CeresDBTest.java b/ceresdb-example/src/test/java/io/ceresdb/CeresDBTest.java index 42a0071..39e9e48 100644 --- a/ceresdb-example/src/test/java/io/ceresdb/CeresDBTest.java +++ b/ceresdb-example/src/test/java/io/ceresdb/CeresDBTest.java @@ -1,28 +1,14 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; -import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; -import java.util.Collection; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -38,26 +24,20 @@ import org.slf4j.LoggerFactory; import io.ceresdb.common.util.MetricsUtil; -import io.ceresdb.common.util.UnsignedUtil; import io.ceresdb.common.util.internal.ThrowUtil; import io.ceresdb.models.Err; -import io.ceresdb.models.FieldValue; -import io.ceresdb.models.QueryOk; -import io.ceresdb.models.QueryRequest; -import io.ceresdb.models.Record; +import io.ceresdb.models.Point; +import io.ceresdb.models.Row; +import io.ceresdb.models.SqlQueryOk; +import io.ceresdb.models.SqlQueryRequest; import io.ceresdb.models.Result; -import io.ceresdb.models.Rows; -import io.ceresdb.models.Series; -import io.ceresdb.models.SqlResult; -import io.ceresdb.models.TagValue; +import io.ceresdb.models.Value; import io.ceresdb.models.WriteOk; +import io.ceresdb.models.WriteRequest; import io.ceresdb.options.CeresDBOptions; import io.ceresdb.rpc.RpcOptions; +import io.ceresdb.util.StreamWriteBuf; -/** - * - * @author jiachun.fjc - */ public class CeresDBTest { private static final Logger LOG = LoggerFactory.getLogger(CeresDBTest.class); @@ -68,14 +48,14 @@ public class CeresDBTest { private CeresDBClient client; @Before - public void before() { + public void before() throws Exception { final RpcOptions rpcOpts = RpcOptions.newDefault(); rpcOpts.setBlockOnLimit(false); rpcOpts.setInitialLimit(32); rpcOpts.setLimitKind(RpcOptions.LimitKind.Gradient); rpcOpts.setLogOnLimitChange(true); - this.opts = CeresDBOptions.newBuilder("127.0.0.1", 8831, 5000) // - .tenant("public", "sub_test", "test_token") // + this.opts = CeresDBOptions.newBuilder("127.0.0.1", 8831, RouteMode.DIRECT) // + .database("public") // .rpcOptions(rpcOpts) // .writeMaxRetries(0) // .readMaxRetries(1) // @@ -85,36 +65,46 @@ public void before() { TEST_TABLE_NAME = String.format(TEST_TABLE_NAME, System.currentTimeMillis()); - final Management management = this.client.management(); - - final SqlResult existsResult = management.executeSql("EXISTS TABLE %s", TEST_TABLE_NAME); - LOG.info("EXISTS TABLE before: {}.", existsResult); - - final SqlResult result = management.executeSql(String.format("CREATE TABLE %s(" + // - "ts TIMESTAMP NOT NULL," + // - "c1 STRING TAG NOT NULL," + // - "c2 INT64 TAG NULL," + // - "c3 DOUBLE NULL," + // - "c4 STRING NULL," + // - "c5 INT64 NULL," + // - "c6 FLOAT NULL," + // - "c7 INT32 NULL," + // - "c8 INT16 NULL," + // - "c9 INT8 NULL," + // - "c10 BOOLEAN NULL," + // - "c11 UINT64 NULL," + // - "c12 UINT32 NULL," + // - "c13 UINT16 NULL," + // - "c14 UINT8 NULL," + // - "c15 TIMESTAMP NULL," + // - "c16 VARBINARY NULL," + // - "TIMESTAMP KEY(ts)) ENGINE=Analytic WITH (ttl='7d')", - TEST_TABLE_NAME)); + final Result existsResult = this.client + .sqlQuery(new SqlQueryRequest("EXISTS TABLE %s", TEST_TABLE_NAME)).get(); + if (!existsResult.isOk()) { + LOG.error("EXISTS TABLE before fail: {}", existsResult.getErr()); + return; + } + LOG.info("EXISTS TABLE before: {}.", existsResult.getOk()); + + final Result createResult = this.client.sqlQuery(new SqlQueryRequest("CREATE TABLE %s(" + // + "ts TIMESTAMP NOT NULL," + // + "tString STRING TAG NOT NULL," + + // + "tInt64 INT64 TAG NULL," + // + "fString STRING NULL," + // + "fBool BOOLEAN NULL," + // + "fDouble DOUBLE NULL," + // + "fFloat FLOAT NULL," + // + "fInt64 INT64 NULL," + // + "fInt32 INT32 NULL," + // + "fInt16 INT16 NULL," + // + "fInt8 INT8 NULL," + // + "fUint64 UINT64 NULL," + // + "fUint32 UINT32 NULL," + // + "fUint16 UINT16 NULL," + // + "fUint8 UINT8 NULL," + // + "fTimestamp TIMESTAMP NULL," + + // + //"fVarbinary VARBINARY NULL, + //" + "TIMESTAMP KEY(ts)) ENGINE=Analytic WITH (ttl='7d')", + TEST_TABLE_NAME)).get(); + if (!createResult.isOk()) { + LOG.error("CREATE TABLE before fail, {}", createResult.getErr()); + return; + } LOG.info("Start CeresDB client {}, with options: {}, create table {}: {}.", result(ret), this.opts, - TEST_TABLE_NAME, result); + TEST_TABLE_NAME, createResult.getOk()); - final SqlResult existsResult2 = management.executeSql("EXISTS TABLE %s", TEST_TABLE_NAME); + final Result existsResult2 = this.client + .sqlQuery(new SqlQueryRequest("EXISTS TABLE %s", TEST_TABLE_NAME)).get(); LOG.info("EXISTS TABLE after: {}.", existsResult2); } @@ -123,9 +113,15 @@ private static String result(final boolean result) { } @After - public void after() { - final SqlResult descResult = this.client.management().executeSql("DROP TABLE %s", TEST_TABLE_NAME); - LOG.info("DROP TABLE: {}.", descResult); + public void after() throws Exception { + final Result dropResult = this.client + .sqlQuery(new SqlQueryRequest("DROP TABLE %s", TEST_TABLE_NAME)).get(); + if (!dropResult.isOk()) { + LOG.error("DROP TABLE after fail, {}", dropResult.getErr()); + return; + } + + LOG.info("DROP TABLE: {}.", dropResult); MetricsUtil.reportImmediately(); this.client.shutdownGracefully(); } @@ -142,64 +138,72 @@ public void comprehensiveTest() throws ExecutionException, InterruptedException final String timeString = format.format(time.getTime()); System.out.println("time=" + timeString + " " + time.getTimeInMillis() + " " + time.getTime()); - final int writeCount = ThreadLocalRandom.current().nextInt(10); + final int writeCount = ThreadLocalRandom.current().nextInt(1, 10); final Result writeR = write(time, writeCount); LOG.info("#comprehensiveTest write result={}.", writeR); Assert.assertTrue(writeR.isOk()); + Assert.assertEquals(writeR.getOk().getSuccess(), writeCount); - final QueryRequest req = QueryRequest.newBuilder() // - .ql("select * from %s where ts < to_timestamp_millis('%s')", TEST_TABLE_NAME, timeString) // + final SqlQueryRequest req = SqlQueryRequest.newBuilder() // + .sql("select * from %s where ts < to_timestamp_millis('%s')", TEST_TABLE_NAME, timeString) // .build(); - final Result queryR = this.client.query(req).get(); + final Result queryR = this.client.sqlQuery(req).get(); LOG.info("#comprehensiveTest query result={}.", queryR); Assert.assertTrue(queryR.isOk()); - final QueryOk ok = queryR.getOk(); - - ok.mapToRecord().forEach(rd -> { - LOG.info("Field descriptor: {}", rd.getFieldDescriptors()); - LOG.info("Data: ts={}, c1={}, c2={}, c3={}, c4={}, c5={}, c6={}, c7={}, c8={}, c9={}," + // - "c10={}, c11={}, c12={}, c13={}, c14={}, c15={}, c16={}", rd.getTimestamp("ts"), // - rd.getString("c1"), // - rd.getInt64("c2"), // - rd.getDouble("c3"), // - rd.getString("c4"), // - rd.getInt64("c5"), // - rd.getFloat("c6"), // - rd.getInteger("c7"), // - rd.getInt16("c8"), // - rd.getInt8("c9"), // - rd.getBoolean("c10"), // - rd.getUInt64("c11"), // - rd.getUInt32("c12"), // - rd.getUInt16("c13"), // - rd.getUInt8("c14"), // - rd.getTimestamp("c15"), rd.getBytes("c16")); + final SqlQueryOk ok = queryR.getOk(); + + ok.stream().forEach(row -> { + LOG.info( + "Data: ts={}, tString={}, tInt64={}, fString={}, fBool={}, fDouble={}, fFloat={}, fInt64={}, fInt32={}, fInt16={}," + + // + "fInt8={}, fUint64={}, fUint32={}, fUint16={}, fUint8={}, fTimestamp={}, fVarbinary={}", // + row.getColumnValue("ts").getTimestamp(), // + row.getColumnValue("tString").getString(), // + row.getColumnValue("tInt64").getInt64(), // + row.getColumnValue("fString").getString(), // + row.getColumnValue("fBool").getBoolean(), // + row.getColumnValue("fDouble").getDouble(), // + row.getColumnValue("fFloat").getFloat(), // + row.getColumnValue("fInt64").getInt64(), // + row.getColumnValue("fInt32").getInt32(), // + row.getColumnValue("fInt16").getInt16(), // + row.getColumnValue("fInt8").getInt8(), // + row.getColumnValue("fUint64").getUInt64(), // + row.getColumnValue("fUint32").getUInt32(), // + row.getColumnValue("fUint16").getUInt16(), // + row.getColumnValue("fUint8").getUInt8(), // + row.getColumnValue("fTimestamp").getTimestamp() // + //row.getColumnValue("fVarbinary").getVarbinary()) + ); }); - final Management management = this.client.management(); - - final SqlResult alterResult1 = management.executeSql("ALTER TABLE %s ADD COLUMN (c18 UINT64, c19 STRING TAG)", - TEST_TABLE_NAME); + final Result alterResult1 = this.client + .sqlQuery( + new SqlQueryRequest("ALTER TABLE %s ADD COLUMN (c18 UINT64, c19 STRING TAG)", TEST_TABLE_NAME)) + .get(); LOG.info("ALTER TABLE 1: {}.", alterResult1); - final SqlResult alterResult2 = management.executeSql("ALTER TABLE %s ADD COLUMN c20 STRING TAG", - TEST_TABLE_NAME); + final Result alterResult2 = this.client + .sqlQuery(new SqlQueryRequest("ALTER TABLE %s ADD COLUMN c20 STRING TAG", TEST_TABLE_NAME)).get(); LOG.info("ALTER TABLE 2: {}.", alterResult2); - final SqlResult descResult = management.executeSql("DESCRIBE %s", TEST_TABLE_NAME); + final Result descResult = this.client + .sqlQuery(new SqlQueryRequest("DESCRIBE %s", TEST_TABLE_NAME)).get(); LOG.info("DESCRIBE TABLE: {}.", descResult); - final SqlResult showResult = management.executeSql("SHOW CREATE TABLE %s", TEST_TABLE_NAME); + final Result showResult = this.client + .sqlQuery(new SqlQueryRequest("SHOW CREATE TABLE %s", TEST_TABLE_NAME)).get(); LOG.info("SHOW CREATE TABLE: {}.", showResult); - final SqlResult queryResult = management.executeSql("SELECT * FROM %s", TEST_TABLE_NAME); + final Result queryResult = this.client + .sqlQuery(new SqlQueryRequest("SELECT * FROM %s", TEST_TABLE_NAME)).get(); LOG.info("QUERY TABLE: {}.", queryResult); } @@ -213,37 +217,28 @@ private Result write(final Calendar time, final int writeCount) { } private CompletableFuture> writeAsync(final Calendar time, final int writeCount) { - return this.client.write(makeRows(time, writeCount)); + return this.client.write(new WriteRequest(makePoints(time, writeCount))); } - private Collection makeRows(final Calendar time, final int count) { + private List makePoints(final Calendar time, final int count) { final long timestamp = time.getTimeInMillis(); - final Collection rows = new ArrayList<>(); + + List points = new ArrayList<>(count); for (long ts = (timestamp - count); ts < timestamp; ts++) { - final Rows.Builder builder = Series.newBuilder(TEST_TABLE_NAME).tag("c1", "first_c1") - .tag("c2", TagValue.withInt64(12)).toRowsBuilder() // - .fields(ts, in -> { - in.put("c3", FieldValue.withDouble(0.1)); - in.put("c4", FieldValue.withString("string value")); - in.put("c5", FieldValue.withInt64(64)); - in.put("c6", FieldValue.withFloat(1.0f)); - in.put("c7", FieldValue.withInt(32)); - in.put("c8", FieldValue.withInt16(16)); - in.put("c9", FieldValue.withInt8(8)); - in.put("c10", FieldValue.withBoolean(true)); - in.put("c11", - FieldValue.withUInt64(UnsignedUtil.getUInt64(Long.MAX_VALUE).add(BigInteger.ONE))); - in.put("c12", FieldValue.withUInt32(33)); - in.put("c13", FieldValue.withUInt16(17)); - in.put("c14", FieldValue.withUInt8(9)); - in.put("c15", FieldValue.withTimestamp(time.getTimeInMillis())); - in.put("c16", FieldValue.withVarbinary("test".getBytes(StandardCharsets.UTF_8))); - }); - - rows.add(builder.build()); + Point point = Point.newPointBuilder(TEST_TABLE_NAME).setTimestamp(ts) + .addTag("tString", Value.withString("first_c1")).addTag("tInt64", Value.withInt64(12)) + .addField("fString", Value.withString("string value")).addField("fBool", Value.withBoolean(true)) + .addField("fDouble", Value.withDouble(0.64)).addField("fFloat", Value.withFloat(0.32f)) + .addField("fInt64", Value.withInt64(-64)).addField("fInt32", Value.withInt32(-32)) + .addField("fInt16", Value.withInt16(-16)).addField("fInt8", Value.withInt8(-8)) + .addField("fUint64", Value.withUInt64(64)).addField("fUint32", Value.withUInt32(32)) + .addField("fUint16", Value.withUInt16(16)).addField("fUint8", Value.withUInt8(8)) + .addField("fTimestamp", Value.withTimestamp(time.getTimeInMillis())) + //.addField("fVarbinary", Value.withVarbinary("test".getBytes(StandardCharsets.UTF_8)) + .build(); + points.add(point); } - - return rows; + return points; } @Ignore @@ -281,13 +276,13 @@ public void loopToWriteTest() throws Exception { @Ignore @Test public void streamWriteTest() { - final StreamWriteBuf writeBuf = this.client.streamWrite(TEST_TABLE_NAME); - final CompletableFuture future = writeBuf.write(makeRows(Calendar.getInstance(), 2)) // - .write(makeRows(Calendar.getInstance(), 3)) // + final StreamWriteBuf writeBuf = this.client.streamWrite(TEST_TABLE_NAME); + final CompletableFuture future = writeBuf.write(makePoints(Calendar.getInstance(), 2)) // + .write(makePoints(Calendar.getInstance(), 3)) // .flush() // - .writeAndFlush(makeRows(Calendar.getInstance(), 10)) // - .write(makeRows(Calendar.getInstance(), 10)) // - .writeAndFlush(makeRows(Calendar.getInstance(), 10)) // + .writeAndFlush(makePoints(Calendar.getInstance(), 10)) // + .write(makePoints(Calendar.getInstance(), 10)) // + .writeAndFlush(makePoints(Calendar.getInstance(), 10)) // .completed(); Assert.assertEquals(35, future.join().getSuccess()); @@ -297,20 +292,20 @@ public void streamWriteTest() { @Test public void streamQueryTest() { final Calendar time = Calendar.getInstance(); - final StreamWriteBuf writeBuf = this.client.streamWrite(TEST_TABLE_NAME); + final StreamWriteBuf writeBuf = this.client.streamWrite(TEST_TABLE_NAME); for (int i = 0; i < 1000; i++) { time.add(Calendar.MILLISECOND, 1); - writeBuf.writeAndFlush(makeRows(time, 1)); + writeBuf.writeAndFlush(makePoints(time, 1)); } final CompletableFuture writeOk = writeBuf.completed(); Assert.assertEquals(1000, writeOk.join().getSuccess()); - final QueryRequest req = QueryRequest.newBuilder().ql("select * from %s", TEST_TABLE_NAME).build(); - final Iterator it = this.client.blockingStreamQuery(req, 3, TimeUnit.SECONDS); + final SqlQueryRequest req = SqlQueryRequest.newBuilder().sql("select * from %s", TEST_TABLE_NAME).build(); + final Iterator it = this.client.blockingStreamSqlQuery(req, 3, TimeUnit.SECONDS); int i = 0; while (it.hasNext()) { - LOG.info("The {} row, timestamp={}", ++i, it.next().getTimestamp("ts")); + LOG.info("The {} row, timestamp={}", ++i, it.next().getColumnValue("ts")); } Assert.assertEquals(1000, i); diff --git a/ceresdb-example/src/test/java/io/ceresdb/ReadmeTest.java b/ceresdb-example/src/test/java/io/ceresdb/ReadmeTest.java index 0b2d41c..8ebeefc 100644 --- a/ceresdb-example/src/test/java/io/ceresdb/ReadmeTest.java +++ b/ceresdb-example/src/test/java/io/ceresdb/ReadmeTest.java @@ -1,43 +1,30 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; import io.ceresdb.models.*; import io.ceresdb.options.CeresDBOptions; +import io.ceresdb.util.StreamWriteBuf; + import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; +import java.util.stream.Stream; -/** - * @author kesheng - */ public class ReadmeTest { @Ignore @Test public void readmeTest() throws ExecutionException, InterruptedException { - final CeresDBOptions opts = CeresDBOptions.newBuilder("127.0.0.1", 8831) // CeresDB default grpc port 8831 - .managementAddress("127.0.0.1", 5000) // CeresDB default http port 3307 - .tenant("public", "sub_test", "test_token") // tenant info + final CeresDBOptions opts = CeresDBOptions.newBuilder("127.0.0.1", 8831, RouteMode.DIRECT) // CeresDB default grpc port 8831 + .database("public") // use database public // maximum retry times when write fails // (only some error codes will be retried, such as the routing table failure) .writeMaxRetries(1) @@ -51,51 +38,101 @@ public void readmeTest() throws ExecutionException, InterruptedException { } // Create table manually, creating table schema ahead of data ingestion is not required - String createTableSql = "CREATE TABLE MY_FIRST_TABLE(" + "ts TIMESTAMP NOT NULL," + "c1 STRING TAG NOT NULL," - + "c2 STRING TAG NOT NULL," + "c3 DOUBLE NULL," + "c4 STRING NULL," + "c5 INT64 NULL," - + "c6 FLOAT NULL," + "c7 INT32 NULL," + "c8 INT16 NULL," + "c9 INT8 NULL," - + "c10 BOOLEAN NULL," + "c11 UINT64 NULL," + "c12 UINT32 NULL," + "c13 UINT16 NULL," - + "c14 UINT8 NULL," + "c15 TIMESTAMP NULL," + "c16 VARBINARY NULL," - + "TIMESTAMP KEY(ts)" + ") ENGINE=Analytic"; - SqlResult result = client.management().executeSql(createTableSql); - - final long t0 = System.currentTimeMillis(); - final long t1 = t0 + 1000; - final long t2 = t1 + 1000; - final Rows data = Series.newBuilder("machine_metric").tag("city", "Singapore").tag("ip", "127.0.0.1") - .toRowsBuilder() - // codes below organizes 3 lines data (3 timestamps) for the `cpu` and `mem` column, this will just transport once through network. CeresDB encourage practices like this, because the SDK could use efficient compression algorithm to reduce network traffic and also be friendly to the sever side. - .field(t0, "cpu", FieldValue.withDouble(0.23)) // first row, first column - .field(t0, "mem", FieldValue.withDouble(0.55)) // first row, second column - .field(t1, "cpu", FieldValue.withDouble(0.25)) // second row, first column - .field(t1, "mem", FieldValue.withDouble(0.56)) // second row, second column - .field(t2, "cpu", FieldValue.withDouble(0.21)) // third row, first column - .field(t2, "mem", FieldValue.withDouble(0.52)) // third row, second column - .build(); + String createTableSql = "CREATE TABLE IF NOT EXISTS machine_table(" + "ts TIMESTAMP NOT NULL," + // + "ts TIMESTAMP NOT NULL," + + "city STRING TAG NOT NULL," + + "ip STRING TAG NOT NULL," + + "cpu DOUBLE NULL," + + "mem DOUBLE NULL," + + "TIMESTAMP KEY(ts)" + // timestamp column must be specified + ") ENGINE=Analytic"; - final CompletableFuture> wf = client.write(data); - // here the `future.get` is just for demonstration, a better async programming practice would be using the CompletableFuture API - final Result wr = wf.get(); + Result createResult = client.sqlQuery(new SqlQueryRequest(createTableSql)).get(); + if (!createResult.isOk()) { + throw new IllegalStateException("Fail to create table"); + } + + final long timestamp = System.currentTimeMillis(); + Point point1 = Point.newPointBuilder("machine_table") + .setTimestamp(timestamp) + .addTag("city", "Singapore") + .addTag("ip", "10.0.0.1") + .addField("cpu", Value.withDouble(0.23)) + .addField("mem", Value.withDouble(0.55)) + .build(); + Point point2 = Point.newPointBuilder("machine_table") + .setTimestamp(timestamp+1000) + .addTag("city", "Singapore") + .addTag("ip", "10.0.0.1") + .addField("cpu", Value.withDouble(0.25)) + .addField("mem", Value.withDouble(0.56)) + .build(); + Point point3 = Point.newPointBuilder("machine_table") + .setTimestamp(timestamp+1000) + .addTag("city", "Shanghai") + .addTag("ip", "10.0.0.2") + .addField("cpu", Value.withDouble(0.21)) + .addField("mem", Value.withDouble(0.52)) + .build(); + List pointList = Arrays.asList(point1, point2, point3); - Assert.assertTrue(wr.isOk()); - Assert.assertEquals(3, wr.getOk().getSuccess()); + final CompletableFuture> wf = client.write(new WriteRequest(pointList)); + // here the `future.get` is just for demonstration, a better async programming practice would be using the CompletableFuture API + final Result writeResult = wf.get(); + Assert.assertTrue(writeResult.isOk()); + Assert.assertEquals(3, writeResult.getOk().getSuccess()); // `Result` class referenced the Rust language practice, provides rich functions (such as mapXXX, andThen) transforming the result value to improve programming efficiency. You can refer to the API docs for detail usage. - Assert.assertEquals(3, wr.mapOr(0, WriteOk::getSuccess).intValue()); - Assert.assertEquals(0, wr.getOk().getFailed()); - Assert.assertEquals(0, wr.mapOr(-1, WriteOk::getFailed).intValue()); + Assert.assertEquals(3, writeResult.mapOr(0, WriteOk::getSuccess).intValue()); + Assert.assertEquals(0, writeResult.getOk().getFailed()); + Assert.assertEquals(0, writeResult.mapOr(-1, WriteOk::getFailed).intValue()); - final QueryRequest queryRequest = QueryRequest.newBuilder().forMetrics("machine_metric") // table name is optional. If not provided, SQL parser will parse the `ql` to get the table name and do the routing automaticly - .ql("select timestamp, cpu, mem from machine_metric") // + final SqlQueryRequest queryRequest = SqlQueryRequest.newBuilder() + .forTables("machine_table") // table name is optional. If not provided, SQL parser will parse the `sql` to get the table name and do the routing automaticly + .sql("select * from machine_table where ts = %d", timestamp) // .build(); - final CompletableFuture> qf = client.query(queryRequest); + final CompletableFuture> qf = client.sqlQuery(queryRequest); // here the `future.get` is just for demonstration, a better async programming practice would be using the CompletableFuture API - final Result qr = qf.get(); + final Result queryResult = qf.get(); - Assert.assertTrue(qr.isOk()); + Assert.assertTrue(queryResult.isOk()); - final QueryOk queryOk = qr.getOk(); + final SqlQueryOk queryOk = queryResult.getOk(); + Assert.assertEquals(1, queryOk.getRowCount()); - final List records = queryOk.mapToRecord().collect(Collectors.toList()); - } + // get rows as list + final List rows = queryOk.getRowList(); + Assert.assertEquals(timestamp, rows.get(0).getColumnValue("ts").getTimestamp()); + Assert.assertEquals("Singapore", rows.get(0).getColumnValue("city").getString()); + Assert.assertEquals("10.0.0.1", rows.get(0).getColumnValue("ip").getString()); + Assert.assertEquals(0.23, rows.get(0).getColumnValue("cpu").getDouble(), 0.0000001); + Assert.assertEquals(0.55, rows.get(0).getColumnValue("mem").getDouble(), 0.0000001); + + // get rows as stream + final Stream rowStream = queryOk.stream(); + rowStream.forEach(row -> System.out.println(row.toString())); + // write with Stream + long start = System.currentTimeMillis(); + long t = start; + final StreamWriteBuf writeBuf = client.streamWrite("machine_table"); + for (int i = 0; i < 1000; i++) { + final Point streamData = Point.newPointBuilder("machine_table") + .setTimestamp(t) + .addTag("city", "Beijing") + .addTag("ip", "10.0.0.3") + .addField("cpu", Value.withDouble(0.42)) + .addField("mem", Value.withDouble(0.67)) + .build(); + writeBuf.writeAndFlush(Collections.singletonList(streamData)); + t = t+1; + } + final CompletableFuture writeOk = writeBuf.completed(); + Assert.assertEquals(1000, writeOk.join().getSuccess()); + + final SqlQueryRequest streamQuerySql = SqlQueryRequest.newBuilder() + .sql("select * from %s where city = '%s' and ts >= %d and ts < %d", "machine_table", "Beijing", start, t).build(); + final Result streamQueryResult = client.sqlQuery(streamQuerySql).get(); + Assert.assertTrue(streamQueryResult.isOk()); + Assert.assertEquals(1000, streamQueryResult.getOk().getRowCount()); + } } diff --git a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/GrpcClient.java b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/GrpcClient.java index 8856b95..89f95f1 100644 --- a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/GrpcClient.java +++ b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/GrpcClient.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc; @@ -52,8 +39,6 @@ import io.ceresdb.common.Endpoint; import io.ceresdb.common.OptKeys; -import io.ceresdb.common.Tenant; -import io.ceresdb.common.util.AuthUtil; import io.ceresdb.common.util.Clock; import io.ceresdb.common.util.Cpus; import io.ceresdb.common.util.ExecutorServiceHelper; @@ -70,7 +55,6 @@ import io.ceresdb.rpc.errors.InvokeTimeoutException; import io.ceresdb.rpc.errors.OnlyErrorMessage; import io.ceresdb.rpc.errors.RemotingException; -import io.ceresdb.rpc.interceptors.AuthHeadersInterceptor; import io.ceresdb.rpc.interceptors.ClientRequestLimitInterceptor; import io.ceresdb.rpc.interceptors.ContextToHeadersInterceptor; import io.ceresdb.rpc.interceptors.MetricInterceptor; @@ -85,7 +69,6 @@ /** * Grpc client implementation. * - * @author jiachun.fjc */ public class GrpcClient implements RpcClient { @@ -122,8 +105,6 @@ public void close(final ExecutorService ins) { private final List connectionObservers = new CopyOnWriteArrayList<>(); private final MarshallerRegistry marshallerRegistry; - private String tenant = "none"; - private String defaultChildTenant = "none"; private RpcOptions opts; private ExecutorService asyncPool; private boolean useSharedAsyncPool; @@ -238,13 +219,12 @@ public void invokeAsync(final Endpoint endpoint, // .withDeadlineAfter(timeout, TimeUnit.MILLISECONDS) // .withExecutor(getObserverExecutor(observer)); - final String childTenant = addChildTenantIntoCtx(ctx); final String methodName = method.getFullMethodName(); final String address = endpoint.toString(); final long startCall = Clock.defaultClock().getTick(); final Channel ch = getCheckedChannel(endpoint, (err) -> { - attachErrMsg(err, UNARY_CALL, methodName, childTenant, address, startCall, -1, ctx); + attachErrMsg(err, UNARY_CALL, methodName, address, startCall, -1, ctx); observer.onError(err); }); @@ -266,7 +246,7 @@ public void onNext(final Message value) { @Override public void onError(final Throwable err) { - attachErrMsg(err, UNARY_CALL, methodName, childTenant, target, startCall, onReceived(true), ctx); + attachErrMsg(err, UNARY_CALL, methodName, target, startCall, onReceived(true), ctx); observer.onError(err); } @@ -277,14 +257,13 @@ public void onCompleted() { private long onReceived(final boolean onError) { final long duration = Clock.defaultClock().duration(startCall); - final String mthAndTnt = MetricsUtil.named(methodName, tenant); - MetricsUtil.timer(REQ_RT, mthAndTnt).update(duration, TimeUnit.MILLISECONDS); - MetricsUtil.timer(REQ_RT, mthAndTnt, address).update(duration, TimeUnit.MILLISECONDS); + MetricsUtil.timer(REQ_RT, method).update(duration, TimeUnit.MILLISECONDS); + MetricsUtil.timer(REQ_RT, method, address).update(duration, TimeUnit.MILLISECONDS); if (onError) { - MetricsUtil.meter(REQ_FAILED, mthAndTnt).mark(); - MetricsUtil.meter(REQ_FAILED, mthAndTnt, address).mark(); + MetricsUtil.meter(REQ_FAILED, method).mark(); + MetricsUtil.meter(REQ_FAILED, method, address).mark(); } return duration; @@ -303,13 +282,12 @@ public void invokeServerStreaming(final Endpoint endpoint, // MethodDescriptor.MethodType.SERVER_STREAMING); final CallOptions callOpts = CallOptions.DEFAULT.withExecutor(getObserverExecutor(observer)); - final String childTenant = addChildTenantIntoCtx(ctx); final String methodName = method.getFullMethodName(); final String address = endpoint.toString(); final long startCall = Clock.defaultClock().getTick(); final Channel ch = getCheckedChannel(endpoint, (err) -> { - attachErrMsg(err, SERVER_STREAMING_CALL, methodName, childTenant, address, startCall, -1, ctx); + attachErrMsg(err, SERVER_STREAMING_CALL, methodName, address, startCall, -1, ctx); observer.onError(err); }); @@ -330,7 +308,7 @@ public void onNext(final Message value) { @Override public void onError(final Throwable err) { - attachErrMsg(err, SERVER_STREAMING_CALL, methodName, childTenant, target, startCall, -1, ctx); + attachErrMsg(err, SERVER_STREAMING_CALL, methodName, target, startCall, -1, ctx); observer.onError(err); } @@ -352,14 +330,13 @@ public Observer invokeClientStreaming(final Endpoint endpoint, MethodDescriptor.MethodType.CLIENT_STREAMING); final CallOptions callOpts = CallOptions.DEFAULT.withExecutor(getObserverExecutor(respObserver)); - final String childTenant = addChildTenantIntoCtx(ctx); final String methodName = method.getFullMethodName(); final String address = endpoint.toString(); final long startCall = Clock.defaultClock().getTick(); final RefCell refErr = new RefCell<>(); final Channel ch = getCheckedChannel(endpoint, (err) -> { - attachErrMsg(err, CLIENT_STREAMING_CALL, methodName, childTenant, address, startCall, -1, ctx); + attachErrMsg(err, CLIENT_STREAMING_CALL, methodName, address, startCall, -1, ctx); refErr.set(err); }); @@ -381,7 +358,7 @@ public void onNext(final Message value) { @Override public void onError(final Throwable err) { - attachErrMsg(err, CLIENT_STREAMING_CALL, methodName, childTenant, target, startCall, -1, ctx); + attachErrMsg(err, CLIENT_STREAMING_CALL, methodName, target, startCall, -1, ctx); respObserver.onError(err); } @@ -419,22 +396,14 @@ private void initInterceptors() { // the last one addInterceptor(new MetricInterceptor()); - // the third + // the second final RpcOptions.LimitKind kind = this.opts.getLimitKind(); if (kind != null && kind != RpcOptions.LimitKind.None) { addInterceptor(createRequestLimitInterceptor(kind)); } - // the second - addInterceptor(new ContextToHeadersInterceptor()); - // the first - final Tenant tenant = this.opts.getTenant(); - if (tenant != null) { - this.tenant = tenant.getTenant(); - this.defaultChildTenant = tenant.getChildTenant(); - addInterceptor(new AuthHeadersInterceptor(tenant)); - } + addInterceptor(new ContextToHeadersInterceptor()); } private ClientRequestLimitInterceptor createRequestLimitInterceptor(final RpcOptions.LimitKind kind) { @@ -491,7 +460,6 @@ private ClientRequestLimitInterceptor createRequestLimitInterceptor(final RpcOpt private void attachErrMsg(final Throwable err, // final String callType, // final String method, // - final String childTenant, // final String target, // final long startCall, // final long duration, // @@ -502,10 +470,6 @@ private void attachErrMsg(final Throwable err, // .append(" got an error,") // .append(" method=") // .append(method) // - .append(", tenant=") // - .append(this.tenant) // - .append(", childTenant=") // - .append(childTenant != null ? childTenant : this.defaultChildTenant) // .append(", target=") // .append(target) // .append(", startCall=") // @@ -528,14 +492,6 @@ private Executor getObserverExecutor(final Observer observer) { return observer.executor() != null ? observer.executor() : this.asyncPool; } - private String addChildTenantIntoCtx(final Context ctx) { - final String childTenant = ctx.remove(AuthUtil.HEAD_ACCESS_CHILD_TENANT); - // null value means clear previous - AuthHeadersInterceptor.setCurrentChildTenant(childTenant); - ContextToHeadersInterceptor.setCurrentCtx(ctx); - return childTenant; - } - private void closeAllChannels() { this.managedChannelPool.values().forEach(ch -> { final boolean ret = ManagedChannelHelper.shutdownAndAwaitTermination(ch); diff --git a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/GrpcFactory.java b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/GrpcFactory.java index fb3910d..4fb1f8c 100644 --- a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/GrpcFactory.java +++ b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/GrpcFactory.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc; @@ -22,7 +9,6 @@ /** * CeresDB grpc impl service factory. * - * @author jiachun.fjc */ @SPI public class GrpcFactory implements RpcFactory { diff --git a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/IdChannel.java b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/IdChannel.java index 14fd79c..75bd204 100644 --- a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/IdChannel.java +++ b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/IdChannel.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc; @@ -28,7 +15,6 @@ /** * A managed channel that has a channel id. * - * @author jiachun.fjc */ public class IdChannel extends ManagedChannel { diff --git a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/ManagedChannelHelper.java b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/ManagedChannelHelper.java index 0808c7f..eb5864c 100644 --- a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/ManagedChannelHelper.java +++ b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/ManagedChannelHelper.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc; @@ -26,7 +13,6 @@ /** * Managed channel shutdown helper. * - * @author jiachun.fjc */ public final class ManagedChannelHelper { diff --git a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/MarshallerRegistry.java b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/MarshallerRegistry.java index 97a654b..bd81c22 100644 --- a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/MarshallerRegistry.java +++ b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/MarshallerRegistry.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc; @@ -27,7 +14,6 @@ /** * Marshaller registry for grpc service. * - * @author jiachun.fjc */ public interface MarshallerRegistry { diff --git a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/interceptors/AuthHeadersInterceptor.java b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/interceptors/AuthHeadersInterceptor.java deleted file mode 100644 index 12079fa..0000000 --- a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/interceptors/AuthHeadersInterceptor.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.rpc.interceptors; - -import java.util.Map; - -import io.grpc.CallOptions; -import io.grpc.Channel; -import io.grpc.ClientCall; -import io.grpc.ClientInterceptor; -import io.grpc.ForwardingClientCall; -import io.grpc.Metadata; -import io.grpc.MethodDescriptor; - -import io.ceresdb.common.Tenant; -import io.ceresdb.common.util.AuthUtil; -import io.ceresdb.common.util.Requires; -import io.ceresdb.common.util.Strings; - -/** - * CeresDB auth interceptor. - * - * @author jiachun.fjc - */ -public class AuthHeadersInterceptor implements ClientInterceptor { - - private static final ThreadLocal CURRENT_CHILD_TENANT = new ThreadLocal<>(); - - private final Tenant tenant; - - public AuthHeadersInterceptor(Tenant tenant) { - this.tenant = Requires.requireNonNull(tenant, "tenant"); - } - - public static void setCurrentChildTenant(final String childTenant) { - CURRENT_CHILD_TENANT.set(childTenant); - } - - public static String getCurrentChildTenant() { - return CURRENT_CHILD_TENANT.get(); - } - - @Override - public ClientCall interceptCall(final MethodDescriptor method, // - final CallOptions callOpts, // - final Channel next) { - return new HeaderAttachingClientCall<>(next.newCall(method, callOpts)); - } - - private final class HeaderAttachingClientCall - extends ForwardingClientCall.SimpleForwardingClientCall { - - // Non private to avoid synthetic class - HeaderAttachingClientCall(ClientCall call) { - super(call); - } - - @Override - public void start(final Listener respListener, final Metadata headers) { - final Map extraHeaders = AuthUtil.authHeaders(tenant); - - final String childTenant = getCurrentChildTenant(); - if (Strings.isNotBlank(childTenant)) { - AuthUtil.replaceChildTenant(extraHeaders, childTenant); - } - - if (!extraHeaders.isEmpty()) { - extraHeaders.forEach((k, v) -> headers.put(Metadata.Key.of(k, Metadata.ASCII_STRING_MARSHALLER), v)); - } - - super.start(respListener, headers); - } - } -} diff --git a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/interceptors/ClientRequestLimitInterceptor.java b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/interceptors/ClientRequestLimitInterceptor.java index 7d75c5f..2a65f92 100644 --- a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/interceptors/ClientRequestLimitInterceptor.java +++ b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/interceptors/ClientRequestLimitInterceptor.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc.interceptors; @@ -41,7 +28,6 @@ * * Refer to `concurrency-limit-grpc` * - * @author jiachun.fjc */ public class ClientRequestLimitInterceptor implements ClientInterceptor { diff --git a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/interceptors/ContextToHeadersInterceptor.java b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/interceptors/ContextToHeadersInterceptor.java index e6a41dc..e4da5db 100644 --- a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/interceptors/ContextToHeadersInterceptor.java +++ b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/interceptors/ContextToHeadersInterceptor.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc.interceptors; @@ -29,7 +16,6 @@ /** * Add RPC context to Grpc headers. * - * @author jiachun.fjc */ public class ContextToHeadersInterceptor implements ClientInterceptor { diff --git a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/interceptors/MetricInterceptor.java b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/interceptors/MetricInterceptor.java index a59ad4b..a2276c2 100644 --- a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/interceptors/MetricInterceptor.java +++ b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/interceptors/MetricInterceptor.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc.interceptors; @@ -32,7 +19,6 @@ /** * Request method metric interceptor. * - * @author jiachun.fjc */ public class MetricInterceptor implements ClientInterceptor { diff --git a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/Gradient2Limit.java b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/Gradient2Limit.java index 1ebd5df..bd0b123 100644 --- a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/Gradient2Limit.java +++ b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/Gradient2Limit.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc.limit; diff --git a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/LimitMetricRegistry.java b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/LimitMetricRegistry.java index 19be5a3..e939799 100644 --- a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/LimitMetricRegistry.java +++ b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/LimitMetricRegistry.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc.limit; @@ -24,7 +11,6 @@ /** * For tracking metrics in the limiters. * - * @author jiachun.fjc */ public class LimitMetricRegistry implements MetricRegistry { diff --git a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/RequestLimitCtx.java b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/RequestLimitCtx.java index 878f5b8..86692d8 100644 --- a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/RequestLimitCtx.java +++ b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/RequestLimitCtx.java @@ -1,25 +1,8 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc.limit; -/** - * - * @author jiachun.fjc - */ public interface RequestLimitCtx { String partitionKey(); diff --git a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/RequestLimiterBuilder.java b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/RequestLimiterBuilder.java index 7f8821d..eae2ede 100644 --- a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/RequestLimiterBuilder.java +++ b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/RequestLimiterBuilder.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc.limit; @@ -25,7 +12,6 @@ /** * Refer to `concurrency-limit-grpc` * - * @author jiachun.fjc */ public class RequestLimiterBuilder extends AbstractPartitionedLimiter.Builder { diff --git a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/VegasLimit.java b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/VegasLimit.java index b1dc3b6..a952569 100644 --- a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/VegasLimit.java +++ b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/limit/VegasLimit.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc.limit; diff --git a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/signal/RpcLimitSignalHandler.java b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/signal/RpcLimitSignalHandler.java index 2899c03..d4b8824 100644 --- a/ceresdb-grpc/src/main/java/io/ceresdb/rpc/signal/RpcLimitSignalHandler.java +++ b/ceresdb-grpc/src/main/java/io/ceresdb/rpc/signal/RpcLimitSignalHandler.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc.signal; @@ -28,7 +15,6 @@ /** * A signal handler that can reset LIMIT_SWITCH by {@link ClientRequestLimitInterceptor#resetLimitSwitch()}. * - * @author jiachun.fjc */ @SPI(priority = 89) public class RpcLimitSignalHandler implements SignalHandler { diff --git a/ceresdb-grpc/src/test/java/io/ceresdb/rpc/LimitTest.java b/ceresdb-grpc/src/test/java/io/ceresdb/rpc/LimitTest.java index 188f4a2..a2e83a5 100644 --- a/ceresdb-grpc/src/test/java/io/ceresdb/rpc/LimitTest.java +++ b/ceresdb-grpc/src/test/java/io/ceresdb/rpc/LimitTest.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc; @@ -27,9 +14,6 @@ import io.ceresdb.rpc.limit.VegasLimit; import com.netflix.concurrency.limits.Limit; -/** - * @author jiachun.fjc - */ public class LimitTest { private static final Logger LOG = LoggerFactory.getLogger(LimitTest.class); diff --git a/ceresdb-grpc/src/test/java/io/ceresdb/rpc/RequestLimiterInterceptorTest.java b/ceresdb-grpc/src/test/java/io/ceresdb/rpc/RequestLimiterInterceptorTest.java index 07fb4f7..908fe8a 100644 --- a/ceresdb-grpc/src/test/java/io/ceresdb/rpc/RequestLimiterInterceptorTest.java +++ b/ceresdb-grpc/src/test/java/io/ceresdb/rpc/RequestLimiterInterceptorTest.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc; @@ -46,7 +33,6 @@ /** * Refer to `concurrency-limit-grpc's test` * - * @author jiachun.fjc */ public class RequestLimiterInterceptorTest { diff --git a/ceresdb-grpc/src/test/java/io/ceresdb/rpc/StringMarshaller.java b/ceresdb-grpc/src/test/java/io/ceresdb/rpc/StringMarshaller.java index b0b8298..6fd9a9e 100644 --- a/ceresdb-grpc/src/test/java/io/ceresdb/rpc/StringMarshaller.java +++ b/ceresdb-grpc/src/test/java/io/ceresdb/rpc/StringMarshaller.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc; diff --git a/ceresdb-http/pom.xml b/ceresdb-http/pom.xml deleted file mode 100644 index 34c6441..0000000 --- a/ceresdb-http/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - 4.0.0 - - ceresdb-client - io.ceresdb - ${revision} - - - ceresdb-http - - - - ${project.groupId} - ceresdb-protocol - - - - - org.jetbrains.kotlin - kotlin-stdlib-common - - - org.jetbrains.kotlin - kotlin-stdlib - - - com.squareup.okio - okio - - - com.squareup.okhttp3 - okhttp - - - - com.google.code.gson - gson - - - diff --git a/ceresdb-http/src/main/java/io/ceresdb/http/HttpManagementClient.java b/ceresdb-http/src/main/java/io/ceresdb/http/HttpManagementClient.java deleted file mode 100644 index c561f4d..0000000 --- a/ceresdb-http/src/main/java/io/ceresdb/http/HttpManagementClient.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.http; - -import java.util.Collection; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.ceresdb.Management; -import io.ceresdb.MetricParser; -import io.ceresdb.MetricParserFactory; -import io.ceresdb.MetricParserFactoryProvider; -import io.ceresdb.Route; -import io.ceresdb.RouterClient; -import io.ceresdb.common.Endpoint; -import io.ceresdb.common.Tenant; -import io.ceresdb.common.util.AuthUtil; -import io.ceresdb.common.util.Requires; -import io.ceresdb.common.util.Strings; -import io.ceresdb.common.util.internal.ThrowUtil; -import io.ceresdb.http.errors.ManagementException; -import io.ceresdb.models.SqlResult; -import io.ceresdb.options.ManagementOptions; -import io.ceresdb.rpc.Context; -import com.google.gson.Gson; - -import okhttp3.Request; -import okhttp3.Response; -import okhttp3.ResponseBody; - -/** - * A management API client that communicates with the server using HTTP protocol. - * - * @author jiachun.fjc - */ -public class HttpManagementClient implements Management { - - private static final Logger LOG = LoggerFactory.getLogger(HttpManagementClient.class); - - private static final String AFFECTED_ROWS = "affected_rows"; - private static final String ROWS = "rows"; - - private final AtomicBoolean started = new AtomicBoolean(false); - - private ManagementOptions opts; - - private Tenant tenant; - private RouterClient routerClient; - - @Override - public boolean init(final ManagementOptions opts) { - if (!this.started.compareAndSet(false, true)) { - throw new IllegalStateException("Http management client has started"); - } - this.opts = Requires.requireNonNull(opts, "Null.opts"); - this.tenant = Requires.requireNonNull(opts.getTenant(), "Null.Tenant"); - this.routerClient = Requires.requireNonNull(opts.getRouterClient(), "Null.RouterClient"); - return true; - } - - @Override - public void shutdownGracefully() { - if (!this.started.compareAndSet(true, false)) { - return; - } - this.opts = null; - } - - @Override - public void display(final Printer out) { - out.println("--- HttpManagementClient ---") // - .print("started=") // - .println(this.started) // - .print("tenant=") // - .println(this.tenant.getTenant()); - } - - @Override - public SqlResult executeSql(final boolean autoRouting, final Context ctx, final String fmtSql, - final Object... args) { - final String sql = getSql(fmtSql, args); - final Endpoint managementAddress = this.opts.getManagementAddress(); - - if (!autoRouting && !this.opts.isCheckSql()) { - return doExecuteSql(sql, managementAddress, ctx); - } - - final MetricParser parser = parseSql(sql); - - if (!autoRouting) { - return doExecuteSql(sql, managementAddress, ctx); - } - - Requires.requireNonNull(parser, "Null.parser"); - - final List tables = parser.metricNames(); - - Requires.requireNonNull(tables, "Null.tables"); - Requires.requireTrue(!tables.isEmpty(), "Empty.tables"); - - try { - final Map routes = this.routerClient.routeFor(tables).get(); - final Endpoint target = getTargetEndpoint(routes.values()); - - return doExecuteSql(sql, target, ctx); - } catch (final Exception e) { - ThrowUtil.throwException(e); - } - - return null; - } - - private SqlResult doExecuteSql(final String sql, final Endpoint endpoint, final Context ctx) { - final Request request = contextToHeaders(newBaseRequestBuilder(), ctx) // - .url(getUrl(endpoint)) // - .post(HttpUtil.requestBody(HttpUtil.params("query", sql))) // - .build(); - - LOG.info("Executing sql: {}, to: {}.", sql, endpoint); - - try (Response resp = HttpUtil.httpClient().newCall(request).execute()) { - if (!resp.isSuccessful()) { - throw new ManagementException( - String.format("Execute sql [%s] error from server %s, err_code=%d, err_msg=%s, detail_msg=%s", // - sql, endpoint, resp.code(), resp.message(), getRespBody(resp))); - } - return toSqlResult(resp); - } catch (final Throwable t) { - LOG.error("Fail to execute sql: {}.", sql, t); - ThrowUtil.throwException(t); - } - return null; // never got here - } - - private Endpoint getTargetEndpoint(final Collection routes) { - final Endpoint managementAddress = this.opts.getManagementAddress(); - - return routes == null ? managementAddress : - routes.stream().findFirst().map(r -> Endpoint.of(r.getEndpoint().getIp(), managementAddress.getPort())) - .orElse(managementAddress); - } - - private String getSql(final String fmtSql, final Object... args) { - return String.format(fmtSql, args); - } - - private MetricParser parseSql(final String sql) { - final MetricParser sqlParser = getSqlParser(sql); - - if (sqlParser == null || !this.opts.isCheckSql()) { - return sqlParser; - } - - return checkStatementType(sqlParser); - } - - private Request.Builder contextToHeaders(final Request.Builder builder, final Context ctx) { - if (ctx == null) { - return builder; - } - - ctx.entrySet().forEach(e -> builder.addHeader(e.getKey(), String.valueOf(e.getValue()))); - return builder; - } - - private Request.Builder newBaseRequestBuilder() { - return authHeaders(new Request.Builder()) // - .addHeader("Content-Type", "application/json"); - } - - private Request.Builder authHeaders(final Request.Builder builder) { - AuthUtil.authHeaders(this.tenant).forEach(builder::addHeader); - return builder; - } - - private static String getUrl(final Endpoint endpoint) { - Requires.requireNonNull(endpoint, "endpoint"); - return String.format("%s://%s:%d/sql", HttpUtil.PROTOCOL, endpoint.getIp(), endpoint.getPort()); - } - - @SuppressWarnings("unchecked") - private static SqlResult toSqlResult(final Response resp) throws IOException { - final String body = getRespBody(resp); - if (Strings.isBlank(body)) { - return SqlResult.EMPTY_RESULT; - } - - final Map mapResult = new Gson().fromJson(body, Map.class); - final Number affectedRows = (Number) mapResult.get(AFFECTED_ROWS); - final List> rows = (List>) mapResult.get(ROWS); - return new SqlResult(affectedRows == null ? 0L : affectedRows.longValue(), rows); - } - - private static String getRespBody(final Response resp) throws IOException { - final ResponseBody body = resp.body(); - return body == null ? "" : body.string(); - } - - private static MetricParser getSqlParser(final String sql) { - final MetricParserFactory factory = MetricParserFactoryProvider.getMetricParserFactory(); - if (MetricParserFactory.DEFAULT == factory) { - return null; - } - - return factory.getParser(sql); - } - - private static MetricParser checkStatementType(final MetricParser sqlParser) { - final MetricParser.StatementType statementType = sqlParser.statementType(); - switch (statementType) { - case Create: - case Select: - case Alter: - case Describe: - case Show: - case Drop: - case Exists: - break; - case Insert: // We must use the SDK to write data, insert statements are not allowed - case Unknown: - default: - throw new UnsupportedOperationException("Unsupported statement: " + statementType); - } - - return sqlParser; - } -} diff --git a/ceresdb-http/src/main/java/io/ceresdb/http/HttpUtil.java b/ceresdb-http/src/main/java/io/ceresdb/http/HttpUtil.java deleted file mode 100644 index a5f285d..0000000 --- a/ceresdb-http/src/main/java/io/ceresdb/http/HttpUtil.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.http; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; - -import io.ceresdb.common.OptKeys; -import io.ceresdb.common.util.SystemPropertyUtil; -import com.google.gson.Gson; - -import okhttp3.MediaType; -import okhttp3.OkHttpClient; -import okhttp3.RequestBody; - -/** - * Http utils. - * - * @author jiachun.fjc - */ -public class HttpUtil { - - public static final String PROTOCOL = "http"; - - private static final long READ_TIMEOUT_MS = SystemPropertyUtil.getLong(OptKeys.HTTP_READ_TIMEOUT_MS, 10000); - private static final long WRITE_TIMEOUT_MS = SystemPropertyUtil.getLong(OptKeys.HTTP_WRITE_TIMEOUT_MS, 10000); - - private static final MediaType JSON_MEDIA_TYPE = MediaType.parse("application/json; charset=utf-8"); - - // The singleton HTTP client. - // - // Shutdown isn’t necessary, the threads and connections that - // are held will be released automatically if they remain idle. - private static final OkHttpClient OK_HTTP_CLIENT = new OkHttpClient.Builder() // - .readTimeout(READ_TIMEOUT_MS, TimeUnit.MILLISECONDS) // - .writeTimeout(WRITE_TIMEOUT_MS, TimeUnit.MILLISECONDS) // - .build(); - - public static OkHttpClient httpClient() { - return OK_HTTP_CLIENT; - } - - public static Map params(final String key, final String val) { - return params(key, val, HashMap::new); - } - - public static Map params(final String key, final String val, - final Supplier> ctx) { - final Map map = ctx.get(); - map.put(key, val); - return map; - } - - public static RequestBody requestBody(final Map params) { - return RequestBody.create(new Gson().toJson(params), JSON_MEDIA_TYPE); - } -} diff --git a/ceresdb-http/src/main/java/io/ceresdb/http/errors/ManagementException.java b/ceresdb-http/src/main/java/io/ceresdb/http/errors/ManagementException.java deleted file mode 100644 index 2d43b41..0000000 --- a/ceresdb-http/src/main/java/io/ceresdb/http/errors/ManagementException.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.http.errors; - -/** - * - * @author jiachun.fjc - */ -public class ManagementException extends RuntimeException { - - private static final long serialVersionUID = -7890279415370105843L; - - public ManagementException() { - } - - public ManagementException(String message) { - super(message); - } - - public ManagementException(String message, Throwable cause) { - super(message, cause); - } - - public ManagementException(Throwable cause) { - super(cause); - } - - public ManagementException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } -} diff --git a/ceresdb-http/src/main/resources/META-INF/services/io.ceresdb.Management b/ceresdb-http/src/main/resources/META-INF/services/io.ceresdb.Management deleted file mode 100644 index d93fcc2..0000000 --- a/ceresdb-http/src/main/resources/META-INF/services/io.ceresdb.Management +++ /dev/null @@ -1 +0,0 @@ -io.ceresdb.http.HttpManagementClient diff --git a/ceresdb-protocol/pom.xml b/ceresdb-protocol/pom.xml index 9fd3d53..1b44304 100644 --- a/ceresdb-protocol/pom.xml +++ b/ceresdb-protocol/pom.xml @@ -27,8 +27,17 @@ - org.apache.avro - avro + com.github.luben + zstd-jni + + + org.apache.arrow + arrow-vector + + + + org.apache.arrow + arrow-memory-unsafe diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/ArrayMapper.java b/ceresdb-protocol/src/main/java/io/ceresdb/ArrayMapper.java deleted file mode 100644 index d29bf32..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/ArrayMapper.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb; - -import java.util.List; -import java.util.function.Function; - -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; - -/** - * Map byte[] to Object[] by avro. - * - * @author jiachun.fjc - */ -public class ArrayMapper extends AvroMapper implements Function { - - ArrayMapper(Schema schema) { - super(schema); - } - - public static ArrayMapper getMapper(final String s) { - final Schema schema = new Schema.Parser().parse(s); - return new ArrayMapper(schema); - } - - @Override - public Object[] apply(final byte[] bytes) { - final GenericRecord record = mapTo(bytes); - final List fields = record.getSchema().getFields(); - final Object[] objects = new Object[fields.size()]; - int i = 0; - for (final Schema.Field f : fields) { - objects[i++] = record.get(f.name()); - } - return objects; - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/AvroMapper.java b/ceresdb-protocol/src/main/java/io/ceresdb/AvroMapper.java deleted file mode 100644 index 7ae9732..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/AvroMapper.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb; - -import java.io.IOException; - -import org.apache.avro.AvroRuntimeException; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.BinaryDecoder; -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.DecoderFactory; - -/** - * Avro mapper tool. - * - * @author jiachun.fjc - */ -public class AvroMapper { - private final Schema schema; - private BinaryDecoder reuseDecoder; - - public AvroMapper(Schema schema) { - this.schema = schema; - } - - public GenericRecord mapTo(final byte[] bytes) { - final DatumReader reader = new GenericDatumReader<>(this.schema); - final BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(bytes, this.reuseDecoder); - if (this.reuseDecoder == null) { - this.reuseDecoder = decoder; - } - try { - return reader.read(null, decoder); - } catch (final IOException e) { - throw new AvroRuntimeException(e); - } - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/BlockingStreamIterator.java b/ceresdb-protocol/src/main/java/io/ceresdb/BlockingStreamIterator.java index 7b825c6..578ee93 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/BlockingStreamIterator.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/BlockingStreamIterator.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; @@ -23,34 +10,33 @@ import java.util.stream.Stream; import io.ceresdb.errors.IteratorException; -import io.ceresdb.models.QueryOk; -import io.ceresdb.models.Record; +import io.ceresdb.models.SqlQueryOk; +import io.ceresdb.models.Row; import io.ceresdb.rpc.Observer; /** * A blocking iterator, the `hasNext` method will be blocked until * the server returns data or the process ends. * - * @author jiachun.fjc */ -public class BlockingStreamIterator implements Iterator> { +public class BlockingStreamIterator implements Iterator> { - private static final QueryOk EOF = QueryOk.emptyOk(); + private static final SqlQueryOk EOF = SqlQueryOk.emptyOk(); private final long timeout; private final TimeUnit unit; private final BlockingQueue staging = new LinkedBlockingQueue<>(); - private final Observer observer; - private QueryOk next; + private final Observer observer; + private SqlQueryOk next; public BlockingStreamIterator(long timeout, TimeUnit unit) { this.timeout = timeout; this.unit = unit; - this.observer = new Observer() { + this.observer = new Observer() { @Override - public void onNext(final QueryOk value) { + public void onNext(final SqlQueryOk value) { staging.offer(value); } @@ -83,7 +69,7 @@ public boolean hasNext() { return reject("Stream iterator got an error", (Throwable) v); } - this.next = (QueryOk) v; + this.next = (SqlQueryOk) v; return this.next != EOF; } catch (final InterruptedException e) { @@ -93,7 +79,7 @@ public boolean hasNext() { } @Override - public Stream next() { + public Stream next() { if (this.next == null) { return reject("Null `next` element"); } @@ -102,10 +88,10 @@ public Stream next() { return reject("Reaches the end of the iterator"); } - return this.next.mapToRecord(); + return this.next.stream(); } - public Observer getObserver() { + public Observer getObserver() { return this.observer; } diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/CeresDBClient.java b/ceresdb-protocol/src/main/java/io/ceresdb/CeresDBClient.java index 0f575e6..08af871 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/CeresDBClient.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/CeresDBClient.java @@ -1,23 +1,9 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -37,13 +23,14 @@ import io.ceresdb.common.util.MetricExecutor; import io.ceresdb.common.util.MetricsUtil; import io.ceresdb.models.Err; -import io.ceresdb.models.QueryOk; -import io.ceresdb.models.QueryRequest; +import io.ceresdb.models.Point; +import io.ceresdb.models.RequestContext; +import io.ceresdb.models.SqlQueryOk; +import io.ceresdb.models.SqlQueryRequest; import io.ceresdb.models.Result; -import io.ceresdb.models.Rows; import io.ceresdb.models.WriteOk; +import io.ceresdb.models.WriteRequest; import io.ceresdb.options.CeresDBOptions; -import io.ceresdb.options.ManagementOptions; import io.ceresdb.options.QueryOptions; import io.ceresdb.options.RouterOptions; import io.ceresdb.options.WriteOptions; @@ -52,13 +39,16 @@ import io.ceresdb.rpc.RpcClient; import io.ceresdb.rpc.RpcFactoryProvider; import io.ceresdb.rpc.RpcOptions; +import io.ceresdb.util.RpcServiceRegister; +import io.ceresdb.util.StreamWriteBuf; +import io.ceresdb.util.Utils; + import com.codahale.metrics.Counter; import com.codahale.metrics.Meter; /** * CeresDB client. * - * @author jiachun.fjc */ public class CeresDBClient implements Write, Query, Lifecycle, Display { @@ -77,9 +67,7 @@ public class CeresDBClient implements Write, Query, Lifecycle, D private RouterClient routerClient; private WriteClient writeClient; private QueryClient queryClient; - // CeresDBClient is only intended to manage the instance and does not - // intend to broker any of its behavior - private Management management; + // Note: We do not close it to free resources, as we view it as shared private Executor asyncWritePool; private Executor asyncReadPool; @@ -112,7 +100,6 @@ public boolean init(final CeresDBOptions opts) { this.asyncReadPool = withMetricPool(this.opts.getAsyncReadPool(), "async_read_pool.time"); this.writeClient = initWriteClient(this.opts, this.routerClient, this.asyncWritePool); this.queryClient = initQueryClient(this.opts, this.routerClient, this.asyncReadPool); - this.management = initManagementClient(this.opts, this.routerClient); INSTANCES.put(this.id, this); @@ -139,10 +126,6 @@ public void shutdownGracefully() { this.routerClient.shutdownGracefully(); } - if (this.management != null) { - this.management.shutdownGracefully(); - } - INSTANCES.remove(this.id); } @@ -155,44 +138,31 @@ public void ensureInitialized() { } @Override - public CompletableFuture> write(final Collection data, final Context ctx) { + public CompletableFuture> write(final WriteRequest req, final Context ctx) { ensureInitialized(); - return this.writeClient.write(data, attachCtx(ctx)); + return this.writeClient.write(req, attachCtx(ctx)); } @Override - public StreamWriteBuf streamWrite(final String metric, final Context ctx) { + public StreamWriteBuf streamWrite(RequestContext reqCtx, final String table, final Context ctx) { ensureInitialized(); - return this.writeClient.streamWrite(metric, attachCtx(ctx)); + return this.writeClient.streamWrite(reqCtx, table, attachCtx(ctx)); } @Override - public CompletableFuture> query(final QueryRequest req, final Context ctx) { + public CompletableFuture> sqlQuery(final SqlQueryRequest req, final Context ctx) { ensureInitialized(); - return this.queryClient.query(req, attachCtx(ctx)); + return this.queryClient.sqlQuery(req, attachCtx(ctx)); } @Override - public void streamQuery(final QueryRequest req, final Context ctx, final Observer observer) { + public void streamSqlQuery(final SqlQueryRequest req, final Context ctx, final Observer observer) { ensureInitialized(); - this.queryClient.streamQuery(req, attachCtx(ctx), observer); + this.queryClient.streamSqlQuery(req, attachCtx(ctx), observer); } - public boolean hasManagement() { - return this.management != null; - } - - public Management management() { - return this.management; - } - - private Executor withMetricPool(final Executor pool, final String name) { - return pool == null ? null : new MetricExecutor(pool, name); - } - - private Context attachCtx(final Context ctx) { - final Context c = ctx == null ? Context.newDefault() : ctx; - return c.with(ID_KEY, id()).with(VERSION_KEY, version()); + public static List instances() { + return new ArrayList<>(INSTANCES.values()); } public int id() { @@ -203,26 +173,10 @@ public String version() { return VERSION; } - public Executor asyncWritePool() { - return this.asyncWritePool; - } - - public Executor asyncReadPool() { - return this.asyncReadPool; - } - public RouterClient routerClient() { return this.routerClient; } - public WriteClient writeClient() { - return this.writeClient; - } - - public QueryClient queryClient() { - return this.queryClient; - } - @Override public void display(final Printer out) { out.println("--- CeresDBClient ---") // @@ -232,8 +186,8 @@ public void display(final Printer out) { .println(version()) // .print("clusterAddress=") // .println(this.opts.getClusterAddress()) // - .print("tenant=") // - .println(this.opts.getTenant().getTenant()) // + .print("database=") // + .println(this.opts.getDatabase()) // .print("userAsyncWritePool=") // .println(this.opts.getAsyncWritePool()) // .print("userAsyncReadPool=") // @@ -254,11 +208,6 @@ public void display(final Printer out) { this.queryClient.display(out); } - if (this.management != null) { - out.println(""); - this.management.display(out); - } - out.println(""); } @@ -266,7 +215,7 @@ public void display(final Printer out) { public String toString() { return "CeresDBClient{" + // "id=" + id + // - "version=" + version() + // + ", version=" + version() + // ", started=" + started + // ", opts=" + opts + // ", writeClient=" + writeClient + // @@ -275,13 +224,17 @@ public String toString() { '}'; } - public static List instances() { - return new ArrayList<>(INSTANCES.values()); + private Executor withMetricPool(final Executor pool, final String name) { + return pool == null ? null : new MetricExecutor(pool, name); + } + + private Context attachCtx(final Context ctx) { + final Context c = ctx == null ? Context.newDefault() : ctx; + return c.with(ID_KEY, id()).with(VERSION_KEY, version()); } private static RpcClient initRpcClient(final CeresDBOptions opts) { final RpcOptions rpcOpts = opts.getRpcOptions(); - rpcOpts.setTenant(opts.getTenant()); final RpcClient rpcClient = RpcFactoryProvider.getRpcFactory().createRpcClient(); if (!rpcClient.init(rpcOpts)) { throw new IllegalStateException("Fail to start RPC client"); @@ -293,8 +246,18 @@ private static RpcClient initRpcClient(final CeresDBOptions opts) { private static RouterClient initRouteClient(final CeresDBOptions opts, final RpcClient rpcClient) { final RouterOptions routerOpts = opts.getRouterOptions(); routerOpts.setRpcClient(rpcClient); - final RouterClient routerClient = routerOpts.getRouteMode().equals(RouteMode.CLUSTER) ? new RouterClient() : - new StandaloneRouterClient(); + + RouterClient routerClient; + switch (routerOpts.getRouteMode()) { + case DIRECT: + routerClient = new RouterClient(); + break; + case PROXY: + routerClient = new ProxyRouterClient(); + break; + default: + throw new IllegalArgumentException("Invalid Route mode " + routerOpts.getRouteMode()); + } if (!routerClient.init(routerOpts)) { throw new IllegalStateException("Fail to start router client"); } @@ -305,6 +268,7 @@ private static WriteClient initWriteClient(final CeresDBOptions opts, // final RouterClient routerClient, // final Executor asyncPool) { final WriteOptions writeOpts = opts.getWriteOptions(); + writeOpts.setDatabase(opts.getDatabase()); writeOpts.setRoutedClient(routerClient); writeOpts.setAsyncPool(asyncPool); final WriteClient writeClient = new WriteClient(); @@ -318,6 +282,7 @@ private static QueryClient initQueryClient(final CeresDBOptions opts, // final RouterClient routerClient, // final Executor asyncPool) { final QueryOptions queryOpts = opts.getQueryOptions(); + queryOpts.setDatabase(opts.getDatabase()); queryOpts.setRouterClient(routerClient); queryOpts.setAsyncPool(asyncPool); final QueryClient queryClient = new QueryClient(); @@ -327,22 +292,6 @@ private static QueryClient initQueryClient(final CeresDBOptions opts, // return queryClient; } - private static Management initManagementClient(final CeresDBOptions opts, final RouterClient routerClient) { - final ManagementOptions mOpts = opts.getManagementOptions(); - if (mOpts == null) { - return null; - } - if (!CeresDBManagementProvider.hasManagement()) { - return null; - } - final Management management = CeresDBManagementProvider.createManagement(); - mOpts.setRouterClient(routerClient); - if (!management.init(mOpts)) { - return null; - } - return management; - } - private static String loadVersion() { try { return Utils // diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/CeresDBManagementProvider.java b/ceresdb-protocol/src/main/java/io/ceresdb/CeresDBManagementProvider.java deleted file mode 100644 index 148ae38..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/CeresDBManagementProvider.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb; - -import io.ceresdb.common.util.Requires; -import io.ceresdb.common.util.ServiceLoader; - -/** - * - * @author jiachun.fjc - */ -final class CeresDBManagementProvider { - - private static final Class MANAGEMENT_CLS; - - static { - Class _managementCls; - try { - _managementCls = ServiceLoader.load(Management.class).firstClass(); - } catch (final Throwable ignored) { - _managementCls = null; - } - MANAGEMENT_CLS = _managementCls; - } - - static boolean hasManagement() { - return MANAGEMENT_CLS != null; - } - - static Management createManagement() { - Requires.requireTrue(hasManagement(), "Do not have a management!"); - try { - return MANAGEMENT_CLS.newInstance(); - } catch (final Throwable t) { - throw new IllegalStateException("Fail to create management instance", t); - } - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/Management.java b/ceresdb-protocol/src/main/java/io/ceresdb/Management.java deleted file mode 100644 index bbd8a71..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/Management.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb; - -import io.ceresdb.common.Display; -import io.ceresdb.common.Lifecycle; -import io.ceresdb.models.SqlResult; -import io.ceresdb.options.ManagementOptions; -import io.ceresdb.rpc.Context; - -/** - * CeresDB client for management API. - * - * @author jiachun.fjc - */ -public interface Management extends Lifecycle, Display { - - /** - * @see #executeSql(boolean, Context, String, Object...) - */ - default SqlResult executeSql(final String fmtSql, final Object... args) { - return executeSql(true, Context.newDefault(), fmtSql, args); - } - - /** - * Execute a SQL. - * - * CREATE TABLE: - * ``` - * CREATE TABLE my_first_table( - * ts TIMESTAMP NOT NULL, - * c1 STRING TAG NOT NULL, - * c2 STRING TAG NOT NULL, - * c3 DOUBLE NULL, - * c4 STRING NULL, - * c5 INT64 NULL, - * c6 FLOAT NULL, - * c7 INT32 NULL, - * c8 INT16 NULL, - * c9 INT8 NULL, - * c10 BOOLEAN NULL, - * c11 UINT64 NULL, - * c12 UINT32 NULL, - * c13 UINT16 NULL, - * c14 UINT8 NULL, - * c15 TIMESTAMP NULL, - * c16 VARBINARY NULL, - * TIMESTAMP KEY(ts) - * ) ENGINE=Analytic - * ``` - * - * DESCRIBE TABLE: - * ``` - * DESCRIBE TABLE my_table - * ``` - * - * @param autoRouting automatic routing to server - * @param ctx the invoke context, will automatically be placed in HTTP Headers - * @param fmtSql format sql string - * @param args arguments referenced by the format specifiers in the format - * sql string. If there are more arguments than format specifiers, - * the extra arguments are ignored. The number of arguments is - * variable and may be zero. - * @return the sql result. - */ - SqlResult executeSql(final boolean autoRouting, final Context ctx, final String fmtSql, final Object... args); -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/MetricParserFactory.java b/ceresdb-protocol/src/main/java/io/ceresdb/MetricParserFactory.java deleted file mode 100644 index 36eb954..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/MetricParserFactory.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb; - -/** - * - * @author jiachun.fjc - */ -public interface MetricParserFactory { - - MetricParserFactory DEFAULT = new NoopFactory(); - - MetricParser getParser(final String ql); - - class NoopFactory implements MetricParserFactory { - - @Override - public MetricParser getParser(final String ql) { - return MetricParser.DEFAULT; - } - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/MetricParserFactoryProvider.java b/ceresdb-protocol/src/main/java/io/ceresdb/MetricParserFactoryProvider.java deleted file mode 100644 index 9a33971..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/MetricParserFactoryProvider.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb; - -import io.ceresdb.common.util.ServiceLoader; - -/** - * - * @author jiachun.fjc - */ -public class MetricParserFactoryProvider { - - private static final MetricParserFactory METRIC_PARSER_FACTORY = ServiceLoader // - .load(MetricParserFactory.class) // - .firstOrDefault(() -> MetricParserFactory.DEFAULT); - - public static MetricParserFactory getMetricParserFactory() { - return METRIC_PARSER_FACTORY; - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/ProxyRouterClient.java b/ceresdb-protocol/src/main/java/io/ceresdb/ProxyRouterClient.java new file mode 100644 index 0000000..87cc6a2 --- /dev/null +++ b/ceresdb-protocol/src/main/java/io/ceresdb/ProxyRouterClient.java @@ -0,0 +1,41 @@ +/* + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + */ +package io.ceresdb; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import io.ceresdb.models.RequestContext; +import io.ceresdb.util.Utils; + +/** + * A route rpc client which implement RouteMode.Proxy + * + * cached nothing about table information, and return + * clusterAddress directly + */ +public class ProxyRouterClient extends RouterClient { + + @Override + public CompletableFuture> routeFor(final RequestContext reqCtx, + final Collection metrics) { + if (metrics == null || metrics.isEmpty()) { + return Utils.completedCf(Collections.emptyMap()); + } + + final Map routeMap = new HashMap<>(); + + metrics.forEach(table -> { + Route route = new Route(); + route.setEndpoint(this.opts.getClusterAddress()); + route.setTable(table); + routeMap.put(table, route); + }); + return Utils.completedCf(routeMap); + } + +} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/Query.java b/ceresdb-protocol/src/main/java/io/ceresdb/Query.java index 947d593..2c2e622 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/Query.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/Query.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; @@ -21,9 +8,9 @@ import java.util.concurrent.TimeUnit; import io.ceresdb.models.Err; -import io.ceresdb.models.QueryOk; -import io.ceresdb.models.QueryRequest; -import io.ceresdb.models.Record; +import io.ceresdb.models.SqlQueryOk; +import io.ceresdb.models.Row; +import io.ceresdb.models.SqlQueryRequest; import io.ceresdb.models.Result; import io.ceresdb.rpc.Context; import io.ceresdb.rpc.Observer; @@ -31,15 +18,14 @@ /** * The query API for CeresDB client. * - * @author jiachun.fjc */ public interface Query { /** - * @see #query(QueryRequest, Context) + * @see #sqlQuery(SqlQueryRequest, Context) */ - default CompletableFuture> query(final QueryRequest req) { - return query(req, Context.newDefault()); + default CompletableFuture> sqlQuery(final SqlQueryRequest req) { + return sqlQuery(req, Context.newDefault()); } /** @@ -49,13 +35,13 @@ default CompletableFuture> query(final QueryRequest req) { * @param ctx the invoke context * @return query result */ - CompletableFuture> query(final QueryRequest req, final Context ctx); + CompletableFuture> sqlQuery(final SqlQueryRequest req, final Context ctx); /** - * @see #streamQuery(QueryRequest, Context, Observer) + * @see #streamSqlQuery(SqlQueryRequest, Context, Observer) */ - default void streamQuery(final QueryRequest req, final Observer observer) { - streamQuery(req, Context.newDefault(), observer); + default void streamSqlQuery(final SqlQueryRequest req, final Observer observer) { + streamSqlQuery(req, Context.newDefault(), observer); } /** @@ -65,15 +51,15 @@ default void streamQuery(final QueryRequest req, final Observer observe * @param observer receives data from an observable stream * @param ctx the invoke context */ - void streamQuery(final QueryRequest req, final Context ctx, final Observer observer); + void streamSqlQuery(final SqlQueryRequest req, final Context ctx, final Observer observer); /** - * @see #blockingStreamQuery(QueryRequest, long, TimeUnit, Context) + * @see #blockingStreamSqlQuery(SqlQueryRequest, long, TimeUnit, Context) */ - default Iterator blockingStreamQuery(final QueryRequest req, // + default Iterator blockingStreamSqlQuery(final SqlQueryRequest req, // final long timeout, // final TimeUnit unit) { - return blockingStreamQuery(req, timeout, unit, Context.newDefault()); + return blockingStreamSqlQuery(req, timeout, unit, Context.newDefault()); } /** @@ -85,12 +71,12 @@ default Iterator blockingStreamQuery(final QueryRequest req, // * @param ctx the invoke context * @return the iterator of record */ - default Iterator blockingStreamQuery(final QueryRequest req, // + default Iterator blockingStreamSqlQuery(final SqlQueryRequest req, // final long timeout, // final TimeUnit unit, // final Context ctx) { final BlockingStreamIterator streams = new BlockingStreamIterator(timeout, unit); - streamQuery(req, ctx, streams.getObserver()); - return new RecordIterator(streams); + streamSqlQuery(req, ctx, streams.getObserver()); + return new RowIterator(streams); } } diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/QueryClient.java b/ceresdb-protocol/src/main/java/io/ceresdb/QueryClient.java index 5d74d9a..adceb11 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/QueryClient.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/QueryClient.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; @@ -21,6 +8,12 @@ import java.util.concurrent.Executor; import java.util.stream.Collectors; +import io.ceresdb.common.parser.SqlParser; +import io.ceresdb.common.parser.SqlParserFactoryProvider; +import io.ceresdb.common.util.Strings; +import io.ceresdb.limit.LimitedPolicy; +import io.ceresdb.limit.QueryLimiter; +import io.ceresdb.models.RequestContext; import io.ceresdb.proto.internal.Storage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,19 +28,20 @@ import io.ceresdb.common.util.SerializingExecutor; import io.ceresdb.errors.StreamException; import io.ceresdb.models.Err; -import io.ceresdb.models.QueryOk; -import io.ceresdb.models.QueryRequest; +import io.ceresdb.models.SqlQueryOk; +import io.ceresdb.models.SqlQueryRequest; import io.ceresdb.models.Result; import io.ceresdb.options.QueryOptions; import io.ceresdb.rpc.Context; import io.ceresdb.rpc.Observer; +import io.ceresdb.util.Utils; + import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; /** * Default Query API impl. * - * @author jiachun.fjc */ public class QueryClient implements Query, Lifecycle, Display { @@ -59,12 +53,12 @@ public class QueryClient implements Query, Lifecycle, Display { private QueryLimiter queryLimiter; static final class InnerMetrics { - static final Histogram READ_ROW_COUNT = MetricsUtil.histogram("read_row_count"); - static final Meter READ_FAILED = MetricsUtil.meter("read_failed"); - static final Meter READ_QPS = MetricsUtil.meter("read_qps"); + static final Histogram READ_ROWS_COUNT = MetricsUtil.histogram("read_rows_count"); + static final Meter READ_FAILED = MetricsUtil.meter("read_failed"); + static final Meter READ_QPS = MetricsUtil.meter("read_qps"); - static Histogram readRowCount() { - return READ_ROW_COUNT; + static Histogram readRowsCount() { + return READ_ROWS_COUNT; } static Meter readFailed() { @@ -98,15 +92,19 @@ public void shutdownGracefully() { } @Override - public CompletableFuture> query(final QueryRequest req, final Context ctx) { + public CompletableFuture> sqlQuery(final SqlQueryRequest req, final Context ctx) { + req.setReqCtx(attachRequestCtx(req.getReqCtx())); + Requires.requireNonNull(req, "Null.request"); + Requires.requireTrue(Strings.isNotBlank(req.getReqCtx().getDatabase()), "No database selected"); + final long startCall = Clock.defaultClock().getTick(); setMetricsIfAbsent(req); return this.queryLimiter.acquireAndDo(req, () -> query0(req, ctx, 0).whenCompleteAsync((r, e) -> { InnerMetrics.readQps().mark(); if (r != null) { - final int rowCount = r.mapOr(0, QueryOk::getRowCount); - InnerMetrics.readRowCount().update(rowCount); + final int rowCount = r.mapOr(0, SqlQueryOk::getRowCount); + InnerMetrics.readRowsCount().update(rowCount); if (Utils.isRwLogging()) { LOG.info("Read from {}, duration={} ms, rowCount={}.", Utils.DB_NAME, Clock.defaultClock().duration(startCall), rowCount); @@ -120,23 +118,36 @@ public CompletableFuture> query(final QueryRequest req, fin } @Override - public void streamQuery(final QueryRequest req, final Context ctx, final Observer observer) { + public void streamSqlQuery(final SqlQueryRequest req, final Context ctx, final Observer observer) { + req.setReqCtx(attachRequestCtx(req.getReqCtx())); + Requires.requireNonNull(req, "Null.request"); Requires.requireNonNull(observer, "Null.observer"); + Requires.requireTrue(Strings.isNotBlank(req.getReqCtx().getDatabase()), "No database selected"); setMetricsIfAbsent(req); - this.routerClient.routeFor(req.getMetrics()) + this.routerClient.routeFor(req.getReqCtx(), req.getTables()) .thenApply(routes -> routes.values().stream().findAny().orElse(this.routerClient.clusterRoute())) .thenAccept(route -> streamQueryFrom(route.getEndpoint(), req, ctx, observer)); } - private CompletableFuture> query0(final QueryRequest req, // - final Context ctx, // - final int retries) { + private RequestContext attachRequestCtx(RequestContext reqCtx) { + if (reqCtx == null) { + reqCtx = new RequestContext(); + } + if (Strings.isNullOrEmpty(reqCtx.getDatabase())) { + reqCtx.setDatabase(this.opts.getDatabase()); + } + return reqCtx; + } + + private CompletableFuture> query0(final SqlQueryRequest req, // + final Context ctx, // + final int retries) { InnerMetrics.readByRetries(retries).mark(); - return this.routerClient.routeFor(req.getMetrics()) // + return this.routerClient.routeFor(req.getReqCtx(), req.getTables()) // .thenApplyAsync(routes -> routes.values() // .stream() // .findAny() // everyone is OK @@ -158,7 +169,7 @@ private CompletableFuture> query0(final QueryRequest req, / // Should refresh route table final Set toRefresh = err.stream() // .filter(Utils::shouldRefreshRouteTable) // - .flatMap(e -> e.getFailedMetrics().stream()) // + .flatMap(e -> e.getFailedTables().stream()) // .collect(Collectors.toSet()); if (toRefresh.isEmpty()) { @@ -166,24 +177,24 @@ private CompletableFuture> query0(final QueryRequest req, / } // Async to refresh route info - return this.routerClient.routeRefreshFor(toRefresh) + return this.routerClient.routeRefreshFor(req.getReqCtx(), toRefresh) .thenComposeAsync(routes -> query0(req, ctx, retries + 1), this.asyncPool); }, this.asyncPool); } - private void setMetricsIfAbsent(final QueryRequest req) { - if (req.getMetrics() != null && !req.getMetrics().isEmpty()) { + private void setMetricsIfAbsent(final SqlQueryRequest req) { + if (req.getTables() != null && !req.getTables().isEmpty()) { return; } - final MetricParser parser = MetricParserFactoryProvider.getMetricParserFactory().getParser(req.getQl()); - req.setMetrics(parser.metricNames()); + final SqlParser parser = SqlParserFactoryProvider.getSqlParserFactory().getParser(req.getSql()); + req.setTables(parser.tableNames()); } private static final class ErrHandler implements Runnable { - private final QueryRequest req; + private final SqlQueryRequest req; - private ErrHandler(QueryRequest req) { + private ErrHandler(SqlQueryRequest req) { this.req = req; } @@ -193,39 +204,41 @@ public void run() { } } - private CompletableFuture> queryFrom(final Endpoint endpoint, // - final QueryRequest req, // - final Context ctx, // - final int retries) { - final Storage.QueryRequest request = Storage.QueryRequest.newBuilder() // - .addAllMetrics(req.getMetrics()) // - .setQl(req.getQl()) // + private CompletableFuture> queryFrom(final Endpoint endpoint, // + final SqlQueryRequest req, // + final Context ctx, // + final int retries) { + final Storage.SqlQueryRequest request = Storage.SqlQueryRequest.newBuilder() // + .setContext(Storage.RequestContext.newBuilder().setDatabase(req.getReqCtx().getDatabase()).build()) // + .addAllTables(req.getTables()) // + .setSql(req.getSql()) // .build(); - final CompletableFuture qrf = this.routerClient.invoke(endpoint, // + final CompletableFuture qrf = this.routerClient.invoke(endpoint, // request, // ctx.with("retries", retries) // server can use this in metrics ); return qrf.thenApplyAsync( - resp -> Utils.toResult(resp, req.getQl(), endpoint, req.getMetrics(), new ErrHandler(req)), + resp -> Utils.toResult(resp, req.getSql(), endpoint, req.getTables(), new ErrHandler(req)), this.asyncPool); } private void streamQueryFrom(final Endpoint endpoint, // - final QueryRequest req, // + final SqlQueryRequest req, // final Context ctx, // - final Observer observer) { - final Storage.QueryRequest request = Storage.QueryRequest.newBuilder() // - .addAllMetrics(req.getMetrics()) // - .setQl(req.getQl()) // + final Observer observer) { + final Storage.SqlQueryRequest request = Storage.SqlQueryRequest.newBuilder() // + .setContext(Storage.RequestContext.newBuilder().setDatabase(req.getReqCtx().getDatabase()).build()) // + .addAllTables(req.getTables()) // + .setSql(req.getSql()) // .build(); - this.routerClient.invokeServerStreaming(endpoint, request, ctx, new Observer() { + this.routerClient.invokeServerStreaming(endpoint, request, ctx, new Observer() { @Override - public void onNext(final Storage.QueryResponse value) { - final Result ret = Utils.toResult(value, req.getQl(), endpoint, req.getMetrics(), + public void onNext(final Storage.SqlQueryResponse value) { + final Result ret = Utils.toResult(value, req.getSql(), endpoint, req.getTables(), new ErrHandler(req)); if (ret.isOk()) { observer.onNext(ret.getOk()); @@ -277,18 +290,18 @@ public DefaultQueryLimiter(int maxInFlight, LimitedPolicy policy) { } @Override - public int calculatePermits(final QueryRequest request) { + public int calculatePermits(final SqlQueryRequest request) { return 1; } @Override - public Result rejected(final QueryRequest request, final RejectedState state) { + public Result rejected(final SqlQueryRequest request, final RejectedState state) { final String errMsg = String.format( "Query limited by client, acquirePermits=%d, maxPermits=%d, availablePermits=%d.", // state.acquirePermits(), // state.maxPermits(), // state.availablePermits()); - return Result.err(Err.queryErr(Result.FLOW_CONTROL, errMsg, null, request.getQl(), request.getMetrics())); + return Result.err(Err.queryErr(Result.FLOW_CONTROL, errMsg, null, request.getSql(), request.getTables())); } } } diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/QueryLimiter.java b/ceresdb-protocol/src/main/java/io/ceresdb/QueryLimiter.java deleted file mode 100644 index 7f91fc4..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/QueryLimiter.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb; - -import io.ceresdb.models.Err; -import io.ceresdb.models.QueryOk; -import io.ceresdb.models.QueryRequest; -import io.ceresdb.models.Result; - -/** - * Like rust: pub type QueryLimiter = CeresDBLimiter> - * - * @author jiachun.fjc - */ -public abstract class QueryLimiter extends CeresDBLimiter> { - - public QueryLimiter(int maxInFlight, LimitedPolicy policy, String metricPrefix) { - super(maxInFlight, policy, metricPrefix); - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/RecordIterator.java b/ceresdb-protocol/src/main/java/io/ceresdb/RecordIterator.java deleted file mode 100644 index cde91f3..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/RecordIterator.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb; - -import java.util.Iterator; - -import io.ceresdb.models.Record; - -/** - * - * @author jiachun.fjc - */ -public class RecordIterator implements Iterator { - - private final BlockingStreamIterator streams; - private Iterator current; - - public RecordIterator(BlockingStreamIterator streams) { - this.streams = streams; - } - - @Override - public boolean hasNext() { - if (this.current != null && this.current.hasNext()) { - return true; - } - while (this.streams.hasNext()) { - this.current = this.streams.next().iterator(); - if (this.current.hasNext()) { - return true; - } - } - return false; - } - - @Override - public Record next() { - return this.current.next(); - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/RecordMapper.java b/ceresdb-protocol/src/main/java/io/ceresdb/RecordMapper.java deleted file mode 100644 index 41aa058..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/RecordMapper.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb; - -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; - -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.util.Utf8; - -import io.ceresdb.common.OptKeys; -import io.ceresdb.common.util.SystemPropertyUtil; -import io.ceresdb.models.Record; - -/** - * Map byte[] to Record by avro. - * - * @author jiachun.fjc - */ -public class RecordMapper extends AvroMapper implements Function { - - private static final boolean NAME_VALIDATE = SystemPropertyUtil.getBool(OptKeys.NAME_VALIDATE, false); - - RecordMapper(Schema schema) { - super(schema); - } - - public static RecordMapper getMapper(final String s) { - final Schema.Parser parser = new Schema.Parser(); - parser.setValidate(NAME_VALIDATE); - final Schema schema = parser.parse(s); - return new RecordMapper(schema); - } - - @Override - public Record apply(final byte[] bytes) { - return new DefaultRecord(mapTo(bytes)); - } - - static final class DefaultRecord implements Record { - - private final GenericRecord gr; - - DefaultRecord(GenericRecord gr) { - this.gr = gr; - } - - @Override - public Object get(final String key) { - return mayCast(this.gr.get(key)); - } - - @Override - public Object get(final int i) { - return mayCast(this.gr.get(i)); - } - - @Override - public boolean hasField(final String key) { - return this.gr.hasField(key); - } - - @Override - public int getFieldCount() { - return this.gr.getSchema().getFields().size(); - } - - @Override - public List getFieldDescriptors() { - return this.gr.getSchema() // - .getFields() // - .stream() // - .map(DefaultRecord::parseFd) // - .collect(Collectors.toList()); - } - - @Override - public Optional getFieldDescriptor(final String field) { - return Optional.ofNullable(parseFd(this.gr.getSchema().getField(field))); - } - - private static FieldDescriptor parseFd(final Schema.Field f) { - if (f == null) { - return null; - } - final Schema sc = f.schema(); - final FieldType type = parseType(sc); - final List subTypes = parseSubTypes(type.getType(), sc); - return new FieldDescriptor(f.name(), type, subTypes); - } - - private static FieldType parseType(final Schema sc) { - final LogicalType logicalType = parseLogicalType(sc.getLogicalType()); - switch (sc.getType()) { - case NULL: - return FieldType.of(Type.Null, logicalType); - case DOUBLE: - return FieldType.of(Type.Double, logicalType); - case FLOAT: - return FieldType.of(Type.Float, logicalType); - case BYTES: - return FieldType.of(Type.Bytes, logicalType); - case STRING: - return FieldType.of(Type.String, logicalType); - case LONG: - return FieldType.of(Type.Long, logicalType); - case INT: - return FieldType.of(Type.Int, logicalType); - case BOOLEAN: - return FieldType.of(Type.Boolean, logicalType); - case UNION: - return FieldType.of(Type.Union, logicalType); - default: - return FieldType.of(Type.Unknown, logicalType); - } - } - - private static List parseSubTypes(final Type type, final Schema schema) { - return type != Type.Union // - ? Collections.emptyList() // - : schema.getTypes().stream().map(DefaultRecord::parseType).collect(Collectors.toList()); - } - - private static LogicalType parseLogicalType(final org.apache.avro.LogicalType lt) { - if (lt == null) { - return LogicalType.Null; - } - - if (lt instanceof LogicalTypes.TimestampMillis) { - return LogicalType.TimestampMillis; - } - - if (lt instanceof LogicalTypes.TimestampMicros) { - return LogicalType.TimestampMicros; - } - - return LogicalType.Unknown; - } - - private static Object mayCast(final Object obj) { - if (obj instanceof Utf8) { - return ((Utf8) obj).toString(); - } - if (obj instanceof ByteBuffer) { - return getBytes((ByteBuffer) obj); - } - return obj; - } - - private static byte[] getBytes(final ByteBuffer buf) { - if (buf.hasArray()) { - return buf.array(); - } - final byte[] bytes = new byte[buf.remaining()]; - buf.get(bytes); - return bytes; - } - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/Route.java b/ceresdb-protocol/src/main/java/io/ceresdb/Route.java index 1664925..5834bc8 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/Route.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/Route.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; @@ -24,10 +11,9 @@ /** * Route info for metric. * - * @author jiachun.fjc */ public class Route { - private String metric; + private String table; private Endpoint endpoint; private Object ext; private final AtomicLong lastHit = new AtomicLong(Clock.defaultClock().getTick()); @@ -46,18 +32,18 @@ public static Route of(final String metric, final Endpoint endpoint) { public static Route of(final String metric, final Endpoint endpoint, final Object ext) { final Route r = new Route(); - r.metric = metric; + r.table = metric; r.endpoint = endpoint; r.ext = ext; return r; } - public String getMetric() { - return metric; + public String getTable() { + return table; } - public void setMetric(String metric) { - this.metric = metric; + public void setTable(String table) { + this.table = table; } public Endpoint getEndpoint() { @@ -90,7 +76,7 @@ public void tryWeekSetHit(final long lastHit) { @Override public String toString() { return "Route{" + // - "metric='" + metric + '\'' + // + "table='" + table + '\'' + // ", endpoint=" + endpoint + // ", ext=" + ext + // ", lastHit=" + lastHit.get() + // diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/RouteMode.java b/ceresdb-protocol/src/main/java/io/ceresdb/RouteMode.java index 605bca5..5a4297a 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/RouteMode.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/RouteMode.java @@ -1,35 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; -/** - * route mode - * @author lee - * @version : RouteMode.java, v 0.1 2023.01.17 14:23 lee Exp $ - */ public enum RouteMode { /** * In this mode, the client does not cache routing information and each request is proxied through the server to the correct server */ - STANDALONE, + PROXY, /** * In this mode, the client cache routing information. Client find the correct server firstly, and then request to the correct server directly. */ - CLUSTER + DIRECT } \ No newline at end of file diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/Router.java b/ceresdb-protocol/src/main/java/io/ceresdb/Router.java index edca99f..20b9dd1 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/Router.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/Router.java @@ -1,27 +1,15 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; import java.util.concurrent.CompletableFuture; +import io.ceresdb.models.RequestContext; + /** * A RPC router for CeresDB. * - * @author jiachun.fjc */ public interface Router { @@ -31,5 +19,5 @@ public interface Router { * @param request request * @return a endpoint for the call */ - CompletableFuture routeFor(final Req request); + CompletableFuture routeFor(final RequestContext reqCtx, final Req request); } diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/RouterClient.java b/ceresdb-protocol/src/main/java/io/ceresdb/RouterClient.java index 69a0dd0..20a6e11 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/RouterClient.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/RouterClient.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; @@ -26,12 +13,11 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; +import io.ceresdb.models.RequestContext; import io.ceresdb.proto.internal.Storage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,7 +30,6 @@ import io.ceresdb.common.util.MetricsUtil; import io.ceresdb.common.util.Requires; import io.ceresdb.common.util.SharedScheduledPool; -import io.ceresdb.common.util.Spines; import io.ceresdb.common.util.TopKSelector; import io.ceresdb.errors.RouteTableException; import io.ceresdb.options.RouterOptions; @@ -52,14 +37,17 @@ import io.ceresdb.rpc.Observer; import io.ceresdb.rpc.RpcClient; import io.ceresdb.rpc.errors.RemotingException; +import io.ceresdb.util.Utils; + import com.codahale.metrics.Histogram; import com.codahale.metrics.Timer; /** - * A route rpc client which cached the routing table information locally + * A route rpc client which implement RouteMode.Direct + * + * cached the routing table information locally * and will refresh when the server returns an error code of INVALID_ROUTE * - * @author jiachun.fjc */ public class RouterClient implements Lifecycle, Display, Iterable { @@ -70,20 +58,16 @@ public class RouterClient implements Lifecycle, Display, Iterable private static final float CLEAN_THRESHOLD = 0.1f; private static final int MAX_CONTINUOUS_GC_TIMES = 3; - private static final int ITEM_COUNT_EACH_REFRESH = 512; - private static final long BLOCKING_ROUTE_TIMEOUT_MS = 3000; - private static final SharedScheduledPool CLEANER_POOL = Utils.getSharedScheduledPool("route_cache_cleaner", 1); private static final SharedScheduledPool REFRESHER_POOL = Utils.getSharedScheduledPool("route_cache_refresher", Math.min(4, Cpus.cpus())); private ScheduledExecutorService cleaner; private ScheduledExecutorService refresher; - - protected RouterOptions opts; - protected RpcClient rpcClient; - protected RouterByMetrics router; - protected InnerMetrics metrics; + protected RouterOptions opts; + protected RpcClient rpcClient; + protected RouterByTables router; + protected InnerMetrics metrics; private final ConcurrentMap routeCache = new ConcurrentHashMap<>(); @@ -93,16 +77,14 @@ static final class InnerMetrics { final Histogram gcTimes; final Histogram gcItems; final Timer gcTimer; - final Timer refreshTimer; private InnerMetrics(final Endpoint name) { final String nameSuffix = name.toString(); - this.refreshedSize = MetricsUtil.histogram("route_for_metrics_refreshed_size", nameSuffix); - this.cachedSize = MetricsUtil.histogram("route_for_metrics_cached_size", nameSuffix); - this.gcTimes = MetricsUtil.histogram("route_for_metrics_gc_times", nameSuffix); - this.gcItems = MetricsUtil.histogram("route_for_metrics_gc_items", nameSuffix); - this.gcTimer = MetricsUtil.timer("route_for_metrics_gc_timer", nameSuffix); - this.refreshTimer = MetricsUtil.timer("route_for_metrics_refresh_timer", nameSuffix); + this.refreshedSize = MetricsUtil.histogram("route_for_tables_refreshed_size", nameSuffix); + this.cachedSize = MetricsUtil.histogram("route_for_tables_cached_size", nameSuffix); + this.gcTimes = MetricsUtil.histogram("route_for_tables_gc_times", nameSuffix); + this.gcItems = MetricsUtil.histogram("route_for_tables_gc_items", nameSuffix); + this.gcTimer = MetricsUtil.timer("route_for_tables_gc_timer", nameSuffix); } Histogram refreshedSize() { @@ -124,10 +106,6 @@ Histogram gcItems() { Timer gcTimer() { return this.gcTimer; } - - Timer refreshTimer() { - return this.refreshTimer; - } } @Override @@ -137,7 +115,7 @@ public boolean init(final RouterOptions opts) { final Endpoint address = Requires.requireNonNull(this.opts.getClusterAddress(), "Null.clusterAddress"); - this.router = new RouterByMetrics(address); + this.router = new RouterByTables(address); this.metrics = new InnerMetrics(address); final long gcPeriod = this.opts.getGcPeriodSeconds(); @@ -148,15 +126,6 @@ public boolean init(final RouterOptions opts) { LOG.info("Route table cache cleaner has been started."); } - final long refreshPeriod = this.opts.getRefreshPeriodSeconds(); - if (refreshPeriod > 0) { - this.refresher = REFRESHER_POOL.getObject(); - this.refresher.scheduleWithFixedDelay(this::refresh, Utils.randomInitialDelay(180), refreshPeriod, - TimeUnit.SECONDS); - - LOG.info("Route table cache refresher has been started."); - } - return true; } @@ -185,20 +154,21 @@ public Route clusterRoute() { return Route.of(this.opts.getClusterAddress()); } - public CompletableFuture> routeFor(final Collection metrics) { - if (metrics == null || metrics.isEmpty()) { + public CompletableFuture> routeFor(final RequestContext reqCtx, + final Collection tables) { + if (tables == null || tables.isEmpty()) { return Utils.completedCf(Collections.emptyMap()); } final Map local = new HashMap<>(); final List misses = new ArrayList<>(); - metrics.forEach(metric -> { - final Route r = this.routeCache.get(metric); + tables.forEach(table -> { + final Route r = this.routeCache.get(table); if (r == null) { - misses.add(metric); + misses.add(table); } else { - local.put(metric, r); + local.put(table, r); } }); @@ -206,7 +176,7 @@ public CompletableFuture> routeFor(final Collection m return Utils.completedCf(local); } - return routeRefreshFor(misses) // refresh from remote + return routeRefreshFor(reqCtx, misses) // refresh from remote .thenApply(remote -> { // then merge result final Map ret; if (remote.size() > local.size()) { @@ -225,35 +195,27 @@ public CompletableFuture> routeFor(final Collection m }); } - public CompletableFuture> routeRefreshFor(final Collection metrics) { + public CompletableFuture> routeRefreshFor(final RequestContext reqCtx, + final Collection tables) { final long startCall = Clock.defaultClock().getTick(); - return this.router.routeFor(metrics).whenComplete((remote, err) -> { + return this.router.routeFor(reqCtx, tables).whenComplete((remote, err) -> { if (err == null) { this.routeCache.putAll(remote); this.metrics.refreshedSize().update(remote.size()); this.metrics.cachedSize().update(this.routeCache.size()); - this.metrics.refreshTimer().update(Clock.defaultClock().duration(startCall), TimeUnit.MILLISECONDS); - LOG.info("Route refreshed: {}, cached_size={}.", metrics, this.routeCache.size()); + LOG.info("Route refreshed: {}, cached_size={}.", tables, this.routeCache.size()); } else { - LOG.warn("Route refresh failed: {}.", metrics, err); + LOG.warn("Route refresh failed: {}.", tables, err); } }); } - private void blockingRouteRefreshFor(final Collection metrics) { - try { - routeRefreshFor(metrics).get(BLOCKING_ROUTE_TIMEOUT_MS, TimeUnit.MILLISECONDS); - } catch (final InterruptedException | ExecutionException | TimeoutException e) { - LOG.error("Fail to blocking refresh route.", e); - } - } - - public void clearRouteCacheBy(final Collection metrics) { - if (metrics == null || metrics.isEmpty()) { + public void clearRouteCacheBy(final Collection tables) { + if (tables == null || tables.isEmpty()) { return; } - metrics.forEach(this.routeCache::remove); + tables.forEach(this.routeCache::remove); } public int clearRouteCache() { @@ -262,28 +224,6 @@ public int clearRouteCache() { return size; } - public void refresh() { - final Collection cachedKeys = this.routeCache.keySet(); - - if (cachedKeys.size() <= ITEM_COUNT_EACH_REFRESH) { - blockingRouteRefreshFor(cachedKeys); - return; - } - - final Collection keysToRefresh = Spines.newBuf(ITEM_COUNT_EACH_REFRESH); - for (final String metric : cachedKeys) { - keysToRefresh.add(metric); - if (keysToRefresh.size() >= ITEM_COUNT_EACH_REFRESH) { - blockingRouteRefreshFor(keysToRefresh); - keysToRefresh.clear(); - } - } - - if (!keysToRefresh.isEmpty()) { - blockingRouteRefreshFor(keysToRefresh); - } - } - public void gc() { this.metrics.gcTimer().time(() -> this.metrics.gcTimes().update(gc0(0))); } @@ -414,28 +354,31 @@ public String toString() { '}'; } - private class RouterByMetrics implements Router, Map> { + private class RouterByTables implements Router, Map> { private final Endpoint endpoint; - private RouterByMetrics(Endpoint endpoint) { + private RouterByTables(Endpoint endpoint) { this.endpoint = endpoint; } @Override - public CompletableFuture> routeFor(final Collection request) { - if (request == null || request.isEmpty()) { + public CompletableFuture> routeFor(final RequestContext reqCtx, + final Collection tables) { + if (tables == null || tables.isEmpty()) { return Utils.completedCf(Collections.emptyMap()); } - final Storage.RouteRequest req = Storage.RouteRequest.newBuilder().addAllMetrics(request).build(); + final Storage.RouteRequest req = Storage.RouteRequest.newBuilder() + .setContext(Storage.RequestContext.newBuilder().setDatabase(reqCtx.getDatabase()).build()) + .addAllTables(tables).build(); final Context ctx = Context.of("call_priority", "100"); // Mysterious trick!!! ＼（＾▽＾）／ final CompletableFuture f = invokeRpc(req, ctx); return f.thenCompose(resp -> { if (Utils.isSuccess(resp.getHeader())) { final Map ret = resp.getRoutesList().stream() - .collect(Collectors.toMap(Storage.Route::getMetric, this::toRouteObj)); + .collect(Collectors.toMap(Storage.Route::getTable, this::toRouteObj)); return Utils.completedCf(ret); } @@ -465,7 +408,7 @@ private CompletableFuture invokeRpc(final Storage.RouteRe private Route toRouteObj(final Storage.Route r) { final Storage.Endpoint ep = Requires.requireNonNull(r.getEndpoint(), "CeresDB.Endpoint"); - return Route.of(r.getMetric(), Endpoint.of(ep.getIp(), ep.getPort()), r.getExt()); + return Route.of(r.getTable(), Endpoint.of(ep.getIp(), ep.getPort()), r.getExt()); } } } diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/RowIterator.java b/ceresdb-protocol/src/main/java/io/ceresdb/RowIterator.java new file mode 100644 index 0000000..cee589b --- /dev/null +++ b/ceresdb-protocol/src/main/java/io/ceresdb/RowIterator.java @@ -0,0 +1,37 @@ +/* + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + */ +package io.ceresdb; + +import java.util.Iterator; + +import io.ceresdb.models.Row; + +public class RowIterator implements Iterator { + + private final BlockingStreamIterator streams; + private Iterator current; + + public RowIterator(BlockingStreamIterator streams) { + this.streams = streams; + } + + @Override + public boolean hasNext() { + if (this.current != null && this.current.hasNext()) { + return true; + } + while (this.streams.hasNext()) { + this.current = this.streams.next().iterator(); + if (this.current.hasNext()) { + return true; + } + } + return false; + } + + @Override + public Row next() { + return this.current.next(); + } +} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/StandaloneRouterClient.java b/ceresdb-protocol/src/main/java/io/ceresdb/StandaloneRouterClient.java deleted file mode 100644 index 4f6e0cb..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/StandaloneRouterClient.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CompletableFuture; - -/** - * A route rpc client which cached nothing about table information, and return - * clusterAddress directly - * @author lee - */ -public class StandaloneRouterClient extends RouterClient { - - public CompletableFuture> routeFor(final Collection metrics) { - if (metrics == null || metrics.isEmpty()) { - return Utils.completedCf(Collections.emptyMap()); - } - - final Map routeMap = new HashMap<>(); - - metrics.forEach(metric -> { - Route route = new Route(); - route.setEndpoint(this.opts.getClusterAddress()); - route.setMetric(metric); - routeMap.put(metric, route); - }); - return Utils.completedCf(routeMap); - - } - -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/Utils.java b/ceresdb-protocol/src/main/java/io/ceresdb/Utils.java deleted file mode 100644 index cbd3803..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/Utils.java +++ /dev/null @@ -1,460 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb; - -import java.io.IOException; -import java.util.Collection; -import java.util.Iterator; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import io.ceresdb.common.Display; -import io.ceresdb.common.Endpoint; -import io.ceresdb.common.OptKeys; -import io.ceresdb.common.util.ExecutorServiceHelper; -import io.ceresdb.common.util.MetricsUtil; -import io.ceresdb.common.util.NamedThreadFactory; -import io.ceresdb.common.util.ObjectPool; -import io.ceresdb.common.util.Requires; -import io.ceresdb.common.util.SharedScheduledPool; -import io.ceresdb.common.util.Spines; -import io.ceresdb.common.util.SystemPropertyUtil; -import io.ceresdb.common.util.ThreadPoolUtil; -import io.ceresdb.models.Err; -import io.ceresdb.models.FieldValue; -import io.ceresdb.models.Keyword; -import io.ceresdb.models.QueryOk; -import io.ceresdb.models.Result; -import io.ceresdb.models.Rows; -import io.ceresdb.models.Schema; -import io.ceresdb.models.TagValue; -import io.ceresdb.models.Value; -import io.ceresdb.models.WriteOk; -import io.ceresdb.proto.internal.Common; -import io.ceresdb.proto.internal.Storage; -import io.ceresdb.rpc.Observer; -import com.google.protobuf.ByteStringHelper; - -/** - * Utils for CeresDBClient. - * - * @author jiachun.fjc - */ -public final class Utils { - - public static final String DB_NAME = "CeresDB"; - - private static final AtomicBoolean RW_LOGGING; - - private static final int REPORT_PERIOD_MIN; - private static final ScheduledExecutorService DISPLAY; - - static { - RW_LOGGING = new AtomicBoolean(SystemPropertyUtil.getBool(OptKeys.RW_LOGGING, true)); - REPORT_PERIOD_MIN = SystemPropertyUtil.getInt(OptKeys.REPORT_PERIOD, 30); - DISPLAY = ThreadPoolUtil.newScheduledBuilder().poolName("display_self") // - .coreThreads(1) // - .enableMetric(true) // - .threadFactory(new NamedThreadFactory("display_self", true)) // - .rejectedHandler(new ThreadPoolExecutor.DiscardOldestPolicy()) // - .build(); - - Runtime.getRuntime() - .addShutdownHook(new Thread(() -> ExecutorServiceHelper.shutdownAndAwaitTermination(DISPLAY))); - } - - /** - * Whether to output concise read/write logs. - * - * @return true or false - */ - public static boolean isRwLogging() { - return RW_LOGGING.get(); - } - - /** - * See {@link #isRwLogging()} - * - * Reset `rwLogging`, set to the opposite of the old value. - * - * @return old value - */ - public static boolean resetRwLogging() { - return RW_LOGGING.getAndSet(!RW_LOGGING.get()); - } - - /** - * Auto report self(CeresDBClient) period. - * - * @return period with minutes - */ - public static int autoReportPeriodMin() { - return REPORT_PERIOD_MIN; - } - - /** - * Only used to schedule to display the self of client. - * - * @param display display - * @param printer to print the display info - */ - public static void scheduleDisplaySelf(final Display display, final Display.Printer printer) { - DISPLAY.scheduleWithFixedDelay(() -> display.display(printer), 0, autoReportPeriodMin(), TimeUnit.MINUTES); - } - - /** - * Create a shared scheduler pool with the given name. - * - * @param name scheduled pool's name - * @param workers the num of workers - * @return new scheduler poll instance - */ - public static SharedScheduledPool getSharedScheduledPool(final String name, final int workers) { - return new SharedScheduledPool(new ObjectPool.Resource() { - - @Override - public ScheduledExecutorService create() { - return ThreadPoolUtil.newScheduledBuilder() // - .poolName(name) // - .coreThreads(workers) // - .enableMetric(true) // - .threadFactory(new NamedThreadFactory(name, true)) // - .rejectedHandler(new ThreadPoolExecutor.DiscardOldestPolicy()) // - .build(); - } - - @Override - public void close(final ScheduledExecutorService instance) { - ExecutorServiceHelper.shutdownAndAwaitTermination(instance); - } - }); - } - - /** - * Merge two given {@link Result} objects. If both Result objects are - * {@link WriteOk} or {@link Err}, then merge their results. If one is - * {@link Err} and the other is {@link WriteOk}, then we will discard - * the {@link WriteOk}. - * - * @param r1 the result - * @param r2 the other result - * @return merged result - */ - public static Result combineResult(final Result r1, final Result r2) { - if (r1.isOk() && r2.isOk()) { - return r1.getOk().combine(r2.getOk()).mapToResult(); - } else if (!r1.isOk() && !r2.isOk()) { - return r1.getErr().combine(r2.getErr()).mapToResult(); - } else { - if (r1.isOk()) { - return r2.getErr().combine(r1.getOk()).mapToResult(); - } else { - return r1.getErr().combine(r2.getOk()).mapToResult(); - } - } - } - - /** - * Converts the given {@link Storage.WriteResponse} to {@link Result} that - * upper-level readable. - * - * @param resp response of the write RPC - * @param to the server address wrote to - * @param rows wrote date in this write - * @return a {@link Result} - */ - public static Result toResult(final Storage.WriteResponse resp, // - final Endpoint to, // - final Collection rows) { - final Common.ResponseHeader header = resp.getHeader(); - final int code = header.getCode(); - final String msg = header.getError(); - final int success = resp.getSuccess(); - final int failed = resp.getFailed(); - - if (code == Result.SUCCESS) { - final Set metrics = rows != null && WriteOk.isCollectWroteDetail() ? - rows.stream().map(Rows::getMetric).collect(Collectors.toSet()) : - null; - return WriteOk.ok(success, failed, metrics).mapToResult(); - } else { - return Err.writeErr(code, msg, to, rows).mapToResult(); - } - } - - /** - * Converts the given {@link Storage.QueryResponse} to {@link Result} that - * upper-level readable. - * - * @param resp response of the write RPC - * @param to the server address wrote to - * @param metrics the metrics who query failed - * @param errHandler the error handler - * @return a {@link Result} - */ - public static Result toResult(final Storage.QueryResponse resp, // - final String ql, // - final Endpoint to, // - final Collection metrics, final Runnable errHandler) { - final Common.ResponseHeader header = resp.getHeader(); - final int code = header.getCode(); - final String msg = header.getError(); - - if (code == Result.SUCCESS) { - final int rowCount = resp.getRowsCount(); - final Stream rows = resp.getRowsList().stream().map(ByteStringHelper::sealByteArray); - return QueryOk.ok(ql, toSchema(resp), rowCount, rows).mapToResult(); - } else { - if (errHandler != null) { - errHandler.run(); - } - return Err.queryErr(code, msg, to, ql, metrics).mapToResult(); - } - } - - private static Schema toSchema(final Storage.QueryResponse resp) { - final Storage.QueryResponse.SchemaType type = resp.getSchemaType(); - final String content = resp.getSchemaContent(); - switch (type) { - case AVRO: - return Schema.schema(Schema.Type.Avro, content); - case JSON: - return Schema.schema(Schema.Type.Json, null); - case UNRECOGNIZED: - default: - throw new IllegalArgumentException("Unrecognized schema type"); - } - } - - /** - * Determine whether the request was successful from the information in the - * response header. - * - * @param header response header - * @return true if response is success - */ - public static boolean isSuccess(final Common.ResponseHeader header) { - return header.getCode() == Result.SUCCESS; - } - - /** - * Returns a new CompletableFuture that is already completed with the given - * value. Same as {@link CompletableFuture#completedFuture(Object)}, only - * rename the method. - * - * @param value the given value - * @param the type of the value - * @return the completed {@link CompletableFuture} - */ - public static CompletableFuture completedCf(final U value) { - return CompletableFuture.completedFuture(value); - } - - /** - * Returns a new CompletableFuture that is already exceptionally with the given - * error. - * - * @param t the given exception - * @param the type of the value - * @return the exceptionally {@link CompletableFuture} - */ - public static CompletableFuture errorCf(final Throwable t) { - final CompletableFuture err = new CompletableFuture<>(); - err.completeExceptionally(t); - return err; - } - - /** - * Break data stream into multiple requests based on the routing table - * information given. - * - * @param data the data to split - * @param routes the route table info - * @return multi data stream - */ - public static Map> splitDataByRoute(final Collection data, // - final Map routes) { - final Map> splits = routes.values() // - .stream() // - .map(Route::getEndpoint) // - .distinct() // - .collect(Collectors.toMap(k -> k, k -> Spines.newBuf(), (v1, v2) -> v1)); - if (splits.size() == 1) { - // fast path, zero copy - splits.replaceAll((ep, empty) -> data); - } else { - data.forEach(rs -> { - final Route route = routes.get(rs.getMetric()); - Requires.requireNonNull(route, "Null.route for " + rs); - final Collection partOf = splits.get(route.getEndpoint()); - Requires.requireNonNull(route, "Invalid.route " + route); - partOf.add(rs); - }); - } - - MetricsUtil.histogram("split_num_per_write").update(splits.size()); - - return splits; - } - - public static boolean shouldNotRetry(final Err err) { - return !shouldRetry(err); - } - - public static boolean shouldRetry(final Err err) { - if (err == null) { - return false; - } - final int errCode = err.getCode(); - return errCode == Result.INVALID_ROUTE || errCode == Result.SHOULD_RETRY; - } - - public static boolean shouldRefreshRouteTable(final Err err) { - return err.getCode() == Result.INVALID_ROUTE; - } - - public static Observer toUnaryObserver(final CompletableFuture future) { - return new Observer() { - - @Override - public void onNext(final V value) { - future.complete(value); - } - - @Override - public void onError(final Throwable err) { - future.completeExceptionally(err); - } - }; - } - - public static Storage.Value toProtoValue(final FieldValue field) { - final Storage.Value.Builder vb = Storage.Value.newBuilder(); - switch (field.getType()) { - case Float64: - return vb.setFloat64Value(field.getFloat64()).build(); - case String: - return vb.setStringValue(field.getString()).build(); - case Int64: - return vb.setInt64Value(field.getInt64()).build(); - case Float32: - return vb.setFloat32Value(field.getFloat32()).build(); - case Int32: - return vb.setInt32Value(field.getInt32()).build(); - case Int16: - return vb.setInt16Value(field.getInt16()).build(); - case Int8: - return vb.setInt8Value(field.getInt8()).build(); - case Boolean: - return vb.setBoolValue(field.getBoolean()).build(); - case UInt64: - return vb.setUint64Value(field.getUInt64()).build(); - case UInt32: - return vb.setUint32Value(field.getUInt32()).build(); - case UInt16: - return vb.setUint16Value(field.getUInt16()).build(); - case UInt8: - return vb.setUint8Value(field.getUInt8()).build(); - case Timestamp: - return vb.setTimestampValue(field.getTimestamp()).build(); - case Varbinary: - return vb.setVarbinaryValue(ByteStringHelper.wrap(field.getVarbinary())).build(); - default: - return invalidType(field); - } - } - - public static Storage.Value toProtoValue(final TagValue tag) { - final Storage.Value.Builder vb = Storage.Value.newBuilder(); - switch (tag.getType()) { - case String: - return vb.setStringValue(tag.getString()).build(); - case Int64: - return vb.setInt64Value(tag.getInt64()).build(); - case Int32: - return vb.setInt32Value(tag.getInt32()).build(); - case Int16: - return vb.setInt16Value(tag.getInt16()).build(); - case Int8: - return vb.setInt8Value(tag.getInt8()).build(); - case Boolean: - return vb.setBoolValue(tag.getBoolean()).build(); - case UInt64: - return vb.setUint64Value(tag.getUInt64()).build(); - case UInt32: - return vb.setUint32Value(tag.getUInt32()).build(); - case UInt16: - return vb.setUint16Value(tag.getUInt16()).build(); - case UInt8: - return vb.setUint8Value(tag.getUInt8()).build(); - case Timestamp: - return vb.setTimestampValue(tag.getTimestamp()).build(); - case Varbinary: - return vb.setVarbinaryValue(ByteStringHelper.wrap(tag.getVarbinary())).build(); - case Float32: - case Float64: - default: - return invalidType(tag); - } - } - - public static long randomInitialDelay(final long delay) { - return ThreadLocalRandom.current().nextLong(delay, delay << 1); - } - - public static Properties loadProperties(final ClassLoader loader, final String name) throws IOException { - final Properties prop = new Properties(); - prop.load(loader.getResourceAsStream(name)); - return prop; - } - - public static T unsupported(final String fmt, final Object... args) { - throw new UnsupportedOperationException(String.format(fmt, args)); - } - - public static void checkKeywords(final Iterator keys) { - if (keys == null) { - return; - } - - while (keys.hasNext()) { - ensureNotKeyword(keys.next()); - } - } - - private static void ensureNotKeyword(final String name) { - if (Keyword.isKeyword(name)) { - throw new IllegalArgumentException("Invalid name, not allow keyword `" + name + '`'); - } - } - - private static T invalidType(final Value value) { - throw new IllegalArgumentException("Invalid type " + value); - } - - private Utils() { - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/Write.java b/ceresdb-protocol/src/main/java/io/ceresdb/Write.java index 841264c..ad95666 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/Write.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/Write.java @@ -1,86 +1,59 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; -import java.util.Collection; -import java.util.Collections; import java.util.concurrent.CompletableFuture; import io.ceresdb.models.Err; +import io.ceresdb.models.Point; +import io.ceresdb.models.RequestContext; import io.ceresdb.models.Result; -import io.ceresdb.models.Rows; import io.ceresdb.models.WriteOk; +import io.ceresdb.models.WriteRequest; import io.ceresdb.rpc.Context; +import io.ceresdb.util.StreamWriteBuf; /** * CeresDB write API. Writes the streaming data to the database, support * failed retries. * - * @author jiachun.fjc */ public interface Write { /** - * @see #write(Rows, Context) + * @see #write(WriteRequest, Context) */ - default CompletableFuture> write(final Rows data) { - return write(Collections.singletonList(data), Context.newDefault()); + default CompletableFuture> write(final WriteRequest req) { + return write(req, Context.newDefault()); } /** - * Write a single series data to database. + * Write the data stream to the database. * - * @param data rows - * @param ctx the invoke context + * @param req write request + * @param ctx the invoked context * @return write result */ - default CompletableFuture> write(final Rows data, final Context ctx) { - return write(Collections.singletonList(data), ctx); - } + CompletableFuture> write(final WriteRequest req, final Context ctx); /** - * @see #write(Collection, Context) + * @see #streamWrite(RequestContext, String, Context) */ - default CompletableFuture> write(final Collection data) { - return write(data, Context.newDefault()); + default StreamWriteBuf streamWrite(final String table) { + return streamWrite(null, table, Context.newDefault()); } - /** - * Write the data stream to the database. - * - * @param data rows - * @param ctx the invoke context - * @return write result - */ - CompletableFuture> write(final Collection data, final Context ctx); - - /** - * @see #streamWrite(String, Context) - */ - default StreamWriteBuf streamWrite(final String metric) { - return streamWrite(metric, Context.newDefault()); + default StreamWriteBuf streamWrite(final RequestContext reqCtx, final String table) { + return streamWrite(reqCtx, table, Context.newDefault()); } /** * Executes a stream-write-call, returns a write request observer for streaming-write. * - * @param metric the metric to write - * @param ctx the invoke context + * @param table the table to write + * @param ctx the invoked context * @return a write request observer for streaming-write */ - StreamWriteBuf streamWrite(final String metric, final Context ctx); + StreamWriteBuf streamWrite(RequestContext reqCtx, final String table, final Context ctx); } diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/WriteClient.java b/ceresdb-protocol/src/main/java/io/ceresdb/WriteClient.java index 0861da0..20f3d76 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/WriteClient.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/WriteClient.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; @@ -43,22 +30,29 @@ import io.ceresdb.common.util.Spines; import io.ceresdb.common.util.Strings; import io.ceresdb.errors.StreamException; +import io.ceresdb.limit.LimitedPolicy; +import io.ceresdb.limit.WriteLimiter; import io.ceresdb.models.Err; +import io.ceresdb.models.Point; +import io.ceresdb.models.RequestContext; import io.ceresdb.models.Result; -import io.ceresdb.models.Rows; import io.ceresdb.models.Value; import io.ceresdb.models.WriteOk; +import io.ceresdb.models.WriteRequest; import io.ceresdb.options.WriteOptions; import io.ceresdb.proto.internal.Storage; import io.ceresdb.rpc.Context; import io.ceresdb.rpc.Observer; +import io.ceresdb.util.StreamWriteBuf; +import io.ceresdb.util.Utils; + import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; +import com.google.common.collect.Lists; /** * Default Write API impl. * - * @author jiachun.fjc */ public class WriteClient implements Write, Lifecycle, Display { @@ -70,22 +64,22 @@ public class WriteClient implements Write, Lifecycle, Display { private WriteLimiter writeLimiter; static final class InnerMetrics { - static final Histogram WRITE_ROWS_SUCCESS = MetricsUtil.histogram("write_rows_success_num"); - static final Histogram WRITE_ROWS_FAILED = MetricsUtil.histogram("write_rows_failed_num"); - static final Histogram METRICS_NUM_PER_WRITE = MetricsUtil.histogram("metrics_num_per_write"); - static final Meter WRITE_FAILED = MetricsUtil.meter("write_failed"); - static final Meter WRITE_QPS = MetricsUtil.meter("write_qps"); - - static Histogram writeRowsSuccess() { - return WRITE_ROWS_SUCCESS; + static final Histogram WRITE_POINTS_SUCCESS = MetricsUtil.histogram("write_points_success_num"); + static final Histogram WRITE_POINTS_FAILED = MetricsUtil.histogram("write_points_failed_num"); + static final Histogram POINTS_NUM_PER_WRITE = MetricsUtil.histogram("points_num_per_write"); + static final Meter WRITE_FAILED = MetricsUtil.meter("write_failed"); + static final Meter WRITE_QPS = MetricsUtil.meter("write_qps"); + + static Histogram writePointsSuccess() { + return WRITE_POINTS_SUCCESS; } - static Histogram writeRowsFailed() { - return WRITE_ROWS_FAILED; + static Histogram writePointsFailed() { + return WRITE_POINTS_FAILED; } - static Histogram metricsNumPerWrite() { - return METRICS_NUM_PER_WRITE; + static Histogram pointsNumPerWrite() { + return POINTS_NUM_PER_WRITE; } static Meter writeFailed() { @@ -108,7 +102,8 @@ public boolean init(final WriteOptions opts) { this.routerClient = this.opts.getRoutedClient(); final Executor pool = this.opts.getAsyncPool(); this.asyncPool = pool != null ? pool : new SerializingExecutor("write_client"); - this.writeLimiter = new DefaultWriteLimiter(this.opts.getMaxInFlightWriteRows(), this.opts.getLimitedPolicy()); + this.writeLimiter = new DefaultWriteLimiter(this.opts.getMaxInFlightWritePoints(), + this.opts.getLimitedPolicy()); return true; } @@ -118,54 +113,63 @@ public void shutdownGracefully() { } @Override - public CompletableFuture> write(final Collection data, final Context ctx) { - Requires.requireNonNull(data, "Null.data"); + public CompletableFuture> write(final WriteRequest req, final Context ctx) { + req.setReqCtx(attachRequestCtx(req.getReqCtx())); + + Requires.requireTrue(Strings.isNotBlank(req.getReqCtx().getDatabase()), "No database selected"); + Requires.requireNonNull(req.getPoints(), "Null.data"); + final long startCall = Clock.defaultClock().getTick(); - return this.writeLimiter.acquireAndDo(data, () -> write0(data, ctx, 0).whenCompleteAsync((r, e) -> { - InnerMetrics.writeQps().mark(); - if (r != null) { - if (Utils.isRwLogging()) { - LOG.info("Write to {}, duration={} ms, result={}.", Utils.DB_NAME, - Clock.defaultClock().duration(startCall), r); - } - if (r.isOk()) { - final WriteOk ok = r.getOk(); - InnerMetrics.writeRowsSuccess().update(ok.getSuccess()); - InnerMetrics.writeRowsFailed().update(ok.getFailed()); - return; - } - } - InnerMetrics.writeFailed().mark(); - }, this.asyncPool)); + return this.writeLimiter.acquireAndDo(req.getPoints(), + () -> write0(req.getReqCtx(), req.getPoints(), ctx, 0).whenCompleteAsync((r, e) -> { + InnerMetrics.writeQps().mark(); + if (r != null) { + if (Utils.isRwLogging()) { + LOG.info("Write to {}, duration={} ms, result={}.", Utils.DB_NAME, + Clock.defaultClock().duration(startCall), r); + } + if (r.isOk()) { + final WriteOk ok = r.getOk(); + InnerMetrics.writePointsSuccess().update(ok.getSuccess()); + InnerMetrics.writePointsFailed().update(ok.getFailed()); + return; + } + } + InnerMetrics.writeFailed().mark(); + }, this.asyncPool)); } @Override - public StreamWriteBuf streamWrite(final String metric, final Context ctx) { - Requires.requireTrue(Strings.isNotBlank(metric), "Blank.metric"); + public StreamWriteBuf streamWrite(final RequestContext reqCtx, final String table, + final Context ctx) { + final RequestContext finalReqCtx = attachRequestCtx(reqCtx); + + Requires.requireTrue(Strings.isNotBlank(finalReqCtx.getDatabase()), "No database selected"); + Requires.requireTrue(Strings.isNotBlank(table), "Blank.table"); final CompletableFuture respFuture = new CompletableFuture<>(); - return this.routerClient.routeFor(Collections.singleton(metric)) - .thenApply(routes -> routes.values().stream().findFirst().orElseGet(() -> Route.invalid(metric))) - .thenApply(route -> streamWriteTo(route, ctx, Utils.toUnaryObserver(respFuture))) - .thenApply(reqObserver -> new StreamWriteBuf() { + return this.routerClient.routeFor(reqCtx, Collections.singleton(table)) + .thenApply(routes -> routes.values().stream().findFirst().orElseGet(() -> Route.invalid(table))) + .thenApply(route -> streamWriteTo(route, finalReqCtx, ctx, Utils.toUnaryObserver(respFuture))) + .thenApply(reqObserver -> new StreamWriteBuf() { - private final Collection buf = Spines.newBuf(); + private final List buf = Spines.newBuf(); @Override - public StreamWriteBuf write(final Rows val) { + public StreamWriteBuf write(final Point val) { this.buf.add(val); return this; } @Override - public StreamWriteBuf write(final Collection c) { + public StreamWriteBuf write(final Collection c) { this.buf.addAll(c); return this; } @Override - public StreamWriteBuf flush() { + public StreamWriteBuf flush() { if (respFuture.isCompletedExceptionally()) { respFuture.getNow(null); // throw the exception now } @@ -177,7 +181,7 @@ public StreamWriteBuf flush() { } @Override - public StreamWriteBuf writeAndFlush(final Collection c) { + public StreamWriteBuf writeAndFlush(final Collection c) { flush(); // flush the previous write reqObserver.onNext(c.stream()); return this; @@ -192,23 +196,33 @@ public CompletableFuture completed() { }).join(); } - private CompletableFuture> write0(final Collection data, // + private RequestContext attachRequestCtx(RequestContext reqCtx) { + if (reqCtx == null) { + reqCtx = new RequestContext(); + } + if (Strings.isNullOrEmpty(reqCtx.getDatabase())) { + reqCtx.setDatabase(this.opts.getDatabase()); + } + return reqCtx; + } + + private CompletableFuture> write0(final RequestContext reqCtx, final List data, // final Context ctx, // final int retries) { InnerMetrics.writeByRetries(retries).mark(); - final Set metrics = data.stream() // - .map(Rows::getMetric) // + final Set tables = data.stream() // + .map(Point::getTable) // .collect(Collectors.toSet()); - InnerMetrics.metricsNumPerWrite().update(metrics.size()); + InnerMetrics.pointsNumPerWrite().update(tables.size()); // 1. Get routes - return this.routerClient.routeFor(metrics) + return this.routerClient.routeFor(reqCtx, tables) // 2. Split data by route info and write to DB .thenComposeAsync(routes -> Utils.splitDataByRoute(data, routes).entrySet().stream() // Write to database - .map(e -> writeTo(e.getKey(), e.getValue(), ctx.copy(), retries)) + .map(e -> writeTo(e.getKey(), reqCtx, e.getValue(), ctx.copy(), retries)) // Reduce and combine write result .reduce((f1, f2) -> f1.thenCombineAsync(f2, Utils::combineResult, this.asyncPool)) .orElse(Utils.completedCf(WriteOk.emptyOk().mapToResult())), this.asyncPool) @@ -230,26 +244,28 @@ private CompletableFuture> write0(final Collection da final Set toRefresh = err.stream() // .filter(Utils::shouldRefreshRouteTable) // .flatMap(e -> e.getFailedWrites().stream()) // - .map(Rows::getMetric) // + .map(Point::getTable) // .collect(Collectors.toSet()); - // Should retries - final List rowsToRetry = err.stream() // + // Should retry + final List pointsToRetry = err.stream() // .filter(Utils::shouldRetry) // .flatMap(e -> e.getFailedWrites().stream()) // .collect(Collectors.toList()); - // Should not retries + // Should not retry final Optional noRetryErr = err.stream() // .filter(Utils::shouldNotRetry) // .reduce(Err::combine); // Async refresh route info - final CompletableFuture> rwf = this.routerClient.routeRefreshFor(toRefresh) + final CompletableFuture> rwf = this.routerClient + .routeRefreshFor(reqCtx, toRefresh) // Even for some data that does not require a refresh of the routing table, // we still wait until the routing table is flushed successfully before // retrying it, in order to give the server a break. - .thenComposeAsync(routes -> write0(rowsToRetry, ctx, retries + 1), this.asyncPool); + .thenComposeAsync(routes -> write0(reqCtx, pointsToRetry, ctx, retries + 1), + this.asyncPool); return noRetryErr.isPresent() ? rwf.thenApplyAsync(ret -> Utils.combineResult(noRetryErr.get().mapToResult(), ret), @@ -260,29 +276,21 @@ private CompletableFuture> write0(final Collection da } private CompletableFuture> writeTo(final Endpoint endpoint, // - final Collection data, // + final RequestContext reqCtx, // + final List data, // final Context ctx, // final int retries) { // The cost is worth it - final int rowCount = data.stream() // - .map(Rows::getRowCount) // - .reduce(0, Integer::sum); + final int rowCount = data.size(); final int maxWriteSize = this.opts.getMaxWriteSize(); if (rowCount <= maxWriteSize) { - return writeTo0(endpoint, data, ctx, retries); + return writeTo0(endpoint, reqCtx, data, ctx, retries); } final Stream.Builder>> fs = Stream.builder(); - final PartBuf partBuf = new PartBuf(); - for (final Rows rs : data) { - final int rc = rs.getRowCount(); - if (partBuf.isNotEmpty() && partBuf.preAdd(rc) > maxWriteSize) { - fs.add(writeTo0(endpoint, partBuf.collectAndReset(), ctx.copy(), retries)); - } - partBuf.add(rs); - } - if (partBuf.isNotEmpty()) { - fs.add(writeTo0(endpoint, partBuf.collectAndReset(), ctx.copy(), retries)); + + for (List part : Lists.partition(data, maxWriteSize)) { + fs.add(writeTo0(endpoint, reqCtx, part, ctx.copy(), retries)); } return fs.build() // @@ -290,50 +298,23 @@ private CompletableFuture> writeTo(final Endpoint endpoint, .orElse(Utils.completedCf(WriteOk.emptyOk().mapToResult())); } - private static class PartBuf { - private Collection buf; - private int count; - - public void add(final Rows rs) { - if (this.buf == null) { - this.buf = Spines.newBuf(); - } - this.buf.add(rs); - this.count += rs.getRowCount(); - } - - public int preAdd(final int c) { - return this.count + c; - } - - public boolean isNotEmpty() { - return this.count > 0; - } - - public Collection collectAndReset() { - final Collection ret = this.buf; - // Cannot reuse the buf, outside will reference it until the response arrives. - this.buf = null; - this.count = 0; - return ret; - } - } - private CompletableFuture> writeTo0(final Endpoint endpoint, // - final Collection data, // + final RequestContext reqCtx, // + final List data, // final Context ctx, // final int retries) { final CompletableFuture wrf = this.routerClient.invoke(endpoint, // - toWriteRequestObj(data.stream()), // + toWriteRequestObj(reqCtx, data.stream()), // ctx.with("retries", retries) // server can use this in metrics ); return wrf.thenApplyAsync(resp -> Utils.toResult(resp, endpoint, data), this.asyncPool); } - private Observer> streamWriteTo(final Route route, // - final Context ctx, // - final Observer respObserver) { + private Observer> streamWriteTo(final Route route, // + final RequestContext reqCtx, // + final Context ctx, // + final Observer respObserver) { final Observer rpcObs = this.routerClient.invokeClientStreaming(route.getEndpoint(), // Storage.WriteRequest.getDefaultInstance(), // ctx, // @@ -360,21 +341,21 @@ public void onCompleted() { } }); - return new Observer>() { + return new Observer>() { - private final String metric = route.getMetric(); + private final String table = route.getTable(); @Override - public void onNext(final Stream value) { - final Stream data = value.filter(rs -> { - if (this.metric.equals(rs.getMetric())) { + public void onNext(final Stream value) { + final Stream data = value.filter(point -> { + if (this.table.equals(point.getTable())) { return true; } throw new StreamException( - String.format("Invalid metric %s, only can write %s.", rs.getMetric(), this.metric)); + String.format("Invalid table %s, only can write %s.", point.getTable(), this.table)); }); - rpcObs.onNext(toWriteRequestObj(data)); + rpcObs.onNext(toWriteRequestObj(reqCtx, data)); } @Override @@ -419,18 +400,20 @@ public String next() { } private static class WriteTuple3 { - Storage.WriteMetric.Builder wmcBui; - NameDict tagDict; - NameDict fieldDict; - - public WriteTuple3(String metric) { - this.wmcBui = Storage.WriteMetric.newBuilder().setMetric(metric); + String table; + Map seriesBuilders; + NameDict tagDict; + NameDict fieldDict; + + public WriteTuple3(String table) { + this.table = table; + this.seriesBuilders = new HashMap<>(); this.tagDict = new NameDict(); this.fieldDict = new NameDict(); } - public Storage.WriteMetric.Builder getWmcBui() { - return wmcBui; + public Map getSeriesBuilders() { + return seriesBuilders; } public NameDict getTagDict() { @@ -441,8 +424,15 @@ public NameDict getFieldDict() { return fieldDict; } - public Storage.WriteMetric build() { - return this.wmcBui // + public Storage.WriteTableRequest build() { + Storage.WriteTableRequest.Builder writeTableRequest = Storage.WriteTableRequest.newBuilder() + .setTable(table); + + seriesBuilders.forEach((key, builder) -> { + writeTableRequest.addEntries(builder.build()); + }); + + return writeTableRequest // .addAllTagNames(this.tagDict.toOrdered()) // .addAllFieldNames(this.fieldDict.toOrdered()) // .build(); @@ -450,45 +440,59 @@ public Storage.WriteMetric build() { } @VisibleForTest - public Storage.WriteRequest toWriteRequestObj(final Stream data) { - final Storage.WriteRequest.Builder wrBui = Storage.WriteRequest.newBuilder(); + public Storage.WriteRequest toWriteRequestObj(final RequestContext reqCtx, final Stream data) { + final Storage.WriteRequest.Builder writeRequestBuilder = Storage.WriteRequest.newBuilder(); final Map tuple3s = new HashMap<>(); - data.forEach(rs -> { - final String metric = rs.getMetric(); - final WriteTuple3 tp3 = tuple3s.computeIfAbsent(metric, WriteTuple3::new); - final Storage.WriteEntry.Builder weyBui = Storage.WriteEntry.newBuilder(); + data.forEach(point -> { + final String table = point.getTable(); + final WriteTuple3 tp3 = tuple3s.computeIfAbsent(table, WriteTuple3::new); final NameDict tagDict = tp3.getTagDict(); - rs.getSeries().getTags().forEach((tagK, tagV) -> { - if (Value.isNull(tagV)) { - return; + point.getTags().forEach((tagK, tagV) -> { + tagDict.insert(tagK); + }); + StringBuffer seriesKeyBuffer = new StringBuffer(); + tagDict.toOrdered().forEach((tagK) -> { + Value tagV = point.getTags().get(tagK); + if (!Value.isNull(tagV)) { + seriesKeyBuffer.append(tagV.getValue().toString()); } - final Storage.Tag.Builder tBui = Storage.Tag.newBuilder().setNameIndex(tagDict.insert(tagK)) - .setValue(Utils.toProtoValue(tagV)); - weyBui.addTags(tBui.build()); }); + Storage.WriteSeriesEntry.Builder seriesEntryBuilder = tp3.getSeriesBuilders() + .computeIfAbsent(seriesKeyBuffer.toString(), seriesKey -> { + final Storage.WriteSeriesEntry.Builder seBuilder = Storage.WriteSeriesEntry.newBuilder(); + point.getTags().forEach((tagK, tagV) -> { + if (Value.isNull(tagV)) { + return; + } + final Storage.Tag.Builder tBui = Storage.Tag.newBuilder().setNameIndex(tagDict.insert(tagK)) + .setValue(Utils.toProtoValue(tagV)); + seBuilder.addTags(tBui.build()); + }); + return seBuilder; + }); final NameDict fieldDict = tp3.getFieldDict(); - rs.getFields().forEach((ts, fields) -> { - final Storage.FieldGroup.Builder fgBui = Storage.FieldGroup.newBuilder().setTimestamp(ts); - fields.forEach((name, field) -> { - if (Value.isNull(field)) { - return; - } - final Storage.Field.Builder fBui = Storage.Field.newBuilder().setNameIndex(fieldDict.insert(name)) - .setValue(Utils.toProtoValue(field)); - fgBui.addFields(fBui.build()); - }); - weyBui.addFieldGroups(fgBui.build()); + final Storage.FieldGroup.Builder fgBui = Storage.FieldGroup.newBuilder().setTimestamp(point.getTimestamp()); + point.getFields().forEach((fieldK, fieldV) -> { + if (Value.isNull(fieldV)) { + return; + } + final Storage.Field.Builder fBui = Storage.Field.newBuilder().setNameIndex(fieldDict.insert(fieldK)) + .setValue(Utils.toProtoValue(fieldV)); + fgBui.addFields(fBui.build()); }); - - tp3.getWmcBui().addEntries(weyBui.build()); + seriesEntryBuilder.addFieldGroups(fgBui.build()); }); - tuple3s.values().forEach(tp3 -> wrBui.addMetrics(tp3.build())); + Storage.RequestContext.Builder ctxBuilder = Storage.RequestContext.newBuilder(); + ctxBuilder.setDatabase(reqCtx.getDatabase()); + writeRequestBuilder.setContext(ctxBuilder.build()); + + tuple3s.values().forEach(tp3 -> writeRequestBuilder.addTableRequests(tp3.build())); - return wrBui.build(); + return writeRequestBuilder.build(); } @Override @@ -519,12 +523,12 @@ public DefaultWriteLimiter(int maxInFlight, LimitedPolicy policy) { } @Override - public int calculatePermits(final Collection in) { - return in == null ? 0 : in.stream().map(Rows::getRowCount).reduce(0, Integer::sum); + public int calculatePermits(final List in) { + return in == null ? 0 : in.size(); } @Override - public Result rejected(final Collection in, final RejectedState state) { + public Result rejected(final List in, final RejectedState state) { final String errMsg = String.format( "Write limited by client, acquirePermits=%d, maxPermits=%d, availablePermits=%d.", // state.acquirePermits(), // diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/WriteLimiter.java b/ceresdb-protocol/src/main/java/io/ceresdb/WriteLimiter.java deleted file mode 100644 index 9c2025d..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/WriteLimiter.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb; - -import java.util.Collection; - -import io.ceresdb.models.Err; -import io.ceresdb.models.Result; -import io.ceresdb.models.Rows; -import io.ceresdb.models.WriteOk; - -/** - * Like rust: pub type WriteLimiter = CeresDBLimiter, Result> - * - * @author jiachun.fjc - */ -public abstract class WriteLimiter extends CeresDBLimiter, Result> { - - public WriteLimiter(int maxInFlight, LimitedPolicy policy, String metric) { - super(maxInFlight, policy, metric); - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/errors/IteratorException.java b/ceresdb-protocol/src/main/java/io/ceresdb/errors/IteratorException.java index c9fe3c9..3d882cd 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/errors/IteratorException.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/errors/IteratorException.java @@ -1,25 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.errors; /** * Iterator timeout error on stream query. * - * @author jiachun.fjc */ public class IteratorException extends RuntimeException { diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/errors/LimitedException.java b/ceresdb-protocol/src/main/java/io/ceresdb/errors/LimitedException.java index 7549fce..15c8c2d 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/errors/LimitedException.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/errors/LimitedException.java @@ -1,25 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.errors; /** * A flow control exception that can be thrown by AbortPolicy. * - * @author jiachun.fjc */ public class LimitedException extends RuntimeException { diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/errors/RouteTableException.java b/ceresdb-protocol/src/main/java/io/ceresdb/errors/RouteTableException.java index 5d7f956..5d50466 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/errors/RouteTableException.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/errors/RouteTableException.java @@ -1,25 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.errors; /** * Get route table exception. * - * @author jiachun.fjc */ public class RouteTableException extends Exception { diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/errors/StreamException.java b/ceresdb-protocol/src/main/java/io/ceresdb/errors/StreamException.java index 97808f8..3b669c7 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/errors/StreamException.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/errors/StreamException.java @@ -1,25 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.errors; /** * Error about stream-query or stream-write. * - * @author jiachun.fjc */ public class StreamException extends RuntimeException { diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/CeresDBLimiter.java b/ceresdb-protocol/src/main/java/io/ceresdb/limit/CeresDBLimiter.java similarity index 79% rename from ceresdb-protocol/src/main/java/io/ceresdb/CeresDBLimiter.java rename to ceresdb-protocol/src/main/java/io/ceresdb/limit/CeresDBLimiter.java index e864df6..0cfa1f5 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/CeresDBLimiter.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/limit/CeresDBLimiter.java @@ -1,24 +1,12 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ -package io.ceresdb; +package io.ceresdb.limit; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; +import io.ceresdb.util.Utils; import io.ceresdb.common.InFlightLimiter; import io.ceresdb.common.Limiter; import io.ceresdb.common.util.MetricsUtil; @@ -33,7 +21,6 @@ * that the available permits are equal to the maximum number of permits, * i.e., there are no in-flight requests. * - * @author jiachun.fjc */ public abstract class CeresDBLimiter { diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/LimitedPolicy.java b/ceresdb-protocol/src/main/java/io/ceresdb/limit/LimitedPolicy.java similarity index 79% rename from ceresdb-protocol/src/main/java/io/ceresdb/LimitedPolicy.java rename to ceresdb-protocol/src/main/java/io/ceresdb/limit/LimitedPolicy.java index 13ff879..1564d2d 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/LimitedPolicy.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/limit/LimitedPolicy.java @@ -1,20 +1,7 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ -package io.ceresdb; +package io.ceresdb.limit; import java.util.concurrent.TimeUnit; @@ -24,7 +11,6 @@ /** * A limited policy using a given {@code Limiter}. * - * @author jiachun.fjc */ public interface LimitedPolicy { diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/limit/QueryLimiter.java b/ceresdb-protocol/src/main/java/io/ceresdb/limit/QueryLimiter.java new file mode 100644 index 0000000..0d91017 --- /dev/null +++ b/ceresdb-protocol/src/main/java/io/ceresdb/limit/QueryLimiter.java @@ -0,0 +1,20 @@ +/* + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + */ +package io.ceresdb.limit; + +import io.ceresdb.models.Err; +import io.ceresdb.models.SqlQueryOk; +import io.ceresdb.models.SqlQueryRequest; +import io.ceresdb.models.Result; + +/** + * Like rust: pub type QueryLimiter = CeresDBLimiter> + * + */ +public abstract class QueryLimiter extends CeresDBLimiter> { + + public QueryLimiter(int maxInFlight, LimitedPolicy policy, String metricPrefix) { + super(maxInFlight, policy, metricPrefix); + } +} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/limit/WriteLimiter.java b/ceresdb-protocol/src/main/java/io/ceresdb/limit/WriteLimiter.java new file mode 100644 index 0000000..bf75df5 --- /dev/null +++ b/ceresdb-protocol/src/main/java/io/ceresdb/limit/WriteLimiter.java @@ -0,0 +1,22 @@ +/* + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + */ +package io.ceresdb.limit; + +import java.util.List; + +import io.ceresdb.models.Err; +import io.ceresdb.models.Point; +import io.ceresdb.models.Result; +import io.ceresdb.models.WriteOk; + +/** + * Like rust: pub type WriteLimiter = CeresDBLimiter, Result> + * + */ +public abstract class WriteLimiter extends CeresDBLimiter, Result> { + + public WriteLimiter(int maxInFlight, LimitedPolicy policy, String table) { + super(maxInFlight, policy, table); + } +} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/Err.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/Err.java index 7a41521..f4b3c7d 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/models/Err.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/models/Err.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.models; @@ -20,6 +7,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -29,7 +17,6 @@ /** * Contains the write or query error value. * - * @author jiachun.fjc */ public class Err implements Streamable { // error code from server @@ -39,13 +26,13 @@ public class Err implements Streamable { // the server address where the error occurred private Endpoint errTo; // the data of wrote failed, can be used to retry - private Collection failedWrites; + private List failedWrites; // other successful server results are merged here private WriteOk subOk; - // the QL failed to query - private String failedQl; + // the SQL failed to query + private String failedSql; // the metrics of failed to query - private Collection failedMetrics; + private Collection failedTables; // child err merged here private Collection children; @@ -61,7 +48,7 @@ public Endpoint getErrTo() { return errTo; } - public Collection getFailedWrites() { + public Collection getFailedWrites() { return failedWrites; } @@ -69,12 +56,12 @@ public WriteOk getSubOk() { return subOk; } - public String getFailedQl() { - return failedQl; + public String getFailedSql() { + return failedSql; } - public Collection getFailedMetrics() { - return failedMetrics; + public Collection getFailedTables() { + return failedTables; } public Err combine(final Err err) { @@ -108,13 +95,13 @@ public Stream stream() { } } - private int failedWriteRowsNum() { + private int failedWritePointsNum() { return this.failedWrites == null ? 0 : this.failedWrites.size(); } - private List failedWriteMetrics() { - return this.failedWrites == null ? Collections.emptyList() // - : this.failedWrites.stream().map(Rows::getMetric).collect(Collectors.toList()); + private Set failedWriteTables() { + return this.failedWrites == null ? Collections.emptySet() // + : this.failedWrites.stream().map(Point::getTable).collect(Collectors.toSet()); } @Override @@ -123,11 +110,11 @@ public String toString() { "code=" + code + // ", error='" + error + '\'' + // ", errTo=" + errTo + // - ", failedWriteRowsNum=" + failedWriteRowsNum() + // - ", failedWriteMetrics=" + failedWriteMetrics() + // + ", failedWritePointsNum=" + failedWritePointsNum() + // + ", failedWriteTables=" + failedWriteTables() + // ", subOk=" + subOk + // - ", failedQl=" + failedQl + // - ", failedMetrics=" + failedMetrics + // + ", failedSql=" + failedSql + // + ", failedTables=" + failedTables + // ", children=" + children + // '}'; } @@ -135,7 +122,7 @@ public String toString() { public static Err writeErr(final int code, // final String error, // final Endpoint errTo, // - final Collection failedWrites) { + final List failedWrites) { final Err err = new Err(); err.code = code; err.error = error; @@ -153,8 +140,8 @@ public static Err queryErr(final int code, // err.code = code; err.error = error; err.errTo = errTo; - err.failedQl = failedQl; - err.failedMetrics = failedMetrics; + err.failedSql = failedQl; + err.failedTables = failedMetrics; return err; } } diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/FieldValue.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/FieldValue.java deleted file mode 100644 index 3fd5228..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/models/FieldValue.java +++ /dev/null @@ -1,341 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.models; - -import java.math.BigInteger; -import java.util.Optional; - -import io.ceresdb.common.util.Requires; -import io.ceresdb.common.util.UnsignedUtil; - -/** - * Field value can be wrote to database. - * - * @author jiachun.fjc - */ -public final class FieldValue extends Value { - - enum NullField { - Float64(Type.Float64), // - String(Type.String), // - Int64(Type.Int64), // - Float32(Type.Float32), // - Int32(Type.Int32), // - Int16(Type.Int16), // - Int8(Type.Int8), // - Boolean(Type.Boolean), // - UInt64(Type.UInt64), // - UInt32(Type.UInt32), // - UInt16(Type.UInt16), // - UInt8(Type.UInt8), // - Timestamp(Type.Timestamp), // - Varbinary(Type.Varbinary); // - - private final FieldValue value; - - NullField(Type type) { - this.value = new FieldValue(type, null); - } - } - - private FieldValue(Type type, Object value) { - super(type, value); - } - - public double getDouble() { - return getFloat64(); - } - - public Optional getDoubleOrNull() { - return getFloat64OrNull(); - } - - public double getFloat64() { - return getCheckedValue(Type.Float64); - } - - public Optional getFloat64OrNull() { - return isNull() ? Optional.empty() : Optional.of(getFloat64()); - } - - public String getString() { - return getCheckedValue(Type.String); - } - - public Optional getStringOrNull() { - return isNull() ? Optional.empty() : Optional.of(getString()); - } - - public long getInt64() { - return getCheckedValue(Type.Int64); - } - - public Optional getInt64OrNull() { - return isNull() ? Optional.empty() : Optional.of(getInt64()); - } - - public float getFloat() { - return getFloat32(); - } - - public Optional getFloatOrNull() { - return getFloat32OrNull(); - } - - public float getFloat32() { - return getCheckedValue(Type.Float32); - } - - public Optional getFloat32OrNull() { - return isNull() ? Optional.empty() : Optional.of(getFloat32()); - } - - public int getInt() { - return getInt32(); - } - - public Optional getIntOrNull() { - return getInt32OrNull(); - } - - public int getInt32() { - return getCheckedValue(Type.Int32); - } - - public Optional getInt32OrNull() { - return isNull() ? Optional.empty() : Optional.of(getInt32()); - } - - public int getInt16() { - return getCheckedValue(Type.Int16); - } - - public Optional getInt16OrNull() { - return isNull() ? Optional.empty() : Optional.of(getInt16()); - } - - public int getInt8() { - return getCheckedValue(Type.Int8); - } - - public Optional getInt8OrNull() { - return isNull() ? Optional.empty() : Optional.of(getInt8()); - } - - public boolean getBoolean() { - return getCheckedValue(Type.Boolean); - } - - public Optional getBooleanOrNull() { - return isNull() ? Optional.empty() : Optional.of(getBoolean()); - } - - public long getUInt64() { - return getCheckedValue(Type.UInt64); - } - - public Optional getUInt64OrNull() { - return isNull() ? Optional.empty() : Optional.of(getUInt64()); - } - - public int getUInt32() { - return getCheckedValue(Type.UInt32); - } - - public Optional getUInt32OrNull() { - return isNull() ? Optional.empty() : Optional.of(getUInt32()); - } - - public int getUInt16() { - return getCheckedValue(Type.UInt16); - } - - public Optional getUInt16OrNull() { - return isNull() ? Optional.empty() : Optional.of(getUInt16()); - } - - public int getUInt8() { - return getCheckedValue(Type.UInt8); - } - - public Optional getUInt8OrNull() { - return isNull() ? Optional.empty() : Optional.of(getUInt8()); - } - - public long getTimestamp() { - return getCheckedValue(Type.Timestamp); - } - - public Optional getTimestampOrNull() { - return isNull() ? Optional.empty() : Optional.of(getTimestamp()); - } - - public byte[] getVarbinary() { - return getCheckedValue(Type.Varbinary); - } - - public Optional getVarbinaryOrNull() { - return isNull() ? Optional.empty() : Optional.of(getVarbinary()); - } - - public static FieldValue withDouble(final double val) { - return withFloat64(val); - } - - public static FieldValue withDoubleOrNull(final Double val) { - return withFloat64OrNull(val); - } - - public static FieldValue withFloat64(final double val) { - return new FieldValue(Type.Float64, val); - } - - public static FieldValue withFloat64OrNull(final Double val) { - return val == null ? NullField.Float64.value : withFloat64(val); - } - - public static FieldValue withString(final String val) { - Requires.requireNonNull(val, "Null.val"); - return new FieldValue(Type.String, val); - } - - public static FieldValue withStringOrNull(final String val) { - return val == null ? NullField.String.value : withString(val); - } - - public static FieldValue withInt64(final long val) { - return new FieldValue(Type.Int64, val); - } - - public static FieldValue withInt64OrNull(final Long val) { - return val == null ? NullField.Int64.value : withInt64(val); - } - - public static FieldValue withFloat(final float val) { - return withFloat32(val); - } - - public static FieldValue withFloatOrNull(final Float val) { - return withFloat32OrNull(val); - } - - public static FieldValue withFloat32(final float val) { - return new FieldValue(Type.Float32, val); - } - - public static FieldValue withFloat32OrNull(final Float val) { - return val == null ? NullField.Float32.value : withFloat32(val); - } - - public static FieldValue withInt(final int val) { - return withInt32(val); - } - - public static FieldValue withIntOrNull(final Integer val) { - return withInt32OrNull(val); - } - - public static FieldValue withInt32(final int val) { - return new FieldValue(Type.Int32, val); - } - - public static FieldValue withInt32OrNull(final Integer val) { - return val == null ? NullField.Int32.value : withInt32(val); - } - - public static FieldValue withInt16(final int val) { - return new FieldValue(Type.Int16, val); - } - - public static FieldValue withInt16OrNull(final Integer val) { - return val == null ? NullField.Int16.value : withInt16(val); - } - - public static FieldValue withInt8(final int val) { - return new FieldValue(Type.Int8, val); - } - - public static FieldValue withInt8OrNull(final Integer val) { - return val == null ? NullField.Int8.value : withInt8(val); - } - - public static FieldValue withBoolean(final boolean val) { - return new FieldValue(Type.Boolean, val); - } - - public static FieldValue withBooleanOrNull(final Boolean val) { - return val == null ? NullField.Boolean.value : withBoolean(val); - } - - public static FieldValue withUInt64(final long val) { - return new FieldValue(Type.UInt64, val); - } - - public static FieldValue withUInt64OrNull(final Long val) { - return val == null ? NullField.UInt64.value : withUInt64(val); - } - - public static FieldValue withUInt64(final BigInteger val) { - Requires.requireNonNull(val, "Null.val"); - return new FieldValue(Type.UInt64, UnsignedUtil.getInt64(val)); - } - - public static FieldValue withUInt64OrNull(final BigInteger val) { - return val == null ? NullField.UInt64.value : withUInt64(val); - } - - public static FieldValue withUInt32(final int val) { - return new FieldValue(Type.UInt32, val); - } - - public static FieldValue withUInt32OrNull(final Integer val) { - return val == null ? NullField.UInt32.value : withUInt32(val); - } - - public static FieldValue withUInt16(final int val) { - return new FieldValue(Type.UInt16, val); - } - - public static FieldValue withUInt16OrNull(final Integer val) { - return val == null ? NullField.UInt16.value : withUInt16(val); - } - - public static FieldValue withUInt8(final int val) { - return new FieldValue(Type.UInt8, val); - } - - public static FieldValue withUInt8OrNull(final Integer val) { - return val == null ? NullField.UInt8.value : withUInt8(val); - } - - public static FieldValue withTimestamp(final long val) { - return new FieldValue(Type.Timestamp, val); - } - - public static FieldValue withTimestampOrNull(final Long val) { - return val == null ? NullField.Timestamp.value : withTimestamp(val); - } - - public static FieldValue withVarbinary(final byte[] val) { - Requires.requireNonNull(val, "Null.val"); - return new FieldValue(Type.Varbinary, val); - } - - public static FieldValue withVarbinaryOrNull(final byte[] val) { - return val == null ? NullField.Varbinary.value : withVarbinary(val); - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/IndexedRecord.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/IndexedRecord.java deleted file mode 100644 index a7610df..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/models/IndexedRecord.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.models; - -import java.math.BigInteger; - -import io.ceresdb.common.util.UnsignedUtil; - -/** - * A record implementation that permits field access by integer index. - * - * @author jiachun.fjc - */ -public interface IndexedRecord { - - /** - * Return the value of a field given its position in the schema. - * - * @param i the field position in the schema - * @return the field value - */ - Object get(final int i); - - /** - * @see #get(int) - */ - default T get(final int i, final Class expectType) { - return expectType.cast(get(i)); - } - - /** - * Get a boolean value for the given index. - * - * @param i the field index - * @return the field value - */ - default Boolean getBoolean(final int i) { - return get(i, Boolean.class); - } - - /** - * @see #getInteger(int) - */ - default Integer getUInt16(final int i) { - return getInteger(i); - } - - /** - * @see #getInteger(int) - */ - default Integer getUInt8(final int i) { - return getInteger(i); - } - - /** - * @see #getInteger(int) - */ - default Integer getInt16(final int i) { - return getInteger(i); - } - - /** - * @see #getInteger(int) - */ - default Integer getInt8(final int i) { - return getInteger(i); - } - - /** - * Get a integer value for the given index. - * - * @param i the field index - * @return the field value - */ - default Integer getInteger(final int i) { - return get(i, Integer.class); - } - - /** - * @see #getLong(int) - */ - default Long getTimestamp(final int i) { - return getLong(i); - } - - /** - * Get a unsigned long value for the given index. - * - * @param i the field index - * @return the field value - */ - default BigInteger getUInt64(final int i) { - final Long value = getLong(i); - return value == null ? null : UnsignedUtil.getUInt64(value); - } - - /** - * @see #getLong(int) - */ - default Long getUInt32(final int i) { - return getLong(i); - } - - /** - * @see #getLong(int) - */ - default Long getInt64(final int i) { - return getLong(i); - } - - /** - * Get a long value for the given index. - * - * @param i the field index - * @return the field value - */ - default Long getLong(final int i) { - return get(i, Long.class); - } - - /** - * Get a float value for the given index. - * - * @param i the field index - * @return the field value - */ - default Float getFloat(final int i) { - return get(i, Float.class); - } - - /** - * Get a double value for the given index. - * - * @param i the field index - * @return the field value - */ - default Double getDouble(final int i) { - return get(i, Double.class); - } - - /** - * Get a string value for the given index. - * - * @param i the field index - * @return the field value - */ - default String getString(final int i) { - return get(i, String.class); - } - - /** - * Get a bytes value for the given index. - * - * @param i the field index - * @return the field value - */ - default byte[] getBytes(final int i) { - return get(i, byte[].class); - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/Keyword.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/Keyword.java index cb7d1b6..d23fd13 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/models/Keyword.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/models/Keyword.java @@ -1,25 +1,8 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.models; -/** - * - * @author jiachun.fjc - */ public enum Keyword { Timestamp, // Tsid; diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/Point.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/Point.java new file mode 100644 index 0000000..9e574e2 --- /dev/null +++ b/ceresdb-protocol/src/main/java/io/ceresdb/models/Point.java @@ -0,0 +1,89 @@ +/* + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + */ +package io.ceresdb.models; + +import java.util.HashMap; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + +import io.ceresdb.util.Utils; +import io.ceresdb.common.util.Requires; + +/** + * A time series data point with multiple fields + * + */ +public class Point { + protected String table; + protected long timestamp; + protected SortedMap tags; + protected Map fields; + + protected Point(String table) { + this.table = table; + this.tags = new TreeMap<>(); + this.fields = new HashMap<>(); + } + + public String getTable() { + return table; + } + + public long getTimestamp() { + return timestamp; + } + + public SortedMap getTags() { + return tags; + } + + public Map getFields() { + return fields; + } + + public static PointBuilder newPointBuilder(String table) { + return new PointBuilder(table); + } + + public static class PointBuilder { + private Point point; + + protected PointBuilder(String table) { + this.point = new Point(table); + } + + public PointBuilder setTimestamp(long timestamp) { + this.point.timestamp = timestamp; + return this; + } + + public PointBuilder addTag(final String tagKey, final Value tagValue) { + this.point.tags.put(tagKey, tagValue); + return this; + } + + public PointBuilder addTag(final String tagKey, final String tagValue) { + this.point.tags.put(tagKey, Value.withStringOrNull(tagValue)); + return this; + } + + public PointBuilder addField(final String fieldKey, final Value fieldValue) { + this.point.fields.put(fieldKey, fieldValue); + return this; + } + + public Point build() { + check(this.point); + return point; + } + + private static void check(final Point point) throws IllegalArgumentException { + Requires.requireNonNull(point.fields, "Null.fields"); + Requires.requireTrue(!point.fields.isEmpty(), "Empty.fields"); + Utils.checkKeywords(point.tags.keySet().stream().iterator()); + Utils.checkKeywords(point.fields.keySet().stream().iterator()); + } + } +} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/QueryOk.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/QueryOk.java deleted file mode 100644 index d56e6e1..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/models/QueryOk.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.models; - -import java.util.function.Function; -import java.util.stream.Stream; - -import io.ceresdb.ArrayMapper; -import io.ceresdb.RecordMapper; -import io.ceresdb.common.Streamable; -import io.ceresdb.common.util.Requires; - -/** - * Contains the success value of query. - * - * @author jiachun.fjc - */ -public class QueryOk implements Streamable { - - private String ql; - private Schema schema; - private int rowCount; - private Stream rows; - - public String getQl() { - return ql; - } - - public Schema getSchema() { - return schema; - } - - public int getRowCount() { - return rowCount; - } - - public Stream map(final Function mapper) { - if (this.rowCount == 0) { - return Stream.empty(); - } - return this.rows.map(mapper); - } - - public Stream mapToRecord() { - if (this.rowCount == 0) { - return Stream.empty(); - } - ensureAvroType(this.schema); - return map(RecordMapper.getMapper(this.schema.getContent())); - } - - public Stream mapToArray() { - if (this.rowCount == 0) { - return Stream.empty(); - } - ensureAvroType(this.schema); - return map(ArrayMapper.getMapper(this.schema.getContent())); - } - - public Result mapToResult() { - return Result.ok(this); - } - - @Override - public Stream stream() { - return this.rows; - } - - @Override - public String toString() { - return "QueryOk{" + // - "ql='" + ql + '\'' + // - ", schema='" + schema + '\'' + // - ", rowCount=" + rowCount + // - '}'; - } - - public static QueryOk emptyOk() { - return ok("", null, 0, Stream.empty()); - } - - public static QueryOk ok(final String ql, final Schema schema, final int rowCount, final Stream rows) { - final QueryOk ok = new QueryOk(); - ok.ql = ql; - ok.schema = schema; - ok.rowCount = rowCount; - ok.rows = rows; - return ok; - } - - private static void ensureAvroType(final Schema schema) { - Requires.requireNonNull(schema, "NUll.schema"); - Requires.requireTrue(schema.getType() == Schema.Type.Avro, "Invalid schema type %s, [Avro] type is required", - schema); - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/QueryRequest.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/QueryRequest.java deleted file mode 100644 index 917bebb..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/models/QueryRequest.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.models; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import io.ceresdb.common.util.Requires; -import io.ceresdb.common.util.Strings; - -/** - * The query request condition. - * - * @author jiachun.fjc - */ -public class QueryRequest { - - private List metrics = Collections.emptyList(); - private String ql; - - public List getMetrics() { - return metrics; - } - - public void setMetrics(List metrics) { - this.metrics = metrics; - } - - public String getQl() { - return ql; - } - - @Override - public String toString() { - return "QueryRequest{" + // - "metrics=" + metrics + // - ", ql='" + ql + '\'' + // - '}'; - } - - public static Builder newBuilder() { - return new Builder(); - } - - public static QueryRequest check(final QueryRequest qr) { - Requires.requireTrue(Strings.isNotBlank(qr.ql), "Empty.ql"); - return qr; - } - - public static class Builder { - private final List metrics = new ArrayList<>(); - private String ql; - - /** - * Client does not parse the QL, so please fill in which metrics you queried. - * - * @param metrics the metrics queried. - * @return this builder - */ - public Builder forMetrics(final String... metrics) { - this.metrics.addAll(Arrays.asList(metrics)); - return this; - } - - /** - * Query language to. - * - * @param ql the query language - * @return this builder - */ - public Builder ql(final String ql) { - this.ql = ql; - return this; - } - - /** - * Query language to, using the specified format string and arguments. - * - * @param fmtQl format ql string - * @param args arguments referenced by the format specifiers in the format - * QL string. If there are more arguments than format specifiers, - * the extra arguments are ignored. The number of arguments is - * variable and may be zero. - * @return this builder - */ - public Builder ql(final String fmtQl, final Object... args) { - this.ql = String.format(fmtQl, args); - return this; - } - - public QueryRequest build() { - final QueryRequest qr = new QueryRequest(); - qr.metrics = this.metrics; - qr.ql = this.ql; - return qr; - } - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/Record.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/Record.java deleted file mode 100644 index 7951169..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/models/Record.java +++ /dev/null @@ -1,298 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.models; - -import java.math.BigInteger; -import java.util.List; -import java.util.Optional; - -import io.ceresdb.common.util.UnsignedUtil; - -/** - * A record, fields are accessible by name as well as by index. - * - * @author jiachun.fjc - */ -public interface Record extends IndexedRecord { - - /** - * Return the value of a field given its name. - * - * @param field the field name - * @return the field value - */ - Object get(final String field); - - /** - * @see #get(String) - */ - default T get(final String field, final Class expectType) { - return expectType.cast(get(field)); - } - - /** - * Get a boolean value for the given {@code field}. - * - * @param field the field name - * @return the field value - */ - default Boolean getBoolean(final String field) { - return get(field, Boolean.class); - } - - /** - * Get a unsigned long value for the given {@code field}. - * - * @param field the field name - * @return the field value - */ - default Integer getUInt16(final String field) { - return getInteger(field); - } - - /** - * @see #getInteger(String) - */ - default Integer getUInt8(final String field) { - return getInteger(field); - } - - /** - * @see #getInteger(String) - */ - default Integer getInt16(final String field) { - return getInteger(field); - } - - /** - * @see #getInteger(String) - */ - default Integer getInt8(final String field) { - return getInteger(field); - } - - /** - * Get a integer value for the given {@code field}. - * - * @param field the field name - * @return the field value - */ - default Integer getInteger(final String field) { - return get(field, Integer.class); - } - - /** - * @see #getLong(String) - */ - default Long getTimestamp(final String field) { - return getLong(field); - } - - /** - * @see #getLong(String) - */ - default BigInteger getUInt64(final String field) { - final Long value = getLong(field); - return value == null ? null : UnsignedUtil.getUInt64(value); - } - - /** - * @see #getLong(String) - */ - default Long getUInt32(final String field) { - return getLong(field); - } - - /** - * @see #getLong(String) - */ - default Long getInt64(final String field) { - return getLong(field); - } - - /** - * Get a long value for the given {@code field}. - * - * @param field the field name - * @return the field value - */ - default Long getLong(final String field) { - return get(field, Long.class); - } - - /** - * Get a float value for the given {@code field}. - * - * @param field the field name - * @return the field value - */ - default Float getFloat(final String field) { - return get(field, Float.class); - } - - /** - * Get a double value for the given {@code field}. - * - * @param field the field name - * @return the field value - */ - default Double getDouble(final String field) { - return get(field, Double.class); - } - - /** - * Get a string value for the given {@code field}. - * - * @param field the field name - * @return the field value - */ - default String getString(final String field) { - return get(field, String.class); - } - - /** - * Get a bytes value for the given {@code field}. - * - * @param field the field name - * @return the field value - */ - default byte[] getBytes(final String field) { - return get(field, byte[].class); - } - - /** - * Return true if record has field with name. - * - * @param field the field name - * @return true if record has field with name: key - */ - boolean hasField(final String field); - - /** - * Return the field count of this record. - * - * @return the field count - */ - int getFieldCount(); - - /** - * Return all field descriptors in the record. - * - * @return all field descriptors - */ - List getFieldDescriptors(); - - /** - * Return the specified field descriptor in the record. - * - * @param field field name - * @return the specified field descriptor - */ - Optional getFieldDescriptor(final String field); - - class FieldDescriptor { - private final String name; - private final FieldType type; - private final List subTypes; // the type is Union - - public FieldDescriptor(String name, FieldType type, List subTypes) { - this.name = name; - this.type = type; - this.subTypes = subTypes; - } - - public String getName() { - return name; - } - - public FieldType getType() { - return type; - } - - public List getSubTypes() { - return subTypes; - } - - public boolean isTimestamp() { - if (isTimestamp(this.type.getLogicalType())) { - return true; - } - - for (final FieldType fieldType : this.subTypes) { - if (isTimestamp(fieldType.getLogicalType())) { - return true; - } - } - - return false; - } - - private static boolean isTimestamp(final LogicalType logicalType) { - return logicalType == LogicalType.TimestampMillis || logicalType == LogicalType.TimestampMicros; - } - - @Override - public String toString() { - return "[" + // - "name='" + name + '\'' + // - ", type='" + type + '\'' + // - ", subTypes='" + subTypes + '\'' + // - ']'; - } - } - - final class FieldType { - private final Type type; - private final LogicalType logicalType; - - public static FieldType of(final Type type, final LogicalType logicalType) { - return new FieldType(type, logicalType); - } - - private FieldType(Type type, LogicalType logicalType) { - this.type = type; - this.logicalType = logicalType; - } - - public Type getType() { - return type; - } - - public LogicalType getLogicalType() { - return logicalType; - } - - @Override - public String toString() { - return "FieldType{" + // - "type=" + type + // - ", logicalType=" + logicalType + // - '}'; - } - } - - /** - * Data types, represents any valid schema. - */ - enum Type { - Null, Double, Float, Bytes, String, Long, Int, Boolean, Union, Unknown - } - - enum LogicalType { - TimestampMillis, TimestampMicros, Null, Unknown - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/RequestContext.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/RequestContext.java new file mode 100644 index 0000000..1040e5b --- /dev/null +++ b/ceresdb-protocol/src/main/java/io/ceresdb/models/RequestContext.java @@ -0,0 +1,20 @@ +/* + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + */ +package io.ceresdb.models; + +/** + * + * RequestContext of business model, such as database + */ +public class RequestContext { + private String database; + + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + } +} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/Result.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/Result.java index 20b5889..d91e132 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/models/Result.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/models/Result.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.models; @@ -23,7 +10,6 @@ /** * `Result` is a type that represents either success ([`Ok`]) or failure ([`Err`]). * - * @author jiachun.fjc */ public final class Result { diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/Row.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/Row.java new file mode 100644 index 0000000..3e86dfc --- /dev/null +++ b/ceresdb-protocol/src/main/java/io/ceresdb/models/Row.java @@ -0,0 +1,41 @@ +/* + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + */ +package io.ceresdb.models; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +public class Row { + private Map values; + + public Row() { + this.values = new HashMap<>(); + } + + public Value getColumnValue(String column) { + return this.values.get(column); + } + + public void setColumnValue(String column, Value value) { + this.values.put(column, value); + } + + public int getColumnCount() { + if (this.values == null) { + return 0; + } + return this.values.size(); + } + + @Override + public String toString() { + if (this.values == null || this.values.isEmpty()) { + return "[Empty Row]"; + } + + return this.values.entrySet().stream().map(entry -> entry.getKey() + ":" + entry.getValue().toString()) + .collect(Collectors.joining("|")); + } +} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/Rows.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/Rows.java deleted file mode 100644 index bf212ba..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/models/Rows.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.models; - -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Objects; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.function.Consumer; - -import io.ceresdb.Utils; -import io.ceresdb.common.util.Requires; - -/** - * Rows with the same series but different timestamps. - * - * @author jiachun.fjc - */ -public class Rows { - - private Series series; - - /** - * Map> - * - * Sorted by timestamp, which is useful for data storage. - */ - private SortedMap> fields; - - public Series getSeries() { - return series; - } - - public SortedMap> getFields() { - return fields; - } - - public String getMetric() { - return this.series == null ? null : this.series.getMetric(); - } - - public int getRowCount() { - return this.fields == null ? 0 : this.fields.size(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Rows rows = (Rows) o; - return Objects.equals(series, rows.series) && Objects.equals(fields, rows.fields); - } - - @Override - public int hashCode() { - return Objects.hash(series, fields); - } - - @Override - public String toString() { - return "Rows{" + // - "series=" + series + // - ", fields=" + fields + // - '}'; - } - - public static Rows check(final Rows rs) { - Requires.requireNonNull(rs.fields, "Null.fields"); - Requires.requireTrue(!rs.fields.isEmpty(), "Empty.fields"); - Utils.checkKeywords(rs.fields.values().stream().flatMap((map) -> map.keySet().stream()).iterator()); - return rs; - } - - public static Builder newBuilder(final Series series) { - return newBuilder(series, false); - } - - public static Builder newBuilder(final Series series, final boolean keepFieldOrder) { - return new Builder(series, keepFieldOrder); - } - - public static class Builder { - private final Series series; - private final boolean keepFieldOrder; // whether keep the fields write order - private final SortedMap> fields; - - public Builder(Series series, boolean keepFieldOrder) { - this.series = series; - this.keepFieldOrder = keepFieldOrder; - this.fields = new TreeMap<>(); - } - - /** - * Sets a field, replacing any existing field of the same name. - * - * @param timestamp timestamp of this filed - * @param name field's name - * @param fieldValue field's value - * @return this builder - */ - public Builder field(final long timestamp, final String name, final FieldValue fieldValue) { - this.fields // - .computeIfAbsent(timestamp, ts -> newFieldMap()) // - .put(name, fieldValue); - return this; - } - - /** - * Sets multi fields, replacing any existing field of the same name. - * - * @param timestamp timestamp of these fields - * @param fields fields map - * @return this builder - */ - public Builder fields(final long timestamp, final Map fields) { - final Map exists = this.fields.get(timestamp); - if (exists == null) { - this.fields.put(timestamp, fields); - } else { - exists.putAll(fields); - } - return this; - } - - /** - * Sets multi fields, replacing any existing field of the same name. - * - * @param timestamp timestamp of these fields - * @param input fields input - * @return this builder - */ - public Builder fields(final long timestamp, final Consumer input) { - final Map map = this.fields.computeIfAbsent(timestamp, k -> newFieldMap()); - input.accept(map::put); - return this; - } - - public interface FieldsInput { - void put(final String name, final FieldValue fieldValue); - } - - private Map newFieldMap() { - return this.keepFieldOrder ? new LinkedHashMap<>() : new HashMap<>(); - } - - /** - * Constructs the series points. - * - * @return Rows - */ - public Rows build() { - final Rows rs = new Rows(); - rs.series = this.series; - rs.fields = Collections.unmodifiableSortedMap(this.fields); - return Rows.check(rs); - } - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/Schema.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/Schema.java deleted file mode 100644 index bd8f67e..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/models/Schema.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.models; - -import io.ceresdb.common.util.Requires; - -/** - * - * @author jiachun.fjc - */ -public final class Schema { - - public enum Type { - Avro(String.class), // - Json(Void.class) // Json do not need schema - ; - - private final Class javaType; - - Type(Class javaType) { - this.javaType = javaType; - } - - public Class getJavaType() { - return javaType; - } - } - - private final Type type; - private final Object content; - - private Schema(Type type, Object content) { - this.type = type; - this.content = content; - } - - public Type getType() { - return type; - } - - @SuppressWarnings("unchecked") - public T getContent() { - switch (this.type) { - case Avro: - case Json: - return (T) this.type.getJavaType().cast(this.content); - default: - throw new IllegalArgumentException("invalid type: " + this.type); - } - } - - @Override - public String toString() { - return "Schema{" + // - "type=" + type + // - ", content=" + content + // - '}'; - } - - public static Schema schema(final Schema.Type type, final Object content) { - Requires.requireNonNull(type, "Null.type"); - type.getJavaType().cast(content); - return new Schema(type, content); - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/Series.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/Series.java deleted file mode 100644 index b4f62d2..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/models/Series.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.models; - -import java.util.Collections; -import java.util.Objects; -import java.util.SortedMap; -import java.util.TreeMap; - -import io.ceresdb.Utils; -import io.ceresdb.common.util.Requires; -import io.ceresdb.common.util.Strings; - -/** - * Series in a metric. - * - * @author jiachun.fjc - */ -public class Series { - - private String metric; - private SortedMap tags; - - public String getMetric() { - return metric; - } - - public SortedMap getTags() { - return tags; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Series series = (Series) o; - return Objects.equals(metric, series.metric) && Objects.equals(tags, series.tags); - } - - @Override - public int hashCode() { - return Objects.hash(metric, tags); - } - - @Override - public String toString() { - return "Series{" + // - "metric='" + metric + '\'' + // - ", tags=" + tags + // - '}'; - } - - public static Series check(final Series ser) { - Requires.requireNonNull(ser, "Null.series"); - Requires.requireTrue(Strings.isNotBlank(ser.metric), "Empty.metric"); - Utils.checkKeywords(ser.tags.keySet().iterator()); - return ser; - } - - public static Builder newBuilder(final String metric) { - return new Builder(metric); - } - - public static class Builder { - private final String metric; - private final SortedMap tags; - - public Builder(final String metric) { - this.metric = metric; - this.tags = new TreeMap<>(); - } - - /** - * Puts a tag(key, value), replacing any existing tag of the same key. - * - * @param name tag's name - * @param value tag's value - * @return this builder - */ - public Builder tag(final String name, final String value) { - this.tags.put(name, TagValue.withString(value)); - return this; - } - - /** - * Puts a tag(key, value), replacing any existing tag of the same key. - * - * @param name tag's name - * @param value tag's value - * @return this builder - */ - public Builder tag(final String name, final TagValue value) { - this.tags.put(name, value); - return this; - } - - /** - * Constructs the series. - * - * @return a constructed series - */ - public Series build() { - final Series ser = new Series(); - ser.metric = this.metric; - ser.tags = Collections.unmodifiableSortedMap(this.tags); - return Series.check(ser); - } - - /** - * Build the series and generate a Rows.Builder to continue adding data fields. - * - * @return rows.builder - */ - public Rows.Builder toRowsBuilder() { - return toRowsBuilder(false); - } - - /** - * Build the series and generate a Rows.Builder to continue adding data fields. - * - * @param keepFieldOrder whether keep the fields write order - * @return rows.builder - */ - public Rows.Builder toRowsBuilder(final boolean keepFieldOrder) { - final Series ser = build(); - return Rows.newBuilder(ser, keepFieldOrder); - } - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/SqlQueryOk.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/SqlQueryOk.java new file mode 100644 index 0000000..2202483 --- /dev/null +++ b/ceresdb-protocol/src/main/java/io/ceresdb/models/SqlQueryOk.java @@ -0,0 +1,81 @@ +/* + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + */ +package io.ceresdb.models; + +import java.util.Collections; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Stream; + +import io.ceresdb.common.Streamable; + +/** + * Contains the success value of query. + * + */ +public class SqlQueryOk implements Streamable { + + private String sql; + private int affectedRows; + private List rows; + + public String getSql() { + return sql; + } + + public int getAffectedRows() { + return affectedRows; + } + + public int getRowCount() { + if (rows == null) { + return 0; + } + return rows.size(); + } + + public List getRowList() { + if (rows == null) { + return Collections.EMPTY_LIST; + } + return rows; + } + + public Stream map(final Function mapper) { + return this.stream().map(mapper); + } + + public Result mapToResult() { + return Result.ok(this); + } + + @Override + public Stream stream() { + if (this.getRowCount() == 0) { + return Stream.empty(); + } + return this.rows.stream(); + } + + @Override + public String toString() { + return "QueryOk{" + // + "sql='" + sql + '\'' + // + ", affectedRows=" + affectedRows + // + ", rows=" + getRowCount() + // + '}'; + } + + public static SqlQueryOk emptyOk() { + return ok("", 0, Collections.EMPTY_LIST); + } + + public static SqlQueryOk ok(final String sql, final int affectedRows, final List rows) { + final SqlQueryOk ok = new SqlQueryOk(); + ok.sql = sql; + ok.affectedRows = affectedRows; + ok.rows = rows; + return ok; + } +} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/SqlQueryRequest.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/SqlQueryRequest.java new file mode 100644 index 0000000..b05297c --- /dev/null +++ b/ceresdb-protocol/src/main/java/io/ceresdb/models/SqlQueryRequest.java @@ -0,0 +1,126 @@ +/* + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + */ +package io.ceresdb.models; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import io.ceresdb.common.util.Requires; +import io.ceresdb.common.util.Strings; + +/** + * The query request condition. + * + */ +public class SqlQueryRequest { + + private RequestContext reqCtx; + private List tables = Collections.emptyList(); + private String sql; + + protected SqlQueryRequest() { + } + + public SqlQueryRequest(String fmtSql, Object... args) { + this.sql = this.getSql(fmtSql, args); + } + + public SqlQueryRequest(List tables, String fmtSql, Object... args) { + this.tables = tables; + this.sql = this.getSql(fmtSql, args); + } + + public RequestContext getReqCtx() { + return reqCtx; + } + + public void setReqCtx(RequestContext reqCtx) { + this.reqCtx = reqCtx; + } + + public List getTables() { + return tables; + } + + public void setTables(List tables) { + this.tables = tables; + } + + public String getSql() { + return sql; + } + + @Override + public String toString() { + return "QueryRequest{" + // + "tables=" + tables + // + ", sql='" + sql + '\'' + // + '}'; + } + + public static Builder newBuilder() { + return new Builder(); + } + + public static SqlQueryRequest check(final SqlQueryRequest qr) { + Requires.requireTrue(qr.tables != null && qr.tables.size() > 0, "Empty.tables"); + Requires.requireTrue(Strings.isNotBlank(qr.sql), "Empty.sql"); + return qr; + } + + private String getSql(final String fmtSql, final Object... args) { + return String.format(fmtSql, args); + } + + public static class Builder { + private final List tables = new ArrayList<>(); + private String sql; + + /** + * Client does not parse the QL, so please fill in which metrics you queried. + * + * @param tables the tables queried. + * @return this builder + */ + public Builder forTables(final String... tables) { + this.tables.addAll(Arrays.asList(tables)); + return this; + } + + /** + * Query language to. + * + * @param sql the query language + * @return this builder + */ + public Builder sql(final String sql) { + this.sql = sql; + return this; + } + + /** + * Query language to, using the specified format string and arguments. + * + * @param fmtQl format ql string + * @param args arguments referenced by the format specifiers in the format + * QL string. If there are more arguments than format specifiers, + * the extra arguments are ignored. The number of arguments is + * variable and may be zero. + * @return this builder + */ + public Builder sql(final String fmtQl, final Object... args) { + this.sql = String.format(fmtQl, args); + return this; + } + + public SqlQueryRequest build() { + final SqlQueryRequest qr = new SqlQueryRequest(); + qr.tables = this.tables; + qr.sql = this.sql; + return qr; + } + } +} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/SqlResult.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/SqlResult.java deleted file mode 100644 index e99d45c..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/models/SqlResult.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.models; - -import java.util.List; -import java.util.Map; - -/** - * - * @author jiachun.fjc - */ -public class SqlResult { - - public static final SqlResult EMPTY_RESULT = new SqlResult(-1, null); - - private final long affectedRows; - private final List> rows; - - public SqlResult(long affectedRows, List> rows) { - this.affectedRows = affectedRows; - this.rows = rows; - } - - public long getAffectedRows() { - return affectedRows; - } - - public List> getRows() { - return rows; - } - - @Override - public String toString() { - return "SqlResult{" + // - "affectedRows=" + affectedRows + // - ", rows=" + rows + // - '}'; - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/TagValue.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/TagValue.java deleted file mode 100644 index 7f2c171..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/models/TagValue.java +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.models; - -import java.math.BigInteger; -import java.util.Optional; - -import io.ceresdb.common.util.Requires; -import io.ceresdb.common.util.UnsignedUtil; - -/** - * Tag value can be wrote to database. - * - * @author jiachun.fjc - */ -public final class TagValue extends Value { - - enum NullTag { - String(Type.String), // - Int64(Type.Int64), // - Int32(Type.Int32), // - Int16(Type.Int16), // - Int8(Type.Int8), // - Boolean(Type.Boolean), // - UInt64(Type.UInt64), // - UInt32(Type.UInt32), // - UInt16(Type.UInt16), // - UInt8(Type.UInt8), // - Timestamp(Type.Timestamp), // - Varbinary(Type.Varbinary); // - - private final TagValue value; - - NullTag(Type type) { - this.value = new TagValue(type, null); - } - } - - private TagValue(Type type, Object value) { - super(type, value); - } - - public String getString() { - return getCheckedValue(Type.String); - } - - public Optional getStringOrNull() { - return isNull() ? Optional.empty() : Optional.of(getString()); - } - - public long getInt64() { - return getCheckedValue(Type.Int64); - } - - public Optional getInt64OrNull() { - return isNull() ? Optional.empty() : Optional.of(getInt64()); - } - - public int getInt() { - return getInt32(); - } - - public Optional getIntOrNull() { - return getInt32OrNull(); - } - - public int getInt32() { - return getCheckedValue(Type.Int32); - } - - public Optional getInt32OrNull() { - return isNull() ? Optional.empty() : Optional.of(getInt32()); - } - - public int getInt16() { - return getCheckedValue(Type.Int16); - } - - public Optional getInt16OrNull() { - return isNull() ? Optional.empty() : Optional.of(getInt16()); - } - - public int getInt8() { - return getCheckedValue(Type.Int8); - } - - public Optional getInt8OrNull() { - return isNull() ? Optional.empty() : Optional.of(getInt8()); - } - - public boolean getBoolean() { - return getCheckedValue(Type.Boolean); - } - - public Optional getBooleanOrNull() { - return isNull() ? Optional.empty() : Optional.of(getBoolean()); - } - - public long getUInt64() { - return getCheckedValue(Type.UInt64); - } - - public Optional getUInt64OrNull() { - return isNull() ? Optional.empty() : Optional.of(getUInt64()); - } - - public int getUInt32() { - return getCheckedValue(Type.UInt32); - } - - public Optional getUInt32OrNull() { - return isNull() ? Optional.empty() : Optional.of(getUInt32()); - } - - public int getUInt16() { - return getCheckedValue(Type.UInt16); - } - - public Optional getUInt16OrNull() { - return isNull() ? Optional.empty() : Optional.of(getUInt16()); - } - - public int getUInt8() { - return getCheckedValue(Type.UInt8); - } - - public Optional getUInt8OrNull() { - return isNull() ? Optional.empty() : Optional.of(getUInt8()); - } - - public long getTimestamp() { - return getCheckedValue(Type.Timestamp); - } - - public Optional getTimestampOrNull() { - return isNull() ? Optional.empty() : Optional.of(getTimestamp()); - } - - public byte[] getVarbinary() { - return getCheckedValue(Type.Varbinary); - } - - public Optional getVarbinaryOrNull() { - return isNull() ? Optional.empty() : Optional.of(getVarbinary()); - } - - public static TagValue withString(final String val) { - Requires.requireNonNull(val, "Null.val"); - return new TagValue(Type.String, val); - } - - public static TagValue withStringOrNull(final String val) { - return val == null ? NullTag.String.value : withString(val); - } - - public static TagValue withInt64(final long val) { - return new TagValue(Type.Int64, val); - } - - public static TagValue withInt64OrNull(final Long val) { - return val == null ? NullTag.Int64.value : withInt64(val); - } - - public static TagValue withInt(final int val) { - return withInt32(val); - } - - public static TagValue withIntOrNull(final Integer val) { - return withInt32OrNull(val); - } - - public static TagValue withInt32(final int val) { - return new TagValue(Type.Int32, val); - } - - public static TagValue withInt32OrNull(final Integer val) { - return val == null ? NullTag.Int32.value : withInt32(val); - } - - public static TagValue withInt16(final int val) { - return new TagValue(Type.Int16, val); - } - - public static TagValue withInt16OrNull(final Integer val) { - return val == null ? NullTag.Int16.value : withInt16(val); - } - - public static TagValue withInt8(final int val) { - return new TagValue(Type.Int8, val); - } - - public static TagValue withInt8OrNull(final Integer val) { - return val == null ? NullTag.Int8.value : withInt8(val); - } - - public static TagValue withBoolean(final boolean val) { - return new TagValue(Type.Boolean, val); - } - - public static TagValue withBooleanOrNull(final Boolean val) { - return val == null ? NullTag.Boolean.value : withBoolean(val); - } - - public static TagValue withUInt64(final long val) { - return new TagValue(Type.UInt64, val); - } - - public static TagValue withUInt64OrNull(final Long val) { - return val == null ? NullTag.UInt64.value : withUInt64(val); - } - - public static TagValue withUInt64(final BigInteger val) { - Requires.requireNonNull(val, "Null.val"); - return new TagValue(Type.UInt64, UnsignedUtil.getInt64(val)); - } - - public static TagValue withUInt64OrNull(final BigInteger val) { - return val == null ? NullTag.UInt64.value : withUInt64(val); - } - - public static TagValue withUInt32(final int val) { - return new TagValue(Type.UInt32, val); - } - - public static TagValue withUInt32OrNull(final Integer val) { - return val == null ? NullTag.UInt32.value : withUInt32(val); - } - - public static TagValue withUInt16(final int val) { - return new TagValue(Type.UInt16, val); - } - - public static TagValue withUInt16OrNull(final Integer val) { - return val == null ? NullTag.UInt16.value : withUInt16(val); - } - - public static TagValue withUInt8(final int val) { - return new TagValue(Type.UInt8, val); - } - - public static TagValue withUInt8OrNull(final Integer val) { - return val == null ? NullTag.UInt8.value : withUInt8(val); - } - - public static TagValue withTimestamp(final long val) { - return new TagValue(Type.Timestamp, val); - } - - public static TagValue withTimestampOrNull(final Long val) { - return val == null ? NullTag.Timestamp.value : withTimestamp(val); - } - - public static TagValue withVarbinary(final byte[] val) { - Requires.requireNonNull(val, "Null.val"); - return new TagValue(Type.Varbinary, val); - } - - public static TagValue withVarbinaryOrNull(final byte[] val) { - return val == null ? NullTag.Varbinary.value : withVarbinary(val); - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/Value.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/Value.java index f87ece0..0f049c2 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/models/Value.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/models/Value.java @@ -1,40 +1,24 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.models; import java.util.Objects; +import java.util.Optional; import io.ceresdb.common.util.Requires; -/** - * - * @author jiachun.fjc - */ public class Value { - public enum Type { - Float64(Double.class), // + public enum DataType { String(String.class), // + Boolean(Boolean.class), // + Double(Double.class), // + Float(Float.class), // Int64(Long.class), // - Float32(Float.class), // Int32(Integer.class), // Int16(Integer.class), // Int8(Integer.class), // - Boolean(Boolean.class), // UInt64(Long.class), // UInt32(Integer.class), // UInt16(Integer.class), // @@ -44,7 +28,7 @@ public enum Type { private final Class javaType; - Type(Class javaType) { + DataType(Class javaType) { this.javaType = javaType; } @@ -53,15 +37,38 @@ public Class getJavaType() { } } - private final Type type; - private final Object value; + enum NullValue { + String(DataType.String), // + Boolean(DataType.Boolean), // + Int64(DataType.Int64), // + Double(DataType.Double), // + Float(DataType.Float), // + Int32(DataType.Int32), // + Int16(DataType.Int16), // + Int8(DataType.Int8), // + UInt64(DataType.UInt64), // + UInt32(DataType.UInt32), // + UInt16(DataType.UInt16), // + UInt8(DataType.UInt8), // + Timestamp(DataType.Timestamp), // + Varbinary(DataType.Varbinary); // + + private final Value value; + + NullValue(DataType type) { + this.value = new Value(type, null); + } + } + + private final DataType type; + private final Object value; - protected Value(Type type, Object value) { + public Value(DataType type, Object value) { this.type = type; this.value = value; } - public Type getType() { + public DataType getDataType() { return type; } @@ -90,7 +97,7 @@ public int hashCode() { public String toString() { return "Value{" + // "type=" + type + // - ", value=" + value + // + ",value=" + value + // '}'; } @@ -98,13 +105,239 @@ public boolean isNull() { return this.value == null; } - public static boolean isNull(final T value) { - return value == null || value.isNull(); + public String getString() { + return getCheckedValue(DataType.String); + } + + public Optional getStringOrNull() { + return isNull() ? Optional.empty() : Optional.of(getString()); + } + + public boolean getBoolean() { + return getCheckedValue(DataType.Boolean); + } + + public Optional getBooleanOrNull() { + return isNull() ? Optional.empty() : Optional.of(getBoolean()); + } + + public double getDouble() { + return getCheckedValue(DataType.Double); + } + + public Optional getDoubleOrNull() { + return isNull() ? Optional.empty() : Optional.of(getDouble()); + } + + public float getFloat() { + return getCheckedValue(DataType.Float); + } + + public Optional getFloatOrNull() { + return isNull() ? Optional.empty() : Optional.of(getFloat()); + } + + public long getInt64() { + return getCheckedValue(DataType.Int64); + } + + public Optional getInt64OrNull() { + return isNull() ? Optional.empty() : Optional.of(getInt64()); + } + + public int getInt32() { + return getCheckedValue(DataType.Int32); + } + + public Optional getInt32OrNull() { + return isNull() ? Optional.empty() : Optional.of(getInt32()); + } + + public int getInt16() { + return getCheckedValue(DataType.Int16); + } + + public Optional getInt16OrNull() { + return isNull() ? Optional.empty() : Optional.of(getInt16()); + } + + public int getInt8() { + return getCheckedValue(DataType.Int8); + } + + public Optional getInt8OrNull() { + return isNull() ? Optional.empty() : Optional.of(getInt8()); + } + + public long getUInt64() { + return getCheckedValue(DataType.UInt64); + } + + public Optional getUInt64OrNull() { + return isNull() ? Optional.empty() : Optional.of(getUInt64()); + } + + public int getUInt32() { + return getCheckedValue(DataType.UInt32); + } + + public Optional getUInt32OrNull() { + return isNull() ? Optional.empty() : Optional.of(getUInt32()); + } + + public int getUInt16() { + return getCheckedValue(DataType.UInt16); + } + + public Optional getUInt16OrNull() { + return isNull() ? Optional.empty() : Optional.of(getUInt16()); + } + + public int getUInt8() { + return getCheckedValue(DataType.UInt8); + } + + public Optional getUInt8OrNull() { + return isNull() ? Optional.empty() : Optional.of(getUInt8()); + } + + public long getTimestamp() { + return getCheckedValue(DataType.Timestamp); + } + + public Optional getTimestampOrNull() { + return isNull() ? Optional.empty() : Optional.of(getTimestamp()); + } + + public byte[] getVarbinary() { + return getCheckedValue(DataType.Varbinary); + } + + public Optional getVarbinaryOrNull() { + return isNull() ? Optional.empty() : Optional.of(getVarbinary()); } @SuppressWarnings("unchecked") - protected T getCheckedValue(final Type type) { + private T getCheckedValue(final DataType type) { Requires.requireTrue(this.type == type, "Invalid type %s, expected is %s", this.type, type); return (T) type.getJavaType().cast(this.value); } + + public static Value withString(final String val) { + Requires.requireNonNull(val, "Null.val"); + return new Value(DataType.String, val); + } + + public static Value withStringOrNull(final String val) { + return val == null ? Value.NullValue.String.value : withString(val); + } + + public static Value withBoolean(final boolean val) { + return new Value(DataType.Boolean, val); + } + + public static Value withBooleanOrNull(final Boolean val) { + return val == null ? Value.NullValue.Boolean.value : withBoolean(val); + } + + public static Value withDouble(final double val) { + return new Value(DataType.Double, val); + } + + public static Value withDoubleOrNull(final Double val) { + return val == null ? Value.NullValue.Double.value : withDouble(val); + } + + public static Value withFloat(final float val) { + return new Value(DataType.Float, val); + } + + public static Value withFloatOrNull(final Float val) { + return val == null ? Value.NullValue.Float.value : withFloat(val); + } + + public static Value withInt64(final long val) { + return new Value(DataType.Int64, val); + } + + public static Value withInt64OrNull(final Long val) { + return val == null ? Value.NullValue.Int64.value : withInt64(val); + } + + public static Value withInt32(final int val) { + return new Value(DataType.Int32, val); + } + + public static Value withInt32OrNull(final Integer val) { + return val == null ? Value.NullValue.Int32.value : withInt32(val); + } + + public static Value withInt16(final int val) { + return new Value(DataType.Int16, val); + } + + public static Value withInt16OrNull(final Integer val) { + return val == null ? Value.NullValue.Int16.value : withInt16(val); + } + + public static Value withInt8(final int val) { + return new Value(DataType.Int8, val); + } + + public static Value withInt8OrNull(final Integer val) { + return val == null ? Value.NullValue.Int8.value : withInt8(val); + } + + public static Value withUInt64(final long val) { + return new Value(DataType.UInt64, val); + } + + public static Value withUInt64OrNull(final Long val) { + return val == null ? Value.NullValue.UInt64.value : withUInt64(val); + } + + public static Value withUInt32(final int val) { + return new Value(DataType.UInt32, val); + } + + public static Value withUInt32OrNull(final Integer val) { + return val == null ? Value.NullValue.UInt32.value : withUInt32(val); + } + + public static Value withUInt16(final int val) { + return new Value(DataType.UInt16, val); + } + + public static Value withUInt16OrNull(final Integer val) { + return val == null ? Value.NullValue.UInt16.value : withUInt16(val); + } + + public static Value withUInt8(final int val) { + return new Value(DataType.UInt8, val); + } + + public static Value withUInt8OrNull(final Integer val) { + return val == null ? Value.NullValue.UInt8.value : withUInt8(val); + } + + public static Value withTimestamp(final long val) { + return new Value(DataType.Timestamp, val); + } + + public static Value withTimestampOrNull(final Long val) { + return val == null ? Value.NullValue.Timestamp.value : withTimestamp(val); + } + + public static Value withVarbinary(final byte[] val) { + Requires.requireNonNull(val, "Null.val"); + return new Value(DataType.Varbinary, val); + } + + public static Value withVarbinaryOrNull(final byte[] val) { + return val == null ? Value.NullValue.Varbinary.value : withVarbinary(val); + } + + public static boolean isNull(final T value) { + return value == null || value.isNull(); + } } diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/WriteOk.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/WriteOk.java index eb03e06..0a65405 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/models/WriteOk.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/models/WriteOk.java @@ -1,22 +1,10 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.models; import java.util.Collection; +import java.util.Set; import io.ceresdb.common.OptKeys; import io.ceresdb.common.util.SystemPropertyUtil; @@ -24,7 +12,6 @@ /** * Contains the success value of write. * - * @author jiachun.fjc */ public class WriteOk { @@ -40,7 +27,7 @@ public static boolean isCollectWroteDetail() { /** * Empty if {@link #COLLECT_WROTE_DETAIL == false}. */ - private Collection metrics; + private Set tables; public int getSuccess() { return success; @@ -58,21 +45,21 @@ public void setFailed(int failed) { this.failed = failed; } - public Collection getMetrics() { - return metrics; + public Collection getTables() { + return tables; } - public void setMetrics(Collection metrics) { - this.metrics = metrics; + public void setTables(Set tables) { + this.tables = tables; } public WriteOk combine(final WriteOk other) { this.success += other.success; this.failed += other.failed; - if (this.metrics == null) { - this.metrics = other.metrics; - } else if (other.metrics != null) { - this.metrics.addAll(other.metrics); + if (this.tables == null) { + this.tables = other.tables; + } else if (other.tables != null) { + this.tables.addAll(other.tables); } return this; } @@ -86,7 +73,7 @@ public String toString() { return "WriteOk{" + // "success=" + success + // ", failed=" + failed + // - ", metrics=" + metrics + // + ", tables=" + tables + // '}'; } @@ -94,11 +81,11 @@ public static WriteOk emptyOk() { return ok(0, 0, null); } - public static WriteOk ok(final int success, final int failed, final Collection metrics) { + public static WriteOk ok(final int success, final int failed, final Set tables) { final WriteOk ok = new WriteOk(); ok.success = success; ok.failed = failed; - ok.metrics = metrics; + ok.tables = tables; return ok; } } diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/models/WriteRequest.java b/ceresdb-protocol/src/main/java/io/ceresdb/models/WriteRequest.java new file mode 100644 index 0000000..4894088 --- /dev/null +++ b/ceresdb-protocol/src/main/java/io/ceresdb/models/WriteRequest.java @@ -0,0 +1,32 @@ +/* + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + */ +package io.ceresdb.models; + +import java.util.List; + +public class WriteRequest { + private RequestContext reqCtx; + private List points; + + public WriteRequest(List points) { + this.points = points; + } + + public WriteRequest(RequestContext reqCtx, List points) { + this.reqCtx = reqCtx; + this.points = points; + } + + public RequestContext getReqCtx() { + return reqCtx; + } + + public void setReqCtx(RequestContext reqCtx) { + this.reqCtx = reqCtx; + } + + public List getPoints() { + return points; + } +} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/options/CeresDBOptions.java b/ceresdb-protocol/src/main/java/io/ceresdb/options/CeresDBOptions.java index b6e2bbd..62d7978 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/options/CeresDBOptions.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/options/CeresDBOptions.java @@ -1,46 +1,30 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.options; import java.util.concurrent.Executor; -import io.ceresdb.LimitedPolicy; +import io.ceresdb.limit.LimitedPolicy; import io.ceresdb.RouteMode; import io.ceresdb.common.Copiable; import io.ceresdb.common.Endpoint; -import io.ceresdb.common.Tenant; import io.ceresdb.common.util.Requires; import io.ceresdb.rpc.RpcOptions; /** * CeresDB client options. * - * @author jiachun.fjc */ public class CeresDBOptions implements Copiable { - private Endpoint clusterAddress; - private Executor asyncWritePool; - private Executor asyncReadPool; - private Tenant tenant; - private RpcOptions rpcOptions; - private RouterOptions routerOptions; - private WriteOptions writeOptions; - private QueryOptions queryOptions; - private ManagementOptions managementOptions; + private Endpoint clusterAddress; + private String database; + private Executor asyncWritePool; + private Executor asyncReadPool; + private RpcOptions rpcOptions; + private RouterOptions routerOptions; + private WriteOptions writeOptions; + private QueryOptions queryOptions; public Endpoint getClusterAddress() { return clusterAddress; @@ -50,6 +34,14 @@ public void setClusterAddress(Endpoint clusterAddress) { this.clusterAddress = clusterAddress; } + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + } + public Executor getAsyncWritePool() { return asyncWritePool; } @@ -66,14 +58,6 @@ public void setAsyncReadPool(Executor asyncReadPool) { this.asyncReadPool = asyncReadPool; } - public Tenant getTenant() { - return tenant; - } - - public void setTenant(Tenant tenant) { - this.tenant = tenant; - } - public RpcOptions getRpcOptions() { return rpcOptions; } @@ -106,23 +90,13 @@ public void setQueryOptions(QueryOptions queryOptions) { this.queryOptions = queryOptions; } - public ManagementOptions getManagementOptions() { - return managementOptions; - } - - public void setManagementOptions(ManagementOptions managementOptions) { - this.managementOptions = managementOptions; - } - @Override public CeresDBOptions copy() { final CeresDBOptions copy = new CeresDBOptions(); copy.clusterAddress = this.clusterAddress; + copy.database = this.database; copy.asyncWritePool = this.asyncWritePool; copy.asyncReadPool = this.asyncReadPool; - if (this.tenant != null) { - copy.tenant = this.tenant.copy(); - } if (this.rpcOptions != null) { copy.rpcOptions = this.rpcOptions.copy(); } @@ -135,9 +109,6 @@ public CeresDBOptions copy() { if (this.queryOptions != null) { copy.queryOptions = this.queryOptions.copy(); } - if (this.managementOptions != null) { - copy.managementOptions = this.managementOptions.copy(); - } return copy; } @@ -145,23 +116,22 @@ public CeresDBOptions copy() { public String toString() { return "CeresDBOptions{" + // "clusterAddress=" + clusterAddress + // + ", database=" + database + // ", asyncWritePool=" + asyncWritePool + // ", asyncReadPool=" + asyncReadPool + // - ", tenant=" + tenant + // ", rpcOptions=" + rpcOptions + // ", routerOptions=" + routerOptions + // ", writeOptions=" + writeOptions + // ", queryOptions=" + queryOptions + // - ", managementOptions=" + managementOptions + // '}'; } public static CeresDBOptions check(final CeresDBOptions opts) { Requires.requireNonNull(opts, "CeresDBOptions.opts"); Requires.requireNonNull(opts.getClusterAddress(), "CeresDBOptions.clusterAddress"); - Requires.requireNonNull(opts.getTenant(), "CeresDBOptions.tenant"); Requires.requireNonNull(opts.getRpcOptions(), "CeresDBOptions.rpcOptions"); Requires.requireNonNull(opts.getRouterOptions(), "CeresDBOptions.RouterOptions"); + Requires.requireNonNull(opts.getRouterOptions().getRouteMode(), "CeresDBOptions.RouterOptions.RouteMode"); Requires.requireNonNull(opts.getWriteOptions(), "CeresDBOptions.writeOptions"); Requires.requireNonNull(opts.getQueryOptions(), "CeresDBOptions.queryOptions"); return opts; @@ -171,10 +141,11 @@ public static CeresDBOptions check(final CeresDBOptions opts) { * Create a new builder for CeresDBOptions. * * @param clusterAddress cluster address, for read/write data + * @Param routeMode direct or proxy in RouteMode * @return builder */ - public static Builder newBuilder(final Endpoint clusterAddress) { - return new Builder(clusterAddress); + public static Builder newBuilder(final Endpoint clusterAddress, RouteMode routeMode) { + return new Builder(clusterAddress, routeMode); } /** @@ -182,35 +153,22 @@ public static Builder newBuilder(final Endpoint clusterAddress) { * * @param clusterHost cluster ip/host, for read/write data * @param clusterPort cluster port + * @param routeMode direct or proxy in RouteMode * @return builder */ - public static Builder newBuilder(final String clusterHost, final int clusterPort) { - return newBuilder(Endpoint.of(clusterHost, clusterPort)); - } - - /** - * Create a new builder for CeresDBOptions. - * - * @param clusterHost cluster ip/host, for read/write data - * @param clusterPort cluster port - * @param managementPort database management port, such as creating tables - * @return builder - */ - public static Builder newBuilder(final String clusterHost, final int clusterPort, final int managementPort) { - return new Builder(Endpoint.of(clusterHost, clusterPort)) // - .managementAddress(Endpoint.of(clusterHost, managementPort)); + public static Builder newBuilder(final String clusterHost, final int clusterPort, RouteMode routeMode) { + return newBuilder(Endpoint.of(clusterHost, clusterPort), routeMode); } public static final class Builder { // The only constant address of this cluster. private final Endpoint clusterAddress; - // Database management address, such as creating tables. - private Endpoint managementAddress; + // The routeMode for sdk, only Proxy and Direct support now. + private RouteMode routeMode; + private String database; // Asynchronous thread pool, which is used to handle various asynchronous tasks in the SDK. private Executor asyncWritePool; private Executor asyncReadPool; - // Tenant - private Tenant tenant; // Rpc options, in general, the default configuration is fine. private RpcOptions rpcOptions = RpcOptions.newDefault(); // Write options @@ -219,7 +177,7 @@ public static final class Builder { // In the case of routing table failure or some other retry able error, a retry of the write is attempted. private int writeMaxRetries = 1; // Write flow control: maximum number of data rows in-flight. - private int maxInFlightWriteRows = 8192; + private int maxInFlightWritePoints = 8192; // Write flow control: limited policy private LimitedPolicy writeLimitedPolicy = LimitedPolicy.defaultWriteLimitedPolicy(); // Query options @@ -238,35 +196,21 @@ public static final class Builder { // all route tables are refreshed every 30 seconds. private long routeTableRefreshPeriodSeconds = 30; - /** Route mode for request - @see RouteMode - **/ - private RouteMode routeMode = RouteMode.CLUSTER; - - public Builder(Endpoint clusterAddress) { + public Builder(Endpoint clusterAddress, RouteMode routeMode) { this.clusterAddress = clusterAddress; + this.routeMode = routeMode; } /** - * Database management address, such as creating tables. * - * @param managementAddress management address, it may have the same IP as - * the cluster address, but it must have a different - * port + * @param database the database name * @return this builder */ - public Builder managementAddress(final Endpoint managementAddress) { - this.managementAddress = managementAddress; + public Builder database(final String database) { + this.database = database; return this; } - /** - * @see #managementAddress(Endpoint) - */ - public Builder managementAddress(final String host, final int port) { - return managementAddress(Endpoint.of(host, port)); - } - /** * Asynchronous thread pool, which is used to handle various asynchronous * tasks in the SDK (You are using a purely asynchronous SDK). If you do not @@ -285,34 +229,6 @@ public Builder asyncPool(final Executor asyncWritePool, final Executor asyncRead return this; } - /** - * @see #tenant(String, String, String) - * - * @param tenant the tenant name - * @param token don't tell the secret to anyone, heaven knows - * and earth knows, you know and I know. ＼（＾▽＾）／ - * @return this builder - */ - public Builder tenant(final String tenant, final String token) { - this.tenant = Tenant.of(tenant, null, token); - return this; - } - - /** - * Sets a tenant. - * - * @param tenant the tenant name - * @param childTenant default subtenant, which is used if you do not - * re-specify a subtenant each time you make a call. - * @param token don't tell the secret to anyone, heaven knows - * and earth knows, you know and I know. ＼（＾▽＾）／ - * @return this builder - */ - public Builder tenant(final String tenant, final String childTenant, final String token) { - this.tenant = Tenant.of(tenant, childTenant, token); - return this; - } - /** * Sets the RPC options, in general, the default configuration is fine. * @@ -350,11 +266,11 @@ public Builder writeMaxRetries(final int maxRetries) { /** * Write flow control: maximum number of data rows in-flight. * - * @param maxInFlightWriteRows maximum number of data rows in-flight + * @param maxInFlightWritePoints maximum number of data rows in-flight * @return this builder */ - public Builder maxInFlightWriteRows(final int maxInFlightWriteRows) { - this.maxInFlightWriteRows = maxInFlightWriteRows; + public Builder maxInFlightWritePoints(final int maxInFlightWritePoints) { + this.maxInFlightWritePoints = maxInFlightWritePoints; return this; } @@ -438,18 +354,6 @@ public Builder routeTableRefreshPeriodSeconds(final long routeTableRefreshPeriod return this; } - /** - * Route mode for request - * @see RouteMode - * - * @param routeMode route mode for request - * @return this builder - */ - public Builder routeMode(final RouteMode routeMode) { - this.routeMode = routeMode; - return this; - } - /** * A good start, happy coding. * @@ -458,9 +362,9 @@ public Builder routeMode(final RouteMode routeMode) { public CeresDBOptions build() { final CeresDBOptions opts = new CeresDBOptions(); opts.clusterAddress = this.clusterAddress; + opts.database = this.database; opts.asyncWritePool = asyncWritePool; opts.asyncReadPool = asyncReadPool; - opts.tenant = this.tenant; opts.rpcOptions = this.rpcOptions; opts.routerOptions = new RouterOptions(); opts.routerOptions.setClusterAddress(this.clusterAddress); @@ -472,18 +376,12 @@ public CeresDBOptions build() { opts.writeOptions = new WriteOptions(); opts.writeOptions.setMaxWriteSize(this.maxWriteSize); opts.writeOptions.setMaxRetries(this.writeMaxRetries); - opts.writeOptions.setMaxInFlightWriteRows(this.maxInFlightWriteRows); + opts.writeOptions.setMaxInFlightWritePoints(this.maxInFlightWritePoints); opts.writeOptions.setLimitedPolicy(this.writeLimitedPolicy); opts.queryOptions = new QueryOptions(); opts.queryOptions.setMaxRetries(this.readMaxRetries); opts.queryOptions.setMaxInFlightQueryRequests(this.maxInFlightQueryRequests); opts.queryOptions.setLimitedPolicy(this.queryLimitedPolicy); - if (this.managementAddress != null) { - final ManagementOptions mOpts = new ManagementOptions(); - mOpts.setManagementAddress(this.managementAddress); - mOpts.setTenant(this.tenant); - opts.managementOptions = mOpts; - } return CeresDBOptions.check(opts); } } diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/options/ManagementOptions.java b/ceresdb-protocol/src/main/java/io/ceresdb/options/ManagementOptions.java deleted file mode 100644 index 3b09fcb..0000000 --- a/ceresdb-protocol/src/main/java/io/ceresdb/options/ManagementOptions.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.options; - -import io.ceresdb.RouterClient; -import io.ceresdb.common.Copiable; -import io.ceresdb.common.Endpoint; -import io.ceresdb.common.Tenant; - -/** - * - * @author jiachun.fjc - */ -public class ManagementOptions implements Copiable { - - private Endpoint managementAddress; - private RouterClient routerClient; - private Tenant tenant; - private boolean checkSql = true; - - public Endpoint getManagementAddress() { - return managementAddress; - } - - public void setManagementAddress(Endpoint managementAddress) { - this.managementAddress = managementAddress; - } - - public RouterClient getRouterClient() { - return routerClient; - } - - public void setRouterClient(RouterClient routerClient) { - this.routerClient = routerClient; - } - - public Tenant getTenant() { - return tenant; - } - - public void setTenant(Tenant tenant) { - this.tenant = tenant; - } - - public boolean isCheckSql() { - return checkSql; - } - - public void setCheckSql(boolean checkSql) { - this.checkSql = checkSql; - } - - @Override - public ManagementOptions copy() { - final ManagementOptions opts = new ManagementOptions(); - opts.managementAddress = this.managementAddress; - opts.routerClient = this.routerClient; - if (this.tenant != null) { - opts.tenant = this.tenant.copy(); - } - opts.checkSql = this.checkSql; - return opts; - } - - @Override - public String toString() { - return "ManagementOptions{" + // - "managementAddress=" + managementAddress + // - "routerClient=" + routerClient + // - ", tenant=" + tenant + // - ", checkSql=" + checkSql + // - '}'; - } -} diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/options/QueryOptions.java b/ceresdb-protocol/src/main/java/io/ceresdb/options/QueryOptions.java index 640efb7..baef4be 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/options/QueryOptions.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/options/QueryOptions.java @@ -1,34 +1,20 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.options; import java.util.concurrent.Executor; -import io.ceresdb.LimitedPolicy; +import io.ceresdb.limit.LimitedPolicy; import io.ceresdb.RouterClient; import io.ceresdb.common.Copiable; /** * Query options. * - * @author jiachun.fjc */ public class QueryOptions implements Copiable { - + private String database; private RouterClient routerClient; private Executor asyncPool; @@ -38,6 +24,14 @@ public class QueryOptions implements Copiable { private int maxInFlightQueryRequests = 8; private LimitedPolicy limitedPolicy = LimitedPolicy.defaultQueryLimitedPolicy(); + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + } + public RouterClient getRouterClient() { return routerClient; } @@ -81,6 +75,7 @@ public void setLimitedPolicy(LimitedPolicy limitedPolicy) { @Override public QueryOptions copy() { final QueryOptions opts = new QueryOptions(); + opts.database = this.database; opts.routerClient = this.routerClient; opts.asyncPool = this.asyncPool; opts.maxRetries = this.maxRetries; @@ -92,11 +87,12 @@ public QueryOptions copy() { @Override public String toString() { return "QueryOptions{" + // - "routerClient=" + routerClient + // - "asyncPool=" + asyncPool + // - "maxRetries=" + maxRetries + // - "maxInFlightQueryRequests=" + maxInFlightQueryRequests + // - "limitedPolicy=" + limitedPolicy + // + ", database=" + database + // + ", routerClient=" + routerClient + // + ", asyncPool=" + asyncPool + // + ", maxRetries=" + maxRetries + // + ", maxInFlightQueryRequests=" + maxInFlightQueryRequests + // + ", limitedPolicy=" + limitedPolicy + // '}'; } } diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/options/RouterOptions.java b/ceresdb-protocol/src/main/java/io/ceresdb/options/RouterOptions.java index 3ec8e3a..4c67dc5 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/options/RouterOptions.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/options/RouterOptions.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.options; @@ -24,12 +11,12 @@ /** * Router options. * - * @author jiachun.fjc */ public class RouterOptions implements Copiable { private RpcClient rpcClient; private Endpoint clusterAddress; + private RouteMode routeMode; // Specifies the maximum number of routing table caches. When the number reaches the limit, the ones that // have not been used for a long time are cleared first private int maxCachedSize = 10_000; @@ -39,8 +26,6 @@ public class RouterOptions implements Copiable { // all route tables are refreshed every 30 seconds. private long refreshPeriodSeconds = 30; - private RouteMode routeMode = RouteMode.CLUSTER; - public RpcClient getRpcClient() { return rpcClient; } diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/options/WriteOptions.java b/ceresdb-protocol/src/main/java/io/ceresdb/options/WriteOptions.java index ee7e945..e2292e0 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/options/WriteOptions.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/options/WriteOptions.java @@ -1,34 +1,21 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.options; import java.util.concurrent.Executor; -import io.ceresdb.LimitedPolicy; +import io.ceresdb.limit.LimitedPolicy; import io.ceresdb.RouterClient; import io.ceresdb.common.Copiable; /** * Write options. * - * @author jiachun.fjc */ public class WriteOptions implements Copiable { + private String database; private RouterClient routerClient; private Executor asyncPool; @@ -36,9 +23,17 @@ public class WriteOptions implements Copiable { private int maxRetries = 1; // In the case of routing table failure or some other retry able error, a retry of the write is attempted. private int maxWriteSize = 512; - // Write flow limit: maximum number of data rows in-flight. - private int maxInFlightWriteRows = 8192; - private LimitedPolicy limitedPolicy = LimitedPolicy.defaultWriteLimitedPolicy(); + // Write flow limit: maximum number of data points in-flight. + private int maxInFlightWritePoints = 8192; + private LimitedPolicy limitedPolicy = LimitedPolicy.defaultWriteLimitedPolicy(); + + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + } public RouterClient getRoutedClient() { return routerClient; @@ -72,12 +67,12 @@ public void setMaxWriteSize(int maxWriteSize) { this.maxWriteSize = maxWriteSize; } - public int getMaxInFlightWriteRows() { - return maxInFlightWriteRows; + public int getMaxInFlightWritePoints() { + return maxInFlightWritePoints; } - public void setMaxInFlightWriteRows(int maxInFlightWriteRows) { - this.maxInFlightWriteRows = maxInFlightWriteRows; + public void setMaxInFlightWritePoints(int maxInFlightWritePoints) { + this.maxInFlightWritePoints = maxInFlightWritePoints; } public LimitedPolicy getLimitedPolicy() { @@ -91,11 +86,12 @@ public void setLimitedPolicy(LimitedPolicy limitedPolicy) { @Override public WriteOptions copy() { final WriteOptions opts = new WriteOptions(); + opts.database = this.database; opts.routerClient = this.routerClient; opts.asyncPool = this.asyncPool; opts.maxRetries = this.maxRetries; opts.maxWriteSize = this.maxWriteSize; - opts.maxInFlightWriteRows = this.maxInFlightWriteRows; + opts.maxInFlightWritePoints = this.maxInFlightWritePoints; opts.limitedPolicy = this.limitedPolicy; return opts; } @@ -103,11 +99,12 @@ public WriteOptions copy() { @Override public String toString() { return "WriteOptions{" + // - "routerClient=" + routerClient + // + ", database=" + database + // + ", routerClient=" + routerClient + // ", globalAsyncPool=" + asyncPool + // ", maxRetries=" + maxRetries + // ", maxWriteSize=" + maxWriteSize + // - ", maxInFlightWriteRows=" + maxInFlightWriteRows + // + ", maxInFlightWritePoints=" + maxInFlightWritePoints + // ", limitedPolicy=" + limitedPolicy + // '}'; } diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/signal/ClearRouteCacheSignalHandler.java b/ceresdb-protocol/src/main/java/io/ceresdb/signal/ClearRouteCacheSignalHandler.java index 2a86918..1e6fec8 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/signal/ClearRouteCacheSignalHandler.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/signal/ClearRouteCacheSignalHandler.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.signal; @@ -30,7 +17,6 @@ /** * A signal handler that can clear the route cache. * - * @author jiachun.fjc */ @SPI(priority = 94) public class ClearRouteCacheSignalHandler implements SignalHandler { diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/signal/DisplaySignalHandler.java b/ceresdb-protocol/src/main/java/io/ceresdb/signal/DisplaySignalHandler.java index 92e7d54..e061934 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/signal/DisplaySignalHandler.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/signal/DisplaySignalHandler.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.signal; @@ -38,7 +25,6 @@ /** * A signal handler that can display all client instance's memory state. * - * @author jiachun.fjc */ @SPI(priority = 98) public class DisplaySignalHandler implements SignalHandler { diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/signal/MetricsSignalHandler.java b/ceresdb-protocol/src/main/java/io/ceresdb/signal/MetricsSignalHandler.java index 12c8692..3094ae0 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/signal/MetricsSignalHandler.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/signal/MetricsSignalHandler.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.signal; @@ -35,7 +22,6 @@ /** * A signal handle that can write the metrics into a file. * - * @author jiachun.fjc */ @SPI(priority = 97) public class MetricsSignalHandler implements SignalHandler { diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/signal/RwLoggingSignalHandler.java b/ceresdb-protocol/src/main/java/io/ceresdb/signal/RwLoggingSignalHandler.java index 1a6f809..9eeac89 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/signal/RwLoggingSignalHandler.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/signal/RwLoggingSignalHandler.java @@ -1,25 +1,12 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.signal; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.ceresdb.Utils; +import io.ceresdb.util.Utils; import io.ceresdb.common.SPI; import io.ceresdb.common.signal.FileSignal; import io.ceresdb.common.signal.Signal; @@ -29,7 +16,6 @@ /** * A signal handler that can reset RW_LOGGING by {@link Utils#resetRwLogging()}. * - * @author jiachun.fjc */ @SPI(priority = 95) public class RwLoggingSignalHandler implements SignalHandler { diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/signal/ShowHelpSignalHandler.java b/ceresdb-protocol/src/main/java/io/ceresdb/signal/ShowHelpSignalHandler.java index 22ef2f2..7e0af3f 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/signal/ShowHelpSignalHandler.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/signal/ShowHelpSignalHandler.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.signal; @@ -25,9 +12,6 @@ import io.ceresdb.common.signal.FileSignals; import io.ceresdb.common.signal.SignalHandler; -/** - * @author jiachun.fjc - */ @SPI(priority = 99) public class ShowHelpSignalHandler implements SignalHandler { diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/signal/ShowRouteCacheSignalHandler.java b/ceresdb-protocol/src/main/java/io/ceresdb/signal/ShowRouteCacheSignalHandler.java index 9d8cec1..447cd3c 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/signal/ShowRouteCacheSignalHandler.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/signal/ShowRouteCacheSignalHandler.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.signal; @@ -38,7 +25,6 @@ /** * A signal handler that can log all of the route cache items to file. * - * @author jiachun.fjc */ @SPI(priority = 96) public class ShowRouteCacheSignalHandler implements SignalHandler { diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/RpcServiceRegister.java b/ceresdb-protocol/src/main/java/io/ceresdb/util/RpcServiceRegister.java similarity index 62% rename from ceresdb-protocol/src/main/java/io/ceresdb/RpcServiceRegister.java rename to ceresdb-protocol/src/main/java/io/ceresdb/util/RpcServiceRegister.java index f2c5dbf..3f52411 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/RpcServiceRegister.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/util/RpcServiceRegister.java @@ -1,20 +1,7 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ -package io.ceresdb; +package io.ceresdb.util; import io.ceresdb.common.OptKeys; import io.ceresdb.common.util.SystemPropertyUtil; @@ -22,10 +9,6 @@ import io.ceresdb.rpc.MethodDescriptor; import io.ceresdb.rpc.RpcFactoryProvider; -/** - * - * @author jiachun.fjc - */ public class RpcServiceRegister { private static final double WRITE_LIMIT_PERCENT = writeLimitPercent(); @@ -53,17 +36,17 @@ public static void registerStorageService() { Storage.WriteRequest.getDefaultInstance(), // Storage.WriteResponse.getDefaultInstance()); RpcFactoryProvider.getRpcFactory().register( - MethodDescriptor.of(String.format(STORAGE_METHOD_TEMPLATE, "Query"), MethodDescriptor.MethodType.UNARY, - 1 - WRITE_LIMIT_PERCENT), // - Storage.QueryRequest.class, // - Storage.QueryRequest.getDefaultInstance(), // - Storage.QueryResponse.getDefaultInstance()); + MethodDescriptor.of(String.format(STORAGE_METHOD_TEMPLATE, "SqlQuery"), + MethodDescriptor.MethodType.UNARY, 1 - WRITE_LIMIT_PERCENT), // + Storage.SqlQueryRequest.class, // + Storage.SqlQueryRequest.getDefaultInstance(), // + Storage.SqlQueryResponse.getDefaultInstance()); RpcFactoryProvider.getRpcFactory().register( - MethodDescriptor.of(String.format(STORAGE_METHOD_TEMPLATE, "StreamQuery"), + MethodDescriptor.of(String.format(STORAGE_METHOD_TEMPLATE, "StreamSqlQuery"), MethodDescriptor.MethodType.SERVER_STREAMING), // - Storage.QueryRequest.class, // - Storage.QueryRequest.getDefaultInstance(), // - Storage.QueryResponse.getDefaultInstance()); + Storage.SqlQueryRequest.class, // + Storage.SqlQueryRequest.getDefaultInstance(), // + Storage.SqlQueryResponse.getDefaultInstance()); } private static double writeLimitPercent() { diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/StreamWriteBuf.java b/ceresdb-protocol/src/main/java/io/ceresdb/util/StreamWriteBuf.java similarity index 57% rename from ceresdb-protocol/src/main/java/io/ceresdb/StreamWriteBuf.java rename to ceresdb-protocol/src/main/java/io/ceresdb/util/StreamWriteBuf.java index 0f2dac0..b3a4865 100644 --- a/ceresdb-protocol/src/main/java/io/ceresdb/StreamWriteBuf.java +++ b/ceresdb-protocol/src/main/java/io/ceresdb/util/StreamWriteBuf.java @@ -1,20 +1,7 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ -package io.ceresdb; +package io.ceresdb.util; import java.util.Collection; import java.util.concurrent.CompletableFuture; @@ -22,7 +9,6 @@ /** * A stream-write buffer. * - * @author jiachun.fjc */ public interface StreamWriteBuf { diff --git a/ceresdb-protocol/src/main/java/io/ceresdb/util/Utils.java b/ceresdb-protocol/src/main/java/io/ceresdb/util/Utils.java new file mode 100644 index 0000000..eb88933 --- /dev/null +++ b/ceresdb-protocol/src/main/java/io/ceresdb/util/Utils.java @@ -0,0 +1,634 @@ +/* + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + */ +package io.ceresdb.util; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import io.ceresdb.Route; +import io.ceresdb.common.Display; +import io.ceresdb.common.Endpoint; +import io.ceresdb.common.OptKeys; +import io.ceresdb.common.util.ExecutorServiceHelper; +import io.ceresdb.common.util.MetricsUtil; +import io.ceresdb.common.util.NamedThreadFactory; +import io.ceresdb.common.util.ObjectPool; +import io.ceresdb.common.util.Requires; +import io.ceresdb.common.util.SharedScheduledPool; +import io.ceresdb.common.util.Spines; +import io.ceresdb.common.util.SystemPropertyUtil; +import io.ceresdb.common.util.ThreadPoolUtil; +import io.ceresdb.models.Err; +import io.ceresdb.models.Keyword; +import io.ceresdb.models.Point; +import io.ceresdb.models.Row; +import io.ceresdb.models.SqlQueryOk; +import io.ceresdb.models.Result; +import io.ceresdb.models.Value; +import io.ceresdb.models.WriteOk; +import io.ceresdb.proto.internal.Common; +import io.ceresdb.proto.internal.Storage; +import io.ceresdb.rpc.Observer; + +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeStampMilliVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.UInt1Vector; +import org.apache.arrow.vector.UInt2Vector; +import org.apache.arrow.vector.UInt4Vector; +import org.apache.arrow.vector.UInt8Vector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdInputStream; +import com.google.protobuf.ByteString; +import com.google.protobuf.ByteStringHelper; + +/** + * Utils for CeresDBClient. + * + */ +public final class Utils { + + public static final String DB_NAME = "CeresDB"; + + private static final AtomicBoolean RW_LOGGING; + + private static final int REPORT_PERIOD_MIN; + private static final ScheduledExecutorService DISPLAY; + + static { + RW_LOGGING = new AtomicBoolean(SystemPropertyUtil.getBool(OptKeys.RW_LOGGING, true)); + REPORT_PERIOD_MIN = SystemPropertyUtil.getInt(OptKeys.REPORT_PERIOD, 30); + DISPLAY = ThreadPoolUtil.newScheduledBuilder().poolName("display_self") // + .coreThreads(1) // + .enableMetric(true) // + .threadFactory(new NamedThreadFactory("display_self", true)) // + .rejectedHandler(new ThreadPoolExecutor.DiscardOldestPolicy()) // + .build(); + + Runtime.getRuntime() + .addShutdownHook(new Thread(() -> ExecutorServiceHelper.shutdownAndAwaitTermination(DISPLAY))); + } + + /** + * Whether to output concise read/write logs. + * + * @return true or false + */ + public static boolean isRwLogging() { + return RW_LOGGING.get(); + } + + /** + * See {@link #isRwLogging()} + * + * Reset `rwLogging`, set to the opposite of the old value. + * + * @return old value + */ + public static boolean resetRwLogging() { + return RW_LOGGING.getAndSet(!RW_LOGGING.get()); + } + + /** + * Auto report self(CeresDBClient) period. + * + * @return period with minutes + */ + public static int autoReportPeriodMin() { + return REPORT_PERIOD_MIN; + } + + /** + * Only used to schedule to display the self of client. + * + * @param display display + * @param printer to print the display info + */ + public static void scheduleDisplaySelf(final Display display, final Display.Printer printer) { + DISPLAY.scheduleWithFixedDelay(() -> display.display(printer), 0, autoReportPeriodMin(), TimeUnit.MINUTES); + } + + /** + * Create a shared scheduler pool with the given name. + * + * @param name scheduled pool's name + * @param workers the num of workers + * @return new scheduler poll instance + */ + public static SharedScheduledPool getSharedScheduledPool(final String name, final int workers) { + return new SharedScheduledPool(new ObjectPool.Resource() { + + @Override + public ScheduledExecutorService create() { + return ThreadPoolUtil.newScheduledBuilder() // + .poolName(name) // + .coreThreads(workers) // + .enableMetric(true) // + .threadFactory(new NamedThreadFactory(name, true)) // + .rejectedHandler(new ThreadPoolExecutor.DiscardOldestPolicy()) // + .build(); + } + + @Override + public void close(final ScheduledExecutorService instance) { + ExecutorServiceHelper.shutdownAndAwaitTermination(instance); + } + }); + } + + /** + * Merge two given {@link Result} objects. If both Result objects are + * {@link WriteOk} or {@link Err}, then merge their results. If one is + * {@link Err} and the other is {@link WriteOk}, then we will discard + * the {@link WriteOk}. + * + * @param r1 the result + * @param r2 the other result + * @return merged result + */ + public static Result combineResult(final Result r1, final Result r2) { + if (r1.isOk() && r2.isOk()) { + return r1.getOk().combine(r2.getOk()).mapToResult(); + } else if (!r1.isOk() && !r2.isOk()) { + return r1.getErr().combine(r2.getErr()).mapToResult(); + } else { + if (r1.isOk()) { + return r2.getErr().combine(r1.getOk()).mapToResult(); + } else { + return r1.getErr().combine(r2.getOk()).mapToResult(); + } + } + } + + /** + * Converts the given {@link Storage.WriteResponse} to {@link Result} that + * upper-level readable. + * + * @param resp response of the write RPC + * @param to the server address wrote to + * @param points wrote date in this write + * @return a {@link Result} + */ + public static Result toResult(final Storage.WriteResponse resp, // + final Endpoint to, // + final List points) { + final Common.ResponseHeader header = resp.getHeader(); + final int code = header.getCode(); + final String msg = header.getError(); + final int success = resp.getSuccess(); + final int failed = resp.getFailed(); + + if (code == Result.SUCCESS) { + final Set tables = points != null && WriteOk.isCollectWroteDetail() ? + points.stream().map(Point::getTable).collect(Collectors.toSet()) : + null; + return WriteOk.ok(success, failed, tables).mapToResult(); + } else { + return Err.writeErr(code, msg, to, points).mapToResult(); + } + } + + /** + * Converts the given {@link Storage.SqlQueryResponse} to {@link Result} that + * upper-level readable. + * + * @param resp response of the write RPC + * @param to the server address wrote to + * @param tables the metrics who query failed + * @param errHandler the error handler + * @return a {@link Result} + */ + public static Result toResult(final Storage.SqlQueryResponse resp, // + final String sql, // + final Endpoint to, // + final Collection tables, final Runnable errHandler) { + final Common.ResponseHeader header = resp.getHeader(); + final int code = header.getCode(); + final String msg = header.getError(); + + if (code != Result.SUCCESS) { + if (errHandler != null) { + errHandler.run(); + } + return Err.queryErr(code, msg, to, sql, tables).mapToResult(); + } + + if (resp.getArrow().getRecordBatchesCount() == 0) { + return SqlQueryOk.ok(sql, resp.getAffectedRows(), null).mapToResult(); + } + + List rows = resp.getArrow().getRecordBatchesList().stream() + .flatMap(batch -> parseArrowBatch(batch, resp.getArrow().getCompression()).stream()) + .collect(Collectors.toList()); + + return SqlQueryOk.ok(sql, resp.getAffectedRows(), rows).mapToResult(); + } + + /** + * Determine whether the request was successful from the information in the + * response header. + * + * @param header response header + * @return true if response is success + */ + public static boolean isSuccess(final Common.ResponseHeader header) { + return header.getCode() == Result.SUCCESS; + } + + /** + * Returns a new CompletableFuture that is already completed with the given + * value. Same as {@link CompletableFuture#completedFuture(Object)}, only + * rename the method. + * + * @param value the given value + * @param the type of the value + * @return the completed {@link CompletableFuture} + */ + public static CompletableFuture completedCf(final U value) { + return CompletableFuture.completedFuture(value); + } + + /** + * Returns a new CompletableFuture that is already exceptionally with the given + * error. + * + * @param t the given exception + * @param the type of the value + * @return the exceptionally {@link CompletableFuture} + */ + public static CompletableFuture errorCf(final Throwable t) { + final CompletableFuture err = new CompletableFuture<>(); + err.completeExceptionally(t); + return err; + } + + /** + * Break data stream into multiple requests based on the routing table + * information given. + * + * @param data the data to split + * @param routes the route table info + * @return multi data stream + */ + public static Map> splitDataByRoute(final List data, // + final Map routes) { + final Map> splits = routes.values() // + .stream() // + .map(Route::getEndpoint) // + .distinct() // + .collect(Collectors.toMap(k -> k, k -> Spines.newBuf(), (v1, v2) -> v1)); + if (splits.size() == 1) { + // fast path, zero copy + splits.replaceAll((ep, empty) -> data); + } else { + data.forEach(rs -> { + final Route route = routes.get(rs.getTable()); + Requires.requireNonNull(route, "Null.route for " + rs); + final Collection partOf = splits.get(route.getEndpoint()); + Requires.requireNonNull(route, "Invalid.route " + route); + partOf.add(rs); + }); + } + + MetricsUtil.histogram("split_num_per_write").update(splits.size()); + + return splits; + } + + public static boolean shouldNotRetry(final Err err) { + return !shouldRetry(err); + } + + public static boolean shouldRetry(final Err err) { + if (err == null) { + return false; + } + final int errCode = err.getCode(); + return errCode == Result.INVALID_ROUTE || errCode == Result.SHOULD_RETRY; + } + + public static boolean shouldRefreshRouteTable(final Err err) { + return err.getCode() == Result.INVALID_ROUTE; + } + + public static Observer toUnaryObserver(final CompletableFuture future) { + return new Observer() { + + @Override + public void onNext(final V value) { + future.complete(value); + } + + @Override + public void onError(final Throwable err) { + future.completeExceptionally(err); + } + }; + } + + public static Storage.Value toProtoValue(final Value value) { + final Storage.Value.Builder vb = Storage.Value.newBuilder(); + switch (value.getDataType()) { + case Double: + return vb.setFloat64Value(value.getDouble()).build(); + case String: + return vb.setStringValue(value.getString()).build(); + case Int64: + return vb.setInt64Value(value.getInt64()).build(); + case Float: + return vb.setFloat32Value(value.getFloat()).build(); + case Int32: + return vb.setInt32Value(value.getInt32()).build(); + case Int16: + return vb.setInt16Value(value.getInt16()).build(); + case Int8: + return vb.setInt8Value(value.getInt8()).build(); + case Boolean: + return vb.setBoolValue(value.getBoolean()).build(); + case UInt64: + return vb.setUint64Value(value.getUInt64()).build(); + case UInt32: + return vb.setUint32Value(value.getUInt32()).build(); + case UInt16: + return vb.setUint16Value(value.getUInt16()).build(); + case UInt8: + return vb.setUint8Value(value.getUInt8()).build(); + case Timestamp: + return vb.setTimestampValue(value.getTimestamp()).build(); + case Varbinary: + return vb.setVarbinaryValue(ByteStringHelper.wrap(value.getVarbinary())).build(); + default: + return invalidType(value); + } + } + + public static long randomInitialDelay(final long delay) { + return ThreadLocalRandom.current().nextLong(delay, delay << 1); + } + + public static Properties loadProperties(final ClassLoader loader, final String name) throws IOException { + final Properties prop = new Properties(); + prop.load(loader.getResourceAsStream(name)); + return prop; + } + + public static T unsupported(final String fmt, final Object... args) { + throw new UnsupportedOperationException(String.format(fmt, args)); + } + + public static void checkKeywords(final Iterator keys) { + if (keys == null) { + return; + } + + while (keys.hasNext()) { + ensureNotKeyword(keys.next()); + } + } + + private static void ensureNotKeyword(final String name) { + if (Keyword.isKeyword(name)) { + throw new IllegalArgumentException("Invalid name, not allow keyword `" + name + '`'); + } + } + + private static T invalidType(final Value value) { + throw new IllegalArgumentException("Invalid type " + value); + } + + private static List parseArrowBatch(ByteString batch, Storage.ArrowPayload.Compression compression) { + try { + InputStream arrowStream = batch.newInput(); + if (compression == Storage.ArrowPayload.Compression.ZSTD) { + byte[] batchBuffer = batch.toByteArray(); + long decompressedSize = Zstd.decompressedSize(batchBuffer); + if (decompressedSize > 0) { + // batch compress mode + byte[] decompressedByteBuffer = Zstd.decompress(batchBuffer, (int) decompressedSize); + arrowStream = new ByteArrayInputStream(decompressedByteBuffer); + } else { + // stream compress mode + ZstdInputStream zstdInputStream = new ZstdInputStream(batch.newInput()); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(batch.size()); + byte[] block = new byte[128 * 1024]; + int size = zstdInputStream.read(block); + byteArrayOutputStream.write(block); + while (size > 0) { + size = zstdInputStream.read(block); + byteArrayOutputStream.write(block); + } + arrowStream = new ByteArrayInputStream(byteArrayOutputStream.toByteArray()); + } + } + + ArrowStreamReader arrowStreamReader = new ArrowStreamReader(arrowStream, new RootAllocator()); + + VectorSchemaRoot readRoot = arrowStreamReader.getVectorSchemaRoot(); + Schema schema = readRoot.getSchema(); + + while (arrowStreamReader.loadNextBatch()) { + return parseArrowRecord(schema, readRoot); + } + return Collections.EMPTY_LIST; + } catch (IOException e) { + return Collections.EMPTY_LIST; + } + } + + private static List parseArrowRecord(Schema schema, VectorSchemaRoot root) { + List rows = Stream.generate(Row::new).limit(root.getRowCount()).collect(Collectors.toList()); + + for (int fieldIdx = 0; fieldIdx < schema.getFields().size(); fieldIdx++) { + Field field = schema.getFields().get(fieldIdx); + FieldVector vector = root.getVector(fieldIdx); + + switch (Types.getMinorTypeForArrowType(field.getType())) { + case VARCHAR: + VarCharVector varCharVector = (VarCharVector) vector; + for (int rowIdx = 0; rowIdx < varCharVector.getValueCount(); rowIdx++) { + if (varCharVector.isNull(rowIdx)) { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withStringOrNull(null)); + } else { + rows.get(rowIdx).setColumnValue(field.getName(), + Value.withString(new String(varCharVector.get(rowIdx)))); + } + } + break; + case BIT: + BitVector bitVector = (BitVector) vector; + for (int rowIdx = 0; rowIdx < bitVector.getValueCount(); rowIdx++) { + if (bitVector.isNull(rowIdx)) { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withBooleanOrNull(null)); + } else { + rows.get(rowIdx).setColumnValue(field.getName(), + Value.withBoolean(bitVector.get(rowIdx) > 0)); + } + } + break; + case FLOAT8: + Float8Vector float8Vector = (Float8Vector) vector; + for (int rowIdx = 0; rowIdx < float8Vector.getValueCount(); rowIdx++) { + if (float8Vector.isNull(rowIdx)) { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withDoubleOrNull(null)); + } else { + rows.get(rowIdx).setColumnValue(field.getName(), + Value.withDouble(float8Vector.get(rowIdx))); + } + } + break; + case FLOAT4: + Float4Vector float4Vector = (Float4Vector) vector; + for (int rowIdx = 0; rowIdx < float4Vector.getValueCount(); rowIdx++) { + if (float4Vector.isNull(rowIdx)) { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withFloatOrNull(null)); + } else { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withFloat(float4Vector.get(rowIdx))); + } + } + break; + case BIGINT: + BigIntVector bigIntVector = (BigIntVector) vector; + for (int rowIdx = 0; rowIdx < bigIntVector.getValueCount(); rowIdx++) { + if (bigIntVector.isNull(rowIdx)) { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withInt64OrNull(null)); + } else { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withInt64(bigIntVector.get(rowIdx))); + } + } + break; + case INT: + IntVector intVector = (IntVector) vector; + for (int rowIdx = 0; rowIdx < intVector.getValueCount(); rowIdx++) { + if (intVector.isNull(rowIdx)) { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withInt32OrNull(null)); + } else { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withInt32(intVector.get(rowIdx))); + } + } + break; + case SMALLINT: + SmallIntVector smallIntVector = (SmallIntVector) vector; + for (int rowIdx = 0; rowIdx < smallIntVector.getValueCount(); rowIdx++) { + if (smallIntVector.isNull(rowIdx)) { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withInt16OrNull(null)); + } else { + rows.get(rowIdx).setColumnValue(field.getName(), + Value.withInt16(smallIntVector.get(rowIdx))); + } + } + break; + case TINYINT: + TinyIntVector tinyIntVector = (TinyIntVector) vector; + for (int rowIdx = 0; rowIdx < tinyIntVector.getValueCount(); rowIdx++) { + if (tinyIntVector.isNull(rowIdx)) { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withInt8OrNull(null)); + } else { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withInt8(tinyIntVector.get(rowIdx))); + } + } + break; + case UINT8: + UInt8Vector uInt8Vector = (UInt8Vector) vector; + for (int rowIdx = 0; rowIdx < uInt8Vector.getValueCount(); rowIdx++) { + if (uInt8Vector.isNull(rowIdx)) { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withUInt64OrNull(null)); + } else { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withUInt64(uInt8Vector.get(rowIdx))); + } + } + break; + case UINT4: + UInt4Vector uInt4Vector = (UInt4Vector) vector; + for (int rowIdx = 0; rowIdx < uInt4Vector.getValueCount(); rowIdx++) { + if (uInt4Vector.isNull(rowIdx)) { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withUInt32OrNull(null)); + } else { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withUInt32(uInt4Vector.get(rowIdx))); + } + } + break; + case UINT2: + UInt2Vector uInt2Vector = (UInt2Vector) vector; + for (int rowIdx = 0; rowIdx < uInt2Vector.getValueCount(); rowIdx++) { + if (uInt2Vector.isNull(rowIdx)) { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withUInt16OrNull(null)); + } else { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withUInt16(uInt2Vector.get(rowIdx))); + } + } + break; + case UINT1: + UInt1Vector uInt1Vector = (UInt1Vector) vector; + for (int rowIdx = 0; rowIdx < uInt1Vector.getValueCount(); rowIdx++) { + if (uInt1Vector.isNull(rowIdx)) { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withUInt8OrNull(null)); + } else { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withUInt8(uInt1Vector.get(rowIdx))); + } + } + break; + case TIMESTAMPMILLI: + TimeStampMilliVector timeStampMilliVector = (TimeStampMilliVector) vector; + for (int rowIdx = 0; rowIdx < timeStampMilliVector.getValueCount(); rowIdx++) { + if (timeStampMilliVector.isNull(rowIdx)) { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withTimestampOrNull(null)); + } else { + rows.get(rowIdx).setColumnValue(field.getName(), + Value.withTimestamp(timeStampMilliVector.get(rowIdx))); + } + } + break; + case VARBINARY: + VarBinaryVector varBinaryVector = (VarBinaryVector) vector; + for (int rowIdx = 0; rowIdx < varBinaryVector.getValueCount(); rowIdx++) { + if (varBinaryVector.isNull(rowIdx)) { + rows.get(rowIdx).setColumnValue(field.getName(), Value.withVarbinaryOrNull(null)); + } else { + rows.get(rowIdx).setColumnValue(field.getName(), + Value.withVarbinaryOrNull(varBinaryVector.get(rowIdx))); + } + } + break; + default: + // paas + + } + } + return rows; + } + + private Utils() { + } +} diff --git a/ceresdb-protocol/src/test/java/io/ceresdb/CeresDBClientTest.java b/ceresdb-protocol/src/test/java/io/ceresdb/CeresDBClientTest.java index bf48c64..8b65e7d 100644 --- a/ceresdb-protocol/src/test/java/io/ceresdb/CeresDBClientTest.java +++ b/ceresdb-protocol/src/test/java/io/ceresdb/CeresDBClientTest.java @@ -1,21 +1,9 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -32,16 +20,14 @@ import io.ceresdb.common.util.internal.ReferenceFieldUpdater; import io.ceresdb.common.util.internal.Updaters; import io.ceresdb.models.Err; +import io.ceresdb.models.Point; import io.ceresdb.models.Result; -import io.ceresdb.models.Rows; import io.ceresdb.models.WriteOk; +import io.ceresdb.models.WriteRequest; import io.ceresdb.options.CeresDBOptions; import io.ceresdb.util.TestUtil; +import io.ceresdb.util.Utils; -/** - * - * @author jiachun.fjc - */ @RunWith(value = MockitoJUnitRunner.class) public class CeresDBClientTest { @@ -55,8 +41,8 @@ public class CeresDBClientTest { @Before public void before() { - this.opts = CeresDBOptions.newBuilder("127.0.0.1", 8081) // - .tenant("test", "sub_test", "test_token") // + this.opts = CeresDBOptions.newBuilder("127.0.0.1", 8081, RouteMode.DIRECT) // + .database("public") // .writeMaxRetries(1) // .readMaxRetries(1) // .build(); @@ -71,8 +57,8 @@ public void after() { @Test(expected = IllegalStateException.class) public void withoutInitTest() { - final Rows rows = TestUtil.newRow("test_metric1_not_init"); - this.client.write(rows); + final List points = TestUtil.newTablePoints("test_table1_not_init"); + this.client.write(new WriteRequest(points)); } @Test(expected = IllegalStateException.class) @@ -94,11 +80,11 @@ public void instancesTest() { public void helloWorldTest() throws ExecutionException, InterruptedException { initAndMockWriteClient(); - final Rows rows = TestUtil.newRow("test_metric1"); + final List points = TestUtil.newTablePoints("test_table1"); - Mockito.when(this.writeClient.write(Mockito.anyList(), Mockito.any())) // + Mockito.when(this.writeClient.write(new WriteRequest(Mockito.anyList()), Mockito.any())) // .thenReturn(Utils.completedCf(WriteOk.ok(2, 0, null).mapToResult())); - final CompletableFuture> f = this.client.write(rows); + final CompletableFuture> f = this.client.write(new WriteRequest(points)); final Result ret = f.get(); Assert.assertTrue(ret.isOk()); final int success = ret.mapOr(-1, WriteOk::getSuccess); diff --git a/ceresdb-protocol/src/test/java/io/ceresdb/QueryClientTest.java b/ceresdb-protocol/src/test/java/io/ceresdb/QueryClientTest.java index 00ecc57..6537a6f 100644 --- a/ceresdb-protocol/src/test/java/io/ceresdb/QueryClientTest.java +++ b/ceresdb-protocol/src/test/java/io/ceresdb/QueryClientTest.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; @@ -22,6 +9,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ForkJoinPool; @@ -29,15 +17,30 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import io.ceresdb.models.Row; import io.ceresdb.proto.internal.Common; import io.ceresdb.proto.internal.Storage; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.BinaryEncoder; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.io.EncoderFactory; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeStampMilliVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.UInt1Vector; +import org.apache.arrow.vector.UInt2Vector; +import org.apache.arrow.vector.UInt4Vector; +import org.apache.arrow.vector.UInt8Vector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowStreamWriter; +import org.apache.arrow.vector.types.pojo.Field; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -52,19 +55,16 @@ import io.ceresdb.errors.IteratorException; import io.ceresdb.errors.StreamException; import io.ceresdb.models.Err; -import io.ceresdb.models.QueryOk; -import io.ceresdb.models.QueryRequest; -import io.ceresdb.models.Record; +import io.ceresdb.models.SqlQueryOk; +import io.ceresdb.models.SqlQueryRequest; import io.ceresdb.models.Result; import io.ceresdb.options.QueryOptions; import io.ceresdb.rpc.Context; import io.ceresdb.rpc.Observer; +import io.ceresdb.util.Utils; import com.google.protobuf.ByteStringHelper; -/** - * @author jiachun.fjc - */ @RunWith(value = MockitoJUnitRunner.class) public class QueryClientTest { @@ -77,6 +77,7 @@ public void before() { final QueryOptions queryOpts = new QueryOptions(); queryOpts.setAsyncPool(ForkJoinPool.commonPool()); queryOpts.setRouterClient(this.routerClient); + queryOpts.setDatabase("public"); this.queryClient = new QueryClient(); this.queryClient.init(queryOpts); @@ -89,51 +90,44 @@ public void after() { } @Test - public void queryOkNoRouteTest() throws ExecutionException, InterruptedException { - final Common.ResponseHeader header = Common.ResponseHeader.newBuilder() // - .setCode(Result.SUCCESS) // - .build(); - final Storage.QueryResponse resp = Storage.QueryResponse.newBuilder().setHeader(header) // - .addRows(ByteStringHelper.wrap(new byte[] { 1, 2, 3 })) // - .build(); + public void queryOkNoRouteTest() throws ExecutionException, InterruptedException, IOException { + final Storage.SqlQueryResponse resp = mockSimpleQueryResponse(1, false); final Endpoint ep = Endpoint.of("127.0.0.1", 8081); Mockito.when(this.routerClient.invoke(Mockito.eq(ep), Mockito.any(), Mockito.any())) // .thenReturn(Utils.completedCf(resp)); - Mockito.when(this.routerClient.routeFor(Mockito.any())) // + Mockito.when(this.routerClient.routeFor(Mockito.any(), Mockito.any())) // .thenReturn(Utils.completedCf(new HashMap<>())); Mockito.when(this.routerClient.clusterRoute()) // .thenReturn(Route.of(ep)); - final QueryRequest req = QueryRequest.newBuilder().forMetrics("query_test_table") // - .ql("select number from query_test_table") // + final SqlQueryRequest req = SqlQueryRequest.newBuilder().forTables("query_test_table") // + .sql("select number from query_test_table") // .build(); - final CompletableFuture> f = this.queryClient.query(req, Context.newDefault()); + final CompletableFuture> f = this.queryClient.sqlQuery(req, Context.newDefault()); - final Result r = f.get(); + final Result r = f.get(); Assert.assertTrue(r.isOk()); - final QueryOk queryOk = r.getOk(); + final SqlQueryOk queryOk = r.getOk(); - final Stream strs = queryOk.map(Arrays::toString); + final Stream strs = queryOk.map(Row::toString); - Assert.assertEquals(Collections.singletonList("[1, 2, 3]"), strs.collect(Collectors.toList())); + Assert.assertEquals( + Collections.singletonList("f1:Value{type=Int32,value=123}|t1:Value{type=String,value=tvtest}"), + strs.collect(Collectors.toList())); } @Test - public void queryOkByValidRouteTest() throws ExecutionException, InterruptedException { - final Common.ResponseHeader header = Common.ResponseHeader.newBuilder() // - .setCode(Result.SUCCESS) // - .build(); - final Storage.QueryResponse resp = Storage.QueryResponse.newBuilder().setHeader(header) // - .addRows(ByteStringHelper.wrap(new byte[] { 1, 2, 3 })) // - .build(); + public void queryOkByValidRouteTest() throws ExecutionException, InterruptedException, IOException { + final Storage.SqlQueryResponse resp = mockSimpleQueryResponse(1, false); final Endpoint ep = Endpoint.of("127.0.0.1", 8081); Mockito.when(this.routerClient.invoke(Mockito.eq(ep), Mockito.any(), Mockito.any())) // .thenReturn(Utils.completedCf(resp)); - Mockito.when(this.routerClient.routeFor(Mockito.eq(Collections.singletonList("query_test_table")))) // + Mockito.when( + this.routerClient.routeFor(Mockito.any(), Mockito.eq(Collections.singletonList("query_test_table")))) // .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = -6260265905617276356L; @@ -142,34 +136,37 @@ public void queryOkByValidRouteTest() throws ExecutionException, InterruptedExce } })); - final QueryRequest req = QueryRequest.newBuilder().forMetrics("query_test_table") // - .ql("select number from query_test_table") // + final SqlQueryRequest req = SqlQueryRequest.newBuilder().forTables("query_test_table") // + .sql("select number from query_test_table") // .build(); - final CompletableFuture> f = this.queryClient.query(req, Context.newDefault()); + final CompletableFuture> f = this.queryClient.sqlQuery(req, Context.newDefault()); - final Result r = f.get(); + final Result r = f.get(); Assert.assertTrue(r.isOk()); - final QueryOk queryOk = r.getOk(); + final SqlQueryOk queryOk = r.getOk(); - final Stream strs = queryOk.map(Arrays::toString); + final Stream strs = queryOk.map(Row::toString); - Assert.assertEquals(Collections.singletonList("[1, 2, 3]"), strs.collect(Collectors.toList())); + Assert.assertEquals( + Collections.singletonList("f1:Value{type=Int32,value=123}|t1:Value{type=String,value=tvtest}"), + strs.collect(Collectors.toList())); } @Test - public void queryFailedTest() throws ExecutionException, InterruptedException { + public void queryFailedTest() throws ExecutionException, InterruptedException, IOException { final Common.ResponseHeader header = Common.ResponseHeader.newBuilder() // .setCode(500) // failed code .build(); - final Storage.QueryResponse resp = Storage.QueryResponse.newBuilder().setHeader(header) // + final Storage.SqlQueryResponse resp = Storage.SqlQueryResponse.newBuilder().setHeader(header) // .build(); final Endpoint ep = Endpoint.of("127.0.0.1", 8081); Mockito.when(this.routerClient.invoke(Mockito.eq(ep), Mockito.any(), Mockito.any())) // .thenReturn(Utils.completedCf(resp)); - Mockito.when(this.routerClient.routeFor(Mockito.eq(Collections.singletonList("query_test_table")))) // + Mockito.when( + this.routerClient.routeFor(Mockito.any(), Mockito.eq(Collections.singletonList("query_test_table")))) // .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = 4517371149948738282L; @@ -177,7 +174,7 @@ public void queryFailedTest() throws ExecutionException, InterruptedException { put("query_test_table", Route.of("query_test_table", ep)); } })); - Mockito.when(this.routerClient.routeRefreshFor(Mockito.any())) + Mockito.when(this.routerClient.routeRefreshFor(Mockito.any(), Mockito.any())) .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = 2347114952231996366L; @@ -186,153 +183,225 @@ public void queryFailedTest() throws ExecutionException, InterruptedException { } })); - final QueryRequest req = QueryRequest.newBuilder().forMetrics("query_test_table") // - .ql("select number from query_test_table") // + final SqlQueryRequest req = SqlQueryRequest.newBuilder().forTables("query_test_table") // + .sql("select number from query_test_table") // .build(); - final CompletableFuture> f = this.queryClient.query(req, Context.newDefault()); + final CompletableFuture> f = this.queryClient.sqlQuery(req, Context.newDefault()); - final Result r = f.get(); + final Result r = f.get(); Assert.assertFalse(r.isOk()); final Err err = r.getErr(); Assert.assertEquals(ep, err.getErrTo()); - Assert.assertEquals(Collections.singletonList("query_test_table"), err.getFailedMetrics()); + Assert.assertEquals(Collections.singletonList("query_test_table"), err.getFailedTables()); } @Test - public void queryByAvroAndReturnGenericRecordTest() throws ExecutionException, InterruptedException, IOException { - final Result r = queryByAvro(); + public void queryByArrowFullTypeTest() throws ExecutionException, InterruptedException, IOException { + final Result r = queryByArrow(); Assert.assertTrue(r.isOk()); - final QueryOk queryOk = r.getOk(); - queryOk.mapToRecord().forEach(this::checkRecord); + final SqlQueryOk queryOk = r.getOk(); + queryOk.stream().forEach(this::checkFullTypeRow); } - private void checkRecord(final Record row) { - Assert.assertEquals(7, row.getFieldCount()); - Assert.assertEquals("f1", row.getFieldDescriptors().get(0).getName()); - Assert.assertEquals("f2", row.getFieldDescriptors().get(1).getName()); - Assert.assertEquals("f3", row.getFieldDescriptors().get(2).getName()); - Assert.assertEquals("f4", row.getFieldDescriptors().get(3).getName()); - Assert.assertEquals("f5", row.getFieldDescriptors().get(4).getName()); - Assert.assertEquals("f6", row.getFieldDescriptors().get(5).getName()); - Assert.assertEquals("f7", row.getFieldDescriptors().get(6).getName()); - - Assert.assertEquals(Record.Type.String, row.getFieldDescriptors().get(0).getType().getType()); - Assert.assertEquals(Record.Type.Int, row.getFieldDescriptors().get(1).getType().getType()); - Assert.assertEquals(Record.Type.Long, row.getFieldDescriptors().get(2).getType().getType()); - Assert.assertEquals(Record.Type.Float, row.getFieldDescriptors().get(3).getType().getType()); - Assert.assertEquals(Record.Type.Double, row.getFieldDescriptors().get(4).getType().getType()); - Assert.assertEquals(Record.Type.Boolean, row.getFieldDescriptors().get(5).getType().getType()); - Assert.assertEquals(Record.Type.Null, row.getFieldDescriptors().get(6).getType().getType()); - - Assert.assertEquals("a string value", row.getString("f1")); - Assert.assertEquals(new Integer(100), row.getInteger("f2")); - Assert.assertEquals(new Long(200), row.getLong("f3")); - Assert.assertEquals(new Float(2.6), row.getFloat("f4")); - Assert.assertEquals(new Double(3.75), row.getDouble("f5")); - Assert.assertEquals(Boolean.TRUE, row.getBoolean("f6")); - Assert.assertNull(row.get("f7")); - - Assert.assertEquals("a string value", row.getString(0)); - Assert.assertEquals(new Integer(100), row.getInteger(1)); - Assert.assertEquals(new Long(200), row.getLong(2)); - Assert.assertEquals(new Float(2.6), row.getFloat(3)); - Assert.assertEquals(new Double(3.75), row.getDouble(4)); - Assert.assertEquals(Boolean.TRUE, row.getBoolean(5)); - Assert.assertNull(row.get(6)); + private void checkFullTypeRow(final Row row) { + Assert.assertEquals(14, row.getColumnCount()); + + Assert.assertEquals("test", row.getColumnValue("fString").getString()); + Assert.assertEquals(Boolean.TRUE, row.getColumnValue("fBool").getBoolean()); + Assert.assertEquals(0.64, row.getColumnValue("fDouble").getDouble(), 0.000001); + Assert.assertEquals(0.32f, row.getColumnValue("fFloat").getFloat(), 0.000001); + Assert.assertEquals(-64, row.getColumnValue("fInt64").getInt64()); + Assert.assertEquals(-32, row.getColumnValue("fInt32").getInt32()); + Assert.assertEquals(-16, row.getColumnValue("fInt16").getInt16()); + Assert.assertEquals(-8, row.getColumnValue("fInt8").getInt8()); + Assert.assertEquals(64, row.getColumnValue("fUint64").getUInt64()); + Assert.assertEquals(32, row.getColumnValue("fUint32").getUInt32()); + Assert.assertEquals(16, row.getColumnValue("fUint16").getUInt16()); + Assert.assertEquals(8, row.getColumnValue("fUint8").getUInt8()); + Assert.assertEquals(1675345488158L, row.getColumnValue("fTimestamp").getTimestamp()); + Assert.assertArrayEquals(new byte[] { 1, 2, 3 }, row.getColumnValue("fVarbinary").getVarbinary()); } @Test - public void queryByAvroAndReturnObjectsTest() throws ExecutionException, InterruptedException, IOException { - final Result r = queryByAvro(); + public void queryByArrowNullTypeTest() throws ExecutionException, InterruptedException, IOException { + final Storage.SqlQueryResponse resp = mockSimpleQueryResponse(1, true); + final Endpoint ep = Endpoint.of("127.0.0.1", 8081); + + Mockito.when(this.routerClient.invoke(Mockito.eq(ep), Mockito.any(), Mockito.any())) // + .thenReturn(Utils.completedCf(resp)); + Mockito.when(this.routerClient.routeFor(Mockito.any(), Mockito.any())) // + .thenReturn(Utils.completedCf(new HashMap<>())); + Mockito.when(this.routerClient.clusterRoute()) // + .thenReturn(Route.of(ep)); + + final SqlQueryRequest req = SqlQueryRequest.newBuilder().forTables("query_test_table") // + .sql("select * from query_test_table") // + .build(); + final CompletableFuture> f = this.queryClient.sqlQuery(req, Context.newDefault()); + + final Result r = f.get(); + Assert.assertTrue(r.isOk()); - final QueryOk queryOk = r.getOk(); - queryOk.mapToArray().forEach(row -> { - Assert.assertEquals(7, row.length); - Assert.assertEquals("a string value", String.valueOf(row[0])); - Assert.assertEquals(100, row[1]); - Assert.assertEquals(200L, row[2]); - Assert.assertEquals((float) 2.6, row[3]); - Assert.assertEquals(3.75, row[4]); - Assert.assertEquals(Boolean.TRUE, row[5]); - Assert.assertNull(row[6]); - }); + + final SqlQueryOk queryOk = r.getOk(); + + Row row = queryOk.stream().findFirst().get(); + Assert.assertEquals("tvtest", row.getColumnValue("t1").getString()); + Assert.assertTrue(row.getColumnValue("f1").isNull()); } - private Result queryByAvro() throws IOException, ExecutionException, InterruptedException { - final Storage.QueryResponse resp = mockQueryResponse(); + private Result queryByArrow() throws IOException, ExecutionException, InterruptedException { + final Storage.SqlQueryResponse resp = mockAllTypeQueryResponse(1); final Endpoint ep = Endpoint.of("127.0.0.1", 8081); Mockito.when(this.routerClient.invoke(Mockito.eq(ep), Mockito.any(), Mockito.any())) // .thenReturn(Utils.completedCf(resp)); - Mockito.when(this.routerClient.routeFor(Mockito.any())) // + Mockito.when(this.routerClient.routeFor(Mockito.any(), Mockito.any())) // .thenReturn(Utils.completedCf(new HashMap<>())); Mockito.when(this.routerClient.clusterRoute()) // .thenReturn(Route.of(ep)); - final QueryRequest req = QueryRequest.newBuilder().forMetrics("query_test_table") // - .ql("select number from query_test_table") // + final SqlQueryRequest req = SqlQueryRequest.newBuilder().forTables("query_test_table") // + .sql("select number from query_test_table") // .build(); - final CompletableFuture> f = this.queryClient.query(req, Context.newDefault()); + final CompletableFuture> f = this.queryClient.sqlQuery(req, Context.newDefault()); return f.get(); } - private QueryOk mockQueryOk() { + private SqlQueryOk mockQueryOk() { try { Thread.sleep(20); - return queryByAvro().unwrapOr(QueryOk.emptyOk()); + return queryByArrow().unwrapOr(SqlQueryOk.emptyOk()); } catch (final Throwable t) { ThrowUtil.throwException(t); } return null; } - private Storage.QueryResponse mockQueryResponse() throws IOException { + private Storage.SqlQueryResponse mockSimpleQueryResponse(int rowCount, boolean hasNullField) throws IOException { final Common.ResponseHeader header = Common.ResponseHeader.newBuilder() // .setCode(Result.SUCCESS) // .build(); - final String userSchema = "{\"type\":\"record\"," + "\"name\":\"my_record\"," + "\"fields\":[" // - + "{\"name\":\"f1\",\"type\":\"string\"}," // - + "{\"name\":\"f2\",\"type\":\"int\"}," // - + "{\"name\":\"f3\",\"type\":\"long\"}," // - + "{\"name\":\"f4\",\"type\":\"float\"}," // - + "{\"name\":\"f5\",\"type\":\"double\"}," // - + "{\"name\":\"f6\",\"type\":\"boolean\"}," // - + "{\"name\":\"f7\",\"type\":\"null\"}" // - + "]}"; - final Schema.Parser parser = new Schema.Parser(); - final Schema schema = parser.parse(userSchema); - final GenericRecord record = new GenericData.Record(schema); - record.put("f1", "a string value"); - record.put("f2", 100); - record.put("f3", 200L); - record.put("f4", 2.6f); - record.put("f5", 3.75d); - record.put("f6", true); - final DatumWriter datumWriter = new GenericDatumWriter<>(schema); - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(baos, null); - datumWriter.write(record, encoder); - encoder.flush(); - baos.flush(); - - return Storage.QueryResponse.newBuilder() // + + BufferAllocator allocator = new RootAllocator(); + VarCharVector varCharVector = new VarCharVector("t1", allocator); + IntVector intVector = new IntVector("f1", allocator); + + for (int i = 0; i < rowCount; i++) { + varCharVector.setSafe(i, "tvtest".getBytes()); + if (hasNullField) { + intVector.setNull(i); + } else { + intVector.setSafe(i, 123); + } + } + varCharVector.setValueCount(rowCount); + intVector.setValueCount(rowCount); + + List fields = Arrays.asList(varCharVector.getField(), intVector.getField()); + List vectors = Arrays.asList(varCharVector, intVector); + VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ArrowStreamWriter writer = new ArrowStreamWriter(root, /*DictionaryProvider=*/null, out); + writer.writeBatch(); + writer.close(); + + Storage.ArrowPayload arrowPayload = Storage.ArrowPayload.newBuilder() + .setCompression(Storage.ArrowPayload.Compression.NONE) + .addRecordBatches(ByteStringHelper.wrap(out.toByteArray())).build(); + + return Storage.SqlQueryResponse.newBuilder() // .setHeader(header) // - .setSchemaType(Storage.QueryResponse.SchemaType.AVRO) // - .setSchemaContent(userSchema) // - .addRows(ByteStringHelper.wrap(baos.toByteArray())) // + .setArrow(arrowPayload).build(); + } + + private Storage.SqlQueryResponse mockAllTypeQueryResponse(int rowCount) throws IOException { + final Common.ResponseHeader header = Common.ResponseHeader.newBuilder() // + .setCode(Result.SUCCESS) // .build(); + + BufferAllocator allocator = new RootAllocator(); + VarCharVector stringVector = new VarCharVector("fString", allocator); + BitVector boolVector = new BitVector("fBool", allocator); + Float8Vector doubleVector = new Float8Vector("fDouble", allocator); + Float4Vector floatVector = new Float4Vector("fFloat", allocator); + BigIntVector int64Vector = new BigIntVector("fInt64", allocator); + IntVector int32Vector = new IntVector("fInt32", allocator); + SmallIntVector int16Vector = new SmallIntVector("fInt16", allocator); + TinyIntVector int8Vector = new TinyIntVector("fInt8", allocator); + UInt8Vector uint64Vector = new UInt8Vector("fUint64", allocator); + UInt4Vector uint32Vector = new UInt4Vector("fUint32", allocator); + UInt2Vector uint16Vector = new UInt2Vector("fUint16", allocator); + UInt1Vector uint8Vector = new UInt1Vector("fUint8", allocator); + TimeStampMilliVector tsVector = new TimeStampMilliVector("fTimestamp", allocator); + VarBinaryVector varBinaryVector = new VarBinaryVector("fVarbinary", allocator); + + for (int i = 0; i < rowCount; i++) { + stringVector.setSafe(i, "test".getBytes()); + boolVector.setSafe(i, 1); + doubleVector.setSafe(i, 0.64); + floatVector.setSafe(i, 0.32f); + int64Vector.setSafe(i, -64); + int32Vector.setSafe(i, -32); + int16Vector.setSafe(i, -16); + int8Vector.setSafe(i, -8); + uint64Vector.setSafe(i, 64); + uint32Vector.setSafe(i, 32); + uint16Vector.setSafe(i, 16); + uint8Vector.setSafe(i, 8); + tsVector.setSafe(i, 1675345488158L); + varBinaryVector.setSafe(i, new byte[] { 1, 2, 3 }); + } + stringVector.setValueCount(rowCount); + boolVector.setValueCount(rowCount); + doubleVector.setValueCount(rowCount); + floatVector.setValueCount(rowCount); + int64Vector.setValueCount(rowCount); + int32Vector.setValueCount(rowCount); + int16Vector.setValueCount(rowCount); + int8Vector.setValueCount(rowCount); + uint64Vector.setValueCount(rowCount); + uint32Vector.setValueCount(rowCount); + uint16Vector.setValueCount(rowCount); + uint8Vector.setValueCount(rowCount); + tsVector.setValueCount(rowCount); + varBinaryVector.setValueCount(rowCount); + + List fields = Arrays.asList(stringVector.getField(), boolVector.getField(), doubleVector.getField(), + floatVector.getField(), int64Vector.getField(), int32Vector.getField(), int16Vector.getField(), + int8Vector.getField(), uint64Vector.getField(), uint32Vector.getField(), uint16Vector.getField(), + uint8Vector.getField(), tsVector.getField(), varBinaryVector.getField()); + + List vectors = Arrays.asList(stringVector, boolVector, doubleVector, floatVector, int64Vector, + int32Vector, int16Vector, int8Vector, uint64Vector, uint32Vector, uint16Vector, uint8Vector, tsVector, + varBinaryVector); + VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ArrowStreamWriter writer = new ArrowStreamWriter(root, /*DictionaryProvider=*/null, out); + writer.writeBatch(); + writer.close(); + + Storage.ArrowPayload arrowPayload = Storage.ArrowPayload.newBuilder() + .setCompression(Storage.ArrowPayload.Compression.NONE) + .addRecordBatches(ByteStringHelper.wrap(out.toByteArray())).build(); + + return Storage.SqlQueryResponse.newBuilder() // + .setHeader(header) // + .setArrow(arrowPayload).build(); } @Test public void blockingStreamQueryTest() { - final QueryOk data = mockQueryOk(); + final SqlQueryOk data = mockQueryOk(); final int respCount = 10; final BlockingStreamIterator streams = new BlockingStreamIterator(1000, TimeUnit.MILLISECONDS); - final Observer obs = streams.getObserver(); + final Observer obs = streams.getObserver(); new Thread(() -> { for (int i = 0; i < respCount; i++) { if (i == 0) { @@ -344,12 +413,12 @@ public void blockingStreamQueryTest() { obs.onCompleted(); }).start(); - final Iterator it = new RecordIterator(streams); + final Iterator it = new RowIterator(streams); int i = 0; while (it.hasNext()) { i++; - checkRecord(it.next()); + checkFullTypeRow(it.next()); } Assert.assertEquals(respCount, i); } @@ -358,7 +427,7 @@ public void blockingStreamQueryTest() { public void blockingStreamQueryOnErrTest() { final int respCount = 10; final BlockingStreamIterator streams = new BlockingStreamIterator(1000, TimeUnit.MILLISECONDS); - final Observer obs = streams.getObserver(); + final Observer obs = streams.getObserver(); new Thread(() -> { for (int i = 0; i < respCount; i++) { if (i == 5) { @@ -370,12 +439,12 @@ public void blockingStreamQueryOnErrTest() { obs.onCompleted(); }).start(); - final Iterator it = new RecordIterator(streams); + final Iterator it = new RowIterator(streams); int i = 0; while (it.hasNext()) { i++; - checkRecord(it.next()); + checkFullTypeRow(it.next()); } Assert.assertEquals(respCount, i); } diff --git a/ceresdb-protocol/src/test/java/io/ceresdb/QueryLimiterTest.java b/ceresdb-protocol/src/test/java/io/ceresdb/QueryLimiterTest.java index 26d5575..be44544 100644 --- a/ceresdb-protocol/src/test/java/io/ceresdb/QueryLimiterTest.java +++ b/ceresdb-protocol/src/test/java/io/ceresdb/QueryLimiterTest.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; @@ -25,20 +12,20 @@ import org.junit.Test; import io.ceresdb.errors.LimitedException; +import io.ceresdb.limit.LimitedPolicy; +import io.ceresdb.limit.QueryLimiter; import io.ceresdb.models.Err; -import io.ceresdb.models.QueryOk; -import io.ceresdb.models.QueryRequest; +import io.ceresdb.models.SqlQueryOk; +import io.ceresdb.models.SqlQueryRequest; import io.ceresdb.models.Result; +import io.ceresdb.util.Utils; -/** - * @author jiachun.fjc - */ public class QueryLimiterTest { @Test(expected = LimitedException.class) public void abortQueryLimitTest() throws ExecutionException, InterruptedException { final QueryLimiter limiter = new QueryClient.DefaultQueryLimiter(1, new LimitedPolicy.AbortPolicy()); - final QueryRequest req = QueryRequest.newBuilder().forMetrics("test").ql("select * from test").build(); + final SqlQueryRequest req = SqlQueryRequest.newBuilder().forTables("test").sql("select * from test").build(); // consume the permits limiter.acquireAndDo(req, CompletableFuture::new); @@ -49,12 +36,12 @@ public void abortQueryLimitTest() throws ExecutionException, InterruptedExceptio @Test public void discardWriteLimitTest() throws ExecutionException, InterruptedException { final QueryLimiter limiter = new QueryClient.DefaultQueryLimiter(1, new LimitedPolicy.DiscardPolicy()); - final QueryRequest req = QueryRequest.newBuilder().forMetrics("test").ql("select * from test").build(); + final SqlQueryRequest req = SqlQueryRequest.newBuilder().forTables("test").sql("select * from test").build(); // consume the permits limiter.acquireAndDo(req, CompletableFuture::new); - final Result ret = limiter.acquireAndDo(req, this::emptyOk).get(); + final Result ret = limiter.acquireAndDo(req, this::emptyOk).get(); Assert.assertFalse(ret.isOk()); Assert.assertEquals(Result.FLOW_CONTROL, ret.getErr().getCode()); @@ -65,7 +52,7 @@ public void discardWriteLimitTest() throws ExecutionException, InterruptedExcept @Test public void blockingWriteLimitTest() throws InterruptedException { final QueryLimiter limiter = new QueryClient.DefaultQueryLimiter(1, new LimitedPolicy.BlockingPolicy()); - final QueryRequest req = QueryRequest.newBuilder().forMetrics("test").ql("select * from test").build(); + final SqlQueryRequest req = SqlQueryRequest.newBuilder().forTables("test").sql("select * from test").build(); // consume the permits limiter.acquireAndDo(req, CompletableFuture::new); @@ -95,13 +82,13 @@ public void blockingTimeoutWriteLimitTest() throws ExecutionException, Interrupt final int timeoutSecs = 2; final QueryLimiter limiter = new QueryClient.DefaultQueryLimiter(1, new LimitedPolicy.BlockingTimeoutPolicy(timeoutSecs, TimeUnit.SECONDS)); - final QueryRequest req = QueryRequest.newBuilder().forMetrics("test").ql("select * from test").build(); + final SqlQueryRequest req = SqlQueryRequest.newBuilder().forTables("test").sql("select * from test").build(); // consume the permits limiter.acquireAndDo(req, CompletableFuture::new); final long start = System.nanoTime(); - final Result ret = limiter.acquireAndDo(req, this::emptyOk).get(); + final Result ret = limiter.acquireAndDo(req, this::emptyOk).get(); Assert.assertEquals(timeoutSecs, TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start), 0.3); Assert.assertFalse(ret.isOk()); @@ -115,7 +102,7 @@ public void abortOnBlockingTimeoutWriteLimitTest() throws ExecutionException, In final int timeoutSecs = 2; final QueryLimiter limiter = new QueryClient.DefaultQueryLimiter(1, new LimitedPolicy.AbortOnBlockingTimeoutPolicy(timeoutSecs, TimeUnit.SECONDS)); - final QueryRequest req = QueryRequest.newBuilder().forMetrics("test").ql("select * from test").build(); + final SqlQueryRequest req = SqlQueryRequest.newBuilder().forTables("test").sql("select * from test").build(); // consume the permits limiter.acquireAndDo(req, CompletableFuture::new); @@ -128,7 +115,7 @@ public void abortOnBlockingTimeoutWriteLimitTest() throws ExecutionException, In } } - private CompletableFuture> emptyOk() { - return Utils.completedCf(Result.ok(QueryOk.emptyOk())); + private CompletableFuture> emptyOk() { + return Utils.completedCf(Result.ok(SqlQueryOk.emptyOk())); } } diff --git a/ceresdb-protocol/src/test/java/io/ceresdb/UtilsTest.java b/ceresdb-protocol/src/test/java/io/ceresdb/UtilsTest.java index ad62ce3..1735508 100644 --- a/ceresdb-protocol/src/test/java/io/ceresdb/UtilsTest.java +++ b/ceresdb-protocol/src/test/java/io/ceresdb/UtilsTest.java @@ -1,24 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -28,21 +15,19 @@ import io.ceresdb.common.Endpoint; import io.ceresdb.models.Err; +import io.ceresdb.models.Point; import io.ceresdb.models.Result; -import io.ceresdb.models.Rows; import io.ceresdb.models.WriteOk; import io.ceresdb.util.TestUtil; +import io.ceresdb.util.Utils; -/** - * - * @author jiachun.fjc - */ public class UtilsTest { @Test public void combineOkResultTest() { final Result ok1 = WriteOk.ok(100, 1, null).mapToResult(); - final Result ok2 = WriteOk.ok(200, 0, Collections.singletonList("m1")).mapToResult(); + final Result ok2 = WriteOk.ok(200, 0, new HashSet<>(Collections.singletonList("t1"))) + .mapToResult(); final Result r = Utils.combineResult(ok1, ok2); Assert.assertEquals(new Integer(300), r.mapOr(0, WriteOk::getSuccess)); @@ -81,34 +66,34 @@ public void combineErrAndOkResultTest() { @Test public void splitDataBySingleRouteTest() { - final List data = TestUtil.newListOfRows("m1", "m2", "m3"); + final List data = TestUtil.newMultiTablePoints("t1", "t2", "t3"); final Map routes = new HashMap<>(); final Endpoint ep = Endpoint.of("127.0.0.1", 9001); - routes.put("m1", Route.of("m1", ep)); - routes.put("m2", Route.of("m2", ep)); - routes.put("m3", Route.of("m3", ep)); + routes.put("t1", Route.of("t1", ep)); + routes.put("t2", Route.of("t2", ep)); + routes.put("t3", Route.of("t3", ep)); - final Map> result = Utils.splitDataByRoute(data, routes); + final Map> result = Utils.splitDataByRoute(data, routes); Assert.assertEquals(1, result.size()); Assert.assertEquals(ep, result.keySet().stream().findFirst().orElse(null)); - Assert.assertEquals(3, result.values().stream().findFirst().orElse(Collections.emptyList()).size()); + Assert.assertEquals(6, result.values().stream().findFirst().orElse(Collections.emptyList()).size()); } @Test public void splitDataByMultiRouteTest() { - final List data = TestUtil.newListOfRows("m1", "m2", "m3"); + final List data = TestUtil.newMultiTablePoints("t1", "t2", "t3"); final Map routes = new HashMap<>(); final Endpoint ep1 = Endpoint.of("127.0.0.1", 9001); final Endpoint ep2 = Endpoint.of("127.0.0.2", 9001); - routes.put("m1", Route.of("m1", ep1)); - routes.put("m2", Route.of("m2", ep1)); - routes.put("m3", Route.of("m3", ep2)); + routes.put("t1", Route.of("t1", ep1)); + routes.put("t2", Route.of("t2", ep1)); + routes.put("t3", Route.of("t3", ep2)); - final Map> result = Utils.splitDataByRoute(data, routes); + final Map> result = Utils.splitDataByRoute(data, routes); Assert.assertEquals(2, result.size()); - Assert.assertEquals(2, result.get(ep1).size()); - Assert.assertEquals(1, result.get(ep2).size()); + Assert.assertEquals(4, result.get(ep1).size()); + Assert.assertEquals(2, result.get(ep2).size()); } } diff --git a/ceresdb-protocol/src/test/java/io/ceresdb/WriteClientTest.java b/ceresdb-protocol/src/test/java/io/ceresdb/WriteClientTest.java index de0e7ac..adc6f8f 100644 --- a/ceresdb-protocol/src/test/java/io/ceresdb/WriteClientTest.java +++ b/ceresdb-protocol/src/test/java/io/ceresdb/WriteClientTest.java @@ -1,24 +1,13 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; @@ -29,6 +18,10 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import io.ceresdb.models.Point; +import io.ceresdb.models.RequestContext; +import io.ceresdb.models.Value; +import io.ceresdb.models.WriteRequest; import io.ceresdb.proto.internal.Common; import io.ceresdb.proto.internal.Storage; import org.junit.After; @@ -43,19 +36,15 @@ import io.ceresdb.common.Endpoint; import io.ceresdb.common.util.Clock; import io.ceresdb.models.Err; -import io.ceresdb.models.FieldValue; import io.ceresdb.models.Result; -import io.ceresdb.models.Rows; -import io.ceresdb.models.Series; import io.ceresdb.models.WriteOk; import io.ceresdb.options.WriteOptions; import io.ceresdb.rpc.Context; import io.ceresdb.rpc.Observer; +import io.ceresdb.util.StreamWriteBuf; import io.ceresdb.util.TestUtil; +import io.ceresdb.util.Utils; -/** - * @author jiachun.fjc - */ @RunWith(value = MockitoJUnitRunner.class) public class WriteClientTest { @@ -68,6 +57,7 @@ public void before() { final WriteOptions writeOpts = new WriteOptions(); writeOpts.setAsyncPool(ForkJoinPool.commonPool()); writeOpts.setRoutedClient(this.routerClient); + writeOpts.setDatabase("public"); this.writeClient = new WriteClient(); this.writeClient.init(writeOpts); @@ -81,9 +71,9 @@ public void after() { @Test public void writeAllSuccessTest() throws ExecutionException, InterruptedException { - final List data = TestUtil.newListOfRows("write_client_test_metric1", // - "write_client_test_metric2", // - "write_client_test_metric3"); + final List data = TestUtil.newMultiTablePoints("write_client_test_table1", // + "write_client_test_table2", // + "write_client_test_table3"); final Endpoint ep1 = Endpoint.of("127.0.0.1", 8081); final Endpoint ep2 = Endpoint.of("127.0.0.2", 8081); @@ -102,18 +92,19 @@ public void writeAllSuccessTest() throws ExecutionException, InterruptedExceptio .thenReturn(Utils.completedCf(resp)); Mockito.when(this.routerClient.invoke(Mockito.eq(ep3), Mockito.any(), Mockito.any())) // .thenReturn(Utils.completedCf(resp)); - Mockito.when(this.routerClient.routeFor(Mockito.any())) // + Mockito.when(this.routerClient.routeFor(Mockito.any(), Mockito.any())) // .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = 4694599978937545735L; { - put("write_client_test_metric1", Route.of("write_client_test_metric1", ep1)); - put("write_client_test_metric2", Route.of("write_client_test_metric2", ep2)); - put("write_client_test_metric3", Route.of("write_client_test_metric3", ep3)); + put("write_client_test_table1", Route.of("write_client_test_table1", ep1)); + put("write_client_test_table2", Route.of("write_client_test_table2", ep2)); + put("write_client_test_table3", Route.of("write_client_test_table3", ep3)); } })); - final CompletableFuture> f = this.writeClient.write(data, Context.newDefault()); + final CompletableFuture> f = this.writeClient.write(new WriteRequest(data), + Context.newDefault()); final Result ret = f.get(); Assert.assertTrue(ret.isOk()); @@ -123,9 +114,9 @@ public void writeAllSuccessTest() throws ExecutionException, InterruptedExceptio @Test public void write3And1InvalidRoute() throws ExecutionException, InterruptedException { - final List data = TestUtil.newListOfRows("write_client_test_metric1", // - "write_client_test_metric2", // - "write_client_test_metric3"); + final List data = TestUtil.newMultiTablePoints("write_client_test_table1", // + "write_client_test_table2", // + "write_client_test_table3"); final Endpoint ep1 = Endpoint.of("127.0.0.1", 8081); final Endpoint ep2 = Endpoint.of("127.0.0.2", 8081); @@ -143,35 +134,36 @@ public void write3And1InvalidRoute() throws ExecutionException, InterruptedExcep .thenReturn(Utils.completedCf(errResp)); Mockito.when(this.routerClient.invoke(Mockito.eq(ep4), Mockito.any(), Mockito.any())) // .thenReturn(Utils.completedCf(resp)); - Mockito.when(this.routerClient.routeFor(Mockito.eq(TestUtil.asSet("write_client_test_metric1", // - "write_client_test_metric2", "write_client_test_metric3")))) // + Mockito.when(this.routerClient.routeFor(Mockito.any(), Mockito.eq(TestUtil.asSet("write_client_test_table1", // + "write_client_test_table2", "write_client_test_table3")))) // .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = -8646902388192715970L; { - put("write_client_test_metric1", Route.of("write_client_test_metric1", ep1)); - put("write_client_test_metric2", Route.of("write_client_test_metric2", ep2)); - put("write_client_test_metric3", Route.of("write_client_test_metric3", ep3)); + put("write_client_test_table1", Route.of("write_client_test_table1", ep1)); + put("write_client_test_table2", Route.of("write_client_test_table2", ep2)); + put("write_client_test_table3", Route.of("write_client_test_table3", ep3)); } })); - Mockito.when(this.routerClient.routeRefreshFor(Mockito.any())) + Mockito.when(this.routerClient.routeRefreshFor(Mockito.any(), Mockito.any())) .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = -3271323053870289591L; { - put("write_client_test_metric3", Route.of("write_client_test_metric3", ep4)); + put("write_client_test_table3", Route.of("write_client_test_table3", ep4)); } })); - Mockito.when(this.routerClient.routeFor(Mockito.eq(TestUtil.asSet("write_client_test_metric3")))) // + Mockito.when(this.routerClient.routeFor(Mockito.any(), Mockito.eq(TestUtil.asSet("write_client_test_table3")))) // .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = 4340010451723257789L; { - put("write_client_test_metric3", Route.of("write_client_test_metric3", ep4)); + put("write_client_test_table3", Route.of("write_client_test_table3", ep4)); } })); - final CompletableFuture> f = this.writeClient.write(data, Context.newDefault()); + final CompletableFuture> f = this.writeClient.write(new WriteRequest(data), + Context.newDefault()); final Result ret = f.get(); Assert.assertTrue(ret.isOk()); @@ -181,9 +173,9 @@ public void write3And1InvalidRoute() throws ExecutionException, InterruptedExcep @Test public void write3And1InvalidRouteAndRetryFailed() throws ExecutionException, InterruptedException { - final List data = TestUtil.newListOfRows("write_client_test_metric1", // - "write_client_test_metric2", // - "write_client_test_metric3"); + final List data = TestUtil.newMultiTablePoints("write_client_test_table1", // + "write_client_test_table2", // + "write_client_test_table3"); final Endpoint ep1 = Endpoint.of("127.0.0.1", 8081); final Endpoint ep2 = Endpoint.of("127.0.0.2", 8081); @@ -198,35 +190,37 @@ public void write3And1InvalidRouteAndRetryFailed() throws ExecutionException, In .thenReturn(Utils.completedCf(resp)); Mockito.when(this.routerClient.invoke(Mockito.eq(ep3), Mockito.any(), Mockito.any())) // .thenReturn(Utils.completedCf(errResp)); - Mockito.when(this.routerClient.routeFor(Mockito.eq( - TestUtil.asSet("write_client_test_metric1", "write_client_test_metric2", "write_client_test_metric3")))) // + Mockito.when(this.routerClient.routeFor(Mockito.any(), + Mockito.eq(TestUtil.asSet("write_client_test_table1", "write_client_test_table2", + "write_client_test_table3")))) // .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = -7535390185627686991L; { - put("write_client_test_metric1", Route.of("write_client_test_metric1", ep1)); - put("write_client_test_metric2", Route.of("write_client_test_metric2", ep2)); - put("write_client_test_metric3", Route.of("write_client_test_metric3", ep3)); + put("write_client_test_table1", Route.of("write_client_test_table1", ep1)); + put("write_client_test_table2", Route.of("write_client_test_table2", ep2)); + put("write_client_test_table3", Route.of("write_client_test_table3", ep3)); } })); - Mockito.when(this.routerClient.routeRefreshFor(Mockito.any())) + Mockito.when(this.routerClient.routeRefreshFor(Mockito.any(), Mockito.any())) .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = -3191375160670801662L; { - put("write_client_test_metric3", Route.of("write_client_test_metric3", ep3)); + put("write_client_test_table3", Route.of("write_client_test_table3", ep3)); } })); - Mockito.when(this.routerClient.routeFor(Mockito.eq(TestUtil.asSet("write_client_test_metric3")))) // + Mockito.when(this.routerClient.routeFor(Mockito.any(), Mockito.eq(TestUtil.asSet("write_client_test_table3")))) // .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = 1341458669202248824L; { - put("write_client_test_metric3", Route.of("write_client_test_metric3", ep3)); + put("write_client_test_table3", Route.of("write_client_test_table3", ep3)); } })); - final CompletableFuture> f = this.writeClient.write(data, Context.newDefault()); + final CompletableFuture> f = this.writeClient.write(new WriteRequest(data), + Context.newDefault()); final Result ret = f.get(); final int success = ret.mapOrElse(err -> -1, WriteOk::getSuccess); @@ -234,14 +228,14 @@ public void write3And1InvalidRouteAndRetryFailed() throws ExecutionException, In Assert.assertFalse(ret.isOk()); Assert.assertEquals(4, ret.getErr().getSubOk().getSuccess()); Assert.assertEquals(0, ret.getErr().getSubOk().getFailed()); - Assert.assertEquals(1, ret.getErr().getFailedWrites().size()); + Assert.assertEquals(2, ret.getErr().getFailedWrites().size()); } @Test public void write3And2FailedAndRetryFailed() throws ExecutionException, InterruptedException { - final List data = TestUtil.newListOfRows("write_client_test_metric1", // - "write_client_test_metric2", // - "write_client_test_metric3"); + final List data = TestUtil.newMultiTablePoints("write_client_test_table1", // + "write_client_test_table2", // + "write_client_test_table3"); final Endpoint ep1 = Endpoint.of("127.0.0.1", 8081); final Endpoint ep2 = Endpoint.of("127.0.0.2", 8081); @@ -256,63 +250,66 @@ public void write3And2FailedAndRetryFailed() throws ExecutionException, Interrup .thenReturn(Utils.completedCf(errResp)); Mockito.when(this.routerClient.invoke(Mockito.eq(ep3), Mockito.any(), Mockito.any())) // .thenReturn(Utils.completedCf(errResp)); - Mockito.when(this.routerClient.routeFor(Mockito.eq( - TestUtil.asSet("write_client_test_metric1", "write_client_test_metric2", "write_client_test_metric3")))) // + Mockito.when(this.routerClient.routeFor(Mockito.any(), + Mockito.eq(TestUtil.asSet("write_client_test_table1", "write_client_test_table2", + "write_client_test_table3")))) // .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = -5936788008084035345L; { - put("write_client_test_metric1", Route.of("write_client_test_metric1", ep1)); - put("write_client_test_metric2", Route.of("write_client_test_metric2", ep2)); - put("write_client_test_metric3", Route.of("write_client_test_metric3", ep3)); + put("write_client_test_table1", Route.of("write_client_test_table1", ep1)); + put("write_client_test_table2", Route.of("write_client_test_table2", ep2)); + put("write_client_test_table3", Route.of("write_client_test_table3", ep3)); } })); - Mockito.when(this.routerClient.routeRefreshFor(Mockito.any())) + Mockito.when(this.routerClient.routeRefreshFor(Mockito.any(), Mockito.any())) .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = -4748944007591733357L; { - put("write_client_test_metric2", Route.of("write_client_test_metric2", ep2)); - put("write_client_test_metric3", Route.of("write_client_test_metric3", ep3)); + put("write_client_test_table2", Route.of("write_client_test_table2", ep2)); + put("write_client_test_table3", Route.of("write_client_test_table3", ep3)); } })); - Mockito.when(this.routerClient - .routeFor(Mockito.eq(TestUtil.asSet("write_client_test_metric2", "write_client_test_metric3")))) // + Mockito.when(this.routerClient.routeFor(Mockito.any(), + Mockito.eq(TestUtil.asSet("write_client_test_table2", "write_client_test_table3")))) // .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = -1811964578845864624L; { - put("write_client_test_metric2", Route.of("write_client_test_metric2", ep2)); - put("write_client_test_metric3", Route.of("write_client_test_metric3", ep3)); + put("write_client_test_table2", Route.of("write_client_test_table2", ep2)); + put("write_client_test_table3", Route.of("write_client_test_table3", ep3)); } })); - Mockito.when(this.routerClient - .routeFor(Mockito.eq(TestUtil.asSet("write_client_test_metric3", "write_client_test_metric2")))) // + Mockito.when(this.routerClient.routeFor(Mockito.any(), + Mockito.eq(TestUtil.asSet("write_client_test_table3", "write_client_test_table2")))) // .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = 3940955382371644111L; { - put("write_client_test_metric2", Route.of("write_client_test_metric2", ep2)); - put("write_client_test_metric3", Route.of("write_client_test_metric3", ep3)); + put("write_client_test_table2", Route.of("write_client_test_table2", ep2)); + put("write_client_test_table3", Route.of("write_client_test_table3", ep3)); } })); - final CompletableFuture> f = this.writeClient.write(data, Context.newDefault()); + final CompletableFuture> f = this.writeClient.write(new WriteRequest(data), + Context.newDefault()); final Result ret = f.get(); Assert.assertFalse(ret.isOk()); Assert.assertEquals(2, ret.getErr().getSubOk().getSuccess()); Assert.assertEquals(0, ret.getErr().getSubOk().getFailed()); - Assert.assertEquals(1, ret.getErr().getFailedWrites().size()); Assert.assertTrue(ret.getErr().stream().findFirst().isPresent()); - Assert.assertEquals(1, ret.getErr().stream().findFirst().get().getFailedWrites().size()); + Assert.assertEquals(2, ret.getErr().stream().count()); + Assert.assertEquals(4, ret.getErr().stream().mapToInt(err -> err.getFailedWrites().size()).sum()); + } @Test public void write3And2FailedAndSomeNoRetry() throws ExecutionException, InterruptedException { - final List data = TestUtil.newListOfRows("write_client_test_metric1", // - "write_client_test_metric2", // - "write_client_test_metric3"); + final List data = TestUtil.newMultiTablePoints("write_client_test_table1", // + "write_client_test_table2", // + "write_client_test_table3"); final Endpoint ep1 = Endpoint.of("127.0.0.1", 8081); final Endpoint ep2 = Endpoint.of("127.0.0.2", 8081); @@ -328,53 +325,55 @@ public void write3And2FailedAndSomeNoRetry() throws ExecutionException, Interrup .thenReturn(Utils.completedCf(errResp1)); Mockito.when(this.routerClient.invoke(Mockito.eq(ep3), Mockito.any(), Mockito.any())) // .thenReturn(Utils.completedCf(errResp2)); - Mockito.when(this.routerClient.routeFor(Mockito.eq( - TestUtil.asSet("write_client_test_metric1", "write_client_test_metric2", "write_client_test_metric3")))) // + Mockito.when(this.routerClient.routeFor(Mockito.any(), + Mockito.eq(TestUtil.asSet("write_client_test_table1", "write_client_test_table2", + "write_client_test_table3")))) // .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = 1040769477529210661L; { - put("write_client_test_metric1", Route.of("write_client_test_metric1", ep1)); - put("write_client_test_metric2", Route.of("write_client_test_metric2", ep2)); - put("write_client_test_metric3", Route.of("write_client_test_metric3", ep3)); + put("write_client_test_table1", Route.of("write_client_test_table1", ep1)); + put("write_client_test_table2", Route.of("write_client_test_table2", ep2)); + put("write_client_test_table3", Route.of("write_client_test_table3", ep3)); } })); - Mockito.when(this.routerClient.routeRefreshFor(Mockito.any())) + Mockito.when(this.routerClient.routeRefreshFor(Mockito.any(), Mockito.any())) .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = -6892083230027668740L; { - put("write_client_test_metric2", Route.of("write_client_test_metric2", ep2)); - put("write_client_test_metric3", Route.of("write_client_test_metric3", ep3)); + put("write_client_test_table2", Route.of("write_client_test_table2", ep2)); + put("write_client_test_table3", Route.of("write_client_test_table3", ep3)); } })); - Mockito.when(this.routerClient.routeFor(Mockito.eq(TestUtil.asSet("write_client_test_metric2")))) // + Mockito.when(this.routerClient.routeFor(Mockito.any(), Mockito.eq(TestUtil.asSet("write_client_test_table2")))) // .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = -9174308983134252825L; { - put("write_client_test_metric2", Route.of("write_client_test_metric2", ep2)); + put("write_client_test_table2", Route.of("write_client_test_table2", ep2)); } })); - final CompletableFuture> f = this.writeClient.write(data, Context.newDefault()); + final CompletableFuture> f = this.writeClient.write(new WriteRequest(data), + Context.newDefault()); final Result ret = f.get(); Assert.assertFalse(ret.isOk()); Assert.assertEquals(2, ret.getErr().getSubOk().getSuccess()); Assert.assertEquals(0, ret.getErr().getSubOk().getFailed()); - Assert.assertEquals(1, ret.getErr().getFailedWrites().size()); - Assert.assertTrue(ret.getErr().stream().findFirst().isPresent()); - Assert.assertEquals(1, ret.getErr().stream().findFirst().get().getFailedWrites().size()); + // TODO + // How to process multi err + //Assert.assertEquals(1, ret.getErr().getFailedWrites().size()); + //Assert.assertTrue(ret.getErr().stream().findFirst().isPresent()); + //Assert.assertEquals(1, ret.getErr().stream().findFirst().get().getFailedWrites().size()); } @Test public void writeSplitTest() throws ExecutionException, InterruptedException { - writeSplit(1, 2); + writeSplit(1, 1); writeSplit(2, 2); - writeSplit(3, 2); writeSplit(4, 4); - writeSplit(5, 4); writeSplit(8, 8); } @@ -383,20 +382,21 @@ private void writeSplit(final int maxWriteSize, final int partOfSuccess) // re-init this.writeClient.shutdownGracefully(); final WriteOptions writeOpts = new WriteOptions(); + writeOpts.setDatabase("public"); writeOpts.setAsyncPool(ForkJoinPool.commonPool()); writeOpts.setRoutedClient(this.routerClient); writeOpts.setMaxWriteSize(maxWriteSize); this.writeClient = new WriteClient(); this.writeClient.init(writeOpts); - final List data = TestUtil.newListOfRows("write_client_test_metric1", // - "write_client_test_metric2", // - "write_client_test_metric3", // - "write_client_test_metric4", // - "write_client_test_metric5", // - "write_client_test_metric6", // - "write_client_test_metric7", // - "write_client_test_metric8"); + final List data = TestUtil.newMultiTablePoints("write_client_test_table1", // + "write_client_test_table2", // + "write_client_test_table3", // + "write_client_test_table4", // + "write_client_test_table5", // + "write_client_test_table6", // + "write_client_test_table7", // + "write_client_test_table8"); final Endpoint ep = Endpoint.of("127.0.0.1", 8081); @@ -409,23 +409,24 @@ private void writeSplit(final int maxWriteSize, final int partOfSuccess) .build(); Mockito.when(this.routerClient.invoke(Mockito.eq(ep), Mockito.any(), Mockito.any())) // .thenReturn(Utils.completedCf(resp)); - Mockito.when(this.routerClient.routeFor(Mockito.any())) // + Mockito.when(this.routerClient.routeFor(Mockito.any(), Mockito.any())) // .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = 4694599978937545735L; { - put("write_client_test_metric1", Route.of("write_client_test_metric1", ep)); - put("write_client_test_metric2", Route.of("write_client_test_metric2", ep)); - put("write_client_test_metric3", Route.of("write_client_test_metric3", ep)); - put("write_client_test_metric4", Route.of("write_client_test_metric4", ep)); - put("write_client_test_metric5", Route.of("write_client_test_metric5", ep)); - put("write_client_test_metric6", Route.of("write_client_test_metric6", ep)); - put("write_client_test_metric7", Route.of("write_client_test_metric7", ep)); - put("write_client_test_metric8", Route.of("write_client_test_metric8", ep)); + put("write_client_test_table1", Route.of("write_client_test_table1", ep)); + put("write_client_test_table2", Route.of("write_client_test_table2", ep)); + put("write_client_test_table3", Route.of("write_client_test_table3", ep)); + put("write_client_test_table4", Route.of("write_client_test_table4", ep)); + put("write_client_test_table5", Route.of("write_client_test_table5", ep)); + put("write_client_test_table6", Route.of("write_client_test_table6", ep)); + put("write_client_test_table7", Route.of("write_client_test_table7", ep)); + put("write_client_test_table8", Route.of("write_client_test_table8", ep)); } })); - final CompletableFuture> f = this.writeClient.write(data, Context.newDefault()); + final CompletableFuture> f = this.writeClient.write(new WriteRequest(data), + Context.newDefault()); final Result ret = f.get(); Assert.assertTrue(ret.isOk()); @@ -435,15 +436,15 @@ private void writeSplit(final int maxWriteSize, final int partOfSuccess) @Test public void streamWriteTest() { - final String testMetric = "stream_metric_test"; + final String testTable = "stream_table_test"; final Endpoint ep = Endpoint.of("127.0.0.1", 8081); - Mockito.when(this.routerClient.routeFor(Mockito.eq(Collections.singleton(testMetric)))) // + Mockito.when(this.routerClient.routeFor(Mockito.any(), Mockito.eq(Collections.singleton(testTable)))) // .thenReturn(Utils.completedCf(new HashMap() { private static final long serialVersionUID = 8473563130528272901L; { - put(testMetric, Route.of(testMetric, ep)); + put(testTable, Route.of(testTable, ep)); } })); final CompletableFuture f = new CompletableFuture<>(); @@ -453,9 +454,9 @@ public void streamWriteTest() { @Override public void onNext(final Storage.WriteRequest value) { - final int c = value.getMetricsList().stream() // - .flatMap(wmc -> wmc.getEntriesList().stream()) // - .map(Storage.WriteEntry::getFieldGroupsCount).reduce(0, Integer::sum); + final int c = value.getTableRequestsList().stream() // + .flatMap(writeTableRequest -> writeTableRequest.getEntriesList().stream()) // + .map(Storage.WriteSeriesEntry::getFieldGroupsCount).reduce(0, Integer::sum); dataCount.addAndGet(c); } @@ -470,15 +471,15 @@ public void onCompleted() { } }); - final StreamWriteBuf writer = this.writeClient.streamWrite(testMetric); + final StreamWriteBuf writer = this.writeClient.streamWrite(testTable); final CompletableFuture ret = writer // - .write(TestUtil.newRow(testMetric)) // - .write(TestUtil.newRow(testMetric)) // - .write(TestUtil.newRow(testMetric)) // + .write(TestUtil.newTablePoints(testTable)) // + .write(TestUtil.newTablePoints(testTable)) // + .write(TestUtil.newTablePoints(testTable)) // .flush() // - .write(TestUtil.newRow(testMetric)) // + .write(TestUtil.newTablePoints(testTable)) // .flush() // - .writeAndFlush(TestUtil.newListOfRows(testMetric, testMetric)) // + .writeAndFlush(TestUtil.newMultiTablePoints(testTable, testTable)) // .completed(); f.whenComplete((r, e) -> { @@ -494,72 +495,65 @@ public void onCompleted() { @Test public void rowsToWriteProtoTest() { - final Rows rs1 = Series.newBuilder("metric1") // - .tag("m1_tag1", "v1") // - .tag("m1_tag2", "v2") // - .tag("m1_tag3", "v3") // - .toRowsBuilder(true) // - .fields(Clock.defaultClock().getTick(), input -> { - input.put("m1_field1", FieldValue.withDouble(0.1)); - input.put("m1_field2", FieldValue.withString("he")); - input.put("m1_field3", FieldValue.withStringOrNull(null)); // null - }) // - .fields(Clock.defaultClock().getTick() + 10, input -> { - input.put("m1_field1", FieldValue.withDouble(0.1)); - input.put("m1_field2", FieldValue.withString("he")); - input.put("m1_field3", FieldValue.withString("surprise!!!")); - }) // - .build(); - - final Rows rs2 = Series.newBuilder("metric2") // - .tag("m2_tag1", "v1") // - .tag("m2_tag2", "v2") // - .tag("m3_tag3", "v3") // - .toRowsBuilder(true) // - .fields(Clock.defaultClock().getTick(), input -> { - input.put("m2_field1", FieldValue.withDouble(0.1)); - input.put("m2_field2", FieldValue.withString("he")); - }) // - .fields(Clock.defaultClock().getTick() + 10, input -> { - input.put("m2_field1", FieldValue.withDouble(0.1)); - input.put("m2_field2", FieldValue.withString("he")); - input.put("m2_field3", FieldValue.withString("surprise!!!")); - }) // - .build(); - - final Rows rs3 = Series.newBuilder("metric1") // - .tag("m1_tag1", "vv1") // - .tag("m1_tag2", "vv2") // - .toRowsBuilder(true) // - .fields(System.currentTimeMillis(), input -> { - input.put("m1_field1", FieldValue.withDouble(0.1)); - input.put("m1_field2", FieldValue.withString("he")); - }) // - .build(); - - final Storage.WriteRequest writeReq = this.writeClient.toWriteRequestObj(Stream.of(rs1, rs2, rs3)); + List table1 = new ArrayList<>(); + table1.add(Point.newPointBuilder("table1") // + .setTimestamp(Clock.defaultClock().getTick()).addTag("t1_tag1", Value.withString("v1")) // + .addTag("t1_tag2", Value.withString("v2")) // + .addTag("t1_tag3", Value.withString("v3")) // + .addField("t1_field1", Value.withDouble(0.1)).addField("t1_field2", Value.withString("he")) + .addField("t1_field3", Value.withStringOrNull(null)).build()); + table1.add(Point.newPointBuilder("table1") // + .setTimestamp(Clock.defaultClock().getTick()).addTag("t1_tag1", Value.withString("v1")) // + .addTag("t1_tag2", Value.withString("v2")) // + .addTag("t1_tag3", Value.withString("v3")) // + .addField("t1_field1", Value.withDouble(0.1)).addField("t1_field2", Value.withString("he")) + .addField("t1_field3", Value.withStringOrNull("surprise!!!")).build()); + + List table2 = new ArrayList<>(); + table2.add(Point.newPointBuilder("table2") // + .setTimestamp(Clock.defaultClock().getTick()).addTag("t2_tag1", Value.withString("v1")) // + .addTag("t2_tag2", Value.withString("v2")) // + .addTag("t2_tag3", Value.withString("v3")) // + .addField("t2_field1", Value.withDouble(0.1)).addField("t2_field2", Value.withString("he")).build()); + table2.add(Point.newPointBuilder("table2").setTimestamp(Clock.defaultClock().getTick() + 10) + .addTag("t2_tag1", Value.withString("v1")) // + .addTag("t2_tag2", Value.withString("v2")) // + .addTag("t2_tag3", Value.withString("v3")) // + .addField("t2_field1", Value.withDouble(0.1)).addField("t2_field2", Value.withString("he")) + .addField("t2_field3", Value.withStringOrNull("surprise!!!")).build()); + + final List table3 = Collections.singletonList(Point.newPointBuilder("table1") + .setTimestamp(System.currentTimeMillis()).addTag("t1_tag1", Value.withString("vv1")) // + .addTag("t1_tag2", Value.withString("vv2")) // + .addField("t1_field1", Value.withDouble(0.1)).addField("t1_field2", Value.withString("he")).build()); + + RequestContext reqCtx = new RequestContext(); + reqCtx.setDatabase("public"); + + final Storage.WriteRequest writeReq = this.writeClient.toWriteRequestObj(reqCtx, + Stream.of(table1, table2, table3).flatMap(List::stream).collect(Collectors.toList()).stream()); Assert.assertNotNull(writeReq); - Assert.assertEquals(2, writeReq.getMetricsCount()); - final List metrics = writeReq.getMetricsList() // + Assert.assertEquals(2, writeReq.getTableRequestsCount()); + final Set tables = writeReq.getTableRequestsList() // .stream() // - .map(Storage.WriteMetric::getMetric) // - .collect(Collectors.toList()); - Assert.assertEquals(Arrays.asList("metric1", "metric2"), metrics); + .map(Storage.WriteTableRequest::getTable) // + .collect(Collectors.toSet()); + Assert.assertEquals(new HashSet<>(Arrays.asList("table1", "table2")), tables); { - final Optional opl = writeReq.getMetricsList().stream() - .filter(wm -> wm.getMetric().equals("metric1")) // + final Optional opl = writeReq.getTableRequestsList().stream() + .filter(wm -> wm.getTable().equals("table1")) // .findFirst(); // Assert.assertTrue(opl.isPresent()); - final Storage.WriteMetric metric = opl.get(); - Assert.assertEquals(3, metric.getTagNamesCount()); - Assert.assertEquals(3, metric.getFieldNamesCount()); - Assert.assertEquals(2, metric.getEntriesCount()); + final Storage.WriteTableRequest tableRequest = opl.get(); + Assert.assertEquals(3, tableRequest.getTagNamesCount()); + Assert.assertEquals(3, tableRequest.getFieldNamesCount()); + Assert.assertEquals(2, tableRequest.getEntriesCount()); { - final Optional opl2 = metric.getEntriesList().stream().filter(w -> { + final Optional opl2 = tableRequest.getEntriesList().stream().filter(w -> { final Set tagVSet = w.getTagsList().stream().map(tag -> tag.getValue().getStringValue()) .collect(Collectors.toSet()); @@ -572,27 +566,22 @@ public void rowsToWriteProtoTest() { Assert.assertEquals(2, opl2.get().getFieldGroupsCount()); Assert.assertEquals(2, opl2.get().getFieldGroups(0).getFieldsCount()); Assert.assertEquals(3, opl2.get().getFieldGroups(1).getFieldsCount()); - Assert.assertEquals(0, opl2.get().getFieldGroups(0).getFields(0).getNameIndex()); - Assert.assertEquals(1, opl2.get().getFieldGroups(0).getFields(1).getNameIndex()); - Assert.assertEquals(0, opl2.get().getFieldGroups(1).getFields(0).getNameIndex()); - Assert.assertEquals(1, opl2.get().getFieldGroups(1).getFields(1).getNameIndex()); - Assert.assertEquals(2, opl2.get().getFieldGroups(1).getFields(2).getNameIndex()); } } { - final Optional opl = writeReq.getMetricsList().stream() - .filter(wm -> wm.getMetric().equals("metric2")) // + final Optional opl = writeReq.getTableRequestsList().stream() + .filter(wm -> wm.getTable().equals("table2")) // .findFirst(); // Assert.assertTrue(opl.isPresent()); - final Storage.WriteMetric metric = opl.get(); - Assert.assertEquals(3, metric.getTagNamesCount()); - Assert.assertEquals(3, metric.getFieldNamesCount()); - Assert.assertEquals(1, metric.getEntriesCount()); + final Storage.WriteTableRequest tableRequest = opl.get(); + Assert.assertEquals(3, tableRequest.getTagNamesCount()); + Assert.assertEquals(3, tableRequest.getFieldNamesCount()); + Assert.assertEquals(1, tableRequest.getEntriesCount()); { - final Optional opl2 = metric.getEntriesList().stream().filter(w -> { + final Optional opl2 = tableRequest.getEntriesList().stream().filter(w -> { final Set tagVSet = w.getTagsList().stream().map(tag -> tag.getValue().getStringValue()) .collect(Collectors.toSet()); @@ -605,11 +594,6 @@ public void rowsToWriteProtoTest() { Assert.assertEquals(2, opl2.get().getFieldGroupsCount()); Assert.assertEquals(2, opl2.get().getFieldGroups(0).getFieldsCount()); Assert.assertEquals(3, opl2.get().getFieldGroups(1).getFieldsCount()); - Assert.assertEquals(0, opl2.get().getFieldGroups(0).getFields(0).getNameIndex()); - Assert.assertEquals(1, opl2.get().getFieldGroups(0).getFields(1).getNameIndex()); - Assert.assertEquals(0, opl2.get().getFieldGroups(1).getFields(0).getNameIndex()); - Assert.assertEquals(1, opl2.get().getFieldGroups(1).getFields(1).getNameIndex()); - Assert.assertEquals(2, opl2.get().getFieldGroups(1).getFields(2).getNameIndex()); } } } diff --git a/ceresdb-protocol/src/test/java/io/ceresdb/WriteLimitTest.java b/ceresdb-protocol/src/test/java/io/ceresdb/WriteLimitTest.java index b7b8294..a7d9524 100644 --- a/ceresdb-protocol/src/test/java/io/ceresdb/WriteLimitTest.java +++ b/ceresdb-protocol/src/test/java/io/ceresdb/WriteLimitTest.java @@ -1,22 +1,9 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb; -import java.util.Collection; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -26,37 +13,37 @@ import org.junit.Test; import io.ceresdb.errors.LimitedException; +import io.ceresdb.limit.LimitedPolicy; +import io.ceresdb.limit.WriteLimiter; import io.ceresdb.models.Err; +import io.ceresdb.models.Point; import io.ceresdb.models.Result; -import io.ceresdb.models.Rows; import io.ceresdb.models.WriteOk; import io.ceresdb.util.TestUtil; +import io.ceresdb.util.Utils; -/** - * @author jiachun.fjc - */ public class WriteLimitTest { @Test(expected = LimitedException.class) public void abortWriteLimitTest() throws ExecutionException, InterruptedException { final WriteLimiter limiter = new WriteClient.DefaultWriteLimiter(1, new LimitedPolicy.AbortPolicy()); - final Collection rows = TestUtil.newListOfRows("test1", "test2"); + final List points = TestUtil.newMultiTablePoints("test1", "test2"); // consume the permits - limiter.acquireAndDo(rows, CompletableFuture::new); + limiter.acquireAndDo(points, CompletableFuture::new); - limiter.acquireAndDo(rows, this::emptyOk).get(); + limiter.acquireAndDo(points, this::emptyOk).get(); } @Test public void discardWriteLimitTest() throws ExecutionException, InterruptedException { final WriteLimiter limiter = new WriteClient.DefaultWriteLimiter(1, new LimitedPolicy.DiscardPolicy()); - final Collection rows = TestUtil.newListOfRows("test1", "test2"); + final List points = TestUtil.newMultiTablePoints("test1", "test2"); // consume the permits - limiter.acquireAndDo(rows, CompletableFuture::new); + limiter.acquireAndDo(points, CompletableFuture::new); - final Result ret = limiter.acquireAndDo(rows, this::emptyOk).get(); + final Result ret = limiter.acquireAndDo(points, this::emptyOk).get(); Assert.assertFalse(ret.isOk()); Assert.assertEquals(Result.FLOW_CONTROL, ret.getErr().getCode()); @@ -67,15 +54,15 @@ public void discardWriteLimitTest() throws ExecutionException, InterruptedExcept @Test public void blockingWriteLimitTest() throws InterruptedException { final WriteLimiter limiter = new WriteClient.DefaultWriteLimiter(1, new LimitedPolicy.BlockingPolicy()); - final Collection rows = TestUtil.newListOfRows("test1", "test2"); + final List points = TestUtil.newMultiTablePoints("test1", "test2"); // consume the permits - limiter.acquireAndDo(rows, CompletableFuture::new); + limiter.acquireAndDo(points, CompletableFuture::new); final AtomicBoolean alwaysFalse = new AtomicBoolean(); final Thread t = new Thread(() -> { try { - limiter.acquireAndDo(rows, this::emptyOk); + limiter.acquireAndDo(points, this::emptyOk); alwaysFalse.set(true); } catch (final Throwable err) { // noinspection ConstantConditions @@ -97,13 +84,13 @@ public void blockingTimeoutWriteLimitTest() throws ExecutionException, Interrupt final int timeoutSecs = 2; final WriteLimiter limiter = new WriteClient.DefaultWriteLimiter(1, new LimitedPolicy.BlockingTimeoutPolicy(timeoutSecs, TimeUnit.SECONDS)); - final Collection rows = TestUtil.newListOfRows("test1", "test2"); + final List points = TestUtil.newMultiTablePoints("test1", "test2"); // consume the permits - limiter.acquireAndDo(rows, CompletableFuture::new); + limiter.acquireAndDo(points, CompletableFuture::new); final long start = System.nanoTime(); - final Result ret = limiter.acquireAndDo(rows, this::emptyOk).get(); + final Result ret = limiter.acquireAndDo(points, this::emptyOk).get(); Assert.assertEquals(timeoutSecs, TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start), 0.3); Assert.assertFalse(ret.isOk()); @@ -117,14 +104,14 @@ public void abortOnBlockingTimeoutWriteLimitTest() throws ExecutionException, In final int timeoutSecs = 2; final WriteLimiter limiter = new WriteClient.DefaultWriteLimiter(1, new LimitedPolicy.AbortOnBlockingTimeoutPolicy(timeoutSecs, TimeUnit.SECONDS)); - final Collection rows = TestUtil.newListOfRows("test1", "test2"); + final List points = TestUtil.newMultiTablePoints("test1", "test2"); // consume the permits - limiter.acquireAndDo(rows, CompletableFuture::new); + limiter.acquireAndDo(points, CompletableFuture::new); final long start = System.nanoTime(); try { - limiter.acquireAndDo(rows, this::emptyOk).get(); + limiter.acquireAndDo(points, this::emptyOk).get(); } finally { Assert.assertEquals(timeoutSecs, TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start), 0.3); } diff --git a/ceresdb-protocol/src/test/java/io/ceresdb/models/ErrTest.java b/ceresdb-protocol/src/test/java/io/ceresdb/models/ErrTest.java index a0a0622..7284f47 100644 --- a/ceresdb-protocol/src/test/java/io/ceresdb/models/ErrTest.java +++ b/ceresdb-protocol/src/test/java/io/ceresdb/models/ErrTest.java @@ -1,25 +1,13 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.models; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import org.junit.Assert; @@ -27,23 +15,23 @@ import io.ceresdb.common.Endpoint; -/** - * @author jiachun.fjc - */ public class ErrTest { @Test public void combineWriteOkTest() { - final List metrics = new ArrayList<>(); - metrics.add("test_metric"); - final WriteOk subOK1 = WriteOk.ok(2, 0, metrics); + final Set tables = new HashSet<>(); + tables.add("test_table"); + final WriteOk subOK1 = WriteOk.ok(2, 0, tables); Err err = Err.writeErr(300, "test_err", Endpoint.of("127.0.0.1", 8081), Collections.emptyList()); err = err.combine(subOK1); Assert.assertEquals(2, err.getSubOk().getSuccess()); - final WriteOk subOK2 = WriteOk.ok(3, 0, Collections.singletonList("test_metric2")); + + final Set tables2 = new HashSet<>(); + tables2.add("test_table2"); + final WriteOk subOK2 = WriteOk.ok(3, 0, tables2); err = err.combine(subOK2); Assert.assertEquals(5, err.getSubOk().getSuccess()); - Assert.assertEquals(Arrays.asList("test_metric", "test_metric2"), err.getSubOk().getMetrics()); + Assert.assertEquals(new HashSet<>(Arrays.asList("test_table", "test_table2")), err.getSubOk().getTables()); } @Test diff --git a/ceresdb-protocol/src/test/java/io/ceresdb/models/FieldValueTest.java b/ceresdb-protocol/src/test/java/io/ceresdb/models/FieldValueTest.java deleted file mode 100644 index 92dd082..0000000 --- a/ceresdb-protocol/src/test/java/io/ceresdb/models/FieldValueTest.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.models; - -import java.math.BigInteger; - -import org.junit.Assert; -import org.junit.Test; - -/** - * - * @author jiachun.fjc - */ -public class FieldValueTest { - - @Test - public void nullableDoubleTest() { - final FieldValue f1 = FieldValue.withDoubleOrNull(0.1); - Assert.assertTrue(f1.getDoubleOrNull().isPresent()); - - final FieldValue f2 = FieldValue.withDoubleOrNull(null); - Assert.assertTrue(f2.isNull()); - Assert.assertFalse(f2.getDoubleOrNull().isPresent()); - } - - @Test - public void nullableStringTest() { - final FieldValue f1 = FieldValue.withStringOrNull("xx"); - Assert.assertTrue(f1.getStringOrNull().isPresent()); - - final FieldValue f2 = FieldValue.withStringOrNull(null); - Assert.assertTrue(f2.isNull()); - Assert.assertFalse(f2.getStringOrNull().isPresent()); - } - - @Test - public void nullableInt64Test() { - final FieldValue f1 = FieldValue.withInt64OrNull(123L); - Assert.assertTrue(f1.getInt64OrNull().isPresent()); - - final FieldValue f2 = FieldValue.withInt64OrNull(null); - Assert.assertTrue(f2.isNull()); - Assert.assertFalse(f2.getInt64OrNull().isPresent()); - } - - @Test - public void nullableFloat32Test() { - final FieldValue f1 = FieldValue.withFloat32OrNull(123.01f); - Assert.assertTrue(f1.getFloat32OrNull().isPresent()); - - final FieldValue f2 = FieldValue.withFloat32OrNull(null); - Assert.assertTrue(f2.isNull()); - Assert.assertFalse(f2.getFloat32OrNull().isPresent()); - } - - @Test - public void nullableInt32Test() { - final FieldValue f1 = FieldValue.withInt32OrNull(123); - Assert.assertTrue(f1.getInt32OrNull().isPresent()); - - final FieldValue f2 = FieldValue.withInt32OrNull(null); - Assert.assertTrue(f2.isNull()); - Assert.assertFalse(f2.getInt32OrNull().isPresent()); - } - - @Test - public void nullableInt16Test() { - final FieldValue f1 = FieldValue.withInt16OrNull(123); - Assert.assertTrue(f1.getInt16OrNull().isPresent()); - - final FieldValue f2 = FieldValue.withInt16OrNull(null); - Assert.assertTrue(f2.isNull()); - Assert.assertFalse(f2.getInt16OrNull().isPresent()); - } - - @Test - public void nullableInt8Test() { - final FieldValue f1 = FieldValue.withInt8OrNull(123); - Assert.assertTrue(f1.getInt8OrNull().isPresent()); - - final FieldValue f2 = FieldValue.withInt8OrNull(null); - Assert.assertTrue(f2.isNull()); - Assert.assertFalse(f2.getInt8OrNull().isPresent()); - } - - @Test - public void nullableBooleanTest() { - final FieldValue f1 = FieldValue.withBooleanOrNull(true); - Assert.assertTrue(f1.getBooleanOrNull().isPresent()); - - final FieldValue f2 = FieldValue.withBooleanOrNull(null); - Assert.assertTrue(f2.isNull()); - Assert.assertFalse(f2.getBooleanOrNull().isPresent()); - } - - @Test - public void nullableUInt64Test() { - final FieldValue f1 = FieldValue.withUInt64OrNull(100L); - Assert.assertTrue(f1.getUInt64OrNull().isPresent()); - - final FieldValue f2 = FieldValue.withUInt64OrNull((Long) null); - Assert.assertTrue(f2.isNull()); - Assert.assertFalse(f2.getUInt64OrNull().isPresent()); - - final FieldValue f3 = FieldValue.withUInt64OrNull((BigInteger) null); - Assert.assertTrue(f3.isNull()); - Assert.assertFalse(f3.getUInt64OrNull().isPresent()); - } - - @Test - public void nullableUInt32Test() { - final FieldValue f1 = FieldValue.withUInt32OrNull(123); - Assert.assertTrue(f1.getUInt32OrNull().isPresent()); - - final FieldValue f2 = FieldValue.withUInt32OrNull(null); - Assert.assertTrue(f2.isNull()); - Assert.assertFalse(f2.getUInt32OrNull().isPresent()); - } - - @Test - public void nullableUInt16Test() { - final FieldValue f1 = FieldValue.withUInt16OrNull(123); - Assert.assertTrue(f1.getUInt16OrNull().isPresent()); - - final FieldValue f2 = FieldValue.withUInt16OrNull(null); - Assert.assertTrue(f2.isNull()); - Assert.assertFalse(f2.getUInt16OrNull().isPresent()); - } - - @Test - public void nullableUInt8Test() { - final FieldValue f1 = FieldValue.withUInt8OrNull(123); - Assert.assertTrue(f1.getUInt8OrNull().isPresent()); - - final FieldValue f2 = FieldValue.withUInt8OrNull(null); - Assert.assertTrue(f2.isNull()); - Assert.assertFalse(f2.getUInt8OrNull().isPresent()); - } - - @Test - public void nullableTimestampTest() { - final FieldValue f1 = FieldValue.withTimestampOrNull(123L); - Assert.assertTrue(f1.getTimestampOrNull().isPresent()); - - final FieldValue f2 = FieldValue.withTimestampOrNull(null); - Assert.assertTrue(f2.isNull()); - Assert.assertFalse(f2.getTimestampOrNull().isPresent()); - } - - @Test - public void nullableVarbinaryTest() { - final FieldValue f1 = FieldValue.withVarbinaryOrNull(new byte[1]); - Assert.assertTrue(f1.getVarbinaryOrNull().isPresent()); - - final FieldValue f2 = FieldValue.withVarbinaryOrNull(null); - Assert.assertTrue(f2.isNull()); - Assert.assertFalse(f2.getVarbinaryOrNull().isPresent()); - } -} diff --git a/ceresdb-protocol/src/test/java/io/ceresdb/models/PointTest.java b/ceresdb-protocol/src/test/java/io/ceresdb/models/PointTest.java new file mode 100644 index 0000000..0ce23c7 --- /dev/null +++ b/ceresdb-protocol/src/test/java/io/ceresdb/models/PointTest.java @@ -0,0 +1,39 @@ +/* + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + */ +package io.ceresdb.models; + +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +import io.ceresdb.common.util.Clock; + +public class PointTest { + @Test + public void PointBuilderTest() { + Point point = Point.newPointBuilder("test_table").setTimestamp(Clock.defaultClock().getTick()) + .addTag("tag1", "t1").addTag("tag2", "t2").addField("f1", Value.withUInt8(123)).build(); + + Assert.assertEquals("t1", point.getTags().get("tag1").getString()); + Assert.assertEquals("t2", point.getTags().get("tag2").getString()); + Assert.assertEquals(123, point.getFields().get("f1").getUInt8()); + } + + @Test(expected = IllegalArgumentException.class) + public void keywordInTagsTest() { + Point.newPointBuilder("test_table").setTimestamp(Clock.defaultClock().getTick()) + .addTag("timestamp", Value.withString("ts")) // + .addTag("tag2", Value.withString("v")) // + .addField("test", Value.withFloat(0.1f)).build(); + } + + @Test(expected = IllegalArgumentException.class) + public void keywordInFieldsTest() { + Point.newPointBuilder("test_table").setTimestamp(Clock.defaultClock().getTick()) + .addTag("tag1", Value.withString("t1")) // + .addTag("tag2", Value.withString("t2")) // + .addField("tsid", Value.withFloat(0.1f)).build(); + } +} diff --git a/ceresdb-protocol/src/test/java/io/ceresdb/models/ResultTest.java b/ceresdb-protocol/src/test/java/io/ceresdb/models/ResultTest.java index 8d36f8a..21de2b3 100644 --- a/ceresdb-protocol/src/test/java/io/ceresdb/models/ResultTest.java +++ b/ceresdb-protocol/src/test/java/io/ceresdb/models/ResultTest.java @@ -1,42 +1,35 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.models; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.IntStream; + import org.junit.Assert; import org.junit.Test; -/** - * @author jiachun.fjc - */ public class ResultTest { @Test - public void mapTest() { + public void mapTest() throws Exception { final Result r1 = Result.ok(WriteOk.ok(2, 0, null)); final Result r2 = r1.map(WriteOk::getSuccess); Assert.assertEquals(2, r2.getOk().intValue()); - final Result r3 = Result.ok(QueryOk.ok(null, null, 5, null)); - final Result r4 = r3.map(QueryOk::getRowCount); + final Result r3 = Result.ok(SqlQueryOk.ok(null, 0, mockRows(5))); + final Result r4 = r3.map(SqlQueryOk::getRowCount); Assert.assertEquals(5, r4.getOk().intValue()); - final Result r5 = Result.err(Err.writeErr(400, null, null, null)); - final Result r6 = r5.map(WriteOk::getSuccess); - Assert.assertFalse(r6.isOk()); + final Result r5 = Result.ok(SqlQueryOk.ok(null, 1, null)); + final Result r6 = r5.map(SqlQueryOk::getAffectedRows); + Assert.assertEquals(1, r6.getOk().intValue()); + + final Result r7 = Result.err(Err.writeErr(400, null, null, null)); + final Result r8 = r7.map(WriteOk::getSuccess); + Assert.assertFalse(r8.isOk()); } @Test @@ -121,4 +114,12 @@ public void unwrapOrElseTest() { final WriteOk r4 = r3.unwrapOrElse(err -> WriteOk.emptyOk()); Assert.assertEquals(0, r4.getSuccess()); } + + private List mockRows(final int rowCount) { + List rows = new ArrayList(rowCount); + for (int i = 0; i < rowCount; i++) { + rows.add(new Row()); + } + return rows; + } } diff --git a/ceresdb-protocol/src/test/java/io/ceresdb/models/RowsTest.java b/ceresdb-protocol/src/test/java/io/ceresdb/models/RowsTest.java deleted file mode 100644 index a7daded..0000000 --- a/ceresdb-protocol/src/test/java/io/ceresdb/models/RowsTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.models; - -import java.util.HashMap; - -import org.junit.Test; - -import io.ceresdb.common.util.Clock; - -/** - * - * @author jiachun.fjc - */ -public class RowsTest { - - @Test(expected = IllegalArgumentException.class) - public void keywordInTagsTest() { - Series.newBuilder("test_metric") // - .tag("timestamp", "ts") // - .tag("tag2", "v") // - .toRowsBuilder() // - .field(Clock.defaultClock().getTick(), "test", FieldValue.withFloat(0.1f)).build(); - } - - @Test(expected = IllegalArgumentException.class) - public void keywordInFieldsTest() { - Series.newBuilder("test_metric") // - .tag("tag1", "ts") // - .tag("tag2", "v") // - .toRowsBuilder() // - .field(Clock.defaultClock().getTick(), "tsid", FieldValue.withFloat(0.1f)) // - .build(); - } - - @Test(expected = UnsupportedOperationException.class) - public void unmodifiableTagsTest() { - final Series series = Series.newBuilder("test_metric") // - .tag("tag1", "ts") // - .tag("tag2", "v") // - .build(); - - series.getTags().put("not_allowed", TagValue.withString("test")); - } - - @Test(expected = UnsupportedOperationException.class) - public void unmodifiableFieldsTest() { - final Rows rs = Series.newBuilder("test_metric") // - .tag("tag1", "ts") // - .tag("tag2", "v") // - .toRowsBuilder() // - .field(Clock.defaultClock().getTick(), "test", FieldValue.withFloat(0.1f)) // - .build(); - - rs.getFields().put(Clock.defaultClock().getTick(), new HashMap<>()); - } -} diff --git a/ceresdb-protocol/src/test/java/io/ceresdb/models/TagValueTest.java b/ceresdb-protocol/src/test/java/io/ceresdb/models/ValueTest.java similarity index 51% rename from ceresdb-protocol/src/test/java/io/ceresdb/models/TagValueTest.java rename to ceresdb-protocol/src/test/java/io/ceresdb/models/ValueTest.java index 466e1de..cde0b48 100644 --- a/ceresdb-protocol/src/test/java/io/ceresdb/models/TagValueTest.java +++ b/ceresdb-protocol/src/test/java/io/ceresdb/models/ValueTest.java @@ -1,152 +1,149 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.models; -import java.math.BigInteger; - import org.junit.Assert; import org.junit.Test; -/** - * - * @author jiachun.fjc - */ -public class TagValueTest { +public class ValueTest { @Test public void nullableStringTest() { - final TagValue f1 = TagValue.withStringOrNull("xx"); + final Value f1 = Value.withStringOrNull("xx"); Assert.assertTrue(f1.getStringOrNull().isPresent()); - final TagValue f2 = TagValue.withStringOrNull(null); + final Value f2 = Value.withStringOrNull(null); Assert.assertTrue(f2.isNull()); Assert.assertFalse(f2.getStringOrNull().isPresent()); } + @Test + public void nullableBooleanTest() { + final Value f1 = Value.withBooleanOrNull(true); + Assert.assertTrue(f1.getBooleanOrNull().isPresent()); + + final Value f2 = Value.withBooleanOrNull(null); + Assert.assertTrue(f2.isNull()); + Assert.assertFalse(f2.getBooleanOrNull().isPresent()); + } + + @Test + public void nullableDoubleTest() { + final Value f1 = Value.withDouble(0.1); + Assert.assertTrue(f1.getDoubleOrNull().isPresent()); + + final Value f2 = Value.withDoubleOrNull(null); + Assert.assertTrue(f2.isNull()); + Assert.assertFalse(f2.getDoubleOrNull().isPresent()); + } + + @Test + public void nullableFloatTest() { + final Value f1 = Value.withFloatOrNull(123.01f); + Assert.assertTrue(f1.getFloatOrNull().isPresent()); + + final Value f2 = Value.withFloatOrNull(null); + Assert.assertTrue(f2.isNull()); + Assert.assertFalse(f2.getFloatOrNull().isPresent()); + } + @Test public void nullableInt64Test() { - final TagValue f1 = TagValue.withInt64OrNull(123L); + final Value f1 = Value.withInt64OrNull(123L); Assert.assertTrue(f1.getInt64OrNull().isPresent()); - final TagValue f2 = TagValue.withInt64OrNull(null); + final Value f2 = Value.withInt64OrNull(null); Assert.assertTrue(f2.isNull()); Assert.assertFalse(f2.getInt64OrNull().isPresent()); } @Test public void nullableInt32Test() { - final TagValue f1 = TagValue.withInt32OrNull(123); + final Value f1 = Value.withInt32OrNull(123); Assert.assertTrue(f1.getInt32OrNull().isPresent()); - final TagValue f2 = TagValue.withInt32OrNull(null); + final Value f2 = Value.withInt32OrNull(null); Assert.assertTrue(f2.isNull()); Assert.assertFalse(f2.getInt32OrNull().isPresent()); } @Test public void nullableInt16Test() { - final TagValue f1 = TagValue.withInt16OrNull(123); + final Value f1 = Value.withInt16OrNull(123); Assert.assertTrue(f1.getInt16OrNull().isPresent()); - final TagValue f2 = TagValue.withInt16OrNull(null); + final Value f2 = Value.withInt16OrNull(null); Assert.assertTrue(f2.isNull()); Assert.assertFalse(f2.getInt16OrNull().isPresent()); } @Test public void nullableInt8Test() { - final TagValue f1 = TagValue.withInt8OrNull(123); + final Value f1 = Value.withInt8OrNull(123); Assert.assertTrue(f1.getInt8OrNull().isPresent()); - final TagValue f2 = TagValue.withInt8OrNull(null); + final Value f2 = Value.withInt8OrNull(null); Assert.assertTrue(f2.isNull()); Assert.assertFalse(f2.getInt8OrNull().isPresent()); } - @Test - public void nullableBooleanTest() { - final TagValue f1 = TagValue.withBooleanOrNull(true); - Assert.assertTrue(f1.getBooleanOrNull().isPresent()); - - final TagValue f2 = TagValue.withBooleanOrNull(null); - Assert.assertTrue(f2.isNull()); - Assert.assertFalse(f2.getBooleanOrNull().isPresent()); - } - @Test public void nullableUInt64Test() { - final TagValue f1 = TagValue.withUInt64OrNull(100L); + final Value f1 = Value.withUInt64OrNull(100L); Assert.assertTrue(f1.getUInt64OrNull().isPresent()); - final TagValue f2 = TagValue.withUInt64OrNull((Long) null); + final Value f2 = Value.withUInt64OrNull((Long) null); Assert.assertTrue(f2.isNull()); Assert.assertFalse(f2.getUInt64OrNull().isPresent()); - - final TagValue f3 = TagValue.withUInt64OrNull((BigInteger) null); - Assert.assertTrue(f3.isNull()); - Assert.assertFalse(f3.getUInt64OrNull().isPresent()); } @Test public void nullableUInt32Test() { - final TagValue f1 = TagValue.withUInt32OrNull(123); + final Value f1 = Value.withUInt32OrNull(123); Assert.assertTrue(f1.getUInt32OrNull().isPresent()); - final TagValue f2 = TagValue.withUInt32OrNull(null); + final Value f2 = Value.withUInt32OrNull(null); Assert.assertTrue(f2.isNull()); Assert.assertFalse(f2.getUInt32OrNull().isPresent()); } @Test public void nullableUInt16Test() { - final TagValue f1 = TagValue.withUInt16OrNull(123); + final Value f1 = Value.withUInt16OrNull(123); Assert.assertTrue(f1.getUInt16OrNull().isPresent()); - final TagValue f2 = TagValue.withUInt16OrNull(null); + final Value f2 = Value.withUInt16OrNull(null); Assert.assertTrue(f2.isNull()); Assert.assertFalse(f2.getUInt16OrNull().isPresent()); } @Test public void nullableUInt8Test() { - final TagValue f1 = TagValue.withUInt8OrNull(123); + final Value f1 = Value.withUInt8OrNull(123); Assert.assertTrue(f1.getUInt8OrNull().isPresent()); - final TagValue f2 = TagValue.withUInt8OrNull(null); + final Value f2 = Value.withUInt8OrNull(null); Assert.assertTrue(f2.isNull()); Assert.assertFalse(f2.getUInt8OrNull().isPresent()); } @Test public void nullableTimestampTest() { - final TagValue f1 = TagValue.withTimestampOrNull(123L); + final Value f1 = Value.withTimestampOrNull(123L); Assert.assertTrue(f1.getTimestampOrNull().isPresent()); - final TagValue f2 = TagValue.withTimestampOrNull(null); + final Value f2 = Value.withTimestampOrNull(null); Assert.assertTrue(f2.isNull()); Assert.assertFalse(f2.getTimestampOrNull().isPresent()); } @Test public void nullableVarbinaryTest() { - final TagValue f1 = TagValue.withVarbinaryOrNull(new byte[1]); + final Value f1 = Value.withVarbinaryOrNull(new byte[1]); Assert.assertTrue(f1.getVarbinaryOrNull().isPresent()); - final TagValue f2 = TagValue.withVarbinaryOrNull(null); + final Value f2 = Value.withVarbinaryOrNull(null); Assert.assertTrue(f2.isNull()); Assert.assertFalse(f2.getVarbinaryOrNull().isPresent()); } diff --git a/ceresdb-protocol/src/test/java/io/ceresdb/models/WriteOkTest.java b/ceresdb-protocol/src/test/java/io/ceresdb/models/WriteOkTest.java index eaa255f..990b619 100644 --- a/ceresdb-protocol/src/test/java/io/ceresdb/models/WriteOkTest.java +++ b/ceresdb-protocol/src/test/java/io/ceresdb/models/WriteOkTest.java @@ -1,58 +1,42 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.models; -import java.util.ArrayList; import java.util.Arrays; -import java.util.List; +import java.util.HashSet; +import java.util.Set; import org.junit.Assert; import org.junit.Test; -/** - * @author jiachun.fjc - */ public class WriteOkTest { @Test public void combineTest() { - final List metrics = new ArrayList<>(); - metrics.add("test1"); - final WriteOk writeOk = WriteOk.ok(200, 2, metrics); - writeOk.combine(WriteOk.ok(100, 0, Arrays.asList("test2", "test3"))); + final Set tables = new HashSet<>(); + tables.add("test1"); + final WriteOk writeOk = WriteOk.ok(200, 2, tables); + writeOk.combine(WriteOk.ok(100, 0, new HashSet<>(Arrays.asList("test2", "test3")))); Assert.assertEquals(300, writeOk.getSuccess()); Assert.assertEquals(2, writeOk.getFailed()); - Assert.assertEquals(Arrays.asList("test1", "test2", "test3"), writeOk.getMetrics()); + Assert.assertEquals(new HashSet<>(Arrays.asList("test1", "test2", "test3")), writeOk.getTables()); } @Test public void combineWithNullMetrics() { final WriteOk writeOk = WriteOk.ok(200, 2, null); - writeOk.combine(WriteOk.ok(100, 0, Arrays.asList("test2", "test3"))); + writeOk.combine(WriteOk.ok(100, 0, new HashSet<>(Arrays.asList("test2", "test3")))); Assert.assertEquals(300, writeOk.getSuccess()); Assert.assertEquals(2, writeOk.getFailed()); - Assert.assertEquals(Arrays.asList("test2", "test3"), writeOk.getMetrics()); + Assert.assertEquals(new HashSet<>(Arrays.asList("test2", "test3")), writeOk.getTables()); writeOk.combine(WriteOk.ok(100, 0, null)); Assert.assertEquals(400, writeOk.getSuccess()); Assert.assertEquals(2, writeOk.getFailed()); - Assert.assertEquals(Arrays.asList("test2", "test3"), writeOk.getMetrics()); + Assert.assertEquals(new HashSet<>(Arrays.asList("test2", "test3")), writeOk.getTables()); } } diff --git a/ceresdb-protocol/src/test/java/io/ceresdb/util/TestUtil.java b/ceresdb-protocol/src/test/java/io/ceresdb/util/TestUtil.java index 4c66132..b02d612 100644 --- a/ceresdb-protocol/src/test/java/io/ceresdb/util/TestUtil.java +++ b/ceresdb-protocol/src/test/java/io/ceresdb/util/TestUtil.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.util; @@ -23,37 +10,38 @@ import java.util.Set; import io.ceresdb.common.util.Clock; -import io.ceresdb.models.FieldValue; +import io.ceresdb.models.Point; import io.ceresdb.models.Result; -import io.ceresdb.models.Rows; -import io.ceresdb.models.Series; +import io.ceresdb.models.Value; import io.ceresdb.proto.internal.Common; import io.ceresdb.proto.internal.Storage; -/** - * @author jiachun.fjc - */ public class TestUtil { - public static Rows newRow(final String metric) { + public static List newTablePoints(final String table) { final long time = Clock.defaultClock().getTick() - 1; - return Series.newBuilder(metric) // - .tag("tag1", "tag_v1") // - .tag("tag2", "tag_v2") // - .toRowsBuilder() // - .field(time, "field1", FieldValue.withDouble(0.1)) // - .field(time, "field2", FieldValue.withString("string_value")) // - .field(time + 1, "field1", FieldValue.withDouble(0.2)) // - .field(time + 1, "field2", FieldValue.withString("string_value_2")) // - .build(); + + List data = new ArrayList<>(); + data.add(Point.newPointBuilder(table) // + .setTimestamp(time).addTag("tag1", Value.withString("tag_v1")) // + .addTag("tag2", Value.withString("tag_v2")) // + .addField("field1", Value.withDouble(0.1)) // + .addField("field2", Value.withString("string_value")) // + .build()); + data.add(Point.newPointBuilder(table).setTimestamp(time + 1).addTag("tag1", Value.withString("tag_v1")) // + .addTag("tag2", Value.withString("tag_v2")) // + .addField("field1", Value.withDouble(0.2)) // + .addField("field2", Value.withString("string_value_2")) // + .build()); + return data; } - public static List newListOfRows(final String... metrics) { - final List rowsList = new ArrayList<>(); - for (final String metric : metrics) { - rowsList.add(newRow(metric)); + public static List newMultiTablePoints(final String... tables) { + final List pointsList = new ArrayList<>(); + for (final String table : tables) { + pointsList.addAll(newTablePoints(table)); } - return rowsList; + return pointsList; } public static Storage.WriteResponse newSuccessWriteResp(final int success) { diff --git a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/Context.java b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/Context.java index a570420..c730bbc 100644 --- a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/Context.java +++ b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/Context.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc; @@ -25,7 +12,6 @@ /** * Invoke context. * - * @author jiachun.fjc */ @SuppressWarnings("unchecked") public class Context implements Copiable { diff --git a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/MethodDescriptor.java b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/MethodDescriptor.java index b5ca5c3..e0c2c06 100644 --- a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/MethodDescriptor.java +++ b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/MethodDescriptor.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc; @@ -21,7 +8,6 @@ /** * Description of a remote method. * - * @author jiachun.fjc */ public class MethodDescriptor { private final String name; diff --git a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/Observer.java b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/Observer.java index cbd42dd..f1a26dc 100644 --- a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/Observer.java +++ b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/Observer.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc; @@ -23,7 +10,6 @@ /** * Receives notifications from an observable stream of messages. * - * @author jiachun.fjc */ public interface Observer { diff --git a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/RpcClient.java b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/RpcClient.java index 32ec7bf..22e22d8 100644 --- a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/RpcClient.java +++ b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/RpcClient.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc; @@ -24,7 +11,6 @@ /** * A common RPC client. * - * @author jiachun.fjc */ public interface RpcClient extends Lifecycle, Display { diff --git a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/RpcFactory.java b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/RpcFactory.java index 5a9872c..02a0770 100644 --- a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/RpcFactory.java +++ b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/RpcFactory.java @@ -1,25 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc; /** * CeresDB's RPC service factory. * - * @author jiachun.fjc */ public interface RpcFactory { diff --git a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/RpcFactoryProvider.java b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/RpcFactoryProvider.java index 0b5a24f..ac33e38 100644 --- a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/RpcFactoryProvider.java +++ b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/RpcFactoryProvider.java @@ -1,27 +1,10 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc; import io.ceresdb.common.util.ServiceLoader; -/** - * - * @author jiachun.fjc - */ public class RpcFactoryProvider { /** diff --git a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/RpcOptions.java b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/RpcOptions.java index c4f9474..e32ed2b 100644 --- a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/RpcOptions.java +++ b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/RpcOptions.java @@ -1,31 +1,16 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc; import java.util.concurrent.TimeUnit; import io.ceresdb.common.Copiable; -import io.ceresdb.common.Tenant; import io.ceresdb.common.util.Cpus; /** * RPC client options. * - * @author jiachun.fjc */ public class RpcOptions implements Copiable { @@ -106,8 +91,6 @@ public class RpcOptions implements Copiable { private boolean logOnLimitChange = true; - private Tenant tenant; - public int getDefaultRpcTimeout() { return defaultRpcTimeout; } @@ -236,14 +219,6 @@ public void setLogOnLimitChange(boolean logOnLimitChange) { this.logOnLimitChange = logOnLimitChange; } - public Tenant getTenant() { - return tenant; - } - - public void setTenant(Tenant tenant) { - this.tenant = tenant; - } - @Override public RpcOptions copy() { final RpcOptions opts = new RpcOptions(); @@ -263,9 +238,6 @@ public RpcOptions copy() { opts.smoothing = this.smoothing; opts.blockOnLimit = this.blockOnLimit; opts.logOnLimitChange = this.logOnLimitChange; - if (this.tenant != null) { - opts.tenant = this.tenant.copy(); - } return opts; } @@ -288,7 +260,6 @@ public String toString() { ", smoothing=" + smoothing + // ", blockOnLimit=" + blockOnLimit + // ", logOnLimitChange=" + logOnLimitChange + // - ", tenant=" + tenant + // '}'; } diff --git a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/errors/ConnectFailException.java b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/errors/ConnectFailException.java index a6b9640..432cac0 100644 --- a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/errors/ConnectFailException.java +++ b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/errors/ConnectFailException.java @@ -1,24 +1,8 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc.errors; -/** - * @author jiachun.fjc - */ public class ConnectFailException extends RemotingException { private static final long serialVersionUID = 3129127065579018606L; diff --git a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/errors/InvokeTimeoutException.java b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/errors/InvokeTimeoutException.java index e4391c8..b5cbea0 100644 --- a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/errors/InvokeTimeoutException.java +++ b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/errors/InvokeTimeoutException.java @@ -1,24 +1,8 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc.errors; -/** - * @author jiachun.fjc - */ public class InvokeTimeoutException extends RemotingException { private static final long serialVersionUID = -4710810309766380565L; diff --git a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/errors/OnlyErrorMessage.java b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/errors/OnlyErrorMessage.java index b0f4619..38e9f07 100644 --- a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/errors/OnlyErrorMessage.java +++ b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/errors/OnlyErrorMessage.java @@ -1,25 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc.errors; /** * Takes some additional error message, no stack trace, can be added to an exists Exception. * - * @author jiachun.fjc */ public class OnlyErrorMessage extends Exception { diff --git a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/errors/RemotingException.java b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/errors/RemotingException.java index daede8d..cbb5ff5 100644 --- a/ceresdb-rpc/src/main/java/io/ceresdb/rpc/errors/RemotingException.java +++ b/ceresdb-rpc/src/main/java/io/ceresdb/rpc/errors/RemotingException.java @@ -1,25 +1,11 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.rpc.errors; /** * Exception for default remoting problems. * - * @author jiachun.fjc */ public class RemotingException extends Exception { diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/AbstractCharStream.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/AbstractCharStream.java index e5a7941..97dc4df 100644 --- a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/AbstractCharStream.java +++ b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/AbstractCharStream.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ /* Generated by: ParserGeneratorCC: Do not edit this line. AbstractCharStream.java Version 1.1 */ /* ParserGeneratorCCOptions:SUPPORT_CLASS_VISIBILITY_PUBLIC=true */ @@ -22,7 +9,6 @@ * An implementation of interface CharStream, where the stream is assumed to * contain only ASCII characters (without unicode processing). */ - public abstract class AbstractCharStream implements CharStream { /** Default buffer size if nothing is specified */ public static final int DEFAULT_BUF_SIZE = 4096; diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBMetricParserFactory.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBMetricParserFactory.java deleted file mode 100644 index eac69ac..0000000 --- a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBMetricParserFactory.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.sql; - -import io.ceresdb.MetricParser; -import io.ceresdb.MetricParserFactory; -import io.ceresdb.common.SPI; - -/** - * - * @author jiachun.fjc - */ -@SPI(priority = 1) -public class CeresDBMetricParserFactory implements MetricParserFactory { - - @Override - public MetricParser getParser(final String ql) { - return new CeresDBMetricParser(ql); - } -} diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBMetricParser.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBParser.java similarity index 65% rename from ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBMetricParser.java rename to ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBParser.java index d9a10b8..a507b82 100644 --- a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBMetricParser.java +++ b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBParser.java @@ -1,42 +1,28 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.sql; import java.util.List; -import io.ceresdb.MetricParser; -import io.ceresdb.Utils; +import io.ceresdb.common.parser.SqlParser; +import io.ceresdb.util.Utils; import io.ceresdb.common.util.internal.ThrowUtil; /** - * Parse metric QL. + * Parse SQL. * - * @author jiachun.fjc */ -public class CeresDBMetricParser implements MetricParser { +public class CeresDBParser implements SqlParser { - private final String ql; + private final String sql; private boolean parsed; private StatementType statementType = StatementType.Unknown; private CeresDBSqlStatement stmt; - public CeresDBMetricParser(String ql) { - this.ql = ql; + public CeresDBParser(String sql) { + this.sql = sql; } @Override @@ -47,7 +33,7 @@ public StatementType statementType() { } @Override - public List metricNames() { + public List tableNames() { parse(); return this.stmt.getTables(); @@ -68,7 +54,7 @@ private void parse() { this.parsed = true; try { - final CeresDBSqlStatement stmt = CeresDBSqlParser.parse(this.ql)[0]; + final CeresDBSqlStatement stmt = CeresDBSqlParser.parse(this.sql)[0]; switch (stmt.getStatementType()) { case SELECT: diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBParserFactory.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBParserFactory.java new file mode 100644 index 0000000..b8f9b02 --- /dev/null +++ b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBParserFactory.java @@ -0,0 +1,17 @@ +/* + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + */ +package io.ceresdb.sql; + +import io.ceresdb.common.parser.SqlParser; +import io.ceresdb.common.parser.SqlParserFactory; +import io.ceresdb.common.SPI; + +@SPI(priority = 1) +public class CeresDBParserFactory implements SqlParserFactory { + + @Override + public SqlParser getParser(final String sql) { + return new CeresDBParser(sql); + } +} diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBSqlParser.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBSqlParser.java index 03b8b88..9063a39 100644 --- a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBSqlParser.java +++ b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBSqlParser.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ /* CeresDBSqlParser.java */ /* Generated by: ParserGeneratorCC: Do not edit this line. CeresDBSqlParser.java */ diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBSqlParserConstants.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBSqlParserConstants.java index b01e7ca..9a5ec6d 100644 --- a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBSqlParserConstants.java +++ b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBSqlParserConstants.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ /* Generated by: ParserGeneratorCC: Do not edit this line. CeresDBSqlParserConstants.java */ package io.ceresdb.sql; diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBSqlParserTokenManager.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBSqlParserTokenManager.java index f77454f..f3eaf32 100644 --- a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBSqlParserTokenManager.java +++ b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBSqlParserTokenManager.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ /* CeresDBSqlParserTokenManager.java */ /* Generated by: ParserGeneratorCC: Do not edit this line. CeresDBSqlParserTokenManager.java */ diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBSqlStatement.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBSqlStatement.java index f8979d4..ad60be9 100644 --- a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBSqlStatement.java +++ b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CeresDBSqlStatement.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.sql; diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CharStream.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CharStream.java index 33a8cc0..5bb6575 100644 --- a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CharStream.java +++ b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/CharStream.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ /* Generated by: ParserGeneratorCC: Do not edit this line. CharStream.java Version 1.1 */ /* ParserGeneratorCCOptions:SUPPORT_CLASS_VISIBILITY_PUBLIC=true */ diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/LanguageType.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/LanguageType.java index 1a79bf0..d3e9bd0 100644 --- a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/LanguageType.java +++ b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/LanguageType.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.sql; diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/OperationType.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/OperationType.java index a0ded5c..c9ebc46 100644 --- a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/OperationType.java +++ b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/OperationType.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.sql; diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/ParseException.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/ParseException.java index 15afd6b..2104c64 100644 --- a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/ParseException.java +++ b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/ParseException.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ /* Generated by: ParserGeneratorCC: Do not edit this line. ParseException.java Version 1.1 */ /* ParserGeneratorCCOptions:KEEP_LINE_COLUMN=true */ diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/ParseHandler.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/ParseHandler.java index 75edf39..f315b0d 100644 --- a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/ParseHandler.java +++ b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/ParseHandler.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.sql; diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/SimpleCharStream.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/SimpleCharStream.java index 31dc66d..af53e95 100644 --- a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/SimpleCharStream.java +++ b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/SimpleCharStream.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ /* Generated by: ParserGeneratorCC: Do not edit this line. SimpleCharStream.java Version 1.1 */ /* ParserGeneratorCCOptions:SUPPORT_CLASS_VISIBILITY_PUBLIC=true */ diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/SqlUtils.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/SqlUtils.java index ed60b44..3ab004b 100644 --- a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/SqlUtils.java +++ b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/SqlUtils.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.sql; diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/StatementType.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/StatementType.java index 67fba5b..a0606bd 100644 --- a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/StatementType.java +++ b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/StatementType.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.sql; diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/Token.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/Token.java index 8599b53..67b33c0 100644 --- a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/Token.java +++ b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/Token.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ /* Generated by: ParserGeneratorCC: Do not edit this line. Token.java Version 1.1 */ /* ParserGeneratorCCOptions:TOKEN_EXTENDS=,KEEP_LINE_COLUMN=true,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */ diff --git a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/TokenMgrException.java b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/TokenMgrException.java index a52b373..d509c77 100644 --- a/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/TokenMgrException.java +++ b/ceresdb-sql-javacc/src/main/java/io/ceresdb/sql/TokenMgrException.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ /* Generated by: ParserGeneratorCC: Do not edit this line. TokenMgrException.java Version 1.1 */ /* ParserGeneratorCCOptions: */ diff --git a/ceresdb-sql-javacc/src/main/resources/META-INF/services/io.ceresdb.MetricParserFactory b/ceresdb-sql-javacc/src/main/resources/META-INF/services/io.ceresdb.MetricParserFactory deleted file mode 100644 index 6415787..0000000 --- a/ceresdb-sql-javacc/src/main/resources/META-INF/services/io.ceresdb.MetricParserFactory +++ /dev/null @@ -1 +0,0 @@ -io.ceresdb.sql.CeresDBMetricParserFactory diff --git a/ceresdb-sql-javacc/src/main/resources/META-INF/services/io.ceresdb.common.parser.SqlParserFactory b/ceresdb-sql-javacc/src/main/resources/META-INF/services/io.ceresdb.common.parser.SqlParserFactory new file mode 100644 index 0000000..052f115 --- /dev/null +++ b/ceresdb-sql-javacc/src/main/resources/META-INF/services/io.ceresdb.common.parser.SqlParserFactory @@ -0,0 +1 @@ +io.ceresdb.sql.CeresDBParserFactory diff --git a/ceresdb-sql-javacc/src/test/java/io/ceresdb/sql/MetricParserTest.java b/ceresdb-sql-javacc/src/test/java/io/ceresdb/sql/MetricParserTest.java deleted file mode 100644 index e8efdf5..0000000 --- a/ceresdb-sql-javacc/src/test/java/io/ceresdb/sql/MetricParserTest.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.sql; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import org.junit.Assert; -import org.junit.Test; - -import io.ceresdb.MetricParser; -import io.ceresdb.MetricParserFactory; -import io.ceresdb.MetricParserFactoryProvider; -import io.ceresdb.common.util.ServiceLoader; - -/** - * @author jiachun.fjc - */ -public class MetricParserTest { - - @Test - public void loadFromSPITest() { - final MetricParserFactory factory = ServiceLoader.load(MetricParserFactory.class) // - .firstOrDefault(() -> MetricParserFactory.DEFAULT); - Assert.assertTrue(factory instanceof CeresDBMetricParserFactory); - } - - @Test - public void alterTableTest() { - final MetricParser parser = getParser("ALTER TABLE test_table_1635254941778 ADD COLUMN c20 UINT64"); - Assert.assertEquals(MetricParser.StatementType.Alter, parser.statementType()); - Assert.assertEquals(Collections.singletonList("test_table_1635254941778"), parser.metricNames()); - } - - @Test - public void descTableTest() { - MetricParser parser = getParser("DESC TABLE test_table_1635254941779"); - Assert.assertEquals(MetricParser.StatementType.Describe, parser.statementType()); - Assert.assertEquals(Collections.singletonList("test_table_1635254941779"), parser.metricNames()); - - parser = getParser("DESC test_table_1635254941779"); - Assert.assertEquals(MetricParser.StatementType.Describe, parser.statementType()); - Assert.assertEquals(Collections.singletonList("test_table_1635254941779"), parser.metricNames()); - } - - @Test - public void describeTableTest() { - MetricParser parser = getParser("DESCRIBE TABLE test_table_1635254941779"); - Assert.assertEquals(MetricParser.StatementType.Describe, parser.statementType()); - Assert.assertEquals(Collections.singletonList("test_table_1635254941779"), parser.metricNames()); - - parser = getParser("DESCRIBE test_table_1635254941779"); - Assert.assertEquals(MetricParser.StatementType.Describe, parser.statementType()); - Assert.assertEquals(Collections.singletonList("test_table_1635254941779"), parser.metricNames()); - } - - @Test - public void extractMetricNamesTest() { - final MetricParser parser = getParser("select func1(table1.a), func2(table2.b), * from table1, table2, table3 " - + "where table1.id=table2.id"); - final List metricNames = parser.metricNames(); - Assert.assertEquals(MetricParser.StatementType.Select, parser.statementType()); - Assert.assertEquals(Arrays.asList("table1", "table2", "table3"), metricNames); - } - - @Test - public void extractMetricNames2Test() { - final MetricParser parser = getParser("select func1(table1.a), func2(table2.b), * from table1 inner join " - + "table2, table3 on table1.id=table2.id and table2.id = table3.id"); - final List metricNames = parser.metricNames(); - Assert.assertEquals(MetricParser.StatementType.Select, parser.statementType()); - Assert.assertEquals(Arrays.asList("table1", "table2", "table3"), metricNames); - } - - @Test - public void extractMetricNames3Test() { - final MetricParser parser = getParser("select func1(table1.a), func2(table2.b), * from table1 join " - + "table2, table3 on table1.id=table2.id and table2.id = table3.id"); - final List metricNames = parser.metricNames(); - Assert.assertEquals(MetricParser.StatementType.Select, parser.statementType()); - Assert.assertEquals(Arrays.asList("table1", "table2", "table3"), metricNames); - } - - @Test - public void extractMetricNames4Test() { - final MetricParser parser = getParser("select func1(table1.a), func2(table2.b), * from table1 left join " - + "table2, table3 on table1.id=table2.id and table2.id = table3.id"); - final List metricNames = parser.metricNames(); - Assert.assertEquals(MetricParser.StatementType.Select, parser.statementType()); - Assert.assertEquals(Arrays.asList("table1", "table2", "table3"), metricNames); - } - - @Test - public void createTableTest() { - final MetricParser parser = getParser("CREATE TABLE with_primary_key(\n" // - + " ts TIMESTAMP NOT NULL,\n" // - + " c1 STRING NOT NULL,\n" // - + " c2 STRING NULL,\n" // - + " c3 DOUBLE NULL,\n" // - + " c4 STRING NULL,\n" // - + " c5 STRING NULL,\n" // - + " TIMESTAMP KEY(ts),\n" // - + " PRIMARY KEY(c1, ts)\n" // - + ")" // - + " ENGINE=Analytic WITH (ttl='7d', update_mode='APPEND');"); - - Assert.assertEquals(MetricParser.StatementType.Create, parser.statementType()); - Assert.assertEquals(Collections.singletonList("with_primary_key"), parser.metricNames()); - } - - @Test - public void createTableIfNotExistsTest() { - final MetricParser parser = getParser( - "CREATE TABLE IF NOT EXISTS with_primary_key(\n" + " ts TIMESTAMP NOT NULL,\n" // - + " c1 STRING NOT NULL,\n" // - + " c2 STRING NULL,\n" // - + " c3 DOUBLE NULL,\n" // - + " c4 STRING NULL,\n" // - + " c5 STRING NULL,\n" // - + " TIMESTAMP KEY(ts),\n" // - + " PRIMARY KEY(c1, ts)\n" + ")" // - + " ENGINE=Analytic WITH (ttl='7d', update_mode='APPEND');"); - - Assert.assertEquals(MetricParser.StatementType.Create, parser.statementType()); - Assert.assertEquals(Collections.singletonList("with_primary_key"), parser.metricNames()); - } - - @Test - public void dropTableTest() { - final MetricParser parser = getParser("DROP TABLE test_table"); - Assert.assertEquals(MetricParser.StatementType.Drop, parser.statementType()); - Assert.assertEquals(Collections.singletonList("test_table"), parser.metricNames()); - } - - @Test - public void dropTableIfExistsTest() { - final MetricParser parser = getParser("DROP TABLE IF EXISTS test_table"); - Assert.assertEquals(MetricParser.StatementType.Drop, parser.statementType()); - Assert.assertEquals(Collections.singletonList("test_table"), parser.metricNames()); - } - - @Test - public void showTest() { - final MetricParser parser = getParser("SHOW xxx_table"); - Assert.assertEquals(MetricParser.StatementType.Show, parser.statementType()); - Assert.assertEquals(Collections.singletonList("xxx_table"), parser.metricNames()); - } - - @Test - public void showCreateTableTest() { - final MetricParser parser = getParser("SHOW CREATE TABLE xxx_table"); - Assert.assertEquals(MetricParser.StatementType.Show, parser.statementType()); - Assert.assertEquals(Collections.singletonList("xxx_table"), parser.metricNames()); - } - - @Test - public void insertTest() { - final MetricParser parser = getParser("INSERT INTO insert_select_test_table (*) VALUES (1, 'a', 1)"); - Assert.assertEquals(MetricParser.StatementType.Insert, parser.statementType()); - Assert.assertEquals(Collections.singletonList("insert_select_test_table"), parser.metricNames()); - } - - @Test - public void existsTest() { - MetricParser parser = getParser("EXISTS TABLE xxx_table"); - Assert.assertEquals(MetricParser.StatementType.Exists, parser.statementType()); - Assert.assertEquals(Collections.singletonList("xxx_table"), parser.metricNames()); - - parser = getParser("EXISTS xxx_table"); - Assert.assertEquals(MetricParser.StatementType.Exists, parser.statementType()); - Assert.assertEquals(Collections.singletonList("xxx_table"), parser.metricNames()); - } - - private static MetricParser getParser(final String sql) { - return MetricParserFactoryProvider.getMetricParserFactory().getParser(sql); - } -} diff --git a/ceresdb-sql-javacc/src/test/java/io/ceresdb/sql/TableParserTest.java b/ceresdb-sql-javacc/src/test/java/io/ceresdb/sql/TableParserTest.java new file mode 100644 index 0000000..7959dd9 --- /dev/null +++ b/ceresdb-sql-javacc/src/test/java/io/ceresdb/sql/TableParserTest.java @@ -0,0 +1,176 @@ +/* + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + */ +package io.ceresdb.sql; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +import io.ceresdb.common.parser.SqlParser; +import io.ceresdb.common.parser.SqlParserFactory; +import io.ceresdb.common.parser.SqlParserFactoryProvider; +import io.ceresdb.common.util.ServiceLoader; + +public class TableParserTest { + + @Test + public void loadFromSPITest() { + final SqlParserFactory factory = ServiceLoader.load(SqlParserFactory.class) // + .firstOrDefault(() -> SqlParserFactory.DEFAULT); + Assert.assertTrue(factory instanceof CeresDBParserFactory); + } + + @Test + public void alterTableTest() { + final SqlParser parser = getParser("ALTER TABLE test_table_1635254941778 ADD COLUMN c20 UINT64"); + Assert.assertEquals(SqlParser.StatementType.Alter, parser.statementType()); + Assert.assertEquals(Collections.singletonList("test_table_1635254941778"), parser.tableNames()); + } + + @Test + public void descTableTest() { + SqlParser parser = getParser("DESC TABLE test_table_1635254941779"); + Assert.assertEquals(SqlParser.StatementType.Describe, parser.statementType()); + Assert.assertEquals(Collections.singletonList("test_table_1635254941779"), parser.tableNames()); + + parser = getParser("DESC test_table_1635254941779"); + Assert.assertEquals(SqlParser.StatementType.Describe, parser.statementType()); + Assert.assertEquals(Collections.singletonList("test_table_1635254941779"), parser.tableNames()); + } + + @Test + public void describeTableTest() { + SqlParser parser = getParser("DESCRIBE TABLE test_table_1635254941779"); + Assert.assertEquals(SqlParser.StatementType.Describe, parser.statementType()); + Assert.assertEquals(Collections.singletonList("test_table_1635254941779"), parser.tableNames()); + + parser = getParser("DESCRIBE test_table_1635254941779"); + Assert.assertEquals(SqlParser.StatementType.Describe, parser.statementType()); + Assert.assertEquals(Collections.singletonList("test_table_1635254941779"), parser.tableNames()); + } + + @Test + public void extractMetricNamesTest() { + final SqlParser parser = getParser("select func1(table1.a), func2(table2.b), * from table1, table2, table3 " + + "where table1.id=table2.id"); + final List metricNames = parser.tableNames(); + Assert.assertEquals(SqlParser.StatementType.Select, parser.statementType()); + Assert.assertEquals(Arrays.asList("table1", "table2", "table3"), metricNames); + } + + @Test + public void extractMetricNames2Test() { + final SqlParser parser = getParser("select func1(table1.a), func2(table2.b), * from table1 inner join " + + "table2, table3 on table1.id=table2.id and table2.id = table3.id"); + final List metricNames = parser.tableNames(); + Assert.assertEquals(SqlParser.StatementType.Select, parser.statementType()); + Assert.assertEquals(Arrays.asList("table1", "table2", "table3"), metricNames); + } + + @Test + public void extractMetricNames3Test() { + final SqlParser parser = getParser("select func1(table1.a), func2(table2.b), * from table1 join " + + "table2, table3 on table1.id=table2.id and table2.id = table3.id"); + final List metricNames = parser.tableNames(); + Assert.assertEquals(SqlParser.StatementType.Select, parser.statementType()); + Assert.assertEquals(Arrays.asList("table1", "table2", "table3"), metricNames); + } + + @Test + public void extractMetricNames4Test() { + final SqlParser parser = getParser("select func1(table1.a), func2(table2.b), * from table1 left join " + + "table2, table3 on table1.id=table2.id and table2.id = table3.id"); + final List metricNames = parser.tableNames(); + Assert.assertEquals(SqlParser.StatementType.Select, parser.statementType()); + Assert.assertEquals(Arrays.asList("table1", "table2", "table3"), metricNames); + } + + @Test + public void createTableTest() { + final SqlParser parser = getParser("CREATE TABLE with_primary_key(\n" // + + " ts TIMESTAMP NOT NULL,\n" // + + " c1 STRING NOT NULL,\n" // + + " c2 STRING NULL,\n" // + + " c3 DOUBLE NULL,\n" // + + " c4 STRING NULL,\n" // + + " c5 STRING NULL,\n" // + + " TIMESTAMP KEY(ts),\n" // + + " PRIMARY KEY(c1, ts)\n" // + + ")" // + + " ENGINE=Analytic WITH (ttl='7d', update_mode='APPEND');"); + + Assert.assertEquals(SqlParser.StatementType.Create, parser.statementType()); + Assert.assertEquals(Collections.singletonList("with_primary_key"), parser.tableNames()); + } + + @Test + public void createTableIfNotExistsTest() { + final SqlParser parser = getParser( + "CREATE TABLE IF NOT EXISTS with_primary_key(\n" + " ts TIMESTAMP NOT NULL,\n" // + + " c1 STRING NOT NULL,\n" // + + " c2 STRING NULL,\n" // + + " c3 DOUBLE NULL,\n" // + + " c4 STRING NULL,\n" // + + " c5 STRING NULL,\n" // + + " TIMESTAMP KEY(ts),\n" // + + " PRIMARY KEY(c1, ts)\n" + ")" // + + " ENGINE=Analytic WITH (ttl='7d', update_mode='APPEND');"); + + Assert.assertEquals(SqlParser.StatementType.Create, parser.statementType()); + Assert.assertEquals(Collections.singletonList("with_primary_key"), parser.tableNames()); + } + + @Test + public void dropTableTest() { + final SqlParser parser = getParser("DROP TABLE test_table"); + Assert.assertEquals(SqlParser.StatementType.Drop, parser.statementType()); + Assert.assertEquals(Collections.singletonList("test_table"), parser.tableNames()); + } + + @Test + public void dropTableIfExistsTest() { + final SqlParser parser = getParser("DROP TABLE IF EXISTS test_table"); + Assert.assertEquals(SqlParser.StatementType.Drop, parser.statementType()); + Assert.assertEquals(Collections.singletonList("test_table"), parser.tableNames()); + } + + @Test + public void showTest() { + final SqlParser parser = getParser("SHOW xxx_table"); + Assert.assertEquals(SqlParser.StatementType.Show, parser.statementType()); + Assert.assertEquals(Collections.singletonList("xxx_table"), parser.tableNames()); + } + + @Test + public void showCreateTableTest() { + final SqlParser parser = getParser("SHOW CREATE TABLE xxx_table"); + Assert.assertEquals(SqlParser.StatementType.Show, parser.statementType()); + Assert.assertEquals(Collections.singletonList("xxx_table"), parser.tableNames()); + } + + @Test + public void insertTest() { + final SqlParser parser = getParser("INSERT INTO insert_select_test_table (*) VALUES (1, 'a', 1)"); + Assert.assertEquals(SqlParser.StatementType.Insert, parser.statementType()); + Assert.assertEquals(Collections.singletonList("insert_select_test_table"), parser.tableNames()); + } + + @Test + public void existsTest() { + SqlParser parser = getParser("EXISTS TABLE xxx_table"); + Assert.assertEquals(SqlParser.StatementType.Exists, parser.statementType()); + Assert.assertEquals(Collections.singletonList("xxx_table"), parser.tableNames()); + + parser = getParser("EXISTS xxx_table"); + Assert.assertEquals(SqlParser.StatementType.Exists, parser.statementType()); + Assert.assertEquals(Collections.singletonList("xxx_table"), parser.tableNames()); + } + + private static SqlParser getParser(final String sql) { + return SqlParserFactoryProvider.getSqlParserFactory().getParser(sql); + } +} diff --git a/ceresdb-sql/src/main/java/io/ceresdb/sql/JSqlMetricParserFactory.java b/ceresdb-sql/src/main/java/io/ceresdb/sql/JSqlMetricParserFactory.java deleted file mode 100644 index 07c7d68..0000000 --- a/ceresdb-sql/src/main/java/io/ceresdb/sql/JSqlMetricParserFactory.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.ceresdb.sql; - -import io.ceresdb.MetricParser; -import io.ceresdb.MetricParserFactory; -import io.ceresdb.common.SPI; - -/** - * - * @author jiachun.fjc - */ -@SPI -public class JSqlMetricParserFactory implements MetricParserFactory { - - @Override - public MetricParser getParser(final String ql) { - return new JSqlMetricParser(ql); - } -} diff --git a/ceresdb-sql/src/main/java/io/ceresdb/sql/JSqlMetricParser.java b/ceresdb-sql/src/main/java/io/ceresdb/sql/JSqlParser.java similarity index 74% rename from ceresdb-sql/src/main/java/io/ceresdb/sql/JSqlMetricParser.java rename to ceresdb-sql/src/main/java/io/ceresdb/sql/JSqlParser.java index 5573b88..953e703 100644 --- a/ceresdb-sql/src/main/java/io/ceresdb/sql/JSqlMetricParser.java +++ b/ceresdb-sql/src/main/java/io/ceresdb/sql/JSqlParser.java @@ -1,25 +1,12 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.sql; import java.util.Collections; import java.util.List; -import io.ceresdb.MetricParser; +import io.ceresdb.common.parser.SqlParser; import io.ceresdb.common.util.internal.ThrowUtil; import net.sf.jsqlparser.JSQLParserException; @@ -34,24 +21,23 @@ import net.sf.jsqlparser.statement.select.Select; /** - * Parse metric QL use JSqlParser. + * Parse SQL use JSqlParser. * - * @author jiachun.fjc */ -public class JSqlMetricParser implements MetricParser { +public class JSqlParser implements SqlParser { - private final String ql; + private final String sql; private boolean parsed; private StatementType statementType = StatementType.Unknown; private Statement stmt; - public JSqlMetricParser(String ql) { - this.ql = ql; + public JSqlParser(String sql) { + this.sql = sql; } - public String getQl() { - return ql; + public String getSql() { + return sql; } @Override @@ -62,7 +48,7 @@ public StatementType statementType() { } @Override - public List metricNames() { + public List tableNames() { parse(); switch (this.statementType) { @@ -103,7 +89,7 @@ private void parse() { this.parsed = true; try { - final Statement stmt = CCJSqlParserUtil.parse(SqlParserUtil.amendSql(this.ql)); + final Statement stmt = CCJSqlParserUtil.parse(SqlParserUtil.amendSql(this.sql)); if (stmt instanceof Select) { this.statementType = StatementType.Select; diff --git a/ceresdb-sql/src/main/java/io/ceresdb/sql/JSqlParserFactory.java b/ceresdb-sql/src/main/java/io/ceresdb/sql/JSqlParserFactory.java new file mode 100644 index 0000000..7d014e9 --- /dev/null +++ b/ceresdb-sql/src/main/java/io/ceresdb/sql/JSqlParserFactory.java @@ -0,0 +1,17 @@ +/* + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. + */ +package io.ceresdb.sql; + +import io.ceresdb.common.parser.SqlParser; +import io.ceresdb.common.parser.SqlParserFactory; +import io.ceresdb.common.SPI; + +@SPI +public class JSqlParserFactory implements SqlParserFactory { + + @Override + public SqlParser getParser(final String sql) { + return new JSqlParser(sql); + } +} diff --git a/ceresdb-sql/src/main/java/io/ceresdb/sql/SqlParserUtil.java b/ceresdb-sql/src/main/java/io/ceresdb/sql/SqlParserUtil.java index 0c3d93c..6130f03 100644 --- a/ceresdb-sql/src/main/java/io/ceresdb/sql/SqlParserUtil.java +++ b/ceresdb-sql/src/main/java/io/ceresdb/sql/SqlParserUtil.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.sql; @@ -20,7 +7,7 @@ import java.util.Set; import java.util.stream.Collectors; -import io.ceresdb.MetricParser; +import io.ceresdb.common.parser.SqlParser; import io.ceresdb.common.util.Requires; import net.sf.jsqlparser.statement.Statement; @@ -29,10 +16,6 @@ import net.sf.jsqlparser.statement.create.table.Index; import net.sf.jsqlparser.util.TablesNamesFinder; -/** - * - * @author jiachun.fjc - */ public class SqlParserUtil { private static final String _TIMESTAMP_KEY_UC = "TIMESTAMP KEY"; @@ -61,7 +44,7 @@ public static String amendSql(final String sql) { return sql; } - public static List extractCreateColumns(final CreateTable createTable) { + public static List extractCreateColumns(final CreateTable createTable) { final String metricName = createTable.getTable().getName(); // timestamp @@ -72,7 +55,7 @@ public static List extractCreateColumns(final CreateTable c final Set tags = createTable.getColumnDefinitions().stream().filter(SqlParserUtil::isTagColumn) .map(ColumnDefinition::getColumnName).collect(Collectors.toSet()); - return createTable.getColumnDefinitions().stream().map(col -> new MetricParser.Column() { + return createTable.getColumnDefinitions().stream().map(col -> new SqlParser.Column() { @Override public String metricName() { diff --git a/ceresdb-sql/src/main/resources/META-INF/services/io.ceresdb.MetricParserFactory b/ceresdb-sql/src/main/resources/META-INF/services/io.ceresdb.MetricParserFactory deleted file mode 100644 index aedfb1f..0000000 --- a/ceresdb-sql/src/main/resources/META-INF/services/io.ceresdb.MetricParserFactory +++ /dev/null @@ -1 +0,0 @@ -io.ceresdb.sql.JSqlMetricParserFactory diff --git a/ceresdb-sql/src/main/resources/META-INF/services/io.ceresdb.common.parser.SqlParserFactory b/ceresdb-sql/src/main/resources/META-INF/services/io.ceresdb.common.parser.SqlParserFactory new file mode 100644 index 0000000..55cd5a5 --- /dev/null +++ b/ceresdb-sql/src/main/resources/META-INF/services/io.ceresdb.common.parser.SqlParserFactory @@ -0,0 +1 @@ +io.ceresdb.sql.JSqlParserFactory diff --git a/ceresdb-sql/src/test/java/io/ceresdb/sql/JSqlMetricParserTest.java b/ceresdb-sql/src/test/java/io/ceresdb/sql/JSqlMetricParserTest.java index 0c36a6a..1f8a981 100644 --- a/ceresdb-sql/src/test/java/io/ceresdb/sql/JSqlMetricParserTest.java +++ b/ceresdb-sql/src/test/java/io/ceresdb/sql/JSqlMetricParserTest.java @@ -1,18 +1,5 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. */ package io.ceresdb.sql; @@ -24,82 +11,78 @@ import org.junit.Assert; import org.junit.Test; -import io.ceresdb.MetricParser; -import io.ceresdb.MetricParserFactory; -import io.ceresdb.MetricParserFactoryProvider; +import io.ceresdb.common.parser.SqlParser; +import io.ceresdb.common.parser.SqlParserFactory; +import io.ceresdb.common.parser.SqlParserFactoryProvider; import io.ceresdb.common.util.ServiceLoader; -/** - * @author jiachun.fjc - */ public class JSqlMetricParserTest { @Test public void loadFromSPITest() { - final MetricParserFactory factory = ServiceLoader.load(MetricParserFactory.class) // - .firstOrDefault(() -> MetricParserFactory.DEFAULT); - Assert.assertTrue(factory instanceof JSqlMetricParserFactory); + final SqlParserFactory factory = ServiceLoader.load(SqlParserFactory.class) // + .firstOrDefault(() -> SqlParserFactory.DEFAULT); + Assert.assertTrue(factory instanceof JSqlParserFactory); } @Test public void alterTableTest() { - final MetricParser parser = getParser("ALTER TABLE test_table_1635254941778 ADD COLUMN c20 UINT64"); - Assert.assertEquals(MetricParser.StatementType.Alter, parser.statementType()); - Assert.assertEquals(Collections.singletonList("test_table_1635254941778"), parser.metricNames()); + final SqlParser parser = getParser("ALTER TABLE test_table_1635254941778 ADD COLUMN c20 UINT64"); + Assert.assertEquals(SqlParser.StatementType.Alter, parser.statementType()); + Assert.assertEquals(Collections.singletonList("test_table_1635254941778"), parser.tableNames()); } @Test public void descTableTest() { - final MetricParser parser = getParser("DESCRIBE test_table_1635254941778"); - Assert.assertEquals(MetricParser.StatementType.Describe, parser.statementType()); - Assert.assertEquals(Collections.singletonList("test_table_1635254941778"), parser.metricNames()); + final SqlParser parser = getParser("DESCRIBE test_table_1635254941778"); + Assert.assertEquals(SqlParser.StatementType.Describe, parser.statementType()); + Assert.assertEquals(Collections.singletonList("test_table_1635254941778"), parser.tableNames()); } @Test public void extractMetricNamesTest() { - final MetricParser parser = getParser( + final SqlParser parser = getParser( "select func1(table1.a), func2(table2.b), * from table1 inner join " + "table2 on table1.id=table2.id"); - final List metricNames = parser.metricNames(); + final List metricNames = parser.tableNames(); Assert.assertEquals(Arrays.asList("table1", "table2"), metricNames); } @Test public void extractMetricNames2Test() { - final MetricParser parser = getParser("select func1(table1.a), func2(table2.b), * from table1 inner join " - + "table2, table3 on table1.id=table2.id and table2.id = table3.id"); - final List metricNames = parser.metricNames(); + final SqlParser parser = getParser("select func1(table1.a), func2(table2.b), * from table1 inner join " + + "table2, table3 on table1.id=table2.id and table2.id = table3.id"); + final List metricNames = parser.tableNames(); Assert.assertEquals(Arrays.asList("table1", "table2", "table3"), metricNames); } @Test public void extractMetricNames3Test() { - final MetricParser parser = getParser("select func1(table1.a), func2(table2.b), * from table1 join " - + "table2, table3 on table1.id=table2.id and table2.id = table3.id"); - final List metricNames = parser.metricNames(); + final SqlParser parser = getParser("select func1(table1.a), func2(table2.b), * from table1 join " + + "table2, table3 on table1.id=table2.id and table2.id = table3.id"); + final List metricNames = parser.tableNames(); Assert.assertEquals(Arrays.asList("table1", "table2", "table3"), metricNames); } @Test public void extractMetricNames4Test() { - final MetricParser parser = getParser("select func1(table1.a), func2(table2.b), * from table1 left join " - + "table2, table3 on table1.id=table2.id and table2.id = table3.id"); - final List metricNames = parser.metricNames(); + final SqlParser parser = getParser("select func1(table1.a), func2(table2.b), * from table1 left join " + + "table2, table3 on table1.id=table2.id and table2.id = table3.id"); + final List metricNames = parser.tableNames(); Assert.assertEquals(Arrays.asList("table1", "table2", "table3"), metricNames); } @Test public void extractFromCreateTableWithPrimaryKeyTest() { - final MetricParser parser = getParser( - "CREATE TABLE with_primary_key(\n" + " ts TIMESTAMP NOT NULL,\n" + " c1 STRING NOT NULL,\n" - + " c2 STRING NULL,\n" + " c3 DOUBLE NULL,\n" - + " c4 STRING NULL,\n" + " c5 STRING NULL,\n" - + " TIMESTAMP KEY(ts),\n" + " PRIMARY KEY(c1, ts)\n" - + ") ENGINE=Analytic WITH (ttl='7d', update_mode='APPEND');"); - final List metricNames = parser.metricNames(); + final SqlParser parser = getParser("CREATE TABLE with_primary_key(\n" + " ts TIMESTAMP NOT NULL,\n" + + " c1 STRING NOT NULL,\n" + " c2 STRING NULL,\n" + + " c3 DOUBLE NULL,\n" + " c4 STRING NULL,\n" + " c5 STRING NULL,\n" + + " TIMESTAMP KEY(ts),\n" + " PRIMARY KEY(c1, ts)\n" + + ") ENGINE=Analytic WITH (ttl='7d', update_mode='APPEND');"); + final List metricNames = parser.tableNames(); Assert.assertEquals(Collections.singletonList("with_primary_key"), metricNames); @@ -112,13 +95,13 @@ public void extractFromCreateTableWithPrimaryKeyTest() { @Test public void extractFromCreateTableWithPrimaryKeyTagTest() { - final MetricParser parser = getParser("CREATE TABLE with_primary_key_tag(\n" + " ts TIMESTAMP NOT NULL,\n" - + " c1 STRING TAG NOT NULL,\n" + " c2 STRING TAG NULL,\n" - + " c3 STRING TAG NULL,\n" + " c4 DOUBLE NULL,\n" - + " c5 STRING NULL,\n" + " c6 STRING NULL,\n" - + " c7 TIMESTAMP NULL,\n" + " TIMESTAMP KEY(ts),\n" - + " PRIMARY KEY(c1, ts)\n" + ") ENGINE=Analytic;"); - final List metricNames = parser.metricNames(); + final SqlParser parser = getParser("CREATE TABLE with_primary_key_tag(\n" + " ts TIMESTAMP NOT NULL,\n" + + " c1 STRING TAG NOT NULL,\n" + " c2 STRING TAG NULL,\n" + + " c3 STRING TAG NULL,\n" + " c4 DOUBLE NULL,\n" + + " c5 STRING NULL,\n" + " c6 STRING NULL,\n" + + " c7 TIMESTAMP NULL,\n" + " TIMESTAMP KEY(ts),\n" + + " PRIMARY KEY(c1, ts)\n" + ") ENGINE=Analytic;"); + final List metricNames = parser.tableNames(); Assert.assertEquals(Collections.singletonList("with_primary_key_tag"), metricNames); @@ -131,12 +114,12 @@ public void extractFromCreateTableWithPrimaryKeyTagTest() { @Test public void extractFromCreateTableWithTagTest() { - final MetricParser parser = getParser("CREATE TABLE with_tag(\n" + " ts TIMESTAMP NOT NULL,\n" - + " c1 STRING TAG NOT NULL,\n" + " c2 STRING TAG NULL,\n" - + " c3 STRING TAG NULL,\n" + " c4 DOUBLE NULL,\n" - + " c5 STRING NULL,\n" + " c6 STRING NULL,\n" - + " TIMESTAMP KEY(ts)\n" + ") ENGINE=Analytic;"); - final List metricNames = parser.metricNames(); + final SqlParser parser = getParser( + "CREATE TABLE with_tag(\n" + " ts TIMESTAMP NOT NULL,\n" + " c1 STRING TAG NOT NULL,\n" + + " c2 STRING TAG NULL,\n" + " c3 STRING TAG NULL,\n" + + " c4 DOUBLE NULL,\n" + " c5 STRING NULL,\n" + " c6 STRING NULL,\n" + + " TIMESTAMP KEY(ts)\n" + ") ENGINE=Analytic;"); + final List metricNames = parser.tableNames(); Assert.assertEquals(Collections.singletonList("with_tag"), metricNames); @@ -149,27 +132,27 @@ public void extractFromCreateTableWithTagTest() { @Test public void dropTableTest() { - final MetricParser parser = getParser("DROP TABLE xxx_table"); - Assert.assertEquals(MetricParser.StatementType.Drop, parser.statementType()); - Assert.assertEquals(Collections.singletonList("xxx_table"), parser.metricNames()); + final SqlParser parser = getParser("DROP TABLE xxx_table"); + Assert.assertEquals(SqlParser.StatementType.Drop, parser.statementType()); + Assert.assertEquals(Collections.singletonList("xxx_table"), parser.tableNames()); } @Test public void showCreateTableTest() { // https://github.com/JSQLParser/JSqlParser/issues/883 - final MetricParser parser = getParser("SHOW xxx_table"); - Assert.assertEquals(MetricParser.StatementType.Show, parser.statementType()); - Assert.assertEquals(Collections.singletonList("xxx_table"), parser.metricNames()); + final SqlParser parser = getParser("SHOW xxx_table"); + Assert.assertEquals(SqlParser.StatementType.Show, parser.statementType()); + Assert.assertEquals(Collections.singletonList("xxx_table"), parser.tableNames()); } @Test public void insertTest() { - final MetricParser parser = getParser("INSERT INTO insert_select_test_table (c1, c2, c3) VALUES (1, 'a', 1)"); - Assert.assertEquals(MetricParser.StatementType.Insert, parser.statementType()); - Assert.assertEquals(Collections.singletonList("insert_select_test_table"), parser.metricNames()); + final SqlParser parser = getParser("INSERT INTO insert_select_test_table (c1, c2, c3) VALUES (1, 'a', 1)"); + Assert.assertEquals(SqlParser.StatementType.Insert, parser.statementType()); + Assert.assertEquals(Collections.singletonList("insert_select_test_table"), parser.tableNames()); } - private static MetricParser getParser(final String sql) { - return MetricParserFactoryProvider.getMetricParserFactory().getParser(sql); + private static SqlParser getParser(final String sql) { + return SqlParserFactoryProvider.getSqlParserFactory().getParser(sql); } } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f91a34f..6b2d0ad 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -3,272 +3,26 @@ ### Format X.Y.Z -| name | desc | -| --- | --- | -| X | major，Representative made incompatible modifications | +| name | desc | +| --- |------------------------------------------------------------------| +| X | major，Representative made incompatible modifications | | Y | minor，Representative has added downward compatible functionality | -| Z | patch，Representative made a downward compatibility correction | +| Z | patch，Representative made a downward compatibility correction | The label below（modifier）： ### Modifier -| name | desc | -| --- | --- | +| name | desc | +|-------| --- | | alpha | The internal test version is generally only used for internal testing, and there may be many bugs | -| beta | Public beta version | -| RC | Release candidate, release candidate, theoretically no new features will be added, mainly bug fixes | +| beta | Public beta version | +| RC | Release candidate, release candidate, theoretically no new features will be added, mainly bug fixes | | Final | Final version | --- -## 0.0.3.RC2 +## 1.0.0.alpha [2023-02-08] - Features - - TageValue add withBooleanOrNull nullable API + - The [Ceresdb](https://github.com/CeresDB/ceresdb/tree/main) java client version of the stable API allowing for the reading, writing, and managing of data tables. - Fixes - - Breaking Changes - - -## 0.0.3.RC -- Features - - FieldValue&TageValue provide nullable API -- Fixes - - -- Breaking Changes - - - -## 0.0.3.beta2 -- Features - - CeresDBClient builder provide a simplified method for constructing parametric version -- Fixes - - -- Breaking Changes - - - -## 0.0.3.beta1 -- Features - - Keyword check during writing (writing `timestamp` and `tsid` is not allowed) -- Fixes - - Fix Union type missing timestamp LogicType -- Breaking Changes - - - -## 0.0.3.alpha -- Features - - Support writing of unsigned Int64 values (through BigInteger) - - `Record` support read unsigned int64 - - Improve the definition of version number rules - - Optimization part document description -- Fixes - - -- Breaking Changes - - The return value of `Record#getUInt64` is changed from `Long` to `BigInteger` to avoid overflow - -## 0.0.2.Final (2022-01-17) -- Features - - Add statistics of total network request / response bytes (Metric) - - Close grpc limiter based on sigusr2 signal - - Occupied sigusr2 signal can be turned off and turned on by default - - Add GRPC channel ID for troubleshooting -- Fixes - - Remove SIG_TERM、SIGHUP signals，Only SIGUSR2 used to avoid incompatibility with some environments. At the same time, a small file is created to realize various signal processing. Please refer to the user's documentation for details - - Update rpc-limiter， maxLimit change from 4096 to 1024，initialLimit change from 512 to 64 - - Add rpc-limiter unit test - - Specify logger output in CeresDBClient for metrics reporter to avoid the burden of configuring logger conf for users - - Fix Stream API NPE bug -- Breaking Changes - - Add OptKeys，summarize all system properties configuration，unify naming rules，and some keys have changed - - Query#blockingStreamQuery 返回值修改为 Iterator （原为 Iterator>） - -## 0.0.2.RC15 (2022-01-04) -- Features - - Change default limit algorithm of RPC from Vegas to Gradient Concurrency-limits -- Fixes - - SerializingExecutor queue print warn log once when it exceeds a certain size(1024 by default) - - RouteCache GC threads avoid print cache detail -- Breaking Changes - - The parameters related to adaptive current limiting in RpcOptions have been redefined and are no longer compatible with the old version - -## 0.0.2.RC14 (2021-12-28) -- Features - - Add query current limiter. The default concurrency is 8 - - Record available permits metric when write/query - - Default Value of maximum number of in transit write in lines change from 16384 to 8192 - - Vegas limiter initialLimit change from 20 to 40，and add vegasInitialLimit option - - Various write parameters and configurations have been optimized - - remove unnecessary metric collect -- Fixes -- Breaking Changes - - QueryOptions#setRoutedClient 改为 QueryOptions#setRouterClient - - QueryOptions#getRoutedClient 改为 QueryOptions#getRouterClient - - CeresDBOptions.Builder.limitedPolicy 改为 CeresDBOptions.Builder.writeLimitedPolicy - -## 0.0.2.RC13 (2021-12-27) -- Features - - Prints table names when writing failed - - RouterClient cleaner and refresher use shard pool to reduce resource used - - CeresDBClient display to the log regularly (once every 30 minutes) - - Default value of maximum single write batch change from 1024 to 512，and maximum in transit write in lines change from 4096 to 16384 - - RouteClient receive SIGUSR2 signal to route cache items to file - - NamedThreadFactory add Factory ID to identify duplicate Factory Name -- Fixes - - Update Log4j to 2.17.0 to fix security hole -- Breaking Changes - -## 0.0.2.RC12 (2021-12-20) -- Features -- Fixes - - Fix the problem that `Management` loaded based on SPI cannot be created repeatedly, resulting in the inability to support multi tenant -- Breaking Changes - -## 0.0.2.RC11 (2021-12-17) -- Features - - Query result `Record` add Type and LogicalType enum，and add an API to determine whether it is a timestamp type -- Fixes -- Breaking Changes - -## 0.0.2.RC10 (2021-12-15) -- Features - - Ignore null value written -- Fixes -- Breaking Changes - -## 0.0.2.RC9 (2021-12-14) -- Features - - TAG support more type. In addition to the original string support, the following types are added： - ``` - Int64, - Int(Int32), - Int16, - Int8, - Boolean, - Uint64, - Uint32, - Uint16, - Uint8, - Timestamp, - Varbinary - ``` -- Fixes - - Update Log4j to 2.16.0 to fix security hole - - Locking OkHttp to avoid jar dependency conflict -- Breaking Changes - -## 0.0.2.RC8 (2021-12-07) -- Features - - SQL Parser support Insert StatementType - - SQL Parser Support `EXISTS [TABLE] xxx_table` - - In a single request，(tenant, token, childTenant) combination can be covered by change context，which is targeted at the user scenario of a single cluster but multiple tenants -- Fixes - - fix 1.0.0.RC4 reference dependency client.version 1.0.0.RC3 - - fix tenant be ignored when token is null -- Breaking Changes - - Remove SQL Parser of whereColumns - -## 0.0.2.RC6 (2021-12-04) -- Features -- Fixes -- Breaking Changes - - Authentication headers key unified to lowercase. Will be incompatible with the previous version of authentication - -## 0.0.2.RC5 (2021-12-02) -- Features - - Support parse SQL `SHOW CREATE TABLE` AND `DROP TABLE` - - ALTER TABLE ADD COLUMN Support adding -- Fixes -- Breaking Changes - - TagValue#string rename to TagValue#withString. The naming style is consistent with FieldValue - -## 0.0.2.RC4 (2021-11-24) -- Features - - ceresdb-sql-javacc module support inner/left/right join SQL parse，Support to resolve multiple queries tables -- Fixes - - Cancel the reuse of avro record when querying data to avoid data being accidentally overwritten -- Breaking Changes - -## 0.0.2.RC3 (2021-11-10) -- Features - - Add new SQL parser based javacc，It is mainly used for table routing of query and client SQL verification of Management API module -- Fixes - - When the query result is returned, the utf8 type is converted to string -- Breaking Changes - -## 0.0.2.RC2 (2021-11-02) -- Features - - By default, avro schema's name validate is turned off to support `select count (*) from` queries without using aliases - - Management API module checks the SQL validity on the client by default -- Fixes - - Because the syntax of creating tables on the ceresdb server is changed, modify the parser of the client to support the latest syntax -- Breaking Changes - -## 0.0.2.RC1 (2021-11-01) -- Features - - Add Management API (support execute SQL) - - @NotSupport Tag streaming API not supported at present - - Add more types of support. All types currently supported: - ``` - Double(Float64), - String, - Int64, - Float(FLoat32), - Int(Int32), - Int16, - Int8, - Boolean, - Uint64, - Uint32, - Uint16, - Uint8, - Timestamp, - Varbinary - ``` -- Fixes - - Fix: the grpc connection cannot be refreshed when the IP under the domain name is changed -- Breaking Changes - - FieldValue some API changes and are not forward compatible - -## 0.0.1.RC6 (2021-09.28) -- Features -- Fixes - - Modify version number format - - Fix the problem that there is no source code in the mvn repository -- Breaking Changes - -## 0.0.1-RC5 (2021-09-28) -- Features - - The keep alive option of gprc is turned off by default. - - Add time cost metrics for the rpc adaptive limiter is blocked in acquire (rpc_limiter_acquire_time_${method_name}) -- Fixes -- Breaking Changes - - The signal triggering clear-route-cache is changed from SIGINT to SIGHUP，and SIGINT signal is nno longer supported（because SIGINT is used for normal interactive exit of JVM） - -## 0.0.1-RC4 (2021-09-13) -- Features - - An adaptive flow restrictor based on TCP Vegas congestion control algorithm is added in the RPC layer. The traffic rates of write and read requests are 70% and 30% respectively. You can choose to configure fast failure or blocking until timeout. Default blocking - - Add abort-on-blocking-timeout (throw exception after being blocked for timeout) write limit strategy. And set abort-on-blocking-timeout(3s) as default policy（the default policy before is blocking-timeout(1s) ） - - Add abort (throw LimitedException directly) write limit strategy - - Optimization of write limit strategy. When the number of data pieces written at one time exceeds the maximum permission of current limiter，if the in transit request is 0 ant this time, the maximum permission of the current limiter will be consumed and the request will be released, this strategy will avoid the direct failure of such large requests -- Fixes -- Breaking Changes - -## 0.0.1-RC3 (2021-09-06) -- Features - - Add the client instance id and client code version to the call context and send them to the server - - Add write flow policy and provide several default implementations, blocking、discard and blocking-timeout. The default is blocking-timeout(1s) -- Fixes -- Breaking Changes - -## 0.0.1-RC2 (2021-09-03) -- Features - - Add `SerializingExecutor` and `DirectExecutor` statistics of task execution time - - Batch function optimization，add `SerializingExecutor` - - Add write flow limiter，the maximum umber of data lines in transit requests is 4096. User configurable parameter `maxInFlightWriteRows`，less than 0 means no current limitation - - `RpcOptions.rpcThreadPoolSize` （rpc max thread number）to avoid occupying too much resource. The default value is adjusted from 64 seconds to 0 second - - `RpcOptions.keepAliveTimeoutSeconds` （ping timeout）the default value is adjusted from 5 seconds to 3 seconds - - Add CHANGELOG.md -- Fixes -- Breaking Changes - -## 0.0.1-RC1 (2021-08-31) -- Features - - The first version of CeresDB client ，contains basic query and write functions -- Fixes -- Breaking Changes diff --git a/docs/CHANGELOG_CN.md b/docs/CHANGELOG_CN.md index 760e172..59055c4 100644 --- a/docs/CHANGELOG_CN.md +++ b/docs/CHANGELOG_CN.md @@ -3,272 +3,27 @@ ### 版本号格式 X.Y.Z -| name | desc | -| --- | --- | -| X | 主版本号（major），一般代表做了不兼容的修改 | -| Y | 此版本号（minor），一般代表做了向下兼容的功能性新增 | -| Z | 修订版本号（patch），一般代表做了向下兼容的问题修正 | +| name | desc | +|------|------------------------------| +| X | 主版本号（major），一般代表做了不兼容的修改 | +| Y | 此版本号（minor），一般代表做了向下兼容的功能性新增 | +| Z | 修订版本号（patch），一般代表做了向下兼容的问题修正 | 再加上下边的标签（modifier）： ### Modifier -| name | desc | -| --- | --- | +| name | desc | +|-------| --- | | alpha | 内测版本，一般只供内部测试用，bug 可能较多 | -| beta | 公开测试版 | -| RC | Release Candidate，发行候选版本，理论上不再加入新的功能，主要修复 bug | +| beta | 公开测试版 | +| RC | Release Candidate，发行候选版本，理论上不再加入新的功能，主要修复 bug | | Final | 最终版本 | --- -## 0.0.3.RC2 +## 1.0.0.alpha [2023-02-08] - Features - - TageValue 追加 withBooleanOrNull 的 nullable API + - [Ceresdb](https://github.com/CeresDB/ceresdb/tree/main) 稳定API的Java客户端版本，允许读写和管理数据表。 - Fixes - - Breaking Changes - -## 0.0.3.RC -- Features - - FieldValue&TageValue 提供 nullable API -- Fixes - - -- Breaking Changes - - - -## 0.0.3.beta2 -- Features - - CeresDBClient builder 提供简化参数版构造方法 -- Fixes - - -- Breaking Changes - - - -## 0.0.3.beta1 -- Features - - 写入期间关键字检查（不允许写入 `timestamp` 和 `tsid`） -- Fixes - - 修复 Union 类型丢失 timestamp LogicType 的问题 -- Breaking Changes - - - -## 0.0.3.alpha -- Features - - 支持 unsigned int64 值的写入（通过 BigInteger） - - `Record` 支持读取 unsigned int64 值 - - 完善版本号规则定义 - - 优化部分文档说明 -- Fixes - - -- Breaking Changes - - `Record#getUInt64` 返回值由 `Long` 改为 `BigInteger` 以避免溢出 - -## 0.0.2.Final (2022-01-17) -- Features - - 新增网络请求/响应 bytes 总量统计 metric - - 基于 SIGUSR2 信号关闭 grpc limiter - - 占用 SIGUSR2 信号可关，默认打开 - - GRPC channel 增加 ID 用于排查问题 -- Fixes - - 移除 SIG_TERM、SIGHUP 信号占用，仅使用 SIGUSR2 信号，避免和某些环境不兼容，同时利用创建一个小文件来实现各种不同的信号处理，详细请参考使用文档 - - 修改 rpc-limiter， maxLimit 调整为 1024（原 4096），initialLimit 调整为 64（原 512） - - 增加 rpc-limiter 单元测试 - - 为 metrics reporter 指定 CeresDBClient 内的 logger output，避免给用户配置 logger conf 带来负担 - - 修复 Stream API NPE bug -- Breaking Changes - - 增加 OptKeys，汇总所有 system properties 配置，统一命名规则，部分 key 已变化 - - Query#blockingStreamQuery 返回值修改为 Iterator （原为 Iterator>） - -## 0.0.2.RC15 (2022-01-04) -- Features - - 修改 RPC 层默认的限流算法，由 Vegas 改为 Gradient Concurrency-limits -- Fixes - - SerializingExecutor queue 超过一定大小（默认 1024）时打印一次 warn 日志 - - RouteCache GC 线程避免打印 cache 明细 -- Breaking Changes - - RpcOptions 中与自适应限流相关的参数均进行了重新定义，不再兼容旧版本 - -## 0.0.2.RC14 (2021-12-28) -- Features - - 增加查询限流器，默认并发度为 8 - - 每次 write/query 记录 available permits 到 metric - - 最大在途写入行数从默认 16384 调整为 8192 - - Vegas limiter 调整 initialLimit 为 40（原 20），同时增加 vegasInitialLimit 选项 - - 调优了各种写入参数和配置 - - 精简掉不必要的 metric 采集 -- Fixes -- Breaking Changes - - QueryOptions#setRoutedClient 改为 QueryOptions#setRouterClient - - QueryOptions#getRoutedClient 改为 QueryOptions#getRouterClient - - CeresDBOptions.Builder.limitedPolicy 改为 CeresDBOptions.Builder.writeLimitedPolicy - -## 0.0.2.RC13 (2021-12-27) -- Features - - 写入失败时日志打印这一批失败的表名 - - RouterClient 的 cleaner 和 refresher 采用共享 Pool 以减少资源占用 - - CeresDBClient 定时 Display 到日志中（30 分钟执行一次） - - 单次写入最大单批从默认 1024 调整为 512，最大在途写入行数从默认 4096 调整为 16384 - - RouteClient 接收 SIGUSR2 信号将 route cache items 输出到文件 - - NamedThreadFactory 增加 Factory ID 来识别重复的 Factory Name -- Fixes - - 升级 Log4j 到 2.17.0 修复安全漏洞 -- Breaking Changes - -## 0.0.2.RC12 (2021-12-20) -- Features -- Fixes - - 修复基于 SPI 加载的 `Management` 不能重复创建从而导致无法支持多租户的问题 -- Breaking Changes - -## 0.0.2.RC11 (2021-12-17) -- Features - - 查询结果 `Record` 增加 Type 和 LogicalType 类型枚举，增加判断是否是 `timestamp` 类型 API -- Fixes -- Breaking Changes - -## 0.0.2.RC10 (2021-12-15) -- Features - - 忽略写入的 null 值 -- Fixes -- Breaking Changes - -## 0.0.2.RC9 (2021-12-14) -- Features - - TAG 增加更多类型支持，除原有支持 String 外新增如下类型： - ``` - Int64, - Int(Int32), - Int16, - Int8, - Boolean, - Uint64, - Uint32, - Uint16, - Uint8, - Timestamp, - Varbinary - ``` -- Fixes - - 升级 Log4j 到 2.16.0 修复安全漏洞 - - 锁定 OkHttp 依赖项版本避免出现 jar 不兼容 -- Breaking Changes - -## 0.0.2.RC8 (2021-12-07) -- Features - - SQL Parser 支持解析 Insert 的 StatementType - - 支持 `EXISTS [TABLE] xxx_table` 语法 - - 在单次请求中，(tenant, token, childTenant) 组合是可以通过请求 Context 覆盖的，针对于单集群却多个租户的用户场景 -- Fixes - - 修复 1.0.0.RC4 依赖 client.version 一直为 1.0.0.RC3 的 bug - - 修复 token 为空时不传租户的问题 -- Breaking Changes - - 移除 SQL Parser 的 whereColumns 解析，目前看没什么用处 - -## 0.0.2.RC6 (2021-12-04) -- Features -- Fixes -- Breaking Changes - - 鉴权相关 Headers 的 Key 统一改为小写，将和前面版本的鉴权不兼容 - -## 0.0.2.RC5 (2021-12-02) -- Features - - 支持 `SHOW CREATE TABLE` 和 `DROP TABLE` 的 SQL 语句解析 - - ALTER TABLE ADD COLUMN 支持添加多列 -- Fixes -- Breaking Changes - - TagValue#string 重命名为 TagValue#withString，与 FieldValue 风格保持一致 - -## 0.0.2.RC4 (2021-11-24) -- Features - - ceresdb-sql-javacc 模块支持解析 inner/left/right join SQL 解析，支持解析多个查询 tables -- Fixes - - 取消查询数据时 avro record 的复用，避免数据被不小心覆盖 -- Breaking Changes - -## 0.0.2.RC3 (2021-11-10) -- Features - - 新增基于 javacc 的 SQL parser，主要用于查询的表路由以及 management API 模块的客户端 sql 校验 -- Fixes - - 在返回查询结果时将 Utf8 类型转换为 String -- Breaking Changes - -## 0.0.2.RC2 (2021-11-02) -- Features - - 默认关闭 avro schema 的 name validate 以支持 `select count(*) from` 这种不必使用别名查询 - - Management API 模块默认在客户端对 sql 有效性进行检查 -- Fixes - - 因为 CeresDB server 端建表语法变更，修改 client 的 parser 以支持最新语法 -- Breaking Changes - -## 0.0.2.RC1 (2021-11-01) -- Features - - 新增 Management API (支持执行 SQL) - - @NotSupport 标记暂不支持的流式 API - - 新增更多类型支持，目前支持的全部类型： - ``` - Double(Float64), - String, - Int64, - Float(FLoat32), - Int(Int32), - Int16, - Int8, - Boolean, - Uint64, - Uint32, - Uint16, - Uint8, - Timestamp, - Varbinary - ``` -- Fixes - - 修复当域名下的 ip 变更时 gRPC 连接无法刷新 -- Breaking Changes - - FieldValue 部分 API 变更并且不向前兼容 - -## 0.0.1.RC6 (2021-09.28) -- Features -- Fixes - - 修改版本号格式 - - 修复发布到 mvn 仓库没有源码的问题新 -- Breaking Changes - -## 0.0.1-RC5 (2021-09-28) -- Features - - 默认关闭 grpc 的 keep-alive 相关选项 - - 增加 rpc 自适应限流器在 acquire 时被 block 的时间统计指标 (rpc_limiter_acquire_time_${method_name}) -- Fixes -- Breaking Changes - - 触发 clear-route-cache 的信号由 SIGINT 改为 SIGHUP，不再支持 SIGINT 信号（因为 JVM 的正常交互式退出使用的是 SIGINT） - -## 0.0.1-RC4 (2021-09-13) -- Features - - 在 RPC 层增加了基于 TCP Vegas 拥塞控制算法的自适应通限流器，写、读请求的通行率占比分别为 70% 和 30%，可选择配置快速失败或者阻塞直到超时，默认阻塞 - - 增加 abort-on-blocking-timeout (被阻塞超时后抛出异常) 写入限流策略，并设置 abort-on-blocking-timeout(3s) 为默认策略（在这之前的默认策略为 blocking-timeout(1s) ） - - 增加 abort (直接抛出 LimitedException) 写入限流策略 - - 写入限流策略优化，当一次性写入的数据条数就超过限流器最大许可的情况下，如果此时在途请求为 0，那么将消耗掉限流器的最大许可并对这次请求放行，该策略将避免这种大请求直接失败 -- Fixes -- Breaking Changes - -## 0.0.1-RC3 (2021-09-06) -- Features - - 将 client 实例 id 和 client code version 加入到调用上下文并发送到服务端 - - 增加写入限流策略，提供几个默认实现分别为 blocking、discard 和 blocking-timeout，默认为 blocking-timeout(1s) -- Fixes -- Breaking Changes - -## 0.0.1-RC2 (2021-09-03) -- Features - - 增加 `SerializingExecutor` 和 `DirectExecutor` 执行任务耗时统计 - - 攒批执行任务优化，增加 `SerializingExecutor` - - 增加写入限流器，默认限制最大在途请求的数据行数为 4096 行，用户可配置参数 `maxInFlightWriteRows`，小于 0 代表不做限流 - - `RpcOptions.rpcThreadPoolSize` （rpc 最大线程数量）默认值由原来的 64 调整为 0，默认不用线程池避免占用过多资源 - - `RpcOptions.keepAliveTimeoutSeconds` （ping timeout）默认值由原来的 5 秒调整为 3 秒 - - 新增 CHANGELOG.md -- Fixes -- Breaking Changes - -## 0.0.1-RC1 (2021-08-31) -- Features - - CeresDB client 的第一个版本，包含基本查询、写入能力 -- Fixes -- Breaking Changes diff --git a/docs/configuration.md b/docs/configuration.md index 301b583..7190461 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,82 +1,63 @@ ## Global Options (System properties / Java -Dxxx) -| name | description | -| --- | --- | -| "CeresDB.available_cpus" | CeresDB 可用 cpu 数量指定，默认使用当前环境的全部 cpu 数量 | -| "CeresDB.grpc.conn.failures.reset_threshold" | gRPC reset connection 的阈值，默认 3 | -| "CeresDB.client.read.write.rw_logging" | 写查的简洁日志输出开关，默认 true | -| "CeresDB.client.write.collect_wrote_detail" | 写入成功后是否保留写入的 metric 列表，默认为 false | -| "CeresDB.client.use_os_signal" | 是否使用 OS Signal，默认会使用 SIGUSR2 | -| "CeresDB.signal.out_dir" | Signal 接收器输出内容到指定磁盘目录，默认为进程启动目录 | -| "CeresDB.avro.name_validate" | 是否开启 Avro name 校验，默认 false | -| "CeresDB.rpc.write.limit_percent" | RPC 层自适应限流器 `写` 操作占比（默认 0.7），剩下的留给 `读` 操作 | -| "CeresDB.reporter.period_minutes" | Metrics reporter 定时输出周期，默认 30 分钟 | -| "CeresDB.http.read_timeout_ms" | 基于 http 协议的管理类 API 的 http 读超时时间，默认 10000ms | -| "CeresDB.http.write_timeout_ms" | 基于 http 协议的管理类 API 的 http 写超时时间，默认 10000ms | +| name | description | +| --- |----------------------------------------------------------------------------------------------------------------------------| +| "CeresDB.available_cpus" | Specify the number of CPU cores available to the SDK, and use all the number of CPUs in the current environment by default | +| "CeresDB.grpc.conn.failures.reset_threshold" | Threshold gRPC reset connection, default is 3 | +| "CeresDB.client.read.write.rw_logging" | Simple log output switch for read and write, default is true | +| "CeresDB.client.write.collect_wrote_detail" | Whether to keep the written table list after the write is successful, default is false | +| "CeresDB.client.use_os_signal" | Whether to use OS Signal, SIGUSR2 is used by default | +| "CeresDB.signal.out_dir" | The Signal receiver outputs the content to the specified disk directory, which defaults to the process startup directory | +| "CeresDB.rpc.write.limit_percent" | RPC adaptive limit `write` ratio (default 0.7), the rest is reserved for `read` | +| "CeresDB.reporter.period_minutes" | Metrics reporter timing output period，default is 30 minutes | ## CeresDBOptions -| name | description | -| --- | --- | -| clusterAddress | 集群地址（稳定可用的地址），路由表将从这个地址获取 | -| asyncWritePool | 作为一个纯异步的 client，需要一个写入调度线程池，用户可自行配置，默认将使用 SerializingExecutor，如果自行配置了 pool 请重点关注性能指标：async_read_pool.time 的数据，即使调整这个调度线程池的配置 | -| asyncReadPool | 作为一个纯异步的 client，需要一个查询调度线程池，用户可自行配置，默认将使用 SerializingExecutor，如果自行配置了 pool 请重点关注性能指标：async_read_pool.time 的数据，即使调整这个调度线程池的配置 | -| tenant | 租户信息 | -| rpcOptions | RPC 的配置选项，详情请参考 RpcOptions | -| routerOptions | 路由表更新组件的配置选项，详情请参考 RouterOptions | -| writeOptions | 写入相关的的配置选项，详情请参考 WriteOptions | -| managementOptions | 数据管理相关配置选项，详情参考 ManagementOptions | +| name | description | +|----------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| clusterAddress | The address of ceresDB server, which can be a stand-alone address or a cluster address | +| routeMode | The route mote，support `DIRECT` and `PROXY`. `DIRECT` mode is directly connected to the server, and the routing information is cached on the SDK side, which has better performance; `PROXY` mode is forwarded through the PROXY module on the server side, which contains performance loss, but it can solve some case that cannot be directly connected, such as network isolation | +| asyncWritePool | A write scheduling thread pool is required in an async client, which can be configured by user，default is `SerializingExecutor`，If you configure the pool, please focus on the performance metric `async_write_pool.time` | +| asyncReadPool | Same as `asyncWritePool` for reading, performance metric is `async_read_pool.time` | +| database | The database used for client, can be overridden by `RequestContext` | +| rpcOptions | The configuration options of the rpc, please refer to `RpcOptions` for details | +| routerOptions | The configuration options of the routing table, please refer to `RouterOptions` for details | +| writeOptions | The configuration options of the writing, please refer to `WriteOptions` for details | ## WriteOptions -写入相关的的配置选项 - -| name | description | -| --- | --- | -| maxRetries | 最大重试次数，sdk 会根据 server 返回的 error code 自行决定是否重试（通常是可能是路由表失效），重试过程对用户透明，完全异步上层无感知 | -| maxWriteSize | 每次写入请求最大的数据条目，超过会被分割成多个请求，默认值为 512 | -| maxInFlightWriteRows | 写入限流参数：最大的在途请求的数据行数，超过会被 block | -| limitedPolicy | 写入限流策略，提供几个实现分别为 blocking、discard 和 blocking-timeout，默认为 abort-blocking-timeout(3s)（阻塞到超时 3s 后失败并抛出异常），用户也可自行扩展 | +| name | description | +|------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| maxRetries | The max retries for write request, if the server returns an error code, the SDK will determine whether to retry the request; the retry process is transparent to the user, completely asynchronous, and imperceptible to the upper layer | +| maxWriteSize | Ihe maximum data points for each write request, if exceeds, it will be divided into multiple requests, default 512 | +| maxInFlightWritePoints | If the maximum number of data points requested in one write request exceeds the current limit, the request will be blocked | +| limitedPolicy | The write limiting policy, provide several implementations is blocking, discard and blocking-timeout，default is abort-blocking-timeout(3s) (Block until timeout 3s and fail with an exception)，Users can also extend the policy | ## QueryOptions -写入相关的的配置选项 - -| name | description | -| --- | --- | -| maxRetries | 最大重试次数，sdk 会在路由表失效时重试，重试过程对用户透明，完全异步上层无感知 | -| maxInFlightQueryRequests | 查询限流参数：最大的在途请求数，超过会被 block | -| limitedPolicy | 查询限流策略，提供几个实现分别为 blocking、discard 和 blocking-timeout，默认为 abort-blocking-timeout(10s)（阻塞到超时 10s 后失败并抛出异常），用户也可自行扩展 | - -# ManagementOptions - -| name | description | -| --- | --- | -| managementAddress | 管理服务地址，通常和集群地址 IP/Host 相同，但端口不同 | -| tenant | 租户信息 | -| checkSql | 是否在客户端提前检查 sql 有效性，默认 true | +| name | description | +|--------------------------|------------------------------------------------------------------------------------------------------------------------------------| +| maxRetries | Same as `WriteOptions.maxRetryies` for query | +| maxInFlightQueryRequests | Same as `WriteOptions.maxInFlightWriteRows` for query | +| limitedPolicy | The query limiting policy, provide implementations smae as `WriteOptions.limitedPolicy`，but default is abort-blocking-timeout(10s) | ## RpcOptions -RPC 相关的配置选项 - -| name | description | -| --- | --- | -| defaultRpcTimeout | 默认的远程调用超时时间，每个请求可单独指定 timeout，如果没有指定，那么将使用这个值，默认为 10s | -| rpcThreadPoolSize | 处理 RPC 调用的线程池最大线程数 | -| rpcThreadPoolQueueSize | 处理 RPC 调用的线程池最大线程数, 如果设置为 0，那么将使用 IO 线程处理 response，默认为 64，线程池的 core_size 要比这个值小，具体公式为：Math.min(Math.max(Cpus.cpus() << 1, 16), rpcThreadPoolQueueSize) | -| maxInboundMessageSize | 上行消息，可接受单个 message 最大的字节数，默认 64M | -| flowControlWindow | 基于 http2.0 的流量控制，默认 64M | -| idleTimeoutSeconds | 设置连接的最大空闲时间，超过时间连接可能失效，默认 5 分钟 | -| keepAliveTimeSeconds | 心跳保鲜，此参数控制在 transport 上发送 keep-alive ping 的时间间隔（以秒为单位默认无限秒，相当于关闭），具备指数避让策略。 | -| keepAliveTimeoutSeconds | 心跳保鲜，此参数控制 keep-alive ping 的发送方等待确认的时间（以秒为单位， 默认 3 秒）。如果在此时间内未收到确认，它将关闭连接。 | -| keepAliveWithoutCalls | 心跳保鲜，如果将此参数设置为 true（默认为 false），则即使没有请求进行，也可以发送 keep-alive ping。这会产生一定消耗，通常情况下，更建议使用 idleTimeoutSeconds 来代替次选项 | -| limitKind | 限流算法类型，支持 Vegas 和 Gradient， 默认为 Gradient | -| initialLimit | Limiter 初始 limit | -| maxLimit | Limiter 最大 limit | -| smoothing | Limiter 平滑因子，默认 0.2 | -| blockOnLimit | 当达到限流值时是否 block 住请求线程？否则快速失败，默认值 false | +| name | description | +|-------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| defaultRpcTimeout | The rpc call timeout, each request can specify timeout separately, if not specified, default is 10s | +| rpcThreadPoolSize | The maximum number of threads in the thread pool that handles RPC calls | +| rpcThreadPoolQueueSize | The maximum number of threads in pool for processing RPC calls, IO threads will be used to process the response if set to 0, default is 64. The core_size of the thread pool should be smaller than this value, the formula is: Math.min(Math.max(Cpus.cpus() << 1, 16), rpcThreadPoolQueueSize) | +| maxInboundMessageSize | The maximum number of bytes that can be accepted for one Inbound message, default is 64M | +| flowControlWindow | Flow control based on http2.0, default is 64M | +| idleTimeoutSeconds | The maximum idle time of the connection, the connection may fail if the time exceeds, default is 5 minutes | +| keepAliveTimeSeconds | The time interval for sending keep-alive ping on the transport (default is infinite seconds, same as turning off), contain exponential avoidance strategy | +| keepAliveTimeoutSeconds | The timeout for waiting keep-alive ping (default is 3 senconds), if no ack is received within this time, it will close the connection. | +| keepAliveWithoutCalls | If enale this param (defaul is disable), keep-alive pings can be sent even if no requests are made. It has overhead, it is recommended to use `idleTimeoutSeconds` instead of this option | +| limitKind | Limiting algorithm type, supports Vegas and Gradient, default is Gradient | +| initialLimit | Limiter initial limit | +| maxLimit | Limiter max limit | +| smoothing | Limiter smoothing factor, default 0.2 | +| blockOnLimit | Block the Request when limit is exceeded, default is false | ## RouterOptions -路由表更新组件的配置选项 - -name | description | -| --- | --- | -| maxCachedSize | 本地最大缓存路由表条目数量，默认值为 10_000，超过会定期 GC | -| gcPeriodSeconds | 定期 GC 触发频率，默认 60 秒 | -| refreshPeriodSeconds | 路由表刷新频率，除了被服务端标记为 invalid 会在调用时主动刷新以外，后台还会定期刷新所有路由表，默认 30 秒执行一次全量刷新 | + name | description | +|----------------------|-------------------------------------------------------------------------------------------------------------| +| maxCachedSize | The maximum number of local cached routing table, default is 10_000, it will be periodically GC if exceeded | +| gcPeriodSeconds | The periodic interval of GC which will clear unused router, default is 60 seconds | +| refreshPeriodSeconds | The periodic interval of refreshing routing table in background, default is 30 seconds. | diff --git a/docs/limit_policy.md b/docs/limit_policy.md index f6921a7..e8dba03 100644 --- a/docs/limit_policy.md +++ b/docs/limit_policy.md @@ -1,28 +1,27 @@ -## 关于 LimitedPolicy -CeresDBClient 默认会对写入/查询进行限流（默认写请求最大在途数据条目 16384，查询最大在途请求数目 8）， -限流方式为对在途请求的数据条数/查询请求数进行限制，可参见 WriteOptions/QueryOptions 中的配置， -其中 LimitedPolicy 为被限流后的处理策略，需要注意的是，如果用户一个写入请求中包含的数据条数就超过设置的最大许可数， -那么当在途请求为 0 的情况下会放行这次请求，下面为所有策略进行一个简单介绍 - -| name | description | -| --- | ---| -| DiscardPolicy | 达到限流值后直接丢弃最新的写入数据 | -| AbortPolicy | 达到限流值后直接抛出 LimitedException 异常到上层 | -| BlockingPolicy | 达到限流值后阻塞当前调用线程直到拿到写入许可（等待在途请求被消化） | -| BlockingTimeoutPolicy | 达到限流值后阻塞当前调用线程直到拿到写入许可（等待在途请求被消化）或者一直等到超时，如果超时，错误信息会体现在返回值的 Result#Err 上，不会抛出异常 | -| AbortOnBlockingTimeoutPolicy | 达到限流值后阻塞当前调用线程直到拿到写入许可（等待在途请求被消化）或者一直等到超时，如果超时会抛出 LimitedException 异常到上层 | -| XXXPolicy | 如果以上均不满足您的需求，请基于 LimitedPolicy 定制吧，祝 coding 的开心 | - -注意：AbortOnBlockingTimeoutPolicy(3s) 为写入限流的默认策略，AbortOnBlockingTimeoutPolicy(10s) 为查询限流的默认策略 - -## 自适应限流 -除了上面介绍的上层限流器，CeresDBClient 还在底层通信层提供了基于 TCP Vegas 和 Gradient Concurrency-limits 算法的自适应限流器。 +## About LimitedPolicy +CeresDBClient will limit the write/query by default (the default maximum number for write requests is 16384, and the default maximum number of requests for queries is 8), +The limiting parameters see the configuration in `WriteOptions` / `QueryOptions`. + +`LimitedPolicy` is the processing policy after the flow is limited. It should be noted that if the number of data contained in one write request exceeds the limit, then the request will be allowed. + +| name | description | +|------------------------------|----------------------------------------------------------------------------------------------------------------------------------------| +| DiscardPolicy | The latest written data is directly discarded when limiting | +| AbortPolicy | Throw a LimitedException directly to the upper layer when limiting | +| BlockingPolicy | Blocking thread while limiting | +| BlockingTimeoutPolicy | Blocking thread util timeout，the error message while be reflected in `Result#Err` when timeout occurs, and no exception will be thrown | +| AbortOnBlockingTimeoutPolicy | Blocking thread util timeout，and throw a `LimitedException` to the upper when timeout occurs | +| XXXPolicy | If none of the above meets your needs, please customize it based on `LimitedPolicy`. Happy coding : ) | + +Note：default write limit policy is `AbortOnBlockingTimeoutPolicy(3s)`，query is `AbortOnBlockingTimeoutPolicy(10s)` + +## Adaptive limiting +CeresDBClient also provides an adaptive current limiter based on TCP `Vegas` and `Gradient` limit algorithms at the underlying communication layer. ### Vegas -Vegas 是一种主动调整 cwnd 的拥塞控制算法，主要思想是设置两个阈值，alpha 和 beta，然后通过计算目标速率 (Expected) -和实际速率 (Actual) 差 (diff)，再比较 diff 与 alpha 和 beta 的关系，对 cwnd 进行调节。 +`Vegas` is a congestion control algorithm that actively adjusts cwnd. The main idea is to set two thresholds, alpha and beta, and then calculate the target rate (Expected) and the actual rate (Actual), then compare the relationship between diff and alpha and beta, and adjust cwnd. -核心算法逻辑如下： +The core algorithm is ``` diff = cwnd*(1-baseRTT/RTT) if (diff < alpha) @@ -34,21 +33,20 @@ else ``` ### Gradient Concurrency-limits -并发限制算法，根据当前平均 RTT 的变化梯度和长期指数平滑平均 RTT 调整限制。与传统的拥塞控制算法不同，使用平均值而不是最小值。 +The `Gradient` concurrency-limit algorithm adjusts the limit according to the change gradient of the current average RTT and the long-term exponentially smoothed average RTT. Unlike traditional congestion control algorithms, average values are used instead of minimum values. -核心算法逻辑如下： +The core algorithm is ```java -// 计算限制范围 [0.5, 1.0] 的梯度以过滤异常值 +// Compute the gradient limited to the range [0.5, 1.0] to filter outliers gradient = max(0.5, min(1.0, longtermRtt / currentRtt)); -// 通过应用渐变并允许某些排队来计算新的限制 +// Calculate the new limit by applying the gradient and allowing some queuing newLimit = gradient * currentLimit + queueSize; -// 使用平滑因子更新限制（平滑因子默认值为 0.2） +// Update limits with smoothing factor (default is 0.2) newLimit = currentLimit * (1 - smoothing) + newLimit * smoothing ``` -- 默认自适应限流算法是 Gradient，可以通过设置 `io.ceresdb.rpc.RpcOptions.limitKind = Vegas` 改为 Vegas - 或者是设置为 设置 `io.ceresdb.rpc.RpcOptions.limitKind = None` 进行关闭 -- 默认达到限流值会快速失败，也可以通过设置 `io.ceresdb.rpc.RpcOptions.blockOnLimit = true` 将策略改为阻塞直至超时 -- 默认情况下写、读请求共享这个限流器，默认写、读分别可以获得 70% 和 30% 的通行许可，可以通过启动参数 -DCeresDB.rpc.write.limit_percent=xxx 来设置写的比例，读的比例为 1 - writeLimitPercent +- The default adaptive current limiting algorithm is `Gradient`，you can set in configuration `io.ceresdb.rpc.RpcOptions.limitKind = Vegas` by `Vegas`, or set `io.ceresdb.rpc.RpcOptions.limitKind = None` to close +- It will fail quickly when the limit is reached by default, and it can also be set by setting `io.ceresdb.rpc.RpcOptions.blockOnLimit = true` +- Write and read requests share this current limiter. By default, 70% and 30% of the access permissions can be obtained for write and read respectively. You can use the startup parameters `-DCeresDB.rpc.write.limit_percent=xxx`，readLimitPercent is `1 - writeLimitPercent` diff --git a/docs/metrics_and_display.md b/docs/metrics_and_display.md index 1eb9de8..a17c2bf 100644 --- a/docs/metrics_and_display.md +++ b/docs/metrics_and_display.md @@ -1,60 +1,59 @@ -## 如何获取 metrics 和 display 数据？ -在程序运行时，可以利用 Linux 平台的 SIGUSR2 信号输出节点的状态信息（display）以及 metric 数据 +## How to get metrics and display data? +When the program is running, the SIGUSR2 signal of the Linux platform can be used to output the status information (display) and metric data of the node -### 执行方式: +### How to use `kill -s SIGUSR2 pid` -相关信息会输出到指定目录，默认在程序工作目录（cwd: lsof -p $pid | grep cwd）生成 2 个文件 -- CeresDB_client_metrics.log 为前缀的文件，记录了当前 client 节点的所有 metrics 信息 -- CeresDB_client_display.log 为前缀的文件，记录的当前 client 节点重要的内存状态信息 +Information will be output to the specified directory，by default, 2 files will be generated in the program working directory (cwd: lsof -p $pid | grep cwd) +- The file prefixed with CeresDB_client_metrics.log records all the metrics information of the current client node +- The file prefixed with CeresDB_client_display.log records important memory status information of the current client node -### Metrics 输出到日志 -- 默认每 10 分钟会输出 metrics 到日志中 +### Metrics output to log +- Metrics will be output to the log every 10 minutes by default -### Metrics 列表（不断更新中） -|name|description| -| ---- | ---- | -| req_bytes | RPC request 总的 bytes 统计 | -| resp_bytes | RPC response 总的 bytes 统计 | -| req_qps_${rpc_method} | RPC request 的 QPS 统计，每个 RPC method 单独统计 | -| req_serialized_bytes_${rpc_method} | 每个 RPC request 的序列化后的大小统计，RPC method 统计粒度 | -| resp_serialized_bytes_${rpc_method} | 每个 RPC response 的序列化后的大小统计，RPC method 统计粒度 | -| req_rt_${rpc_method}_${tenant}| 请求 RT，方法 + 租户粒度 | -| req_rt_${rpc_method}_${tenant}_${address} | 请求 RT，方法 + 租户 + Server 地址粒度 | -| req_failed_${rpc_method}_${tenant}| 请求失败统计，方法 + 租户粒度 | -| req_failed_${rpc_method}_${tenant}_${address} | 请求失败统计，方法 + 租户 + Server 地址粒度 | -| thread_pool.${thread_pool_name} | 线程池执行任务耗时统计 | -| scheduled_thread_pool.${schedule_thread_pool_name} | Schedule 线程池执行任务耗时统计 | -| split_num_per_write | 根据路由表，每批写入会被分割成多个请求，这个指标统计的是写入被分割的情况 | -| route_for_metrics_refreshed_size_${address} | 每次向 server 请求路由表的 metric 数量 | -| route_for_metrics_cached_size_${address} | 本地缓存的路由表的 metric 数量 | -| route_for_metrics_gc_times_${address} | 路由表每次 GC 需要循环次数的统计 | -| route_for_metrics_gc_items_${address} | 路由表每次 GC 释放掉的 items 数量统计 | -| route_for_metrics_gc_timer_${address} | 路由表 GC 时间统计 | -| route_for_metrics_refresh_timer_${address} | 路由表远程刷新时间统计 | -| write_rows_success_num | 写入成功的数据条数统计 | -| write_rows_failed_num | 写入失败的数据条数统计 | -| write_failed | 写入失败的次数统计 | -| write_qps | 写入请求 QPS | -| metrics_num_per_write | 每个写入请求的 metrics 数量统计 | -| async_write_pool.time | SDK 中执行异步写入任务的 async pool 耗时统计，这个很重要，需要特别关注 | -| async_read_pool.time | SDK 中执行异步读取任务的 async pool 耗时统计，这个很重要，需要特别关注 | -| connection_counter | SDK 与 server 建立的连接个数 | -| connection_failures | SDK 与 server 失败连接的统计 | -| read_row_count | 每次查询数据条数的统计 | -| read_failed | 查询失败的次数统计 | -| read_qps | 查询 QPS | -| write_by_retries_${n} | 第 n 次重试写入的 qps，n == 0 表示是第一次写入（非重试），n > 3 将视为 n == 3 来统计 | -| read_by_retries_${n} | 第 n 次重试查询的 qps，n == 0 表示是第一次查询（非重试），n > 3 将视为 n == 3 来统计 | -| write_limiter_acquire_wait_time | 被写入限流器 block 的时间统计 | -| write_limiter_acquire_available_permits | 写入限流器 available_permits 统计 | -| query_limiter_acquire_wait_time | 被查询限流器 block 的时间统计 | -| query_limiter_acquire_available_permits | 查询限流器 available_permits 统计 | -| direct_executor_timer_${name} | direct executor 任务执行耗时统计 | -| serializing_executor_single_task_timer_${name} | serializing executor 单个任务执行耗时统计 | -| serializing_executor_drain_timer_${name} | serializing executor 排干全部任务耗时统计 | -| serializing_executor_drain_num_${name} | serializing executor 每次排干任务数量 histogram 统计 | -| rpc_limiter_${name} | 基于 TCP Vegas 算法的 rpc 限流器相关统计指标 | +### Metrics description （updating） +| name | description | +|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------| +| req_bytes | RPC request total bytes statistics | +| resp_bytes | RPC response total bytes statistics | +| req_qps_${rpc_method} | QPS statistics of RPC request, separate for each RPC method | +| req_serialized_bytes_${rpc_method} | The serialized size statistics of each RPC request, separate for each method | +| resp_serialized_bytes_${rpc_method} | The serialized size statistics of each RPC response, separate for each method | +| req_rt_${rpc_method}_ | Request RT, method | +| req_rt_${rpc_method}_${address} | Request RT, method + server | +| req_failed_${rpc_method} | Request failed, method | +| req_failed_${rpc_method}_${address} | Request failed, method + server | +| thread_pool.${thread_pool_name} | Time of thread pool execution tasks | +| scheduled_thread_pool.${schedule_thread_pool_name} | Time of thread schedule pool execution tasks | +| split_num_per_write | Each batch of writes will be divided into multiple requests by router, and this indicator counts the splitting of writes | +| route_for_tables_refreshed_size_${address} | The count of tables requested for the router from the server each time | +| route_for_tables_cached_size_${address} | The count of tables in the locally cached router | +| route_for_tables_gc_times_${address} | The count of cycles required for each GC of the router | +| route_for_tables_gc_items_${address} | The count of items released by each GC of the router | +| route_for_tables_gc_timer_${address} | The time of GC time | +| write_points_success_num | The count of successfully written point | +| write_points_failed_num | The count of failed written points | +| write_failed | The count of failed write requests | +| write_qps | Write QPS | +| points_num_per_write | The count of points for each write request | +| async_write_pool.time | The async pool time of executing asynchronous write tasks in the SDK is very important and requires special attention | +| async_read_pool.time | Same as `async_write_pool.time` for reading | +| connection_counter | The count of connections established between the SDK and the server | +| connection_failures | The count of failed connections established between the SDK and the server | +| read_rows_count | The count of rows per query | +| read_failed | The count of failed queried requests | +| read_qps | Query QPS | +| write_by_retries_${n} | The QPS of the nth retry write, n == 0 means it is the first write (not retry), n > 3 will be counted as n == 3 | +| read_by_retries_${n} | Same as `write_by_retries_${n}` for reading | +| write_limiter_acquire_wait_time | Time written to the current limiter block | +| write_limiter_acquire_available_permits | Write limiter available_permits | +| query_limiter_acquire_wait_time | The time of the queried limiter block | +| query_limiter_acquire_available_permits | Query limiter available_permits statistics | +| direct_executor_timer_${name} | The task execution time of direct executor | +| serializing_executor_single_task_timer_${name} | The task exeution time for serializing executor | +| serializing_executor_drain_timer_${name} | Drain all task time statistics | +| serializing_executor_drain_num_${name} | Serializing executor histogram statistics on the number of tasks drained each time | +| rpc_limiter_${name} | The rpc metrics on TCP Vegas limiter | ### Metrics demo: ``` @@ -72,7 +71,7 @@ rpc_limiter_call_id_grpc_call_status_success count = 3160 -- CeresDB -- Histograms ------------------------------------------------------------------ -metrics_num_per_write +points_num_per_write count = 1585 min = 1 max = 1 @@ -144,7 +143,7 @@ resp_serialized_bytes_storage.StorageService/Write 98% <= 8.00 99% <= 8.00 99.9% <= 8.00 -route_for_metrics_cached_size_127.0.0.1:8831 +route_for_tables_cached_size_127.0.0.1:8831 count = 1 min = 1 max = 1 @@ -156,7 +155,7 @@ route_for_metrics_cached_size_127.0.0.1:8831 98% <= 1.00 99% <= 1.00 99.9% <= 1.00 -route_for_metrics_gc_items_127.0.0.1:8831 +route_for_tables_gc_items_127.0.0.1:8831 count = 0 min = 0 max = 0 @@ -168,7 +167,7 @@ route_for_metrics_gc_items_127.0.0.1:8831 98% <= 0.00 99% <= 0.00 99.9% <= 0.00 -route_for_metrics_gc_times_127.0.0.1:8831 +route_for_tables_gc_times_127.0.0.1:8831 count = 0 min = 0 max = 0 @@ -180,7 +179,7 @@ route_for_metrics_gc_times_127.0.0.1:8831 98% <= 0.00 99% <= 0.00 99.9% <= 0.00 -route_for_metrics_refreshed_size_127.0.0.1:8831 +route_for_tables_refreshed_size_127.0.0.1:8831 count = 1 min = 1 max = 1 @@ -252,7 +251,7 @@ write_limiter_acquire_available_permits 98% <= 7168.00 99% <= 7168.00 99.9% <= 7168.00 -write_rows_failed_num +write_points_failed_num count = 1580 min = 0 max = 0 @@ -264,7 +263,7 @@ write_rows_failed_num 98% <= 0.00 99% <= 0.00 99.9% <= 0.00 -write_rows_success_num +write_points_success_num count = 1580 min = 1024 max = 1024 @@ -436,7 +435,7 @@ req_rt_storage.StorageService/Write_public_127.0.0.1:8831 98% <= 229.00 milliseconds 99% <= 385.00 milliseconds 99.9% <= 546.00 milliseconds -route_for_metrics_gc_timer_127.0.0.1:8831 +route_for_tables_gc_timer_127.0.0.1:8831 count = 0 mean rate = 0.00 calls/second 1-minute rate = 0.00 calls/second @@ -452,22 +451,6 @@ route_for_metrics_gc_timer_127.0.0.1:8831 98% <= 0.00 milliseconds 99% <= 0.00 milliseconds 99.9% <= 0.00 milliseconds -route_for_metrics_refresh_timer_127.0.0.1:8831 - count = 1 - mean rate = 0.02 calls/second - 1-minute rate = 0.07 calls/second - 5-minute rate = 0.16 calls/second - 15-minute rate = 0.19 calls/second - min = 322.00 milliseconds - max = 322.00 milliseconds - mean = 322.00 milliseconds - stddev = 0.00 milliseconds - median = 322.00 milliseconds - 75% <= 322.00 milliseconds - 95% <= 322.00 milliseconds - 98% <= 322.00 milliseconds - 99% <= 322.00 milliseconds - 99.9% <= 322.00 milliseconds rpc_limiter_acquire_time_storage.StorageService/Write count = 3170 mean rate = 48.96 calls/second @@ -589,7 +572,7 @@ write_limiter_acquire_wait_time id=1 version=1.0.0.Final clusterAddress=127.0.0.1:8831 -tenant=public +database=public userAsyncWritePool=null userAsyncReadPool=null @@ -599,10 +582,10 @@ routeCache.size=1 --- GrpcClient --- started=true -opts=RpcOptions{defaultRpcTimeout=10000, rpcThreadPoolSize=0, rpcThreadPoolQueueSize=16, maxInboundMessageSize=67108864, flowControlWindow=67108864, idleTimeoutSeconds=300, keepAliveTimeSeconds=3, keepAliveTimeoutSeconds=3, keepAliveWithoutCalls=true, openVegasLimiter=true, vegasInitialLimit=50, blockOnLimit=false, tenant=Tenant{tenant='public', childTenant='sub_test', token='test_token'}} +opts=RpcOptions{defaultRpcTimeout=10000, rpcThreadPoolSize=0, rpcThreadPoolQueueSize=16, maxInboundMessageSize=67108864, flowControlWindow=67108864, idleTimeoutSeconds=300, keepAliveTimeSeconds=3, keepAliveTimeoutSeconds=3, keepAliveWithoutCalls=true, openVegasLimiter=true, vegasInitialLimit=50, blockOnLimit=false} connectionObservers=[io.ceresdb.CeresDBClient$RpcConnectionObserver@465f5824] asyncPool=DirectExecutor{name='grpc_executor'} -interceptors=[io.ceresdb.rpc.interceptors.MetricInterceptor@62b47ad1, io.ceresdb.rpc.interceptors.ClientRequestLimitInterceptor@77533e32, io.ceresdb.rpc.interceptors.ContextToHeadersInterceptor@1a52427d, io.ceresdb.rpc.interceptors.AuthHeadersInterceptor@2670a76b] +interceptors=[io.ceresdb.rpc.interceptors.MetricInterceptor@62b47ad1, io.ceresdb.rpc.interceptors.ClientRequestLimitInterceptor@77533e32, io.ceresdb.rpc.interceptors.ContextToHeadersInterceptor@1a52427d] managedChannelPool={127.0.0.1:8831=ManagedChannelOrphanWrapper{delegate=ManagedChannelImpl{logId=1, target=127.0.0.1:8831}}} transientFailures={} @@ -614,10 +597,4 @@ asyncPool=SerializingExecutor{name='write_client'} --- QueryClient --- maxRetries=1 asyncPool=SerializingExecutor{name='query_client'} - ---- HttpManagementClient --- -started=true -tenant=public - - ``` diff --git a/docs/other.md b/docs/other.md index 9e21598..f01dc6b 100644 --- a/docs/other.md +++ b/docs/other.md @@ -1,13 +1,13 @@ -## 一些运维工具 +## Some maintenance tools -### 如何使用 `kill -s SIGUSR2 $pid` +### How to use `kill -s SIGUSR2 $pid` -第一次执行 `kill -s SIGUSR2 $pid` 会从日志输出上看到如下帮助信息，包括: -1. 清除本地路由表缓存 -2. 开/关简洁版写查 response 日志的输出 -3. 输出内存中的 metrics、路由表 cache 和重要对象的内存状态信息到本地文件中 +The first time you execute `kill -s SIGUSR2 $pid`, will see the following help information from the log output, including: +1. Clear the local router cache +2. Turn on/off simple output of the response log +3. Output in-memory metrics, router cache, and memory state information of important objects to local files -按照帮助信息执行即可 +Follow the help information to execute ``` - -- CeresDBClient Signal Help -- diff --git a/docs/read.md b/docs/read.md index 2c70b0f..8cc4fef 100644 --- a/docs/read.md +++ b/docs/read.md @@ -1,4 +1,4 @@ -## 查询流程 +## Query process ``` ┌─────────────────────┐ @@ -38,19 +38,19 @@ └─────────────────────┘ └─────────────────────┘ └─────────────────────┘ ``` -## 名词解释 +## Description - CeresDBClient - - CeresDB 的 java client 实现，面向使用用户，提供写入、查询等 API + - The java client implementation of CeresDB is oriented to users and provides APIs such as writing and querying - QueryClient - - 查询的默认实现，纯异步 - - 包含异步获取路由表，路由表失效自动重试 + - The default implementation of queries, purely asynchronous + - Including asynchronous acquisition of routing table, automatic retry when routing table fails - RouterClient - - 路由表客户端，会在本地维护路由表信息，并从服务端刷新路由表 + - The router client will maintain the routing table information locally and refresh the routing table from the server - RpcClient - - 一个纯异步的高性能 RPC 客户端，默认传输层基于 gRPC 实现 + - A pure asynchronous high-performance RPC client, the default transport layer is implemented based on gRPC -## 查询 API 说明 +## How to use ```java /** @@ -60,142 +60,37 @@ * @param ctx the invoke context * @return query result */ -CompletableFuture> query(QueryRequest req, Context ctx); +CompletableFuture> sqlQuery(SqlQueryRequest req, Context ctx); ``` -### 参数说明: -| name | desc | -| --- | --- | -| `QueryRequest req` | 查询条件，包含 metrics 和 ql 字段，metrics 为建议字段，填写话会有更高效的路由, 不填写的话会自动解析 ql 语句以便进行路由(需要引入 ceresdb-sql 模块); ql 为查询语言的文本表示。 | -| `Context ctx` | 调用上下文，实现一些特殊需求，ctx 中的内容会写入 gRPC 的 headers metadata | +### Parameters +| name | desc | +|-----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `SqlQueryRequest req` | Query request, including `tables` and `sql` fields, `tables` is a suggested field, if filled in, there will be more efficient routing, if not filled, the sql statement will be automatically parsed for routing (need to introduce the ceresdb-sql module) | +| `Context ctx` | Call context, to achieve some special requirements, the content in ctx will be written into the headers metadata of gRPC | -### 返回值说明 +### Return -```java -CompletableFuture>: 返回结果一个 future，因为 query 是纯异步 API，整个链路上没有一处是阻塞的。 - -Result: Result 的灵感来自 Rust 中的 Result，其中 QueryOk 和 Err 同时只能一个有值。 -QueryOk: 查询成功的结果展示，包含一个 rowCount 用于反映这次查询的数据条数；还有一个 Stream rows 表示此次查询的数据流。 -可以调用 QueryOk#mapToRecord 将每一行转成 Record 或是 QueryOk#mapToArray 将每一行转成一个 Object[]。 -实际上，也可以让服务端返回 json，在发起查询调用时在 Context -中设置数据协议，即 ctx.with("data_protocol", "json")，设置后服务端将返回 UTF8 的 byte[]，用户基于 QueryOk#map 自行将 byte[] -转成 String 再解析 json 即可。 -代码示例： - // Record - final Stream records = queryOk.mapToRecord(); - // 其中 parseUser 先基于 bytes 构建 String 再 parseJson 到 pojo 对象 - final Stream users = queryOk.map(bytes -> parseUser(bytes)); - -Err: 查询失败结果展示，包含错误状态码、错误文本信息、抛错的服务器地址。 -``` - -Record: -```java -public interface Record extends IndexedRecord { - - /** - * Return the value of a field given its name. - */ - Object get(final String field); - - default T get(final String field, final Class expectType) { - return expectType.cast(get(field)); - } - - /** - * Get a boolean value for the given {@code field}. - */ - default Boolean getBoolean(final String field) { - return get(field, Boolean.class); - } - - default Integer getUInt16(final String field) { - return getInteger(field); - } - - default Integer getUInt8(final String field) { - return getInteger(field); - } +`CompletableFuture>` +- Return a future, because query is a pure asynchronous API, and no part of the entire link is blocked. - default Integer getInt16(final String field) { - return getInteger(field); - } +`Result` +- Result is inspired by Result in Rust, where QueryOk and Err can only have a value at the same time - default Integer getInt8(final String field) { - return getInteger(field); - } +`SqlQueryOk` + - The result of a successful query, contains + - sql in SqlQueryRequest + - affectedRows is the number of affected data rows returned when updating or deleting data + - rows is the statement returned by the query, a class of Sql will only return affectedRows Or List, will not return both values - /** - * Get a integer value for the given {@code field}. - */ - default Integer getInteger(final String field) { - return get(field, Integer.class); - } +`Row` +- When processing the results returned by the query, the user can directly obtain `List`, or process it through stream. +- Row is a collection of Value, which is a very simple data structure +- Note: When Value gets the java primitive value, you need to pass in the type method that matches the table creation, otherwise an error will be reported +- Example to use `Row`: `row.getColumnValue("cpu_util").getDouble() + ` - default Long getTimestamp(final String field) { - return getLong(field); - } +`Err` +- The result of the query failure is displayed, including the error status code, error text information, and the address of the server where the error was thrown. - default Long getUInt64(final String field) { - return getLong(field); - } - default Long getUInt32(final String field) { - return getLong(field); - } - - default Long getInt64(final String field) { - return getLong(field); - } - - /** - * Get a long value for the given {@code field}. - */ - default Long getLong(final String field) { - return get(field, Long.class); - } - - /** - * Get a float value for the given {@code field}. - */ - default Float getFloat(final String field) { - return get(field, Float.class); - } - - /** - * Get a double value for the given {@code field}. - */ - default Double getDouble(final String field) { - return get(field, Double.class); - } - - /** - * Get a string value for the given {@code field}. - */ - default String getString(final String field) { - return get(field, String.class); - } - - /** - * Get a bytes value for the given {@code field}. - */ - default byte[] getBytes(final String field) { - return get(field, byte[].class); - } - - /** - * Return true if record has field with name. - */ - boolean hasField(final String field); - - /** - * Return the field count of this record. - */ - int getFieldCount(); - - /** - * Return all field descriptors in the record. - */ - List getFieldDescriptors(); -} -``` diff --git a/docs/streaming.md b/docs/streaming.md index aa8fad0..2166135 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -1,29 +1,28 @@ -## 流式 API -CeresDB 支持流式查询和写入，其中流式查询为 response streaming，适合用来大规模拉取数据；流式写入为 request streaming，适合大批量导入数据使用。 +## Stream API +CeresDB supports streaming query and writing which is suitable for large amounts of data. -### 流式写入 API 说明 +### Stream Write API ```java /** * Executes a streaming-write-call, returns a write request observer for streaming-write. * - * @param metric the metric to write + * @param table the table to write * @param ctx the invoke context * @return a write request observer for streaming-write */ -StreamWriteBuf streamWrite(final String metric, final Context ctx); +StreamWriteBuf streamWrite(final String table, final Context ctx); ``` -#### 参数说明 +#### Parameters -| name | desc | -| --- | --- | -| `String metric` | 必须要指定一个 metric，并且限制只能流式写入这个 metric 的数据，这样做是为了高效，支持多个 metric 数据同时流式写入并不能做到高效，意义不大，所以不支持 | -| `Context ctx` | 调用上下文，实现一些特殊需求，ctx 中的内容会写入 gRPC 的 headers metadata | +| name | desc | +|----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `String table` | A table must be specified, and the data that can only be streamed into this table for efficiency. Supporting multiple table data streamed at the same time is not efficient and meaningless, so it is not supported | +| `Context ctx` | Call context, to achieve some special requirements, the content in ctx will be written into the headers metadata of gRPC | -#### 返回值说明 -返回值: -`StreamWriteBuf`: 一个支持流式写入的 buffer 如下： +#### Return +`StreamWriteBuf` ```java public interface StreamWriteBuf { @@ -61,23 +60,25 @@ public interface StreamWriteBuf { } ``` -用户不断的多次调用 write 将数据写入缓存，在 flush 时真正发起一次真正的流式写入调用，可以多次 flush，建议 write 一定条数再 flush 一次，当写入完成时再调用 -completed 告知 server 完成，接下来 server 端会返回一个汇总过的 result(`CompletableFuture`) +You can continuously calls write multiple times to write data into the cache, and actually initiates a real streaming write call when flushing, which can be flushed multiple times. +It is recommended to write batch of data and then flush once, and call again when the writing is completed + +`completed` method tells the server to complete, and then the server will return a summarized result(`CompletableFuture`) Example: ```java -final StreamWriteBuf writer = this.writeClient.streamWrite("test_metric"); +final StreamWriteBuf writer = this.writeClient.streamWrite("test_table"); final CompletableFuture ret = writer - .write(Util.generateRow("test_metric")) // 写入随机生成的数据，这里只作为示例 - .write(Util.generateRow("test_metric")) // 可以链式调用 - .write(Util.generateRow("test_metric")) // - .flush() // flush 一次，后台会将数据发送到 server - .write(Util.generateRow("test_metric")) // - .flush() // 再一次 flush，整个流式调用可以多次 flush，每次 flush 数据的大小可根据业务场景定夺 - .completed(); // 调用 completed 会结束这个`流`，server 会返回总体的写入结果 + .write(Util.generatePoints("test_table")) + .write(Util.generatePoints("test_table")) + .write(Util.generatePoints("test_table")) + .flush() // flush, the background will send the data to the server + .write(Util.generatePoints("test_table")) // + .flush() // flush again + .completed(); // completed will end the `stream`, and the server will return the overall write result ``` -### 流式查询 API 说明 +### Stream Query API ```java /** @@ -87,7 +88,7 @@ final CompletableFuture ret = writer * @param observer receives data from an observable stream * @param ctx the invoke context */ -void streamQuery(QueryRequest req, Context ctx, Observer observer); +void streamSqlQuery(SqlQueryRequest req, Context ctx, Observer observer); /** * Executes a stream-query-call with a streaming response. @@ -98,22 +99,23 @@ void streamQuery(QueryRequest req, Context ctx, Observer observer); * @param ctx the invoke context * @return the iterator of record data */ -Iterator blockingStreamQuery(QueryRequest req, long timeout, TimeUnit unit, Context ctx); +Iterator blockingStreamSqlQuery(SqlQueryRequest req, long timeout, TimeUnit unit, Context ctx); ``` -流式查询有两种 API，一种是基于 Observer 回调，灵活性更高，适合非阻塞的异步场景； -另一种是返回一个 Iterator，每个 element 即是一行数据(Record)，hasNext 方法有可能被阻塞直到服务端返回数据流或者数据流结束。 +Stream query has two APIs. +- one is based on `Observer` callback, which is more flexible and suitable for non-blocking asynchronous scenarios +- the other is to return an `Iterator`, each element is a row of data (Row), the hasNext method may be blocked until the server returns the data stream or the data stream ends. -#### 参数说明 +#### Parameters -| name | desc | -| --- | --- | -| `QueryRequest req` | 查询条件，这一点和普通查询没有任何区别 | -| `Context ctx` | 调用上下文，实现一些特殊需求，ctx 中的内容会写入 gRPC 的 headers metadata | -| `Observer observer` | response 回调观察者，可以不断的接收 server 端返回的数据，在 server 端把数据吐完后 onCompleted 会被调用 | -| `timeout` | 调用 `Iterator#hasNext` 的最大等待时间（因为是基于 buffer 的惰性拉取数据，在 buffer 为空时会从 server 现拉取数据） | +| name | desc | +|---------------------------------| -- | +| `SqlQueryRequest req` | Query request, which are no different from ordinary queries | +| `Context ctx` | Same as ordinary queries `Context` | +| `Observer observer` | The response callback observer can continuously receive the data returned by the server, and `onCompleted` will be called after all the data is sent on the server | +| `timeout` | The maximum waiting time for calling `Iterator#hasNext` (because it is a buffer-based lazy data pull, when the buffer is empty, it will pull data from the server now) | -Observer 的 API 如下： +Observer API ```java public interface Observer { diff --git a/docs/table.md b/docs/table.md index eebf276..d89a0a9 100644 --- a/docs/table.md +++ b/docs/table.md @@ -1,102 +1,2 @@ -## 如何创建一张表 -CeresDB 是一个 Schema-less 的时序数据引擎，你可以不必创建 schema 就立刻写入数据（CeresDB 会根据你的第一次写入帮你创建一个默认的 schema）。 -当然你也可以自行创建一个 schema 来更精细化的管理的表（比如索引等） - -### 表规范 -#### CeresDB 的表要有以下约束： -- 必须包含主键 -- 必须包含时间列 -- 主键必须包含时间列，并且只能包含一个时间列，该时间列为主时间列（默认时间列） -- 主键列必须是非空列，因此，时间列也必须是非空列 -- 可以指定若干 tag 列 - -#### 主时间列 -每个表都必须要有一个主时间列，用来映射到时序数据里的时间戳，如 opentsdb/prometheus 里的 timestamp 。建表时，用户必须指定表中一个 Timestamp -类型的列为主时间列，该列会自动映射到 opentsdb/prometheus 等协议里的 timestamp 。需要注意的是，主时间列的名字可以自定义，不需要一定叫 timestamp - -#### 约束和选项 -建表语句可以 -- 通过 PRIMARY KEY 约束指定主键 -- 通过 TIMESTAMP KEY 约束指定主时间列（默认时间列） -- 通过 TAG 选项指定 tag 列 - -上述约束可以全大写或者小写，在建表后均不可修改 - - -## 建表例子 - -1. 指定 primary key 为 (c1, ts) ，这是一个复合主键。同时，指定主时间列为 ts -``` -CREATE TABLE with_primary_key( - ts TIMESTAMP NOT NULL, - c1 STRING NOT NULL, - c2 STRING NULL, - c3 DOUBLE NULL, - c4 STRING NULL, - c5 STRING NULL, - TIMESTAMP KEY(ts), - PRIMARY KEY(c1, ts) -) ENGINE=Analytic WITH (ttl='7d'); -``` - -2. 指定 primary key 为 (c1, ts)，指定主时间列为 ts ，指定 tag 为 c1，c2，c2。另外，可以注意到允许有其他时间戳列，这个列作为普通的数据列存在。 -``` -CREATE TABLE with_primary_key_tag( - ts TIMESTAMP NOT NULL, - c1 STRING TAG NOT NULL, - c2 STRING TAG NULL, - c3 STRING TAG NULL, - c4 DOUBLE NULL, - c5 STRING NULL, - c6 STRING NULL, - c7 TIMESTAMP NULL, - TIMESTAMP KEY(ts), - PRIMARY KEY(c1, ts) -) ENGINE=Analytic WITH (ttl='7d'); -``` - -3. 部分场景下，可能用户并没有指定主键，但是会指定 tags 。这种情况下，会隐式地为用户创建一个叫 tsid 的列，并自动地设置主键为 (tsid, timestamp) - 其中 timestamp 为用户的主时间列，而 tsid 则是通过 tags 哈希生成。本质上这也是一种自动生成 id 的机制 -``` -CREATE TABLE with_tag( - ts TIMESTAMP NOT NULL, - c1 STRING TAG NOT NULL, - c2 STRING TAG NULL, - c3 STRING TAG NULL, - c4 DOUBLE NULL, - c5 STRING NULL, - c6 STRING NULL, - TIMESTAMP KEY(ts) -) ENGINE=Analytic WITH (ttl='7d'); -``` - -因此，实际逻辑上该语句会变成类似以下形式的语句（注意并不存在这种语法） -``` -CREATE TABLE with_tag( - tsid AUTO_HASH_BY_TAGS, - ts TIMESTAMP NOT NULL, - c1 STRING TAG NOT NULL, - c2 STRING TAG NULL, - c3 STRING TAG NULL, - c4 DOUBLE NULL, - c5 STRING NULL, - c6 STRING NULL, - TIMESTAMP KEY(ts), - PRIMARY KEY(tsid, ts) -) ENGINE=Analytic; -``` - -## DESCRIBE TABLE -``` -DESCRIBE TABLE MY_FIRST_TABLE -``` - -## ALTER TABLE -``` -ALTER TABLE MY_FIRST_TABLE ADD COLUMN col_20 UINT64 -``` - -# EXISTS TABLE -``` -EXISTS [TABLE] xxx_table -``` +## Table specification +See [Table specification doc](https://github.com/CeresDB/docs/blob/main/docs/src/en/sql/model/special_columns.md) \ No newline at end of file diff --git a/docs/write.md b/docs/write.md index 565d792..dad4e2a 100644 --- a/docs/write.md +++ b/docs/write.md @@ -1,5 +1,5 @@ -## 写入流程图 +## Write process ``` ┌─────────────────────┐ @@ -39,34 +39,35 @@ └─────────────────────┘ └─────────────────────┘ └─────────────────────┘ ``` -## 名词解释 +## Description - CeresDBClient - - CeresDB 的 java client 实现，面向使用用户，提供写入、查询等 API + - The java client implementation of CeresDB is oriented to users and provides APIs such as writing and querying - WriteClient - - 写数据的默认实现，纯异步写入 - - 包含异步获取路由表，自动拆分请求 - - 支持异步重试以及异步合并多个服务端的响应 + - The default implementation of writing data, pure asynchronous writing + - Contains asynchronously fetching router, automatically splitting requests + - Support asynchronous retry + - Asynchronously merge responses from multiple servers - RouterClient - - 路由表客户端，会在本地维护路由表信息，并从服务端刷新路由表 + - The router client will maintain the routing table information locally and refresh the routing table from the server - RpcClient - - 一个纯异步的高性能 RPC 客户端，默认传输层基于 gRPC 实现 + - A pure asynchronous high-performance RPC client, the default transport layer is implemented based on gRPC -## 创建表 -CeresDB 是一个 schema-less 时序数据库，如果你不指定 schema，会在你第一次写入时自动创建默认的 schema，你也可以自己指定 schema，下面是建表语句的一个例子，包含了所有支持的类型： +## CreateTable +Create table example ``` CREATE TABLE my_first_table( ts TIMESTAMP NOT NULL, c1 STRING TAG NOT NULL, c2 STRING TAG NOT NULL, - c3 DOUBLE NULL, - c4 STRING NULL, - c5 INT64 NULL, - c6 FLOAT NULL, - c7 INT32 NULL, - c8 INT16 NULL, - c9 INT8 NULL, - c10 BOOLEAN NULL, + c3 STRING NULL, + c4 BOOLEAN NULL, + c5 DOUBLE NULL, + c6 FLOAT NULL, + c7 INT64 NULL, + c8 INT32 NULL, + c9 INT16 NULL, + c10 INT8 NULL, c11 UINT64 NULL, c12 UINT32 NULL, c13 UINT16 NULL, @@ -77,70 +78,52 @@ CREATE TABLE my_first_table( ) ENGINE=Analytic ``` -## 写入 API 说明 +## Write API ```java /** * Write the data stream to the database. * - * @param data rows + * @param req the write request * @param ctx the invoke context * @return write result */ -CompletableFuture> write(Collection data, Context ctx); +CompletableFuture> write(WriteRequest req, Context ctx); ``` -### 参数说明 -| name | desc | -| --- | --- | -| `Collection data` | 要写入的数据，每一个 Rows 为一条时间线上多个时间点的点的集合，对于按照时间线来写入的场景这个结构将很大程度减少数据传输的大小，不过通常监控场景更多的是写一个横截面，整体上这个结构是一个综合考虑的结果。| -| `Context ctx` | 调用上下文，实现一些特殊需求，ctx 中的内容会写入 gRPC 的 headers metadata | +### Parameters +| name | desc | +|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `WriteRuest req` | Write request is a written Point collection, where Point is a data point that supports multiple values, allowing data points from different tables to be written together | +| `Context ctx` | Call context, to achieve some special requirements, the content in ctx will be written into the headers metadata of gRPC | -### 返回值说明 -CompletableFuture>: 返回结果一个 future，因为 write 是纯异步 API，整个链路上没有一处是阻塞的。 +### Return -Result: Result 的灵感来自 Rust 中的 Result，其中 WriteOk 和 Err 同时只能一个有值，由于 Write API 还会根据路由表拆分请求，所以 Result 必须有合并的能力。 -WriteOk: 写入成功的结果展示，包含 success(成功条数) 和 failed(失败条数) 以及写入的 metrics 列表，多个 WriteOk 支持合并 -Err: 写入失败结果展示，包含错误状态码、错误文本信息、抛错的服务器地址，同样支持多个 Err 合并 +`CompletableFuture>` +- Return a future, because write is a pure asynchronous API, and no part of the entire link is blocked -### Rows 结构 -| rows | tag1 | tag2 | tag3 | timestamp | field1 | field2 | -| --- | --- | --- | --- | --- | --- | --- | -|0| a1 | a2 | a3 | 2021-07-15 18:00:00 | fa1 | fa2 | -|0| a1 | a2 | a3 | 2021-07-15 18:00:01 | fb1 | fb2 | -|0| a1 | a2 | a3 | 2021-07-15 18:00:02 | fc1 | fc2 | -|1| d1 | d2 | d3 | 2021-07-15 18:00:02 | fe1 | fe2 | +`Result` +- Result is inspired by Result in Rust, where QueryOk and Err can only have a value at the same time +- Since the Write API also splits the request according to the router, the `Result` must have the ability to merge -如上表格，4 行数据都是相同 metric，其中前 3 条数据是相同时间线 `metric-a1-a2-a3` 所以前 3 条是一个 Rows，第 4 条数据是另一个 Rows +`WriteOk` +- The display of successful writing results, including success (number of successful points) and failed (number of failed points) and a list of written tables +- Multiple WriteOk supports merging -### 如果构建一个 Rows? -```java -// -final long time = System.currentTimeMillis() - 1; -final Rows rows = Series.newBuilder(metric) // 指定 metric - .tag("tag1", "tag_v1") // tagK1 = tagV1 - .tag("tag2", "tag_v2") // tagK2 = tagV2 - .toRowsBuilder() // 时间线已经确定，下面转为 Rows 的 Builder - .field(time, "field1", FieldValue.withDouble(0.1)) // 添加一个时间点为 `time` 的 field1 - .field(time, "field2", FieldValue.withString("string_value")) // 添加一个时间点为 `time` 的 field2，与上面的 filed1 是同一行数据 - .field(time + 1, "field1", FieldValue.withDouble(0.2)) // 添加一个时间点为 `time + 1` 的 field1，这是第二行数据了 - .field(time + 1, "field2", FieldValue.withString("string_value_2")) // 添加一个时间点为 `time + 1` 的 field2，与上面的 filed1 是同一行数据 - .build(); -``` -以下代码也具备相同效果 +`Err` +- The result of the query failure is displayed, including the error status code, error text information, and the address of the server where the error was thrown + +### How to build `Point`? ```java final long time = System.currentTimeMillis() - 1; -final Rows rows = Series.newBuilder(metric) // 指定 metric - .tag("tag1", "tag_v1") // tagK1 = tagV1 - .tag("tag2", "tag_v2") // tagK2 = tagV2 - .toRowsBuilder() // 时间线已经确定，下面转为 Rows 的 Builder - .fields(time, input -> { // 一次写一个时间点的一整行数据 - input.put("field1", FieldValue.withDouble(0.1)); - input.put("field2", FieldValue.withString("string_value")); - }) - .fields(time + 1, input -> { // 一次写一个时间点的一整行数据 - input.put("field1", FieldValue.withDouble(0.2)); - input.put("field2", FieldValue.withString("string_value_2")); - }) - .build(); + +// build single point once +final Point point = Point.newPointBuilder(table) // set table + .setTimestamp(time) // set first point timestamp + .addTag("tag1", "tag_v1") // add point tag + .addTag("tag2", "tag_v2") + .addField("field1", Value.withDouble(0.64)) // add point value + .addField("field2", Value.withString("string_value")) + .build() // complete the building and check ``` + diff --git a/pom.xml b/pom.xml index f1eda01..ff4d913 100644 --- a/pom.xml +++ b/pom.xml @@ -36,7 +36,6 @@ ceresdb-rpc ceresdb-sql ceresdb-sql-javacc - ceresdb-http @@ -57,7 +56,8 @@ - 1.10.2 + 9.0.0 + 1.5.2-5 2.8.9 1.23.0 1.20 @@ -74,10 +74,10 @@ 4.9.1 2.8.0 UTF-8 - 0.1.0 + 0.3.0 3.21.7 - 0.2.0-alpha + 1.0.0-alpha 1.7.21 @@ -93,11 +93,6 @@ ceresdb-grpc ${project.version} - - ${project.groupId} - ceresdb-http - ${project.version} - ${project.groupId} ceresdb-protocol @@ -136,9 +131,21 @@ - org.apache.avro - avro - ${arvo.version} + com.github.luben + zstd-jni + ${zstd.version} + + + + org.apache.arrow + arrow-vector + ${arrow.version} + + + + org.apache.arrow + arrow-memory-unsafe + ${arrow.version} @@ -328,6 +335,7 @@ **/sql/SimpleCharStream.java **/sql/Token.java **/sql/TokenMgrException.java + **/ReadmeTest.java diff --git a/tools/codestyle/HEADER b/tools/codestyle/HEADER index 0ed96c4..a99399c 100644 --- a/tools/codestyle/HEADER +++ b/tools/codestyle/HEADER @@ -1,14 +1 @@ -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file +Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0. \ No newline at end of file