From fe4132a928583fd8e2c13fb51b89d2a02fa8a9fa Mon Sep 17 00:00:00 2001 From: moonsphere Date: Thu, 29 Oct 2020 20:24:30 +0800 Subject: [PATCH 01/12] fix transaction too large error when use TiDB as storage --- .../jdbc/hikaricp/JDBCHikariCPClient.java | 6 +- .../jdbc/h2/dao/H2HistoryDeleteDAO.java | 2 +- .../jdbc/mysql/TiDBHistoryDeleteDAO.java | 56 +++++++++++++++++++ .../jdbc/mysql/TiDBStorageProvider.java | 25 +++++++++ ...g.oap.server.library.module.ModuleProvider | 3 +- 5 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBHistoryDeleteDAO.java create mode 100644 oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBStorageProvider.java diff --git a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/jdbc/hikaricp/JDBCHikariCPClient.java b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/jdbc/hikaricp/JDBCHikariCPClient.java index 502eed3847cf..03bf6ac36e51 100644 --- a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/jdbc/hikaricp/JDBCHikariCPClient.java +++ b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/jdbc/hikaricp/JDBCHikariCPClient.java @@ -90,14 +90,14 @@ public void execute(Connection connection, String sql) throws JDBCClientExceptio } } - public boolean execute(Connection connection, String sql, Object... params) throws JDBCClientException { + public int executeUpdate(Connection connection, String sql, Object... params) throws JDBCClientException { LOGGER.debug("execute query with result: {}", sql); - boolean result; + int result; PreparedStatement statement = null; try { statement = connection.prepareStatement(sql); setStatementParam(statement, params); - result = statement.execute(); + result = statement.executeUpdate(); statement.closeOnCompletion(); healthChecker.health(); } catch (SQLException e) { diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java index 31ffe5032581..4c8510d6a4c8 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java @@ -66,7 +66,7 @@ public void deleteHistory(Model model, String timeBucketColumnName, int ttl) thr return; } } - client.execute(connection, dataDeleteSQL.toString(), deadline, minTimeBucket); + client.executeUpdate(connection, dataDeleteSQL.toString(), deadline, minTimeBucket); } catch (JDBCClientException | SQLException e) { throw new IOException(e.getMessage(), e); } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBHistoryDeleteDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBHistoryDeleteDAO.java new file mode 100644 index 000000000000..eac392b9ef9b --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBHistoryDeleteDAO.java @@ -0,0 +1,56 @@ +package org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql; + +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.library.client.jdbc.JDBCClientException; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariCPClient; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.SQLBuilder; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2HistoryDeleteDAO; +import org.joda.time.DateTime; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; + +public class TiDBHistoryDeleteDAO extends H2HistoryDeleteDAO { + + public TiDBHistoryDeleteDAO(JDBCHikariCPClient client) { + super(client); + } + + @Override + public void deleteHistory(Model model, String timeBucketColumnName, int ttl) throws IOException { + SQLBuilder dataDeleteSQL = new SQLBuilder("delete from " + model.getName() + " where ") + .append(timeBucketColumnName).append("<= ? and ") + .append(timeBucketColumnName).append(">= ?") + .append(" limit 10000"); + long minTimeBucket = 0; + DateTime minDate = new DateTime(1900, 1, 1, 0, 0); + + try (Connection connection = client.getConnection()) { + long deadline; + if (model.isRecord()) { + deadline = Long.valueOf(new DateTime().plusDays(0 - ttl).toString("yyyyMMddHHmmss")); + } else { + switch (model.getDownsampling()) { + case Minute: + deadline = Long.valueOf(new DateTime().plusDays(0 - ttl).toString("yyyyMMddHHmm")); + minTimeBucket = Long.valueOf(minDate.toString("yyyyMMddHHmm")); + break; + case Hour: + deadline = Long.valueOf(new DateTime().plusDays(0 - ttl).toString("yyyyMMddHH")); + minTimeBucket = Long.valueOf(minDate.toString("yyyyMMddHH")); + break; + case Day: + deadline = Long.valueOf(new DateTime().plusDays(0 - ttl).toString("yyyyMMdd")); + minTimeBucket = Long.valueOf(minDate.toString("yyyyMMdd")); + break; + default: + return; + } + } + while(client.executeUpdate(connection, dataDeleteSQL.toString(), deadline, minTimeBucket) > 0) {} + } catch (JDBCClientException | SQLException e) { + throw new IOException(e.getMessage(), e); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBStorageProvider.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBStorageProvider.java new file mode 100644 index 000000000000..89eaa1738f56 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBStorageProvider.java @@ -0,0 +1,25 @@ +package org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql; + +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO; +import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; + +/** + * TiDB storage enhanced and came from MySQLStorageProvider to support TiDB. + * + * cause: need add "useAffectedRows=true" to jdbc url. + */ +@Slf4j +public class TiDBStorageProvider extends MySQLStorageProvider { + + @Override + public String name() { + return "tidb"; + } + + @Override + public void prepare() throws ServiceNotProvidedException { + super.prepare(); + this.registerServiceImplementation(IHistoryDeleteDAO.class, new TiDBHistoryDeleteDAO(this.mysqlClient)); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider index c8fa0b9f81a8..d9b7df5f7276 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider @@ -17,4 +17,5 @@ # org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.H2StorageProvider -org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLStorageProvider \ No newline at end of file +org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLStorageProvider +org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.TiDBStorageProvider \ No newline at end of file From 2953a069a2d1409eff001eb8aef3d2f8ff092600 Mon Sep 17 00:00:00 2001 From: moonsphere Date: Thu, 29 Oct 2020 20:46:22 +0800 Subject: [PATCH 02/12] add CHANGES.md --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index e9a0e0addb21..8e5b58117408 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,7 @@ Release Notes. #### OAP-Backend * Add the `@SuperDataset` annotation for BrowserErrorLog. * Improve Kubernetes service registry for ALS analysis. +* Fix "transaction too large error" when use TiDB as storage. #### UI From 4f3e3db27673b2add4c3c612892474f66a6e6e72 Mon Sep 17 00:00:00 2001 From: moonsphere Date: Thu, 29 Oct 2020 21:08:50 +0800 Subject: [PATCH 03/12] fix typo --- .../server/storage/plugin/jdbc/mysql/TiDBStorageProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBStorageProvider.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBStorageProvider.java index 89eaa1738f56..2ac2dcef75d5 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBStorageProvider.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBStorageProvider.java @@ -7,7 +7,7 @@ /** * TiDB storage enhanced and came from MySQLStorageProvider to support TiDB. * - * cause: need add "useAffectedRows=true" to jdbc url. + * caution: need add "useAffectedRows=true" to jdbc url. */ @Slf4j public class TiDBStorageProvider extends MySQLStorageProvider { From 7187c7e54451547f24f82c4a555fb7b9db48765c Mon Sep 17 00:00:00 2001 From: JaredTan95 Date: Sat, 14 Nov 2020 20:37:25 +0800 Subject: [PATCH 04/12] add tidb storage provider module. --- docs/en/setup/backend/backend-storage.md | 15 +- oap-server/server-bootstrap/pom.xml | 5 + .../src/main/resources/application.yml | 13 ++ oap-server/server-storage-plugin/pom.xml | 1 + .../plugin/jdbc/mysql/MySQLStorageConfig.java | 2 +- .../jdbc/mysql/TiDBHistoryDeleteDAO.java | 56 ------ .../jdbc/mysql/TiDBStorageProvider.java | 25 --- ...g.oap.server.library.module.ModuleProvider | 3 +- .../storage-tidb-plugin/pom.xml | 41 ++++ .../jdbc/tidb/TiDBHistoryDeleteDAO.java | 76 ++++++++ .../plugin/jdbc/tidb/TiDBStorageConfig.java | 28 +++ .../plugin/jdbc/tidb/TiDBStorageProvider.java | 177 ++++++++++++++++++ ...g.oap.server.library.module.ModuleProvider | 19 ++ 13 files changed, 371 insertions(+), 90 deletions(-) delete mode 100644 oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBHistoryDeleteDAO.java delete mode 100644 oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBStorageProvider.java create mode 100644 oap-server/server-storage-plugin/storage-tidb-plugin/pom.xml create mode 100644 oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBHistoryDeleteDAO.java create mode 100644 oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBStorageConfig.java create mode 100644 oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBStorageProvider.java create mode 100644 oap-server/server-storage-plugin/storage-tidb-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider diff --git a/docs/en/setup/backend/backend-storage.md b/docs/en/setup/backend/backend-storage.md index a189afc56570..70297bef1283 100644 --- a/docs/en/setup/backend/backend-storage.md +++ b/docs/en/setup/backend/backend-storage.md @@ -223,22 +223,25 @@ All connection related settings including link url, username and password are in Here are some of the settings, please follow [HikariCP](https://github.com/brettwooldridge/HikariCP) connection pool document for all the settings. ## TiDB -Currently tested TiDB in version 2.0.9, and Mysql Client driver in version 8.0.13. -Active TiDB as storage, set storage provider to **mysql**. +Tested TiDB Server 4.0.8 version and Mysql Client driver 8.0.13 version currently. +Active TiDB as storage, set storage provider to **tidb**. ```yaml storage: - selector: ${SW_STORAGE:mysql} - mysql: + selector: ${SW_STORAGE:tidb} + tidb: properties: - jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:3306/swtest"} + jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:4000/swtest"} dataSource.user: ${SW_DATA_SOURCE_USER:root} - dataSource.password: ${SW_DATA_SOURCE_PASSWORD:root@1234} + dataSource.password: ${SW_DATA_SOURCE_PASSWORD:""} dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true} dataSource.prepStmtCacheSize: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250} dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048} dataSource.useServerPrepStmts: ${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true} + dataSource.useAffectedRows: ${SW_DATA_SOURCE_USE_AFFECTED_ROWS:true} metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000} + maxSizeOfArrayColumn: ${SW_STORAGE_MAX_SIZE_OF_ARRAY_COLUMN:20} + numOfSearchableValuesPerTag: ${SW_STORAGE_NUM_OF_SEARCHABLE_VALUES_PER_TAG:2} ``` All connection related settings including link url, username and password are in `application.yml`. These settings can refer to the configuration of *MySQL* above. diff --git a/oap-server/server-bootstrap/pom.xml b/oap-server/server-bootstrap/pom.xml index 954426b4e148..6c8fb3abacc2 100644 --- a/oap-server/server-bootstrap/pom.xml +++ b/oap-server/server-bootstrap/pom.xml @@ -157,6 +157,11 @@ storage-influxdb-plugin ${project.version} + + org.apache.skywalking + storage-tidb-plugin + ${project.version} + diff --git a/oap-server/server-bootstrap/src/main/resources/application.yml b/oap-server/server-bootstrap/src/main/resources/application.yml index 5cc427ea2cc6..050e3ee270ee 100755 --- a/oap-server/server-bootstrap/src/main/resources/application.yml +++ b/oap-server/server-bootstrap/src/main/resources/application.yml @@ -171,6 +171,19 @@ storage: metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000} maxSizeOfArrayColumn: ${SW_STORAGE_MAX_SIZE_OF_ARRAY_COLUMN:20} numOfSearchableValuesPerTag: ${SW_STORAGE_NUM_OF_SEARCHABLE_VALUES_PER_TAG:2} + tidb: + properties: + jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:4000/tidbswtest"} + dataSource.user: ${SW_DATA_SOURCE_USER:root} + dataSource.password: ${SW_DATA_SOURCE_PASSWORD:""} + dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true} + dataSource.prepStmtCacheSize: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250} + dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048} + dataSource.useServerPrepStmts: ${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true} + dataSource.useAffectedRows: ${SW_DATA_SOURCE_USE_AFFECTED_ROWS:true} + metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000} + maxSizeOfArrayColumn: ${SW_STORAGE_MAX_SIZE_OF_ARRAY_COLUMN:20} + numOfSearchableValuesPerTag: ${SW_STORAGE_NUM_OF_SEARCHABLE_VALUES_PER_TAG:2} influxdb: # InfluxDB configuration url: ${SW_STORAGE_INFLUXDB_URL:http://localhost:8086} diff --git a/oap-server/server-storage-plugin/pom.xml b/oap-server/server-storage-plugin/pom.xml index 1c2c38b0642f..16b7d1d5ba8a 100644 --- a/oap-server/server-storage-plugin/pom.xml +++ b/oap-server/server-storage-plugin/pom.xml @@ -34,6 +34,7 @@ storage-zipkin-plugin storage-jaeger-plugin storage-influxdb-plugin + storage-tidb-plugin \ No newline at end of file diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageConfig.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageConfig.java index eb9b1ae98109..bf0cccb69425 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageConfig.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageConfig.java @@ -25,7 +25,7 @@ @Setter @Getter -public final class MySQLStorageConfig extends ModuleConfig { +public class MySQLStorageConfig extends ModuleConfig { private int metadataQueryMaxSize = 5000; /** * Inherit from {@link org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.H2StorageConfig#getMaxSizeOfArrayColumn()} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBHistoryDeleteDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBHistoryDeleteDAO.java deleted file mode 100644 index eac392b9ef9b..000000000000 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBHistoryDeleteDAO.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql; - -import org.apache.skywalking.oap.server.core.storage.model.Model; -import org.apache.skywalking.oap.server.library.client.jdbc.JDBCClientException; -import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariCPClient; -import org.apache.skywalking.oap.server.storage.plugin.jdbc.SQLBuilder; -import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2HistoryDeleteDAO; -import org.joda.time.DateTime; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.SQLException; - -public class TiDBHistoryDeleteDAO extends H2HistoryDeleteDAO { - - public TiDBHistoryDeleteDAO(JDBCHikariCPClient client) { - super(client); - } - - @Override - public void deleteHistory(Model model, String timeBucketColumnName, int ttl) throws IOException { - SQLBuilder dataDeleteSQL = new SQLBuilder("delete from " + model.getName() + " where ") - .append(timeBucketColumnName).append("<= ? and ") - .append(timeBucketColumnName).append(">= ?") - .append(" limit 10000"); - long minTimeBucket = 0; - DateTime minDate = new DateTime(1900, 1, 1, 0, 0); - - try (Connection connection = client.getConnection()) { - long deadline; - if (model.isRecord()) { - deadline = Long.valueOf(new DateTime().plusDays(0 - ttl).toString("yyyyMMddHHmmss")); - } else { - switch (model.getDownsampling()) { - case Minute: - deadline = Long.valueOf(new DateTime().plusDays(0 - ttl).toString("yyyyMMddHHmm")); - minTimeBucket = Long.valueOf(minDate.toString("yyyyMMddHHmm")); - break; - case Hour: - deadline = Long.valueOf(new DateTime().plusDays(0 - ttl).toString("yyyyMMddHH")); - minTimeBucket = Long.valueOf(minDate.toString("yyyyMMddHH")); - break; - case Day: - deadline = Long.valueOf(new DateTime().plusDays(0 - ttl).toString("yyyyMMdd")); - minTimeBucket = Long.valueOf(minDate.toString("yyyyMMdd")); - break; - default: - return; - } - } - while(client.executeUpdate(connection, dataDeleteSQL.toString(), deadline, minTimeBucket) > 0) {} - } catch (JDBCClientException | SQLException e) { - throw new IOException(e.getMessage(), e); - } - } -} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBStorageProvider.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBStorageProvider.java deleted file mode 100644 index 2ac2dcef75d5..000000000000 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBStorageProvider.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql; - -import lombok.extern.slf4j.Slf4j; -import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO; -import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; - -/** - * TiDB storage enhanced and came from MySQLStorageProvider to support TiDB. - * - * caution: need add "useAffectedRows=true" to jdbc url. - */ -@Slf4j -public class TiDBStorageProvider extends MySQLStorageProvider { - - @Override - public String name() { - return "tidb"; - } - - @Override - public void prepare() throws ServiceNotProvidedException { - super.prepare(); - this.registerServiceImplementation(IHistoryDeleteDAO.class, new TiDBHistoryDeleteDAO(this.mysqlClient)); - } -} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider index d9b7df5f7276..c8fa0b9f81a8 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider @@ -17,5 +17,4 @@ # org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.H2StorageProvider -org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLStorageProvider -org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.TiDBStorageProvider \ No newline at end of file +org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLStorageProvider \ No newline at end of file diff --git a/oap-server/server-storage-plugin/storage-tidb-plugin/pom.xml b/oap-server/server-storage-plugin/storage-tidb-plugin/pom.xml new file mode 100644 index 000000000000..e9893858dac3 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-tidb-plugin/pom.xml @@ -0,0 +1,41 @@ + + + + server-storage-plugin + org.apache.skywalking + 8.3.0-SNAPSHOT + + 4.0.0 + + storage-tidb-plugin + jar + + + + + org.apache.skywalking + server-core + ${project.version} + + + + org.apache.skywalking + library-client + ${project.version} + + + + org.apache.skywalking + storage-jdbc-hikaricp-plugin + ${project.version} + + + + + \ No newline at end of file diff --git a/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBHistoryDeleteDAO.java b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBHistoryDeleteDAO.java new file mode 100644 index 000000000000..eae6cff3feb7 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBHistoryDeleteDAO.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.tidb; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.library.client.jdbc.JDBCClientException; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariCPClient; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.SQLBuilder; +import org.joda.time.DateTime; + +public class TiDBHistoryDeleteDAO implements IHistoryDeleteDAO { + + private final JDBCHikariCPClient client; + + public TiDBHistoryDeleteDAO(JDBCHikariCPClient client) { + this.client = client; + } + + @Override + public void deleteHistory(Model model, String timeBucketColumnName, int ttl) throws IOException { + SQLBuilder dataDeleteSQL = new SQLBuilder("delete from " + model.getName() + " where ") + .append(timeBucketColumnName).append("<= ? and ") + .append(timeBucketColumnName).append(">= ?") + .append(" limit 10000"); + long minTimeBucket = 0; + DateTime minDate = new DateTime(1900, 1, 1, 0, 0); + + try (Connection connection = client.getConnection()) { + long deadline; + if (model.isRecord()) { + deadline = Long.parseLong(new DateTime().plusDays(0 - ttl).toString("yyyyMMddHHmmss")); + } else { + switch (model.getDownsampling()) { + case Minute: + deadline = Long.parseLong(new DateTime().plusDays(0 - ttl).toString("yyyyMMddHHmm")); + minTimeBucket = Long.parseLong(minDate.toString("yyyyMMddHHmm")); + break; + case Hour: + deadline = Long.parseLong(new DateTime().plusDays(0 - ttl).toString("yyyyMMddHH")); + minTimeBucket = Long.parseLong(minDate.toString("yyyyMMddHH")); + break; + case Day: + deadline = Long.parseLong(new DateTime().plusDays(0 - ttl).toString("yyyyMMdd")); + minTimeBucket = Long.parseLong(minDate.toString("yyyyMMdd")); + break; + default: + return; + } + } + while (client.executeUpdate(connection, dataDeleteSQL.toString(), deadline, minTimeBucket) > 0) { + } + } catch (JDBCClientException | SQLException e) { + throw new IOException(e.getMessage(), e); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBStorageConfig.java b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBStorageConfig.java new file mode 100644 index 000000000000..197187c44fec --- /dev/null +++ b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBStorageConfig.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.tidb; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLStorageConfig; + +@Setter +@Getter +public class TiDBStorageConfig extends MySQLStorageConfig { +} diff --git a/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBStorageProvider.java b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBStorageProvider.java new file mode 100644 index 000000000000..30d4fa857187 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBStorageProvider.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.tidb; + +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.config.ConfigService; +import org.apache.skywalking.oap.server.core.storage.IBatchDAO; +import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO; +import org.apache.skywalking.oap.server.core.storage.StorageDAO; +import org.apache.skywalking.oap.server.core.storage.StorageException; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.cache.INetworkAddressAliasDAO; +import org.apache.skywalking.oap.server.core.storage.management.UITemplateManagementDAO; +import org.apache.skywalking.oap.server.core.storage.model.ModelCreator; +import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profile.IProfileThreadSnapshotQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IAggregationQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IBrowserLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.ITopNRecordsQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.ITopologyQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.ITraceQueryDAO; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariCPClient; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; +import org.apache.skywalking.oap.server.library.module.ModuleDefine; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2BatchDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2HistoryDeleteDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2MetadataQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2MetricsQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2NetworkAddressAliasDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2ProfileTaskLogQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2ProfileTaskQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2ProfileThreadSnapshotQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2StorageDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2TopNRecordsQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2TopologyQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2UITemplateManagementDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLAggregationQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLAlarmQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLLogQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLTableInstaller; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLTraceQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MysqlBrowserLogQueryDAO; + +/** + * TiDB storage enhanced and came from MySQLStorageProvider to support TiDB. + * + * caution: need add "useAffectedRows=true" to jdbc url. + */ +@Slf4j +public class TiDBStorageProvider extends ModuleProvider { + + private TiDBStorageConfig config; + private JDBCHikariCPClient mysqlClient; + + public TiDBStorageProvider() { + config = new TiDBStorageConfig(); + } + + @Override + public String name() { + return "tidb"; + } + + @Override + public Class module() { + return StorageModule.class; + } + + @Override + public ModuleConfig createConfigBeanIfAbsent() { + return config; + } + + @Override + public void prepare() throws ServiceNotProvidedException { + mysqlClient = new JDBCHikariCPClient(config.getProperties()); + + this.registerServiceImplementation(IBatchDAO.class, new H2BatchDAO(mysqlClient)); + this.registerServiceImplementation( + StorageDAO.class, + new H2StorageDAO( + getManager(), mysqlClient, config.getMaxSizeOfArrayColumn(), config.getNumOfSearchableValuesPerTag()) + ); + this.registerServiceImplementation( + INetworkAddressAliasDAO.class, new H2NetworkAddressAliasDAO(mysqlClient)); + + this.registerServiceImplementation(ITopologyQueryDAO.class, new H2TopologyQueryDAO(mysqlClient)); + this.registerServiceImplementation(IMetricsQueryDAO.class, new H2MetricsQueryDAO(mysqlClient)); + this.registerServiceImplementation( + ITraceQueryDAO.class, + new MySQLTraceQueryDAO( + getManager(), + mysqlClient, + config.getMaxSizeOfArrayColumn(), + config.getNumOfSearchableValuesPerTag() + ) + ); + this.registerServiceImplementation(IBrowserLogQueryDAO.class, new MysqlBrowserLogQueryDAO(mysqlClient)); + this.registerServiceImplementation( + IMetadataQueryDAO.class, new H2MetadataQueryDAO(mysqlClient, config.getMetadataQueryMaxSize())); + this.registerServiceImplementation(IAggregationQueryDAO.class, new MySQLAggregationQueryDAO(mysqlClient)); + this.registerServiceImplementation(IAlarmQueryDAO.class, new MySQLAlarmQueryDAO(mysqlClient)); + this.registerServiceImplementation( + IHistoryDeleteDAO.class, new H2HistoryDeleteDAO(mysqlClient)); + this.registerServiceImplementation(ITopNRecordsQueryDAO.class, new H2TopNRecordsQueryDAO(mysqlClient)); + this.registerServiceImplementation(ILogQueryDAO.class, new MySQLLogQueryDAO(mysqlClient)); + + this.registerServiceImplementation(IProfileTaskQueryDAO.class, new H2ProfileTaskQueryDAO(mysqlClient)); + this.registerServiceImplementation(IProfileTaskLogQueryDAO.class, new H2ProfileTaskLogQueryDAO(mysqlClient)); + this.registerServiceImplementation( + IProfileThreadSnapshotQueryDAO.class, new H2ProfileThreadSnapshotQueryDAO(mysqlClient)); + this.registerServiceImplementation(UITemplateManagementDAO.class, new H2UITemplateManagementDAO(mysqlClient)); + + this.registerServiceImplementation(IHistoryDeleteDAO.class, new TiDBHistoryDeleteDAO(mysqlClient)); + } + + @Override + public void start() throws ServiceNotProvidedException, ModuleStartException { + final ConfigService configService = getManager().find(CoreModule.NAME) + .provider() + .getService(ConfigService.class); + final int numOfSearchableTags = configService.getSearchableTracesTags().split(Const.COMMA).length; + if (numOfSearchableTags * config.getNumOfSearchableValuesPerTag() > config.getMaxSizeOfArrayColumn()) { + throw new ModuleStartException("Size of searchableTracesTags[" + numOfSearchableTags + + "] * numOfSearchableValuesPerTag[" + config.getNumOfSearchableValuesPerTag() + + "] > maxSizeOfArrayColumn[" + config.getMaxSizeOfArrayColumn() + + "]. Potential out of bound in the runtime."); + } + + try { + mysqlClient.connect(); + + MySQLTableInstaller installer = new MySQLTableInstaller( + mysqlClient, getManager(), config.getMaxSizeOfArrayColumn(), config.getNumOfSearchableValuesPerTag() + ); + getManager().find(CoreModule.NAME).provider().getService(ModelCreator.class).addModelListener(installer); + } catch (StorageException e) { + throw new ModuleStartException(e.getMessage(), e); + } + } + + @Override + public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException { + + } + + @Override + public String[] requiredModules() { + return new String[] {CoreModule.NAME}; + } +} diff --git a/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider new file mode 100644 index 000000000000..cfe7e5f9f81f --- /dev/null +++ b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.storage.plugin.jdbc.tidb.TiDBStorageProvider \ No newline at end of file From c887749781e71370ab85af952500b301b8719622 Mon Sep 17 00:00:00 2001 From: JaredTan95 Date: Sun, 15 Nov 2020 10:32:47 +0800 Subject: [PATCH 05/12] add e2e. --- .github/workflows/e2e.storages.yaml | 2 +- .../storage-tidb-plugin/pom.xml | 18 + .../docker/storage/docker-compose.tidb.yml | 89 ++++ .../docker/storage/tidbconfig/pd.toml | 86 +++ .../docker/storage/tidbconfig/tidb.toml | 239 +++++++++ .../docker/storage/tidbconfig/tikv.toml | 497 ++++++++++++++++++ 6 files changed, 930 insertions(+), 1 deletion(-) create mode 100644 test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml create mode 100644 test/e2e/e2e-test/docker/storage/tidbconfig/pd.toml create mode 100644 test/e2e/e2e-test/docker/storage/tidbconfig/tidb.toml create mode 100644 test/e2e/e2e-test/docker/storage/tidbconfig/tikv.toml diff --git a/.github/workflows/e2e.storages.yaml b/.github/workflows/e2e.storages.yaml index e82aca5329f1..26c63fc05fc9 100644 --- a/.github/workflows/e2e.storages.yaml +++ b/.github/workflows/e2e.storages.yaml @@ -34,7 +34,7 @@ jobs: timeout-minutes: 90 strategy: matrix: - storage: ['mysql', 'es6', 'es7.0', 'es7.9', 'influxdb'] + storage: ['mysql', 'es6', 'es7.0', 'es7.9', 'influxdb','tidb'] env: SW_STORAGE: ${{ matrix.storage }} steps: diff --git a/oap-server/server-storage-plugin/storage-tidb-plugin/pom.xml b/oap-server/server-storage-plugin/storage-tidb-plugin/pom.xml index e9893858dac3..bf7096776c9f 100644 --- a/oap-server/server-storage-plugin/storage-tidb-plugin/pom.xml +++ b/oap-server/server-storage-plugin/storage-tidb-plugin/pom.xml @@ -1,4 +1,22 @@ + + diff --git a/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml b/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml new file mode 100644 index 000000000000..d421167be8d0 --- /dev/null +++ b/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + pd0: + image: pingcap/pd:latest + ports: + - "2379" + volumes: + - ./tidbconfig/pd.toml:/pd.toml:ro + command: + - --name=pd0 + - --client-urls=http://0.0.0.0:2379 + - --peer-urls=http://0.0.0.0:2380 + - --advertise-client-urls=http://pd0:2379 + - --advertise-peer-urls=http://pd0:2380 + - --initial-cluster=pd0=http://pd0:2380 + - --data-dir=/data/pd0 + - --config=/pd.toml + - --log-file=/logs/pd0.log + restart: on-failure + tikv0: + image: pingcap/tikv:latest + volumes: + - ./tidbconfig/tikv.toml:/tikv.toml:ro + command: + - --addr=0.0.0.0:20160 + - --advertise-addr=tikv0:20160 + - --data-dir=/data/tikv0 + - --pd=pd0:2379 + - --config=/tikv.toml + - --log-file=/logs/tikv0.log + depends_on: + - "pd0" + restart: on-failure + + tidb: + image: pingcap/tidb:latest + ports: + - "4000:4000" + - "10080:10080" + volumes: + - ./tidbconfig/tidb.toml:/tidb.toml:ro + command: + - --store=tikv + - --path=pd0:2379 + - --config=/tidb.toml + - --log-file=/logs/tidb.log + - --advertise-address=tidb + depends_on: + - "tikv0" + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/4000 "] + interval: 5s + timeout: 60s + retries: 120 + restart: on-failure + + oap: + extends: + file: ../base-compose.yml + service: oap + environment: + SW_STORAGE: tidb + SW_PROMETHEUS_FETCHER_ACTIVE: "true" + SW_TELEMETRY: prometheus + SW_JDBC_URL: "jdbc:mysql://tidb:4000/INFORMATION_SCHEMA" + SW_DATA_SOURCE_PASSWORD: "" + depends_on: + tidb: + condition: service_healthy + entrypoint: ['sh', '-c', '/download-mysql.sh && /skywalking/docker-entrypoint.sh'] + +networks: + e2e: diff --git a/test/e2e/e2e-test/docker/storage/tidbconfig/pd.toml b/test/e2e/e2e-test/docker/storage/tidbconfig/pd.toml new file mode 100644 index 000000000000..b1562a569e30 --- /dev/null +++ b/test/e2e/e2e-test/docker/storage/tidbconfig/pd.toml @@ -0,0 +1,86 @@ +# PD Configuration. + +name = "pd" +data-dir = "default.pd" + +client-urls = "http://127.0.0.1:2379" +# if not set, use ${client-urls} +advertise-client-urls = "" + +peer-urls = "http://127.0.0.1:2380" +# if not set, use ${peer-urls} +advertise-peer-urls = "" + +initial-cluster = "pd=http://127.0.0.1:2380" +initial-cluster-state = "new" + +lease = 3 +tso-save-interval = "3s" + +[security] +# Path of file that contains list of trusted SSL CAs. if set, following four settings shouldn't be empty +cacert-path = "" +# Path of file that contains X509 certificate in PEM format. +cert-path = "" +# Path of file that contains X509 key in PEM format. +key-path = "" + +[log] +level = "error" + +# log format, one of json, text, console +#format = "text" + +# disable automatic timestamps in output +#disable-timestamp = false + +# file logging +[log.file] +#filename = "" +# max log file size in MB +#max-size = 300 +# max log file keep days +#max-days = 28 +# maximum number of old log files to retain +#max-backups = 7 +# rotate log by day +#log-rotate = true + +[metric] +# prometheus client push interval, set "0s" to disable prometheus. +interval = "15s" +# prometheus pushgateway address, leaves it empty will disable prometheus. +address = "pushgateway:9091" + +[schedule] +max-merge-region-size = 0 +split-merge-interval = "1h" +max-snapshot-count = 3 +max-pending-peer-count = 16 +max-store-down-time = "30m" +leader-schedule-limit = 4 +region-schedule-limit = 4 +replica-schedule-limit = 8 +merge-schedule-limit = 8 +tolerant-size-ratio = 5.0 + +# customized schedulers, the format is as below +# if empty, it will use balance-leader, balance-region, hot-region as default +# [[schedule.schedulers]] +# type = "evict-leader" +# args = ["1"] + +[replication] +# The number of replicas for each region. +max-replicas = 3 +# The label keys specified the location of a store. +# The placement priorities is implied by the order of label keys. +# For example, ["zone", "rack"] means that we should place replicas to +# different zones first, then to different racks if we don't have enough zones. +location-labels = [] + +[label-property] +# Do not assign region leaders to stores that have these tags. +# [[label-property.reject-leader]] +# key = "zone" +# value = "cn1 diff --git a/test/e2e/e2e-test/docker/storage/tidbconfig/tidb.toml b/test/e2e/e2e-test/docker/storage/tidbconfig/tidb.toml new file mode 100644 index 000000000000..9f881b489177 --- /dev/null +++ b/test/e2e/e2e-test/docker/storage/tidbconfig/tidb.toml @@ -0,0 +1,239 @@ +# TiDB Configuration. + +# TiDB server host. +host = "0.0.0.0" + +# TiDB server port. +port = 4000 + +# Registered store name, [tikv, mocktikv] +store = "mocktikv" + +# TiDB storage path. +path = "/tmp/tidb" + +# The socket file to use for connection. +socket = "" + +# Run ddl worker on this tidb-server. +run-ddl = true + +# Schema lease duration, very dangerous to change only if you know what you do. +lease = "0" + +# When create table, split a separated region for it. It is recommended to +# turn off this option if there will be a large number of tables created. +split-table = true + +# The limit of concurrent executed sessions. +token-limit = 1000 + +# Only print a log when out of memory quota. +# Valid options: ["log", "cancel"] +oom-action = "log" + +# Set the memory quota for a query in bytes. Default: 32GB +mem-quota-query = 34359738368 + +# Enable coprocessor streaming. +enable-streaming = false + +# Set system variable 'lower_case_table_names' +lower-case-table-names = 2 + +[log] +# Log level: debug, info, warn, error, fatal. +level = "error" + +# Log format, one of json, text, console. +format = "text" + +# Disable automatic timestamp in output +disable-timestamp = false + +# Stores slow query log into separated files. +slow-query-file = "" + +# Queries with execution time greater than this value will be logged. (Milliseconds) +slow-threshold = 300 + +# Queries with internal result greater than this value will be logged. +expensive-threshold = 10000 + +# Maximum query length recorded in log. +query-log-max-len = 2048 + +# File logging. +[log.file] +# Log file name. +filename = "" + +# Max log file size in MB (upper limit to 4096MB). +max-size = 300 + +# Max log file keep days. No clean up by default. +max-days = 0 + +# Maximum number of old log files to retain. No clean up by default. +max-backups = 0 + +# Rotate log by day +log-rotate = true + +[security] +# Path of file that contains list of trusted SSL CAs for connection with mysql client. +ssl-ca = "" + +# Path of file that contains X509 certificate in PEM format for connection with mysql client. +ssl-cert = "" + +# Path of file that contains X509 key in PEM format for connection with mysql client. +ssl-key = "" + +# Path of file that contains list of trusted SSL CAs for connection with cluster components. +cluster-ssl-ca = "" + +# Path of file that contains X509 certificate in PEM format for connection with cluster components. +cluster-ssl-cert = "" + +# Path of file that contains X509 key in PEM format for connection with cluster components. +cluster-ssl-key = "" + +[status] +# If enable status report HTTP service. +report-status = true + +# TiDB status port. +status-port = 10080 + +# Prometheus pushgateway address, leaves it empty will disable prometheus push. +metrics-addr = "pushgateway:9091" + +# Prometheus client push interval in second, set \"0\" to disable prometheus push. +metrics-interval = 15 + +[performance] +# Max CPUs to use, 0 use number of CPUs in the machine. +max-procs = 0 +# StmtCountLimit limits the max count of statement inside a transaction. +stmt-count-limit = 5000 + +# Set keep alive option for tcp connection. +tcp-keep-alive = true + +# The maximum number of retries when commit a transaction. +retry-limit = 10 + +# Whether support cartesian product. +cross-join = true + +# Stats lease duration, which influences the time of analyze and stats load. +stats-lease = "3s" + +# Run auto analyze worker on this tidb-server. +run-auto-analyze = true + +# Probability to use the query feedback to update stats, 0 or 1 for always false/true. +feedback-probability = 0.0 + +# The max number of query feedback that cache in memory. +query-feedback-limit = 1024 + +# Pseudo stats will be used if the ratio between the modify count and +# row count in statistics of a table is greater than it. +pseudo-estimate-ratio = 0.7 + +[proxy-protocol] +# PROXY protocol acceptable client networks. +# Empty string means disable PROXY protocol, * means all networks. +networks = "" + +# PROXY protocol header read timeout, unit is second +header-timeout = 5 + +[plan-cache] +enabled = false +capacity = 2560 +shards = 256 + +[prepared-plan-cache] +enabled = false +capacity = 100 + +[opentracing] +# Enable opentracing. +enable = false + +# Whether to enable the rpc metrics. +rpc-metrics = false + +[opentracing.sampler] +# Type specifies the type of the sampler: const, probabilistic, rateLimiting, or remote +type = "const" + +# Param is a value passed to the sampler. +# Valid values for Param field are: +# - for "const" sampler, 0 or 1 for always false/true respectively +# - for "probabilistic" sampler, a probability between 0 and 1 +# - for "rateLimiting" sampler, the number of spans per second +# - for "remote" sampler, param is the same as for "probabilistic" +# and indicates the initial sampling rate before the actual one +# is received from the mothership +param = 1.0 + +# SamplingServerURL is the address of jaeger-agent's HTTP sampling server +sampling-server-url = "" + +# MaxOperations is the maximum number of operations that the sampler +# will keep track of. If an operation is not tracked, a default probabilistic +# sampler will be used rather than the per operation specific sampler. +max-operations = 0 + +# SamplingRefreshInterval controls how often the remotely controlled sampler will poll +# jaeger-agent for the appropriate sampling strategy. +sampling-refresh-interval = 0 + +[opentracing.reporter] +# QueueSize controls how many spans the reporter can keep in memory before it starts dropping +# new spans. The queue is continuously drained by a background go-routine, as fast as spans +# can be sent out of process. +queue-size = 0 + +# BufferFlushInterval controls how often the buffer is force-flushed, even if it's not full. +# It is generally not useful, as it only matters for very low traffic services. +buffer-flush-interval = 0 + +# LogSpans, when true, enables LoggingReporter that runs in parallel with the main reporter +# and logs all submitted spans. Main Configuration.Logger must be initialized in the code +# for this option to have any effect. +log-spans = false + +# LocalAgentHostPort instructs reporter to send spans to jaeger-agent at this address +local-agent-host-port = "" + +[tikv-client] +# Max gRPC connections that will be established with each tikv-server. +grpc-connection-count = 16 + +# After a duration of this time in seconds if the client doesn't see any activity it pings +# the server to see if the transport is still alive. +grpc-keepalive-time = 10 + +# After having pinged for keepalive check, the client waits for a duration of Timeout in seconds +# and if no activity is seen even after that the connection is closed. +grpc-keepalive-timeout = 3 + +# max time for commit command, must be twice bigger than raft election timeout. +commit-timeout = "41s" + +[binlog] + +# Socket file to write binlog. +binlog-socket = "" + +# WriteTimeout specifies how long it will wait for writing binlog to pump. +write-timeout = "15s" + +# If IgnoreError is true, when writting binlog meets error, TiDB would stop writting binlog, +# but still provide service. +ignore-error = false diff --git a/test/e2e/e2e-test/docker/storage/tidbconfig/tikv.toml b/test/e2e/e2e-test/docker/storage/tidbconfig/tikv.toml new file mode 100644 index 000000000000..6606cd9cd91d --- /dev/null +++ b/test/e2e/e2e-test/docker/storage/tidbconfig/tikv.toml @@ -0,0 +1,497 @@ +# TiKV config template +# Human-readable big numbers: +# File size(based on byte): KB, MB, GB, TB, PB +# e.g.: 1_048_576 = "1MB" +# Time(based on ms): ms, s, m, h +# e.g.: 78_000 = "1.3m" + +# log level: trace, debug, info, warn, error, off. +log-level = "error" +# file to store log, write to stderr if it's empty. +# log-file = "" + +[readpool.storage] +# size of thread pool for high-priority operations +# high-concurrency = 4 +# size of thread pool for normal-priority operations +# normal-concurrency = 4 +# size of thread pool for low-priority operations +# low-concurrency = 4 +# max running high-priority operations, reject if exceed +# max-tasks-high = 8000 +# max running normal-priority operations, reject if exceed +# max-tasks-normal = 8000 +# max running low-priority operations, reject if exceed +# max-tasks-low = 8000 +# size of stack size for each thread pool +# stack-size = "10MB" + +[readpool.coprocessor] +# Notice: if CPU_NUM > 8, default thread pool size for coprocessors +# will be set to CPU_NUM * 0.8. + +# high-concurrency = 8 +# normal-concurrency = 8 +# low-concurrency = 8 +# max-tasks-high = 16000 +# max-tasks-normal = 16000 +# max-tasks-low = 16000 +# stack-size = "10MB" + +[server] +# set listening address. +# addr = "127.0.0.1:20160" +# set advertise listening address for client communication, if not set, use addr instead. +# advertise-addr = "" +# notify capacity, 40960 is suitable for about 7000 regions. +# notify-capacity = 40960 +# maximum number of messages can be processed in one tick. +# messages-per-tick = 4096 + +# compression type for grpc channel, available values are no, deflate and gzip. +# grpc-compression-type = "no" +# size of thread pool for grpc server. +# grpc-concurrency = 4 +# The number of max concurrent streams/requests on a client connection. +# grpc-concurrent-stream = 1024 +# The number of connections with each tikv server to send raft messages. +# grpc-raft-conn-num = 10 +# Amount to read ahead on individual grpc streams. +# grpc-stream-initial-window-size = "2MB" + +# How many snapshots can be sent concurrently. +# concurrent-send-snap-limit = 32 +# How many snapshots can be recv concurrently. +# concurrent-recv-snap-limit = 32 + +# max count of tasks being handled, new tasks will be rejected. +# end-point-max-tasks = 2000 + +# max recursion level allowed when decoding dag expression +# end-point-recursion-limit = 1000 + +# max time to handle coprocessor request before timeout +# end-point-request-max-handle-duration = "60s" + +# the max bytes that snapshot can be written to disk in one second, +# should be set based on your disk performance +# snap-max-write-bytes-per-sec = "100MB" + +# set attributes about this server, e.g. { zone = "us-west-1", disk = "ssd" }. +# labels = {} + +[storage] +# set the path to rocksdb directory. +# data-dir = "/tmp/tikv/store" + +# notify capacity of scheduler's channel +# scheduler-notify-capacity = 10240 + +# maximum number of messages can be processed in one tick +# scheduler-messages-per-tick = 1024 + +# the number of slots in scheduler latches, concurrency control for write. +# scheduler-concurrency = 2048000 + +# scheduler's worker pool size, should increase it in heavy write cases, +# also should less than total cpu cores. +# scheduler-worker-pool-size = 4 + +# When the pending write bytes exceeds this threshold, +# the "scheduler too busy" error is displayed. +# scheduler-pending-write-threshold = "100MB" + +[pd] +# pd endpoints +# endpoints = [] + +[metric] +# the Prometheus client push interval. Setting the value to 0s stops Prometheus client from pushing. +# interval = "15s" +# the Prometheus pushgateway address. Leaving it empty stops Prometheus client from pushing. +address = "pushgateway:9091" +# the Prometheus client push job name. Note: A node id will automatically append, e.g., "tikv_1". +# job = "tikv" + +[raftstore] +# true (default value) for high reliability, this can prevent data loss when power failure. +# sync-log = true + +# set the path to raftdb directory, default value is data-dir/raft +# raftdb-path = "" + +# set store capacity, if no set, use disk capacity. +# capacity = 0 + +# notify capacity, 40960 is suitable for about 7000 regions. +# notify-capacity = 40960 + +# maximum number of messages can be processed in one tick. +# messages-per-tick = 4096 + +# Region heartbeat tick interval for reporting to pd. +# pd-heartbeat-tick-interval = "60s" +# Store heartbeat tick interval for reporting to pd. +# pd-store-heartbeat-tick-interval = "10s" + +# When region size changes exceeds region-split-check-diff, we should check +# whether the region should be split or not. +# region-split-check-diff = "6MB" + +# Interval to check region whether need to be split or not. +# split-region-check-tick-interval = "10s" + +# When raft entry exceed the max size, reject to propose the entry. +# raft-entry-max-size = "8MB" + +# Interval to gc unnecessary raft log. +# raft-log-gc-tick-interval = "10s" +# A threshold to gc stale raft log, must >= 1. +# raft-log-gc-threshold = 50 +# When entry count exceed this value, gc will be forced trigger. +# raft-log-gc-count-limit = 72000 +# When the approximate size of raft log entries exceed this value, gc will be forced trigger. +# It's recommanded to set it to 3/4 of region-split-size. +# raft-log-gc-size-limit = "72MB" + +# When a peer hasn't been active for max-peer-down-duration, +# we will consider this peer to be down and report it to pd. +# max-peer-down-duration = "5m" + +# Interval to check whether start manual compaction for a region, +# region-compact-check-interval = "5m" +# Number of regions for each time to check. +# region-compact-check-step = 100 +# The minimum number of delete tombstones to trigger manual compaction. +# region-compact-min-tombstones = 10000 +# Interval to check whether should start a manual compaction for lock column family, +# if written bytes reach lock-cf-compact-threshold for lock column family, will fire +# a manual compaction for lock column family. +# lock-cf-compact-interval = "10m" +# lock-cf-compact-bytes-threshold = "256MB" + +# Interval (s) to check region whether the data are consistent. +# consistency-check-interval = 0 + +# Use delete range to drop a large number of continuous keys. +# use-delete-range = false + +# delay time before deleting a stale peer +# clean-stale-peer-delay = "10m" + +# Interval to cleanup import sst files. +# cleanup-import-sst-interval = "10m" + +[coprocessor] +# When it is true, it will try to split a region with table prefix if +# that region crosses tables. It is recommended to turn off this option +# if there will be a large number of tables created. +# split-region-on-table = true +# When the region's size exceeds region-max-size, we will split the region +# into two which the left region's size will be region-split-size or a little +# bit smaller. +# region-max-size = "144MB" +# region-split-size = "96MB" + +[rocksdb] +# Maximum number of concurrent background jobs (compactions and flushes) +# max-background-jobs = 8 + +# This value represents the maximum number of threads that will concurrently perform a +# compaction job by breaking it into multiple, smaller ones that are run simultaneously. +# Default: 1 (i.e. no subcompactions) +# max-sub-compactions = 1 + +# Number of open files that can be used by the DB. You may need to +# increase this if your database has a large working set. Value -1 means +# files opened are always kept open. You can estimate number of files based +# on target_file_size_base and target_file_size_multiplier for level-based +# compaction. +# If max-open-files = -1, RocksDB will prefetch index and filter blocks into +# block cache at startup, so if your database has a large working set, it will +# take several minutes to open the db. +max-open-files = 1024 + +# Max size of rocksdb's MANIFEST file. +# For detailed explanation please refer to https://github.com/facebook/rocksdb/wiki/MANIFEST +# max-manifest-file-size = "20MB" + +# If true, the database will be created if it is missing. +# create-if-missing = true + +# rocksdb wal recovery mode +# 0 : TolerateCorruptedTailRecords, tolerate incomplete record in trailing data on all logs; +# 1 : AbsoluteConsistency, We don't expect to find any corruption in the WAL; +# 2 : PointInTimeRecovery, Recover to point-in-time consistency; +# 3 : SkipAnyCorruptedRecords, Recovery after a disaster; +# wal-recovery-mode = 2 + +# rocksdb write-ahead logs dir path +# This specifies the absolute dir path for write-ahead logs (WAL). +# If it is empty, the log files will be in the same dir as data. +# When you set the path to rocksdb directory in memory like in /dev/shm, you may want to set +# wal-dir to a directory on a persistent storage. +# See https://github.com/facebook/rocksdb/wiki/How-to-persist-in-memory-RocksDB-database +# wal-dir = "/tmp/tikv/store" + +# The following two fields affect how archived write-ahead logs will be deleted. +# 1. If both set to 0, logs will be deleted asap and will not get into the archive. +# 2. If wal-ttl-seconds is 0 and wal-size-limit is not 0, +# WAL files will be checked every 10 min and if total size is greater +# then wal-size-limit, they will be deleted starting with the +# earliest until size_limit is met. All empty files will be deleted. +# 3. If wal-ttl-seconds is not 0 and wal-size-limit is 0, then +# WAL files will be checked every wal-ttl-seconds / 2 and those that +# are older than wal-ttl-seconds will be deleted. +# 4. If both are not 0, WAL files will be checked every 10 min and both +# checks will be performed with ttl being first. +# When you set the path to rocksdb directory in memory like in /dev/shm, you may want to set +# wal-ttl-seconds to a value greater than 0 (like 86400) and backup your db on a regular basis. +# See https://github.com/facebook/rocksdb/wiki/How-to-persist-in-memory-RocksDB-database +# wal-ttl-seconds = 0 +# wal-size-limit = 0 + +# rocksdb max total wal size +# max-total-wal-size = "4GB" + +# Rocksdb Statistics provides cumulative stats over time. +# Turn statistics on will introduce about 5%-10% overhead for RocksDB, +# but it is worthy to know the internal status of RocksDB. +# enable-statistics = true + +# Dump statistics periodically in information logs. +# Same as rocksdb's default value (10 min). +# stats-dump-period = "10m" + +# Due to Rocksdb FAQ: https://github.com/facebook/rocksdb/wiki/RocksDB-FAQ, +# If you want to use rocksdb on multi disks or spinning disks, you should set value at +# least 2MB; +# compaction-readahead-size = 0 + +# This is the maximum buffer size that is used by WritableFileWrite +# writable-file-max-buffer-size = "1MB" + +# Use O_DIRECT for both reads and writes in background flush and compactions +# use-direct-io-for-flush-and-compaction = false + +# Limit the disk IO of compaction and flush. Compaction and flush can cause +# terrible spikes if they exceed a certain threshold. Consider setting this to +# 50% ~ 80% of the disk throughput for a more stable result. However, in heavy +# write workload, limiting compaction and flush speed can cause write stalls too. +# rate-bytes-per-sec = 0 + +# Enable or disable the pipelined write +# enable-pipelined-write = true + +# Allows OS to incrementally sync files to disk while they are being +# written, asynchronously, in the background. +# bytes-per-sync = "0MB" + +# Allows OS to incrementally sync WAL to disk while it is being written. +# wal-bytes-per-sync = "0KB" + +# Specify the maximal size of the Rocksdb info log file. If the log file +# is larger than `max_log_file_size`, a new info log file will be created. +# If max_log_file_size == 0, all logs will be written to one log file. +# Default: 1GB +# info-log-max-size = "1GB" + +# Time for the Rocksdb info log file to roll (in seconds). +# If specified with non-zero value, log file will be rolled +# if it has been active longer than `log_file_time_to_roll`. +# Default: 0 (disabled) +# info-log-roll-time = "0" + +# Maximal Rocksdb info log files to be kept. +# Default: 10 +# info-log-keep-log-file-num = 10 + +# This specifies the Rocksdb info LOG dir. +# If it is empty, the log files will be in the same dir as data. +# If it is non empty, the log files will be in the specified dir, +# and the db data dir's absolute path will be used as the log file +# name's prefix. +# Default: empty +# info-log-dir = "" + +# Column Family default used to store actual data of the database. +[rocksdb.defaultcf] +# compression method (if any) is used to compress a block. +# no: kNoCompression +# snappy: kSnappyCompression +# zlib: kZlibCompression +# bzip2: kBZip2Compression +# lz4: kLZ4Compression +# lz4hc: kLZ4HCCompression +# zstd: kZSTD + +# per level compression +# compression-per-level = ["no", "no", "lz4", "lz4", "lz4", "zstd", "zstd"] + +# Approximate size of user data packed per block. Note that the +# block size specified here corresponds to uncompressed data. +# block-size = "64KB" + +# If you're doing point lookups you definitely want to turn bloom filters on, We use +# bloom filters to avoid unnecessary disk reads. Default bits_per_key is 10, which +# yields ~1% false positive rate. Larger bits_per_key values will reduce false positive +# rate, but increase memory usage and space amplification. +# bloom-filter-bits-per-key = 10 + +# false means one sst file one bloom filter, true means evry block has a corresponding bloom filter +# block-based-bloom-filter = false + +# level0-file-num-compaction-trigger = 4 + +# Soft limit on number of level-0 files. We start slowing down writes at this point. +# level0-slowdown-writes-trigger = 20 + +# Maximum number of level-0 files. We stop writes at this point. +# level0-stop-writes-trigger = 36 + +# Amount of data to build up in memory (backed by an unsorted log +# on disk) before converting to a sorted on-disk file. +# write-buffer-size = "128MB" + +# The maximum number of write buffers that are built up in memory. +# max-write-buffer-number = 5 + +# The minimum number of write buffers that will be merged together +# before writing to storage. +# min-write-buffer-number-to-merge = 1 + +# Control maximum total data size for base level (level 1). +# max-bytes-for-level-base = "512MB" + +# Target file size for compaction. +# target-file-size-base = "8MB" + +# Max bytes for compaction.max_compaction_bytes +# max-compaction-bytes = "2GB" + +# There are four different algorithms to pick files to compact. +# 0 : ByCompensatedSize +# 1 : OldestLargestSeqFirst +# 2 : OldestSmallestSeqFirst +# 3 : MinOverlappingRatio +# compaction-pri = 3 + +# block-cache used to cache uncompressed blocks, big block-cache can speed up read. +# in normal cases should tune to 30%-50% system's total memory. +# block-cache-size = "1GB" + +# Indicating if we'd put index/filter blocks to the block cache. +# If not specified, each "table reader" object will pre-load index/filter block +# during table initialization. +# cache-index-and-filter-blocks = true + +# Pin level0 filter and index blocks in cache. +# pin-l0-filter-and-index-blocks = true + +# Enable read amplication statistics. +# value => memory usage (percentage of loaded blocks memory) +# 1 => 12.50 % +# 2 => 06.25 % +# 4 => 03.12 % +# 8 => 01.56 % +# 16 => 00.78 % +# read-amp-bytes-per-bit = 0 + +# Pick target size of each level dynamically. +# dynamic-level-bytes = true + +# Options for Column Family write +# Column Family write used to store commit informations in MVCC model +[rocksdb.writecf] +# compression-per-level = ["no", "no", "lz4", "lz4", "lz4", "zstd", "zstd"] +# block-size = "64KB" +# write-buffer-size = "128MB" +# max-write-buffer-number = 5 +# min-write-buffer-number-to-merge = 1 +# max-bytes-for-level-base = "512MB" +# target-file-size-base = "8MB" + +# in normal cases should tune to 10%-30% system's total memory. +# block-cache-size = "256MB" +# level0-file-num-compaction-trigger = 4 +# level0-slowdown-writes-trigger = 20 +# level0-stop-writes-trigger = 36 +# cache-index-and-filter-blocks = true +# pin-l0-filter-and-index-blocks = true +# compaction-pri = 3 +# read-amp-bytes-per-bit = 0 +# dynamic-level-bytes = true + +[rocksdb.lockcf] +# compression-per-level = ["no", "no", "no", "no", "no", "no", "no"] +# block-size = "16KB" +# write-buffer-size = "128MB" +# max-write-buffer-number = 5 +# min-write-buffer-number-to-merge = 1 +# max-bytes-for-level-base = "128MB" +# target-file-size-base = "8MB" +# block-cache-size = "256MB" +# level0-file-num-compaction-trigger = 1 +# level0-slowdown-writes-trigger = 20 +# level0-stop-writes-trigger = 36 +# cache-index-and-filter-blocks = true +# pin-l0-filter-and-index-blocks = true +# compaction-pri = 0 +# read-amp-bytes-per-bit = 0 +# dynamic-level-bytes = true + +[raftdb] +# max-sub-compactions = 1 +max-open-files = 1024 +# max-manifest-file-size = "20MB" +# create-if-missing = true + +# enable-statistics = true +# stats-dump-period = "10m" + +# compaction-readahead-size = 0 +# writable-file-max-buffer-size = "1MB" +# use-direct-io-for-flush-and-compaction = false +# enable-pipelined-write = true +# allow-concurrent-memtable-write = false +# bytes-per-sync = "0MB" +# wal-bytes-per-sync = "0KB" + +# info-log-max-size = "1GB" +# info-log-roll-time = "0" +# info-log-keep-log-file-num = 10 +# info-log-dir = "" + +[raftdb.defaultcf] +# compression-per-level = ["no", "no", "lz4", "lz4", "lz4", "zstd", "zstd"] +# block-size = "64KB" +# write-buffer-size = "128MB" +# max-write-buffer-number = 5 +# min-write-buffer-number-to-merge = 1 +# max-bytes-for-level-base = "512MB" +# target-file-size-base = "8MB" + +# should tune to 256MB~2GB. +# block-cache-size = "256MB" +# level0-file-num-compaction-trigger = 4 +# level0-slowdown-writes-trigger = 20 +# level0-stop-writes-trigger = 36 +# cache-index-and-filter-blocks = true +# pin-l0-filter-and-index-blocks = true +# compaction-pri = 0 +# read-amp-bytes-per-bit = 0 +# dynamic-level-bytes = true + +[security] +# set the path for certificates. Empty string means disabling secure connectoins. +# ca-path = "" +# cert-path = "" +# key-path = "" + +[import] +# the directory to store importing kv data. +# import-dir = "/tmp/tikv/import" +# number of threads to handle RPC requests. +# num-threads = 8 +# stream channel window size, stream will be blocked on channel full. +# stream-channel-window = 128 From 8171ee17612d0d67aa0a9660b2df77fac37884c7 Mon Sep 17 00:00:00 2001 From: JaredTan95 Date: Sun, 15 Nov 2020 10:53:02 +0800 Subject: [PATCH 06/12] revert ui submodule. --- skywalking-ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skywalking-ui b/skywalking-ui index 4e6d3871dd79..00fedacc5fe8 160000 --- a/skywalking-ui +++ b/skywalking-ui @@ -1 +1 @@ -Subproject commit 4e6d3871dd79835dca1bbf7c5814e350aa10cc3d +Subproject commit 00fedacc5fe842f44952f58719fc6a9659147367 From 0926e4c32180e784a277cb4123b5ffa1abcffbc9 Mon Sep 17 00:00:00 2001 From: JaredTan95 Date: Sun, 15 Nov 2020 10:56:59 +0800 Subject: [PATCH 07/12] fix license. --- .../e2e-test/docker/storage/tidbconfig/pd.toml | 15 +++++++++++++++ .../e2e-test/docker/storage/tidbconfig/tidb.toml | 15 +++++++++++++++ .../e2e-test/docker/storage/tidbconfig/tikv.toml | 15 +++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/test/e2e/e2e-test/docker/storage/tidbconfig/pd.toml b/test/e2e/e2e-test/docker/storage/tidbconfig/pd.toml index b1562a569e30..a81a3df8ce01 100644 --- a/test/e2e/e2e-test/docker/storage/tidbconfig/pd.toml +++ b/test/e2e/e2e-test/docker/storage/tidbconfig/pd.toml @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # PD Configuration. name = "pd" diff --git a/test/e2e/e2e-test/docker/storage/tidbconfig/tidb.toml b/test/e2e/e2e-test/docker/storage/tidbconfig/tidb.toml index 9f881b489177..693ba1f91a08 100644 --- a/test/e2e/e2e-test/docker/storage/tidbconfig/tidb.toml +++ b/test/e2e/e2e-test/docker/storage/tidbconfig/tidb.toml @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # TiDB Configuration. # TiDB server host. diff --git a/test/e2e/e2e-test/docker/storage/tidbconfig/tikv.toml b/test/e2e/e2e-test/docker/storage/tidbconfig/tikv.toml index 6606cd9cd91d..31c95de15bc3 100644 --- a/test/e2e/e2e-test/docker/storage/tidbconfig/tikv.toml +++ b/test/e2e/e2e-test/docker/storage/tidbconfig/tikv.toml @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # TiKV config template # Human-readable big numbers: # File size(based on byte): KB, MB, GB, TB, PB From 692e5529d0b03ef0a7ffcb14f7c6dc90941b5259 Mon Sep 17 00:00:00 2001 From: JaredTan95 Date: Sun, 15 Nov 2020 12:18:26 +0800 Subject: [PATCH 08/12] update e2e. --- .../jdbc/tidb/TiDBHistoryDeleteDAO.java | 18 +- .../docker/storage/docker-compose.tidb.yml | 42 +- .../docker/storage/tidbconfig/pd.toml | 101 ---- .../docker/storage/tidbconfig/tikv.toml | 512 ------------------ 4 files changed, 7 insertions(+), 666 deletions(-) delete mode 100644 test/e2e/e2e-test/docker/storage/tidbconfig/pd.toml delete mode 100644 test/e2e/e2e-test/docker/storage/tidbconfig/tikv.toml diff --git a/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBHistoryDeleteDAO.java b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBHistoryDeleteDAO.java index eae6cff3feb7..be3208b84fb7 100644 --- a/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBHistoryDeleteDAO.java +++ b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBHistoryDeleteDAO.java @@ -39,35 +39,29 @@ public TiDBHistoryDeleteDAO(JDBCHikariCPClient client) { @Override public void deleteHistory(Model model, String timeBucketColumnName, int ttl) throws IOException { SQLBuilder dataDeleteSQL = new SQLBuilder("delete from " + model.getName() + " where ") - .append(timeBucketColumnName).append("<= ? and ") - .append(timeBucketColumnName).append(">= ?") + .append(timeBucketColumnName).append("<= ? ") .append(" limit 10000"); - long minTimeBucket = 0; - DateTime minDate = new DateTime(1900, 1, 1, 0, 0); try (Connection connection = client.getConnection()) { long deadline; if (model.isRecord()) { - deadline = Long.parseLong(new DateTime().plusDays(0 - ttl).toString("yyyyMMddHHmmss")); + deadline = Long.parseLong(new DateTime().plusDays(-ttl).toString("yyyyMMddHHmmss")); } else { switch (model.getDownsampling()) { case Minute: - deadline = Long.parseLong(new DateTime().plusDays(0 - ttl).toString("yyyyMMddHHmm")); - minTimeBucket = Long.parseLong(minDate.toString("yyyyMMddHHmm")); + deadline = Long.parseLong(new DateTime().plusDays(-ttl).toString("yyyyMMddHHmm")); break; case Hour: - deadline = Long.parseLong(new DateTime().plusDays(0 - ttl).toString("yyyyMMddHH")); - minTimeBucket = Long.parseLong(minDate.toString("yyyyMMddHH")); + deadline = Long.parseLong(new DateTime().plusDays(-ttl).toString("yyyyMMddHH")); break; case Day: - deadline = Long.parseLong(new DateTime().plusDays(0 - ttl).toString("yyyyMMdd")); - minTimeBucket = Long.parseLong(minDate.toString("yyyyMMdd")); + deadline = Long.parseLong(new DateTime().plusDays(-ttl).toString("yyyyMMdd")); break; default: return; } } - while (client.executeUpdate(connection, dataDeleteSQL.toString(), deadline, minTimeBucket) > 0) { + while (client.executeUpdate(connection, dataDeleteSQL.toString(), deadline) > 0) { } } catch (JDBCClientException | SQLException e) { throw new IOException(e.getMessage(), e); diff --git a/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml b/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml index d421167be8d0..70826ff00963 100644 --- a/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml +++ b/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml @@ -16,38 +16,6 @@ version: '2.1' services: - pd0: - image: pingcap/pd:latest - ports: - - "2379" - volumes: - - ./tidbconfig/pd.toml:/pd.toml:ro - command: - - --name=pd0 - - --client-urls=http://0.0.0.0:2379 - - --peer-urls=http://0.0.0.0:2380 - - --advertise-client-urls=http://pd0:2379 - - --advertise-peer-urls=http://pd0:2380 - - --initial-cluster=pd0=http://pd0:2380 - - --data-dir=/data/pd0 - - --config=/pd.toml - - --log-file=/logs/pd0.log - restart: on-failure - tikv0: - image: pingcap/tikv:latest - volumes: - - ./tidbconfig/tikv.toml:/tikv.toml:ro - command: - - --addr=0.0.0.0:20160 - - --advertise-addr=tikv0:20160 - - --data-dir=/data/tikv0 - - --pd=pd0:2379 - - --config=/tikv.toml - - --log-file=/logs/tikv0.log - depends_on: - - "pd0" - restart: on-failure - tidb: image: pingcap/tidb:latest ports: @@ -55,20 +23,12 @@ services: - "10080:10080" volumes: - ./tidbconfig/tidb.toml:/tidb.toml:ro - command: - - --store=tikv - - --path=pd0:2379 - - --config=/tidb.toml - - --log-file=/logs/tidb.log - - --advertise-address=tidb - depends_on: - - "tikv0" + restart: on-failure healthcheck: test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/4000 "] interval: 5s timeout: 60s retries: 120 - restart: on-failure oap: extends: diff --git a/test/e2e/e2e-test/docker/storage/tidbconfig/pd.toml b/test/e2e/e2e-test/docker/storage/tidbconfig/pd.toml deleted file mode 100644 index a81a3df8ce01..000000000000 --- a/test/e2e/e2e-test/docker/storage/tidbconfig/pd.toml +++ /dev/null @@ -1,101 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# PD Configuration. - -name = "pd" -data-dir = "default.pd" - -client-urls = "http://127.0.0.1:2379" -# if not set, use ${client-urls} -advertise-client-urls = "" - -peer-urls = "http://127.0.0.1:2380" -# if not set, use ${peer-urls} -advertise-peer-urls = "" - -initial-cluster = "pd=http://127.0.0.1:2380" -initial-cluster-state = "new" - -lease = 3 -tso-save-interval = "3s" - -[security] -# Path of file that contains list of trusted SSL CAs. if set, following four settings shouldn't be empty -cacert-path = "" -# Path of file that contains X509 certificate in PEM format. -cert-path = "" -# Path of file that contains X509 key in PEM format. -key-path = "" - -[log] -level = "error" - -# log format, one of json, text, console -#format = "text" - -# disable automatic timestamps in output -#disable-timestamp = false - -# file logging -[log.file] -#filename = "" -# max log file size in MB -#max-size = 300 -# max log file keep days -#max-days = 28 -# maximum number of old log files to retain -#max-backups = 7 -# rotate log by day -#log-rotate = true - -[metric] -# prometheus client push interval, set "0s" to disable prometheus. -interval = "15s" -# prometheus pushgateway address, leaves it empty will disable prometheus. -address = "pushgateway:9091" - -[schedule] -max-merge-region-size = 0 -split-merge-interval = "1h" -max-snapshot-count = 3 -max-pending-peer-count = 16 -max-store-down-time = "30m" -leader-schedule-limit = 4 -region-schedule-limit = 4 -replica-schedule-limit = 8 -merge-schedule-limit = 8 -tolerant-size-ratio = 5.0 - -# customized schedulers, the format is as below -# if empty, it will use balance-leader, balance-region, hot-region as default -# [[schedule.schedulers]] -# type = "evict-leader" -# args = ["1"] - -[replication] -# The number of replicas for each region. -max-replicas = 3 -# The label keys specified the location of a store. -# The placement priorities is implied by the order of label keys. -# For example, ["zone", "rack"] means that we should place replicas to -# different zones first, then to different racks if we don't have enough zones. -location-labels = [] - -[label-property] -# Do not assign region leaders to stores that have these tags. -# [[label-property.reject-leader]] -# key = "zone" -# value = "cn1 diff --git a/test/e2e/e2e-test/docker/storage/tidbconfig/tikv.toml b/test/e2e/e2e-test/docker/storage/tidbconfig/tikv.toml deleted file mode 100644 index 31c95de15bc3..000000000000 --- a/test/e2e/e2e-test/docker/storage/tidbconfig/tikv.toml +++ /dev/null @@ -1,512 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# TiKV config template -# Human-readable big numbers: -# File size(based on byte): KB, MB, GB, TB, PB -# e.g.: 1_048_576 = "1MB" -# Time(based on ms): ms, s, m, h -# e.g.: 78_000 = "1.3m" - -# log level: trace, debug, info, warn, error, off. -log-level = "error" -# file to store log, write to stderr if it's empty. -# log-file = "" - -[readpool.storage] -# size of thread pool for high-priority operations -# high-concurrency = 4 -# size of thread pool for normal-priority operations -# normal-concurrency = 4 -# size of thread pool for low-priority operations -# low-concurrency = 4 -# max running high-priority operations, reject if exceed -# max-tasks-high = 8000 -# max running normal-priority operations, reject if exceed -# max-tasks-normal = 8000 -# max running low-priority operations, reject if exceed -# max-tasks-low = 8000 -# size of stack size for each thread pool -# stack-size = "10MB" - -[readpool.coprocessor] -# Notice: if CPU_NUM > 8, default thread pool size for coprocessors -# will be set to CPU_NUM * 0.8. - -# high-concurrency = 8 -# normal-concurrency = 8 -# low-concurrency = 8 -# max-tasks-high = 16000 -# max-tasks-normal = 16000 -# max-tasks-low = 16000 -# stack-size = "10MB" - -[server] -# set listening address. -# addr = "127.0.0.1:20160" -# set advertise listening address for client communication, if not set, use addr instead. -# advertise-addr = "" -# notify capacity, 40960 is suitable for about 7000 regions. -# notify-capacity = 40960 -# maximum number of messages can be processed in one tick. -# messages-per-tick = 4096 - -# compression type for grpc channel, available values are no, deflate and gzip. -# grpc-compression-type = "no" -# size of thread pool for grpc server. -# grpc-concurrency = 4 -# The number of max concurrent streams/requests on a client connection. -# grpc-concurrent-stream = 1024 -# The number of connections with each tikv server to send raft messages. -# grpc-raft-conn-num = 10 -# Amount to read ahead on individual grpc streams. -# grpc-stream-initial-window-size = "2MB" - -# How many snapshots can be sent concurrently. -# concurrent-send-snap-limit = 32 -# How many snapshots can be recv concurrently. -# concurrent-recv-snap-limit = 32 - -# max count of tasks being handled, new tasks will be rejected. -# end-point-max-tasks = 2000 - -# max recursion level allowed when decoding dag expression -# end-point-recursion-limit = 1000 - -# max time to handle coprocessor request before timeout -# end-point-request-max-handle-duration = "60s" - -# the max bytes that snapshot can be written to disk in one second, -# should be set based on your disk performance -# snap-max-write-bytes-per-sec = "100MB" - -# set attributes about this server, e.g. { zone = "us-west-1", disk = "ssd" }. -# labels = {} - -[storage] -# set the path to rocksdb directory. -# data-dir = "/tmp/tikv/store" - -# notify capacity of scheduler's channel -# scheduler-notify-capacity = 10240 - -# maximum number of messages can be processed in one tick -# scheduler-messages-per-tick = 1024 - -# the number of slots in scheduler latches, concurrency control for write. -# scheduler-concurrency = 2048000 - -# scheduler's worker pool size, should increase it in heavy write cases, -# also should less than total cpu cores. -# scheduler-worker-pool-size = 4 - -# When the pending write bytes exceeds this threshold, -# the "scheduler too busy" error is displayed. -# scheduler-pending-write-threshold = "100MB" - -[pd] -# pd endpoints -# endpoints = [] - -[metric] -# the Prometheus client push interval. Setting the value to 0s stops Prometheus client from pushing. -# interval = "15s" -# the Prometheus pushgateway address. Leaving it empty stops Prometheus client from pushing. -address = "pushgateway:9091" -# the Prometheus client push job name. Note: A node id will automatically append, e.g., "tikv_1". -# job = "tikv" - -[raftstore] -# true (default value) for high reliability, this can prevent data loss when power failure. -# sync-log = true - -# set the path to raftdb directory, default value is data-dir/raft -# raftdb-path = "" - -# set store capacity, if no set, use disk capacity. -# capacity = 0 - -# notify capacity, 40960 is suitable for about 7000 regions. -# notify-capacity = 40960 - -# maximum number of messages can be processed in one tick. -# messages-per-tick = 4096 - -# Region heartbeat tick interval for reporting to pd. -# pd-heartbeat-tick-interval = "60s" -# Store heartbeat tick interval for reporting to pd. -# pd-store-heartbeat-tick-interval = "10s" - -# When region size changes exceeds region-split-check-diff, we should check -# whether the region should be split or not. -# region-split-check-diff = "6MB" - -# Interval to check region whether need to be split or not. -# split-region-check-tick-interval = "10s" - -# When raft entry exceed the max size, reject to propose the entry. -# raft-entry-max-size = "8MB" - -# Interval to gc unnecessary raft log. -# raft-log-gc-tick-interval = "10s" -# A threshold to gc stale raft log, must >= 1. -# raft-log-gc-threshold = 50 -# When entry count exceed this value, gc will be forced trigger. -# raft-log-gc-count-limit = 72000 -# When the approximate size of raft log entries exceed this value, gc will be forced trigger. -# It's recommanded to set it to 3/4 of region-split-size. -# raft-log-gc-size-limit = "72MB" - -# When a peer hasn't been active for max-peer-down-duration, -# we will consider this peer to be down and report it to pd. -# max-peer-down-duration = "5m" - -# Interval to check whether start manual compaction for a region, -# region-compact-check-interval = "5m" -# Number of regions for each time to check. -# region-compact-check-step = 100 -# The minimum number of delete tombstones to trigger manual compaction. -# region-compact-min-tombstones = 10000 -# Interval to check whether should start a manual compaction for lock column family, -# if written bytes reach lock-cf-compact-threshold for lock column family, will fire -# a manual compaction for lock column family. -# lock-cf-compact-interval = "10m" -# lock-cf-compact-bytes-threshold = "256MB" - -# Interval (s) to check region whether the data are consistent. -# consistency-check-interval = 0 - -# Use delete range to drop a large number of continuous keys. -# use-delete-range = false - -# delay time before deleting a stale peer -# clean-stale-peer-delay = "10m" - -# Interval to cleanup import sst files. -# cleanup-import-sst-interval = "10m" - -[coprocessor] -# When it is true, it will try to split a region with table prefix if -# that region crosses tables. It is recommended to turn off this option -# if there will be a large number of tables created. -# split-region-on-table = true -# When the region's size exceeds region-max-size, we will split the region -# into two which the left region's size will be region-split-size or a little -# bit smaller. -# region-max-size = "144MB" -# region-split-size = "96MB" - -[rocksdb] -# Maximum number of concurrent background jobs (compactions and flushes) -# max-background-jobs = 8 - -# This value represents the maximum number of threads that will concurrently perform a -# compaction job by breaking it into multiple, smaller ones that are run simultaneously. -# Default: 1 (i.e. no subcompactions) -# max-sub-compactions = 1 - -# Number of open files that can be used by the DB. You may need to -# increase this if your database has a large working set. Value -1 means -# files opened are always kept open. You can estimate number of files based -# on target_file_size_base and target_file_size_multiplier for level-based -# compaction. -# If max-open-files = -1, RocksDB will prefetch index and filter blocks into -# block cache at startup, so if your database has a large working set, it will -# take several minutes to open the db. -max-open-files = 1024 - -# Max size of rocksdb's MANIFEST file. -# For detailed explanation please refer to https://github.com/facebook/rocksdb/wiki/MANIFEST -# max-manifest-file-size = "20MB" - -# If true, the database will be created if it is missing. -# create-if-missing = true - -# rocksdb wal recovery mode -# 0 : TolerateCorruptedTailRecords, tolerate incomplete record in trailing data on all logs; -# 1 : AbsoluteConsistency, We don't expect to find any corruption in the WAL; -# 2 : PointInTimeRecovery, Recover to point-in-time consistency; -# 3 : SkipAnyCorruptedRecords, Recovery after a disaster; -# wal-recovery-mode = 2 - -# rocksdb write-ahead logs dir path -# This specifies the absolute dir path for write-ahead logs (WAL). -# If it is empty, the log files will be in the same dir as data. -# When you set the path to rocksdb directory in memory like in /dev/shm, you may want to set -# wal-dir to a directory on a persistent storage. -# See https://github.com/facebook/rocksdb/wiki/How-to-persist-in-memory-RocksDB-database -# wal-dir = "/tmp/tikv/store" - -# The following two fields affect how archived write-ahead logs will be deleted. -# 1. If both set to 0, logs will be deleted asap and will not get into the archive. -# 2. If wal-ttl-seconds is 0 and wal-size-limit is not 0, -# WAL files will be checked every 10 min and if total size is greater -# then wal-size-limit, they will be deleted starting with the -# earliest until size_limit is met. All empty files will be deleted. -# 3. If wal-ttl-seconds is not 0 and wal-size-limit is 0, then -# WAL files will be checked every wal-ttl-seconds / 2 and those that -# are older than wal-ttl-seconds will be deleted. -# 4. If both are not 0, WAL files will be checked every 10 min and both -# checks will be performed with ttl being first. -# When you set the path to rocksdb directory in memory like in /dev/shm, you may want to set -# wal-ttl-seconds to a value greater than 0 (like 86400) and backup your db on a regular basis. -# See https://github.com/facebook/rocksdb/wiki/How-to-persist-in-memory-RocksDB-database -# wal-ttl-seconds = 0 -# wal-size-limit = 0 - -# rocksdb max total wal size -# max-total-wal-size = "4GB" - -# Rocksdb Statistics provides cumulative stats over time. -# Turn statistics on will introduce about 5%-10% overhead for RocksDB, -# but it is worthy to know the internal status of RocksDB. -# enable-statistics = true - -# Dump statistics periodically in information logs. -# Same as rocksdb's default value (10 min). -# stats-dump-period = "10m" - -# Due to Rocksdb FAQ: https://github.com/facebook/rocksdb/wiki/RocksDB-FAQ, -# If you want to use rocksdb on multi disks or spinning disks, you should set value at -# least 2MB; -# compaction-readahead-size = 0 - -# This is the maximum buffer size that is used by WritableFileWrite -# writable-file-max-buffer-size = "1MB" - -# Use O_DIRECT for both reads and writes in background flush and compactions -# use-direct-io-for-flush-and-compaction = false - -# Limit the disk IO of compaction and flush. Compaction and flush can cause -# terrible spikes if they exceed a certain threshold. Consider setting this to -# 50% ~ 80% of the disk throughput for a more stable result. However, in heavy -# write workload, limiting compaction and flush speed can cause write stalls too. -# rate-bytes-per-sec = 0 - -# Enable or disable the pipelined write -# enable-pipelined-write = true - -# Allows OS to incrementally sync files to disk while they are being -# written, asynchronously, in the background. -# bytes-per-sync = "0MB" - -# Allows OS to incrementally sync WAL to disk while it is being written. -# wal-bytes-per-sync = "0KB" - -# Specify the maximal size of the Rocksdb info log file. If the log file -# is larger than `max_log_file_size`, a new info log file will be created. -# If max_log_file_size == 0, all logs will be written to one log file. -# Default: 1GB -# info-log-max-size = "1GB" - -# Time for the Rocksdb info log file to roll (in seconds). -# If specified with non-zero value, log file will be rolled -# if it has been active longer than `log_file_time_to_roll`. -# Default: 0 (disabled) -# info-log-roll-time = "0" - -# Maximal Rocksdb info log files to be kept. -# Default: 10 -# info-log-keep-log-file-num = 10 - -# This specifies the Rocksdb info LOG dir. -# If it is empty, the log files will be in the same dir as data. -# If it is non empty, the log files will be in the specified dir, -# and the db data dir's absolute path will be used as the log file -# name's prefix. -# Default: empty -# info-log-dir = "" - -# Column Family default used to store actual data of the database. -[rocksdb.defaultcf] -# compression method (if any) is used to compress a block. -# no: kNoCompression -# snappy: kSnappyCompression -# zlib: kZlibCompression -# bzip2: kBZip2Compression -# lz4: kLZ4Compression -# lz4hc: kLZ4HCCompression -# zstd: kZSTD - -# per level compression -# compression-per-level = ["no", "no", "lz4", "lz4", "lz4", "zstd", "zstd"] - -# Approximate size of user data packed per block. Note that the -# block size specified here corresponds to uncompressed data. -# block-size = "64KB" - -# If you're doing point lookups you definitely want to turn bloom filters on, We use -# bloom filters to avoid unnecessary disk reads. Default bits_per_key is 10, which -# yields ~1% false positive rate. Larger bits_per_key values will reduce false positive -# rate, but increase memory usage and space amplification. -# bloom-filter-bits-per-key = 10 - -# false means one sst file one bloom filter, true means evry block has a corresponding bloom filter -# block-based-bloom-filter = false - -# level0-file-num-compaction-trigger = 4 - -# Soft limit on number of level-0 files. We start slowing down writes at this point. -# level0-slowdown-writes-trigger = 20 - -# Maximum number of level-0 files. We stop writes at this point. -# level0-stop-writes-trigger = 36 - -# Amount of data to build up in memory (backed by an unsorted log -# on disk) before converting to a sorted on-disk file. -# write-buffer-size = "128MB" - -# The maximum number of write buffers that are built up in memory. -# max-write-buffer-number = 5 - -# The minimum number of write buffers that will be merged together -# before writing to storage. -# min-write-buffer-number-to-merge = 1 - -# Control maximum total data size for base level (level 1). -# max-bytes-for-level-base = "512MB" - -# Target file size for compaction. -# target-file-size-base = "8MB" - -# Max bytes for compaction.max_compaction_bytes -# max-compaction-bytes = "2GB" - -# There are four different algorithms to pick files to compact. -# 0 : ByCompensatedSize -# 1 : OldestLargestSeqFirst -# 2 : OldestSmallestSeqFirst -# 3 : MinOverlappingRatio -# compaction-pri = 3 - -# block-cache used to cache uncompressed blocks, big block-cache can speed up read. -# in normal cases should tune to 30%-50% system's total memory. -# block-cache-size = "1GB" - -# Indicating if we'd put index/filter blocks to the block cache. -# If not specified, each "table reader" object will pre-load index/filter block -# during table initialization. -# cache-index-and-filter-blocks = true - -# Pin level0 filter and index blocks in cache. -# pin-l0-filter-and-index-blocks = true - -# Enable read amplication statistics. -# value => memory usage (percentage of loaded blocks memory) -# 1 => 12.50 % -# 2 => 06.25 % -# 4 => 03.12 % -# 8 => 01.56 % -# 16 => 00.78 % -# read-amp-bytes-per-bit = 0 - -# Pick target size of each level dynamically. -# dynamic-level-bytes = true - -# Options for Column Family write -# Column Family write used to store commit informations in MVCC model -[rocksdb.writecf] -# compression-per-level = ["no", "no", "lz4", "lz4", "lz4", "zstd", "zstd"] -# block-size = "64KB" -# write-buffer-size = "128MB" -# max-write-buffer-number = 5 -# min-write-buffer-number-to-merge = 1 -# max-bytes-for-level-base = "512MB" -# target-file-size-base = "8MB" - -# in normal cases should tune to 10%-30% system's total memory. -# block-cache-size = "256MB" -# level0-file-num-compaction-trigger = 4 -# level0-slowdown-writes-trigger = 20 -# level0-stop-writes-trigger = 36 -# cache-index-and-filter-blocks = true -# pin-l0-filter-and-index-blocks = true -# compaction-pri = 3 -# read-amp-bytes-per-bit = 0 -# dynamic-level-bytes = true - -[rocksdb.lockcf] -# compression-per-level = ["no", "no", "no", "no", "no", "no", "no"] -# block-size = "16KB" -# write-buffer-size = "128MB" -# max-write-buffer-number = 5 -# min-write-buffer-number-to-merge = 1 -# max-bytes-for-level-base = "128MB" -# target-file-size-base = "8MB" -# block-cache-size = "256MB" -# level0-file-num-compaction-trigger = 1 -# level0-slowdown-writes-trigger = 20 -# level0-stop-writes-trigger = 36 -# cache-index-and-filter-blocks = true -# pin-l0-filter-and-index-blocks = true -# compaction-pri = 0 -# read-amp-bytes-per-bit = 0 -# dynamic-level-bytes = true - -[raftdb] -# max-sub-compactions = 1 -max-open-files = 1024 -# max-manifest-file-size = "20MB" -# create-if-missing = true - -# enable-statistics = true -# stats-dump-period = "10m" - -# compaction-readahead-size = 0 -# writable-file-max-buffer-size = "1MB" -# use-direct-io-for-flush-and-compaction = false -# enable-pipelined-write = true -# allow-concurrent-memtable-write = false -# bytes-per-sync = "0MB" -# wal-bytes-per-sync = "0KB" - -# info-log-max-size = "1GB" -# info-log-roll-time = "0" -# info-log-keep-log-file-num = 10 -# info-log-dir = "" - -[raftdb.defaultcf] -# compression-per-level = ["no", "no", "lz4", "lz4", "lz4", "zstd", "zstd"] -# block-size = "64KB" -# write-buffer-size = "128MB" -# max-write-buffer-number = 5 -# min-write-buffer-number-to-merge = 1 -# max-bytes-for-level-base = "512MB" -# target-file-size-base = "8MB" - -# should tune to 256MB~2GB. -# block-cache-size = "256MB" -# level0-file-num-compaction-trigger = 4 -# level0-slowdown-writes-trigger = 20 -# level0-stop-writes-trigger = 36 -# cache-index-and-filter-blocks = true -# pin-l0-filter-and-index-blocks = true -# compaction-pri = 0 -# read-amp-bytes-per-bit = 0 -# dynamic-level-bytes = true - -[security] -# set the path for certificates. Empty string means disabling secure connectoins. -# ca-path = "" -# cert-path = "" -# key-path = "" - -[import] -# the directory to store importing kv data. -# import-dir = "/tmp/tikv/import" -# number of threads to handle RPC requests. -# num-threads = 8 -# stream channel window size, stream will be blocked on channel full. -# stream-channel-window = 128 From 6dd6562235a968fc48b2124917bdb2354d094dbc Mon Sep 17 00:00:00 2001 From: JaredTan95 Date: Sun, 15 Nov 2020 13:52:08 +0800 Subject: [PATCH 09/12] update e2e. --- .github/workflows/e2e.storages.yaml | 2 +- test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.storages.yaml b/.github/workflows/e2e.storages.yaml index 26c63fc05fc9..370c4eece7bf 100644 --- a/.github/workflows/e2e.storages.yaml +++ b/.github/workflows/e2e.storages.yaml @@ -34,7 +34,7 @@ jobs: timeout-minutes: 90 strategy: matrix: - storage: ['mysql', 'es6', 'es7.0', 'es7.9', 'influxdb','tidb'] + storage: ['mysql', 'es6', 'es7.0', 'es7.9', 'influxdb', 'tidb'] env: SW_STORAGE: ${{ matrix.storage }} steps: diff --git a/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml b/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml index 70826ff00963..3f231c43c867 100644 --- a/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml +++ b/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml @@ -25,7 +25,7 @@ services: - ./tidbconfig/tidb.toml:/tidb.toml:ro restart: on-failure healthcheck: - test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/4000 "] + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/4000"] interval: 5s timeout: 60s retries: 120 From dd36b362eb4e0913b4732d1b247dcf3664a154c0 Mon Sep 17 00:00:00 2001 From: JaredTan95 Date: Sun, 15 Nov 2020 14:10:57 +0800 Subject: [PATCH 10/12] update e2e. --- test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml b/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml index 3f231c43c867..299e715c4392 100644 --- a/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml +++ b/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml @@ -18,9 +18,8 @@ version: '2.1' services: tidb: image: pingcap/tidb:latest - ports: - - "4000:4000" - - "10080:10080" + expose: + - 4000 volumes: - ./tidbconfig/tidb.toml:/tidb.toml:ro restart: on-failure @@ -29,6 +28,8 @@ services: interval: 5s timeout: 60s retries: 120 + networks: + - e2e oap: extends: From eb3a582c795df9397302ec677f883ca65666f4a3 Mon Sep 17 00:00:00 2001 From: kezhenxu94 Date: Sun, 22 Nov 2020 10:45:09 +0800 Subject: [PATCH 11/12] Fix wrong health check test --- test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml b/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml index 299e715c4392..a570ef8f7d06 100644 --- a/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml +++ b/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml @@ -24,7 +24,7 @@ services: - ./tidbconfig/tidb.toml:/tidb.toml:ro restart: on-failure healthcheck: - test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/4000"] + test: ["CMD", "sh", "-c", "nc -zn 127.0.0.1 4000"] interval: 5s timeout: 60s retries: 120 @@ -39,7 +39,7 @@ services: SW_STORAGE: tidb SW_PROMETHEUS_FETCHER_ACTIVE: "true" SW_TELEMETRY: prometheus - SW_JDBC_URL: "jdbc:mysql://tidb:4000/INFORMATION_SCHEMA" + SW_JDBC_URL: "jdbc:mysql://tidb:4000/test" SW_DATA_SOURCE_PASSWORD: "" depends_on: tidb: From 590bea76202fa8a1bb42ad4eaea8bf9ad0c9d3ee Mon Sep 17 00:00:00 2001 From: JaredTan95 Date: Sun, 22 Nov 2020 22:37:12 +0800 Subject: [PATCH 12/12] add tidb ttl e2e. --- .github/workflows/e2e.ttl.yaml | 2 +- .../storage-tidb-plugin/pom.xml | 6 --- .../docker/ttl/docker-compose.tidb.yml | 51 +++++++++++++++++++ 3 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 test/e2e/e2e-test/docker/ttl/docker-compose.tidb.yml diff --git a/.github/workflows/e2e.ttl.yaml b/.github/workflows/e2e.ttl.yaml index 645fb8856e12..89c3b1cc9cac 100644 --- a/.github/workflows/e2e.ttl.yaml +++ b/.github/workflows/e2e.ttl.yaml @@ -37,7 +37,7 @@ jobs: timeout-minutes: 90 strategy: matrix: - storage: ['es6', 'es7', 'influxdb'] + storage: ['es6', 'es7', 'influxdb', 'tidb'] env: SW_STORAGE: ${{ matrix.storage }} steps: diff --git a/oap-server/server-storage-plugin/storage-tidb-plugin/pom.xml b/oap-server/server-storage-plugin/storage-tidb-plugin/pom.xml index bf7096776c9f..1c973318c1f3 100644 --- a/oap-server/server-storage-plugin/storage-tidb-plugin/pom.xml +++ b/oap-server/server-storage-plugin/storage-tidb-plugin/pom.xml @@ -49,11 +49,5 @@ storage-jdbc-hikaricp-plugin ${project.version} - - \ No newline at end of file diff --git a/test/e2e/e2e-test/docker/ttl/docker-compose.tidb.yml b/test/e2e/e2e-test/docker/ttl/docker-compose.tidb.yml new file mode 100644 index 000000000000..9746298d06a6 --- /dev/null +++ b/test/e2e/e2e-test/docker/ttl/docker-compose.tidb.yml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + tidb: + image: pingcap/tidb:latest + expose: + - 4000 + volumes: + - ./tidbconfig/tidb.toml:/tidb.toml:ro + restart: on-failure + healthcheck: + test: ["CMD", "sh", "-c", "nc -zn 127.0.0.1 4000"] + interval: 5s + timeout: 60s + retries: 120 + networks: + - e2e + + oap: + extends: + file: ../base-compose.yml + service: oap + environment: + SW_STORAGE: tidb + SW_PROMETHEUS_FETCHER_ACTIVE: "true" + SW_TELEMETRY: prometheus + SW_JDBC_URL: "jdbc:mysql://tidb:4000/test" + SW_DATA_SOURCE_PASSWORD: "" + SW_CORE_DATA_KEEPER_EXECUTE_PERIOD: 1 + depends_on: + tidb: + condition: service_healthy + entrypoint: ['sh', '-c', '/download-mysql.sh && /skywalking/docker-entrypoint.sh'] + +networks: + e2e: