add Tidb provider module.

.github/workflows/e2e.storages.yaml

@@ -34,7 +34,7 @@ jobs:
     timeout-minutes: 90
     strategy:
       matrix:
-        storage: ['mysql', 'es6', 'es7.0', 'es7.9', 'influxdb']
+        storage: ['mysql', 'es6', 'es7.0', 'es7.9', 'influxdb', 'tidb']
     env:
       SW_STORAGE: ${{ matrix.storage }}
     steps:

.github/workflows/e2e.ttl.yaml

@@ -37,7 +37,7 @@ jobs:
     timeout-minutes: 90
     strategy:
       matrix:
-        storage: ['es6', 'es7', 'influxdb']
+        storage: ['es6', 'es7', 'influxdb', 'tidb']
     env:
       SW_STORAGE: ${{ matrix.storage }}
     steps:

CHANGES.md

@@ -50,6 +50,7 @@ Release Notes.
 * Fix that chunked string is incorrect while the tag contains colon.
 * Fix the incorrect dynamic configuration key bug of `endpoint-name-grouping`.
 * Remove unused min date timebucket in jdbc deletehistory logical
+* Fix "transaction too large error" when use TiDB as storage.
 
 #### UI
 * Fix incorrect label in radial chart in topology.

docs/en/setup/backend/backend-storage.md

@@ -223,22 +223,25 @@ All connection related settings including link url, username and password are in
 Here are some of the settings, please follow [HikariCP](https://github.com/brettwooldridge/HikariCP) connection pool document for all the settings.
 
 ## TiDB
-Currently tested TiDB in version 2.0.9, and Mysql Client driver in version 8.0.13.
-Active TiDB as storage, set storage provider to **mysql**. 
+Tested TiDB Server 4.0.8 version and Mysql Client driver 8.0.13 version currently.
+Active TiDB as storage, set storage provider to **tidb**. 
 
 ```yaml
 storage:
-  selector: ${SW_STORAGE:mysql}
-  mysql:
+  selector: ${SW_STORAGE:tidb}
+  tidb:
     properties:
-      jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:3306/swtest"}
+      jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:4000/swtest"}
       dataSource.user: ${SW_DATA_SOURCE_USER:root}
-      dataSource.password: ${SW_DATA_SOURCE_PASSWORD:root@1234}
+      dataSource.password: ${SW_DATA_SOURCE_PASSWORD:""}
       dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true}
       dataSource.prepStmtCacheSize: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250}
       dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048}
       dataSource.useServerPrepStmts: ${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true}
+      dataSource.useAffectedRows: ${SW_DATA_SOURCE_USE_AFFECTED_ROWS:true}
     metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000}
+    maxSizeOfArrayColumn: ${SW_STORAGE_MAX_SIZE_OF_ARRAY_COLUMN:20}
+    numOfSearchableValuesPerTag: ${SW_STORAGE_NUM_OF_SEARCHABLE_VALUES_PER_TAG:2}
 ```
 All connection related settings including link url, username and password are in `application.yml`. 
 These settings can refer to the configuration of *MySQL* above.

oap-server/server-bootstrap/pom.xml

@@ -157,6 +157,11 @@
             <artifactId>storage-influxdb-plugin</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.skywalking</groupId>
+            <artifactId>storage-tidb-plugin</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <!-- storage module -->
 
         <!-- queryBuild module -->

oap-server/server-bootstrap/src/main/resources/application.yml

@@ -171,6 +171,19 @@ storage:
     metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000}
     maxSizeOfArrayColumn: ${SW_STORAGE_MAX_SIZE_OF_ARRAY_COLUMN:20}
     numOfSearchableValuesPerTag: ${SW_STORAGE_NUM_OF_SEARCHABLE_VALUES_PER_TAG:2}
+  tidb:
+    properties:
+      jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:4000/tidbswtest"}
+      dataSource.user: ${SW_DATA_SOURCE_USER:root}
+      dataSource.password: ${SW_DATA_SOURCE_PASSWORD:""}
+      dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true}
+      dataSource.prepStmtCacheSize: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250}
+      dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048}
+      dataSource.useServerPrepStmts: ${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true}
+      dataSource.useAffectedRows: ${SW_DATA_SOURCE_USE_AFFECTED_ROWS:true}
+    metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000}
+    maxSizeOfArrayColumn: ${SW_STORAGE_MAX_SIZE_OF_ARRAY_COLUMN:20}
+    numOfSearchableValuesPerTag: ${SW_STORAGE_NUM_OF_SEARCHABLE_VALUES_PER_TAG:2}
   influxdb:
     # InfluxDB configuration
     url: ${SW_STORAGE_INFLUXDB_URL:http://localhost:8086}

oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/jdbc/hikaricp/JDBCHikariCPClient.java

@@ -90,14 +90,14 @@ public void execute(Connection connection, String sql) throws JDBCClientExceptio
         }
     }
 
-    public boolean execute(Connection connection, String sql, Object... params) throws JDBCClientException {
+    public int executeUpdate(Connection connection, String sql, Object... params) throws JDBCClientException {
         LOGGER.debug("execute query with result: {}", sql);
-        boolean result;
+        int result;
         PreparedStatement statement = null;
         try {
             statement = connection.prepareStatement(sql);
             setStatementParam(statement, params);
-            result = statement.execute();
+            result = statement.executeUpdate();
             statement.closeOnCompletion();
             healthChecker.health();
         } catch (SQLException e) {

oap-server/server-storage-plugin/pom.xml

@@ -34,6 +34,7 @@
         <module>storage-zipkin-plugin</module>
         <module>storage-jaeger-plugin</module>
         <module>storage-influxdb-plugin</module>
+        <module>storage-tidb-plugin</module>
     </modules>
 
 </project>

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java

@@ -60,7 +60,7 @@ public void deleteHistory(Model model, String timeBucketColumnName, int ttl) thr
                         return;
                 }
             }
-            client.execute(connection, dataDeleteSQL.toString(), deadline);
+            client.executeUpdate(connection, dataDeleteSQL.toString(), deadline);
         } catch (JDBCClientException | SQLException e) {
             throw new IOException(e.getMessage(), e);
         }

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageConfig.java

@@ -25,7 +25,7 @@
 
 @Setter
 @Getter
-public final class MySQLStorageConfig extends ModuleConfig {
+public class MySQLStorageConfig extends ModuleConfig {
     private int metadataQueryMaxSize = 5000;
     /**
      * Inherit from {@link org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.H2StorageConfig#getMaxSizeOfArrayColumn()}

oap-server/server-storage-plugin/storage-tidb-plugin/pom.xml

@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>server-storage-plugin</artifactId>
+        <groupId>org.apache.skywalking</groupId>
+        <version>8.3.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>storage-tidb-plugin</artifactId>
+    <packaging>jar</packaging>
+
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.skywalking</groupId>
+            <artifactId>server-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.skywalking</groupId>
+            <artifactId>library-client</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.skywalking</groupId>
+            <artifactId>storage-jdbc-hikaricp-plugin</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+</project>

oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBHistoryDeleteDAO.java

@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.storage.plugin.jdbc.tidb;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.SQLException;
+import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO;
+import org.apache.skywalking.oap.server.core.storage.model.Model;
+import org.apache.skywalking.oap.server.library.client.jdbc.JDBCClientException;
+import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariCPClient;
+import org.apache.skywalking.oap.server.storage.plugin.jdbc.SQLBuilder;
+import org.joda.time.DateTime;
+
+public class TiDBHistoryDeleteDAO implements IHistoryDeleteDAO {
+
+    private final JDBCHikariCPClient client;
+
+    public TiDBHistoryDeleteDAO(JDBCHikariCPClient client) {
+        this.client = client;
+    }
+
+    @Override
+    public void deleteHistory(Model model, String timeBucketColumnName, int ttl) throws IOException {
+        SQLBuilder dataDeleteSQL = new SQLBuilder("delete from " + model.getName() + " where ")
+            .append(timeBucketColumnName).append("<= ? ")
+            .append(" limit 10000");
+
+        try (Connection connection = client.getConnection()) {
+            long deadline;
+            if (model.isRecord()) {
+                deadline = Long.parseLong(new DateTime().plusDays(-ttl).toString("yyyyMMddHHmmss"));
+            } else {
+                switch (model.getDownsampling()) {
+                    case Minute:
+                        deadline = Long.parseLong(new DateTime().plusDays(-ttl).toString("yyyyMMddHHmm"));
+                        break;
+                    case Hour:
+                        deadline = Long.parseLong(new DateTime().plusDays(-ttl).toString("yyyyMMddHH"));
+                        break;
+                    case Day:
+                        deadline = Long.parseLong(new DateTime().plusDays(-ttl).toString("yyyyMMdd"));
+                        break;
+                    default:
+                        return;
+                }
+            }
+            while (client.executeUpdate(connection, dataDeleteSQL.toString(), deadline) > 0) {
+            }
+        } catch (JDBCClientException | SQLException e) {
+            throw new IOException(e.getMessage(), e);
+        }
+    }
+}

oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBStorageConfig.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.storage.plugin.jdbc.tidb;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLStorageConfig;
+
+@Setter
+@Getter
+public class TiDBStorageConfig extends MySQLStorageConfig {
+}