diff --git a/CHANGES.md b/CHANGES.md index 9d53966c8003..19054abb251f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -28,6 +28,7 @@ Release Notes. * Fix deadlock problem when using elasticsearch-client-7.0.0. * Fix storage-jdbc isExists not set dbname. * Fix `searchService` bug in the InfluxDB storage implementation. +* Fix "transaction too large error" when use TiDB as storage. #### UI diff --git a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/jdbc/hikaricp/JDBCHikariCPClient.java b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/jdbc/hikaricp/JDBCHikariCPClient.java index 502eed3847cf..03bf6ac36e51 100644 --- a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/jdbc/hikaricp/JDBCHikariCPClient.java +++ b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/jdbc/hikaricp/JDBCHikariCPClient.java @@ -90,14 +90,14 @@ public void execute(Connection connection, String sql) throws JDBCClientExceptio } } - public boolean execute(Connection connection, String sql, Object... params) throws JDBCClientException { + public int executeUpdate(Connection connection, String sql, Object... params) throws JDBCClientException { LOGGER.debug("execute query with result: {}", sql); - boolean result; + int result; PreparedStatement statement = null; try { statement = connection.prepareStatement(sql); setStatementParam(statement, params); - result = statement.execute(); + result = statement.executeUpdate(); statement.closeOnCompletion(); healthChecker.health(); } catch (SQLException e) { diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java index 31ffe5032581..4c8510d6a4c8 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java @@ -66,7 +66,7 @@ public void deleteHistory(Model model, String timeBucketColumnName, int ttl) thr return; } } - client.execute(connection, dataDeleteSQL.toString(), deadline, minTimeBucket); + client.executeUpdate(connection, dataDeleteSQL.toString(), deadline, minTimeBucket); } catch (JDBCClientException | SQLException e) { throw new IOException(e.getMessage(), e); } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBHistoryDeleteDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBHistoryDeleteDAO.java new file mode 100644 index 000000000000..eac392b9ef9b --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBHistoryDeleteDAO.java @@ -0,0 +1,56 @@ +package org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql; + +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.library.client.jdbc.JDBCClientException; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariCPClient; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.SQLBuilder; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2HistoryDeleteDAO; +import org.joda.time.DateTime; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; + +public class TiDBHistoryDeleteDAO extends H2HistoryDeleteDAO { + + public TiDBHistoryDeleteDAO(JDBCHikariCPClient client) { + super(client); + } + + @Override + public void deleteHistory(Model model, String timeBucketColumnName, int ttl) throws IOException { + SQLBuilder dataDeleteSQL = new SQLBuilder("delete from " + model.getName() + " where ") + .append(timeBucketColumnName).append("<= ? and ") + .append(timeBucketColumnName).append(">= ?") + .append(" limit 10000"); + long minTimeBucket = 0; + DateTime minDate = new DateTime(1900, 1, 1, 0, 0); + + try (Connection connection = client.getConnection()) { + long deadline; + if (model.isRecord()) { + deadline = Long.valueOf(new DateTime().plusDays(0 - ttl).toString("yyyyMMddHHmmss")); + } else { + switch (model.getDownsampling()) { + case Minute: + deadline = Long.valueOf(new DateTime().plusDays(0 - ttl).toString("yyyyMMddHHmm")); + minTimeBucket = Long.valueOf(minDate.toString("yyyyMMddHHmm")); + break; + case Hour: + deadline = Long.valueOf(new DateTime().plusDays(0 - ttl).toString("yyyyMMddHH")); + minTimeBucket = Long.valueOf(minDate.toString("yyyyMMddHH")); + break; + case Day: + deadline = Long.valueOf(new DateTime().plusDays(0 - ttl).toString("yyyyMMdd")); + minTimeBucket = Long.valueOf(minDate.toString("yyyyMMdd")); + break; + default: + return; + } + } + while(client.executeUpdate(connection, dataDeleteSQL.toString(), deadline, minTimeBucket) > 0) {} + } catch (JDBCClientException | SQLException e) { + throw new IOException(e.getMessage(), e); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBStorageProvider.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBStorageProvider.java new file mode 100644 index 000000000000..2ac2dcef75d5 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/TiDBStorageProvider.java @@ -0,0 +1,25 @@ +package org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql; + +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO; +import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; + +/** + * TiDB storage enhanced and came from MySQLStorageProvider to support TiDB. + * + * caution: need add "useAffectedRows=true" to jdbc url. + */ +@Slf4j +public class TiDBStorageProvider extends MySQLStorageProvider { + + @Override + public String name() { + return "tidb"; + } + + @Override + public void prepare() throws ServiceNotProvidedException { + super.prepare(); + this.registerServiceImplementation(IHistoryDeleteDAO.class, new TiDBHistoryDeleteDAO(this.mysqlClient)); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider index c8fa0b9f81a8..d9b7df5f7276 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider @@ -17,4 +17,5 @@ # org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.H2StorageProvider -org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLStorageProvider \ No newline at end of file +org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLStorageProvider +org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.TiDBStorageProvider \ No newline at end of file