From 6f1922e8d344e7e09e1cc9f013795206357a091b Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Fri, 23 Aug 2024 14:08:19 +0800 Subject: [PATCH] [fix](jdbc catalog) Fix Memory Leak by Enabling Weak References in HikariCP (#39582) This PR addresses a memory leak issue caused by FastList objects in HikariCP being retained by ThreadLocal variables, which are not easily garbage collected in long-running JNI threads. To mitigate this, a system property com.zaxxer.hikari.useWeakReferences is set to true, ensuring that WeakReference is used for ThreadLocal objects, allowing the garbage collector to reclaim memory more effectively. Even though setting this will affect some performance, solving resource leaks is relatively more important Performance difference before and after setting Before setting: 10 concurrency 0.02-0.05 100 concurrency 0.18-0.4 After setting: 10 concurrency 0.02-0.07 100 concurrency 0.18-0.7 --- .../main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java | 3 ++- .../src/main/java/org/apache/doris/jdbc/JdbcDataSource.java | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java index d1cd0c9d33e78d..67a0cf5a636b34 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java @@ -93,6 +93,7 @@ public BaseJdbcExecutor(byte[] thriftParams) throws Exception { .setConnectionPoolMaxLifeTime(request.connection_pool_max_life_time) .setConnectionPoolKeepAlive(request.connection_pool_keep_alive); JdbcDataSource.getDataSource().setCleanupInterval(request.connection_pool_cache_clear_time); + System.setProperty("com.zaxxer.hikari.useWeakReferences", "true"); init(config, request.statement); this.jdbcDriverVersion = getJdbcDriverVersion(); } @@ -308,7 +309,7 @@ private void init(JdbcDataSourceConfig config, String sql) throws JdbcExecutorEx ds.setKeepaliveTime(config.getConnectionPoolMaxLifeTime() / 5L); // default 6 min } hikariDataSource = ds; - JdbcDataSource.getDataSource().putSource(hikariDataSourceKey, ds); + JdbcDataSource.getDataSource().putSource(hikariDataSourceKey, hikariDataSource); LOG.info("JdbcClient set" + " ConnectionPoolMinSize = " + config.getConnectionPoolMinSize() + ", ConnectionPoolMaxSize = " + config.getConnectionPoolMaxSize() diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java index 447566596a7e92..1aea763852fcc3 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java @@ -59,8 +59,10 @@ public Map getSourcesMap() { } public void setCleanupInterval(long interval) { - this.cleanupInterval = interval * 1000L; - restartCleanupTask(); + if (this.cleanupInterval != interval * 1000L) { + this.cleanupInterval = interval * 1000L; + restartCleanupTask(); + } } private synchronized void restartCleanupTask() {