From 9d04861be11d93446d4e9e5e7e4e01261a02c6cd Mon Sep 17 00:00:00 2001 From: kyligence-git Date: Tue, 14 Jan 2025 23:09:05 +0000 Subject: [PATCH 1/3] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20250115) --- cpp-ch/clickhouse.version | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index 79ca4eb69bb1..45287fab517c 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20250110 -CH_COMMIT=eafc5ef70b3 +CH_BRANCH=rebase_ch/20250115 +CH_COMMIT=8e0d5eaf0fc From bf2b836343af8ad966bb67a3da68f270ad8014a4 Mon Sep 17 00:00:00 2001 From: Chang Chen Date: Tue, 14 Jan 2025 12:00:04 +0800 Subject: [PATCH 2/3] Fix build due to https://github.com/ClickHouse/ClickHouse/pull/73294 --- .../Storages/MergeTree/MergeSparkMergeTreeTask.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp-ch/local-engine/Storages/MergeTree/MergeSparkMergeTreeTask.cpp b/cpp-ch/local-engine/Storages/MergeTree/MergeSparkMergeTreeTask.cpp index ee6930e4de51..cb0b3f2054da 100644 --- a/cpp-ch/local-engine/Storages/MergeTree/MergeSparkMergeTreeTask.cpp +++ b/cpp-ch/local-engine/Storages/MergeTree/MergeSparkMergeTreeTask.cpp @@ -17,6 +17,7 @@ #include "MergeSparkMergeTreeTask.h" #include +#include #include #include #include @@ -149,8 +150,7 @@ void MergeSparkMergeTreeTask::prepare() task_context, //merge_mutate_entry->tagger->reserved_space, storage.tryReserveSpace( - MergeTreeDataMergerMutator::estimateNeededDiskSpace(future_part->parts), - future_part->parts[0]->getDataPartStorage()), + CompactionStatistics::estimateNeededDiskSpace(future_part->parts), future_part->parts[0]->getDataPartStorage()), deduplicate, deduplicate_by_columns, cleanup, From 89a72a14f0b59eb8bc8411baeb720cbb60877d56 Mon Sep 17 00:00:00 2001 From: Chang Chen Date: Tue, 14 Jan 2025 12:00:43 +0800 Subject: [PATCH 3/3] Fix UT due to https://github.com/ClickHouse/ClickHouse/pull/70102 --- cpp-ch/local-engine/Common/CHUtil.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index ace3a515a2c4..b2dfa3cf449d 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -77,6 +77,10 @@ namespace ServerSetting extern const ServerSettingsString primary_index_cache_policy; extern const ServerSettingsUInt64 primary_index_cache_size; extern const ServerSettingsDouble primary_index_cache_size_ratio; +extern const ServerSettingsString skipping_index_cache_policy; +extern const ServerSettingsUInt64 skipping_index_cache_size; +extern const ServerSettingsUInt64 skipping_index_cache_max_entries; +extern const ServerSettingsDouble skipping_index_cache_size_ratio; } namespace Setting { @@ -762,11 +766,11 @@ void BackendInitializerUtil::initContexts(DB::Context::ConfigurationPtr config) global_context->setConfig(config); auto tmp_path = config->getString("tmp_path", PathConfig::DEFAULT_TEMP_FILE_PATH); - if(config->getBool(PathConfig::USE_CURRENT_DIRECTORY_AS_TMP, false)) + if (config->getBool(PathConfig::USE_CURRENT_DIRECTORY_AS_TMP, false)) { char buffer[PATH_MAX]; if (getcwd(buffer, sizeof(buffer)) != nullptr) - tmp_path = std::string(buffer) + tmp_path; + tmp_path = std::string(buffer) + tmp_path; }; global_context->setTemporaryStoragePath(tmp_path, 0); @@ -805,6 +809,14 @@ void BackendInitializerUtil::initContexts(DB::Context::ConfigurationPtr config) double index_mark_cache_size_ratio = config->getDouble("index_mark_cache_size_ratio", DEFAULT_INDEX_MARK_CACHE_SIZE_RATIO); global_context->setIndexMarkCache(index_mark_cache_policy, index_mark_cache_size, index_mark_cache_size_ratio); + String skipping_index_cache_policy = server_settings[ServerSetting::skipping_index_cache_policy]; + size_t skipping_index_cache_size = server_settings[ServerSetting::skipping_index_cache_size]; + size_t skipping_index_cache_max_entries = server_settings[ServerSetting::skipping_index_cache_max_entries]; + double skipping_index_cache_size_ratio = server_settings[ServerSetting::skipping_index_cache_size_ratio]; + LOG_INFO(log, "Skipping index cache size to {}", formatReadableSizeWithBinarySuffix(skipping_index_cache_size)); + global_context->setSkippingIndexCache( + skipping_index_cache_policy, skipping_index_cache_size, skipping_index_cache_max_entries, skipping_index_cache_size_ratio); + size_t mmap_cache_size = config->getUInt64("mmap_cache_size", DEFAULT_MMAP_CACHE_MAX_SIZE); global_context->setMMappedFileCache(mmap_cache_size);