diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index c0b98650c3e0d2..933de634350ad0 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -28,6 +28,7 @@ #include "exec/schema_scanner/schema_active_queries_scanner.h" #include "exec/schema_scanner/schema_backend_active_tasks.h" +#include "exec/schema_scanner/schema_catalog_meta_cache_stats_scanner.h" #include "exec/schema_scanner/schema_charsets_scanner.h" #include "exec/schema_scanner/schema_collations_scanner.h" #include "exec/schema_scanner/schema_columns_scanner.h" @@ -236,6 +237,8 @@ std::unique_ptr SchemaScanner::create(TSchemaTableType::type type return SchemaBackendWorkloadGroupResourceUsage::create_unique(); case TSchemaTableType::SCH_TABLE_PROPERTIES: return SchemaTablePropertiesScanner::create_unique(); + case TSchemaTableType::SCH_CATALOG_META_CACHE_STATISTICS: + return SchemaCatalogMetaCacheStatsScanner::create_unique(); default: return SchemaDummyScanner::create_unique(); break; diff --git a/be/src/exec/schema_scanner/schema_catalog_meta_cache_stats_scanner.cpp b/be/src/exec/schema_scanner/schema_catalog_meta_cache_stats_scanner.cpp new file mode 100644 index 00000000000000..4c067057729f21 --- /dev/null +++ b/be/src/exec/schema_scanner/schema_catalog_meta_cache_stats_scanner.cpp @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/schema_scanner/schema_catalog_meta_cache_stats_scanner.h" + +#include "runtime/client_cache.h" +#include "runtime/exec_env.h" +#include "runtime/query_context.h" +#include "runtime/runtime_state.h" +#include "util/thrift_rpc_helper.h" +#include "vec/common/string_ref.h" +#include "vec/core/block.h" +#include "vec/data_types/data_type_factory.hpp" + +namespace doris { +std::vector SchemaCatalogMetaCacheStatsScanner::_s_tbls_columns = { + {"CATALOG_NAME", TYPE_STRING, sizeof(StringRef), true}, + {"CACHE_NAME", TYPE_STRING, sizeof(StringRef), true}, + {"METRIC_NAME", TYPE_STRING, sizeof(StringRef), true}, + {"METRIC_VALUE", TYPE_STRING, sizeof(StringRef), true}, +}; + +SchemaCatalogMetaCacheStatsScanner::SchemaCatalogMetaCacheStatsScanner() + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_CATALOG_META_CACHE_STATISTICS) {} + +SchemaCatalogMetaCacheStatsScanner::~SchemaCatalogMetaCacheStatsScanner() {} + +Status SchemaCatalogMetaCacheStatsScanner::start(RuntimeState* state) { + _block_rows_limit = state->batch_size(); + _rpc_timeout = state->execution_timeout() * 1000; + _fe_addr = state->get_query_ctx()->current_connect_fe; + return Status::OK(); +} + +Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { + TSchemaTableRequestParams schema_table_request_params; + for (int i = 0; i < _s_tbls_columns.size(); i++) { + schema_table_request_params.__isset.columns_name = true; + schema_table_request_params.columns_name.emplace_back(_s_tbls_columns[i].name); + } + schema_table_request_params.__set_current_user_ident(*_param->common_param->current_user_ident); + + TFetchSchemaTableDataRequest request; + request.__set_schema_table_name(TSchemaTableName::CATALOG_META_CACHE_STATS); + request.__set_schema_table_params(schema_table_request_params); + + TFetchSchemaTableDataResult result; + + RETURN_IF_ERROR(ThriftRpcHelper::rpc( + _fe_addr.hostname, _fe_addr.port, + [&request, &result](FrontendServiceConnection& client) { + client->fetchSchemaTableData(result, request); + }, + _rpc_timeout)); + + Status status(Status::create(result.status)); + if (!status.ok()) { + LOG(WARNING) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname + << ") failed, errmsg=" << status; + return status; + } + std::vector result_data = result.data_batch; + + _block = vectorized::Block::create_unique(); + for (int i = 0; i < _s_tbls_columns.size(); ++i) { + TypeDescriptor descriptor(_s_tbls_columns[i].type); + auto data_type = vectorized::DataTypeFactory::instance().create_data_type(descriptor, true); + _block->insert(vectorized::ColumnWithTypeAndName(data_type->create_column(), data_type, + _s_tbls_columns[i].name)); + } + + _block->reserve(_block_rows_limit); + + if (result_data.size() > 0) { + int col_size = result_data[0].column_value.size(); + if (col_size != _s_tbls_columns.size()) { + return Status::InternalError( + "catalog meta cache stats schema is not match for FE and BE"); + } + } + + for (int i = 0; i < result_data.size(); i++) { + TRow row = result_data[i]; + for (int j = 0; j < _s_tbls_columns.size(); j++) { + RETURN_IF_ERROR(insert_block_column(row.column_value[j], j, _block.get(), + _s_tbls_columns[j].type)); + } + } + return Status::OK(); +} + +Status SchemaCatalogMetaCacheStatsScanner::get_next_block_internal(vectorized::Block* block, + bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + + if (_block == nullptr) { + RETURN_IF_ERROR(_get_meta_cache_from_fe()); + _total_rows = _block->rows(); + } + + if (_row_idx == _total_rows) { + *eos = true; + return Status::OK(); + } + + int current_batch_rows = std::min(_block_rows_limit, _total_rows - _row_idx); + vectorized::MutableBlock mblock = vectorized::MutableBlock::build_mutable_block(block); + RETURN_IF_ERROR(mblock.add_rows(_block.get(), _row_idx, current_batch_rows)); + _row_idx += current_batch_rows; + + *eos = _row_idx == _total_rows; + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/schema_scanner/schema_catalog_meta_cache_stats_scanner.h b/be/src/exec/schema_scanner/schema_catalog_meta_cache_stats_scanner.h new file mode 100644 index 00000000000000..dd6c908deb326d --- /dev/null +++ b/be/src/exec/schema_scanner/schema_catalog_meta_cache_stats_scanner.h @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "exec/schema_scanner.h" + +namespace doris { +class RuntimeState; +namespace vectorized { +class Block; +} // namespace vectorized + +class SchemaCatalogMetaCacheStatsScanner : public SchemaScanner { + ENABLE_FACTORY_CREATOR(SchemaCatalogMetaCacheStatsScanner); + +public: + SchemaCatalogMetaCacheStatsScanner(); + ~SchemaCatalogMetaCacheStatsScanner() override; + + Status start(RuntimeState* state) override; + Status get_next_block_internal(vectorized::Block* block, bool* eos) override; + + static std::vector _s_tbls_columns; + +private: + Status _get_meta_cache_from_fe(); + + TNetworkAddress _fe_addr; + + int _block_rows_limit = 4096; + int _row_idx = 0; + int _total_rows = 0; + std::unique_ptr _block = nullptr; + int _rpc_timeout = 3000; +}; +}; // namespace doris diff --git a/be/src/vec/exec/scan/vmeta_scanner.h b/be/src/vec/exec/scan/vmeta_scanner.h index 8f952661b35b0d..b44511826f2c50 100644 --- a/be/src/vec/exec/scan/vmeta_scanner.h +++ b/be/src/vec/exec/scan/vmeta_scanner.h @@ -93,7 +93,6 @@ class VMetaScanner : public VScanner { TFetchSchemaTableDataRequest* request); Status _build_queries_metadata_request(const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request); - bool _meta_eos; TupleId _tuple_id; TUserIdentity _user_identity; diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index d5939715f3e511..b8cc93a4a5e38d 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2028,7 +2028,7 @@ public class Config extends ConfigBase { * Decrease this value if FE's memory is small */ @ConfField(mutable = false, masterOnly = false) - public static long max_external_file_cache_num = 100000; + public static long max_external_file_cache_num = 10000; /** * Max cache num of external table's schema diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java index b33659fbc66d2a..2b6bd0f089e091 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java @@ -85,7 +85,9 @@ public enum SchemaTableType { SCH_WORKLOAD_GROUP_RESOURCE_USAGE("WORKLOAD_GROUP_RESOURCE_USAGE", "WORKLOAD_GROUP_RESOURCE_USAGE", TSchemaTableType.SCH_WORKLOAD_GROUP_RESOURCE_USAGE), SCH_TABLE_PROPERTIES("TABLE_PROPERTIES", "TABLE_PROPERTIES", - TSchemaTableType.SCH_TABLE_PROPERTIES); + TSchemaTableType.SCH_TABLE_PROPERTIES), + SCH_CATALOG_META_CACHE_STATISTICS("CATALOG_META_CACHE_STATISTICS", "CATALOG_META_CACHE_STATISTICS", + TSchemaTableType.SCH_CATALOG_META_CACHE_STATISTICS); private static final String dbName = "INFORMATION_SCHEMA"; private static SelectList fullSelectLists; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java index 8fb071feb96de9..eeff956658fc31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java @@ -550,6 +550,14 @@ public class SchemaTable extends Table { .column("PROPERTY_NAME", ScalarType.createStringType()) .column("PROPERTY_VALUE", ScalarType.createStringType()) .build())) + .put("catalog_meta_cache_statistics", + new SchemaTable(SystemIdGenerator.getNextId(), "catalog_meta_cache_statistics", TableType.SCHEMA, + builder().column("CATALOG_NAME", ScalarType.createStringType()) + .column("CACHE_NAME", ScalarType.createStringType()) + .column("METRIC_NAME", ScalarType.createStringType()) + .column("METRIC_VALUE", ScalarType.createStringType()) + .build()) + ) .build(); private boolean fetchAllFe = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 513fc951672f9f..8759f9f5ef2f04 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -36,6 +36,7 @@ import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.RemovalListener; +import com.github.benmanes.caffeine.cache.stats.CacheStats; import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -296,4 +297,15 @@ public MetaCache buildMetaCache(String name, maxSize, namesCacheLoader, metaObjCacheLoader, removalListener); return metaCache; } + + public static Map getCacheStats(CacheStats cacheStats, long estimatedSize) { + Map stats = Maps.newHashMap(); + stats.put("hit_ratio", String.valueOf(cacheStats.hitRate())); + stats.put("hit_count", String.valueOf(cacheStats.hitCount())); + stats.put("read_count", String.valueOf(cacheStats.hitCount() + cacheStats.missCount())); + stats.put("eviction_count", String.valueOf(cacheStats.evictionCount())); + stats.put("average_load_penalty", String.valueOf(cacheStats.averageLoadPenalty())); + stats.put("estimated_size", String.valueOf(estimatedSize)); + return stats; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index 006ed83413a233..218197de51dcd0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -36,6 +36,7 @@ import org.apache.doris.common.util.LocationPath; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CacheException; +import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.hive.AcidInfo.DeleteDeltaInfo; import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.fs.FileSystemCache; @@ -141,7 +142,7 @@ private void init() { OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_hive_partition_table_cache_num, - false, + true, null); partitionValuesCache = partitionValuesCacheFactory.buildCache(key -> loadPartitionValues(key), null, refreshExecutor); @@ -150,7 +151,7 @@ private void init() { OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_hive_partition_cache_num, - false, + true, null); partitionCache = partitionCacheFactory.buildCache(new CacheLoader() { @Override @@ -183,7 +184,7 @@ public void setNewFileCache() { ? fileMetaCacheTtlSecond : 28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_external_file_cache_num, - false, + true, null); CacheLoader loader = new CacheBulkLoader() { @@ -1125,4 +1126,19 @@ public HivePartitionValues copy() { return copy; } } + + /** + * get cache stats + * @return metric value>> + */ + public Map> getStats() { + Map> res = Maps.newHashMap(); + res.put("hive_partition_values_cache", ExternalMetaCacheMgr.getCacheStats(partitionValuesCache.stats(), + partitionCache.estimatedSize())); + res.put("hive_partition_cache", + ExternalMetaCacheMgr.getCacheStats(partitionCache.stats(), partitionCache.estimatedSize())); + res.put("hive_file_cache", + ExternalMetaCacheMgr.getCacheStats(fileCacheRef.get().stats(), fileCacheRef.get().estimatedSize())); + return res; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java index 2372b88e0db32e..62094b21c2ba1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java @@ -21,6 +21,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CacheException; +import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.TablePartitionValues; import org.apache.doris.datasource.TablePartitionValues.TablePartitionKey; import org.apache.doris.datasource.hive.HMSExternalCatalog; @@ -28,6 +29,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; @@ -37,6 +39,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.OptionalLong; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; @@ -55,7 +58,7 @@ public HudiCachedPartitionProcessor(long catalogId, ExecutorService executor) { OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60), Config.max_external_table_cache_num, - false, + true, null); this.partitionCache = partitionCacheFactory.buildCache(key -> new TablePartitionValues(), null, executor); } @@ -167,4 +170,11 @@ public TablePartitionValues getPartitionValues(HMSExternalTable table, HoodieTab throw new CacheException("Failed to get hudi partitions: " + Util.getRootCauseMessage(e), e); } } + + public Map> getCacheStats() { + Map> res = Maps.newHashMap(); + res.put("hudi_partition_cache", ExternalMetaCacheMgr.getCacheStats(partitionCache.stats(), + partitionCache.estimatedSize())); + return res; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java index 13bd9650978f81..a35c73dc296db5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java @@ -22,6 +22,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.UserException; import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper; import org.apache.doris.datasource.property.constants.HMSProperties; @@ -31,6 +32,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.ManifestFiles; import org.apache.iceberg.SerializableTable; @@ -58,7 +60,7 @@ public IcebergMetadataCache(ExecutorService executor) { OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60), Config.max_external_table_cache_num, - false, + true, null); this.snapshotListCache = snapshotListCacheFactory.buildCache(key -> loadSnapshots(key), null, executor); @@ -66,7 +68,7 @@ public IcebergMetadataCache(ExecutorService executor) { OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60), Config.max_external_table_cache_num, - false, + true, null); this.tableCache = tableCacheFactory.buildCache(key -> loadTable(key), null, executor); } @@ -246,4 +248,13 @@ public int hashCode() { return Objects.hash(catalog.getId(), dbName, tableName); } } + + public Map> getCacheStats() { + Map> res = Maps.newHashMap(); + res.put("iceberg_snapshot_cache", ExternalMetaCacheMgr.getCacheStats(snapshotListCache.stats(), + snapshotListCache.estimatedSize())); + res.put("iceberg_table_cache", ExternalMetaCacheMgr.getCacheStats(tableCache.stats(), + tableCache.estimatedSize())); + return res; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index c8105a81dcf743..929497cbaab665 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -38,8 +38,12 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HiveMetaStoreCache; +import org.apache.doris.datasource.hudi.source.HudiCachedPartitionProcessor; +import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergMetadataCache; import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog; import org.apache.doris.job.common.JobType; @@ -112,6 +116,8 @@ public class MetadataGenerator { private static final ImmutableMap TABLE_PROPERTIES_COLUMN_TO_INDEX; + private static final ImmutableMap META_CACHE_STATS_COLUMN_TO_INDEX; + static { ImmutableMap.Builder activeQueriesbuilder = new ImmutableMap.Builder(); List activeQueriesColList = SchemaTable.TABLE_MAP.get("active_queries").getFullSchema(); @@ -152,6 +158,13 @@ public class MetadataGenerator { propertiesBuilder.put(propertiesColList.get(i).getName().toLowerCase(), i); } TABLE_PROPERTIES_COLUMN_TO_INDEX = propertiesBuilder.build(); + + ImmutableMap.Builder metaCacheBuilder = new ImmutableMap.Builder(); + List metaCacheColList = SchemaTable.TABLE_MAP.get("catalog_meta_cache_statistics").getFullSchema(); + for (int i = 0; i < metaCacheColList.size(); i++) { + metaCacheBuilder.put(metaCacheColList.get(i).getName().toLowerCase(), i); + } + META_CACHE_STATS_COLUMN_TO_INDEX = metaCacheBuilder.build(); } public static TFetchSchemaTableDataResult getMetadataTable(TFetchSchemaTableDataRequest request) throws TException { @@ -239,6 +252,10 @@ public static TFetchSchemaTableDataResult getSchemaTableData(TFetchSchemaTableDa result = tablePropertiesMetadataResult(schemaTableParams); columnIndex = TABLE_PROPERTIES_COLUMN_TO_INDEX; break; + case CATALOG_META_CACHE_STATS: + result = metaCacheStatsMetadataResult(schemaTableParams); + columnIndex = META_CACHE_STATS_COLUMN_TO_INDEX; + break; default: return errorResult("invalid schema table name."); } @@ -1124,4 +1141,50 @@ private static TFetchSchemaTableDataResult tablePropertiesMetadataResult(TSchema result.setStatus(new TStatus(TStatusCode.OK)); return result; } + + private static TFetchSchemaTableDataResult metaCacheStatsMetadataResult(TSchemaTableRequestParams params) { + List dataBatch = Lists.newArrayList(); + TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult(); + ExternalMetaCacheMgr mgr = Env.getCurrentEnv().getExtMetaCacheMgr(); + for (CatalogIf catalogIf : Env.getCurrentEnv().getCatalogMgr().getCopyOfCatalog()) { + if (catalogIf instanceof HMSExternalCatalog) { + HMSExternalCatalog catalog = (HMSExternalCatalog) catalogIf; + // 1. hive metastore cache + HiveMetaStoreCache cache = mgr.getMetaStoreCache(catalog); + if (cache != null) { + fillBatch(dataBatch, cache.getStats(), catalog.getName()); + } + // 2. hudi cache + HudiCachedPartitionProcessor processor + = (HudiCachedPartitionProcessor) mgr.getHudiPartitionProcess(catalog); + fillBatch(dataBatch, processor.getCacheStats(), catalog.getName()); + } else if (catalogIf instanceof IcebergExternalCatalog) { + // 3. iceberg cache + IcebergMetadataCache icebergCache = mgr.getIcebergMetadataCache(); + fillBatch(dataBatch, icebergCache.getCacheStats(), catalogIf.getName()); + } + } + + result.setDataBatch(dataBatch); + result.setStatus(new TStatus(TStatusCode.OK)); + return result; + } + + private static void fillBatch(List dataBatch, Map> stats, + String catalogName) { + for (Map.Entry> entry : stats.entrySet()) { + String cacheName = entry.getKey(); + Map cacheStats = entry.getValue(); + for (Map.Entry cacheStatsEntry : cacheStats.entrySet()) { + String metricName = cacheStatsEntry.getKey(); + String metricValue = cacheStatsEntry.getValue(); + TRow trow = new TRow(); + trow.addToColumnValue(new TCell().setStringVal(catalogName)); // CATALOG_NAME + trow.addToColumnValue(new TCell().setStringVal(cacheName)); // CACHE_NAME + trow.addToColumnValue(new TCell().setStringVal(metricName)); // METRIC_NAME + trow.addToColumnValue(new TCell().setStringVal(metricValue)); // METRIC_VALUE + dataBatch.add(trow); + } + } + } } diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index 753c178bbef1df..9382bb1e2c44f9 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -136,7 +136,8 @@ enum TSchemaTableType { SCH_WORKLOAD_GROUP_PRIVILEGES = 48, SCH_WORKLOAD_GROUP_RESOURCE_USAGE = 49, SCH_TABLE_PROPERTIES = 50, - SCH_FILE_CACHE_STATISTICS = 51 + SCH_FILE_CACHE_STATISTICS = 51, + SCH_CATALOG_META_CACHE_STATISTICS = 52; } enum THdfsCompression { diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index c0176ef5135cb1..157b0a40ddeff6 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -957,6 +957,7 @@ enum TSchemaTableName { TABLE_OPTIONS = 6, WORKLOAD_GROUP_PRIVILEGES = 7, TABLE_PROPERTIES = 8, + CATALOG_META_CACHE_STATS = 9, } struct TMetadataTableRequestParams { @@ -971,6 +972,7 @@ struct TMetadataTableRequestParams { 9: optional PlanNodes.TJobsMetadataParams jobs_metadata_params 10: optional PlanNodes.TTasksMetadataParams tasks_metadata_params 11: optional PlanNodes.TPartitionsMetadataParams partitions_metadata_params + 12: optional PlanNodes.TMetaCacheStatsParams meta_cache_stats_params } struct TSchemaTableRequestParams { diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index a060f5efab44c5..d6690fe77a12f7 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -528,6 +528,9 @@ struct TQueriesMetadataParams { 6: optional TPartitionsMetadataParams partitions_params } +struct TMetaCacheStatsParams { +} + struct TMetaScanRange { 1: optional Types.TMetadataType metadata_type 2: optional TIcebergMetadataParams iceberg_params @@ -538,6 +541,7 @@ struct TMetaScanRange { 7: optional TJobsMetadataParams jobs_params 8: optional TTasksMetadataParams tasks_params 9: optional TPartitionsMetadataParams partitions_params + 10: optional TMetaCacheStatsParams meta_cache_stats_params } // Specification of an individual data range which is held in its entirety diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index eef033ff51818d..9290768e0dbe65 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -712,7 +712,7 @@ enum TMetadataType { JOBS, TASKS, WORKLOAD_SCHED_POLICY, - PARTITIONS + PARTITIONS; } enum TIcebergQueryType { diff --git a/regression-test/suites/external_table_p0/hive/test_external_catalog_hive.groovy b/regression-test/suites/external_table_p0/hive/test_external_catalog_hive.groovy index b920168198acb7..b71c72957e9c02 100644 --- a/regression-test/suites/external_table_p0/hive/test_external_catalog_hive.groovy +++ b/regression-test/suites/external_table_p0/hive/test_external_catalog_hive.groovy @@ -178,5 +178,9 @@ suite("test_external_catalog_hive", "p0,external,hive,external_docker,external_d """ exception "Failed to init access controller: bound must be positive" } + + // test catalog_meta_cache_statistics + sql """select * from internal.information_schema.catalog_meta_cache_statistics;""" + sql """select * from ${catalog_name}.information_schema.catalog_meta_cache_statistics where catalog_name="${catalog_name}";""" } } diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy index 15c23fa74d8fa4..2c37ba91207ec1 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy @@ -62,6 +62,10 @@ suite("test_iceberg_table_stats", "p0,external,doris,external_docker,external_do assert_stats("sample_mor_orc", "1000") assert_stats("sample_mor_parquet", "1000") + // test catalog_meta_cache_statistics + sql """select * from information_schema.catalog_meta_cache_statistics;""" + sql """select * from information_schema.catalog_meta_cache_statistics where catalog_name="${catalog_name}";""" + } finally { } }