From 7486df064e8faf291bc04aeb05ad9d3df6f6e3e6 Mon Sep 17 00:00:00 2001 From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com> Date: Thu, 15 Aug 2024 22:05:00 +0800 Subject: [PATCH] [improvement](statistics)Support drop cached stats. (#39367) Support drop cached stats. Usage: `drop cached stats table_name` --- fe/fe-core/src/main/cup/sql_parser.cup | 4 + .../doris/analysis/DropCachedStatsStmt.java | 116 ++++++++++++++++++ .../doris/analysis/ShowColumnStatsStmt.java | 4 +- .../java/org/apache/doris/qe/DdlExecutor.java | 3 + .../org/apache/doris/qe/ShowExecutor.java | 6 +- .../doris/statistics/AnalysisManager.java | 28 ++++- .../statistics/test_drop_cached_stats.groovy | 57 +++++++++ 7 files changed, 212 insertions(+), 6 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/DropCachedStatsStmt.java create mode 100644 regression-test/suites/statistics/test_drop_cached_stats.groovy diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 7feb912a0c7c27..1c2d83473987a9 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -3103,6 +3103,10 @@ drop_stmt ::= {: RESULT = new DropStatsStmt(tbl, cols); :} + | KW_DROP KW_CACHED KW_STATS table_name:tbl + {: + RESULT = new DropCachedStatsStmt(tbl); + :} | KW_DROP KW_EXPIRED KW_STATS {: RESULT = new DropStatsStmt(true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropCachedStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropCachedStatsStmt.java new file mode 100644 index 00000000000000..0e721bef3f0e3b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropCachedStatsStmt.java @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.UserException; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + +/** + * Manually drop cached statistics for table and its mv. + *

+ * syntax: + * DROP CACHED STATS TableName; + */ +public class DropCachedStatsStmt extends DdlStmt { + + private final TableName tableName; + + private long catalogId; + private long dbId; + private long tblId; + + public DropCachedStatsStmt(TableName tableName) { + this.tableName = tableName; + } + + @Override + public void analyze(Analyzer analyzer) throws UserException { + super.analyze(analyzer); + if (tableName == null) { + throw new UserException("Should specify a valid table name."); + } + tableName.analyze(analyzer); + String catalogName = tableName.getCtl(); + String dbName = tableName.getDb(); + String tblName = tableName.getTbl(); + CatalogIf catalog = analyzer.getEnv().getCatalogMgr() + .getCatalogOrAnalysisException(catalogName); + DatabaseIf db = catalog.getDbOrAnalysisException(dbName); + TableIf table = db.getTableOrAnalysisException(tblName); + tblId = table.getId(); + dbId = db.getId(); + catalogId = catalog.getId(); + // check permission + checkAnalyzePriv(catalogName, db.getFullName(), table.getName()); + } + + public long getTblId() { + return tblId; + } + + public long getDbId() { + return dbId; + } + + public long getCatalogIdId() { + return catalogId; + } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder(); + sb.append("DROP CACHED STATS "); + + if (tableName != null) { + sb.append(tableName.toSql()); + } + + return sb.toString(); + } + + @Override + public String toString() { + return toSql(); + } + + private void checkAnalyzePriv(String catalogName, String dbName, String tblName) throws AnalysisException { + if (!Env.getCurrentEnv().getAccessManager() + .checkTblPriv(ConnectContext.get(), catalogName, dbName, tblName, + PrivPredicate.DROP)) { + ErrorReport.reportAnalysisException( + ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, + "DROP", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), + dbName + "." + tblName); + } + } + + @Override + public RedirectStatus getRedirectStatus() { + return RedirectStatus.NO_FORWARD; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java index a8d0284c138823..d30b2efcb87e1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java @@ -143,8 +143,8 @@ public ShowResultSet constructResultSet(List, ColumnSt return; } List row = Lists.newArrayList(); - row.add(p.first.first); row.add(p.first.second); + row.add(p.first.first); row.add(String.valueOf(p.second.count)); row.add(String.valueOf(p.second.ndv)); row.add(String.valueOf(p.second.numNulls)); @@ -153,7 +153,7 @@ public ShowResultSet constructResultSet(List, ColumnSt row.add(String.valueOf(p.second.minExpr == null ? "N/A" : p.second.minExpr.toSql())); row.add(String.valueOf(p.second.maxExpr == null ? "N/A" : p.second.maxExpr.toSql())); ColStatsMeta colStatsMeta = Env.getCurrentEnv().getAnalysisManager().findColStatsMeta(table.getId(), - p.first.first); + p.first.second); row.add(String.valueOf(colStatsMeta == null ? "N/A" : colStatsMeta.analysisMethod)); row.add(String.valueOf(colStatsMeta == null ? "N/A" : colStatsMeta.analysisType)); row.add(String.valueOf(colStatsMeta == null ? "N/A" : colStatsMeta.jobType)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java index c06ddde23ac9c4..a9bc7924b68368 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java @@ -75,6 +75,7 @@ import org.apache.doris.analysis.DdlStmt; import org.apache.doris.analysis.DeleteStmt; import org.apache.doris.analysis.DropAnalyzeJobStmt; +import org.apache.doris.analysis.DropCachedStatsStmt; import org.apache.doris.analysis.DropCatalogStmt; import org.apache.doris.analysis.DropDbStmt; import org.apache.doris.analysis.DropEncryptKeyStmt; @@ -331,6 +332,8 @@ public static void execute(Env env, DdlStmt ddlStmt) throws Exception { ProfileManager.getInstance().cleanProfile(); } else if (ddlStmt instanceof DropStatsStmt) { env.getAnalysisManager().dropStats((DropStatsStmt) ddlStmt); + } else if (ddlStmt instanceof DropCachedStatsStmt) { + env.getAnalysisManager().dropCachedStats((DropCachedStatsStmt) ddlStmt); } else if (ddlStmt instanceof KillAnalysisJobStmt) { env.getAnalysisManager().handleKillAnalyzeStmt((KillAnalysisJobStmt) ddlStmt); } else if (ddlStmt instanceof CleanQueryStatsStmt) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 8e1c8a847324d3..7a1c3f0d0ab01d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -2559,16 +2559,16 @@ private void getStatsForSpecifiedColumns(List, ColumnS ColumnStatistic columnStatistic = Env.getCurrentEnv().getStatisticsCache().getColumnStatistics( tableIf.getDatabase().getCatalog().getId(), tableIf.getDatabase().getId(), tableIf.getId(), indexId, colName); - columnStatistics.add(Pair.of(Pair.of(colName, indexName), columnStatistic)); + columnStatistics.add(Pair.of(Pair.of(indexName, colName), columnStatistic)); } else if (partitionNames == null) { ColumnStatistic columnStatistic = StatisticsRepository.queryColumnStatisticsByName(tableIf.getId(), indexId, colName); - columnStatistics.add(Pair.of(Pair.of(colName, indexName), columnStatistic)); + columnStatistics.add(Pair.of(Pair.of(indexName, colName), columnStatistic)); } else { String finalIndexName = indexName; columnStatistics.addAll(StatisticsRepository.queryColumnStatisticsByPartitions(tableName, colName, partitionNames.getPartitionNames()) - .stream().map(s -> Pair.of(Pair.of(colName, finalIndexName), s)) + .stream().map(s -> Pair.of(Pair.of(finalIndexName, colName), s)) .collect(Collectors.toList())); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index 1289b11bb85c54..e7634af135732c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.AnalyzeStmt; import org.apache.doris.analysis.AnalyzeTblStmt; import org.apache.doris.analysis.DropAnalyzeJobStmt; +import org.apache.doris.analysis.DropCachedStatsStmt; import org.apache.doris.analysis.DropStatsStmt; import org.apache.doris.analysis.KillAnalysisJobStmt; import org.apache.doris.analysis.ShowAnalyzeStmt; @@ -676,6 +677,13 @@ private ThreadPoolExecutor createThreadPoolForSyncAnalyze() { StatisticsUtil.getAnalyzeTimeout())); } + public void dropCachedStats(DropCachedStatsStmt stmt) { + long catalogId = stmt.getCatalogIdId(); + long dbId = stmt.getDbId(); + long tblId = stmt.getTblId(); + dropCachedStats(catalogId, dbId, tblId); + } + public void dropStats(DropStatsStmt dropStatsStmt) throws DdlException { if (dropStatsStmt.dropExpired) { Env.getCurrentEnv().getStatisticsCleaner().clear(); @@ -711,8 +719,26 @@ public void dropStats(TableIf table) throws DdlException { StatisticsRepository.dropStatistics(table.getId(), cols); } + public void dropCachedStats(long catalogId, long dbId, long tableId) { + TableIf table = StatisticsUtil.findTable(catalogId, dbId, tableId); + StatisticsCache statsCache = Env.getCurrentEnv().getStatisticsCache(); + Set columns = table.getSchemaAllIndexes(false) + .stream().map(Column::getName).collect(Collectors.toSet()); + for (String column : columns) { + List indexIds = Lists.newArrayList(); + if (table instanceof OlapTable) { + indexIds = ((OlapTable) table).getMvColumnIndexIds(column); + } else { + indexIds.add(-1L); + } + for (long indexId : indexIds) { + statsCache.invalidate(tableId, indexId, column); + } + } + } + public void invalidateLocalStats(long catalogId, long dbId, long tableId, - Set columns, TableStatsMeta tableStats) { + Set columns, TableStatsMeta tableStats) { if (tableStats == null) { return; } diff --git a/regression-test/suites/statistics/test_drop_cached_stats.groovy b/regression-test/suites/statistics/test_drop_cached_stats.groovy new file mode 100644 index 00000000000000..a9d328b19fbfd8 --- /dev/null +++ b/regression-test/suites/statistics/test_drop_cached_stats.groovy @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_drop_cached_stats") { + + sql """drop database if exists test_drop_cached_stats""" + sql """create database test_drop_cached_stats""" + sql """use test_drop_cached_stats""" + sql """set global enable_auto_analyze=false""" + + sql """CREATE TABLE drop_cache_test ( + key1 int NOT NULL, + value1 varchar(25) NOT NULL, + value2 varchar(125) NOT NULL + )ENGINE=OLAP + DUPLICATE KEY(`key1`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`key1`) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ) + """ + createMV("create materialized view mv1 as select key1 from drop_cache_test;") + + sql """insert into drop_cache_test values (1, "1", "1")""" + sql """analyze table drop_cache_test with sync""" + + def result = sql """show column stats drop_cache_test""" + assertEquals(4, result.size()) + result = sql """show column cached stats drop_cache_test""" + assertEquals(4, result.size()) + + sql """drop cached stats drop_cache_test""" + result = sql """show column cached stats drop_cache_test""" + assertEquals(0, result.size()) + + result = sql """show column stats drop_cache_test""" + assertEquals(4, result.size()) + + + sql """drop database if exists test_drop_cached_stats""" +} +