diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index 63ab9232341a1e..59a2035d197500 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -174,8 +174,7 @@ public void createAnalysisJobs(AnalyzeDBStmt analyzeDBStmt, boolean proxy) throw } } - public List buildAnalysisInfosForDB(DatabaseIf db, AnalyzeProperties analyzeProperties) - throws DdlException { + public List buildAnalysisInfosForDB(DatabaseIf db, AnalyzeProperties analyzeProperties) { List tbls = db.getTables(); List analysisInfos = new ArrayList<>(); db.readLock(); @@ -195,12 +194,18 @@ public List buildAnalysisInfosForDB(DatabaseIf db, Analyz try { analyzeTblStmt.check(); } catch (AnalysisException analysisException) { - throw new DdlException(analysisException.getMessage(), analysisException); + LOG.warn("Failed to build analyze job: {}", + analysisException.getMessage(), analysisException); } analyzeStmts.add(analyzeTblStmt); } for (AnalyzeTblStmt analyzeTblStmt : analyzeStmts) { - analysisInfos.add(buildAndAssignJob(analyzeTblStmt)); + try { + analysisInfos.add(buildAndAssignJob(analyzeTblStmt)); + } catch (DdlException e) { + LOG.warn("Failed to build analyze job: {}", + e.getMessage(), e); + } } } finally { db.readUnlock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java index 02fe96ec71a5ca..ebe687d78e80b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java @@ -73,8 +73,8 @@ protected void runAfterCatalogReady() { return; } + analyzePeriodically(); if (!Config.enable_full_auto_analyze) { - analyzePeriodically(); analyzeAutomatically(); } else { analyzeAll(); @@ -85,26 +85,28 @@ protected void runAfterCatalogReady() { private void analyzeAll() { Set catalogs = Env.getCurrentEnv().getCatalogMgr().getCopyOfCatalog(); for (CatalogIf ctl : catalogs) { - try { - Collection dbs = ctl.getAllDbs(); - for (DatabaseIf databaseIf : dbs) { - if (StatisticConstants.STATISTICS_DB_BLACK_LIST.contains(databaseIf.getFullName())) { + + Collection dbs = ctl.getAllDbs(); + for (DatabaseIf databaseIf : dbs) { + if (StatisticConstants.STATISTICS_DB_BLACK_LIST.contains(databaseIf.getFullName())) { + continue; + } + AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); + List analysisInfos = analysisManager.buildAnalysisInfosForDB(databaseIf, + AnalyzeProperties.DEFAULT_PROP); + for (AnalysisInfo analysisInfo : analysisInfos) { + analysisInfo = getReAnalyzeRequiredPart(analysisInfo); + if (analysisInfo == null) { continue; } - AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); - List analysisInfos = analysisManager.buildAnalysisInfosForDB(databaseIf, - AnalyzeProperties.DEFAULT_PROP); - for (AnalysisInfo analysisInfo : analysisInfos) { - analysisInfo = getReAnalyzeRequiredPart(analysisInfo); - if (analysisInfo == null) { - continue; - } + try { analysisManager.createSystemAnalysisJob(analysisInfo, analysisTaskExecutor); + } catch (Exception e) { + LOG.warn("Failed to create analysis job", e); } } - } catch (Throwable t) { - LOG.warn("Failed to analyze all statistics.", t); } + } analyzePeriodically(); @@ -160,16 +162,20 @@ private void analyzeAutomatically() { * @return new job info after check * @throws Throwable failed to check */ - private AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) throws Throwable { + private AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) { long lastExecTimeInMs = jobInfo.lastExecTimeInMs; TableIf table = StatisticsUtil .findTable(jobInfo.catalogName, jobInfo.dbName, jobInfo.tblName); - TableStatistic tblStats = StatisticsRepository.fetchTableLevelStats(table.getId()); + TableStatistic tblStats = null; + try { + tblStats = StatisticsRepository.fetchTableLevelStats(table.getId()); + } catch (Throwable t) { + LOG.warn("Failed to fetch table stats", t); + return null; + } if (tblStats == TableStatistic.UNKNOWN) { - LOG.warn("Failed to automatically analyze statistics, " - + "no corresponding table statistics for job: {}", jobInfo.toString()); - throw new DdlException("No corresponding table statistics for automatic job."); + return jobInfo; } if (!needReanalyzeTable(table, tblStats)) { @@ -200,7 +206,7 @@ private boolean needReanalyzeTable(TableIf table, TableStatistic tblStats) { } private void checkAnalyzedPartitions(TableIf table, Set statsPartitions, - Set needRunPartitions, long lastExecTimeInMs) throws DdlException { + Set needRunPartitions, long lastExecTimeInMs) { for (String statsPartition : statsPartitions) { Partition partition = table.getPartition(statsPartition); if (partition == null) { @@ -209,12 +215,17 @@ private void checkAnalyzedPartitions(TableIf table, Set statsPartitions, needRunPartitions.add(statsPartition); continue; } - TableStatistic partitionStats = StatisticsRepository + TableStatistic partitionStats = null; + try { + partitionStats = StatisticsRepository .fetchTableLevelOfPartStats(partition.getId()); - if (partitionStats == TableStatistic.UNKNOWN) { + } catch (DdlException e) { + LOG.warn("Failed to fetch part stats", e); continue; } - if (needReanalyzePartition(lastExecTimeInMs, partition, partitionStats)) { + + if (needReanalyzePartition(lastExecTimeInMs, partition, partitionStats) + || partitionStats == TableStatistic.UNKNOWN) { needRunPartitions.add(partition.getName()); } }