From 13772d5c5272633e92e649b7c94ab007c966c8fa Mon Sep 17 00:00:00 2001 From: kikyo Date: Wed, 2 Aug 2023 17:26:55 +0800 Subject: [PATCH] handle exception properly --- .../doris/statistics/AnalysisManager.java | 27 +++++-- .../statistics/StatisticsAutoAnalyzer.java | 74 +++++++++++++++++-- 2 files changed, 85 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index e549bd6b0bda5a..ef909728c78af6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -18,6 +18,7 @@ package org.apache.doris.statistics; import org.apache.doris.analysis.AnalyzeDBStmt; +import org.apache.doris.analysis.AnalyzeProperties; import org.apache.doris.analysis.AnalyzeStmt; import org.apache.doris.analysis.AnalyzeTblStmt; import org.apache.doris.analysis.DropAnalyzeJobStmt; @@ -169,6 +170,13 @@ public void createAnalyze(AnalyzeStmt analyzeStmt, boolean proxy) throws DdlExce public void createAnalysisJobs(AnalyzeDBStmt analyzeDBStmt, boolean proxy) throws DdlException { DatabaseIf db = analyzeDBStmt.getDb(); + List analysisInfos = buildAnalysisInfosForDB(db, analyzeDBStmt.getAnalyzeProperties()); + if (!analyzeDBStmt.isSync()) { + sendJobId(analysisInfos, proxy); + } + } + + public List buildAnalysisInfosForDB(DatabaseIf db, AnalyzeProperties analyzeProperties) { List tbls = db.getTables(); List analysisInfos = new ArrayList<>(); db.readLock(); @@ -178,27 +186,30 @@ public void createAnalysisJobs(AnalyzeDBStmt analyzeDBStmt, boolean proxy) throw if (table instanceof View) { continue; } - TableName tableName = new TableName(analyzeDBStmt.getCtlIf().getName(), db.getFullName(), + TableName tableName = new TableName(db.getCatalog().getName(), db.getFullName(), table.getName()); - AnalyzeTblStmt analyzeTblStmt = new AnalyzeTblStmt(analyzeDBStmt.getAnalyzeProperties(), tableName, + AnalyzeTblStmt analyzeTblStmt = new AnalyzeTblStmt(analyzeProperties, tableName, null, db.getId(), table); try { analyzeTblStmt.check(); } catch (AnalysisException analysisException) { - throw new DdlException(analysisException.getMessage(), analysisException); + LOG.warn("Failed to build analyze job: {}", + analysisException.getMessage(), analysisException); } analyzeStmts.add(analyzeTblStmt); } for (AnalyzeTblStmt analyzeTblStmt : analyzeStmts) { - analysisInfos.add(buildAndAssignJob(analyzeTblStmt)); - } - if (!analyzeDBStmt.isSync()) { - sendJobId(analysisInfos, proxy); + try { + analysisInfos.add(buildAndAssignJob(analyzeTblStmt)); + } catch (DdlException e) { + LOG.warn("Failed to build analyze job: {}", + e.getMessage(), e); + } } + return analysisInfos; } finally { db.readUnlock(); } - } // Each analyze stmt corresponding to an analysis job. diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java index aae783ca8b278b..b29897304a4f84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java @@ -56,14 +56,59 @@ protected void runAfterCatalogReady() { if (!StatisticsUtil.statsTblAvailable()) { return; } +<<<<<<< HEAD if (Config.enable_auto_collect_statistics) { analyzePeriodically(); +======= + if (!checkAnalyzeTime(LocalTime.now(TimeUtils.getTimeZone().toZoneId()))) { + return; + } + + if (!analysisTaskExecutor.idle()) { + return; + } + + analyzePeriodically(); + if (!Config.enable_full_auto_analyze) { +>>>>>>> 43614cc44f (handle exception properly) analyzeAutomatically(); } } +<<<<<<< HEAD public void autoAnalyzeStats(DdlStmt ddlStmt) { // TODO Monitor some DDL statements, and then trigger automatic analysis tasks +======= + @SuppressWarnings({"rawtypes", "unchecked"}) + private void analyzeAll() { + Set catalogs = Env.getCurrentEnv().getCatalogMgr().getCopyOfCatalog(); + for (CatalogIf ctl : catalogs) { + + Collection dbs = ctl.getAllDbs(); + for (DatabaseIf databaseIf : dbs) { + if (StatisticConstants.STATISTICS_DB_BLACK_LIST.contains(databaseIf.getFullName())) { + continue; + } + AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); + List analysisInfos = analysisManager.buildAnalysisInfosForDB(databaseIf, + AnalyzeProperties.DEFAULT_PROP); + for (AnalysisInfo analysisInfo : analysisInfos) { + analysisInfo = getReAnalyzeRequiredPart(analysisInfo); + if (analysisInfo == null) { + continue; + } + try { + analysisManager.createSystemAnalysisJob(analysisInfo, analysisTaskExecutor); + } catch (Exception e) { + LOG.warn("Failed to create analysis job", e); + } + } + } + + } + + analyzePeriodically(); +>>>>>>> 43614cc44f (handle exception properly) } private void analyzePeriodically() { @@ -116,16 +161,24 @@ private void analyzeAutomatically() { * @return new job info after check * @throws Throwable failed to check */ +<<<<<<< HEAD private AnalysisInfo checkAutomaticJobInfo(AnalysisInfo jobInfo) throws Throwable { +======= + private AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) { +>>>>>>> 43614cc44f (handle exception properly) long lastExecTimeInMs = jobInfo.lastExecTimeInMs; TableIf table = StatisticsUtil .findTable(jobInfo.catalogName, jobInfo.dbName, jobInfo.tblName); - TableStatistic tblStats = StatisticsRepository.fetchTableLevelStats(table.getId()); + TableStatistic tblStats = null; + try { + tblStats = StatisticsRepository.fetchTableLevelStats(table.getId()); + } catch (Throwable t) { + LOG.warn("Failed to fetch table stats", t); + return null; + } if (tblStats == TableStatistic.UNKNOWN) { - LOG.warn("Failed to automatically analyze statistics, " - + "no corresponding table statistics for job: {}", jobInfo.toString()); - throw new DdlException("No corresponding table statistics for automatic job."); + return jobInfo; } if (!needReanalyzeTable(table, tblStats)) { @@ -156,7 +209,7 @@ private boolean needReanalyzeTable(TableIf table, TableStatistic tblStats) { } private void checkAnalyzedPartitions(TableIf table, Set statsPartitions, - Set needRunPartitions, long lastExecTimeInMs) throws DdlException { + Set needRunPartitions, long lastExecTimeInMs) { for (String statsPartition : statsPartitions) { Partition partition = table.getPartition(statsPartition); if (partition == null) { @@ -165,12 +218,17 @@ private void checkAnalyzedPartitions(TableIf table, Set statsPartitions, needRunPartitions.add(statsPartition); continue; } - TableStatistic partitionStats = StatisticsRepository + TableStatistic partitionStats = null; + try { + partitionStats = StatisticsRepository .fetchTableLevelOfPartStats(partition.getId()); - if (partitionStats == TableStatistic.UNKNOWN) { + } catch (DdlException e) { + LOG.warn("Failed to fetch part stats", e); continue; } - if (needReanalyzePartition(lastExecTimeInMs, partition, partitionStats)) { + + if (needReanalyzePartition(lastExecTimeInMs, partition, partitionStats) + || partitionStats == TableStatistic.UNKNOWN) { needRunPartitions.add(partition.getName()); } }