diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 962d9c0ac56567..7224b82d91d267 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -434,6 +434,8 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_AUTO_ANALYZE = "enable_auto_analyze"; + public static final String AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD = "auto_analyze_table_width_threshold"; + public static final String FASTER_FLOAT_CONVERT = "faster_float_convert"; public static final String ENABLE_DECIMAL256 = "enable_decimal256"; @@ -1315,6 +1317,13 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { flag = VariableMgr.GLOBAL) public boolean enableAutoAnalyze = true; + @VariableMgr.VarAttr(name = AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD, + description = {"参与自动收集的最大表宽度,列数多于这个参数的表不参与自动收集", + "Maximum table width to enable auto analyze, " + + "table with more columns than this value will not be auto analyzed."}, + flag = VariableMgr.GLOBAL) + public int autoAnalyzeTableWidthThreshold = 70; + @VariableMgr.VarAttr(name = AUTO_ANALYZE_START_TIME, needForward = true, checker = "checkAnalyzeTimeFormat", description = {"该参数定义自动ANALYZE例程的开始时间", "This parameter defines the start time for the automatic ANALYZE routine."}, diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java index 9f1bd3bf681896..c9eee3a01b07b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java @@ -98,6 +98,8 @@ public class StatisticConstants { public static final int SUBMIT_JOB_LIMIT = 5; + public static final int AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD = 70; + static { SYSTEM_DBS.add(SystemInfoService.DEFAULT_CLUSTER + ClusterNamespace.CLUSTER_DELIMITER + FeConstants.INTERNAL_DB_NAME); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java index 7f8dd18d50eb28..97061018ae4f99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java @@ -181,9 +181,13 @@ protected void createAnalyzeJobForTbl(DatabaseIf db, protected AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) { TableIf table = StatisticsUtil .findTable(jobInfo.catalogId, jobInfo.dbId, jobInfo.tblId); + // Skip tables that are too width. + if (table.getBaseSchema().size() > StatisticsUtil.getAutoAnalyzeTableWidthThreshold()) { + return null; + } + AnalysisManager analysisManager = Env.getServingEnv().getAnalysisManager(); TableStatsMeta tblStats = analysisManager.findTableStatsStatus(table.getId()); - if (!table.needReAnalyzeTable(tblStats)) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index 1763c33028c9fb..6d274e0f4c60dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -939,6 +939,16 @@ public static int getAnalyzeTimeout() { return StatisticConstants.ANALYZE_TIMEOUT_IN_SEC; } + public static int getAutoAnalyzeTableWidthThreshold() { + try { + return findConfigFromGlobalSessionVar(SessionVariable.AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD) + .autoAnalyzeTableWidthThreshold; + } catch (Exception e) { + LOG.warn("Failed to get value of auto_analyze_table_width_threshold, return default", e); + } + return StatisticConstants.AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD; + } + public static String encodeValue(ResultRow row, int index) { if (row == null || row.getValues().size() <= index) { return "NULL"; diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java index 14c6f41384f100..732196ef31b861 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java @@ -221,6 +221,37 @@ public AnalysisInfo getAnalysisJobInfo(AnalysisInfo jobInfo, TableIf table, Assertions.assertNotNull(statisticsAutoCollector.getReAnalyzeRequiredPart(analysisInfo2)); } + @Test + public void testSkipWideTable() { + + TableIf tableIf = new OlapTable(); + + new MockUp() { + @Mock + public List getBaseSchema() { + return Lists.newArrayList(new Column("col1", Type.INT), new Column("col2", Type.INT)); + } + }; + + new MockUp() { + int count = 0; + int [] thresholds = {1, 10}; + @Mock + public TableIf findTable(long catalogName, long dbName, long tblName) { + return tableIf; + } + + @Mock + public int getAutoAnalyzeTableWidthThreshold() { + return thresholds[count++]; + } + }; + AnalysisInfo analysisInfo = new AnalysisInfoBuilder().build(); + StatisticsAutoCollector statisticsAutoCollector = new StatisticsAutoCollector(); + Assertions.assertNull(statisticsAutoCollector.getReAnalyzeRequiredPart(analysisInfo)); + Assertions.assertNotNull(statisticsAutoCollector.getReAnalyzeRequiredPart(analysisInfo)); + } + @Test public void testLoop() { AtomicBoolean timeChecked = new AtomicBoolean();