[fix](stats) Fix data size calculation of auto sample

docs/en/docs/query-acceleration/statistics.md

@@ -237,16 +237,16 @@ Automatic collection tasks do not support viewing the completion status and fail
 |full_auto_analyze_end_time|End time for automatic statistics collection|02:00:00|
 |enable_auto_sample|Enable automatic sampling for large tables. When enabled, statistics will be automatically collected through sampling for tables larger than the `huge_table_lower_bound_size_in_bytes` threshold.| false|
 |auto_analyze_job_record_count|Controls the persistence of records for automatically triggered statistics collection jobs.|20000|
-|huge_table_default_sample_rows|Defines the number of sample rows for large tables when automatic sampling is enabled.|200000|
+|huge_table_default_sample_rows|Defines the number of sample rows for large tables when automatic sampling is enabled.|4194304|
 |huge_table_lower_bound_size_in_bytes|Defines the lower size threshold for large tables. When `enable_auto_sample` is enabled, statistics will be automatically collected through sampling for tables larger than this value.|5368 709120|
 |huge_table_auto_analyze_interval_in_millis|Controls the minimum time interval for automatic ANALYZE on large tables. Within this interval, tables larger than `huge_table_lower_bound_size_in_bytes` will only be analyzed once.|43200000|
 |table_stats_health_threshold|Takes a value between 0-100. When the data update volume reaches (100 - table_stats_health_threshold)% since the last statistics collection operation, the statistics for the table are considered outdated.|80|
-|enable_full_auto_analyze|Enable automatic collection functionality|true|
 
 |Session Variable|Description|Default Value|
 |---|---|---|
 |full_auto_analyze_start_time|Start time for automatic statistics collection|00:00:00|
 |full_auto_analyze_end_time|End time for automatic statistics collection|02:00:00|
+|enable_full_auto_analyze|Enable automatic collection functionality|true|
 
 Please note that when both FE configuration and global session variables are configured for the same parameter, the value of the global session variable takes precedence.
 

docs/zh-CN/docs/query-acceleration/statistics.md

@@ -249,16 +249,16 @@ SHOW AUTO ANALYZE [ptable_name]
 |full_auto_analyze_end_time|自动统计信息收集结束时间|02:00:00|
 |enable_auto_sample|是否开启大表自动sample，开启后对于大小超过huge_table_lower_bound_size_in_bytes会自动通过采样收集| false|
 |auto_analyze_job_record_count|控制统计信息的自动触发作业执行记录的持久化行数|20000|
-|huge_table_default_sample_rows|定义开启开启大表自动sample后，对大表的采样行数|200000|
+|huge_table_default_sample_rows|定义开启开启大表自动sample后，对大表的采样行数|4194304|
 |huge_table_lower_bound_size_in_bytes|定义大表的大小下界，在开启enable_auto_sample的情况下，大小超过该值的表将会自动通过采样收集统计信息|5368 709120|
 |huge_table_auto_analyze_interval_in_millis|控制对大表的自动ANALYZE的最小时间间隔，在该时间间隔内大小超过huge_table_lower_bound_size_in_bytes的表仅ANALYZE一次|43200000|
 |table_stats_health_threshold|取值在0-100之间，当自上次统计信息收集操作之后，数据更新量达到 (100 - table_stats_health_threshold)% ，认为该表的统计信息已过时|80|
-|enable_full_auto_analyze|开启自动收集功能|true|
 
 |会话变量|说明|默认值|
 |---|---|---|
 |full_auto_analyze_start_time|自动统计信息收集开始时间|00:00:00|
 |full_auto_analyze_end_time|自动统计信息收集结束时间|02:00:00|
+|enable_full_auto_analyze|开启自动收集功能|true|
 
 注意，对于fe配置和全局会话变量中均可配置的参数都设置的情况下，优先使用全局会话变量参数值。
 

fe/fe-common/src/main/java/org/apache/doris/common/Config.java

@@ -2079,15 +2079,6 @@ public class Config extends ConfigBase {
         "Sample size for hive row count estimation."})
     public static int hive_stats_partition_sample_size = 3000;
 
-    @ConfField
-    public static boolean enable_full_auto_analyze = true;
-
-    @ConfField
-    public static String full_auto_analyze_start_time = "00:00:00";
-
-    @ConfField
-    public static String full_auto_analyze_end_time = "02:00:00";
-
     @ConfField
     public static int statistics_sql_parallel_exec_instance_num = 1;
 
@@ -2184,10 +2175,10 @@ public class Config extends ConfigBase {
             + "statistics through sampling"})
     public static long huge_table_lower_bound_size_in_bytes = 5L * 1024 * 1024 * 1024;
 
-    @ConfField(description = {"定义开启开启大表自动sample后，对大表的采样行数",
-            "This defines the number of sample rows for large tables when automatic sampling for"
+    @ConfField(description = {"定义开启开启大表自动sample后，对大表的采样比例",
+            "This defines the number of sample percent for large tables when automatic sampling for"
                     + "large tables is enabled"})
-    public static int huge_table_default_sample_rows = 20_0000;
+    public static int huge_table_default_sample_rows = 4194304;
 
     @ConfField(description = {"是否开启大表自动sample，开启后对于大小超过huge_table_lower_bound_size_in_bytes会自动通过采样收集"
             + "统计信息", "Whether to enable automatic sampling for large tables, which, when enabled, automatically"

fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java

@@ -1568,14 +1568,18 @@ public Partition replacePartition(Partition newPartition) {
         return oldPartition;
     }
 
-    public long getDataSize() {
+    public long getDataSize(boolean singleReplica) {
         long dataSize = 0;
         for (Partition partition : getAllPartitions()) {
-            dataSize += partition.getDataSize(false);
+            dataSize += partition.getDataSize(singleReplica);
         }
         return dataSize;
     }
 
+    public long getDataSize() {
+        return getDataSize(false);
+    }
+
     public long getRemoteDataSize() {
         long remoteDataSize = 0;
         for (Partition partition : getAllPartitions()) {

fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java

@@ -246,7 +246,7 @@ default long getLastUpdateTime() {
         return -1L;
     }
 
-    default long getDataSize() {
+    default long getDataSize(boolean singleReplica) {
         // TODO: Each tableIf should impl it by itself.
         return 0;
     }

fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java

@@ -413,6 +413,8 @@ public class SessionVariable implements Serializable, Writable {
 
     public static final String TEST_QUERY_CACHE_HIT = "test_query_cache_hit";
 
+    public static final String ENABLE_FULL_AUTO_ANALYZE = "enable_full_auto_analyze";
+
     public static final List<String> DEBUG_VARIABLES = ImmutableList.of(
             SKIP_DELETE_PREDICATE,
             SKIP_DELETE_BITMAP,
@@ -1198,13 +1200,13 @@ public void setMaxJoinNumberOfReorder(int maxJoinNumberOfReorder) {
             description = {"该参数定义自动ANALYZE例程的开始时间",
                     "This parameter defines the start time for the automatic ANALYZE routine."},
             flag = VariableMgr.GLOBAL)
-    public String fullAutoAnalyzeStartTime = "";
+    public String fullAutoAnalyzeStartTime = "00:00:00";
 
     @VariableMgr.VarAttr(name = FULL_AUTO_ANALYZE_END_TIME, needForward = true, checker = "checkAnalyzeTimeFormat",
             description = {"该参数定义自动ANALYZE例程的结束时间",
                     "This parameter defines the end time for the automatic ANALYZE routine."},
             flag = VariableMgr.GLOBAL)
-    public String fullAutoAnalyzeEndTime = "";
+    public String fullAutoAnalyzeEndTime = "02:00:00";
 
     @VariableMgr.VarAttr(name = ENABLE_UNIQUE_KEY_PARTIAL_UPDATE, needForward = false)
     public boolean enableUniqueKeyPartialUpdate = false;
@@ -1216,6 +1218,11 @@ public void setMaxJoinNumberOfReorder(int maxJoinNumberOfReorder) {
             options = {"none", "sql_cache", "partition_cache"})
     public String testQueryCacheHit = "none";
 
+    @VariableMgr.VarAttr(name = ENABLE_FULL_AUTO_ANALYZE,
+            description = {"该参数控制是否开启自动收集", "Set false to disable auto analyze"},
+            flag = VariableMgr.GLOBAL)
+    public boolean enableFullAutoAnalyze = true;
+
     // If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables,
     // not the default value set in the code.
     public void initFuzzyModeVariables() {

fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskWrapper.java

@@ -56,7 +56,7 @@ public void run() {
             if (task.killed) {
                 return;
             }
-            if (task.info.scheduleType.equals(ScheduleType.AUTOMATIC) && !StatisticsUtil.checkAnalyzeTime(
+            if (task.info.scheduleType.equals(ScheduleType.AUTOMATIC) && !StatisticsUtil.inAnalyzeTime(
                     LocalTime.now(TimeUtils.getTimeZone().toZoneId()))) {
                 // TODO: Do we need a separate AnalysisState here?
                 Env.getCurrentEnv().getAnalysisManager()

fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java

@@ -225,7 +225,8 @@ protected String getSampleExpression() {
         }
         int sampleRows = info.sampleRows;
         if (info.analysisMethod == AnalysisMethod.FULL) {
-            if (Config.enable_auto_sample && tbl.getDataSize() > Config.huge_table_lower_bound_size_in_bytes) {
+            if (Config.enable_auto_sample
+                    && tbl.getDataSize(true) > Config.huge_table_lower_bound_size_in_bytes) {
                 sampleRows = Config.huge_table_default_sample_rows;
             } else {
                 return "";

fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java

@@ -81,6 +81,9 @@ public class StatisticConstants {
     // union more relation than 512 may cause StackOverFlowException in the future.
     public static final int UNION_ALL_LIMIT = 512;
 
+    public static final String FULL_AUTO_ANALYZE_START_TIME = "00:00:00";
+    public static final String FULL_AUTO_ANALYZE_END_TIME = "23:59:59";
+
     static {
         SYSTEM_DBS.add(SystemInfoService.DEFAULT_CLUSTER
                 + ClusterNamespace.CLUSTER_DELIMITER + FeConstants.INTERNAL_DB_NAME);

fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java

@@ -58,11 +58,11 @@ public StatisticsAutoCollector() {
 
     @Override
     protected void collect() {
-        if (!StatisticsUtil.checkAnalyzeTime(LocalTime.now(TimeUtils.getTimeZone().toZoneId()))) {
+        if (!StatisticsUtil.inAnalyzeTime(LocalTime.now(TimeUtils.getTimeZone().toZoneId()))) {
             analysisTaskExecutor.clear();
             return;
         }
-        if (Config.enable_full_auto_analyze) {
+        if (StatisticsUtil.enableAutoAnalyze()) {
             analyzeAll();
         }
     }
@@ -115,7 +115,7 @@ protected boolean skip(TableIf table) {
         if (!(table instanceof OlapTable || table instanceof ExternalTable)) {
             return true;
         }
-        if (table.getDataSize() < Config.huge_table_lower_bound_size_in_bytes) {
+        if (table.getDataSize(true) < Config.huge_table_lower_bound_size_in_bytes) {
             return false;
         }
         TableStats tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(table.getId());
@@ -124,7 +124,7 @@ protected boolean skip(TableIf table) {
 
     protected void createAnalyzeJobForTbl(DatabaseIf<? extends TableIf> db,
             List<AnalysisInfo> analysisInfos, TableIf table) {
-        AnalysisMethod analysisMethod = table.getDataSize() > Config.huge_table_lower_bound_size_in_bytes
+        AnalysisMethod analysisMethod = table.getDataSize(true) > Config.huge_table_lower_bound_size_in_bytes
                 ? AnalysisMethod.SAMPLE : AnalysisMethod.FULL;
         TableName tableName = new TableName(db.getCatalog().getName(), db.getFullName(),
                 table.getName());
@@ -141,7 +141,7 @@ protected void createAnalyzeJobForTbl(DatabaseIf<? extends TableIf> db,
                 .setAnalysisType(AnalysisInfo.AnalysisType.FUNDAMENTALS)
                 .setAnalysisMode(AnalysisInfo.AnalysisMode.INCREMENTAL)
                 .setAnalysisMethod(analysisMethod)
-                .setSamplePercent(Config.huge_table_default_sample_rows)
+                .setSampleRows(Config.huge_table_default_sample_rows)
                 .setScheduleType(ScheduleType.AUTOMATIC)
                 .setState(AnalysisState.PENDING)
                 .setTaskIds(new ArrayList<>())

fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java

@@ -23,12 +23,15 @@
 import org.apache.doris.statistics.util.StatisticsUtil;
 
 import org.apache.hudi.common.util.VisibleForTesting;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 import java.util.HashMap;
 import java.util.Map;
 
 public abstract class StatisticsCollector extends MasterDaemon {
 
+    private static final Logger LOG = LogManager.getLogger(StatisticsCollector.class);
 
     protected final AnalysisTaskExecutor analysisTaskExecutor;
 
@@ -45,13 +48,15 @@ protected void runAfterCatalogReady() {
             return;
         }
         if (!StatisticsUtil.statsTblAvailable()) {
+            LOG.info("Stats table not available, skip");
             return;
         }
         if (Env.isCheckpointThread()) {
             return;
         }
 
         if (!analysisTaskExecutor.idle()) {
+            LOG.info("Analyze tasks those submitted in last time is not finished, skip");
             return;
         }
         collect();

fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java

@@ -174,6 +174,7 @@ public static AutoCloseConnectContext buildConnectContext(boolean limitScan) {
         sessionVariable.setMaxExecMemByte(Config.statistics_sql_mem_limit_in_bytes);
         sessionVariable.cpuResourceLimit = Config.cpu_resource_limit_per_analyze_task;
         sessionVariable.setEnableInsertStrict(true);
+        sessionVariable.enablePageCache = false;
         sessionVariable.parallelExecInstanceNum = Config.statistics_sql_parallel_exec_instance_num;
         sessionVariable.parallelPipelineTaskNum = Config.statistics_sql_parallel_exec_instance_num;
         sessionVariable.setEnableNereidsPlanner(false);
@@ -729,15 +730,14 @@ public static boolean isExternalTable(String catalogName, String dbName, String
         return table instanceof ExternalTable;
     }
 
-    public static boolean checkAnalyzeTime(LocalTime now) {
+    public static boolean inAnalyzeTime(LocalTime now) {
         try {
             Pair<LocalTime, LocalTime> range = findRangeFromGlobalSessionVar();
-            DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("HH:mm:ss");
-            LocalTime start = range == null
-                    ? LocalTime.parse(Config.full_auto_analyze_start_time, timeFormatter) : range.first;
-            LocalTime end = range == null
-                    ? LocalTime.parse(Config.full_auto_analyze_end_time, timeFormatter) : range.second;
-
+            if (range == null) {
+                return false;
+            }
+            LocalTime start = range.first;
+            LocalTime end = range.second;
             if (start.isAfter(end) && (now.isAfter(start) || now.isBefore(end))) {
                 return true;
             } else {
@@ -754,13 +754,14 @@ private static Pair<LocalTime, LocalTime> findRangeFromGlobalSessionVar() {
             String startTime =
                     findRangeFromGlobalSessionVar(SessionVariable.FULL_AUTO_ANALYZE_START_TIME)
                             .fullAutoAnalyzeStartTime;
+            // For compatibility
             if (StringUtils.isEmpty(startTime)) {
-                return null;
+                startTime = StatisticConstants.FULL_AUTO_ANALYZE_START_TIME;
             }
             String endTime = findRangeFromGlobalSessionVar(SessionVariable.FULL_AUTO_ANALYZE_END_TIME)
                     .fullAutoAnalyzeEndTime;
             if (StringUtils.isEmpty(startTime)) {
-                return null;
+                endTime = StatisticConstants.FULL_AUTO_ANALYZE_END_TIME;
             }
             DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("HH:mm:ss");
             return Pair.of(LocalTime.parse(startTime, timeFormatter), LocalTime.parse(endTime, timeFormatter));
@@ -775,4 +776,13 @@ private static SessionVariable findRangeFromGlobalSessionVar(String varName) thr
         VariableMgr.getValue(sessionVariable, variableExpr);
         return sessionVariable;
     }
+
+    public static boolean enableAutoAnalyze() {
+        try {
+            return findRangeFromGlobalSessionVar(SessionVariable.ENABLE_FULL_AUTO_ANALYZE).enableFullAutoAnalyze;
+        } catch (Exception e) {
+            LOG.warn("Fail to get value of enable auto analyze, return false by default", e);
+        }
+        return false;
+    }
 }

fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java

@@ -34,7 +34,7 @@ public class OlapAnalysisTaskTest {
     public void testAutoSample(@Mocked CatalogIf catalogIf, @Mocked DatabaseIf databaseIf, @Mocked TableIf tableIf) {
         new Expectations() {
             {
-                tableIf.getDataSize();
+                tableIf.getDataSize(true);
                 result = 60_0000_0000L;
             }
         };
@@ -46,11 +46,11 @@ public void testAutoSample(@Mocked CatalogIf catalogIf, @Mocked DatabaseIf datab
         olapAnalysisTask.tbl = tableIf;
         Config.enable_auto_sample = true;
         String sampleExpr = olapAnalysisTask.getSampleExpression();
-        Assertions.assertEquals("TABLESAMPLE(200000 ROWS)", sampleExpr);
+        Assertions.assertEquals("TABLESAMPLE(4194304 ROWS)", sampleExpr);
 
         new Expectations() {
             {
-                tableIf.getDataSize();
+                tableIf.getDataSize(true);
                 result = 1_0000_0000L;
             }
         };