From 602ade159203720ae131b5710e78553f56b52435 Mon Sep 17 00:00:00 2001
From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com>
Date: Fri, 8 Nov 2024 14:57:58 +0800
Subject: [PATCH] [improvement](statistics)Remove useless stats validation
 check. (#43279)

Remove useless stats validation check.
Before, we will not load the column stats when table rowCount > 0 and
ndv == 0 and nullCount != rowCount. This is to avoid using invalid
stats.
Now, we remove this validation because the planner side added validation
to the column stats, (see https://github.com/apache/doris/pull/41790).
Besides, after remove of the validation, it is easier to add regression
test using stats injection.
---
 .../java/org/apache/doris/statistics/ColStatsData.java    | 5 -----
 .../doris/statistics/ColumnStatisticsCacheLoader.java     | 8 --------
 .../java/org/apache/doris/statistics/StatisticsCache.java | 6 +-----
 regression-test/suites/statistics/analyze_stats.groovy    | 4 +---
 4 files changed, 2 insertions(+), 21 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsData.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsData.java
index bb8263994583fd..cc79f779f6e872 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsData.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsData.java
@@ -131,11 +131,6 @@ public String toSQL(boolean roundByParentheses) {
     }
 
     public ColumnStatistic toColumnStatistic() {
-        // For non-empty table, return UNKNOWN if we can't collect ndv value.
-        // Because inaccurate ndv is very misleading.
-        if (count > 0 && ndv == 0 && count != nullCount) {
-            return ColumnStatistic.UNKNOWN;
-        }
         try {
             ColumnStatisticBuilder columnStatisticBuilder = new ColumnStatisticBuilder();
             columnStatisticBuilder.setCount(count);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java
index 42d5d542359880..1876cacc003f30 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java
@@ -51,14 +51,6 @@ protected Optional<ColumnStatistic> doLoad(StatisticsCacheKey key) {
             // it will trigger load function again without cache an empty value.
             return null;
         }
-        if (columnStatistic.isPresent()) {
-            // For non-empty table, return UNKNOWN if we can't collect ndv value.
-            // Because inaccurate ndv is very misleading.
-            ColumnStatistic stats = columnStatistic.get();
-            if (stats.count > 0 && stats.ndv == 0 && stats.count != stats.numNulls) {
-                columnStatistic = Optional.of(ColumnStatistic.UNKNOWN);
-            }
-        }
         return columnStatistic;
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
index af5958d740205e..fae6fd0555fab6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
@@ -171,12 +171,8 @@ private void doPreHeat() {
                 long tblId = statsId.tblId;
                 long idxId = statsId.idxId;
                 String colId = statsId.colId;
-                final StatisticsCacheKey k =
-                        new StatisticsCacheKey(tblId, idxId, colId);
+                final StatisticsCacheKey k = new StatisticsCacheKey(tblId, idxId, colId);
                 ColumnStatistic c = ColumnStatistic.fromResultRow(r);
-                if (c.count > 0 && c.ndv == 0 && c.count != c.numNulls) {
-                    c = ColumnStatistic.UNKNOWN;
-                }
                 putCache(k, c);
             } catch (Throwable t) {
                 LOG.warn("Error when preheating stats cache. reason: [{}]. Row:[{}]", t.getMessage(), r);
diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy
index 993e6f531da8eb..c828274a40357f 100644
--- a/regression-test/suites/statistics/analyze_stats.groovy
+++ b/regression-test/suites/statistics/analyze_stats.groovy
@@ -2706,9 +2706,7 @@ PARTITION `p599` VALUES IN (599)
     alter_result = sql """show column stats alter_test(id)"""
     assertEquals(1, alter_result.size())
     alter_result = sql """show column cached stats alter_test(id)"""
-    assertEquals(0, alter_result.size())
-    alter_result = sql """show column cached stats alter_test(id)"""
-    assertEquals(0, alter_result.size())
+    assertEquals(1, alter_result.size())
     sql """alter table alter_test modify column id set stats ('row_count'='100', 'ndv'='0', 'num_nulls'='100', 'data_size'='2.69975443E8', 'min_value'='1', 'max_value'='2');"""
     alter_result = sql """show column stats alter_test(id)"""
     assertEquals(1, alter_result.size())