[improvement and fix](statistics)Load the cache for external table row count while init table.

fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java

@@ -150,6 +150,21 @@ public ShowResultSet constructResultSet(TableStats tableStatistic) {
         return new ShowResultSet(getMetaData(), result);
     }
 
+    public ShowResultSet constructResultSet(long rowCount) {
+        List<List<String>> result = Lists.newArrayList();
+        List<String> row = Lists.newArrayList();
+        row.add("");
+        row.add("");
+        row.add(String.valueOf(rowCount));
+        row.add("");
+        row.add("");
+        row.add("");
+        row.add("");
+        row.add("");
+        result.add(row);
+        return new ShowResultSet(getMetaData(), result);
+    }
+
     public boolean isCached() {
         return cached;
     }

fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java

@@ -72,6 +72,7 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable {
     @SerializedName(value = "lastUpdateTime")
     protected long lastUpdateTime;
 
+    protected long dbId;
     protected boolean objectCreated;
     protected ExternalCatalog catalog;
     protected ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true);
@@ -113,6 +114,7 @@ protected void makeSureInitialized() {
         try {
             // getDbOrAnalysisException will call makeSureInitialized in ExternalCatalog.
             ExternalDatabase db = catalog.getDbOrAnalysisException(dbName);
+            dbId = db.getId();
             db.makeSureInitialized();
         } catch (AnalysisException e) {
             Util.logAndThrowRuntimeException(LOG, String.format("Exception to get db %s", dbName), e);

fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java

@@ -149,6 +149,7 @@ protected synchronized void makeSureInitialized() {
                 }
             }
             objectCreated = true;
+            estimatedRowCount = getRowCountFromExternalSource();
         }
     }
 
@@ -272,6 +273,15 @@ public long getUpdateTime() {
     @Override
     public long getRowCount() {
         makeSureInitialized();
+        long rowCount = getRowCountFromExternalSource();
+        if (rowCount == -1) {
+            LOG.debug("Will estimate row count from file list.");
+            rowCount = StatisticsUtil.getRowCountFromFileList(this);
+        }
+        return rowCount;
+    }
+
+    private long getRowCountFromExternalSource() {
         long rowCount;
         switch (dlaType) {
             case HIVE:
@@ -284,10 +294,6 @@ public long getRowCount() {
                 LOG.warn("getRowCount for dlaType {} is not supported.", dlaType);
                 rowCount = -1;
         }
-        if (rowCount == -1) {
-            LOG.debug("Will estimate row count from file list.");
-            rowCount = StatisticsUtil.getRowCountFromFileList(this);
-        }
         return rowCount;
     }
 

fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java

@@ -437,7 +437,9 @@ public PlanFragment visitPhysicalFileScan(PhysicalFileScan fileScan, PlanTransla
         TableRef ref = new TableRef(tableName, null, null);
         BaseTableRef tableRef = new BaseTableRef(ref, table, tableName);
         tupleDescriptor.setRef(tableRef);
-
+        if (fileScan.getStats() != null) {
+            scanNode.setCardinality((long) fileScan.getStats().getRowCount());
+        }
         Utils.execWithUncheckedException(scanNode::init);
         context.addScanNode(scanNode);
         ScanNode finalScanNode = scanNode;

fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java

@@ -2451,7 +2451,16 @@ private void handleShowTableStats() {
         ShowTableStatsStmt showTableStatsStmt = (ShowTableStatsStmt) stmt;
         TableIf tableIf = showTableStatsStmt.getTable();
         TableStats tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(tableIf.getId());
-        resultSet = showTableStatsStmt.constructResultSet(tableStats);
+        /*
+           HMSExternalTable table will fetch row count from HMS
+           or estimate with file size and schema if it's not analyzed.
+           tableStats == null means it's not analyzed, in this case show the estimated row count.
+         */
+        if (tableStats == null && tableIf instanceof HMSExternalTable) {
+            resultSet = showTableStatsStmt.constructResultSet(tableIf.estimatedRowCount());
+        } else {
+            resultSet = showTableStatsStmt.constructResultSet(tableStats);
+        }
     }
 
     private void handleShowColumnStats() throws AnalysisException {

fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java

@@ -32,6 +32,7 @@
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.catalog.View;
+import org.apache.doris.catalog.external.ExternalTable;
 import org.apache.doris.catalog.external.HMSExternalTable;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.Config;
@@ -285,9 +286,7 @@ public List<AnalysisInfo> buildAnalysisInfosForDB(DatabaseIf<TableIf> db, Analyz
                 // columnNames null means to add all visitable columns.
                 // Will get all the visible columns in analyzeTblStmt.check()
                 AnalyzeTblStmt analyzeTblStmt = new AnalyzeTblStmt(analyzeProperties, tableName,
-                        table.getBaseSchema().stream().filter(c -> !StatisticsUtil.isUnsupportedType(c.getType())).map(
-                                Column::getName).collect(
-                                Collectors.toList()), db.getId(), table);
+                        null, db.getId(), table);
                 try {
                     analyzeTblStmt.check();
                 } catch (AnalysisException analysisException) {
@@ -334,11 +333,10 @@ public AnalysisInfo buildAndAssignJob(AnalyzeTblStmt stmt) throws DdlException {
 
         boolean isSync = stmt.isSync();
         Map<Long, BaseAnalysisTask> analysisTaskInfos = new HashMap<>();
+        createTaskForEachColumns(jobInfo, analysisTaskInfos, isSync);
         if (stmt.isAllColumns()
                 && StatisticsUtil.isExternalTable(jobInfo.catalogName, jobInfo.dbName, jobInfo.tblName)) {
-            createTaskForExternalTable(jobInfo, analysisTaskInfos, isSync);
-        } else {
-            createTaskForEachColumns(jobInfo, analysisTaskInfos, isSync);
+            createTableLevelTaskForExternalTable(jobInfo, analysisTaskInfos, isSync);
         }
         if (isSync) {
             syncExecute(analysisTaskInfos.values());
@@ -583,7 +581,7 @@ public void logCreateAnalysisJob(AnalysisInfo analysisJob) {
     }
 
     @VisibleForTesting
-    public void createTaskForExternalTable(AnalysisInfo jobInfo,
+    public void createTableLevelTaskForExternalTable(AnalysisInfo jobInfo,
             Map<Long, BaseAnalysisTask> analysisTasks,
             boolean isSync) throws DdlException {
 
@@ -616,6 +614,10 @@ public void updateTaskStatus(AnalysisInfo info, AnalysisState taskState, String
     public void updateTableStats(AnalysisInfo jobInfo) {
         TableIf tbl = StatisticsUtil.findTable(jobInfo.catalogName,
                 jobInfo.dbName, jobInfo.tblName);
+        // External Table update table stats after table level task finished.
+        if (tbl instanceof ExternalTable) {
+            return;
+        }
         // TODO: set updatedRows to 0, when loadedRows of transaction info is ready.
         updateTableStatsStatus(new TableStats(tbl.getId(), tbl.estimatedRowCount(), jobInfo));
     }

fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java

@@ -123,7 +123,7 @@ private void getTableStats() throws Exception {
         String rowCount = columnResult.get(0).get(0);
         Env.getCurrentEnv().getAnalysisManager()
                 .updateTableStatsStatus(
-                        new TableStats(table.getId(), Long.parseLong(rowCount), null));
+                        new TableStats(table.getId(), Long.parseLong(rowCount), info));
     }
 
     /**
@@ -269,6 +269,10 @@ private void setParameterData(Map<String, String> parameters, Map<String, String
 
     @Override
     protected void afterExecution() {
+        // Table level task doesn't need to sync any value to sync stats, it stores the value in metadata.
+        if (isTableLevelTask) {
+            return;
+        }
         Env.getCurrentEnv().getStatisticsCache().syncLoadColStats(tbl.getId(), -1, col.getName());
     }
 }

regression-test/suites/external_table_p2/hive/test_hive_statistic.groovy

@@ -234,11 +234,11 @@ suite("test_hive_statistic", "p2,external,hive,external_remote,external_remote_h
         sql """analyze database `statistics` with sync"""
         result = sql """show table stats statistics"""
         assertTrue(result.size() == 1)
-        assertTrue(result[0][0] == "100")
+        assertTrue(result[0][2] == "100")
 
         result = sql """show table cached stats statistics"""
         assertTrue(result.size() == 1)
-        assertTrue(result[0][0] == "100")
+        assertTrue(result[0][2] == "100")
 
         sql """drop stats statistics"""
         result = sql """show column cached stats statistics"""

regression-test/suites/external_table_p2/hive/test_hive_statistic_cache.groovy

@@ -29,6 +29,40 @@ suite("test_hive_statistic_cache", "p2,external,hive,external_remote,external_re
                 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
             );
         """
+        sql """use ${catalog_name}.tpch_1000_parquet"""
+        sql """desc customer""";
+        sql """desc lineitem""";
+        sql """desc region""";
+        sql """desc nation""";
+        sql """desc orders""";
+        sql """desc part""";
+        sql """desc partsupp""";
+        sql """desc supplier""";
+        Thread.sleep(1000);
+        def result = sql """show table cached stats customer"""
+        assertTrue(result[0][2] == "150000000")
+
+        result = sql """show table cached stats lineitem"""
+        assertTrue(result[0][2] == "5999989709")
+
+        result = sql """show table cached stats region"""
+        assertTrue(result[0][2] == "5")
+
+        result = sql """show table cached stats nation"""
+        assertTrue(result[0][2] == "25")
+
+        result = sql """show table cached stats orders"""
+        assertTrue(result[0][2] == "1500000000")
+
+        result = sql """show table cached stats part"""
+        assertTrue(result[0][2] == "200000000")
+
+        result = sql """show table cached stats partsupp"""
+        assertTrue(result[0][2] == "800000000")
+
+        result = sql """show table cached stats supplier"""
+        assertTrue(result[0][2] == "10000000")
+
         logger.info("catalog " + catalog_name + " created")
         sql """switch ${catalog_name};"""
         logger.info("switched to catalog " + catalog_name)
@@ -37,7 +71,7 @@ suite("test_hive_statistic_cache", "p2,external,hive,external_remote,external_re
         sql """analyze table `stats` with sync;"""
         sql """select count(*) from stats"""
         Thread.sleep(5000);
-        def result = sql """show column cached stats `stats` (lo_orderkey)"""
+        result = sql """show column cached stats `stats` (lo_orderkey)"""
         assertTrue(result[0][0] == "lo_orderkey")
         assertTrue(result[0][1] == "100.0")
         assertTrue(result[0][2] == "26.0")