From 092f0b5f172bd9b369d903dfc52fb9df31e1911d Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Mon, 17 Jul 2023 08:54:52 +0800 Subject: [PATCH 01/10] add last sync time --- .../doris/catalog/external/ExternalDatabase.java | 15 +++++++++++++++ .../doris/catalog/external/ExternalTable.java | 7 +++++++ .../apache/doris/datasource/ExternalCatalog.java | 16 +++++++++++++++- .../apache/doris/datasource/InitCatalogLog.java | 3 +++ .../apache/doris/datasource/InitDatabaseLog.java | 3 +++ 5 files changed, 43 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java index e1b582a5403bcc..cbeba87dac2339 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java @@ -73,6 +73,8 @@ public abstract class ExternalDatabase protected Map tableNameToId = Maps.newConcurrentMap(); @SerializedName(value = "idToTbl") protected Map idToTbl = Maps.newConcurrentMap(); + @SerializedName(value = "lastSyncTime") + protected long lastSyncTime; protected final InitDatabaseLog.Type dbLogType; protected ExternalCatalog extCatalog; protected boolean invalidCacheInInit = true; @@ -147,6 +149,7 @@ public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) { } tableNameToId = tmpTableNameToId; idToTbl = tmpIdToTbl; + lastSyncTime = log.getLastSyncTime(); initialized = true; } @@ -179,6 +182,10 @@ protected void init() { tableNameToId = tmpTableNameToId; idToTbl = tmpIdToTbl; } + + long currentTime = System.currentTimeMillis(); + lastSyncTime = currentTime; + initDatabaseLog.setLastSyncTime(lastSyncTime); initialized = true; Env.getCurrentEnv().getEditLog().logInitExternalDb(initDatabaseLog); } @@ -307,6 +314,14 @@ public T getTableNullable(long tableId) { return idToTbl.get(tableId); } + public long getLastSyncTime() { + return lastSyncTime; + } + + public void setLastSyncTime(long lastSyncTime) { + this.lastSyncTime = lastSyncTime; + } + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java index 88ba9bc94e2804..166c7ea3e69056 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java @@ -69,6 +69,8 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable { protected long timestamp; @SerializedName(value = "dbName") protected String dbName; + @SerializedName(value = "lastSyncTime") + protected long lastSyncTime; protected boolean objectCreated; protected ExternalCatalog catalog; @@ -342,6 +344,11 @@ public Optional getColumnStatistic(String colName) { * * @return */ + public List initSchemaAndUpdateSyncTime() { + lastSyncTime = System.currentTimeMillis(); + return initSchema(); + } + public List initSchema() { throw new NotImplementedException("implement in sub class"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index cf2de864946a03..4cd740ad59abdd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -84,6 +84,8 @@ public abstract class ExternalCatalog private boolean initialized = false; @SerializedName(value = "idToDb") protected Map> idToDb = Maps.newConcurrentMap(); + @SerializedName(value = "lastSyncTime") + protected long lastSyncTime; // db name does not contains "default_cluster" protected Map dbNameToId = Maps.newConcurrentMap(); private boolean objectCreated = false; @@ -253,6 +255,9 @@ protected void init() { } dbNameToId = tmpDbNameToId; idToDb = tmpIdToDb; + long currentTime = System.currentTimeMillis(); + lastSyncTime = currentTime; + initCatalogLog.setLastSyncTime(lastSyncTime); Env.getCurrentEnv().getEditLog().logInitCatalog(initCatalogLog); } @@ -275,7 +280,7 @@ public final List getSchema(String dbName, String tblName) { if (db.isPresent()) { Optional table = db.get().getTable(tblName); if (table.isPresent()) { - return table.get().initSchema(); + return table.get().initSchemaAndUpdateSyncTime(); } } // return one column with unsupported type. @@ -391,6 +396,14 @@ private void modifyComment(Map props) { props.remove("comment"); } + public long getLastSyncTime() { + return lastSyncTime; + } + + public void setLastSyncTime(long lastSyncTime) { + this.lastSyncTime = lastSyncTime; + } + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); @@ -425,6 +438,7 @@ public void replayInitCatalog(InitCatalogLog log) { } dbNameToId = tmpDbNameToId; idToDb = tmpIdToDb; + lastSyncTime = log.getLastSyncTime(); initialized = true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java index 73fbeeb78112c8..509409b2ba2e69 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java @@ -65,6 +65,9 @@ public enum Type { @SerializedName(value = "type") private Type type; + @SerializedName(value = "lastSyncTime") + private long lastSyncTime; + public InitCatalogLog() { refreshCount = 0; createCount = 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java index 14cd4410ec4c1e..349cb5f3369faa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java @@ -68,6 +68,9 @@ public enum Type { @SerializedName(value = "type") private Type type; + @SerializedName(value = "lastSyncTime") + protected long lastSyncTime; + public InitDatabaseLog() { refreshCount = 0; createCount = 0; From 565e32cecf8a8cf9b15569a7a74d2f220a0adec1 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Mon, 17 Jul 2023 14:30:56 +0800 Subject: [PATCH 02/10] show proc catalog add sync time --- .../java/org/apache/doris/common/proc/CatalogsProcDir.java | 2 +- .../src/main/java/org/apache/doris/datasource/CatalogIf.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java index 48e6b6ab46ad7e..b1fbcba79a3294 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java @@ -37,7 +37,7 @@ */ public class CatalogsProcDir implements ProcDirInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() - .add("CatalogIds").add("CatalogName").add("DatabaseNum") + .add("CatalogIds").add("CatalogName").add("DatabaseNum").add("lastSyncTime") .build(); private Env env; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index ce13c9881e5c28..54e2a345598fd6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -152,6 +152,10 @@ default void onClose() { String getComment(); + default long getLastSyncTime() { + return -1l; + } + default CatalogLog constructEditLog() { CatalogLog log = new CatalogLog(); log.setCatalogId(getId()); From 4e2acd84768e8fa71d487ce24aeb32c78b3c140f Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Mon, 17 Jul 2023 14:40:54 +0800 Subject: [PATCH 03/10] show proc catalog add sync time --- .../java/org/apache/doris/common/proc/CatalogsProcDir.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java index b1fbcba79a3294..8b10c982230213 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java @@ -27,6 +27,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -81,6 +82,7 @@ public ProcResult fetchResult() throws AnalysisException { List catalogIds = env.getCatalogMgr().getCatalogIds(); // get info List> catalogInfos = Lists.newArrayList(); + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); for (long catalogId : catalogIds) { CatalogIf catalog = env.getCatalogMgr().getCatalog(catalogId); if (catalog == null) { @@ -90,6 +92,8 @@ public ProcResult fetchResult() throws AnalysisException { catalogInfo.add(catalog.getId()); catalogInfo.add(catalog.getName()); catalogInfo.add(catalog.getDbNames().size()); + long lastSyncTime = catalog.getLastSyncTime(); + catalogInfo.add(lastSyncTime > 0 ? simpleDateFormat.format(lastSyncTime) : ""); catalogInfos.add(catalogInfo); } From 67d54c3092ff68f7e87007ecd3886c997e62c57c Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Mon, 17 Jul 2023 14:41:45 +0800 Subject: [PATCH 04/10] show proc catalog add sync time --- .../src/main/java/org/apache/doris/datasource/CatalogIf.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index 54e2a345598fd6..fd74ada0e3db37 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -153,7 +153,7 @@ default void onClose() { String getComment(); default long getLastSyncTime() { - return -1l; + return -1L; } default CatalogLog constructEditLog() { From e0d721cae5653fbfb1bf79cb2567d2407f423d1f Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Mon, 17 Jul 2023 15:14:00 +0800 Subject: [PATCH 05/10] show proc catalog add sync time --- .../java/org/apache/doris/analysis/ShowCatalogStmt.java | 1 + .../src/main/java/org/apache/doris/catalog/DatabaseIf.java | 4 ++++ .../src/main/java/org/apache/doris/catalog/TableIf.java | 4 ++++ .../java/org/apache/doris/common/proc/CatalogsProcDir.java | 4 ++-- .../main/java/org/apache/doris/common/proc/DbsProcDir.java | 7 +++++-- .../java/org/apache/doris/common/proc/TablesProcDir.java | 7 +++++-- .../main/java/org/apache/doris/datasource/CatalogMgr.java | 5 ++++- 7 files changed, 25 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java index 8c32bafe5fafdd..2f20c2d12e8cef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java @@ -32,6 +32,7 @@ public class ShowCatalogStmt extends ShowStmt { .addColumn(new Column("Type", ScalarType.createStringType())) .addColumn(new Column("IsCurrent", ScalarType.createStringType())) .addColumn(new Column("CreateTime", ScalarType.createStringType())) + .addColumn(new Column("LastSyncTime", ScalarType.createStringType())) .addColumn(new Column("Comment", ScalarType.createStringType())) .build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java index 7672b8b76d2006..5f739ece59721f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java @@ -235,4 +235,8 @@ default OlapTable getOlapTableOrAnalysisException(String tableName) throws Analy void dropTable(String tableName); CatalogIf getCatalog(); + + default long getLastSyncTime() { + return -1L; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index 0f12cc25509bd0..58f65b7d64607b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -228,5 +228,9 @@ default Partition getPartition(String name) { default boolean isManagedTable() { return getType() == TableType.OLAP || getType() == TableType.MATERIALIZED_VIEW; } + + default long getLastSyncTime() { + return -1L; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java index 8b10c982230213..f4a1db992929da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java @@ -38,7 +38,7 @@ */ public class CatalogsProcDir implements ProcDirInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() - .add("CatalogIds").add("CatalogName").add("DatabaseNum").add("lastSyncTime") + .add("CatalogIds").add("CatalogName").add("DatabaseNum").add("LastSyncTime") .build(); private Env env; @@ -93,7 +93,7 @@ public ProcResult fetchResult() throws AnalysisException { catalogInfo.add(catalog.getName()); catalogInfo.add(catalog.getDbNames().size()); long lastSyncTime = catalog.getLastSyncTime(); - catalogInfo.add(lastSyncTime > 0 ? simpleDateFormat.format(lastSyncTime) : ""); + catalogInfo.add(lastSyncTime > 0 ? simpleDateFormat.format(lastSyncTime) : "UNRECORDED"); catalogInfos.add(catalogInfo); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java index 431c618a6f3301..2f4d791891e152 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java @@ -31,6 +31,7 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -43,7 +44,7 @@ public class DbsProcDir implements ProcDirInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("DbId").add("DbName").add("TableNum").add("Size").add("Quota") .add("LastConsistencyCheckTime").add("ReplicaCount").add("ReplicaQuota") - .add("TransactionQuota") + .add("TransactionQuota").add("LastSyncTime") .build(); private Env env; @@ -94,6 +95,7 @@ public ProcResult fetchResult() throws AnalysisException { // get info List> dbInfos = new ArrayList<>(); + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); for (String dbName : dbNames) { DatabaseIf db = catalog.getDbNullable(dbName); if (db == null) { @@ -122,7 +124,8 @@ public ProcResult fetchResult() throws AnalysisException { dbInfo.add(replicaCount); dbInfo.add(replicaQuota); dbInfo.add(transactionQuota); - + long lastSyncTime = db.getLastSyncTime(); + dbInfo.add(lastSyncTime > 0 ? simpleDateFormat.format(lastSyncTime) : "UNRECORDED"); } finally { db.readUnlock(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java index 3b43ec887102cb..91e489039eb69d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java @@ -33,6 +33,7 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -45,6 +46,7 @@ public class TablesProcDir implements ProcDirInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("TableId").add("TableName").add("IndexNum").add("PartitionColumnName") .add("PartitionNum").add("State").add("Type").add("LastConsistencyCheckTime").add("ReplicaCount") + .add("LastSyncTime") .build(); private DatabaseIf db; @@ -83,6 +85,7 @@ public ProcResult fetchResult() throws AnalysisException { // get info List> tableInfos = new ArrayList>(); List tableList = db.getTables(); + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); for (TableIf table : tableList) { List tableInfo = new ArrayList(); int partitionNum = 1; @@ -128,8 +131,8 @@ public ProcResult fetchResult() throws AnalysisException { tableInfo.add(FeConstants.null_string); tableInfo.add(replicaCount); } - - + long lastSyncTime = table.getLastSyncTime(); + tableInfo.add(lastSyncTime > 0 ? simpleDateFormat.format(lastSyncTime) : "UNRECORDED"); tableInfos.add(tableInfo); } finally { table.readUnlock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 96a7c1eae3e784..d08b64b454189d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -60,6 +60,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -375,7 +376,7 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg) matcher = PatternMatcherWrapper.createMysqlPattern(showStmt.getPattern(), CaseSensibility.CATALOG.getCaseSensibility()); } - + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); for (CatalogIf catalog : nameToCatalog.values()) { if (Env.getCurrentEnv().getAccessManager() .checkCtlPriv(ConnectContext.get(), catalog.getName(), PrivPredicate.SHOW)) { @@ -396,6 +397,8 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg) Map props = catalog.getProperties(); String createTime = props.getOrDefault(CreateCatalogStmt.CREATE_TIME_PROP, "UNRECORDED"); row.add(createTime); + long lastSyncTime = catalog.getLastSyncTime(); + row.add(lastSyncTime > 0 ? simpleDateFormat.format(lastSyncTime) : "UNRECORDED"); row.add(catalog.getComment()); rows.add(row); } From 68c7f277eecc02ff99e1dbd5392360acebdfc226 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Mon, 17 Jul 2023 16:15:41 +0800 Subject: [PATCH 06/10] resolve comment --- .../doris/analysis/ShowCatalogStmt.java | 2 +- .../org/apache/doris/catalog/DatabaseIf.java | 2 +- .../org/apache/doris/catalog/TableIf.java | 2 +- .../catalog/external/ExternalDatabase.java | 18 ++++++++--------- .../doris/catalog/external/ExternalTable.java | 8 ++++---- .../doris/common/proc/CatalogsProcDir.java | 9 ++++----- .../apache/doris/common/proc/DbsProcDir.java | 8 +++----- .../doris/common/proc/TablesProcDir.java | 8 +++----- .../apache/doris/datasource/CatalogIf.java | 2 +- .../apache/doris/datasource/CatalogMgr.java | 7 +++---- .../doris/datasource/ExternalCatalog.java | 20 +++++++++---------- .../doris/datasource/InitCatalogLog.java | 4 ++-- .../doris/datasource/InitDatabaseLog.java | 4 ++-- 13 files changed, 44 insertions(+), 50 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java index 2f20c2d12e8cef..191c5ba9a40ed5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java @@ -32,7 +32,7 @@ public class ShowCatalogStmt extends ShowStmt { .addColumn(new Column("Type", ScalarType.createStringType())) .addColumn(new Column("IsCurrent", ScalarType.createStringType())) .addColumn(new Column("CreateTime", ScalarType.createStringType())) - .addColumn(new Column("LastSyncTime", ScalarType.createStringType())) + .addColumn(new Column("LastUpdateTime", ScalarType.createStringType())) .addColumn(new Column("Comment", ScalarType.createStringType())) .build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java index 5f739ece59721f..f9c5d94601f787 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java @@ -236,7 +236,7 @@ default OlapTable getOlapTableOrAnalysisException(String tableName) throws Analy CatalogIf getCatalog(); - default long getLastSyncTime() { + default long getLastUpdateTime() { return -1L; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index 58f65b7d64607b..fa833f359d267d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -229,7 +229,7 @@ default boolean isManagedTable() { return getType() == TableType.OLAP || getType() == TableType.MATERIALIZED_VIEW; } - default long getLastSyncTime() { + default long getLastUpdateTime() { return -1L; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java index cbeba87dac2339..0a82d37ff3d5cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java @@ -73,8 +73,8 @@ public abstract class ExternalDatabase protected Map tableNameToId = Maps.newConcurrentMap(); @SerializedName(value = "idToTbl") protected Map idToTbl = Maps.newConcurrentMap(); - @SerializedName(value = "lastSyncTime") - protected long lastSyncTime; + @SerializedName(value = "lastUpdateTime") + protected long lastUpdateTime; protected final InitDatabaseLog.Type dbLogType; protected ExternalCatalog extCatalog; protected boolean invalidCacheInInit = true; @@ -149,7 +149,7 @@ public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) { } tableNameToId = tmpTableNameToId; idToTbl = tmpIdToTbl; - lastSyncTime = log.getLastSyncTime(); + lastUpdateTime = log.getLastUpdateTime(); initialized = true; } @@ -184,8 +184,8 @@ protected void init() { } long currentTime = System.currentTimeMillis(); - lastSyncTime = currentTime; - initDatabaseLog.setLastSyncTime(lastSyncTime); + lastUpdateTime = currentTime; + initDatabaseLog.setLastUpdateTime(lastUpdateTime); initialized = true; Env.getCurrentEnv().getEditLog().logInitExternalDb(initDatabaseLog); } @@ -314,12 +314,12 @@ public T getTableNullable(long tableId) { return idToTbl.get(tableId); } - public long getLastSyncTime() { - return lastSyncTime; + public long getLastUpdateTime() { + return lastUpdateTime; } - public void setLastSyncTime(long lastSyncTime) { - this.lastSyncTime = lastSyncTime; + public void setLastUpdateTime(long lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java index 166c7ea3e69056..f4c76cda7a98a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java @@ -69,8 +69,8 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable { protected long timestamp; @SerializedName(value = "dbName") protected String dbName; - @SerializedName(value = "lastSyncTime") - protected long lastSyncTime; + @SerializedName(value = "lastUpdateTime") + protected long lastUpdateTime; protected boolean objectCreated; protected ExternalCatalog catalog; @@ -344,8 +344,8 @@ public Optional getColumnStatistic(String colName) { * * @return */ - public List initSchemaAndUpdateSyncTime() { - lastSyncTime = System.currentTimeMillis(); + public List initSchemaAndUpdateTime() { + lastUpdateTime = System.currentTimeMillis(); return initSchema(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java index f4a1db992929da..e66afe7b1f370e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.util.ListComparator; +import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.CatalogIf; import com.google.common.base.Preconditions; @@ -27,7 +28,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -38,7 +38,7 @@ */ public class CatalogsProcDir implements ProcDirInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() - .add("CatalogIds").add("CatalogName").add("DatabaseNum").add("LastSyncTime") + .add("CatalogIds").add("CatalogName").add("DatabaseNum").add("LastUpdateTime") .build(); private Env env; @@ -82,7 +82,6 @@ public ProcResult fetchResult() throws AnalysisException { List catalogIds = env.getCatalogMgr().getCatalogIds(); // get info List> catalogInfos = Lists.newArrayList(); - SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); for (long catalogId : catalogIds) { CatalogIf catalog = env.getCatalogMgr().getCatalog(catalogId); if (catalog == null) { @@ -92,8 +91,8 @@ public ProcResult fetchResult() throws AnalysisException { catalogInfo.add(catalog.getId()); catalogInfo.add(catalog.getName()); catalogInfo.add(catalog.getDbNames().size()); - long lastSyncTime = catalog.getLastSyncTime(); - catalogInfo.add(lastSyncTime > 0 ? simpleDateFormat.format(lastSyncTime) : "UNRECORDED"); + long lastUpdateTime = catalog.getlastUpdateTime(); + catalogInfo.add(lastUpdateTime > 0 ? TimeUtils.longToTimeString(lastUpdateTime) : "UNRECORDED"); catalogInfos.add(catalogInfo); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java index 2f4d791891e152..6e7532b4bf44cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java @@ -31,7 +31,6 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -44,7 +43,7 @@ public class DbsProcDir implements ProcDirInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("DbId").add("DbName").add("TableNum").add("Size").add("Quota") .add("LastConsistencyCheckTime").add("ReplicaCount").add("ReplicaQuota") - .add("TransactionQuota").add("LastSyncTime") + .add("TransactionQuota").add("LastUpdateTime") .build(); private Env env; @@ -95,7 +94,6 @@ public ProcResult fetchResult() throws AnalysisException { // get info List> dbInfos = new ArrayList<>(); - SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); for (String dbName : dbNames) { DatabaseIf db = catalog.getDbNullable(dbName); if (db == null) { @@ -124,8 +122,8 @@ public ProcResult fetchResult() throws AnalysisException { dbInfo.add(replicaCount); dbInfo.add(replicaQuota); dbInfo.add(transactionQuota); - long lastSyncTime = db.getLastSyncTime(); - dbInfo.add(lastSyncTime > 0 ? simpleDateFormat.format(lastSyncTime) : "UNRECORDED"); + long lastUpdateTime = db.getLastUpdateTime(); + dbInfo.add(lastUpdateTime > 0 ? TimeUtils.longToTimeString(lastUpdateTime) : "UNRECORDED"); } finally { db.readUnlock(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java index 91e489039eb69d..d3ee9dceacac6a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java @@ -33,7 +33,6 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -46,7 +45,7 @@ public class TablesProcDir implements ProcDirInterface { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("TableId").add("TableName").add("IndexNum").add("PartitionColumnName") .add("PartitionNum").add("State").add("Type").add("LastConsistencyCheckTime").add("ReplicaCount") - .add("LastSyncTime") + .add("LastUpdateTime") .build(); private DatabaseIf db; @@ -85,7 +84,6 @@ public ProcResult fetchResult() throws AnalysisException { // get info List> tableInfos = new ArrayList>(); List tableList = db.getTables(); - SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); for (TableIf table : tableList) { List tableInfo = new ArrayList(); int partitionNum = 1; @@ -131,8 +129,8 @@ public ProcResult fetchResult() throws AnalysisException { tableInfo.add(FeConstants.null_string); tableInfo.add(replicaCount); } - long lastSyncTime = table.getLastSyncTime(); - tableInfo.add(lastSyncTime > 0 ? simpleDateFormat.format(lastSyncTime) : "UNRECORDED"); + long lastUpdateTime = table.getLastUpdateTime(); + tableInfo.add(lastUpdateTime > 0 ? TimeUtils.longToTimeString(lastUpdateTime) : "UNRECORDED"); tableInfos.add(tableInfo); } finally { table.readUnlock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index fd74ada0e3db37..b66b9c77b27c15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -152,7 +152,7 @@ default void onClose() { String getComment(); - default long getLastSyncTime() { + default long getlastUpdateTime() { return -1L; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index c345904734d836..f979af1abe7b9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -43,6 +43,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.PrintableMap; +import org.apache.doris.common.util.TimeUtils; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.persist.OperationType; import org.apache.doris.persist.gson.GsonPostProcessable; @@ -60,7 +61,6 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -376,7 +376,6 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg) matcher = PatternMatcherWrapper.createMysqlPattern(showStmt.getPattern(), CaseSensibility.CATALOG.getCaseSensibility()); } - SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); for (CatalogIf catalog : nameToCatalog.values()) { if (Env.getCurrentEnv().getAccessManager() .checkCtlPriv(ConnectContext.get(), catalog.getName(), PrivPredicate.SHOW)) { @@ -397,8 +396,8 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg) Map props = catalog.getProperties(); String createTime = props.getOrDefault(CreateCatalogStmt.CREATE_TIME_PROP, "UNRECORDED"); row.add(createTime); - long lastSyncTime = catalog.getLastSyncTime(); - row.add(lastSyncTime > 0 ? simpleDateFormat.format(lastSyncTime) : "UNRECORDED"); + long lastUpdateTime = catalog.getlastUpdateTime(); + row.add(lastUpdateTime > 0 ? TimeUtils.longToTimeString(lastUpdateTime) : "UNRECORDED"); row.add(catalog.getComment()); rows.add(row); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 4cd740ad59abdd..855f64fb8b3621 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -84,8 +84,8 @@ public abstract class ExternalCatalog private boolean initialized = false; @SerializedName(value = "idToDb") protected Map> idToDb = Maps.newConcurrentMap(); - @SerializedName(value = "lastSyncTime") - protected long lastSyncTime; + @SerializedName(value = "lastUpdateTime") + protected long lastUpdateTime; // db name does not contains "default_cluster" protected Map dbNameToId = Maps.newConcurrentMap(); private boolean objectCreated = false; @@ -256,8 +256,8 @@ protected void init() { dbNameToId = tmpDbNameToId; idToDb = tmpIdToDb; long currentTime = System.currentTimeMillis(); - lastSyncTime = currentTime; - initCatalogLog.setLastSyncTime(lastSyncTime); + lastUpdateTime = currentTime; + initCatalogLog.setLastUpdateTime(lastUpdateTime); Env.getCurrentEnv().getEditLog().logInitCatalog(initCatalogLog); } @@ -280,7 +280,7 @@ public final List getSchema(String dbName, String tblName) { if (db.isPresent()) { Optional table = db.get().getTable(tblName); if (table.isPresent()) { - return table.get().initSchemaAndUpdateSyncTime(); + return table.get().initSchemaAndUpdateTime(); } } // return one column with unsupported type. @@ -396,12 +396,12 @@ private void modifyComment(Map props) { props.remove("comment"); } - public long getLastSyncTime() { - return lastSyncTime; + public long getlastUpdateTime() { + return lastUpdateTime; } - public void setLastSyncTime(long lastSyncTime) { - this.lastSyncTime = lastSyncTime; + public void setLastUpdateTime(long lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; } @Override @@ -438,7 +438,7 @@ public void replayInitCatalog(InitCatalogLog log) { } dbNameToId = tmpDbNameToId; idToDb = tmpIdToDb; - lastSyncTime = log.getLastSyncTime(); + lastUpdateTime = log.getLastUpdateTime(); initialized = true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java index 509409b2ba2e69..dd30fbf43c95ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java @@ -65,8 +65,8 @@ public enum Type { @SerializedName(value = "type") private Type type; - @SerializedName(value = "lastSyncTime") - private long lastSyncTime; + @SerializedName(value = "lastUpdateTime") + private long lastUpdateTime; public InitCatalogLog() { refreshCount = 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java index 349cb5f3369faa..3a85fb1edc54c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java @@ -68,8 +68,8 @@ public enum Type { @SerializedName(value = "type") private Type type; - @SerializedName(value = "lastSyncTime") - protected long lastSyncTime; + @SerializedName(value = "lastUpdateTime") + protected long lastUpdateTime; public InitDatabaseLog() { refreshCount = 0; From a8053bd60a8cc1adadde2de12bbe3f6c2c5623f3 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Mon, 17 Jul 2023 22:28:23 +0800 Subject: [PATCH 07/10] add ut --- .../doris/common/proc/CatalogsProcDir.java | 2 +- .../apache/doris/datasource/CatalogIf.java | 2 +- .../apache/doris/datasource/CatalogMgr.java | 2 +- .../doris/datasource/ExternalCatalog.java | 2 +- .../doris/catalog/RefreshTableTest.java | 34 ++++++++++++++++++- 5 files changed, 37 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java index e66afe7b1f370e..832497c086d848 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java @@ -91,7 +91,7 @@ public ProcResult fetchResult() throws AnalysisException { catalogInfo.add(catalog.getId()); catalogInfo.add(catalog.getName()); catalogInfo.add(catalog.getDbNames().size()); - long lastUpdateTime = catalog.getlastUpdateTime(); + long lastUpdateTime = catalog.getLastUpdateTime(); catalogInfo.add(lastUpdateTime > 0 ? TimeUtils.longToTimeString(lastUpdateTime) : "UNRECORDED"); catalogInfos.add(catalogInfo); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index b66b9c77b27c15..8e3304e54ac38a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -152,7 +152,7 @@ default void onClose() { String getComment(); - default long getlastUpdateTime() { + default long getLastUpdateTime() { return -1L; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index f979af1abe7b9d..e439013b538398 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -396,7 +396,7 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg) Map props = catalog.getProperties(); String createTime = props.getOrDefault(CreateCatalogStmt.CREATE_TIME_PROP, "UNRECORDED"); row.add(createTime); - long lastUpdateTime = catalog.getlastUpdateTime(); + long lastUpdateTime = catalog.getLastUpdateTime(); row.add(lastUpdateTime > 0 ? TimeUtils.longToTimeString(lastUpdateTime) : "UNRECORDED"); row.add(catalog.getComment()); rows.add(row); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 855f64fb8b3621..fe61dda59e509f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -396,7 +396,7 @@ private void modifyComment(Map props) { props.remove("comment"); } - public long getlastUpdateTime() { + public long getLastUpdateTime() { return lastUpdateTime; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java index 11e37ce238f05b..55aa583de1ce61 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java @@ -24,6 +24,7 @@ import org.apache.doris.analysis.RefreshTableStmt; import org.apache.doris.analysis.TableName; import org.apache.doris.catalog.external.ExternalDatabase; +import org.apache.doris.catalog.external.TestExternalDatabase; import org.apache.doris.catalog.external.TestExternalTable; import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.CatalogIf; @@ -71,7 +72,13 @@ protected void runAfterAll() throws Exception { @Test public void testRefreshCatalog() throws Exception { CatalogIf test1 = env.getCatalogMgr().getCatalog("test1"); + // init is 0 + long l1 = test1.getLastUpdateTime(); + Assertions.assertTrue(l1 == 0); TestExternalTable table = (TestExternalTable) test1.getDbNullable("db1").getTable("tbl11").get(); + // getDb() triggered init method + long l2 = test1.getLastUpdateTime(); + Assertions.assertTrue(l2 > l1); Assertions.assertFalse(table.isObjectCreated()); table.makeSureInitialized(); Assertions.assertTrue(table.isObjectCreated()); @@ -82,6 +89,9 @@ public void testRefreshCatalog() throws Exception { } catch (Exception e) { // Do nothing } + // not triggered init method + long l3 = test1.getLastUpdateTime(); + Assertions.assertTrue(l3 == l2); Assertions.assertTrue(table.isObjectCreated()); test1.getDbNullable("db1").getTables(); Assertions.assertFalse(table.isObjectCreated()); @@ -93,6 +103,9 @@ public void testRefreshCatalog() throws Exception { Assertions.assertFalse(((ExternalCatalog) test1).isInitialized()); table.makeSureInitialized(); Assertions.assertTrue(((ExternalCatalog) test1).isInitialized()); + // table.makeSureInitialized() triggered init method + long l4 = test1.getLastUpdateTime(); + Assertions.assertTrue(l4 > l3); try { DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); } catch (Exception e) { @@ -104,7 +117,12 @@ public void testRefreshCatalog() throws Exception { @Test public void testRefreshDatabase() throws Exception { CatalogIf test1 = env.getCatalogMgr().getCatalog("test1"); - TestExternalTable table = (TestExternalTable) test1.getDbNullable("db1").getTable("tbl11").get(); + TestExternalDatabase db1 = (TestExternalDatabase) test1.getDbNullable("db1"); + long l1 = db1.getLastUpdateTime(); + Assertions.assertTrue(l1 == 0); + TestExternalTable table = db1.getTable("tbl11").get(); + long l2 = db1.getLastUpdateTime(); + Assertions.assertTrue(l2 > l1); Assertions.assertFalse(table.isObjectCreated()); table.makeSureInitialized(); Assertions.assertTrue(table.isObjectCreated()); @@ -114,6 +132,8 @@ public void testRefreshDatabase() throws Exception { } catch (Exception e) { // Do nothing } + long l3 = db1.getLastUpdateTime(); + Assertions.assertTrue(l3 == l2); Assertions.assertTrue(table.isObjectCreated()); test1.getDbNullable("db1").getTables(); Assertions.assertFalse(table.isObjectCreated()); @@ -124,6 +144,8 @@ public void testRefreshDatabase() throws Exception { } Assertions.assertFalse(((ExternalDatabase) test1.getDbNullable("db1")).isInitialized()); table.makeSureInitialized(); + long l4 = db1.getLastUpdateTime(); + Assertions.assertTrue(l4 > l3); Assertions.assertTrue(((ExternalDatabase) test1.getDbNullable("db1")).isInitialized()); try { DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt); @@ -137,8 +159,12 @@ public void testRefreshTable() throws Exception { CatalogIf test1 = env.getCatalogMgr().getCatalog("test1"); TestExternalTable table = (TestExternalTable) test1.getDbNullable("db1").getTable("tbl11").get(); Assertions.assertFalse(table.isObjectCreated()); + long l1 = table.getLastUpdateTime(); + Assertions.assertTrue(l1 == 0); table.makeSureInitialized(); Assertions.assertTrue(table.isObjectCreated()); + long l2 = table.getLastUpdateTime(); + Assertions.assertTrue(l2 == l1); RefreshTableStmt refreshTableStmt = new RefreshTableStmt(new TableName("test1", "db1", "tbl11")); try { DdlExecutor.execute(Env.getCurrentEnv(), refreshTableStmt); @@ -146,6 +172,12 @@ public void testRefreshTable() throws Exception { // Do nothing } Assertions.assertFalse(table.isObjectCreated()); + long l3 = table.getLastUpdateTime(); + Assertions.assertTrue(l3 == l2); + table.getFullSchema(); + // only table.getFullSchema() can change table.lastUpdateTime + long l4 = table.getLastUpdateTime(); + Assertions.assertTrue(l4 > l3); } public static class RefreshTableProvider implements TestExternalCatalog.TestCatalogProvider { From 326893d467d0dedcb9fb315f0731fce0844ffdfc Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Mon, 17 Jul 2023 22:50:20 +0800 Subject: [PATCH 08/10] update time when sync event --- .../src/main/java/org/apache/doris/datasource/CatalogMgr.java | 4 ++++ .../java/org/apache/doris/datasource/ExternalObjectLog.java | 3 +++ 2 files changed, 7 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 4d0b8f6c7f600d..f4ca2f4b0aec48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -694,6 +694,7 @@ public void dropExternalTable(String dbName, String tableName, String catalogNam log.setCatalogId(catalog.getId()); log.setDbId(db.getId()); log.setTableId(table.getId()); + log.setLastUpdateTime(System.currentTimeMillis()); replayDropExternalTable(log); Env.getCurrentEnv().getEditLog().logDropExternalTable(log); } @@ -719,6 +720,7 @@ public void replayDropExternalTable(ExternalObjectLog log) { db.writeLock(); try { db.dropTable(table.getName()); + db.setLastUpdateTime(log.getLastUpdateTime()); } finally { db.writeUnlock(); } @@ -768,6 +770,7 @@ public void createExternalTableFromEvent(String dbName, String tableName, String log.setDbId(db.getId()); log.setTableName(tableName); log.setTableId(Env.getCurrentEnv().getNextId()); + log.setLastUpdateTime(System.currentTimeMillis()); replayCreateExternalTableFromEvent(log); Env.getCurrentEnv().getEditLog().logCreateExternalTable(log); } @@ -788,6 +791,7 @@ public void replayCreateExternalTableFromEvent(ExternalObjectLog log) { db.writeLock(); try { db.replayCreateTableFromEvent(log.getTableName(), log.getTableId()); + db.setLastUpdateTime(log.getLastUpdateTime()); } finally { db.writeUnlock(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java index 04d5b06036b494..4ebb137a607c2d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalObjectLog.java @@ -56,6 +56,9 @@ public class ExternalObjectLog implements Writable { @SerializedName(value = "partitionNames") private List partitionNames; + @SerializedName(value = "lastUpdateTime") + private long lastUpdateTime; + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); From 83ff342cda2776bc890476a8c50973b24c8da89a Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Tue, 18 Jul 2023 08:38:12 +0800 Subject: [PATCH 09/10] fix ut --- .../doris/analysis/ShowCatalogStmtTest.java | 6 +- .../doris/catalog/RefreshCatalogTest.java | 142 ++++++++++++++++++ .../apache/doris/catalog/RefreshDbTest.java | 138 +++++++++++++++++ .../doris/catalog/RefreshTableTest.java | 90 ----------- .../doris/common/proc/DbsProcDirTest.java | 12 +- 5 files changed, 290 insertions(+), 98 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshCatalogTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCatalogStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCatalogStmtTest.java index 68b7ae49ff1952..56e33f1b1f9907 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCatalogStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCatalogStmtTest.java @@ -26,18 +26,18 @@ public class ShowCatalogStmtTest { @Test public void testNormal() throws UserException, AnalysisException { - final Analyzer analyzer = AccessTestUtil.fetchBlockAnalyzer(); + final Analyzer analyzer = AccessTestUtil.fetchBlockAnalyzer(); ShowCatalogStmt stmt = new ShowCatalogStmt(); stmt.analyze(analyzer); Assert.assertNull(stmt.getCatalogName()); - Assert.assertEquals(6, stmt.getMetaData().getColumnCount()); + Assert.assertEquals(7, stmt.getMetaData().getColumnCount()); Assert.assertEquals("SHOW CATALOGS", stmt.toSql()); stmt = new ShowCatalogStmt(null, "%hive%"); stmt.analyze(analyzer); Assert.assertNull(stmt.getCatalogName()); Assert.assertNotNull(stmt.getPattern()); - Assert.assertEquals(6, stmt.getMetaData().getColumnCount()); + Assert.assertEquals(7, stmt.getMetaData().getColumnCount()); Assert.assertEquals("SHOW CATALOGS LIKE '%hive%'", stmt.toSql()); stmt = new ShowCatalogStmt("testCatalog", null); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshCatalogTest.java new file mode 100644 index 00000000000000..1a663d7a26a6b7 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshCatalogTest.java @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.analysis.CreateCatalogStmt; +import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.RefreshCatalogStmt; +import org.apache.doris.catalog.external.TestExternalTable; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.DdlExecutor; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class RefreshCatalogTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + FeConstants.runningUnitTest = true; + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog + CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.catalog.RefreshTableTest$RefreshTableProvider\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + DropCatalogStmt stmt = (DropCatalogStmt) parseAndAnalyzeStmt("drop catalog test1"); + env.getCatalogMgr().dropCatalog(stmt); + } + + @Test + public void testRefreshCatalog() throws Exception { + CatalogIf test1 = env.getCatalogMgr().getCatalog("test1"); + // init is 0 + long l1 = test1.getLastUpdateTime(); + Assertions.assertTrue(l1 == 0); + TestExternalTable table = (TestExternalTable) test1.getDbNullable("db1").getTable("tbl11").get(); + // getDb() triggered init method + long l2 = test1.getLastUpdateTime(); + Assertions.assertTrue(l2 > l1); + Assertions.assertFalse(table.isObjectCreated()); + table.makeSureInitialized(); + Assertions.assertTrue(table.isObjectCreated()); + RefreshCatalogStmt refreshCatalogStmt = new RefreshCatalogStmt("test1", null); + Assertions.assertTrue(refreshCatalogStmt.isInvalidCache()); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + // not triggered init method + long l3 = test1.getLastUpdateTime(); + Assertions.assertTrue(l3 == l2); + Assertions.assertTrue(table.isObjectCreated()); + test1.getDbNullable("db1").getTables(); + Assertions.assertFalse(table.isObjectCreated()); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + Assertions.assertFalse(((ExternalCatalog) test1).isInitialized()); + table.makeSureInitialized(); + Assertions.assertTrue(((ExternalCatalog) test1).isInitialized()); + // table.makeSureInitialized() triggered init method + long l4 = test1.getLastUpdateTime(); + Assertions.assertTrue(l4 > l3); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + Assertions.assertFalse(((ExternalCatalog) test1).isInitialized()); + } + + public static class RefreshTableProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tblSchemaMap1 = Maps.newHashMap(); + // db1 + tblSchemaMap1.put("tbl11", Lists.newArrayList( + new Column("a11", PrimitiveType.BIGINT), + new Column("a12", PrimitiveType.STRING), + new Column("a13", PrimitiveType.FLOAT))); + tblSchemaMap1.put("tbl12", Lists.newArrayList( + new Column("b21", PrimitiveType.BIGINT), + new Column("b22", PrimitiveType.STRING), + new Column("b23", PrimitiveType.FLOAT))); + MOCKED_META.put("db1", tblSchemaMap1); + // db2 + Map> tblSchemaMap2 = Maps.newHashMap(); + tblSchemaMap2.put("tbl21", Lists.newArrayList( + new Column("c11", PrimitiveType.BIGINT), + new Column("c12", PrimitiveType.STRING), + new Column("c13", PrimitiveType.FLOAT))); + MOCKED_META.put("db2", tblSchemaMap2); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java new file mode 100644 index 00000000000000..23def21689c70d --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.analysis.CreateCatalogStmt; +import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.RefreshDbStmt; +import org.apache.doris.catalog.external.ExternalDatabase; +import org.apache.doris.catalog.external.TestExternalDatabase; +import org.apache.doris.catalog.external.TestExternalTable; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.DdlExecutor; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class RefreshDbTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + FeConstants.runningUnitTest = true; + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog + CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.catalog.RefreshTableTest$RefreshTableProvider\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + DropCatalogStmt stmt = (DropCatalogStmt) parseAndAnalyzeStmt("drop catalog test1"); + env.getCatalogMgr().dropCatalog(stmt); + } + + @Test + public void testRefreshDatabase() throws Exception { + CatalogIf test1 = env.getCatalogMgr().getCatalog("test1"); + TestExternalDatabase db1 = (TestExternalDatabase) test1.getDbNullable("db1"); + long l1 = db1.getLastUpdateTime(); + Assertions.assertTrue(l1 == 0); + TestExternalTable table = db1.getTable("tbl11").get(); + long l2 = db1.getLastUpdateTime(); + Assertions.assertTrue(l2 > l1); + Assertions.assertFalse(table.isObjectCreated()); + table.makeSureInitialized(); + Assertions.assertTrue(table.isObjectCreated()); + RefreshDbStmt refreshDbStmt = new RefreshDbStmt("test1", "db1", null); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt); + } catch (Exception e) { + // Do nothing + } + long l3 = db1.getLastUpdateTime(); + Assertions.assertTrue(l3 == l2); + Assertions.assertTrue(table.isObjectCreated()); + test1.getDbNullable("db1").getTables(); + Assertions.assertFalse(table.isObjectCreated()); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt); + } catch (Exception e) { + // Do nothing + } + Assertions.assertFalse(((ExternalDatabase) test1.getDbNullable("db1")).isInitialized()); + table.makeSureInitialized(); + long l4 = db1.getLastUpdateTime(); + Assertions.assertTrue(l4 > l3); + Assertions.assertTrue(((ExternalDatabase) test1.getDbNullable("db1")).isInitialized()); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt); + } catch (Exception e) { + // Do nothing + } + } + + public static class RefreshTableProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tblSchemaMap1 = Maps.newHashMap(); + // db1 + tblSchemaMap1.put("tbl11", Lists.newArrayList( + new Column("a11", PrimitiveType.BIGINT), + new Column("a12", PrimitiveType.STRING), + new Column("a13", PrimitiveType.FLOAT))); + tblSchemaMap1.put("tbl12", Lists.newArrayList( + new Column("b21", PrimitiveType.BIGINT), + new Column("b22", PrimitiveType.STRING), + new Column("b23", PrimitiveType.FLOAT))); + MOCKED_META.put("db1", tblSchemaMap1); + // db2 + Map> tblSchemaMap2 = Maps.newHashMap(); + tblSchemaMap2.put("tbl21", Lists.newArrayList( + new Column("c11", PrimitiveType.BIGINT), + new Column("c12", PrimitiveType.STRING), + new Column("c13", PrimitiveType.FLOAT))); + MOCKED_META.put("db2", tblSchemaMap2); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java index 55aa583de1ce61..305bfa32c43cf2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java @@ -19,16 +19,11 @@ import org.apache.doris.analysis.CreateCatalogStmt; import org.apache.doris.analysis.DropCatalogStmt; -import org.apache.doris.analysis.RefreshCatalogStmt; -import org.apache.doris.analysis.RefreshDbStmt; import org.apache.doris.analysis.RefreshTableStmt; import org.apache.doris.analysis.TableName; -import org.apache.doris.catalog.external.ExternalDatabase; -import org.apache.doris.catalog.external.TestExternalDatabase; import org.apache.doris.catalog.external.TestExternalTable; import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.CatalogIf; -import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.DdlExecutor; @@ -69,91 +64,6 @@ protected void runAfterAll() throws Exception { env.getCatalogMgr().dropCatalog(stmt); } - @Test - public void testRefreshCatalog() throws Exception { - CatalogIf test1 = env.getCatalogMgr().getCatalog("test1"); - // init is 0 - long l1 = test1.getLastUpdateTime(); - Assertions.assertTrue(l1 == 0); - TestExternalTable table = (TestExternalTable) test1.getDbNullable("db1").getTable("tbl11").get(); - // getDb() triggered init method - long l2 = test1.getLastUpdateTime(); - Assertions.assertTrue(l2 > l1); - Assertions.assertFalse(table.isObjectCreated()); - table.makeSureInitialized(); - Assertions.assertTrue(table.isObjectCreated()); - RefreshCatalogStmt refreshCatalogStmt = new RefreshCatalogStmt("test1", null); - Assertions.assertTrue(refreshCatalogStmt.isInvalidCache()); - try { - DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); - } catch (Exception e) { - // Do nothing - } - // not triggered init method - long l3 = test1.getLastUpdateTime(); - Assertions.assertTrue(l3 == l2); - Assertions.assertTrue(table.isObjectCreated()); - test1.getDbNullable("db1").getTables(); - Assertions.assertFalse(table.isObjectCreated()); - try { - DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); - } catch (Exception e) { - // Do nothing - } - Assertions.assertFalse(((ExternalCatalog) test1).isInitialized()); - table.makeSureInitialized(); - Assertions.assertTrue(((ExternalCatalog) test1).isInitialized()); - // table.makeSureInitialized() triggered init method - long l4 = test1.getLastUpdateTime(); - Assertions.assertTrue(l4 > l3); - try { - DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); - } catch (Exception e) { - // Do nothing - } - Assertions.assertFalse(((ExternalCatalog) test1).isInitialized()); - } - - @Test - public void testRefreshDatabase() throws Exception { - CatalogIf test1 = env.getCatalogMgr().getCatalog("test1"); - TestExternalDatabase db1 = (TestExternalDatabase) test1.getDbNullable("db1"); - long l1 = db1.getLastUpdateTime(); - Assertions.assertTrue(l1 == 0); - TestExternalTable table = db1.getTable("tbl11").get(); - long l2 = db1.getLastUpdateTime(); - Assertions.assertTrue(l2 > l1); - Assertions.assertFalse(table.isObjectCreated()); - table.makeSureInitialized(); - Assertions.assertTrue(table.isObjectCreated()); - RefreshDbStmt refreshDbStmt = new RefreshDbStmt("test1", "db1", null); - try { - DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt); - } catch (Exception e) { - // Do nothing - } - long l3 = db1.getLastUpdateTime(); - Assertions.assertTrue(l3 == l2); - Assertions.assertTrue(table.isObjectCreated()); - test1.getDbNullable("db1").getTables(); - Assertions.assertFalse(table.isObjectCreated()); - try { - DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt); - } catch (Exception e) { - // Do nothing - } - Assertions.assertFalse(((ExternalDatabase) test1.getDbNullable("db1")).isInitialized()); - table.makeSureInitialized(); - long l4 = db1.getLastUpdateTime(); - Assertions.assertTrue(l4 > l3); - Assertions.assertTrue(((ExternalDatabase) test1.getDbNullable("db1")).isInitialized()); - try { - DdlExecutor.execute(Env.getCurrentEnv(), refreshDbStmt); - } catch (Exception e) { - // Do nothing - } - } - @Test public void testRefreshTable() throws Exception { CatalogIf test1 = env.getCatalogMgr().getCatalog("test1"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java index 07e8a679dc19c7..0fe6bf18a59a51 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java @@ -193,11 +193,13 @@ public void testFetchResultNormal() throws AnalysisException { Assert.assertTrue(result instanceof BaseProcResult); Assert.assertEquals(Lists.newArrayList("DbId", "DbName", "TableNum", "Size", "Quota", - "LastConsistencyCheckTime", "ReplicaCount", "ReplicaQuota", "TransactionQuota"), + "LastConsistencyCheckTime", "ReplicaCount", "ReplicaQuota", "TransactionQuota", "LastUpdateTime"), result.getColumnNames()); List> rows = Lists.newArrayList(); - rows.add(Arrays.asList(String.valueOf(db1.getId()), db1.getFullName(), "0", "0.000 ", "1024.000 TB", FeConstants.null_string, "0", "1073741824", "1000")); - rows.add(Arrays.asList(String.valueOf(db2.getId()), db2.getFullName(), "0", "0.000 ", "1024.000 TB", FeConstants.null_string, "0", "1073741824", "1000")); + rows.add(Arrays.asList(String.valueOf(db1.getId()), db1.getFullName(), "0", "0.000 ", "1024.000 TB", + FeConstants.null_string, "0", "1073741824", "1000", "UNRECORDED")); + rows.add(Arrays.asList(String.valueOf(db2.getId()), db2.getFullName(), "0", "0.000 ", "1024.000 TB", + FeConstants.null_string, "0", "1073741824", "1000", "UNRECORDED")); Assert.assertEquals(rows, result.getRows()); } @@ -228,8 +230,8 @@ public void testFetchResultInvalid() throws AnalysisException { dir = new DbsProcDir(env, catalog); result = dir.fetchResult(); Assert.assertEquals(Lists.newArrayList("DbId", "DbName", "TableNum", "Size", "Quota", - "LastConsistencyCheckTime", "ReplicaCount", "ReplicaQuota", "TransactionQuota"), - result.getColumnNames()); + "LastConsistencyCheckTime", "ReplicaCount", "ReplicaQuota", "TransactionQuota", "LastUpdateTime"), + result.getColumnNames()); List> rows = Lists.newArrayList(); Assert.assertEquals(rows, result.getRows()); } From 1926e653d6aa449f6e2489d421269cfb1e66b008 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Wed, 19 Jul 2023 11:09:45 +0800 Subject: [PATCH 10/10] unrecored ==> \n --- .../java/org/apache/doris/common/proc/CatalogsProcDir.java | 3 +-- .../main/java/org/apache/doris/common/proc/DbsProcDir.java | 3 +-- .../main/java/org/apache/doris/common/proc/TablesProcDir.java | 3 +-- .../src/main/java/org/apache/doris/datasource/CatalogMgr.java | 3 +-- .../java/org/apache/doris/common/proc/DbsProcDirTest.java | 4 ++-- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java index 832497c086d848..854b4dddc796df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java @@ -91,8 +91,7 @@ public ProcResult fetchResult() throws AnalysisException { catalogInfo.add(catalog.getId()); catalogInfo.add(catalog.getName()); catalogInfo.add(catalog.getDbNames().size()); - long lastUpdateTime = catalog.getLastUpdateTime(); - catalogInfo.add(lastUpdateTime > 0 ? TimeUtils.longToTimeString(lastUpdateTime) : "UNRECORDED"); + catalogInfo.add(TimeUtils.longToTimeString(catalog.getLastUpdateTime())); catalogInfos.add(catalogInfo); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java index 6e7532b4bf44cf..a3818099d8e641 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java @@ -122,8 +122,7 @@ public ProcResult fetchResult() throws AnalysisException { dbInfo.add(replicaCount); dbInfo.add(replicaQuota); dbInfo.add(transactionQuota); - long lastUpdateTime = db.getLastUpdateTime(); - dbInfo.add(lastUpdateTime > 0 ? TimeUtils.longToTimeString(lastUpdateTime) : "UNRECORDED"); + dbInfo.add(TimeUtils.longToTimeString(db.getLastUpdateTime())); } finally { db.readUnlock(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java index d3ee9dceacac6a..be51b157dedf6e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java @@ -129,8 +129,7 @@ public ProcResult fetchResult() throws AnalysisException { tableInfo.add(FeConstants.null_string); tableInfo.add(replicaCount); } - long lastUpdateTime = table.getLastUpdateTime(); - tableInfo.add(lastUpdateTime > 0 ? TimeUtils.longToTimeString(lastUpdateTime) : "UNRECORDED"); + tableInfo.add(TimeUtils.longToTimeString(table.getLastUpdateTime())); tableInfos.add(tableInfo); } finally { table.readUnlock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index f4ca2f4b0aec48..1de6dbfe972210 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -398,8 +398,7 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg) Map props = catalog.getProperties(); String createTime = props.getOrDefault(CreateCatalogStmt.CREATE_TIME_PROP, "UNRECORDED"); row.add(createTime); - long lastUpdateTime = catalog.getLastUpdateTime(); - row.add(lastUpdateTime > 0 ? TimeUtils.longToTimeString(lastUpdateTime) : "UNRECORDED"); + row.add(TimeUtils.longToTimeString(catalog.getLastUpdateTime())); row.add(catalog.getComment()); rows.add(row); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java index 0fe6bf18a59a51..b3052643d2e463 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java @@ -197,9 +197,9 @@ public void testFetchResultNormal() throws AnalysisException { result.getColumnNames()); List> rows = Lists.newArrayList(); rows.add(Arrays.asList(String.valueOf(db1.getId()), db1.getFullName(), "0", "0.000 ", "1024.000 TB", - FeConstants.null_string, "0", "1073741824", "1000", "UNRECORDED")); + FeConstants.null_string, "0", "1073741824", "1000", FeConstants.null_string)); rows.add(Arrays.asList(String.valueOf(db2.getId()), db2.getFullName(), "0", "0.000 ", "1024.000 TB", - FeConstants.null_string, "0", "1073741824", "1000", "UNRECORDED")); + FeConstants.null_string, "0", "1073741824", "1000", FeConstants.null_string)); Assert.assertEquals(rows, result.getRows()); }