From b2686c9dd78ed13627dc0ed51dd6422671b8f16c Mon Sep 17 00:00:00 2001 From: yangzhg Date: Tue, 29 Jun 2021 16:53:58 +0800 Subject: [PATCH 1/8] add update time to show table status --- .../org/apache/doris/catalog/OlapTable.java | 11 ++++++ .../java/org/apache/doris/catalog/Table.java | 4 ++ .../apache/doris/catalog/TempPartitions.java | 9 +++++ .../org/apache/doris/qe/ShowExecutor.java | 38 ++++++++++++++++--- 4 files changed, 57 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 72b0d9cb0ae8fe..3a21cd5d90d0a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -387,6 +387,17 @@ public Column getVisibleColumn(String columnName) { return null; } + @Override + public long getUpdateTime() { + long updateTime = tempPartitions.getUpdateTime(); + for (Partition p : idToPartition.values()) { + if (p.getVisibleVersionTime() > updateTime) { + updateTime = p.getVisibleVersionTime(); + } + } + return updateTime; + } + // this is only for schema change. public void renameIndexForSchemaChange(String name, String newName) { long idxId = indexNameToId.remove(name); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java index ec7e2813418ba4..ed0eaf8f1d85fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java @@ -232,6 +232,10 @@ public long getCreateTime() { return createTime; } + public long getUpdateTime() { + return -1L; + } + public TTableDescriptor toThrift() { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TempPartitions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TempPartitions.java index b5fb03c87454d9..fc5251336a41ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TempPartitions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TempPartitions.java @@ -57,6 +57,15 @@ public void addPartition(Partition partition) { idToPartition.put(partition.getId(), partition); nameToPartition.put(partition.getName(), partition); } + public long getUpdateTime() { + long updateTime = -1l; + for (Partition p : idToPartition.values()) { + if (p.getVisibleVersionTime() > updateTime) { + updateTime = p.getVisibleVersionTime(); + } + } + return updateTime; + } /* * Drop temp partitions. diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 42e035c800b6f7..0ca1cd3684f659 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -154,10 +154,12 @@ import java.io.InputStreamReader; import java.net.URL; import java.net.URLConnection; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -628,16 +630,42 @@ private void handleShowTableStatus() throws AnalysisException { PrivPredicate.SHOW)) { continue; } - List row = Lists.newArrayList(); // Name row.add(table.getName()); // Engine row.add(table.getEngine()); - // version, ra - for (int i = 0; i < 15; ++i) { - row.add(null); - } + // version + row.add(null); + // Row_format + row.add(null); + // Rows + row.add(null); + // Avg_row_length + row.add(null); + // Data_length + row.add(null); + // Max_data_length + row.add(null); + // Index_length + row.add(null); + // Data_free + row.add(null); + // Auto_increment + row.add(null); + // Create_time + row.add(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(table.getCreateTime() * 1000))); + // Update_time + row.add(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(table.getUpdateTime() * 1000))); + // Check_time + row.add(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(table.getLastCheckTime() * 1000))); + // Collation + row.add("utf-8"); + // Checksum + row.add(null); + // Create_options + row.add(null); + row.add(table.getComment()); rows.add(row); } From 90f18d7f23b158c83bd7eb1b8e4b9518e7aaf5b6 Mon Sep 17 00:00:00 2001 From: yangzhg Date: Tue, 29 Jun 2021 17:13:04 +0800 Subject: [PATCH 2/8] add update time to show table status --- fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 0ca1cd3684f659..39e9835e236546 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -656,7 +656,7 @@ private void handleShowTableStatus() throws AnalysisException { // Create_time row.add(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(table.getCreateTime() * 1000))); // Update_time - row.add(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(table.getUpdateTime() * 1000))); + row.add(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(table.getUpdateTime()))); // Check_time row.add(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(table.getLastCheckTime() * 1000))); // Collation From 6fb091ef4042d195e5fdcc970b8c1ff906158ade Mon Sep 17 00:00:00 2001 From: yangzhg Date: Tue, 29 Jun 2021 18:36:55 +0800 Subject: [PATCH 3/8] add update time to show table status --- .../schema_scanner/schema_tables_scanner.cpp | 26 +++++++++++++++++-- .../doris/service/FrontendServiceImpl.java | 3 +++ gensrc/thrift/FrontendService.thrift | 3 +++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_tables_scanner.cpp index 178424e86fe1b3..9e0296c17d8a88 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp @@ -169,7 +169,17 @@ Status SchemaTablesScanner::fill_one_row(Tuple* tuple, MemPool* pool) { } } // update_time - { tuple->set_null(_tuple_desc->slots()[15]->null_indicator_offset()); } + if (tbl_status.__isset.update_time) { + int64_t update_time = tbl_status.update_time; + if (update_time <= 0) { + tuple->set_null(_tuple_desc->slots()[15]->null_indicator_offset()); + } else { + tuple->set_not_null(_tuple_desc->slots()[15]->null_indicator_offset()); + void* slot = tuple->get_slot(_tuple_desc->slots()[15]->tuple_offset()); + DateTimeValue* time_slot = reinterpret_cast(slot); + time_slot->from_unixtime(update_time, TimezoneUtils::default_time_zone); + } + } // check_time if (tbl_status.__isset.last_check_time) { int64_t check_time = tbl_status.last_check_time; @@ -183,7 +193,19 @@ Status SchemaTablesScanner::fill_one_row(Tuple* tuple, MemPool* pool) { } } // collation - { tuple->set_null(_tuple_desc->slots()[17]->null_indicator_offset()); } + if (tbl_status.__isset.collation) { + void* slot = tuple->get_slot(_tuple_desc->slots()[17]->tuple_offset()); + StringValue* str_slot = reinterpret_cast(slot); + const std::string* src = &tbl_status.collation; + str_slot->len = src->length(); + str_slot->ptr = (char*)pool->allocate(str_slot->len); + if (NULL == str_slot->ptr) { + return Status::InternalError("Allocate memcpy failed."); + } + memcpy(str_slot->ptr, src->c_str(), str_slot->len); + } else { + tuple->set_null(_tuple_desc->slots()[17]->null_indicator_offset()); + } // checksum { tuple->set_null(_tuple_desc->slots()[18]->null_indicator_offset()); } // create_options diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 260a64108a1139..8af381f31d5158 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -285,6 +285,9 @@ public TListTableStatusResult listTableStatus(TGetTablesParams params) throws TE status.setComment(table.getComment()); status.setCreateTime(table.getCreateTime()); status.setLastCheckTime(table.getLastCheckTime()); + status.setUpdateTime(table.getUpdateTime()/1000); + status.setCheckTime(table.getLastCheckTime()); + status.setCollation("utf-8"); tablesResult.add(status); } finally { table.readUnlock(); diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 8c1328a1f22c4b..d22a486dde01e6 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -314,6 +314,9 @@ struct TTableStatus { 5: optional i64 last_check_time 6: optional i64 create_time 7: optional string ddl_sql + 8: optional i64 update_time + 9: optional i64 check_time + 10: optional string collation } struct TListTableStatusResult { From ac7300d9c99af668e4d11f50f8e4adaaacaa35e6 Mon Sep 17 00:00:00 2001 From: yangzhg Date: Tue, 29 Jun 2021 18:44:02 +0800 Subject: [PATCH 4/8] add update time to show table status --- .../main/java/org/apache/doris/qe/ShowExecutor.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 39e9835e236546..780011871b140c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -656,9 +656,17 @@ private void handleShowTableStatus() throws AnalysisException { // Create_time row.add(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(table.getCreateTime() * 1000))); // Update_time - row.add(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(table.getUpdateTime()))); + if (table.getUpdateTime() > 0) { + row.add(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(table.getUpdateTime()))); + } else { + row.add(null); + } // Check_time - row.add(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(table.getLastCheckTime() * 1000))); + if (table.getLastCheckTime() > 0) { + row.add(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(table.getLastCheckTime() * 1000))); + } else { + row.add(null); + } // Collation row.add("utf-8"); // Checksum From 830bbe383ab2c6528dc8c9aa0fd45f8b0e67cf8b Mon Sep 17 00:00:00 2001 From: yangzhg Date: Tue, 29 Jun 2021 19:25:47 +0800 Subject: [PATCH 5/8] add update time to show table status --- .../schema_scanner/schema_tables_scanner.cpp | 22 +++++++++++++--- .../org/apache/doris/catalog/OlapTable.java | 25 +++++++++++++++++++ .../java/org/apache/doris/catalog/Table.java | 13 ++++++++++ .../org/apache/doris/qe/ShowExecutor.java | 6 ++--- .../doris/service/FrontendServiceImpl.java | 3 +++ gensrc/thrift/FrontendService.thrift | 3 +++ 6 files changed, 65 insertions(+), 7 deletions(-) diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_tables_scanner.cpp index 9e0296c17d8a88..96e95a5b8ff765 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp @@ -143,12 +143,26 @@ Status SchemaTablesScanner::fill_one_row(Tuple* tuple, MemPool* pool) { // row_format { tuple->set_null(_tuple_desc->slots()[6]->null_indicator_offset()); } // rows - { tuple->set_null(_tuple_desc->slots()[7]->null_indicator_offset()); } + if (tbl_status.__isset.rows) { + void* slot = tuple->get_slot(_tuple_desc->slots()[7]->tuple_offset()); + *(reinterpret_cast(slot)) = tbl_status.rows; + } else { + tuple->set_null(_tuple_desc->slots()[7]->null_indicator_offset()); + } // avg_row_length - { tuple->set_null(_tuple_desc->slots()[8]->null_indicator_offset()); } + if (tbl_status.__isset.avg_row_length) { + void* slot = tuple->get_slot(_tuple_desc->slots()[8]->tuple_offset()); + *(reinterpret_cast(slot)) = tbl_status.avg_row_length; + } else { + tuple->set_null(_tuple_desc->slots()[8]->null_indicator_offset()); + } // data_length - { tuple->set_null(_tuple_desc->slots()[9]->null_indicator_offset()); } - // max_data_length + if (tbl_status.__isset.avg_row_length) { + void* slot = tuple->get_slot(_tuple_desc->slots()[9]->tuple_offset()); + *(reinterpret_cast(slot)) = tbl_status.data_length; + } else { + tuple->set_null(_tuple_desc->slots()[9]->null_indicator_offset()); + } // max_data_length { tuple->set_null(_tuple_desc->slots()[10]->null_indicator_offset()); } // index_length { tuple->set_null(_tuple_desc->slots()[11]->null_indicator_offset()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 3a21cd5d90d0a1..478e07622586ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -933,6 +933,7 @@ public TTableDescriptor toThrift() { return tTableDescriptor; } + @Override public long getRowCount() { long rowCount = 0; for (Map.Entry entry : idToPartition.entrySet()) { @@ -941,6 +942,30 @@ public long getRowCount() { return rowCount; } + @Override + public long getAvgRowLength() { + long rowCount = 0; + long dataSize = 0; + for (Map.Entry entry : idToPartition.entrySet()) { + rowCount += entry.getValue().getBaseIndex().getRowCount(); + dataSize += entry.getValue().getBaseIndex().getDataSize(); + } + if (rowCount > 0) { + return dataSize / rowCount; + } else { + return 0; + } + } + + @Override + public long getDataLength() { + long dataSize = 0; + for (Map.Entry entry : idToPartition.entrySet()) { + dataSize += entry.getValue().getBaseIndex().getDataSize(); + } + return dataSize; + } + @Override public CreateTableStmt toCreateTableStmt(String dbName) { throw new RuntimeException("Don't support anymore"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java index ed0eaf8f1d85fc..152b0859d38e4c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java @@ -236,6 +236,19 @@ public long getUpdateTime() { return -1L; } + public long getRowCount() { + return 0; + } + + public long getAvgRowLength() { + return 0; + } + + public long getDataLength() { + return 0; + } + + public TTableDescriptor toThrift() { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 780011871b140c..b4f4e1274adc91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -640,11 +640,11 @@ private void handleShowTableStatus() throws AnalysisException { // Row_format row.add(null); // Rows - row.add(null); + row.add(String.valueOf(table.getRowCount())); // Avg_row_length - row.add(null); + row.add(String.valueOf(table.getAvgRowLength())); // Data_length - row.add(null); + row.add(String.valueOf(table.getDataLength())); // Max_data_length row.add(null); // Index_length diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 8af381f31d5158..bdc70b0359e250 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -288,6 +288,9 @@ public TListTableStatusResult listTableStatus(TGetTablesParams params) throws TE status.setUpdateTime(table.getUpdateTime()/1000); status.setCheckTime(table.getLastCheckTime()); status.setCollation("utf-8"); + status.setRows(table.getRowCount()); + status.dataLength(table.getDataLength()); + status.setAvgRowLength(table.getAvgRowLength()); tablesResult.add(status); } finally { table.readUnlock(); diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index d22a486dde01e6..92463922740842 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -317,6 +317,9 @@ struct TTableStatus { 8: optional i64 update_time 9: optional i64 check_time 10: optional string collation + 11: optional i64 rows; + 12: optional i64 avg_row_length + 13: optional i64 data_length; } struct TListTableStatusResult { From c25d88693694f002a7234999ac853cc5edd69f4b Mon Sep 17 00:00:00 2001 From: yangzhg Date: Tue, 29 Jun 2021 19:30:25 +0800 Subject: [PATCH 6/8] add update time to show table status --- .../main/java/org/apache/doris/service/FrontendServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index bdc70b0359e250..1bc16e71402b28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -289,7 +289,7 @@ public TListTableStatusResult listTableStatus(TGetTablesParams params) throws TE status.setCheckTime(table.getLastCheckTime()); status.setCollation("utf-8"); status.setRows(table.getRowCount()); - status.dataLength(table.getDataLength()); + status.setDataLength(table.getDataLength()); status.setAvgRowLength(table.getAvgRowLength()); tablesResult.add(status); } finally { From aba26645eac2d6ed2de04f0274fab618271bb921 Mon Sep 17 00:00:00 2001 From: yangzhg Date: Mon, 5 Jul 2021 10:13:51 +0800 Subject: [PATCH 7/8] use TimeUtils --- .../src/main/java/org/apache/doris/qe/ShowExecutor.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index b4f4e1274adc91..44225f1a5466ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -124,6 +124,7 @@ import org.apache.doris.common.util.OrderByPair; import org.apache.doris.common.util.ProfileManager; import org.apache.doris.common.util.RuntimeProfile; +import org.apache.doris.common.util.TimeUtils; import org.apache.doris.load.DeleteHandler; import org.apache.doris.load.ExportJob; import org.apache.doris.load.ExportMgr; @@ -654,16 +655,16 @@ private void handleShowTableStatus() throws AnalysisException { // Auto_increment row.add(null); // Create_time - row.add(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(table.getCreateTime() * 1000))); + row.add(TimeUtils.longToTimeString(table.getCreateTime() * 1000)); // Update_time if (table.getUpdateTime() > 0) { - row.add(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(table.getUpdateTime()))); + row.add(TimeUtils.longToTimeString(table.getUpdateTime())); } else { row.add(null); } // Check_time if (table.getLastCheckTime() > 0) { - row.add(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(table.getLastCheckTime() * 1000))); + row.add(TimeUtils.longToTimeString(table.getLastCheckTime() * 1000)); } else { row.add(null); } From f31b45d84674553d198522374d532e88e4e54d38 Mon Sep 17 00:00:00 2001 From: yangzhg Date: Tue, 6 Jul 2021 13:30:41 +0800 Subject: [PATCH 8/8] fix compile error --- fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 44225f1a5466ef..cf3090b0656698 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -155,12 +155,10 @@ import java.io.InputStreamReader; import java.net.URL; import java.net.URLConnection; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Map;