From a4111d27824a5e8828e7541701259e7d1a29d2d7 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt <952130278@qq.com> Date: Wed, 14 Jul 2021 20:31:15 +0800 Subject: [PATCH 01/12] add query 'show trash' for display backends trash used capacity --- be/src/olap/storage_engine.cpp | 16 +++- be/src/olap/storage_engine.h | 2 + be/src/service/backend_service.cpp | 14 +++ be/src/service/backend_service.h | 2 + fe/fe-core/src/main/cup/sql_parser.cup | 12 ++- .../apache/doris/analysis/ShowTrashStmt.java | 93 +++++++++++++++++++ .../apache/doris/common/util/DebugUtil.java | 2 +- .../org/apache/doris/qe/ShowExecutor.java | 49 ++++++++++ fe/fe-core/src/main/jflex/sql_scanner.flex | 1 + .../apache/doris/common/GenericPoolTest.java | 6 ++ .../doris/utframe/MockedBackendFactory.java | 5 + gensrc/thrift/BackendService.thrift | 2 + 12 files changed, 201 insertions(+), 3 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index 9d265e87f5e73e..1806b5984ba9cf 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -331,7 +331,7 @@ std::vector StorageEngine::get_stores() { template std::vector StorageEngine::get_stores(); template std::vector StorageEngine::get_stores(); -OLAPStatus StorageEngine::get_all_data_dir_info(vector* data_dir_infos, +OLAPStatus StorageEngine::get_all_data_dir_info(std::vector* data_dir_infos, bool need_update) { OLAPStatus res = OLAP_SUCCESS; data_dir_infos->clear(); @@ -377,6 +377,20 @@ OLAPStatus StorageEngine::get_all_data_dir_info(vector* data_dir_in return res; } +int64_t StorageEngine::get_file_or_directory_size(std::filesystem::path file_path) { + if (!std::filesystem::exists(file_path)) { + return 0; + } + if (!std::filesystem::is_directory(file_path)) { + return std::filesystem::file_size(file_path); + } + int64_t sum_size = 0; + for (auto& it : std::filesystem::directory_iterator(file_path)) { + sum_size += get_file_or_directory_size(it.path()); + } + return sum_size; +} + void StorageEngine::_start_disk_stat_monitor() { for (auto& it : _store_map) { it.second->health_check(); diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 1162260fc2038f..feb2b1f1d8f930 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -100,6 +100,8 @@ class StorageEngine { // @brief 获取所有root_path信息 OLAPStatus get_all_data_dir_info(std::vector* data_dir_infos, bool need_update); + int64_t get_file_or_directory_size(std::filesystem::path file_path); + // get root path for creating tablet. The returned vector of root path should be random, // for avoiding that all the tablet would be deployed one disk. std::vector get_stores_for_create_tablet(TStorageMedium::type storage_medium); diff --git a/be/src/service/backend_service.cpp b/be/src/service/backend_service.cpp index f4e4fd0b979a3d..296e52305c616b 100644 --- a/be/src/service/backend_service.cpp +++ b/be/src/service/backend_service.cpp @@ -210,6 +210,20 @@ void BackendService::get_tablet_stat(TTabletStatResult& result) { StorageEngine::instance()->tablet_manager()->get_tablet_stat(&result); } +int64_t BackendService::get_trash_used_capacity() { + int64_t result = 0; + + std::vector data_dir_infos; + StorageEngine::instance()->get_all_data_dir_info(&data_dir_infos, false /*do not update */); + + for (auto& root_path_info : data_dir_infos) { + std::string lhs_trash_path = root_path_info.path + TRASH_PREFIX; + std::filesystem::path trash_path(lhs_trash_path); + result += StorageEngine::instance()->get_file_or_directory_size(trash_path); + } + return result; +} + void BackendService::submit_routine_load_task(TStatus& t_status, const std::vector& tasks) { for (auto& task : tasks) { diff --git a/be/src/service/backend_service.h b/be/src/service/backend_service.h index 166a1366d82b04..03f7279f0b85b2 100644 --- a/be/src/service/backend_service.h +++ b/be/src/service/backend_service.h @@ -129,6 +129,8 @@ class BackendService : public BackendServiceIf { virtual void get_tablet_stat(TTabletStatResult& result) override; + virtual int64_t get_trash_used_capacity() override; + virtual void submit_routine_load_task(TStatus& t_status, const std::vector& tasks) override; diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index c4543364574e96..b338b19e80dc57 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -264,7 +264,7 @@ terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALL, KW_ALTER, KW_A KW_S3, KW_SCHEMA, KW_SCHEMAS, KW_SECOND, KW_SELECT, KW_SEMI, KW_SERIALIZABLE, KW_SESSION, KW_SET, KW_SETS, KW_SET_VAR, KW_SHOW, KW_SIGNED, KW_SMALLINT, KW_SNAPSHOT, KW_SONAME, KW_SPLIT, KW_START, KW_STATUS, KW_STOP, KW_STORAGE, KW_STREAM, KW_STRING, KW_STRUCT, KW_SUM, KW_SUPERUSER, KW_SYNC, KW_SYSTEM, - KW_TABLE, KW_TABLES, KW_TABLET, KW_TASK, KW_TEMPORARY, KW_TERMINATED, KW_THAN, KW_TIME, KW_THEN, KW_TIMESTAMP, KW_TINYINT, + KW_TABLE, KW_TABLES, KW_TABLET, KW_TASK, KW_TEMPORARY, KW_TERMINATED, KW_THAN, KW_TIME, KW_THEN, KW_TIMESTAMP, KW_TINYINT,KW_TRASH, KW_TO, KW_TRANSACTION, KW_TRIGGERS, KW_TRIM, KW_TRUE, KW_TRUNCATE, KW_TYPE, KW_TYPES, KW_UNCOMMITTED, KW_UNBOUNDED, KW_UNION, KW_UNIQUE, KW_UNSIGNED, KW_USE, KW_USER, KW_USING, KW_UNINSTALL, KW_VALUE, KW_VALUES, KW_VARCHAR, KW_VARIABLES, KW_VERBOSE, KW_VIEW, @@ -331,6 +331,8 @@ nonterminal InsertStmt insert_stmt; nonterminal InsertTarget insert_target; nonterminal InsertSource insert_source; +nonterminal opt_trash_string_list; + nonterminal BackupStmt backup_stmt; nonterminal AbstractBackupTableRefClause opt_backup_table_ref_list; nonterminal Boolean backup_exclude_or_not; @@ -2554,6 +2556,14 @@ show_param ::= {: RESULT = new ShowBackendsStmt(); :} + | KW_TRASH KW_ON string_list:backends + {: + RESULT = new ShowTrashStmt(backends); + :} + | KW_TRASH + {: + RESULT = new ShowTrashStmt(null); + :} | KW_FRONTENDS {: RESULT = new ShowFrontendsStmt(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java new file mode 100644 index 00000000000000..4ccf0a7eab3589 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.qe.ShowResultSetMetaData; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.Backend; +import org.apache.doris.mysql.privilege.PrivPredicate; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ShowTrashStmt extends ShowStmt { + public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder().add("BackendId") + .add("BackendHost").add("BackendHeartbeatPort").add("TrashUsedCapacity").build(); + private List backends = Lists.newArrayList(); + + public ShowTrashStmt(List backends) { + ImmutableMap backendsInfo = Catalog.getCurrentSystemInfo().getIdToBackend(); + Map backendsID = new HashMap(); + for (Backend backend : backendsInfo.values()) { + backendsID.put(String.valueOf(backend.getHost()) + ":" + String.valueOf(backend.getHeartbeatPort()), backend.getId()); + } + if (backends == null) { + for (Backend backend : backendsInfo.values()) { + this.backends.add(backend); + } + } else { + for (String backend : backends) { + if (backendsID.get(backend) != null) { + this.backends.add(backendsInfo.get(backendsID.get(backend))); + } + } + } + } + + public List getBackends() { + return backends; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN) + && !Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), + PrivPredicate.OPERATOR)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN/OPERATOR"); + } + } + + @Override + public ShowResultSetMetaData getMetaData() { + ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); + for (String title : TITLE_NAMES) { + builder.addColumn(new Column(title, ScalarType.createVarchar(30))); + } + return builder.build(); + } + + @Override + public RedirectStatus getRedirectStatus() { + if (ConnectContext.get().getSessionVariable().getForwardToMaster()) { + return RedirectStatus.FORWARD_NO_SYNC; + } else { + return RedirectStatus.NO_FORWARD; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java index 8d4b960dad251e..41d911820ee471 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java @@ -27,7 +27,7 @@ import java.util.UUID; public class DebugUtil { - public static final DecimalFormat DECIMAL_FORMAT_SCALE_3 = new DecimalFormat("#.000"); + public static final DecimalFormat DECIMAL_FORMAT_SCALE_3 = new DecimalFormat("0.000"); public static int THOUSAND = 1000; public static int MILLION = 1000 * THOUSAND; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index e526b0001f9559..830e4e60304c2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -71,6 +71,7 @@ import org.apache.doris.analysis.ShowTableStmt; import org.apache.doris.analysis.ShowTabletStmt; import org.apache.doris.analysis.ShowTransactionStmt; +import org.apache.doris.analysis.ShowTrashStmt; import org.apache.doris.analysis.ShowUserPropertyStmt; import org.apache.doris.analysis.ShowVariablesStmt; import org.apache.doris.analysis.ShowViewStmt; @@ -103,12 +104,14 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.CaseSensibility; import org.apache.doris.common.ConfigBase; +import org.apache.doris.common.ClientPool; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.PatternMatcher; +import org.apache.doris.common.Pair; import org.apache.doris.common.proc.BackendsProcDir; import org.apache.doris.common.proc.FrontendsProcNode; import org.apache.doris.common.proc.LoadProcDir; @@ -119,6 +122,7 @@ import org.apache.doris.common.proc.TabletsProcDir; import org.apache.doris.common.profile.ProfileTreeNode; import org.apache.doris.common.profile.ProfileTreePrinter; +import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; @@ -139,6 +143,8 @@ import org.apache.doris.system.Backend; import org.apache.doris.system.SystemInfoService; import org.apache.doris.thrift.TUnit; +import org.apache.doris.thrift.BackendService; +import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.transaction.GlobalTransactionMgr; import com.google.common.base.Preconditions; @@ -272,6 +278,8 @@ public ShowResultSet execute() throws AnalysisException { handleShowGrants(); } else if (stmt instanceof ShowRolesStmt) { handleShowRoles(); + } else if (stmt instanceof ShowTrashStmt) { + handleShowTrash(); } else if (stmt instanceof AdminShowReplicaStatusStmt) { handleAdminShowTabletStatus(); } else if (stmt instanceof AdminShowReplicaDistributionStmt) { @@ -1690,6 +1698,47 @@ private void handleShowRoles() { List> infos = Catalog.getCurrentCatalog().getAuth().getRoleInfo(); resultSet = new ShowResultSet(showStmt.getMetaData(), infos); } + + private void handleShowTrash() { + ShowTrashStmt showStmt = (ShowTrashStmt) stmt; + List> results = Lists.newArrayList(); + + for (Backend backend : showStmt.getBackends()) { + BackendService.Client client = null; + TNetworkAddress address = null; + Long trashUsedCapacityB = null; + boolean ok = false; + try { + long start = System.currentTimeMillis(); + address = new TNetworkAddress(backend.getHost(), backend.getBePort()); + client = ClientPool.backendPool.borrowObject(address); + trashUsedCapacityB = client.getTrashUsedCapacity(); + LOG.debug("get trash used capacity from backend: {}, trashUsedCapacity: {}, cost {}", backend.getId(), + trashUsedCapacityB, System.currentTimeMillis() - start); + ok = true; + } catch (Exception e) { + LOG.warn("task exec error. backend[{}]", backend.getId(), e); + } finally { + if (ok) { + ClientPool.backendPool.returnObject(address, client); + } else { + ClientPool.backendPool.invalidateObject(address, client); + } + } + + if (trashUsedCapacityB != null) { + List backendInfo = new ArrayList(); + backendInfo.add(String.valueOf(backend.getId())); + backendInfo.add(backend.getHost()); + backendInfo.add(String.valueOf(backend.getHeartbeatPort())); + Pair trashUsedCapacity = DebugUtil.getByteUint(trashUsedCapacityB); + backendInfo.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(trashUsedCapacity.first) + " " + trashUsedCapacity.second); + results.add(backendInfo); + } + } + + resultSet = new ShowResultSet(showStmt.getMetaData(), results); + } private void handleAdminShowTabletStatus() throws AnalysisException { AdminShowReplicaStatusStmt showStmt = (AdminShowReplicaStatusStmt) stmt; diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex index 53c6c703c2693f..d2002d947d99f5 100644 --- a/fe/fe-core/src/main/jflex/sql_scanner.flex +++ b/fe/fe-core/src/main/jflex/sql_scanner.flex @@ -103,6 +103,7 @@ import org.apache.doris.qe.SqlModeHelper; keywordMap.put("array", new Integer(SqlParserSymbols.KW_ARRAY)); keywordMap.put("backend", new Integer(SqlParserSymbols.KW_BACKEND)); keywordMap.put("backends", new Integer(SqlParserSymbols.KW_BACKENDS)); + keywordMap.put("trash", new Integer(SqlParserSymbols.KW_TRASH)); keywordMap.put("backup", new Integer(SqlParserSymbols.KW_BACKUP)); keywordMap.put("begin", new Integer(SqlParserSymbols.KW_BEGIN)); keywordMap.put("between", new Integer(SqlParserSymbols.KW_BETWEEN)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java index b8a6152151e35b..188d0b8e9e93ae 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java @@ -193,6 +193,12 @@ public TStatus eraseExportTask(TUniqueId task_id) throws TException { return null; } + @Override + public long getTrashUsedCapacity() throws TException { + // TODO Auto-generated method stub + return 0l; + } + @Override public TTabletStatResult getTabletStat() throws TException { // TODO Auto-generated method stub diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java index a65461d89ac0d2..7792f98391d292 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java @@ -244,6 +244,11 @@ public TStatus eraseExportTask(TUniqueId task_id) throws TException { return new TStatus(TStatusCode.OK); } + @Override + public long getTrashUsedCapacity() throws TException { + return 0l; + } + @Override public TTabletStatResult getTabletStat() throws TException { return new TTabletStatResult(Maps.newHashMap()); diff --git a/gensrc/thrift/BackendService.thrift b/gensrc/thrift/BackendService.thrift index 200a8ead956534..ffbd26667e8134 100644 --- a/gensrc/thrift/BackendService.thrift +++ b/gensrc/thrift/BackendService.thrift @@ -147,6 +147,8 @@ service BackendService { Status.TStatus erase_export_task(1:Types.TUniqueId task_id); TTabletStatResult get_tablet_stat(); + + i64 get_trash_used_capacity(); Status.TStatus submit_routine_load_task(1:list tasks); From 80ba2c31e944a92139513850dc276300e9b2bf4e Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt <952130278@qq.com> Date: Thu, 15 Jul 2021 13:41:05 +0800 Subject: [PATCH 02/12] merge column 'BackendHost','BackendHeartBeatPort' into 'Backend' && adjust query define at 'sql_parser' && remove duplicates input backend --- fe/fe-core/src/main/cup/sql_parser.cup | 4 +--- .../main/java/org/apache/doris/analysis/ShowTrashStmt.java | 3 ++- .../src/main/java/org/apache/doris/qe/ShowExecutor.java | 3 +-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index b338b19e80dc57..3c13f363aba5f8 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -331,8 +331,6 @@ nonterminal InsertStmt insert_stmt; nonterminal InsertTarget insert_target; nonterminal InsertSource insert_source; -nonterminal opt_trash_string_list; - nonterminal BackupStmt backup_stmt; nonterminal AbstractBackupTableRefClause opt_backup_table_ref_list; nonterminal Boolean backup_exclude_or_not; @@ -2556,7 +2554,7 @@ show_param ::= {: RESULT = new ShowBackendsStmt(); :} - | KW_TRASH KW_ON string_list:backends + | KW_TRASH KW_ON LPAREN string_list:backends RPAREN {: RESULT = new ShowTrashStmt(backends); :} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java index 4ccf0a7eab3589..b841716660f775 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java @@ -38,7 +38,7 @@ public class ShowTrashStmt extends ShowStmt { public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder().add("BackendId") - .add("BackendHost").add("BackendHeartbeatPort").add("TrashUsedCapacity").build(); + .add("Backend").add("TrashUsedCapacity").build(); private List backends = Lists.newArrayList(); public ShowTrashStmt(List backends) { @@ -55,6 +55,7 @@ public ShowTrashStmt(List backends) { for (String backend : backends) { if (backendsID.get(backend) != null) { this.backends.add(backendsInfo.get(backendsID.get(backend))); + backendsID.remove(backend); // avoid repetition } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 830e4e60304c2a..fd0301f186c0f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -1729,8 +1729,7 @@ private void handleShowTrash() { if (trashUsedCapacityB != null) { List backendInfo = new ArrayList(); backendInfo.add(String.valueOf(backend.getId())); - backendInfo.add(backend.getHost()); - backendInfo.add(String.valueOf(backend.getHeartbeatPort())); + backendInfo.add(backend.getHost() + ":" + String.valueOf(backend.getHeartbeatPort())); Pair trashUsedCapacity = DebugUtil.getByteUint(trashUsedCapacityB); backendInfo.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(trashUsedCapacity.first) + " " + trashUsedCapacity.second); results.add(backendInfo); From f92938d83b113412616b1d5d0f9f258b72c8cfe2 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt <952130278@qq.com> Date: Thu, 15 Jul 2021 19:15:28 +0800 Subject: [PATCH 03/12] 1. change sql_scannner.flex to keep the alpha order. 2. display null when connect fail to client 3. trash information to show proc, and support show proc '/trash' 4. add document of show trash stmt --- .../Administration/SHOW TRASH.md | 54 +++++++++ .../Administration/SHOW TRASH.md | 50 ++++++++ .../apache/doris/analysis/ShowTrashStmt.java | 3 +- .../apache/doris/common/proc/ProcService.java | 1 + .../doris/common/proc/TrashProcNode.java | 110 ++++++++++++++++++ .../org/apache/doris/qe/ShowExecutor.java | 45 +------ fe/fe-core/src/main/jflex/sql_scanner.flex | 2 +- 7 files changed, 222 insertions(+), 43 deletions(-) create mode 100644 docs/en/sql-reference/sql-statements/Administration/SHOW TRASH.md create mode 100644 docs/zh-CN/sql-reference/sql-statements/Administration/SHOW TRASH.md create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java diff --git a/docs/en/sql-reference/sql-statements/Administration/SHOW TRASH.md b/docs/en/sql-reference/sql-statements/Administration/SHOW TRASH.md new file mode 100644 index 00000000000000..b443100cac717d --- /dev/null +++ b/docs/en/sql-reference/sql-statements/Administration/SHOW TRASH.md @@ -0,0 +1,54 @@ +--- +{ + "title": "SHOW BACKENDS", + "language": "zh-CN" +} +--- + + + +# SHOW TRASH +## description + +This statement is used to view trash used capacity on some backends. + + Syntax: + + SHOW TRASH [ON ("BackendHost1:BackendHeartBeatPort1", "BackendHost2:BackendHeartBeatPort2", ...)]; + + Explain: + + 1. BackendId Indicates the ID of the backend node. + 2. Backend The format is BackendHost:BackendHeartBeatPort of the node. + 3. TrashUsedCapacity Indicates that the garbage data of the node occupies space. + +## example + + 1. View the space occupied by garbage data of all be nodes. + + SHOW TRASH; + + 2. Check the space occupied by garbage data of '192.168.0.1:9050' and '192.168.0.2:9050'. + + SHOW TRASH ON ("192.168.0.1:9050","192.168.0.2:9050"); + +## keyword + SHOW, TRASH + diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW TRASH.md b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW TRASH.md new file mode 100644 index 00000000000000..38e397dafb9e30 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW TRASH.md @@ -0,0 +1,50 @@ +--- +{ + "title": "SHOW BACKENDS", + "language": "zh-CN" +} +--- + + + +# SHOW TRASH +## description + 该语句用于查看 backend 内的垃圾数据占用空间。 + 语法： + SHOW TRASH [ON ("BackendHost1:BackendHeartBeatPort1", "BackendHost2:BackendHeartBeatPort2", ...)]; + + 说明： + 1. BackendId 表示该be节点的ID。 + 2. Backend 格式为该节点的BackendHost:BackendHeartBeatPort。 + 3. TrashUsedCapacity 表示该节点垃圾数据占用空间。 + +## example + + 1. 查看所有be节点的垃圾数据占用空间。 + + SHOW TRASH; + + 2. 查看'192.168.0.1:9050'和'192.168.0.2:9050'的垃圾数据占用空间。 + + SHOW TRASH ON ("192.168.0.1:9050","192.168.0.2:9050"); + +## keyword + SHOW, TRASH + diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java index b841716660f775..913da50aff5e0b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java @@ -23,6 +23,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.proc.TrashProcNode; import org.apache.doris.qe.ShowResultSetMetaData; import org.apache.doris.qe.ConnectContext; import org.apache.doris.system.Backend; @@ -77,7 +78,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException { @Override public ShowResultSetMetaData getMetaData() { ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); - for (String title : TITLE_NAMES) { + for (String title : TrashProcNode.TITLE_NAMES) { builder.addColumn(new Column(title, ScalarType.createVarchar(30))); } return builder.build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/ProcService.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/ProcService.java index c93b137917fd42..c45116052cb40e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/ProcService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/ProcService.java @@ -45,6 +45,7 @@ private ProcService() { root.register("resources", Catalog.getCurrentCatalog().getResourceMgr().getProcNode()); root.register("load_error_hub", new LoadErrorHubProcNode(Catalog.getCurrentCatalog())); root.register("transactions", new TransDbProcDir()); + root.register("trash", new TrashProcNode()); root.register("monitor", new MonitorProcDir()); root.register("current_queries", new CurrentQueryStatisticsProcDir()); root.register("current_backend_instances", new CurrentQueryBackendInstanceProcDir()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java new file mode 100644 index 00000000000000..ea37ec9bba9c00 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.proc; + +import org.apache.doris.thrift.BackendService; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.system.Backend; +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.Pair; +import org.apache.doris.common.ClientPool; +import org.apache.doris.common.util.DebugUtil; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; + +/* + * Show trash + * SHOW PROC /trash/ + */ +public class TrashProcNode implements ProcNodeInterface { + private static final Logger LOG = LogManager.getLogger(TrashProcNode.class); + + public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() + .add("BackendId").add("Backend").add("TrashUsedCapacity").build(); + + private List backends = Lists.newArrayList(); + + public TrashProcNode() { + ImmutableMap backendsInfo = Catalog.getCurrentSystemInfo().getIdToBackend(); + for (Backend backend : backendsInfo.values()) { + this.backends.add(backend); + } + } + + @Override + public ProcResult fetchResult() { + BaseProcResult result = new BaseProcResult(); + result.setNames(TITLE_NAMES); + + List> infos = Lists.newArrayList(); + + getTrashInfo(backends, infos); + + for (List info : infos) { + result.addRow(info); + } + + return result; + } + + public static void getTrashInfo(List backends, List> infos) { + + for (Backend backend : backends) { + BackendService.Client client = null; + TNetworkAddress address = null; + Long trashUsedCapacityB = null; + boolean ok = false; + try { + long start = System.currentTimeMillis(); + address = new TNetworkAddress(backend.getHost(), backend.getBePort()); + client = ClientPool.backendPool.borrowObject(address); + trashUsedCapacityB = client.getTrashUsedCapacity(); + ok = true; + } catch (Exception e) { + LOG.warn("task exec error. backend[{}]", backend.getId(), e); + } finally { + if (ok) { + ClientPool.backendPool.returnObject(address, client); + } else { + ClientPool.backendPool.invalidateObject(address, client); + } + } + + List backendInfo = new ArrayList(); + backendInfo.add(String.valueOf(backend.getId())); + backendInfo.add(backend.getHost() + ":" + String.valueOf(backend.getHeartbeatPort())); + if (trashUsedCapacityB != null) { + Pair trashUsedCapacity = DebugUtil.getByteUint(trashUsedCapacityB); + backendInfo.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(trashUsedCapacity.first) + " " + trashUsedCapacity.second); + } else { + backendInfo.add("null"); + } + infos.add(backendInfo); + } + + } +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index fd0301f186c0f2..3088499fd98984 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -104,14 +104,12 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.CaseSensibility; import org.apache.doris.common.ConfigBase; -import org.apache.doris.common.ClientPool; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.PatternMatcher; -import org.apache.doris.common.Pair; import org.apache.doris.common.proc.BackendsProcDir; import org.apache.doris.common.proc.FrontendsProcNode; import org.apache.doris.common.proc.LoadProcDir; @@ -120,9 +118,9 @@ import org.apache.doris.common.proc.RollupProcDir; import org.apache.doris.common.proc.SchemaChangeProcDir; import org.apache.doris.common.proc.TabletsProcDir; +import org.apache.doris.common.proc.TrashProcNode; import org.apache.doris.common.profile.ProfileTreeNode; import org.apache.doris.common.profile.ProfileTreePrinter; -import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; @@ -143,8 +141,6 @@ import org.apache.doris.system.Backend; import org.apache.doris.system.SystemInfoService; import org.apache.doris.thrift.TUnit; -import org.apache.doris.thrift.BackendService; -import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.transaction.GlobalTransactionMgr; import com.google.common.base.Preconditions; @@ -1701,42 +1697,9 @@ private void handleShowRoles() { private void handleShowTrash() { ShowTrashStmt showStmt = (ShowTrashStmt) stmt; - List> results = Lists.newArrayList(); - - for (Backend backend : showStmt.getBackends()) { - BackendService.Client client = null; - TNetworkAddress address = null; - Long trashUsedCapacityB = null; - boolean ok = false; - try { - long start = System.currentTimeMillis(); - address = new TNetworkAddress(backend.getHost(), backend.getBePort()); - client = ClientPool.backendPool.borrowObject(address); - trashUsedCapacityB = client.getTrashUsedCapacity(); - LOG.debug("get trash used capacity from backend: {}, trashUsedCapacity: {}, cost {}", backend.getId(), - trashUsedCapacityB, System.currentTimeMillis() - start); - ok = true; - } catch (Exception e) { - LOG.warn("task exec error. backend[{}]", backend.getId(), e); - } finally { - if (ok) { - ClientPool.backendPool.returnObject(address, client); - } else { - ClientPool.backendPool.invalidateObject(address, client); - } - } - - if (trashUsedCapacityB != null) { - List backendInfo = new ArrayList(); - backendInfo.add(String.valueOf(backend.getId())); - backendInfo.add(backend.getHost() + ":" + String.valueOf(backend.getHeartbeatPort())); - Pair trashUsedCapacity = DebugUtil.getByteUint(trashUsedCapacityB); - backendInfo.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(trashUsedCapacity.first) + " " + trashUsedCapacity.second); - results.add(backendInfo); - } - } - - resultSet = new ShowResultSet(showStmt.getMetaData(), results); + List> infos = Lists.newArrayList(); + TrashProcNode.getTrashInfo(showStmt.getBackends(), infos); + resultSet = new ShowResultSet(showStmt.getMetaData(), infos); } private void handleAdminShowTabletStatus() throws AnalysisException { diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex index d2002d947d99f5..94a0251f74bb9f 100644 --- a/fe/fe-core/src/main/jflex/sql_scanner.flex +++ b/fe/fe-core/src/main/jflex/sql_scanner.flex @@ -103,7 +103,6 @@ import org.apache.doris.qe.SqlModeHelper; keywordMap.put("array", new Integer(SqlParserSymbols.KW_ARRAY)); keywordMap.put("backend", new Integer(SqlParserSymbols.KW_BACKEND)); keywordMap.put("backends", new Integer(SqlParserSymbols.KW_BACKENDS)); - keywordMap.put("trash", new Integer(SqlParserSymbols.KW_TRASH)); keywordMap.put("backup", new Integer(SqlParserSymbols.KW_BACKUP)); keywordMap.put("begin", new Integer(SqlParserSymbols.KW_BEGIN)); keywordMap.put("between", new Integer(SqlParserSymbols.KW_BETWEEN)); @@ -366,6 +365,7 @@ import org.apache.doris.qe.SqlModeHelper; keywordMap.put("tinyint", new Integer(SqlParserSymbols.KW_TINYINT)); keywordMap.put("to", new Integer(SqlParserSymbols.KW_TO)); keywordMap.put("transaction", new Integer(SqlParserSymbols.KW_TRANSACTION)); + keywordMap.put("trash", new Integer(SqlParserSymbols.KW_TRASH)); keywordMap.put("triggers", new Integer(SqlParserSymbols.KW_TRIGGERS)); keywordMap.put("trim", new Integer(SqlParserSymbols.KW_TRIM)); keywordMap.put("true", new Integer(SqlParserSymbols.KW_TRUE)); From b89cd13cf468bd5b96f47d2687ef8288f2b29498 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt <952130278@qq.com> Date: Thu, 15 Jul 2021 19:22:26 +0800 Subject: [PATCH 04/12] fix bug about document of show trash stmt --- docs/.vuepress/sidebar/en.js | 1 + docs/.vuepress/sidebar/zh-CN.js | 1 + .../sql-statements/Administration/SHOW TRASH.md | 6 +++--- .../sql-statements/Administration/SHOW TRASH.md | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/.vuepress/sidebar/en.js b/docs/.vuepress/sidebar/en.js index bf40bd33352d69..544711140c3259 100644 --- a/docs/.vuepress/sidebar/en.js +++ b/docs/.vuepress/sidebar/en.js @@ -427,6 +427,7 @@ module.exports = [ "SHOW MIGRATIONS", "SHOW PLUGINS", "SHOW TABLE STATUS", + "SHOW TRASH", "UNINSTALL PLUGIN", ], }, diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js index e9ef97be8fb541..6edfbf318ebd3a 100644 --- a/docs/.vuepress/sidebar/zh-CN.js +++ b/docs/.vuepress/sidebar/zh-CN.js @@ -432,6 +432,7 @@ module.exports = [ "SHOW MIGRATIONS", "SHOW PLUGINS", "SHOW TABLE STATUS", + "SHOW TRASH", "UNINSTALL PLUGIN", ], }, diff --git a/docs/en/sql-reference/sql-statements/Administration/SHOW TRASH.md b/docs/en/sql-reference/sql-statements/Administration/SHOW TRASH.md index b443100cac717d..b9fc5ffab9bff2 100644 --- a/docs/en/sql-reference/sql-statements/Administration/SHOW TRASH.md +++ b/docs/en/sql-reference/sql-statements/Administration/SHOW TRASH.md @@ -1,7 +1,7 @@ --- { - "title": "SHOW BACKENDS", - "language": "zh-CN" + "title": "SHOW TRASH", + "language": "en" } --- @@ -34,7 +34,7 @@ This statement is used to view trash used capacity on some backends. SHOW TRASH [ON ("BackendHost1:BackendHeartBeatPort1", "BackendHost2:BackendHeartBeatPort2", ...)]; Explain: - + 1. BackendId Indicates the ID of the backend node. 2. Backend The format is BackendHost:BackendHeartBeatPort of the node. 3. TrashUsedCapacity Indicates that the garbage data of the node occupies space. diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW TRASH.md b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW TRASH.md index 38e397dafb9e30..b3a0f3ad2bcf6d 100644 --- a/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW TRASH.md +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW TRASH.md @@ -1,6 +1,6 @@ --- { - "title": "SHOW BACKENDS", + "title": "SHOW TRASH", "language": "zh-CN" } --- From ec5c79eefb7ed15501e31dd0cf875016228f3ea7 Mon Sep 17 00:00:00 2001 From: Pxl <952130278@qq.com> Date: Thu, 15 Jul 2021 19:24:23 +0800 Subject: [PATCH 05/12] Update be/src/olap/storage_engine.cpp Co-authored-by: Zhengguo Yang <780531911@qq.com> --- be/src/olap/storage_engine.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index 1806b5984ba9cf..db357607041166 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -385,7 +385,7 @@ int64_t StorageEngine::get_file_or_directory_size(std::filesystem::path file_pat return std::filesystem::file_size(file_path); } int64_t sum_size = 0; - for (auto& it : std::filesystem::directory_iterator(file_path)) { + for (const auto& it : std::filesystem::directory_iterator(file_path)) { sum_size += get_file_or_directory_size(it.path()); } return sum_size; From 7a410fe62dc818f1198d240618f2fef5097840e1 Mon Sep 17 00:00:00 2001 From: Pxl <952130278@qq.com> Date: Thu, 15 Jul 2021 19:24:32 +0800 Subject: [PATCH 06/12] Update be/src/service/backend_service.cpp Co-authored-by: Zhengguo Yang <780531911@qq.com> --- be/src/service/backend_service.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/service/backend_service.cpp b/be/src/service/backend_service.cpp index 296e52305c616b..b7ef4b7f3b8de5 100644 --- a/be/src/service/backend_service.cpp +++ b/be/src/service/backend_service.cpp @@ -216,7 +216,7 @@ int64_t BackendService::get_trash_used_capacity() { std::vector data_dir_infos; StorageEngine::instance()->get_all_data_dir_info(&data_dir_infos, false /*do not update */); - for (auto& root_path_info : data_dir_infos) { + for (const auto& root_path_info : data_dir_infos) { std::string lhs_trash_path = root_path_info.path + TRASH_PREFIX; std::filesystem::path trash_path(lhs_trash_path); result += StorageEngine::instance()->get_file_or_directory_size(trash_path); From f84e758ae1b6e4b6f1601c2b8b0639961c81b223 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt <952130278@qq.com> Date: Thu, 15 Jul 2021 19:43:57 +0800 Subject: [PATCH 07/12] remove useless TITLENAME define --- .../src/main/java/org/apache/doris/analysis/ShowTrashStmt.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java index 913da50aff5e0b..4fd32f70ba9be2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java @@ -29,7 +29,6 @@ import org.apache.doris.system.Backend; import org.apache.doris.mysql.privilege.PrivPredicate; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -38,8 +37,6 @@ import java.util.Map; public class ShowTrashStmt extends ShowStmt { - public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder().add("BackendId") - .add("Backend").add("TrashUsedCapacity").build(); private List backends = Lists.newArrayList(); public ShowTrashStmt(List backends) { From 2440f2e5a4983d9aeab65a6699dac3aaf0b997d1 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt <952130278@qq.com> Date: Fri, 16 Jul 2021 11:01:31 +0800 Subject: [PATCH 08/12] change literal null to empty string --- .../main/java/org/apache/doris/common/proc/TrashProcNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java index ea37ec9bba9c00..d165794a76571d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java @@ -100,7 +100,7 @@ public static void getTrashInfo(List backends, List> infos Pair trashUsedCapacity = DebugUtil.getByteUint(trashUsedCapacityB); backendInfo.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(trashUsedCapacity.first) + " " + trashUsedCapacity.second); } else { - backendInfo.add("null"); + backendInfo.add(""); } infos.add(backendInfo); } From a2b971830a0114217407758af144f84b789b86fd Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt <952130278@qq.com> Date: Mon, 26 Jul 2021 15:57:41 +0800 Subject: [PATCH 09/12] add support for 'SHOW PROC /trash/backendId' --- .../doris/common/proc/TrashProcNode.java | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java index d165794a76571d..67f9d583467aee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java @@ -21,10 +21,12 @@ import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.system.Backend; import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Pair; import org.apache.doris.common.ClientPool; import org.apache.doris.common.util.DebugUtil; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -37,9 +39,10 @@ /* * Show trash - * SHOW PROC /trash/ + * SHOW PROC /trash + * SHOW PROC /trash/backendId */ -public class TrashProcNode implements ProcNodeInterface { +public class TrashProcNode implements ProcDirInterface { private static final Logger LOG = LogManager.getLogger(TrashProcNode.class); public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() @@ -53,6 +56,10 @@ public TrashProcNode() { this.backends.add(backend); } } + + public TrashProcNode(Backend backend) { + backends.add(backend); + } @Override public ProcResult fetchResult() { @@ -104,7 +111,32 @@ public static void getTrashInfo(List backends, List> infos } infos.add(backendInfo); } + } + + @Override + public boolean register(String name, ProcNodeInterface node) { + return false; + } + + @Override + public ProcNodeInterface lookup(String beIdStr) throws AnalysisException { + if (Strings.isNullOrEmpty(beIdStr)) { + throw new AnalysisException("Backend id is null"); + } + + long backendId = -1L; + try { + backendId = Long.parseLong(beIdStr); + } catch (NumberFormatException e) { + throw new AnalysisException("Invalid backend id format: " + beIdStr); + } + + Backend backend = Catalog.getCurrentSystemInfo().getBackend(backendId); + if (backend == null) { + throw new AnalysisException("Backend[" + backendId + "] does not exist."); + } + return new TrashProcNode(backend); } } From 0293c5300d07c2c38cc2c3c16b8d194fe95b7a84 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt <952130278@qq.com> Date: Tue, 27 Jul 2021 10:20:52 +0800 Subject: [PATCH 10/12] add limit about query format like 'trash/backendId/backendId' --- .../org/apache/doris/common/proc/TrashProcNode.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java index 67f9d583467aee..e5eb53e7046d76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java @@ -39,18 +39,21 @@ /* * Show trash - * SHOW PROC /trash - * SHOW PROC /trash/backendId + * SHOW PROC '/trash' + * SHOW PROC '/trash/backendId' */ public class TrashProcNode implements ProcDirInterface { private static final Logger LOG = LogManager.getLogger(TrashProcNode.class); + private boolean isNode; + public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("BackendId").add("Backend").add("TrashUsedCapacity").build(); private List backends = Lists.newArrayList(); public TrashProcNode() { + isNode = false; ImmutableMap backendsInfo = Catalog.getCurrentSystemInfo().getIdToBackend(); for (Backend backend : backendsInfo.values()) { this.backends.add(backend); @@ -58,6 +61,7 @@ public TrashProcNode() { } public TrashProcNode(Backend backend) { + isNode = true; backends.add(backend); } @@ -120,6 +124,10 @@ public boolean register(String name, ProcNodeInterface node) { @Override public ProcNodeInterface lookup(String beIdStr) throws AnalysisException { + if (isNode) { + throw new AnalysisException("Invalid query format"); + } + if (Strings.isNullOrEmpty(beIdStr)) { throw new AnalysisException("Backend id is null"); } From c484ec8570666288b6b9a6c6f509a132eec42a95 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt <952130278@qq.com> Date: Fri, 30 Jul 2021 15:14:33 +0800 Subject: [PATCH 11/12] 1. Removed the syntax for querying multiple backends. 2. Support for querying the specific disk information of a single backend. 3. Updated related interfaces and documents. --- be/src/service/backend_service.cpp | 20 +++ be/src/service/backend_service.h | 6 +- .../Administration/SHOW TRASH.md | 13 +- .../Administration/SHOW TRASH.md | 11 +- fe/fe-core/src/main/cup/sql_parser.cup | 6 +- .../doris/analysis/ShowTrashDiskStmt.java | 79 ++++++++++ .../apache/doris/analysis/ShowTrashStmt.java | 21 +-- .../apache/doris/common/proc/ProcService.java | 2 +- .../doris/common/proc/TrashProcDir.java | 138 ++++++++++++++++++ .../doris/common/proc/TrashProcNode.java | 125 ++++++---------- .../org/apache/doris/qe/ShowExecutor.java | 13 +- .../apache/doris/common/GenericPoolTest.java | 7 + .../doris/utframe/MockedBackendFactory.java | 6 + gensrc/thrift/BackendService.thrift | 8 + 14 files changed, 334 insertions(+), 121 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashDiskStmt.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcDir.java diff --git a/be/src/service/backend_service.cpp b/be/src/service/backend_service.cpp index b7ef4b7f3b8de5..87ad432d1c554d 100644 --- a/be/src/service/backend_service.cpp +++ b/be/src/service/backend_service.cpp @@ -224,6 +224,26 @@ int64_t BackendService::get_trash_used_capacity() { return result; } +void BackendService::get_disk_trash_used_capacity(std::vector& diskTrashInfos) { + std::vector data_dir_infos; + StorageEngine::instance()->get_all_data_dir_info(&data_dir_infos, false /*do not update */); + + for (const auto& root_path_info : data_dir_infos) { + TDiskTrashInfo diskTrashInfo; + + diskTrashInfo.__set_root_path(root_path_info.path); + + diskTrashInfo.__set_state(root_path_info.is_used ? "ONLINE" : "OFFLINE"); + + std::string lhs_trash_path = root_path_info.path + TRASH_PREFIX; + std::filesystem::path trash_path(lhs_trash_path); + diskTrashInfo.__set_trash_used_capacity( + StorageEngine::instance()->get_file_or_directory_size(trash_path)); + + diskTrashInfos.push_back(diskTrashInfo); + } +} + void BackendService::submit_routine_load_task(TStatus& t_status, const std::vector& tasks) { for (auto& task : tasks) { diff --git a/be/src/service/backend_service.h b/be/src/service/backend_service.h index 03f7279f0b85b2..1464fe46f6269b 100644 --- a/be/src/service/backend_service.h +++ b/be/src/service/backend_service.h @@ -61,6 +61,7 @@ class TQueryOptions; class TExportTaskRequest; class TExportStatusResult; class TStreamLoadRecordResult; +class TDiskTrashInfo; // This class just forward rpc for actual handler // make this class because we can bind multiple service on single point @@ -131,6 +132,8 @@ class BackendService : public BackendServiceIf { virtual int64_t get_trash_used_capacity() override; + virtual void get_disk_trash_used_capacity(std::vector& diskTrashInfos) override; + virtual void submit_routine_load_task(TStatus& t_status, const std::vector& tasks) override; @@ -143,7 +146,8 @@ class BackendService : public BackendServiceIf { // used for external service, close some context and release resource related with this context virtual void close_scanner(TScanCloseResult& result_, const TScanCloseParams& params); - virtual void get_stream_load_record(TStreamLoadRecordResult& result, const int64_t last_stream_record_time) override; + virtual void get_stream_load_record(TStreamLoadRecordResult& result, + const int64_t last_stream_record_time) override; private: Status start_plan_fragment_execution(const TExecPlanFragmentParams& exec_params); diff --git a/docs/en/sql-reference/sql-statements/Administration/SHOW TRASH.md b/docs/en/sql-reference/sql-statements/Administration/SHOW TRASH.md index b9fc5ffab9bff2..b1081d524fb5e1 100644 --- a/docs/en/sql-reference/sql-statements/Administration/SHOW TRASH.md +++ b/docs/en/sql-reference/sql-statements/Administration/SHOW TRASH.md @@ -31,23 +31,22 @@ This statement is used to view trash used capacity on some backends. Syntax: - SHOW TRASH [ON ("BackendHost1:BackendHeartBeatPort1", "BackendHost2:BackendHeartBeatPort2", ...)]; + SHOW TRASH [ON "BackendHost:BackendHeartBeatPort"]; Explain: - 1. BackendId Indicates the ID of the backend node. - 2. Backend The format is BackendHost:BackendHeartBeatPort of the node. - 3. TrashUsedCapacity Indicates that the garbage data of the node occupies space. + 1. Backend The format is BackendHost:BackendHeartBeatPort of the node. + 2. TrashUsedCapacity Indicates that the trash data of the node occupies space. ## example - 1. View the space occupied by garbage data of all be nodes. + 1. View the space occupied by trash data of all be nodes. SHOW TRASH; - 2. Check the space occupied by garbage data of '192.168.0.1:9050' and '192.168.0.2:9050'. + 2. Check the space occupied by trash data of '192.168.0.1:9050'(The specific disk information will be displayed). - SHOW TRASH ON ("192.168.0.1:9050","192.168.0.2:9050"); + SHOW TRASH ON "192.168.0.1:9050"; ## keyword SHOW, TRASH diff --git a/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW TRASH.md b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW TRASH.md index b3a0f3ad2bcf6d..5f2d3dfb7874ea 100644 --- a/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW TRASH.md +++ b/docs/zh-CN/sql-reference/sql-statements/Administration/SHOW TRASH.md @@ -28,12 +28,11 @@ under the License. ## description 该语句用于查看 backend 内的垃圾数据占用空间。 语法： - SHOW TRASH [ON ("BackendHost1:BackendHeartBeatPort1", "BackendHost2:BackendHeartBeatPort2", ...)]; + SHOW TRASH [ON BackendHost:BackendHeartBeatPort]; 说明： - 1. BackendId 表示该be节点的ID。 - 2. Backend 格式为该节点的BackendHost:BackendHeartBeatPort。 - 3. TrashUsedCapacity 表示该节点垃圾数据占用空间。 + 1. Backend 格式为该节点的BackendHost:BackendHeartBeatPort。 + 2. TrashUsedCapacity 表示该节点垃圾数据占用空间。 ## example @@ -41,9 +40,9 @@ under the License. SHOW TRASH; - 2. 查看'192.168.0.1:9050'和'192.168.0.2:9050'的垃圾数据占用空间。 + 2. 查看'192.168.0.1:9050'的垃圾数据占用空间(会显示具体磁盘信息)。 - SHOW TRASH ON ("192.168.0.1:9050","192.168.0.2:9050"); + SHOW TRASH ON "192.168.0.1:9050"; ## keyword SHOW, TRASH diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 3c13f363aba5f8..984ab65dc115ea 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -2554,13 +2554,13 @@ show_param ::= {: RESULT = new ShowBackendsStmt(); :} - | KW_TRASH KW_ON LPAREN string_list:backends RPAREN + | KW_TRASH KW_ON STRING_LITERAL:backend {: - RESULT = new ShowTrashStmt(backends); + RESULT = new ShowTrashDiskStmt(backend); :} | KW_TRASH {: - RESULT = new ShowTrashStmt(null); + RESULT = new ShowTrashStmt(); :} | KW_FRONTENDS {: diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashDiskStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashDiskStmt.java new file mode 100644 index 00000000000000..1a8fa94124fc22 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashDiskStmt.java @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.proc.TrashProcNode; +import org.apache.doris.qe.ShowResultSetMetaData; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.Backend; +import org.apache.doris.mysql.privilege.PrivPredicate; + +import com.google.common.collect.ImmutableMap; + +public class ShowTrashDiskStmt extends ShowStmt { + + private Backend backend; + + public ShowTrashDiskStmt(String backendQuery) { + ImmutableMap backendsInfo = Catalog.getCurrentSystemInfo().getIdToBackend(); + for (Backend backend : backendsInfo.values()) { + String backendStr = String.valueOf(backend.getHost()) + ":" + String.valueOf(backend.getHeartbeatPort()); + if (backendQuery.equals(backendStr)) { + this.backend = backend; + break; + } + } + } + + public Backend getBackend() { + return backend; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN) + && !Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), + PrivPredicate.OPERATOR)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN/OPERATOR"); + } + } + + @Override + public ShowResultSetMetaData getMetaData() { + ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); + for (String title : TrashProcNode.TITLE_NAMES) { + builder.addColumn(new Column(title, ScalarType.createVarchar(30))); + } + return builder.build(); + } + + @Override + public RedirectStatus getRedirectStatus() { + if (ConnectContext.get().getSessionVariable().getForwardToMaster()) { + return RedirectStatus.FORWARD_NO_SYNC; + } else { + return RedirectStatus.NO_FORWARD; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java index 4fd32f70ba9be2..fd86874ff8c2e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java @@ -32,30 +32,15 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import java.util.HashMap; import java.util.List; -import java.util.Map; public class ShowTrashStmt extends ShowStmt { private List backends = Lists.newArrayList(); - public ShowTrashStmt(List backends) { + public ShowTrashStmt() { ImmutableMap backendsInfo = Catalog.getCurrentSystemInfo().getIdToBackend(); - Map backendsID = new HashMap(); for (Backend backend : backendsInfo.values()) { - backendsID.put(String.valueOf(backend.getHost()) + ":" + String.valueOf(backend.getHeartbeatPort()), backend.getId()); - } - if (backends == null) { - for (Backend backend : backendsInfo.values()) { - this.backends.add(backend); - } - } else { - for (String backend : backends) { - if (backendsID.get(backend) != null) { - this.backends.add(backendsInfo.get(backendsID.get(backend))); - backendsID.remove(backend); // avoid repetition - } - } + this.backends.add(backend); } } @@ -67,7 +52,7 @@ public List getBackends() { public void analyze(Analyzer analyzer) throws AnalysisException { if (!Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN) && !Catalog.getCurrentCatalog().getAuth().checkGlobalPriv(ConnectContext.get(), - PrivPredicate.OPERATOR)) { + PrivPredicate.OPERATOR)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN/OPERATOR"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/ProcService.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/ProcService.java index c45116052cb40e..247e46e80b8c67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/ProcService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/ProcService.java @@ -45,7 +45,7 @@ private ProcService() { root.register("resources", Catalog.getCurrentCatalog().getResourceMgr().getProcNode()); root.register("load_error_hub", new LoadErrorHubProcNode(Catalog.getCurrentCatalog())); root.register("transactions", new TransDbProcDir()); - root.register("trash", new TrashProcNode()); + root.register("trash", new TrashProcDir()); root.register("monitor", new MonitorProcDir()); root.register("current_queries", new CurrentQueryStatisticsProcDir()); root.register("current_backend_instances", new CurrentQueryBackendInstanceProcDir()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcDir.java new file mode 100644 index 00000000000000..e1b0b859b3376e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcDir.java @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.proc; + +import org.apache.doris.thrift.BackendService; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.system.Backend; +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Pair; +import org.apache.doris.common.ClientPool; +import org.apache.doris.common.util.DebugUtil; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; + +/* + * Show trash + * SHOW PROC '/trash' + * SHOW PROC '/trash/backendId' + */ +public class TrashProcDir implements ProcDirInterface { + private static final Logger LOG = LogManager.getLogger(TrashProcNode.class); + + public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder().add("BackendId") + .add("Backend").add("TrashUsedCapacity").build(); + + private List backends = Lists.newArrayList(); + + public TrashProcDir() { + ImmutableMap backendsInfo = Catalog.getCurrentSystemInfo().getIdToBackend(); + for (Backend backend : backendsInfo.values()) { + this.backends.add(backend); + } + } + + @Override + public ProcResult fetchResult() { + BaseProcResult result = new BaseProcResult(); + result.setNames(TITLE_NAMES); + + List> infos = Lists.newArrayList(); + + getTrashInfo(backends, infos); + + for (List info : infos) { + result.addRow(info); + } + + return result; + } + + public static void getTrashInfo(List backends, List> infos) { + + for (Backend backend : backends) { + BackendService.Client client = null; + TNetworkAddress address = null; + Long trashUsedCapacityB = null; + boolean ok = false; + try { + long start = System.currentTimeMillis(); + address = new TNetworkAddress(backend.getHost(), backend.getBePort()); + client = ClientPool.backendPool.borrowObject(address); + trashUsedCapacityB = client.getTrashUsedCapacity(); + ok = true; + } catch (Exception e) { + LOG.warn("task exec error. backend[{}]", backend.getId(), e); + } finally { + if (ok) { + ClientPool.backendPool.returnObject(address, client); + } else { + ClientPool.backendPool.invalidateObject(address, client); + } + } + + List backendInfo = new ArrayList(); + backendInfo.add(String.valueOf(backend.getId())); + backendInfo.add(backend.getHost() + ":" + String.valueOf(backend.getHeartbeatPort())); + if (trashUsedCapacityB != null) { + Pair trashUsedCapacity = DebugUtil.getByteUint(trashUsedCapacityB); + backendInfo.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(trashUsedCapacity.first) + " " + + trashUsedCapacity.second); + } else { + backendInfo.add(""); + } + infos.add(backendInfo); + } + } + + @Override + public boolean register(String name, ProcNodeInterface node) { + return false; + } + + @Override + public ProcNodeInterface lookup(String backendHostAndPort) throws AnalysisException { + if (Strings.isNullOrEmpty(backendHostAndPort)) { + throw new AnalysisException("BackendHost:HeartBeatPort is null"); + } + String backendHost; + int backendHeartBeatPort; + try { + backendHost = backendHostAndPort.split(":")[0]; + backendHeartBeatPort = Integer.parseInt(backendHostAndPort.split(":")[1]); + } catch (NumberFormatException e) { + throw new AnalysisException("Invalid backend format: " + backendHostAndPort); + } + Backend backend = Catalog.getCurrentSystemInfo().getBackendWithHeartbeatPort(backendHost, backendHeartBeatPort); + if (backend == null) { + throw new AnalysisException("Backend[" + backendHostAndPort + "] does not exist."); + } + + return new TrashProcNode(backend); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java index e5eb53e7046d76..d01e3ccceca55b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TrashProcNode.java @@ -18,17 +18,15 @@ package org.apache.doris.common.proc; import org.apache.doris.thrift.BackendService; +import org.apache.doris.thrift.TDiskTrashInfo; import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.system.Backend; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Pair; import org.apache.doris.common.ClientPool; import org.apache.doris.common.util.DebugUtil; -import com.google.common.base.Strings; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import org.apache.logging.log4j.LogManager; @@ -37,42 +35,28 @@ import java.util.ArrayList; import java.util.List; -/* - * Show trash - * SHOW PROC '/trash' - * SHOW PROC '/trash/backendId' - */ -public class TrashProcNode implements ProcDirInterface { +public class TrashProcNode implements ProcNodeInterface { private static final Logger LOG = LogManager.getLogger(TrashProcNode.class); - private boolean isNode; - - public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() - .add("BackendId").add("Backend").add("TrashUsedCapacity").build(); - - private List backends = Lists.newArrayList(); - - public TrashProcNode() { - isNode = false; - ImmutableMap backendsInfo = Catalog.getCurrentSystemInfo().getIdToBackend(); - for (Backend backend : backendsInfo.values()) { - this.backends.add(backend); - } - } + public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder().add("RootPath") + .add("State").add("TrashUsedCapacity").build(); + + private Backend backend; public TrashProcNode(Backend backend) { - isNode = true; - backends.add(backend); + this.backend = backend; } - + @Override public ProcResult fetchResult() { + Preconditions.checkNotNull(backend); + BaseProcResult result = new BaseProcResult(); result.setNames(TITLE_NAMES); - + List> infos = Lists.newArrayList(); - getTrashInfo(backends, infos); + getTrashDiskInfo(backend, infos); for (List info : infos) { result.addRow(info); @@ -81,70 +65,43 @@ public ProcResult fetchResult() { return result; } - public static void getTrashInfo(List backends, List> infos) { - - for (Backend backend : backends) { - BackendService.Client client = null; - TNetworkAddress address = null; - Long trashUsedCapacityB = null; - boolean ok = false; - try { - long start = System.currentTimeMillis(); - address = new TNetworkAddress(backend.getHost(), backend.getBePort()); - client = ClientPool.backendPool.borrowObject(address); - trashUsedCapacityB = client.getTrashUsedCapacity(); - ok = true; - } catch (Exception e) { - LOG.warn("task exec error. backend[{}]", backend.getId(), e); - } finally { - if (ok) { - ClientPool.backendPool.returnObject(address, client); - } else { - ClientPool.backendPool.invalidateObject(address, client); - } - } + public static void getTrashDiskInfo(Backend backend, List> infos) { - List backendInfo = new ArrayList(); - backendInfo.add(String.valueOf(backend.getId())); - backendInfo.add(backend.getHost() + ":" + String.valueOf(backend.getHeartbeatPort())); - if (trashUsedCapacityB != null) { - Pair trashUsedCapacity = DebugUtil.getByteUint(trashUsedCapacityB); - backendInfo.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(trashUsedCapacity.first) + " " + trashUsedCapacity.second); + BackendService.Client client = null; + TNetworkAddress address = null; + boolean ok = false; + List diskTrashInfos = null; + try { + address = new TNetworkAddress(backend.getHost(), backend.getBePort()); + client = ClientPool.backendPool.borrowObject(address); + diskTrashInfos = client.getDiskTrashUsedCapacity(); + ok = true; + } catch (Exception e) { + LOG.warn("task exec error. backend[{}]", backend.getId(), e); + } finally { + if (ok) { + ClientPool.backendPool.returnObject(address, client); } else { - backendInfo.add(""); + ClientPool.backendPool.invalidateObject(address, client); } - infos.add(backendInfo); } - } - @Override - public boolean register(String name, ProcNodeInterface node) { - return false; - } - - @Override - public ProcNodeInterface lookup(String beIdStr) throws AnalysisException { - if (isNode) { - throw new AnalysisException("Invalid query format"); + if (diskTrashInfos == null) { + return; } + for (TDiskTrashInfo diskTrashInfo : diskTrashInfos) { + List diskInfo = new ArrayList(); - if (Strings.isNullOrEmpty(beIdStr)) { - throw new AnalysisException("Backend id is null"); - } + diskInfo.add(diskTrashInfo.getRootPath()); - long backendId = -1L; - try { - backendId = Long.parseLong(beIdStr); - } catch (NumberFormatException e) { - throw new AnalysisException("Invalid backend id format: " + beIdStr); - } + diskInfo.add(diskTrashInfo.getState()); - Backend backend = Catalog.getCurrentSystemInfo().getBackend(backendId); - if (backend == null) { - throw new AnalysisException("Backend[" + backendId + "] does not exist."); - } + long trashUsedCapacityB = diskTrashInfo.getTrashUsedCapacity(); + Pair trashUsedCapacity = DebugUtil.getByteUint(trashUsedCapacityB); + diskInfo.add( + DebugUtil.DECIMAL_FORMAT_SCALE_3.format(trashUsedCapacity.first) + " " + trashUsedCapacity.second); - return new TrashProcNode(backend); + infos.add(diskInfo); + } } } - diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 3088499fd98984..07025ac831063a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -72,6 +72,7 @@ import org.apache.doris.analysis.ShowTabletStmt; import org.apache.doris.analysis.ShowTransactionStmt; import org.apache.doris.analysis.ShowTrashStmt; +import org.apache.doris.analysis.ShowTrashDiskStmt; import org.apache.doris.analysis.ShowUserPropertyStmt; import org.apache.doris.analysis.ShowVariablesStmt; import org.apache.doris.analysis.ShowViewStmt; @@ -118,6 +119,7 @@ import org.apache.doris.common.proc.RollupProcDir; import org.apache.doris.common.proc.SchemaChangeProcDir; import org.apache.doris.common.proc.TabletsProcDir; +import org.apache.doris.common.proc.TrashProcDir; import org.apache.doris.common.proc.TrashProcNode; import org.apache.doris.common.profile.ProfileTreeNode; import org.apache.doris.common.profile.ProfileTreePrinter; @@ -276,6 +278,8 @@ public ShowResultSet execute() throws AnalysisException { handleShowRoles(); } else if (stmt instanceof ShowTrashStmt) { handleShowTrash(); + } else if (stmt instanceof ShowTrashDiskStmt) { + handleShowTrashDisk(); } else if (stmt instanceof AdminShowReplicaStatusStmt) { handleAdminShowTabletStatus(); } else if (stmt instanceof AdminShowReplicaDistributionStmt) { @@ -1698,7 +1702,14 @@ private void handleShowRoles() { private void handleShowTrash() { ShowTrashStmt showStmt = (ShowTrashStmt) stmt; List> infos = Lists.newArrayList(); - TrashProcNode.getTrashInfo(showStmt.getBackends(), infos); + TrashProcDir.getTrashInfo(showStmt.getBackends(), infos); + resultSet = new ShowResultSet(showStmt.getMetaData(), infos); + } + + private void handleShowTrashDisk() { + ShowTrashDiskStmt showStmt = (ShowTrashDiskStmt) stmt; + List> infos = Lists.newArrayList(); + TrashProcNode.getTrashDiskInfo(showStmt.getBackend(), infos); resultSet = new ShowResultSet(showStmt.getMetaData(), infos); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java index 188d0b8e9e93ae..e134e812b9377e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java @@ -25,6 +25,7 @@ import org.apache.doris.thrift.TCancelPlanFragmentParams; import org.apache.doris.thrift.TCancelPlanFragmentResult; import org.apache.doris.thrift.TDeleteEtlFilesRequest; +import org.apache.doris.thrift.TDiskTrashInfo; import org.apache.doris.thrift.TExecPlanFragmentParams; import org.apache.doris.thrift.TExecPlanFragmentResult; import org.apache.doris.thrift.TExportStatusResult; @@ -199,6 +200,12 @@ public long getTrashUsedCapacity() throws TException { return 0l; } + @Override + public List getDiskTrashUsedCapacity() throws TException { + // TODO Auto-generated method stub + return null; + } + @Override public TTabletStatResult getTabletStat() throws TException { // TODO Auto-generated method stub diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java index 7792f98391d292..f14d59b3e4adbe 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java @@ -33,6 +33,7 @@ import org.apache.doris.thrift.TCancelPlanFragmentParams; import org.apache.doris.thrift.TCancelPlanFragmentResult; import org.apache.doris.thrift.TDeleteEtlFilesRequest; +import org.apache.doris.thrift.TDiskTrashInfo; import org.apache.doris.thrift.TEtlState; import org.apache.doris.thrift.TExecPlanFragmentParams; import org.apache.doris.thrift.TExecPlanFragmentResult; @@ -249,6 +250,11 @@ public long getTrashUsedCapacity() throws TException { return 0l; } + @Override + public List getDiskTrashUsedCapacity() throws TException { + return null; + } + @Override public TTabletStatResult getTabletStat() throws TException { return new TTabletStatResult(Maps.newHashMap()); diff --git a/gensrc/thrift/BackendService.thrift b/gensrc/thrift/BackendService.thrift index ffbd26667e8134..1288a2bed21d56 100644 --- a/gensrc/thrift/BackendService.thrift +++ b/gensrc/thrift/BackendService.thrift @@ -105,6 +105,12 @@ struct TStreamLoadRecordResult { 1: required map stream_load_record } +struct TDiskTrashInfo { + 1: required string root_path + 2: required string state + 3: required i64 trash_used_capacity +} + service BackendService { // Called by coord to start asynchronous execution of plan fragment in backend. // Returns as soon as all incoming data streams have been set up. @@ -149,6 +155,8 @@ service BackendService { TTabletStatResult get_tablet_stat(); i64 get_trash_used_capacity(); + + list get_disk_trash_used_capacity(); Status.TStatus submit_routine_load_task(1:list tasks); From c3f7c7e989b8dfc659577ee7622b2acac9f8e62b Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt <952130278@qq.com> Date: Thu, 5 Aug 2021 14:54:01 +0800 Subject: [PATCH 12/12] change Redirect to 'NO_FORWARD' at ShowTrashStmt/ShowTrashDiskStmt --- .../java/org/apache/doris/analysis/ShowTrashDiskStmt.java | 6 +----- .../main/java/org/apache/doris/analysis/ShowTrashStmt.java | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashDiskStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashDiskStmt.java index 1a8fa94124fc22..3cca857ed7ad6a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashDiskStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashDiskStmt.java @@ -70,10 +70,6 @@ public ShowResultSetMetaData getMetaData() { @Override public RedirectStatus getRedirectStatus() { - if (ConnectContext.get().getSessionVariable().getForwardToMaster()) { - return RedirectStatus.FORWARD_NO_SYNC; - } else { - return RedirectStatus.NO_FORWARD; - } + return RedirectStatus.NO_FORWARD; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java index fd86874ff8c2e5..a2c744428a2b70 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTrashStmt.java @@ -68,10 +68,6 @@ public ShowResultSetMetaData getMetaData() { @Override public RedirectStatus getRedirectStatus() { - if (ConnectContext.get().getSessionVariable().getForwardToMaster()) { - return RedirectStatus.FORWARD_NO_SYNC; - } else { - return RedirectStatus.NO_FORWARD; - } + return RedirectStatus.NO_FORWARD; } }