From 1a5fcfbe0890173bec015bcb2cd78eac1c532965 Mon Sep 17 00:00:00 2001 From: yangzhg Date: Mon, 1 Jun 2020 18:54:00 +0800 Subject: [PATCH 1/7] xxx --- fe/src/main/java/org/apache/doris/catalog/Replica.java | 4 ++++ .../java/org/apache/doris/common/proc/TabletsProcDir.java | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/fe/src/main/java/org/apache/doris/catalog/Replica.java b/fe/src/main/java/org/apache/doris/catalog/Replica.java index c9cab33710142a..ef16cef1e1d05a 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Replica.java +++ b/fe/src/main/java/org/apache/doris/catalog/Replica.java @@ -225,6 +225,10 @@ public long getPathHash() { return pathHash; } + public String getMetaUrl(long tabletId) { + return String.format("%s/api/meta/header/%d/%d", this.backendId, tabletId, this.schemaHash); + } + public void setPathHash(long pathHash) { this.pathHash = pathHash; } diff --git a/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java b/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java index a19a498f606534..c969e7ce8b1a35 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java +++ b/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java @@ -45,7 +45,7 @@ public class TabletsProcDir implements ProcDirInterface { .add("LstFailedVersion").add("LstFailedVersionHash").add("LstFailedTime") .add("DataSize").add("RowCount").add("State") .add("LstConsistencyCheckTime").add("CheckVersion").add("CheckVersionHash") - .add("VersionCount").add("PathHash") + .add("VersionCount").add("PathHash").add("MetaUrl") .build(); private Database db; @@ -88,6 +88,7 @@ public List> fetchComparableResult(long version, long backendId tabletInfo.add(-1); // check version hash tabletInfo.add(-1); // version count tabletInfo.add(-1); // path hash + tabletInfo.add("N/A"); // meta url tabletInfos.add(tabletInfo); } else { @@ -119,6 +120,7 @@ public List> fetchComparableResult(long version, long backendId tabletInfo.add(tablet.getCheckedVersionHash()); tabletInfo.add(replica.getVersionCount()); tabletInfo.add(replica.getPathHash()); + tabletInfo.add(replica.getMetaUrl(tabletId)); tabletInfos.add(tabletInfo); } From 65cb21024c46f879ffef9bacb0180d3ba3ced660 Mon Sep 17 00:00:00 2001 From: yangzhg Date: Mon, 1 Jun 2020 19:27:37 +0800 Subject: [PATCH 2/7] xxx --- fe/src/main/java/org/apache/doris/catalog/Replica.java | 4 ---- .../org/apache/doris/common/proc/TabletsProcDir.java | 10 +++++++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/catalog/Replica.java b/fe/src/main/java/org/apache/doris/catalog/Replica.java index ef16cef1e1d05a..c9cab33710142a 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Replica.java +++ b/fe/src/main/java/org/apache/doris/catalog/Replica.java @@ -225,10 +225,6 @@ public long getPathHash() { return pathHash; } - public String getMetaUrl(long tabletId) { - return String.format("%s/api/meta/header/%d/%d", this.backendId, tabletId, this.schemaHash); - } - public void setPathHash(long pathHash) { this.pathHash = pathHash; } diff --git a/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java b/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java index c969e7ce8b1a35..e8ebf9ee547b5d 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java +++ b/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java @@ -26,9 +26,11 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.system.Backend; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import java.util.ArrayList; import java.util.Collections; @@ -59,6 +61,7 @@ public TabletsProcDir(Database db, MaterializedIndex index) { public List> fetchComparableResult(long version, long backendId, Replica.ReplicaState state) { Preconditions.checkNotNull(db); Preconditions.checkNotNull(index); + ImmutableMap backendMap = Catalog.getCurrentSystemInfo().getIdToBackend(); List> tabletInfos = new ArrayList>(); db.readLock(); @@ -120,7 +123,12 @@ public List> fetchComparableResult(long version, long backendId tabletInfo.add(tablet.getCheckedVersionHash()); tabletInfo.add(replica.getVersionCount()); tabletInfo.add(replica.getPathHash()); - tabletInfo.add(replica.getMetaUrl(tabletId)); + String metaUrl = String.format("http://%s:%d/api/meta/header/%d/%d", + backendMap.get(backendId).getHost(), + backendMap.get(backendId).getHttpPort(), + tabletId, + replica.getSchemaHash()); + tabletInfo.add(metaUrl); tabletInfos.add(tabletInfo); } From 2c1b7afd334ac35da8136d299e4f7ed52d9ca775 Mon Sep 17 00:00:00 2001 From: yangzhg Date: Mon, 1 Jun 2020 19:33:48 +0800 Subject: [PATCH 3/7] xxx --- .../java/org/apache/doris/common/proc/TabletsProcDir.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java b/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java index e8ebf9ee547b5d..acc1338c642263 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java +++ b/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java @@ -124,8 +124,8 @@ public List> fetchComparableResult(long version, long backendId tabletInfo.add(replica.getVersionCount()); tabletInfo.add(replica.getPathHash()); String metaUrl = String.format("http://%s:%d/api/meta/header/%d/%d", - backendMap.get(backendId).getHost(), - backendMap.get(backendId).getHttpPort(), + backendMap.get(replica.getBackendId()).getHost(), + backendMap.get(replica.getBackendId()).getHttpPort(), tabletId, replica.getSchemaHash()); tabletInfo.add(metaUrl); From a47331af373550477b549e6b500a8a5f50ff63e5 Mon Sep 17 00:00:00 2001 From: yangzhg Date: Mon, 1 Jun 2020 20:37:09 +0800 Subject: [PATCH 4/7] xxx --- .../java/org/apache/doris/http/action/SystemAction.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fe/src/main/java/org/apache/doris/http/action/SystemAction.java b/fe/src/main/java/org/apache/doris/http/action/SystemAction.java index 5a18667878a9e9..a0ca3f36575179 100644 --- a/fe/src/main/java/org/apache/doris/http/action/SystemAction.java +++ b/fe/src/main/java/org/apache/doris/http/action/SystemAction.java @@ -17,6 +17,7 @@ package org.apache.doris.http.action; +import org.apache.commons.validator.routines.UrlValidator; import org.apache.doris.analysis.RedirectStatus; import org.apache.doris.catalog.Catalog; import org.apache.doris.common.AnalysisException; @@ -132,6 +133,7 @@ private void appendSystemInfo(StringBuilder buffer, String procPath, String path } private void appendSystemTableBody(StringBuilder buffer, List> rows, boolean isDir, String path) { + UrlValidator validator = new UrlValidator(); for ( List strList : rows) { buffer.append(""); int columnIndex = 1; @@ -142,6 +144,11 @@ private void appendSystemTableBody(StringBuilder buffer, List> rows buffer.append(""); buffer.append(str); buffer.append(""); + } else if (validator.isValid(str)) { + String escapeStr = str.replace("%", "%25"); + buffer.append(""); + buffer.append("URL"); + buffer.append(""); } else { buffer.append(str.replaceAll("\\n", "
")); } From 29ad9285051572a526944d364a1d2b9613dfc8fc Mon Sep 17 00:00:00 2001 From: yangzhg Date: Mon, 1 Jun 2020 20:58:01 +0800 Subject: [PATCH 5/7] add meta data header link to doris web list --- fe/src/main/java/org/apache/doris/http/action/SystemAction.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/src/main/java/org/apache/doris/http/action/SystemAction.java b/fe/src/main/java/org/apache/doris/http/action/SystemAction.java index a0ca3f36575179..bfa403f20025da 100644 --- a/fe/src/main/java/org/apache/doris/http/action/SystemAction.java +++ b/fe/src/main/java/org/apache/doris/http/action/SystemAction.java @@ -145,7 +145,6 @@ private void appendSystemTableBody(StringBuilder buffer, List> rows buffer.append(str); buffer.append(""); } else if (validator.isValid(str)) { - String escapeStr = str.replace("%", "%25"); buffer.append(""); buffer.append("URL"); buffer.append(""); From 70177280c0d35dc8992b5c24b54e5ca643092388 Mon Sep 17 00:00:00 2001 From: yangzhg Date: Tue, 2 Jun 2020 12:27:07 +0800 Subject: [PATCH 6/7] add compaction status --- .../operation/tablet-repair-and-balance.md | 16 ++++++++-------- .../operation/tablet-repair-and-balance.md | 14 +++++++------- .../apache/doris/common/proc/TabletsProcDir.java | 10 ++++++++-- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/docs/en/administrator-guide/operation/tablet-repair-and-balance.md b/docs/en/administrator-guide/operation/tablet-repair-and-balance.md index 802228bb978f1d..2ea37676485b16 100644 --- a/docs/en/administrator-guide/operation/tablet-repair-and-balance.md +++ b/docs/en/administrator-guide/operation/tablet-repair-and-balance.md @@ -303,14 +303,14 @@ Duplicate status view mainly looks at the status of the duplicate, as well as th `SHOW TABLET FROM tbl1;` ``` - +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ - | TabletId | ReplicaId | BackendId | Version | VersionHash | LstSuccessVersion | LstSuccessVersionHash | LstFailedVersion | LstFailedVersionHash | LstFailedTime | DataSize | RowCount | State | LstConsistencyCheckTime | CheckVersion | CheckVersionHash | VersionCount | PathHash | - +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ - | 29502429 | 29502432 | 10006 | 2 | 0 | 2 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -5822326203532286804 | - | 29502429 | 36885996 | 10002 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -1441285706148429853 | - | 29502429 | 48100551 | 10007 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -4784691547051455525 | - +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ - ``` + +----------+-----------+-----------+------------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------------+--------------+----------------------+----------------------+----------------------+ + | TabletId | ReplicaId | BackendId | SchemaHash | Version | VersionHash | LstSuccessVersion | LstSuccessVersionHash | LstFailedVersion | LstFailedVersionHash | LstFailedTime | DataSize | RowCount | State | LstConsistencyCheckTime | CheckVersion | CheckVersionHash | VersionCount | PathHash | MetaUrl | CompactionStatus | + +----------+-----------+-----------+------------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------------+--------------+----------------------+----------------------+----------------------+ + | 29502429 | 29502432 | 10006 | 1421156361 | 2 | 0 | 2 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -5822326203532286804 | url | url | + | 29502429 | 36885996 | 10002 | 1421156361 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -1441285706148429853 | url | url | + | 29502429 | 48100551 | 10007 | 1421156361 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -4784691547051455525 | url | url | + +----------+-----------+-----------+------------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------------+--------------+----------------------+----------------------+----------------------+ + ``` The figure above shows some additional information, including copy size, number of rows, number of versions, where the data path is located. diff --git a/docs/zh-CN/administrator-guide/operation/tablet-repair-and-balance.md b/docs/zh-CN/administrator-guide/operation/tablet-repair-and-balance.md index 87b3ddc4da9c80..c991bb0610f288 100644 --- a/docs/zh-CN/administrator-guide/operation/tablet-repair-and-balance.md +++ b/docs/zh-CN/administrator-guide/operation/tablet-repair-and-balance.md @@ -304,13 +304,13 @@ TabletScheduler 在每轮调度时，都会通过 LoadBalancer 来选择一定 `SHOW TABLET FROM tbl1;` ``` - +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ - | TabletId | ReplicaId | BackendId | Version | VersionHash | LstSuccessVersion | LstSuccessVersionHash | LstFailedVersion | LstFailedVersionHash | LstFailedTime | DataSize | RowCount | State | LstConsistencyCheckTime | CheckVersion | CheckVersionHash | VersionCount | PathHash | - +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ - | 29502429 | 29502432 | 10006 | 2 | 0 | 2 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -5822326203532286804 | - | 29502429 | 36885996 | 10002 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -1441285706148429853 | - | 29502429 | 48100551 | 10007 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -4784691547051455525 | - +----------+-----------+-----------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------- -+--------------+----------------------+ + +----------+-----------+-----------+------------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------------+--------------+----------------------+----------------------+----------------------+ + | TabletId | ReplicaId | BackendId | SchemaHash | Version | VersionHash | LstSuccessVersion | LstSuccessVersionHash | LstFailedVersion | LstFailedVersionHash | LstFailedTime | DataSize | RowCount | State | LstConsistencyCheckTime | CheckVersion | CheckVersionHash | VersionCount | PathHash | MetaUrl | CompactionStatus | + +----------+-----------+-----------+------------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------------+--------------+----------------------+----------------------+----------------------+ + | 29502429 | 29502432 | 10006 | 1421156361 | 2 | 0 | 2 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -5822326203532286804 | url | url | + | 29502429 | 36885996 | 10002 | 1421156361 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -1441285706148429853 | url | url | + | 29502429 | 48100551 | 10007 | 1421156361 | 2 | 0 | -1 | 0 | -1 | 0 | N/A | 784 | 0 | NORMAL | N/A | -1 | -1 | 2 | -4784691547051455525 | url | url | + +----------+-----------+-----------+------------+---------+-------------+-------------------+-----------------------+------------------+----------------------+---------------+----------+----------+--------+-------------------------+--------------+----------------------+--------------+----------------------+----------------------+----------------------+ ``` 上图展示了包括副本大小、行数、版本数量、所在数据路径等一些额外的信息。 diff --git a/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java b/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java index acc1338c642263..7df618242a6fed 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java +++ b/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java @@ -47,7 +47,7 @@ public class TabletsProcDir implements ProcDirInterface { .add("LstFailedVersion").add("LstFailedVersionHash").add("LstFailedTime") .add("DataSize").add("RowCount").add("State") .add("LstConsistencyCheckTime").add("CheckVersion").add("CheckVersionHash") - .add("VersionCount").add("PathHash").add("MetaUrl") + .add("VersionCount").add("PathHash").add("MetaUrl").add("CompactionStatus") .build(); private Database db; @@ -92,6 +92,7 @@ public List> fetchComparableResult(long version, long backendId tabletInfo.add(-1); // version count tabletInfo.add(-1); // path hash tabletInfo.add("N/A"); // meta url + tabletInfo.add("N/A"); // compaction status tabletInfos.add(tabletInfo); } else { @@ -129,7 +130,12 @@ public List> fetchComparableResult(long version, long backendId tabletId, replica.getSchemaHash()); tabletInfo.add(metaUrl); - + String compactionUrl = String.format( + "http://%s:%d/api/compaction/show?tablet_id=%d&schema_hash=%d", + backendMap.get(replica.getBackendId()).getHost(), + backendMap.get(replica.getBackendId()).getHttpPort(), + tabletId, + replica.getSchemaHash()); tabletInfos.add(tabletInfo); } } From 9f801446bcc5ef407a57223e48f45eee56729412 Mon Sep 17 00:00:00 2001 From: yangzhg Date: Tue, 2 Jun 2020 12:34:31 +0800 Subject: [PATCH 7/7] add compaction status --- .../main/java/org/apache/doris/common/proc/TabletsProcDir.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java b/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java index 7df618242a6fed..3c4bd74862326c 100644 --- a/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java +++ b/fe/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java @@ -136,6 +136,7 @@ public List> fetchComparableResult(long version, long backendId backendMap.get(replica.getBackendId()).getHttpPort(), tabletId, replica.getSchemaHash()); + tabletInfo.add(compactionUrl); tabletInfos.add(tabletInfo); } }