From a673ab7d6ae696a15795d636c1ff7661b328671d Mon Sep 17 00:00:00 2001 From: Dino Zhang Date: Thu, 22 Jul 2021 18:15:40 +0800 Subject: [PATCH 1/2] fix show load like match --- .../main/java/org/apache/doris/load/Load.java | 19 ++++++++++++------- .../apache/doris/load/loadv2/LoadManager.java | 13 +++++++++---- .../doris/analysis/ShowLoadStmtTest.java | 6 ++++-- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java index aefe73e2e54704..e7f96260c0b5d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java @@ -65,6 +65,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.CaseSensibility; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; @@ -75,6 +76,7 @@ import org.apache.doris.common.LoadException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.Pair; +import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.UserException; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.MetaLockUtils; @@ -1591,7 +1593,7 @@ private boolean checkMultiLabelUsed(long dbId, String label, long timestamp) thr return false; } - public boolean isLabelExist(String dbName, String labelValue, boolean isAccurateMatch) throws DdlException { + public boolean isLabelExist(String dbName, String labelValue, boolean isAccurateMatch) throws DdlException, AnalysisException { // get load job and check state Database db = Catalog.getCurrentCatalog().getDb(dbName); if (db == null) { @@ -1610,7 +1612,8 @@ public boolean isLabelExist(String dbName, String labelValue, boolean isAccurate } } else { for (Map.Entry> entry : labelToLoadJobs.entrySet()) { - if (entry.getKey().contains(labelValue)) { + PatternMatcher matcher = PatternMatcher.createMysqlPattern(labelValue, CaseSensibility.LABEL.getCaseSensibility()); + if (matcher.match(entry.getKey())) { loadJobs.addAll(entry.getValue()); } } @@ -1627,7 +1630,7 @@ public boolean isLabelExist(String dbName, String labelValue, boolean isAccurate } } - public boolean cancelLoadJob(CancelLoadStmt stmt, boolean isAccurateMatch) throws DdlException { + public boolean cancelLoadJob(CancelLoadStmt stmt, boolean isAccurateMatch) throws DdlException, AnalysisException { // get params String dbName = stmt.getDbName(); String label = stmt.getLabel(); @@ -1654,8 +1657,9 @@ public boolean cancelLoadJob(CancelLoadStmt stmt, boolean isAccurateMatch) throw } } else { for (Map.Entry> entry : labelToLoadJobs.entrySet()) { - if (entry.getKey().contains(label)) { - matchLoadJobs.addAll(entry.getValue()); + PatternMatcher matcher = PatternMatcher.createMysqlPattern(label, CaseSensibility.LABEL.getCaseSensibility()); + if (matcher.match(entry.getKey())) { + loadJobs.addAll(entry.getValue()); } } } @@ -1904,7 +1908,7 @@ public LoadJob getLoadJob(long jobId) { } public LinkedList> getLoadJobInfosByDb(long dbId, String dbName, String labelValue, - boolean accurateMatch, Set states) { + boolean accurateMatch, Set states) throws AnalysisException { LinkedList> loadJobInfos = new LinkedList>(); readLock(); try { @@ -1926,7 +1930,8 @@ public LinkedList> getLoadJobInfosByDb(long dbId, String dbName continue; } } else { - if (!label.contains(labelValue)) { + PatternMatcher matcher = PatternMatcher.createMysqlPattern(labelValue, CaseSensibility.LABEL.getCaseSensibility()); + if (!matcher.match(label)) { continue; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java index a223ae42be1886..6e1c486730c7c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java @@ -23,12 +23,15 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Table; import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.CaseSensibility; import org.apache.doris.common.Config; import org.apache.doris.common.DataQualityException; import org.apache.doris.common.DdlException; import org.apache.doris.common.DuplicatedRequestException; import org.apache.doris.common.LabelAlreadyUsedException; import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.LogBuilder; @@ -294,7 +297,7 @@ public void recordFinishedLoadJob(String label, String dbName, long tableId, Etl Catalog.getCurrentCatalog().getEditLog().logCreateLoadJob(loadJob); } - public void cancelLoadJob(CancelLoadStmt stmt, boolean isAccurateMatch) throws DdlException { + public void cancelLoadJob(CancelLoadStmt stmt, boolean isAccurateMatch) throws DdlException, AnalysisException { Database db = Catalog.getCurrentCatalog().getDb(stmt.getDbName()); if (db == null) { throw new DdlException("Db does not exist. name: " + stmt.getDbName()); @@ -317,7 +320,8 @@ public void cancelLoadJob(CancelLoadStmt stmt, boolean isAccurateMatch) throws D } } else { for (Map.Entry> entry : labelToLoadJobs.entrySet()) { - if (entry.getKey().contains(stmt.getLabel())) { + PatternMatcher matcher = PatternMatcher.createMysqlPattern(stmt.getLabel(), CaseSensibility.LABEL.getCaseSensibility()); + if (matcher.match(entry.getKey())) { matchLoadJobs.addAll(entry.getValue()); } } @@ -503,7 +507,7 @@ public void processLoadingStateJobs() { * The result is unordered. */ public List> getLoadJobInfosByDb(long dbId, String labelValue, - boolean accurateMatch, Set statesValue) { + boolean accurateMatch, Set statesValue) throws AnalysisException { LinkedList> loadJobInfos = new LinkedList>(); if (!dbIdToLabelToLoadJobs.containsKey(dbId)) { return loadJobInfos; @@ -539,7 +543,8 @@ public List> getLoadJobInfosByDb(long dbId, String labelValue, } else { // non-accurate match for (Map.Entry> entry : labelToLoadJobs.entrySet()) { - if (entry.getKey().contains(labelValue)) { + PatternMatcher matcher = PatternMatcher.createMysqlPattern(labelValue, CaseSensibility.LABEL.getCaseSensibility()); + if (matcher.match(entry.getKey())) { loadJobList.addAll(entry.getValue()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowLoadStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowLoadStmtTest.java index a45687575fbc21..b9027f360be5ef 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowLoadStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowLoadStmtTest.java @@ -108,10 +108,12 @@ public void testWhere() throws UserException, AnalysisException { stmt.analyze(analyzer); Assert.assertEquals("SHOW LOAD FROM `testCluster:testDb` WHERE `label` = \'abc\' LIMIT 10", stmt.toString()); + StringLiteral stringLiteralLike = new StringLiteral("ab%"); LikePredicate likePredicate = new LikePredicate(org.apache.doris.analysis.LikePredicate.Operator.LIKE, - slotRef, stringLiteral); + slotRef, stringLiteralLike); + stmt = new ShowLoadStmt(null, likePredicate, null, new LimitElement(10)); stmt.analyze(analyzer); - Assert.assertEquals("SHOW LOAD FROM `testCluster:testDb` WHERE `label` LIKE \'abc\' LIMIT 10", stmt.toString()); + Assert.assertEquals("SHOW LOAD FROM `testCluster:testDb` WHERE `label` LIKE \'ab%\' LIMIT 10", stmt.toString()); } } From cf08b130be6ff54a8b6c0c7088ab71f1f688e0a1 Mon Sep 17 00:00:00 2001 From: Dino Zhang Date: Wed, 28 Jul 2021 16:19:54 +0800 Subject: [PATCH 2/2] Compatible with historical issues --- .../java/org/apache/doris/analysis/CancelLoadStmt.java | 3 +++ .../java/org/apache/doris/analysis/ShowLoadStmt.java | 3 +++ .../src/main/java/org/apache/doris/load/Load.java | 10 +++++++--- .../java/org/apache/doris/load/loadv2/LoadManager.java | 4 ++-- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelLoadStmt.java index 4fda341e115cb1..545ed9d18768e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelLoadStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelLoadStmt.java @@ -115,6 +115,9 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { valid = false; break; } + if (!isAccurateMatch && !label.contains("%")) { + label = "%" + label + "%"; + } } while (false); if (!valid) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadStmt.java index 523d10a58519c8..610477e4dfe2cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadStmt.java @@ -219,6 +219,9 @@ private void analyzeSubPredicate(Expr subExpr) throws AnalysisException { break CHECK; } + if (!isAccurateMatch && !value.contains("%")) { + value = "%" + value + "%"; + } if (hasLabel) { labelValue = value; } else if (hasState) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java index e7f96260c0b5d9..bc78e44ba610da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java @@ -1611,8 +1611,8 @@ public boolean isLabelExist(String dbName, String labelValue, boolean isAccurate loadJobs.addAll(labelToLoadJobs.get(labelValue)); } } else { + PatternMatcher matcher = PatternMatcher.createMysqlPattern(labelValue, CaseSensibility.LABEL.getCaseSensibility()); for (Map.Entry> entry : labelToLoadJobs.entrySet()) { - PatternMatcher matcher = PatternMatcher.createMysqlPattern(labelValue, CaseSensibility.LABEL.getCaseSensibility()); if (matcher.match(entry.getKey())) { loadJobs.addAll(entry.getValue()); } @@ -1656,8 +1656,8 @@ public boolean cancelLoadJob(CancelLoadStmt stmt, boolean isAccurateMatch) throw matchLoadJobs.addAll(labelToLoadJobs.get(label)); } } else { + PatternMatcher matcher = PatternMatcher.createMysqlPattern(label, CaseSensibility.LABEL.getCaseSensibility()); for (Map.Entry> entry : labelToLoadJobs.entrySet()) { - PatternMatcher matcher = PatternMatcher.createMysqlPattern(label, CaseSensibility.LABEL.getCaseSensibility()); if (matcher.match(entry.getKey())) { loadJobs.addAll(entry.getValue()); } @@ -1919,6 +1919,11 @@ public LinkedList> getLoadJobInfosByDb(long dbId, String dbName long start = System.currentTimeMillis(); LOG.debug("begin to get load job info, size: {}", loadJobs.size()); + PatternMatcher matcher = null; + if (labelValue != null && !accurateMatch) { + matcher = PatternMatcher.createMysqlPattern(labelValue, CaseSensibility.LABEL.getCaseSensibility()); + } + for (LoadJob loadJob : loadJobs) { // filter first String label = loadJob.getLabel(); @@ -1930,7 +1935,6 @@ public LinkedList> getLoadJobInfosByDb(long dbId, String dbName continue; } } else { - PatternMatcher matcher = PatternMatcher.createMysqlPattern(labelValue, CaseSensibility.LABEL.getCaseSensibility()); if (!matcher.match(label)) { continue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java index 6e1c486730c7c4..b8d8cabdf855bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java @@ -319,8 +319,8 @@ public void cancelLoadJob(CancelLoadStmt stmt, boolean isAccurateMatch) throws D matchLoadJobs.addAll(labelToLoadJobs.get(stmt.getLabel())); } } else { + PatternMatcher matcher = PatternMatcher.createMysqlPattern(stmt.getLabel(), CaseSensibility.LABEL.getCaseSensibility()); for (Map.Entry> entry : labelToLoadJobs.entrySet()) { - PatternMatcher matcher = PatternMatcher.createMysqlPattern(stmt.getLabel(), CaseSensibility.LABEL.getCaseSensibility()); if (matcher.match(entry.getKey())) { matchLoadJobs.addAll(entry.getValue()); } @@ -542,8 +542,8 @@ public List> getLoadJobInfosByDb(long dbId, String labelValue, loadJobList.addAll(labelToLoadJobs.get(labelValue)); } else { // non-accurate match + PatternMatcher matcher = PatternMatcher.createMysqlPattern(labelValue, CaseSensibility.LABEL.getCaseSensibility()); for (Map.Entry> entry : labelToLoadJobs.entrySet()) { - PatternMatcher matcher = PatternMatcher.createMysqlPattern(labelValue, CaseSensibility.LABEL.getCaseSensibility()); if (matcher.match(entry.getKey())) { loadJobList.addAll(entry.getValue()); }