From e649b8f3c06fbfb0bbd1c1f52a6ef09d2fd46c15 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 5 Jul 2024 12:06:43 +0800 Subject: [PATCH 1/4] 1 --- .../java/org/apache/doris/policy/Policy.java | 3 ++- .../java/org/apache/doris/policy/RowPolicy.java | 17 ++++++++++++----- .../org/apache/doris/policy/PolicyTest.java | 2 +- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java b/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java index 81016e94826606..01c5399d4ab974 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java @@ -117,7 +117,8 @@ public static Policy fromCreateStmt(CreatePolicyStmt stmt) throws AnalysisExcept } return new RowPolicy(policyId, stmt.getPolicyName(), stmt.getTableName().getCtl(), stmt.getTableName().getDb(), stmt.getTableName().getTbl(), userIdent, stmt.getRoleName(), - stmt.getOrigStmt().originStmt, stmt.getFilterType(), stmt.getWherePredicate()); + stmt.getOrigStmt().originStmt, stmt.getOrigStmt().idx, stmt.getFilterType(), + stmt.getWherePredicate()); default: throw new AnalysisException("Unknown policy type: " + stmt.getType()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/RowPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/policy/RowPolicy.java index 2b52c03b7d5aa1..9361ab1595e32a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/RowPolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/RowPolicy.java @@ -102,6 +102,8 @@ public class RowPolicy extends Policy implements RowFilterPolicy { **/ @SerializedName(value = "originStmt") private String originStmt; + @SerializedName(value = "stmtIdx") + private int stmtIdx; private Expr wherePredicate = null; @@ -123,7 +125,7 @@ public RowPolicy() { * @param wherePredicate where predicate */ public RowPolicy(long policyId, final String policyName, long dbId, UserIdentity user, String roleName, - String originStmt, + String originStmt, int stmtIdx, final long tableId, final FilterType filterType, final Expr wherePredicate) { super(policyId, PolicyTypeEnum.ROW, policyName); this.user = user; @@ -132,12 +134,13 @@ public RowPolicy(long policyId, final String policyName, long dbId, UserIdentity this.tableId = tableId; this.filterType = filterType; this.originStmt = originStmt; + this.stmtIdx = stmtIdx; this.wherePredicate = wherePredicate; } public RowPolicy(long policyId, final String policyName, String ctlName, String dbName, String tableName, UserIdentity user, String roleName, - String originStmt, final FilterType filterType, final Expr wherePredicate) { + String originStmt, int stmtIdx, final FilterType filterType, final Expr wherePredicate) { super(policyId, PolicyTypeEnum.ROW, policyName); this.user = user; this.roleName = roleName; @@ -146,6 +149,7 @@ public RowPolicy(long policyId, final String policyName, String ctlName, String this.tableName = tableName; this.filterType = filterType; this.originStmt = originStmt; + this.stmtIdx = stmtIdx; this.wherePredicate = wherePredicate; } @@ -166,16 +170,19 @@ public void gsonPostProcess() throws IOException { try { SqlScanner input = new SqlScanner(new StringReader(originStmt), 0L); SqlParser parser = new SqlParser(input); - CreatePolicyStmt stmt = (CreatePolicyStmt) SqlParserUtils.getFirstStmt(parser); + CreatePolicyStmt stmt = (CreatePolicyStmt) SqlParserUtils.getStmt(parser, stmtIdx); wherePredicate = stmt.getWherePredicate(); } catch (Exception e) { - throw new IOException("table policy parse originStmt error", e); + String errorMsg = String.format("table policy parse originStmt error, originStmt: %s, stmtIdx: %s.", + originStmt, stmtIdx); + throw new IOException(errorMsg, e); } } @Override public RowPolicy clone() { return new RowPolicy(this.id, this.policyName, this.dbId, this.user, this.roleName, this.originStmt, + this.stmtIdx, this.tableId, this.filterType, this.wherePredicate); } @@ -218,7 +225,7 @@ public boolean isInvalid() { public Expression getFilterExpression() throws AnalysisException { NereidsParser nereidsParser = new NereidsParser(); String sql = getOriginStmt(); - CreatePolicyCommand command = (CreatePolicyCommand) nereidsParser.parseSingle(sql); + CreatePolicyCommand command = (CreatePolicyCommand) nereidsParser.parseMultiple(sql).get(stmtIdx).first; Optional wherePredicate = command.getWherePredicate(); if (!wherePredicate.isPresent()) { throw new AnalysisException("Invalid row policy [" + getPolicyIdent() + "], " + sql); diff --git a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java index ddf0d68fd2b171..d9c726775d4304 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java @@ -341,7 +341,7 @@ public void testReadWrite() throws IOException, AnalysisException { FilterType filterType = FilterType.PERMISSIVE; Expr wherePredicate = null; - Policy rowPolicy = new RowPolicy(10000, policyName, dbId, user, null, originStmt, tableId, filterType, + Policy rowPolicy = new RowPolicy(10000, policyName, dbId, user, null, originStmt, 0, tableId, filterType, wherePredicate); ByteArrayOutputStream emptyOutputStream = new ByteArrayOutputStream(); From 1c6ffdc157344807cc96b75a6a8fa1bed5b20836 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 5 Jul 2024 15:10:23 +0800 Subject: [PATCH 2/4] 1 --- .../main/java/org/apache/doris/policy/Policy.java | 3 ++- .../java/org/apache/doris/policy/RowPolicy.java | 15 +++++++++++---- .../org/apache/doris/qe/ConnectProcessor.java | 2 +- .../java/org/apache/doris/policy/PolicyTest.java | 2 +- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java b/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java index 81016e94826606..01c5399d4ab974 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java @@ -117,7 +117,8 @@ public static Policy fromCreateStmt(CreatePolicyStmt stmt) throws AnalysisExcept } return new RowPolicy(policyId, stmt.getPolicyName(), stmt.getTableName().getCtl(), stmt.getTableName().getDb(), stmt.getTableName().getTbl(), userIdent, stmt.getRoleName(), - stmt.getOrigStmt().originStmt, stmt.getFilterType(), stmt.getWherePredicate()); + stmt.getOrigStmt().originStmt, stmt.getOrigStmt().idx, stmt.getFilterType(), + stmt.getWherePredicate()); default: throw new AnalysisException("Unknown policy type: " + stmt.getType()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/RowPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/policy/RowPolicy.java index 2b52c03b7d5aa1..4805345de9db27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/RowPolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/RowPolicy.java @@ -102,6 +102,8 @@ public class RowPolicy extends Policy implements RowFilterPolicy { **/ @SerializedName(value = "originStmt") private String originStmt; + @SerializedName(value = "stmtIdx") + private int stmtIdx; private Expr wherePredicate = null; @@ -123,7 +125,7 @@ public RowPolicy() { * @param wherePredicate where predicate */ public RowPolicy(long policyId, final String policyName, long dbId, UserIdentity user, String roleName, - String originStmt, + String originStmt, int stmtIdx, final long tableId, final FilterType filterType, final Expr wherePredicate) { super(policyId, PolicyTypeEnum.ROW, policyName); this.user = user; @@ -132,12 +134,13 @@ public RowPolicy(long policyId, final String policyName, long dbId, UserIdentity this.tableId = tableId; this.filterType = filterType; this.originStmt = originStmt; + this.stmtIdx = stmtIdx; this.wherePredicate = wherePredicate; } public RowPolicy(long policyId, final String policyName, String ctlName, String dbName, String tableName, UserIdentity user, String roleName, - String originStmt, final FilterType filterType, final Expr wherePredicate) { + String originStmt, int stmtIdx, final FilterType filterType, final Expr wherePredicate) { super(policyId, PolicyTypeEnum.ROW, policyName); this.user = user; this.roleName = roleName; @@ -146,6 +149,7 @@ public RowPolicy(long policyId, final String policyName, String ctlName, String this.tableName = tableName; this.filterType = filterType; this.originStmt = originStmt; + this.stmtIdx = stmtIdx; this.wherePredicate = wherePredicate; } @@ -166,16 +170,19 @@ public void gsonPostProcess() throws IOException { try { SqlScanner input = new SqlScanner(new StringReader(originStmt), 0L); SqlParser parser = new SqlParser(input); - CreatePolicyStmt stmt = (CreatePolicyStmt) SqlParserUtils.getFirstStmt(parser); + CreatePolicyStmt stmt = (CreatePolicyStmt) SqlParserUtils.getStmt(parser, stmtIdx); wherePredicate = stmt.getWherePredicate(); } catch (Exception e) { - throw new IOException("table policy parse originStmt error", e); + String errorMsg = String.format("table policy parse originStmt error, originStmt: %s, stmtIdx: %s.", + originStmt, stmtIdx); + throw new IOException(errorMsg, e); } } @Override public RowPolicy clone() { return new RowPolicy(this.id, this.policyName, this.dbId, this.user, this.roleName, this.originStmt, + this.stmtIdx, this.tableId, this.filterType, this.wherePredicate); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index d25f541a7af510..164ed98a51ceed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -374,7 +374,7 @@ public void executeQuery(MysqlCommand mysqlCommand, String originStmt) throws Ex } StatementBase parsedStmt = stmts.get(i); - parsedStmt.setOrigStmt(new OriginStatement(convertedStmt, i)); + parsedStmt.setOrigStmt(new OriginStatement(auditStmt, usingOrigSingleStmt ? 0 : i)); parsedStmt.setUserInfo(ctx.getCurrentUserIdentity()); executor = new StmtExecutor(ctx, parsedStmt); executor.getProfile().getSummaryProfile().setParseSqlStartTime(parseSqlStartTime); diff --git a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java index ddf0d68fd2b171..d9c726775d4304 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java @@ -341,7 +341,7 @@ public void testReadWrite() throws IOException, AnalysisException { FilterType filterType = FilterType.PERMISSIVE; Expr wherePredicate = null; - Policy rowPolicy = new RowPolicy(10000, policyName, dbId, user, null, originStmt, tableId, filterType, + Policy rowPolicy = new RowPolicy(10000, policyName, dbId, user, null, originStmt, 0, tableId, filterType, wherePredicate); ByteArrayOutputStream emptyOutputStream = new ByteArrayOutputStream(); From 278fed95ed44a9e6723173ac658eb71070e7410f Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 5 Jul 2024 19:51:31 +0800 Subject: [PATCH 3/4] 1 --- .../org/apache/doris/policy/RowPolicy.java | 6 +++++- .../org/apache/doris/policy/PolicyTest.java | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/RowPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/policy/RowPolicy.java index 9361ab1595e32a..71312518af036c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/RowPolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/RowPolicy.java @@ -225,7 +225,11 @@ public boolean isInvalid() { public Expression getFilterExpression() throws AnalysisException { NereidsParser nereidsParser = new NereidsParser(); String sql = getOriginStmt(); - CreatePolicyCommand command = (CreatePolicyCommand) nereidsParser.parseMultiple(sql).get(stmtIdx).first; + if (getStmtIdx() != 0) { + // Under normal circumstances, the index will only be equal to 0 + throw new AnalysisException("Invalid row policy [" + getPolicyIdent() + "], " + sql); + } + CreatePolicyCommand command = (CreatePolicyCommand) nereidsParser.parseSingle(sql); Optional wherePredicate = command.getWherePredicate(); if (!wherePredicate.isPresent()) { throw new AnalysisException("Invalid row policy [" + getPolicyIdent() + "], " + sql); diff --git a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java index d9c726775d4304..8d6a2a48ae36a2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java @@ -34,6 +34,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.FeConstants; +import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.Lists; @@ -364,4 +365,22 @@ public void testReadWrite() throws IOException, AnalysisException { Assertions.assertEquals(tableId, newRowPolicy.getTableId()); Assertions.assertEquals(filterType, newRowPolicy.getFilterType()); } + + @Test + public void testCompatibility() { + String s1 = "{\n" + + " \"clazz\": \"RowPolicy\",\n" + + " \"roleName\": \"role1\",\n" + + " \"dbId\": 2,\n" + + " \"tableId\": 2,\n" + + " \"filterType\": \"PERMISSIVE\",\n" + + " \"originStmt\": \"CREATE ROW POLICY test_row_policy ON test.table1 AS PERMISSIVE TO test_policy USING (k1 \\u003d 1)\",\n" + + " \"id\": 1,\n" + + " \"type\": \"ROW\",\n" + + " \"policyName\": \"cc\",\n" + + " \"version\": 0\n" + + "}"; + RowPolicy rowPolicy = GsonUtils.GSON.fromJson(s1, RowPolicy.class); + Assertions.assertEquals(rowPolicy.getStmtIdx(), 0); + } } From a2833f473f174f319c1550f9a7b993ae66944884 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Mon, 8 Jul 2024 15:07:42 +0800 Subject: [PATCH 4/4] 1 --- .../src/main/java/org/apache/doris/policy/RowPolicy.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/RowPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/policy/RowPolicy.java index 71312518af036c..2b8a492e82cfec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/RowPolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/RowPolicy.java @@ -175,7 +175,8 @@ public void gsonPostProcess() throws IOException { } catch (Exception e) { String errorMsg = String.format("table policy parse originStmt error, originStmt: %s, stmtIdx: %s.", originStmt, stmtIdx); - throw new IOException(errorMsg, e); + // Only print logs to avoid cluster failure to start + LOG.warn(errorMsg, e); } }