Skip to content

[feat](Nereids) support nereids hint position detaction #39113

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morrySnow merged 3 commits into from
Aug 15, 2024
Merged

[feat](Nereids) support nereids hint position detaction #39113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
52d16cd
[feat](Nereids) support nereids hint position detaction
LiBinfeng-01 Aug 8, 2024
5e7f7af
fix fe ut
LiBinfeng-01 Aug 8, 2024
fd3c9ad
fix comments
LiBinfeng-01 Aug 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,19 @@ lexer grammar DorisLexer;
public void markUnclosedComment() {
has_unclosed_bracketed_comment = true;
}

// This variable will hold the external state
private boolean channel2;
Copy link
Contributor

@morrySnow morrySnow Aug 9, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why need this var? i think we should always let hint go to channel 2 ?

Copy link
Contributor Author

@LiBinfeng-01 LiBinfeng-01 Aug 10, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

when i let channel 2 always open, it would have problem when parsing because in channel lexer would add some char before original string to let parser ignore it. For example: it would make "/* leading(t1 t2) */" into "[@-1,15:36 = '/leading(t1 t2) */', xxx" which when i want to parse it, i would meet eof first then parse exit. So it make a switch to control this behavior


// Method to set the external state
public void setChannel2(boolean value) {
this.channel2 = value;
}

// Method to decide the channel based on external state
private boolean isChannel2() {
return this.channel2;
}
}

SEMICOLON: ';';
Expand Down Expand Up @@ -686,6 +699,11 @@ BRACKETED_COMMENT
: '/*' {!isHint()}? ( BRACKETED_COMMENT | . )*? ('*/' | {markUnclosedComment();} EOF) -> channel(HIDDEN)
;

HINT_WITH_CHANNEL
: {isChannel2()}? HINT_START .*? HINT_END -> channel(2)
;


FROM_DUAL
: 'FROM' WS+ 'DUAL' -> channel(HIDDEN);

Expand Down
126 changes: 72 additions & 54 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -514,6 +514,12 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
return pos1.getCharPositionInLine() - pos2.getCharPositionInLine();
});

private final Map<Integer, ParserRuleContext> selectHintMap;

public LogicalPlanBuilder(Map<Integer, ParserRuleContext> selectHintMap) {
this.selectHintMap = selectHintMap;
}

@SuppressWarnings("unchecked")
protected <T> T typedVisit(ParseTree ctx) {
return (T) ctx.accept(this);
Expand Down Expand Up @@ -1352,7 +1358,16 @@ public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationConte
Optional.ofNullable(ctx.aggClause()),
Optional.ofNullable(ctx.havingClause()));
selectPlan = withQueryOrganization(selectPlan, ctx.queryOrganization());
return withSelectHint(selectPlan, selectCtx.selectHint());
if ((selectHintMap == null) || selectHintMap.isEmpty()) {
return selectPlan;
}
List<ParserRuleContext> selectHintContexts = Lists.newArrayList();
for (Integer key : selectHintMap.keySet()) {
if (key > selectCtx.getStart().getStopIndex() && key < selectCtx.getStop().getStartIndex()) {
selectHintContexts.add(selectHintMap.get(key));
}
}
return withSelectHint(selectPlan, selectHintContexts);
});
}

Expand Down Expand Up @@ -3110,67 +3125,70 @@ private LogicalPlan withJoinRelations(LogicalPlan input, RelationContext ctx) {
return last;
}

private LogicalPlan withSelectHint(LogicalPlan logicalPlan, SelectHintContext hintContext) {
if (hintContext == null) {
private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List<ParserRuleContext> hintContexts) {
if (hintContexts.isEmpty()) {
return logicalPlan;
}
Map<String, SelectHint> hints = Maps.newLinkedHashMap();
for (HintStatementContext hintStatement : hintContext.hintStatements) {
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
switch (hintName) {
case "set_var":
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
Optional<String> value = Optional.empty();
if (kv.constantValue != null) {
Literal literal = (Literal) visit(kv.constantValue);
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
} else if (kv.identifierValue != null) {
// maybe we should throw exception when the identifierValue is quoted identifier
value = Optional.ofNullable(kv.identifierValue.getText());
for (ParserRuleContext hintContext : hintContexts) {
SelectHintContext selectHintContext = (SelectHintContext) hintContext;
for (HintStatementContext hintStatement : selectHintContext.hintStatements) {
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
switch (hintName) {
case "set_var":
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
Optional<String> value = Optional.empty();
if (kv.constantValue != null) {
Literal literal = (Literal) visit(kv.constantValue);
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
} else if (kv.identifierValue != null) {
// maybe we should throw exception when the identifierValue is quoted identifier
value = Optional.ofNullable(kv.identifierValue.getText());
}
parameters.put(parameterName, value);
}
parameters.put(parameterName, value);
}
}
hints.put(hintName, new SelectHintSetVar(hintName, parameters));
break;
case "leading":
List<String> leadingParameters = new ArrayList<String>();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
leadingParameters.add(parameterName);
hints.put(hintName, new SelectHintSetVar(hintName, parameters));
break;
case "leading":
List<String> leadingParameters = new ArrayList<String>();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
leadingParameters.add(parameterName);
}
}
}
hints.put(hintName, new SelectHintLeading(hintName, leadingParameters));
break;
case "ordered":
hints.put(hintName, new SelectHintOrdered(hintName));
break;
case "use_cbo_rule":
List<String> useRuleParameters = new ArrayList<String>();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
useRuleParameters.add(parameterName);
hints.put(hintName, new SelectHintLeading(hintName, leadingParameters));
break;
case "ordered":
hints.put(hintName, new SelectHintOrdered(hintName));
break;
case "use_cbo_rule":
List<String> useRuleParameters = new ArrayList<String>();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
useRuleParameters.add(parameterName);
}
}
}
hints.put(hintName, new SelectHintUseCboRule(hintName, useRuleParameters, false));
break;
case "no_use_cbo_rule":
List<String> noUseRuleParameters = new ArrayList<String>();
for (HintAssignmentContext kv : hintStatement.parameters) {
String parameterName = visitIdentifierOrText(kv.key);
if (kv.key != null) {
noUseRuleParameters.add(parameterName);
hints.put(hintName, new SelectHintUseCboRule(hintName, useRuleParameters, false));
break;
case "no_use_cbo_rule":
List<String> noUseRuleParameters = new ArrayList<String>();
for (HintAssignmentContext kv : hintStatement.parameters) {
String parameterName = visitIdentifierOrText(kv.key);
if (kv.key != null) {
noUseRuleParameters.add(parameterName);
}
}
}
hints.put(hintName, new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
break;
default:
break;
hints.put(hintName, new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
break;
default:
break;
}
}
}
return new LogicalSelectHint<>(hints, logicalPlan);
Expand Down
5 changes: 5 additions & 0 deletions ...e-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForCreateView.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,15 @@
import com.google.common.collect.ImmutableList;
import org.antlr.v4.runtime.ParserRuleContext;

import java.util.Map;
import java.util.Optional;

/**LogicalPlanBuilderForCreateView*/
public class LogicalPlanBuilderForCreateView extends LogicalPlanBuilder {
public LogicalPlanBuilderForCreateView(Map<Integer, ParserRuleContext> selectHintMap) {
super(selectHintMap);
}

@Override
protected LogicalPlan withGenerate(LogicalPlan plan, LateralViewContext ctx) {
ConnectContext.get().getStatementContext().addIndexInSqlToString(
Expand Down
5 changes: 5 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForSyncMv.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import org.apache.doris.nereids.util.PlanUtils;

import com.google.common.collect.ImmutableList;
import org.antlr.v4.runtime.ParserRuleContext;

import java.util.List;
import java.util.Map;
Expand All @@ -49,6 +50,10 @@
public class LogicalPlanBuilderForSyncMv extends LogicalPlanBuilder {
private Optional<String> querySql;

public LogicalPlanBuilderForSyncMv(Map<Integer, ParserRuleContext> selectHintMap) {
super(selectHintMap);
}

@Override
public Expression visitFunctionCallExpression(DorisParser.FunctionCallExpressionContext ctx) {
Expression expression = super.visitFunctionCallExpression(ctx);
Expand Down
37 changes: 34 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.apache.doris.qe.SessionVariable;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.ParserRuleContext;
Expand Down Expand Up @@ -273,26 +274,56 @@ private <T> T parse(String sql, @Nullable LogicalPlanBuilder logicalPlanBuilder,
Function<DorisParser, ParserRuleContext> parseFunction) {
ParserRuleContext tree = toAst(sql, parseFunction);
LogicalPlanBuilder realLogicalPlanBuilder = logicalPlanBuilder == null
? new LogicalPlanBuilder() : logicalPlanBuilder;
? new LogicalPlanBuilder(getHintMap(sql, DorisParser::selectHint)) : logicalPlanBuilder;
return (T) realLogicalPlanBuilder.visit(tree);
}

public LogicalPlan parseForCreateView(String sql) {
ParserRuleContext tree = toAst(sql, DorisParser::singleStatement);
LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForCreateView();
LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForCreateView(
getHintMap(sql, DorisParser::selectHint));
return (LogicalPlan) realLogicalPlanBuilder.visit(tree);
}

public Optional<String> parseForSyncMv(String sql) {
ParserRuleContext tree = toAst(sql, DorisParser::singleStatement);
LogicalPlanBuilderForSyncMv logicalPlanBuilderForSyncMv = new LogicalPlanBuilderForSyncMv();
LogicalPlanBuilderForSyncMv logicalPlanBuilderForSyncMv = new LogicalPlanBuilderForSyncMv(
getHintMap(sql, DorisParser::selectHint));
logicalPlanBuilderForSyncMv.visit(tree);
return logicalPlanBuilderForSyncMv.getQuerySql();
}

/** get hint map */
public static Map<Integer, ParserRuleContext> getHintMap(String sql,
Function<DorisParser, ParserRuleContext> parseFunction) {
// parse hint first round
DorisLexer hintLexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql)));
hintLexer.setChannel2(true);
CommonTokenStream hintTokenStream = new CommonTokenStream(hintLexer);
Comment on lines +300 to +302
Copy link
Contributor

@morrySnow morrySnow Aug 9, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

antlr could generate one stream and get specific channel token from it by getHiddenTokensToRight, so could we use it to get hint in visitRegularQuerySpecification?

Copy link
Contributor Author

@LiBinfeng-01 LiBinfeng-01 Aug 10, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

getHiddenTokensToRight does not work in this case, I tried to get it but it failed. I try to use List hintTokens = hintTokenStream.getHiddenTokensToRight(0, 2); but it failed to get token list out from channel2


Map<Integer, ParserRuleContext> selectHintMap = Maps.newHashMap();

Token hintToken = hintTokenStream.getTokenSource().nextToken();
while (hintToken != null && hintToken.getType() != DorisLexer.EOF) {
int tokenType = hintToken.getType();
if (tokenType == DorisLexer.HINT_WITH_CHANNEL) {
String hintSql = sql.substring(hintToken.getStartIndex(), hintToken.getStopIndex() + 1);
DorisLexer newHintLexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(hintSql)));
newHintLexer.setChannel2(false);
CommonTokenStream newHintTokenStream = new CommonTokenStream(newHintLexer);
DorisParser hintParser = new DorisParser(newHintTokenStream);
ParserRuleContext hintContext = parseFunction.apply(hintParser);
Comment on lines +313 to +315
Copy link
Contributor

@morrySnow morrySnow Aug 9, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we could write a new parser to process hint seperately?

Copy link
Contributor Author

@LiBinfeng-01 LiBinfeng-01 Aug 10, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it is no need to write a new parser but use exist parser logic as hint syntax, the only thing we need to care about is position and new hint type, position is solved by selectHintMap we added, new hint type is filtered by original hint parser. So it is no needed to add a new parser with same logic, we can only use channel 2 to deal with hint seperately

selectHintMap.put(hintToken.getStartIndex(), hintContext);
}
hintToken = hintTokenStream.getTokenSource().nextToken();
}
return selectHintMap;
}

/** toAst */
public static ParserRuleContext toAst(String sql, Function<DorisParser, ParserRuleContext> parseFunction) {
DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql)));
lexer.setChannel2(true);
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
DorisParser parser = new DorisParser(tokenStream);

Expand Down
4 changes: 4 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/parser/plsql/PLSqlLogicalPlanBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@
*/
public class PLSqlLogicalPlanBuilder extends LogicalPlanBuilder {

public PLSqlLogicalPlanBuilder() {
super(null);
Copy link
Contributor

@morrySnow morrySnow Aug 9, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same as create view

Copy link
Contributor Author

@LiBinfeng-01 LiBinfeng-01 Aug 10, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

}

public List<String> visitMultipartIdentifier(MultipartIdentifierContext ctx) {
return ctx.parts.stream()
.map(RuleContext::getText)
Expand Down
8 changes: 0 additions & 8 deletions fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -352,16 +352,10 @@ public void testJoinHint() {
parsePlan("select * from t1 join [broadcast] t2 on t1.keyy=t2.keyy")
.matches(logicalJoin().when(j -> j.getDistributeHint().distributeType == DistributeType.BROADCAST_RIGHT));

parsePlan("select * from t1 join /*+ broadcast */ t2 on t1.keyy=t2.keyy")
.matches(logicalJoin().when(j -> j.getDistributeHint().distributeType == DistributeType.BROADCAST_RIGHT));

// invalid hint position
parsePlan("select * from [shuffle] t1 join t2 on t1.keyy=t2.keyy")
.assertThrowsExactly(ParseException.class);

parsePlan("select * from /*+ shuffle */ t1 join t2 on t1.keyy=t2.keyy")
.assertThrowsExactly(ParseException.class);

// invalid hint content
parsePlan("select * from t1 join [bucket] t2 on t1.keyy=t2.keyy")
.assertThrowsExactly(ParseException.class)
Expand All @@ -372,8 +366,6 @@ public void testJoinHint() {
+ "----------------------^^^");

// invalid multiple hints
parsePlan("select * from t1 join /*+ shuffle , broadcast */ t2 on t1.keyy=t2.keyy")
.assertThrowsExactly(ParseException.class);

parsePlan("select * from t1 join [shuffle,broadcast] t2 on t1.keyy=t2.keyy")
.assertThrowsExactly(ParseException.class);
Expand Down
Loading