Skip to content

Rewrite special comment parser (#711) #837

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
tangenta wants to merge 1 commit into from
Closed

Rewrite special comment parser (#711) #837

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 11 additions & 26 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,41 +1,26 @@
.PHONY: all parser clean

ARCH:="`uname -s`"
MAC:="Darwin"
LINUX:="Linux"
all: fmt parser

all: parser.go fmt

test: parser.go fmt
test: fmt parser
sh test.sh

parser.go: parser.y
make parser

parser: bin/goyacc
bin/goyacc -o /dev/null parser.y
bin/goyacc -o parser.go parser.y 2>&1 | egrep "(shift|reduce)/reduce" | awk '{print} END {if (NR > 0) {print "Find conflict in parser.y. Please check y.output for more information."; exit 1;}}'
rm -f y.output

@if [ $(ARCH) = $(LINUX) ]; \
then \
sed -i -e 's|//line.*||' -e 's/yyEofCode/yyEOFCode/' parser.go; \
elif [ $(ARCH) = $(MAC) ]; \
then \
/usr/bin/sed -i "" 's|//line.*||' parser.go; \
/usr/bin/sed -i "" 's/yyEofCode/yyEOFCode/' parser.go; \
fi
parser: parser.go hintparser.go

@awk 'BEGIN{print "// Code generated by goyacc DO NOT EDIT."} {print $0}' parser.go > tmp_parser.go && mv tmp_parser.go parser.go;
%arser.go: prefix = $(@:parser.go=)
%arser.go: %arser.y bin/goyacc
@echo "bin/goyacc -o $@ -p yy$(prefix) -t $(prefix)Parser $<"
@bin/goyacc -o $@ -p yy$(prefix) -t $(prefix)Parser $< || ( rm -f $@ && echo 'Please check y.output for more information' && exit 1 )
@rm -f y.output

bin/goyacc: goyacc/main.go
GO111MODULE=on go build -o bin/goyacc goyacc/main.go

fmt:
fmt: bin/goyacc
@echo "gofmt (simplify)"
@ gofmt -s -l -w . 2>&1 | awk '{print} END{if(NR>0) {exit 1}}'
@gofmt -s -l -w . 2>&1 | awk '{print} END{if(NR>0) {exit 1}}'

clean:
go clean -i ./...
rm -rf *.out
rm parser.go
rm -f parser.go hintparser.go
83 changes: 79 additions & 4 deletions ast/misc.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2087,24 +2087,99 @@ type TableOptimizerHint struct {
// Table hints has no schema info
// It allows only table name or alias (if table has an alias)
HintName model.CIStr
Tables []model.CIStr
// QBName is the default effective query block of this hint.
QBName model.CIStr
Tables []HintTable
Indexes []model.CIStr
StoreType model.CIStr
// Statement Execution Time Optimizer Hints
// See https://dev.mysql.com/doc/refman/5.7/en/optimizer-hints.html#optimizer-hints-execution-time
MaxExecutionTime uint64
MemoryQuota int64
QueryType model.CIStr
HintFlag bool
}

// HintTable is table in the hint. It may have query block info.
type HintTable struct {
DBName model.CIStr
TableName model.CIStr
QBName model.CIStr
}

func (ht *HintTable) Restore(ctx *RestoreCtx) {
if ht.DBName.L != "" {
ctx.WriteName(ht.DBName.String())
ctx.WriteKeyWord(".")
}
ctx.WriteName(ht.TableName.String())
if ht.QBName.L != "" {
ctx.WriteKeyWord("@")
ctx.WriteName(ht.QBName.String())
}
}

// Restore implements Node interface.
func (n *TableOptimizerHint) Restore(ctx *RestoreCtx) error {
ctx.WriteKeyWord(n.HintName.String())
ctx.WritePlain("(")
if n.HintName.L == "max_execution_time" {
if n.QBName.L != "" {
if n.HintName.L != "qb_name" {
ctx.WriteKeyWord("@")
}
ctx.WriteName(n.QBName.String())
}
// Hints without args except query block.
switch n.HintName.L {
case "hash_agg", "stream_agg", "agg_to_cop", "read_consistent_replica", "no_index_merge", "qb_name":
ctx.WritePlain(")")
return nil
}
if n.QBName.L != "" {
ctx.WritePlain(" ")
}
// Hints with args except query block.
switch n.HintName.L {
case "max_execution_time":
ctx.WritePlainf("%d", n.MaxExecutionTime)
} else {
case "tidb_hj", "tidb_smj", "tidb_inlj", "hash_join", "sm_join", "inl_join":
for i, table := range n.Tables {
if i != 0 {
ctx.WritePlain(", ")
}
ctx.WriteName(table.String())
table.Restore(ctx)
}
case "use_index", "ignore_index", "use_index_merge":
n.Tables[0].Restore(ctx)
ctx.WritePlain(" ")
for i, index := range n.Indexes {
if i != 0 {
ctx.WritePlain(", ")
}
ctx.WriteName(index.String())
}
case "use_toja", "enable_plan_cache":
if n.HintFlag {
ctx.WritePlain("TRUE")
} else {
ctx.WritePlain("FALSE")
}
case "query_type":
ctx.WriteKeyWord(n.QueryType.String())
case "memory_quota":
ctx.WritePlainf("%d MB", n.MemoryQuota/1024/1024)
case "read_from_storage":
ctx.WriteKeyWord(n.StoreType.String())
for i, table := range n.Tables {
if i == 0 {
ctx.WritePlain("[")
}
table.Restore(ctx)
if i == len(n.Tables)-1 {
ctx.WritePlain("]")
} else {
ctx.WritePlain(", ")
}
}
}
ctx.WritePlain(")")
Expand Down
8 changes: 8 additions & 0 deletions ast/misc_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,14 @@ func (ts *testMiscSuite) TestUserSpec(c *C) {

func (ts *testMiscSuite) TestTableOptimizerHintRestore(c *C) {
testCases := []NodeRestoreTestCase{
{"USE_INDEX(t1 c1)", "USE_INDEX(`t1` `c1`)"},
{"USE_INDEX(test.t1 c1)", "USE_INDEX(`test`.`t1` `c1`)"},
{"USE_INDEX(@sel_1 t1 c1)", "USE_INDEX(@`sel_1` `t1` `c1`)"},
{"USE_INDEX(t1@sel_1 c1)", "USE_INDEX(`t1`@`sel_1` `c1`)"},
{"IGNORE_INDEX(t1 c1)", "IGNORE_INDEX(`t1` `c1`)"},
{"USE_INDEX(test.t1@sel_1 c1)", "USE_INDEX(`test`.`t1`@`sel_1` `c1`)"},
{"IGNORE_INDEX(@sel_1 t1 c1)", "IGNORE_INDEX(@`sel_1` `t1` `c1`)"},
{"IGNORE_INDEX(t1@sel_1 c1)", "IGNORE_INDEX(`t1`@`sel_1` `c1`)"},
{"TIDB_SMJ(`t1`)", "TIDB_SMJ(`t1`)"},
{"TIDB_SMJ(t1)", "TIDB_SMJ(`t1`)"},
{"TIDB_SMJ(t1,t2)", "TIDB_SMJ(`t1`, `t2`)"},
Expand Down
12 changes: 1 addition & 11 deletions digester.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,17 +173,7 @@ func (d *sqlDigester) normalize(sql string) {

func (d *sqlDigester) reduceOptimizerHint(tok *token) (reduced bool) {
// ignore /*+..*/
if tok.tok == hintBegin {
for {
tok, _, _ := d.lexer.scan()
if tok == 0 || (tok == unicode.ReplacementChar && d.lexer.r.eof()) {
break
}
if tok == hintEnd {
reduced = true
break
}
}
if tok.tok == hintComment {
return
}

Expand Down
20 changes: 10 additions & 10 deletions goyacc/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ var (
oResolved = flag.Bool("ex", false, "explain how were conflicts resolved")
oXErrors = flag.String("xe", "", "generate eXtra errors from examples source file")
oXErrorsGen = flag.String("xegen", "", "generate error from examples source file automatically from the grammar")
oParserType = flag.String("t", "Parser", "name of the parser in the generated yyParse() function")
)

func main() {
Expand Down Expand Up @@ -300,7 +301,7 @@ func main1(in string) (err error) {

p, err := y.ProcessFile(token.NewFileSet(), in, &y.Options{
//NoDefault: *oNoDefault,
AllowConflicts: true,
AllowConflicts: false,
Closures: *oClosures,
LA: *oLA,
Reducible: *oReducible,
Expand Down Expand Up @@ -364,7 +365,7 @@ func main1(in string) (err error) {

// ----------------------------------------------------------- Prologue
f := strutil.IndentFormatter(out, "\t")
mustFormat(f, "// CAUTION: Generated file - DO NOT EDIT.\n\n")
mustFormat(f, "// Code generated by goyacc DO NOT EDIT.\n\n")
mustFormat(f, "%s", injectImport(p.Prologue))
mustFormat(f, `
type %[1]sSymType %i%s%u
Expand Down Expand Up @@ -546,13 +547,12 @@ func %[1]slex1(yylex %[1]sLexer, lval *%[1]sSymType) (n int) {
return n
}

func %[1]sParse(yylex %[1]sLexer, parser *Parser) int {
func %[1]sParse(yylex %[1]sLexer, parser *%[5]s) int {
const yyError = %[2]d

yyEx, _ := yylex.(%[1]sLexerEx)
var yyn int
parser.yylval = %[1]sSymType{}
parser.yyVAL = %[1]sSymType{}
yyS := parser.cache

Nerrs := 0 /* number of errors */
Expand Down Expand Up @@ -580,13 +580,13 @@ ret1:
yystack:
/* put a state and value onto the stack */
yyp++
if yyp >= len(yyS) {
if yyp+1 >= len(yyS) {
nyys := make([]%[1]sSymType, len(yyS)*2)
copy(nyys, yyS)
yyS = nyys
parser.cache = yyS
}
yyS[yyp] = parser.yyVAL
parser.yyVAL = &yyS[yyp+1]
yyS[yyp].yys = yystate

yynewstate:
Expand Down Expand Up @@ -614,7 +614,7 @@ yynewstate:
switch {
case yyn > 0: // shift
yychar = -1
parser.yyVAL = parser.yylval
*parser.yyVAL = parser.yylval
yystate = yyn
yyshift = yyn
if %[1]sDebug >= 2 {
Expand Down Expand Up @@ -712,7 +712,7 @@ yynewstate:
yyS = nyys
parser.cache = yyS
}
parser.yyVAL = yyS[yyp+1]
parser.yyVAL = &yyS[yyp+1]

/* consult goto table to find next state */
exState := yystate
Expand All @@ -724,7 +724,7 @@ yynewstate:

switch r {%i
`,
*oPref, errSym, *oDlvalf, *oDlval)
*oPref, errSym, *oDlvalf, *oDlval, *oParserType)
for r, rule := range p.Rules {
if rule.Action == nil {
continue
Expand Down Expand Up @@ -781,7 +781,7 @@ yynewstate:
mustFormat(f, `%u
}

if yyEx != nil && yyEx.Reduced(r, exState, &parser.yyVAL) {
if yyEx != nil && yyEx.Reduced(r, exState, parser.yyVAL) {
return -1
}
goto yystack /* stack new state and value */
Expand Down
Loading