cherry-pick sql hint related PRs into release-3.1

ast/dml.go

@@ -766,6 +766,8 @@ type SelectStmt struct {
 	IsAfterUnionDistinct bool
 	// IsInBraces indicates whether it's a stmt in brace.
 	IsInBraces bool
+	// QueryBlockOffset indicates the order of this SelectStmt if counted from left to right in the sql text.
+	QueryBlockOffset int
 }
 
 // Restore implements Node interface.
@@ -908,6 +910,14 @@ func (n *SelectStmt) Accept(v Visitor) (Node, bool) {
 		n.TableHints = newHints
 	}
 
+	if n.Fields != nil {
+		node, ok := n.Fields.Accept(v)
+		if !ok {
+			return n, false
+		}
+		n.Fields = node.(*FieldList)
+	}
+
 	if n.From != nil {
 		node, ok := n.From.Accept(v)
 		if !ok {
@@ -924,14 +934,6 @@ func (n *SelectStmt) Accept(v Visitor) (Node, bool) {
 		n.Where = node.(ExprNode)
 	}
 
-	if n.Fields != nil {
-		node, ok := n.Fields.Accept(v)
-		if !ok {
-			return n, false
-		}
-		n.Fields = node.(*FieldList)
-	}
-
 	if n.GroupBy != nil {
 		node, ok := n.GroupBy.Accept(v)
 		if !ok {

ast/misc.go

@@ -2022,24 +2022,94 @@ type TableOptimizerHint struct {
 	// Table hints has no schema info
 	// It allows only table name or alias (if table has an alias)
 	HintName model.CIStr
-	Tables   []model.CIStr
+	// QBName is the default effective query block of this hint.
+	QBName    model.CIStr
+	Tables    []HintTable
+	Indexes   []model.CIStr
+	StoreType model.CIStr
 	// Statement Execution Time Optimizer Hints
 	// See https://dev.mysql.com/doc/refman/5.7/en/optimizer-hints.html#optimizer-hints-execution-time
 	MaxExecutionTime uint64
+	MemoryQuota      int64
+	QueryType        model.CIStr
+	HintFlag         bool
+}
+
+// HintTable is table in the hint. It may have query block info.
+type HintTable struct {
+	TableName model.CIStr
+	QBName    model.CIStr
+}
+
+func (ht *HintTable) Restore(ctx *RestoreCtx) {
+	ctx.WriteName(ht.TableName.String())
+	if ht.QBName.L != "" {
+		ctx.WriteKeyWord("@")
+		ctx.WriteName(ht.QBName.String())
+	}
 }
 
 // Restore implements Node interface.
 func (n *TableOptimizerHint) Restore(ctx *RestoreCtx) error {
 	ctx.WriteKeyWord(n.HintName.String())
 	ctx.WritePlain("(")
-	if n.HintName.L == "max_execution_time" {
+	if n.QBName.L != "" {
+		if n.HintName.L != "qb_name" {
+			ctx.WriteKeyWord("@")
+		}
+		ctx.WriteName(n.QBName.String())
+	}
+	// Hints without args except query block.
+	switch n.HintName.L {
+	case "hash_agg", "stream_agg", "agg_to_cop", "read_consistent_replica", "no_index_merge", "qb_name":
+		ctx.WritePlain(")")
+		return nil
+	}
+	if n.QBName.L != "" {
+		ctx.WritePlain(" ")
+	}
+	// Hints with args except query block.
+	switch n.HintName.L {
+	case "max_execution_time":
 		ctx.WritePlainf("%d", n.MaxExecutionTime)
-	} else {
+	case "tidb_hj", "tidb_smj", "tidb_inlj", "hash_join", "sm_join", "inl_join":
 		for i, table := range n.Tables {
 			if i != 0 {
 				ctx.WritePlain(", ")
 			}
-			ctx.WriteName(table.String())
+			table.Restore(ctx)
+		}
+	case "use_index", "ignore_index", "use_index_merge":
+		n.Tables[0].Restore(ctx)
+		ctx.WritePlain(" ")
+		for i, index := range n.Indexes {
+			if i != 0 {
+				ctx.WritePlain(", ")
+			}
+			ctx.WriteName(index.String())
+		}
+	case "use_toja", "enable_plan_cache":
+		if n.HintFlag {
+			ctx.WritePlain("TRUE")
+		} else {
+			ctx.WritePlain("FALSE")
+		}
+	case "query_type":
+		ctx.WriteKeyWord(n.QueryType.String())
+	case "memory_quota":
+		ctx.WritePlainf("%d MB", n.MemoryQuota/1024/1024)
+	case "read_from_storage":
+		ctx.WriteKeyWord(n.StoreType.String())
+		for i, table := range n.Tables {
+			if i == 0 {
+				ctx.WritePlain("[")
+			}
+			table.Restore(ctx)
+			if i == len(n.Tables)-1 {
+				ctx.WritePlain("]")
+			} else {
+				ctx.WritePlain(", ")
+			}
 		}
 	}
 	ctx.WritePlain(")")

ast/misc_test.go

@@ -205,18 +205,58 @@ func (ts *testMiscSuite) TestUserSpec(c *C) {
 
 func (ts *testMiscSuite) TestTableOptimizerHintRestore(c *C) {
 	testCases := []NodeRestoreTestCase{
+		{"USE_INDEX(t1 c1)", "USE_INDEX(`t1` `c1`)"},
+		{"USE_INDEX(@sel_1 t1 c1)", "USE_INDEX(@`sel_1` `t1` `c1`)"},
+		{"USE_INDEX(t1@sel_1 c1)", "USE_INDEX(`t1`@`sel_1` `c1`)"},
+		{"IGNORE_INDEX(t1 c1)", "IGNORE_INDEX(`t1` `c1`)"},
+		{"IGNORE_INDEX(@sel_1 t1 c1)", "IGNORE_INDEX(@`sel_1` `t1` `c1`)"},
+		{"IGNORE_INDEX(t1@sel_1 c1)", "IGNORE_INDEX(`t1`@`sel_1` `c1`)"},
 		{"TIDB_SMJ(`t1`)", "TIDB_SMJ(`t1`)"},
 		{"TIDB_SMJ(t1)", "TIDB_SMJ(`t1`)"},
 		{"TIDB_SMJ(t1,t2)", "TIDB_SMJ(`t1`, `t2`)"},
+		{"TIDB_SMJ(@sel1 t1,t2)", "TIDB_SMJ(@`sel1` `t1`, `t2`)"},
+		{"TIDB_SMJ(t1@sel1,t2@sel2)", "TIDB_SMJ(`t1`@`sel1`, `t2`@`sel2`)"},
 		{"TIDB_INLJ(t1,t2)", "TIDB_INLJ(`t1`, `t2`)"},
+		{"TIDB_INLJ(@sel1 t1,t2)", "TIDB_INLJ(@`sel1` `t1`, `t2`)"},
+		{"TIDB_INLJ(t1@sel1,t2@sel2)", "TIDB_INLJ(`t1`@`sel1`, `t2`@`sel2`)"},
 		{"TIDB_HJ(t1,t2)", "TIDB_HJ(`t1`, `t2`)"},
+		{"TIDB_HJ(@sel1 t1,t2)", "TIDB_HJ(@`sel1` `t1`, `t2`)"},
+		{"TIDB_HJ(t1@sel1,t2@sel2)", "TIDB_HJ(`t1`@`sel1`, `t2`@`sel2`)"},
+		{"SM_JOIN(t1,t2)", "SM_JOIN(`t1`, `t2`)"},
+		{"INL_JOIN(t1,t2)", "INL_JOIN(`t1`, `t2`)"},
+		{"HASH_JOIN(t1,t2)", "HASH_JOIN(`t1`, `t2`)"},
 		{"MAX_EXECUTION_TIME(3000)", "MAX_EXECUTION_TIME(3000)"},
+		{"MAX_EXECUTION_TIME(@sel1 3000)", "MAX_EXECUTION_TIME(@`sel1` 3000)"},
+		{"USE_INDEX_MERGE(t1 c1)", "USE_INDEX_MERGE(`t1` `c1`)"},
+		{"USE_INDEX_MERGE(@sel1 t1 c1)", "USE_INDEX_MERGE(@`sel1` `t1` `c1`)"},
+		{"USE_INDEX_MERGE(t1@sel1 c1)", "USE_INDEX_MERGE(`t1`@`sel1` `c1`)"},
+		{"USE_TOJA(TRUE)", "USE_TOJA(TRUE)"},
+		{"USE_TOJA(FALSE)", "USE_TOJA(FALSE)"},
+		{"USE_TOJA(@sel1 TRUE)", "USE_TOJA(@`sel1` TRUE)"},
+		{"QUERY_TYPE(OLAP)", "QUERY_TYPE(OLAP)"},
+		{"QUERY_TYPE(OLTP)", "QUERY_TYPE(OLTP)"},
+		{"QUERY_TYPE(@sel1 OLTP)", "QUERY_TYPE(@`sel1` OLTP)"},
+		{"MEMORY_QUOTA(1 GB)", "MEMORY_QUOTA(1024 MB)"},
+		{"MEMORY_QUOTA(@sel1 1 GB)", "MEMORY_QUOTA(@`sel1` 1024 MB)"},
+		{"HASH_AGG()", "HASH_AGG()"},
+		{"HASH_AGG(@sel1)", "HASH_AGG(@`sel1`)"},
+		{"STREAM_AGG()", "STREAM_AGG()"},
+		{"STREAM_AGG(@sel1)", "STREAM_AGG(@`sel1`)"},
+		{"AGG_TO_COP()", "AGG_TO_COP()"},
+		{"AGG_TO_COP(@sel_1)", "AGG_TO_COP(@`sel_1`)"},
+		{"NO_INDEX_MERGE()", "NO_INDEX_MERGE()"},
+		{"NO_INDEX_MERGE(@sel1)", "NO_INDEX_MERGE(@`sel1`)"},
+		{"READ_CONSISTENT_REPLICA()", "READ_CONSISTENT_REPLICA()"},
+		{"READ_CONSISTENT_REPLICA(@sel1)", "READ_CONSISTENT_REPLICA(@`sel1`)"},
+		{"QB_NAME(sel1)", "QB_NAME(`sel1`)"},
+		{"READ_FROM_STORAGE(@sel TIFLASH[t1, t2])", "READ_FROM_STORAGE(@`sel` TIFLASH[`t1`, `t2`])"},
 	}
 	extractNodeFunc := func(node Node) Node {
 		return node.(*SelectStmt).TableHints[0]
 	}
 	RunNodeRestoreTest(c, testCases, "select /*+ %s */ * from t1 join t2", extractNodeFunc)
 }
+
 func (ts *testMiscSuite) TestChangeStmtRestore(c *C) {
 	testCases := []NodeRestoreTestCase{
 		{"CHANGE PUMP TO NODE_STATE ='paused' FOR NODE_ID '127.0.0.1:9090'", "CHANGE PUMP TO NODE_STATE ='paused' FOR NODE_ID '127.0.0.1:9090'"},

circle.yml

@@ -46,7 +46,7 @@ jobs:
           cd /go/src/github.com/pingcap/
           git clone git@github.com:pingcap/tidb.git
           cd tidb
-          git checkout origin/release-3.0
+          git checkout origin/release-3.1
           rm go.sum
           GO111MODULE=on go mod edit -replace github.com/pingcap/parser=github.com/${CIRCLE_PR_USERNAME:-$CIRCLE_PROJECT_USERNAME}/${CIRCLE_PR_REPONAME:-$CIRCLE_PROJECT_REPONAME}@$CIRCLE_SHA1
           make gotest

misc.go

@@ -89,6 +89,8 @@ func init() {
 	initTokenByte('<', int('<'))
 	initTokenByte('(', int('('))
 	initTokenByte(')', int(')'))
+	initTokenByte('[', int('['))
+	initTokenByte(']', int(']'))
 	initTokenByte(';', int(';'))
 	initTokenByte(',', int(','))
 	initTokenByte('&', int('&'))
@@ -138,6 +140,7 @@ var tokenMap = map[string]int{
 	"ADDDATE":                  addDate,
 	"ADMIN":                    admin,
 	"AFTER":                    after,
+	"AGG_TO_COP":               hintAggToCop,
 	"ALL":                      all,
 	"ALGORITHM":                algorithm,
 	"ALTER":                    alter,
@@ -248,6 +251,7 @@ var tokenMap = map[string]int{
 	"DYNAMIC":                  dynamic,
 	"ELSE":                     elseKwd,
 	"ENABLE":                   enable,
+	"ENABLE_PLAN_CACHE":        hintEnablePlanCache,
 	"ENCLOSED":                 enclosed,
 	"END":                      end,
 	"ENGINE":                   engine,
@@ -288,6 +292,8 @@ var tokenMap = map[string]int{
 	"GROUP":                    group,
 	"GROUP_CONCAT":             groupConcat,
 	"HASH":                     hash,
+	"HASH_AGG":                 hintHASHAGG,
+	"HASH_JOIN":                hintHJ,
 	"HAVING":                   having,
 	"HIGH_PRIORITY":            highPriority,
 	"HISTORY":                  history,
@@ -298,11 +304,13 @@ var tokenMap = map[string]int{
 	"IDENTIFIED":               identified,
 	"IF":                       ifKwd,
 	"IGNORE":                   ignore,
+	"IGNORE_INDEX":             hintIgnoreIndex,
 	"IN":                       in,
 	"INCREMENTAL":              incremental,
 	"INDEX":                    index,
 	"INDEXES":                  indexes,
 	"INFILE":                   infile,
+	"INL_JOIN":                 hintINLJ,
 	"INNER":                    inner,
 	"INPLACE":                  inplace,
 	"INSTANT":                  instant,
@@ -363,6 +371,7 @@ var tokenMap = map[string]int{
 	"MEDIUMINT":                mediumIntType,
 	"MEDIUMTEXT":               mediumtextType,
 	"MEMORY":                   memory,
+	"MEMORY_QUOTA":             hintMemoryQuota,
 	"MERGE":                    merge,
 	"MICROSECOND":              microsecond,
 	"MIN":                      min,
@@ -380,6 +389,7 @@ var tokenMap = map[string]int{
 	"NEVER":                    never,
 	"NEXT_ROW_ID":              next_row_id,
 	"NO":                       no,
+	"NO_INDEX_MERGE":           hintNoIndexMerge,
 	"NO_WRITE_TO_BINLOG":       noWriteToBinLog,
 	"NODE_ID":                  nodeID,
 	"NODE_STATE":               nodeState,
@@ -392,6 +402,8 @@ var tokenMap = map[string]int{
 	"NUMERIC":                  numericType,
 	"NVARCHAR":                 nvarcharType,
 	"OFFSET":                   offset,
+	"OLAP":                     hintOLAP,
+	"OLTP":                     hintOLTP,
 	"ON":                       on,
 	"ONLY":                     only,
 	"OPTIMISTIC":               optimistic,
@@ -419,15 +431,19 @@ var tokenMap = map[string]int{
 	"PROFILE":                  profile,
 	"PROFILES":                 profiles,
 	"PUMP":                     pump,
+	"QB_NAME":                  hintQBName,
 	"QUARTER":                  quarter,
 	"QUERY":                    query,
+	"QUERY_TYPE":               hintQueryType,
 	"QUERIES":                  queries,
 	"QUICK":                    quick,
 	"SHARD_ROW_ID_BITS":        shardRowIDBits,
 	"PRE_SPLIT_REGIONS":        preSplitRegions,
 	"RANGE":                    rangeKwd,
 	"RECOVER":                  recover,
 	"READ":                     read,
+	"READ_CONSISTENT_REPLICA":  hintReadConsistentReplica,
+	"READ_FROM_STORAGE":        hintReadFromStorage,
 	"REAL":                     realType,
 	"RECENT":                   recent,
 	"REDUNDANT":                redundant,
@@ -469,6 +485,7 @@ var tokenMap = map[string]int{
 	"SIGNED":                   signed,
 	"SLAVE":                    slave,
 	"SLOW":                     slow,
+	"SM_JOIN":                  hintSMJ,
 	"SMALLINT":                 smallIntType,
 	"SNAPSHOT":                 snapshot,
 	"SOME":                     some,
@@ -501,6 +518,7 @@ var tokenMap = map[string]int{
 	"STDDEV_SAMP":              stddevSamp,
 	"STORED":                   stored,
 	"STRAIGHT_JOIN":            straightJoin,
+	"STREAM_AGG":               hintSTREAMAGG,
 	"SUBDATE":                  subDate,
 	"SUBJECT":                  subject,
 	"SUBPARTITION":             subpartition,
@@ -519,9 +537,11 @@ var tokenMap = map[string]int{
 	"THAN":                     than,
 	"THEN":                     then,
 	"TIDB":                     tidb,
-	"TIDB_HJ":                  tidbHJ,
-	"TIDB_INLJ":                tidbINLJ,
-	"TIDB_SMJ":                 tidbSMJ,
+	"TIDB_HJ":                  hintHJ,
+	"TIDB_INLJ":                hintINLJ,
+	"TIDB_SMJ":                 hintSMJ,
+	"TIKV":                     hintTiKV,
+	"TIFLASH":                  hintTiFlash,
 	"TIME":                     timeType,
 	"TIMESTAMP":                timestampType,
 	"TIMESTAMPADD":             timestampAdd,
@@ -559,6 +579,10 @@ var tokenMap = map[string]int{
 	"UPDATE":                   update,
 	"USAGE":                    usage,
 	"USE":                      use,
+	"USE_INDEX":                hintUseIndex,
+	"USE_INDEX_MERGE":          hintUseIndexMerge,
+	"USE_PLAN_CACHE":           hintUsePlanCache,
+	"USE_TOJA":                 hintUseToja,
 	"USER":                     user,
 	"USING":                    using,
 	"UTC_DATE":                 utcDate,
@@ -648,10 +672,13 @@ var windowFuncTokenMap = map[string]int{
 
 // aliases are strings directly map to another string and use the same token.
 var aliases = map[string]string{
-	"SCHEMA":  "DATABASE",
-	"SCHEMAS": "DATABASES",
-	"DEC":     "DECIMAL",
-	"SUBSTR":  "SUBSTRING",
+	"SCHEMA":    "DATABASE",
+	"SCHEMAS":   "DATABASES",
+	"DEC":       "DECIMAL",
+	"SUBSTR":    "SUBSTRING",
+	"TIDB_HJ":   "HASH_JOIN",
+	"TIDB_INLJ": "INL_JOIN",
+	"TIDB_SMJ":  "SM_JOIN",
 }
 
 func (s *Scanner) isTokenIdentifier(lit string, offset int) int {