From 22cb2475d3bededb67a708bd2a6a4df3d12d373c Mon Sep 17 00:00:00 2001 From: mmyj Date: Wed, 11 Nov 2020 00:25:38 +0800 Subject: [PATCH 1/3] try to improve the performance of appending not fixed columns --- util/chunk/chunk.go | 4 ++- util/chunk/chunk_test.go | 77 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/util/chunk/chunk.go b/util/chunk/chunk.go index ddd01ada18b97..7757eae8ac123 100644 --- a/util/chunk/chunk.go +++ b/util/chunk/chunk.go @@ -505,8 +505,10 @@ func (c *Chunk) Append(other *Chunk, begin, end int) { } else { beginOffset, endOffset := src.offsets[begin], src.offsets[end] dst.data = append(dst.data, src.data[beginOffset:endOffset]...) + lastOffset := dst.offsets[len(dst.offsets)-1] for i := begin; i < end; i++ { - dst.offsets = append(dst.offsets, dst.offsets[len(dst.offsets)-1]+src.offsets[i+1]-src.offsets[i]) + lastOffset += src.offsets[i+1] - src.offsets[i] + dst.offsets = append(dst.offsets, lastOffset) } } for i := begin; i < end; i++ { diff --git a/util/chunk/chunk_test.go b/util/chunk/chunk_test.go index 68ef2bad196f9..178d9b5960f36 100644 --- a/util/chunk/chunk_test.go +++ b/util/chunk/chunk_test.go @@ -1059,3 +1059,80 @@ func benchmarkChunkGrow(t benchChunkGrowCase) func(b *testing.B) { } } } + +func BenchmarkAppendRows(b *testing.B) { + b.ReportAllocs() + rowChk := newChunk(8, 8, 0, 0) + + for i := 0; i < 4096; i++ { + rowChk.AppendNull(0) + rowChk.AppendInt64(1, 1) + rowChk.AppendString(2, "abcd") + rowChk.AppendBytes(3, []byte("abcd")) + } + + type testCaseConf struct { + batchSize int + } + testCaseConfs := []testCaseConf{ + {batchSize: 2}, + {batchSize: 8}, + {batchSize: 16}, + {batchSize: 100}, + {batchSize: 1000}, + {batchSize: 4000}, + } + + chk := newChunk(8, 8, 0, 0) + for _, conf := range testCaseConfs { + b.ResetTimer() + b.Run(fmt.Sprintf("row-%d", conf.batchSize), func(b *testing.B) { + for i := 0; i < b.N; i++ { + chk.Reset() + for j := 0; j < conf.batchSize; j++ { + chk.AppendRow(rowChk.GetRow(j)) + } + } + }) + b.ResetTimer() + b.Run(fmt.Sprintf("column-%d", conf.batchSize), func(b *testing.B) { + for i := 0; i < b.N; i++ { + chk.Reset() + chk.Append(rowChk, 0, conf.batchSize) + } + }) + } +} + +func BenchmarkAppend(b *testing.B) { + b.ReportAllocs() + rowChk := newChunk(0, 0) + + for i := 0; i < 4096; i++ { + rowChk.AppendString(0, "abcd") + rowChk.AppendBytes(1, []byte("abcd")) + } + + type testCaseConf struct { + batchSize int + } + testCaseConfs := []testCaseConf{ + {batchSize: 2}, + {batchSize: 8}, + {batchSize: 16}, + {batchSize: 100}, + {batchSize: 1000}, + {batchSize: 4000}, + } + + chk := newChunk(0, 0) + for _, conf := range testCaseConfs { + b.ResetTimer() + b.Run(fmt.Sprintf("column-%d", conf.batchSize), func(b *testing.B) { + for i := 0; i < b.N; i++ { + chk.Reset() + chk.Append(rowChk, 0, conf.batchSize) + } + }) + } +} From 618f09a361595670dd1323c5c9a4e603761b8c0e Mon Sep 17 00:00:00 2001 From: mmyj Date: Wed, 11 Nov 2020 13:11:39 +0800 Subject: [PATCH 2/3] add BenchmarkIndexLookUpGetResultChunk --- executor/distsql_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/executor/distsql_test.go b/executor/distsql_test.go index ce2013a04ac2c..71083a53a7496 100644 --- a/executor/distsql_test.go +++ b/executor/distsql_test.go @@ -22,6 +22,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/parser/model" + "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/kv" @@ -255,3 +256,19 @@ func (s *testSuite3) TestPushLimitDownIndexLookUpReader(c *C) { tk.MustQuery("select * from tbl use index(idx_b_c) where b > 1 order by b desc limit 2,1").Check(testkit.Rows("3 3 3")) tk.MustQuery("select * from tbl use index(idx_b_c) where b > 1 and c > 1 limit 2,1").Check(testkit.Rows("4 4 4")) } + +func (s *testSuite3) BenchmarkIndexLookUpGetResultChunk(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists tbl") + tk.MustExec("create table tbl(a int, b int, c int, d text, key idx_b_c(b,c))") + for i := 0; i < 1000; i ++ { + tk.MustExec("insert into tbl values(1,1,1,'11111 11111'),(2,2,2,'22222 22222'),(3,3,3,'33333 33333'),(4,4,4,'444444 44444'),(5,5,5,'55555 55555')") + } + c.ResetTimer() + for i := 0; i < c.N; i++ { + tk.MustQuery("select * from tbl use index(idx_b_c) where b = 1") + tk.MustQuery("select * from tbl use index(idx_b_c) where b > 2") + tk.MustQuery("select * from tbl use index(idx_b_c) where b = 3 limit 10") + } +} From 30631bf26c0ae4aaacd98bec06744f46cec86d55 Mon Sep 17 00:00:00 2001 From: mmyj Date: Fri, 19 Feb 2021 13:25:13 +0800 Subject: [PATCH 3/3] fix --- executor/distsql_test.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/executor/distsql_test.go b/executor/distsql_test.go index df51d98a15c60..e8116d8a85c5f 100644 --- a/executor/distsql_test.go +++ b/executor/distsql_test.go @@ -283,19 +283,3 @@ func (s *testSuite3) TestIndexLookUpGetResultChunk(c *C) { tk.MustQuery("select * from tbl use index(idx_a) where a > 99 order by a asc limit 1").Check(testkit.Rows("100 100 100")) tk.MustQuery("select * from tbl use index(idx_a) where a > 10 order by a asc limit 4,1").Check(testkit.Rows("15 15 15")) } - -func (s *testSuite3) BenchmarkIndexLookUpGetResultChunk(c *C) { - tk := testkit.NewTestKit(c, s.store) - tk.MustExec("use test") - tk.MustExec("drop table if exists tbl") - tk.MustExec("create table tbl(a int, b int, c int, d text, key idx_b_c(b,c))") - for i := 0; i < 1000; i ++ { - tk.MustExec("insert into tbl values(1,1,1,'11111 11111'),(2,2,2,'22222 22222'),(3,3,3,'33333 33333'),(4,4,4,'444444 44444'),(5,5,5,'55555 55555')") - } - c.ResetTimer() - for i := 0; i < c.N; i++ { - tk.MustQuery("select * from tbl use index(idx_b_c) where b = 1") - tk.MustQuery("select * from tbl use index(idx_b_c) where b > 2") - tk.MustQuery("select * from tbl use index(idx_b_c) where b = 3 limit 10") - } -}