From 7ed7dcb9539179bfbfbe5bac025aaff817432d01 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Tue, 2 Feb 2021 16:03:04 +0800 Subject: [PATCH 01/10] planner, statistics: merge the partition-level stats to global-level stats --- executor/analyze.go | 54 +++++++++++++++++ planner/core/common_plans.go | 1 - statistics/handle/handle.go | 112 +++++++++++++++++++++++++++++++++++ statistics/table.go | 10 ++++ 4 files changed, 176 insertions(+), 1 deletion(-) diff --git a/executor/analyze.go b/executor/analyze.go index b9c799f55b72f..f1348656c31d5 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -96,6 +96,21 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { close(taskCh) statsHandle := domain.GetDomain(e.ctx).StatsHandle() panicCnt := 0 + + pruneMode := variable.PartitionPruneMode(e.ctx.GetSessionVars().PartitionPruneMode.Load()) + // needGlobalStats used to indicate whether we should merge the partition-level stats to global-level stats. + needGlobalStats := pruneMode == variable.DynamicOnly || pruneMode == variable.StaticButPrepareDynamic + type additionGlobalStatsInfo struct { + tableID int64 + isIndex int + idxID int64 + statsVersion int + } + // globalStatsMap is a map used to store which partition tables and the corresponding indexes need global-level stats. + // The meaning of key in map is the combination of tableID and indexID. + // The meaning of value in map is some additional information needed to build global-level stats. + globalStatsMap := make(map[string]additionGlobalStatsInfo) + for panicCnt < concurrency { result, ok := <-resultCh if !ok { @@ -113,6 +128,17 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { } statisticsID := result.TableID.GetStatisticsID() for i, hg := range result.Hist { + if result.TableID.IsPartitionTable() && needGlobalStats { + // If it does not belong to the statistics of index, we need to set it to -1 to distinguish. + idxID := int64(-1) + if result.IsIndex != 0 { + idxID = hg.ID + } + globalStatsKey := fmt.Sprintf("%d_%d", result.TableID.TableID, idxID) + if _, ok := globalStatsMap[globalStatsKey]; !ok { + globalStatsMap[globalStatsKey] = additionGlobalStatsInfo{result.TableID.TableID, result.IsIndex, hg.ID, result.StatsVer} + } + } err1 := statsHandle.SaveStatsToStorage(statisticsID, result.Count, result.IsIndex, hg, result.Cms[i], result.TopNs[i], result.StatsVer, 1) if err1 != nil { err = err1 @@ -135,6 +161,34 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { if err != nil { return err } + if needGlobalStats { + for _, info := range globalStatsMap { + globalStats, succ := statsHandle.MergePartitionStats2GlobalStats(infoschema.GetInfoSchema(e.ctx), info.tableID, info.isIndex, info.idxID) + if succ { + for i := 0; i < globalStats.Num; i++ { + hg := globalStats.Hg[i] + cms := globalStats.Cms[i] + topN := globalStats.TopN[i] + if hg == nil || cms == nil || topN == nil { + succ = false + break + } + err = statsHandle.SaveStatsToStorage(info.tableID, globalStats.Count, info.isIndex, hg, cms, topN, info.statsVersion, 1) + if err != nil { + logutil.Logger(ctx).Error("save global-level stats to storage failed", zap.Error(err)) + succ = false + } + } + } + if !succ { + if info.isIndex != 0 { + e.ctx.GetSessionVars().StmtCtx.AppendWarning(fmt.Errorf("build global-level statistics for table %d index %d failed", info.tableID, info.idxID)) + } else { + e.ctx.GetSessionVars().StmtCtx.AppendWarning(fmt.Errorf("build global-level statistics for table %d failed", info.tableID)) + } + } + } + } return statsHandle.Update(infoschema.GetInfoSchema(e.ctx)) } diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 820adb468284c..61e91bb112ad5 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -779,7 +779,6 @@ func (h *AnalyzeTableID) GetStatisticsID() int64 { } // IsPartitionTable indicates whether the table is partition table. -// for new partition implementation is TRUE but FALSE for old partition implementation func (h *AnalyzeTableID) IsPartitionTable() bool { return h.PartitionID != -1 } diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index fd1d06e73b0ae..da045e49e4342 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -221,6 +221,118 @@ func (h *Handle) Update(is infoschema.InfoSchema) error { return nil } +// GlobalStats is used to store the statistics contained in the global-level stats +// which is generated by the merge of partition-level stats. +// It will both store the column stats and index stats. +// In the column statistics, the variable `num` is equal to the number of columns in the partition table. +// In the index statistics, the variable `num` is always equal to one. +type GlobalStats struct { + Num int + Count int64 + Hg []*statistics.Histogram + Cms []*statistics.CMSketch + TopN []*statistics.TopN +} + +// MergePartitionStats2GlobalStats merge the partition-level stats to global-level stats based on the tableID. +func (h *Handle) MergePartitionStats2GlobalStats(is infoschema.InfoSchema, physicalID int64, isIndex int, idxID int64) (globalStats *GlobalStats, succ bool) { + // get the partition table IDs + h.mu.Lock() + globalTable, ok := h.getTableByPhysicalID(is, physicalID) + h.mu.Unlock() + if !ok { + logutil.BgLogger().Debug("unknown physical ID in stats meta table, maybe it has been dropped", zap.Int64("ID", physicalID)) + return + } + globalTableInfo := globalTable.Meta() + partitionNum := globalTableInfo.Partition.Num + partitionIDs := make([]int64, 0, partitionNum) + for i := uint64(0); i < partitionNum; i++ { + partitionIDs = append(partitionIDs, globalTableInfo.Partition.Definitions[i].ID) + } + + // initialized the globalStats + globalStats = new(GlobalStats) + if isIndex == 0 { + globalStats.Num = len(globalTableInfo.Columns) + } else { + globalStats.Num = 1 + } + globalStats.Count = 0 + globalStats.Hg = make([]*statistics.Histogram, globalStats.Num) + globalStats.Cms = make([]*statistics.CMSketch, globalStats.Num) + globalStats.TopN = make([]*statistics.TopN, globalStats.Num) + + // The first dimension of slice is means the number of stats in the globalStats. + // The second dimension of slice is means the number of partition tables. + // We should store all of the partition-level stats first, and merge them together. + allHg := make([][]*statistics.Histogram, globalStats.Num) + allCms := make([][]*statistics.CMSketch, globalStats.Num) + allTopN := make([][]*statistics.TopN, globalStats.Num) + for i := 0; i < globalStats.Num; i++ { + allHg[i] = make([]*statistics.Histogram, 0, partitionNum) + allCms[i] = make([]*statistics.CMSketch, 0, partitionNum) + allTopN[i] = make([]*statistics.TopN, 0, partitionNum) + } + + for _, partitionID := range partitionIDs { + h.mu.Lock() + partitionTable, ok := h.getTableByPhysicalID(is, partitionID) + h.mu.Unlock() + if !ok { + logutil.BgLogger().Debug("unknown physical ID in stats meta table, maybe it has been dropped", zap.Int64("ID", partitionID)) + return + } + tableInfo := partitionTable.Meta() + partitionStats, err := h.TableStatsFromStorage(tableInfo, partitionID, false, nil) + // Error is not nil may mean that there are some ddl changes on this table, we will not update it. + if err != nil { + logutil.BgLogger().Error("[stats] error occurred when read table stats", zap.String("table", tableInfo.Name.O), zap.Error(err)) + return + } + if partitionStats == nil { + return + } + globalStats.Count += partitionStats.Count + for i := 0; i < globalStats.Num; i++ { + ID := tableInfo.Columns[i].ID + if isIndex != 0 { + // If the statistics is the index stats, we should use the index ID to replace the column ID. + ID = idxID + } + hg, cms, topN := partitionStats.GetStatsInfo(ID, isIndex) + allHg[i] = append(allHg[i], hg) + allCms[i] = append(allCms[i], cms) + allTopN[i] = append(allTopN[i], topN) + } + } + + // After collect all of the statistics from the partition-level stats, + // we should merge them together. + for i := 0; i < globalStats.Num; i++ { + // Merge CMSketch + globalStats.Cms[i] = allCms[i][0].Copy() + for j := uint64(1); j < partitionNum; j++ { + err := globalStats.Cms[i].MergeCMSketch(allCms[i][j]) + if err != nil { + logutil.BgLogger().Debug("failed to merge the CMSketch when we merge the partition-level stats to global-level stats") + return + } + } + + // Merge topN. We need to merge TopN before merging the histogram. + // Because after merging TopN, some numbers will be left. + // These left numbers should be inserted into the histogram. + + // Merge histogram + + // TODO: Merge NDV + } + // Build the global-level stats success. + succ = true + return +} + func (h *Handle) getTableByPhysicalID(is infoschema.InfoSchema, physicalID int64) (table.Table, bool) { if is.SchemaMetaVersion() != h.mu.schemaVersion { h.mu.schemaVersion = is.SchemaMetaVersion() diff --git a/statistics/table.go b/statistics/table.go index a48346e1a7cde..8170bfe18d934 100644 --- a/statistics/table.go +++ b/statistics/table.go @@ -197,6 +197,16 @@ func (t *Table) ColumnByName(colName string) *Column { return nil } +// GetStatsInfo returns their statistics according to the ID of the column or index, including histogram, CMSketch and TopN. +func (t *Table) GetStatsInfo(ID int64, isIndex int) (*Histogram, *CMSketch, *TopN) { + if isIndex == 0 { + colStatsInfo := t.Columns[ID] + return colStatsInfo.Histogram.Copy(), colStatsInfo.CMSketch.Copy(), colStatsInfo.TopN.Copy() + } + idxStatsInfo := t.Indices[ID] + return idxStatsInfo.Histogram.Copy(), idxStatsInfo.CMSketch.Copy(), idxStatsInfo.TopN.Copy() +} + type tableColumnID struct { TableID int64 ColumnID int64 From c934224b82786148dff569ecdca415ae25816131 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Wed, 3 Feb 2021 12:00:54 +0800 Subject: [PATCH 02/10] address comments --- executor/analyze.go | 39 ++++++++++++++++++------------------- statistics/handle/handle.go | 2 +- statistics/table.go | 12 ++++++------ 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index f1348656c31d5..d5ef9c46ead61 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -100,16 +100,21 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { pruneMode := variable.PartitionPruneMode(e.ctx.GetSessionVars().PartitionPruneMode.Load()) // needGlobalStats used to indicate whether we should merge the partition-level stats to global-level stats. needGlobalStats := pruneMode == variable.DynamicOnly || pruneMode == variable.StaticButPrepareDynamic - type additionGlobalStatsInfo struct { - tableID int64 - isIndex int + type globalStatsKey struct { + tableID int64 + indexID int64 + } + type globalStatsInfo struct { + isIndex int + // When the `isIndex == 0`, the idxID will be the column ID. + // Otherwise, the idxID will be the index ID. idxID int64 statsVersion int } // globalStatsMap is a map used to store which partition tables and the corresponding indexes need global-level stats. - // The meaning of key in map is the combination of tableID and indexID. + // The meaning of key in map is the structure that used to store the tableID and indexID. // The meaning of value in map is some additional information needed to build global-level stats. - globalStatsMap := make(map[string]additionGlobalStatsInfo) + globalStatsMap := make(map[globalStatsKey]globalStatsInfo) for panicCnt < concurrency { result, ok := <-resultCh @@ -134,9 +139,9 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { if result.IsIndex != 0 { idxID = hg.ID } - globalStatsKey := fmt.Sprintf("%d_%d", result.TableID.TableID, idxID) - if _, ok := globalStatsMap[globalStatsKey]; !ok { - globalStatsMap[globalStatsKey] = additionGlobalStatsInfo{result.TableID.TableID, result.IsIndex, hg.ID, result.StatsVer} + globalStatsID := globalStatsKey{result.TableID.TableID, idxID} + if _, ok := globalStatsMap[globalStatsID]; !ok { + globalStatsMap[globalStatsID] = globalStatsInfo{result.IsIndex, hg.ID, result.StatsVer} } } err1 := statsHandle.SaveStatsToStorage(statisticsID, result.Count, result.IsIndex, hg, result.Cms[i], result.TopNs[i], result.StatsVer, 1) @@ -162,18 +167,12 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { return err } if needGlobalStats { - for _, info := range globalStatsMap { - globalStats, succ := statsHandle.MergePartitionStats2GlobalStats(infoschema.GetInfoSchema(e.ctx), info.tableID, info.isIndex, info.idxID) + for globalStatsID, info := range globalStatsMap { + globalStats, succ := statsHandle.MergePartitionStats2GlobalStats(infoschema.GetInfoSchema(e.ctx), globalStatsID.tableID, info.isIndex, info.idxID) if succ { for i := 0; i < globalStats.Num; i++ { - hg := globalStats.Hg[i] - cms := globalStats.Cms[i] - topN := globalStats.TopN[i] - if hg == nil || cms == nil || topN == nil { - succ = false - break - } - err = statsHandle.SaveStatsToStorage(info.tableID, globalStats.Count, info.isIndex, hg, cms, topN, info.statsVersion, 1) + hg, cms, topN := globalStats.Hg[i], globalStats.Cms[i], globalStats.TopN[i] + err = statsHandle.SaveStatsToStorage(globalStatsID.tableID, globalStats.Count, info.isIndex, hg, cms, topN, info.statsVersion, 1) if err != nil { logutil.Logger(ctx).Error("save global-level stats to storage failed", zap.Error(err)) succ = false @@ -182,9 +181,9 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { } if !succ { if info.isIndex != 0 { - e.ctx.GetSessionVars().StmtCtx.AppendWarning(fmt.Errorf("build global-level statistics for table %d index %d failed", info.tableID, info.idxID)) + e.ctx.GetSessionVars().StmtCtx.AppendWarning(fmt.Errorf("build global-level statistics for table %d index %d failed", globalStatsID.tableID, info.idxID)) } else { - e.ctx.GetSessionVars().StmtCtx.AppendWarning(fmt.Errorf("build global-level statistics for table %d failed", info.tableID)) + e.ctx.GetSessionVars().StmtCtx.AppendWarning(fmt.Errorf("build global-level statistics for table %d failed", globalStatsID.tableID)) } } } diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index da045e49e4342..45de5202f3370 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -300,7 +300,7 @@ func (h *Handle) MergePartitionStats2GlobalStats(is infoschema.InfoSchema, physi // If the statistics is the index stats, we should use the index ID to replace the column ID. ID = idxID } - hg, cms, topN := partitionStats.GetStatsInfo(ID, isIndex) + hg, cms, topN := partitionStats.GetStatsInfo(ID, isIndex == 1) allHg[i] = append(allHg[i], hg) allCms[i] = append(allCms[i], cms) allTopN[i] = append(allTopN[i], topN) diff --git a/statistics/table.go b/statistics/table.go index 8170bfe18d934..9469b32b3be42 100644 --- a/statistics/table.go +++ b/statistics/table.go @@ -198,13 +198,13 @@ func (t *Table) ColumnByName(colName string) *Column { } // GetStatsInfo returns their statistics according to the ID of the column or index, including histogram, CMSketch and TopN. -func (t *Table) GetStatsInfo(ID int64, isIndex int) (*Histogram, *CMSketch, *TopN) { - if isIndex == 0 { - colStatsInfo := t.Columns[ID] - return colStatsInfo.Histogram.Copy(), colStatsInfo.CMSketch.Copy(), colStatsInfo.TopN.Copy() +func (t *Table) GetStatsInfo(ID int64, isIndex bool) (*Histogram, *CMSketch, *TopN) { + if isIndex { + idxStatsInfo := t.Indices[ID] + return idxStatsInfo.Histogram.Copy(), idxStatsInfo.CMSketch.Copy(), idxStatsInfo.TopN.Copy() } - idxStatsInfo := t.Indices[ID] - return idxStatsInfo.Histogram.Copy(), idxStatsInfo.CMSketch.Copy(), idxStatsInfo.TopN.Copy() + colStatsInfo := t.Columns[ID] + return colStatsInfo.Histogram.Copy(), colStatsInfo.CMSketch.Copy(), colStatsInfo.TopN.Copy() } type tableColumnID struct { From 72325cacd95f9774e8af3e96c177e2c9e62139a0 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Wed, 3 Feb 2021 14:59:05 +0800 Subject: [PATCH 03/10] address comments --- executor/analyze.go | 23 ++++++++--------------- statistics/handle/handle.go | 23 +++++++++++------------ 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index d5ef9c46ead61..eda5ba039cc59 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -168,22 +168,15 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { } if needGlobalStats { for globalStatsID, info := range globalStatsMap { - globalStats, succ := statsHandle.MergePartitionStats2GlobalStats(infoschema.GetInfoSchema(e.ctx), globalStatsID.tableID, info.isIndex, info.idxID) - if succ { - for i := 0; i < globalStats.Num; i++ { - hg, cms, topN := globalStats.Hg[i], globalStats.Cms[i], globalStats.TopN[i] - err = statsHandle.SaveStatsToStorage(globalStatsID.tableID, globalStats.Count, info.isIndex, hg, cms, topN, info.statsVersion, 1) - if err != nil { - logutil.Logger(ctx).Error("save global-level stats to storage failed", zap.Error(err)) - succ = false - } - } + globalStats, err := statsHandle.MergePartitionStats2GlobalStats(infoschema.GetInfoSchema(e.ctx), globalStatsID.tableID, info.isIndex, info.idxID) + if err != nil { + return err } - if !succ { - if info.isIndex != 0 { - e.ctx.GetSessionVars().StmtCtx.AppendWarning(fmt.Errorf("build global-level statistics for table %d index %d failed", globalStatsID.tableID, info.idxID)) - } else { - e.ctx.GetSessionVars().StmtCtx.AppendWarning(fmt.Errorf("build global-level statistics for table %d failed", globalStatsID.tableID)) + for i := 0; i < globalStats.Num; i++ { + hg, cms, topN := globalStats.Hg[i], globalStats.Cms[i], globalStats.TopN[i] + err = statsHandle.SaveStatsToStorage(globalStatsID.tableID, globalStats.Count, info.isIndex, hg, cms, topN, info.statsVersion, 1) + if err != nil { + return err } } } diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 45de5202f3370..89e8c180dcff3 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -235,13 +235,13 @@ type GlobalStats struct { } // MergePartitionStats2GlobalStats merge the partition-level stats to global-level stats based on the tableID. -func (h *Handle) MergePartitionStats2GlobalStats(is infoschema.InfoSchema, physicalID int64, isIndex int, idxID int64) (globalStats *GlobalStats, succ bool) { +func (h *Handle) MergePartitionStats2GlobalStats(is infoschema.InfoSchema, physicalID int64, isIndex int, idxID int64) (globalStats *GlobalStats, err error) { // get the partition table IDs h.mu.Lock() globalTable, ok := h.getTableByPhysicalID(is, physicalID) h.mu.Unlock() if !ok { - logutil.BgLogger().Debug("unknown physical ID in stats meta table, maybe it has been dropped", zap.Int64("ID", physicalID)) + err = errors.Errorf("unknown physical ID in stats meta table, maybe it has been dropped", zap.Int64("ID", physicalID)) return } globalTableInfo := globalTable.Meta() @@ -263,9 +263,10 @@ func (h *Handle) MergePartitionStats2GlobalStats(is infoschema.InfoSchema, physi globalStats.Cms = make([]*statistics.CMSketch, globalStats.Num) globalStats.TopN = make([]*statistics.TopN, globalStats.Num) - // The first dimension of slice is means the number of stats in the globalStats. + // The first dimension of slice is means the number of column or index stats in the globalStats. // The second dimension of slice is means the number of partition tables. - // We should store all of the partition-level stats first, and merge them together. + // Because all topN and histograms need to be collected before they can be merged. + // So we should store all of the partition-level stats first, and merge them together. allHg := make([][]*statistics.Histogram, globalStats.Num) allCms := make([][]*statistics.CMSketch, globalStats.Num) allTopN := make([][]*statistics.TopN, globalStats.Num) @@ -280,17 +281,18 @@ func (h *Handle) MergePartitionStats2GlobalStats(is infoschema.InfoSchema, physi partitionTable, ok := h.getTableByPhysicalID(is, partitionID) h.mu.Unlock() if !ok { - logutil.BgLogger().Debug("unknown physical ID in stats meta table, maybe it has been dropped", zap.Int64("ID", partitionID)) + err = errors.Errorf("unknown physical ID in stats meta table, maybe it has been dropped", zap.Int64("ID", partitionID)) return } tableInfo := partitionTable.Meta() - partitionStats, err := h.TableStatsFromStorage(tableInfo, partitionID, false, nil) - // Error is not nil may mean that there are some ddl changes on this table, we will not update it. + var partitionStats *statistics.Table + partitionStats, err = h.TableStatsFromStorage(tableInfo, partitionID, false, nil) + // Error is not nil may mean that there are some ddl changes on this table, we will not use it. if err != nil { - logutil.BgLogger().Error("[stats] error occurred when read table stats", zap.String("table", tableInfo.Name.O), zap.Error(err)) return } if partitionStats == nil { + err = errors.Errorf("[stats] error occurred when read partition table stats", zap.Int64("ID", partitionID)) return } globalStats.Count += partitionStats.Count @@ -313,9 +315,8 @@ func (h *Handle) MergePartitionStats2GlobalStats(is infoschema.InfoSchema, physi // Merge CMSketch globalStats.Cms[i] = allCms[i][0].Copy() for j := uint64(1); j < partitionNum; j++ { - err := globalStats.Cms[i].MergeCMSketch(allCms[i][j]) + err = globalStats.Cms[i].MergeCMSketch(allCms[i][j]) if err != nil { - logutil.BgLogger().Debug("failed to merge the CMSketch when we merge the partition-level stats to global-level stats") return } } @@ -328,8 +329,6 @@ func (h *Handle) MergePartitionStats2GlobalStats(is infoschema.InfoSchema, physi // TODO: Merge NDV } - // Build the global-level stats success. - succ = true return } From edbab927718794b75281e0b3683b643ed9e16f9c Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Wed, 3 Feb 2021 15:13:04 +0800 Subject: [PATCH 04/10] update the error message --- statistics/handle/handle.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 89e8c180dcff3..5f1f65df57bef 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -241,7 +241,7 @@ func (h *Handle) MergePartitionStats2GlobalStats(is infoschema.InfoSchema, physi globalTable, ok := h.getTableByPhysicalID(is, physicalID) h.mu.Unlock() if !ok { - err = errors.Errorf("unknown physical ID in stats meta table, maybe it has been dropped", zap.Int64("ID", physicalID)) + err = errors.Errorf("unknown physical ID %d in stats meta table, maybe it has been dropped", physicalID) return } globalTableInfo := globalTable.Meta() @@ -281,18 +281,17 @@ func (h *Handle) MergePartitionStats2GlobalStats(is infoschema.InfoSchema, physi partitionTable, ok := h.getTableByPhysicalID(is, partitionID) h.mu.Unlock() if !ok { - err = errors.Errorf("unknown physical ID in stats meta table, maybe it has been dropped", zap.Int64("ID", partitionID)) + err = errors.Errorf("unknown physical ID %d in stats meta table, maybe it has been dropped", partitionID) return } tableInfo := partitionTable.Meta() var partitionStats *statistics.Table partitionStats, err = h.TableStatsFromStorage(tableInfo, partitionID, false, nil) - // Error is not nil may mean that there are some ddl changes on this table, we will not use it. if err != nil { return } if partitionStats == nil { - err = errors.Errorf("[stats] error occurred when read partition table stats", zap.Int64("ID", partitionID)) + err = errors.Errorf("[stats] error occurred when read partition-level stats of the table with tableID %d and partitionID %d", physicalID, partitionID) return } globalStats.Count += partitionStats.Count @@ -327,7 +326,6 @@ func (h *Handle) MergePartitionStats2GlobalStats(is infoschema.InfoSchema, physi // Merge histogram - // TODO: Merge NDV } return } From d36c5d9a6da3eb73ed7a06d8f6b9a3c01050bf86 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Wed, 3 Feb 2021 15:24:36 +0800 Subject: [PATCH 05/10] update the error message --- statistics/handle/handle.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index ce53f922755dc..ba334008f2019 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -357,9 +357,16 @@ func (h *Handle) MergePartitionStats2GlobalStats(is infoschema.InfoSchema, physi // Merge topN. We need to merge TopN before merging the histogram. // Because after merging TopN, some numbers will be left. // These left numbers should be inserted into the histogram. + err = errors.Errorf("TODO: The merge function of the topN structure has not been implemented yet") + if err != nil { + return + } // Merge histogram - + err = errors.Errorf("TODO: The merge function of the histogram structure has not been implemented yet") + if err != nil { + return + } } return } From 8dd83b134e1ea48034fbe0160a2a9476e882ea7d Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Wed, 3 Feb 2021 15:26:48 +0800 Subject: [PATCH 06/10] fix UT --- statistics/handle/handle.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index ba334008f2019..8f9624e285757 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -320,7 +320,7 @@ func (h *Handle) MergePartitionStats2GlobalStats(is infoschema.InfoSchema, physi } tableInfo := partitionTable.Meta() var partitionStats *statistics.Table - partitionStats, err = h.TableStatsFromStorage(tableInfo, partitionID, false, nil) + partitionStats, err = h.TableStatsFromStorage(tableInfo, partitionID, false, 0) if err != nil { return } From 5f5265e2f19a977b1990cef9f972e9117870fd91 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Thu, 4 Feb 2021 16:34:00 +0800 Subject: [PATCH 07/10] add the test for build global-level stats --- executor/analyze.go | 5 ++-- statistics/handle/handle_test.go | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index eda5ba039cc59..d93fbc6ce2026 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -170,13 +170,14 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { for globalStatsID, info := range globalStatsMap { globalStats, err := statsHandle.MergePartitionStats2GlobalStats(infoschema.GetInfoSchema(e.ctx), globalStatsID.tableID, info.isIndex, info.idxID) if err != nil { - return err + logutil.Logger(ctx).Error("merge partition-level stats to global-level stats failed", zap.Error(err)) + continue } for i := 0; i < globalStats.Num; i++ { hg, cms, topN := globalStats.Hg[i], globalStats.Cms[i], globalStats.TopN[i] err = statsHandle.SaveStatsToStorage(globalStatsID.tableID, globalStats.Count, info.isIndex, hg, cms, topN, info.statsVersion, 1) if err != nil { - return err + logutil.Logger(ctx).Error("save global-level stats to storage failed", zap.Error(err)) } } } diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index a319b2f0f6426..6efb69b0cff92 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -676,6 +676,49 @@ func (s *testStatsSuite) TestCorrelation(c *C) { c.Assert(result.Rows()[0][9], Equals, "0") } +func (s *testStatsSuite) TestBuildGlobalLevelStats(c *C) { + defer cleanEnv(c, s.store, s.do) + testKit := testkit.NewTestKit(c, s.store) + testKit.MustExec("use test") + testKit.MustExec("drop table if exists t, t1;") + testKit.MustExec("set @@tidb_partition_prune_mode = 'static-only';") + testKit.MustExec("create table t(a int, b int, c int) PARTITION BY HASH(a) PARTITIONS 3;") + testKit.MustExec("create table t1(a int) PARTITION BY HASH(a) PARTITIONS 2;") + testKit.MustExec("insert into t values(1,1,1),(3,12,3),(4,20,4),(2,7,2),(5,21,5);") + testKit.MustExec("insert into t1 values(1),(3),(4),(2),(5);") + testKit.MustExec("create index idx_t_ab on t(a, b);") + testKit.MustExec("create index idx_t_b on t(b);") + testKit.MustExec("analyze table t;") + result := testKit.MustQuery("show stats_meta where table_name = 't';").Sort() + c.Assert(len(result.Rows()), Equals, 3) + c.Assert(result.Rows()[0][5], Equals, "1") + c.Assert(result.Rows()[1][5], Equals, "2") + c.Assert(result.Rows()[2][5], Equals, "2") + + result = testKit.MustQuery("show stats_meta where table_name = 't1';").Sort() + c.Assert(len(result.Rows()), Equals, 0) + + testKit.MustExec("set @@tidb_partition_prune_mode = 'dynamic-only';") + testKit.MustExec("analyze table t, t1;") + result = testKit.MustQuery("show stats_meta where table_name = 't'").Sort() + c.Assert(len(result.Rows()), Equals, 3) + c.Assert(result.Rows()[0][5], Equals, "1") + c.Assert(result.Rows()[1][5], Equals, "2") + c.Assert(result.Rows()[2][5], Equals, "2") + + result = testKit.MustQuery("show stats_meta where table_name = 't1';").Sort() + c.Assert(len(result.Rows()), Equals, 2) + c.Assert(result.Rows()[0][5], Equals, "3") + c.Assert(result.Rows()[1][5], Equals, "2") + + testKit.MustExec("analyze table t index idx_t_ab, idx_t_b;") + result = testKit.MustQuery("show stats_meta where table_name = 't'").Sort() + c.Assert(len(result.Rows()), Equals, 3) + c.Assert(result.Rows()[0][5], Equals, "1") + c.Assert(result.Rows()[1][5], Equals, "2") + c.Assert(result.Rows()[2][5], Equals, "2") +} + func (s *testStatsSuite) TestExtendedStatsDefaultSwitch(c *C) { defer cleanEnv(c, s.store, s.do) tk := testkit.NewTestKit(c, s.store) From d59d5b5de92a6268365dc2092c757d61e5609fe4 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Thu, 4 Feb 2021 16:44:39 +0800 Subject: [PATCH 08/10] fix UT --- statistics/handle/handle.go | 159 ++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 98858c853ba93..986da8a00f013 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -115,6 +115,16 @@ func (h *Handle) execRestrictedSQL(ctx context.Context, sql string, params ...in }) } +func (h *Handle) execRestrictedSQLWithStatsVer(ctx context.Context, statsVer int, sql string, params ...interface{}) ([]chunk.Row, []*ast.ResultField, error) { + return h.withRestrictedSQLExecutor(ctx, func(ctx context.Context, exec sqlexec.RestrictedSQLExecutor) ([]chunk.Row, []*ast.ResultField, error) { + stmt, err := exec.ParseWithParams(ctx, sql, params...) + if err != nil { + return nil, nil, errors.Trace(err) + } + return exec.ExecRestrictedStmt(ctx, stmt, execOptionForAnalyze[statsVer]) + }) +} + func (h *Handle) execRestrictedSQLWithSnapshot(ctx context.Context, sql string, snapshot uint64, params ...interface{}) ([]chunk.Row, []*ast.ResultField, error) { return h.withRestrictedSQLExecutor(ctx, func(ctx context.Context, exec sqlexec.RestrictedSQLExecutor) ([]chunk.Row, []*ast.ResultField, error) { stmt, err := exec.ParseWithParams(ctx, sql, params...) @@ -255,6 +265,138 @@ func (h *Handle) Update(is infoschema.InfoSchema) error { return nil } +// UpdateSessionVar updates the necessary session variables for the stats reader. +func (h *Handle) UpdateSessionVar() error { + verInString, err := h.mu.ctx.GetSessionVars().GlobalVarsAccessor.GetGlobalSysVar(variable.TiDBAnalyzeVersion) + if err != nil { + return err + } + h.mu.Lock() + defer h.mu.Unlock() + ver, err := strconv.ParseInt(verInString, 10, 64) + if err != nil { + return err + } + h.mu.ctx.GetSessionVars().AnalyzeVersion = int(ver) + return err +} + +// GlobalStats is used to store the statistics contained in the global-level stats +// which is generated by the merge of partition-level stats. +// It will both store the column stats and index stats. +// In the column statistics, the variable `num` is equal to the number of columns in the partition table. +// In the index statistics, the variable `num` is always equal to one. +type GlobalStats struct { + Num int + Count int64 + Hg []*statistics.Histogram + Cms []*statistics.CMSketch + TopN []*statistics.TopN +} + +// MergePartitionStats2GlobalStats merge the partition-level stats to global-level stats based on the tableID. +func (h *Handle) MergePartitionStats2GlobalStats(is infoschema.InfoSchema, physicalID int64, isIndex int, idxID int64) (globalStats *GlobalStats, err error) { + // get the partition table IDs + h.mu.Lock() + globalTable, ok := h.getTableByPhysicalID(is, physicalID) + h.mu.Unlock() + if !ok { + err = errors.Errorf("unknown physical ID %d in stats meta table, maybe it has been dropped", physicalID) + return + } + globalTableInfo := globalTable.Meta() + partitionNum := globalTableInfo.Partition.Num + partitionIDs := make([]int64, 0, partitionNum) + for i := uint64(0); i < partitionNum; i++ { + partitionIDs = append(partitionIDs, globalTableInfo.Partition.Definitions[i].ID) + } + + // initialized the globalStats + globalStats = new(GlobalStats) + if isIndex == 0 { + globalStats.Num = len(globalTableInfo.Columns) + } else { + globalStats.Num = 1 + } + globalStats.Count = 0 + globalStats.Hg = make([]*statistics.Histogram, globalStats.Num) + globalStats.Cms = make([]*statistics.CMSketch, globalStats.Num) + globalStats.TopN = make([]*statistics.TopN, globalStats.Num) + + // The first dimension of slice is means the number of column or index stats in the globalStats. + // The second dimension of slice is means the number of partition tables. + // Because all topN and histograms need to be collected before they can be merged. + // So we should store all of the partition-level stats first, and merge them together. + allHg := make([][]*statistics.Histogram, globalStats.Num) + allCms := make([][]*statistics.CMSketch, globalStats.Num) + allTopN := make([][]*statistics.TopN, globalStats.Num) + for i := 0; i < globalStats.Num; i++ { + allHg[i] = make([]*statistics.Histogram, 0, partitionNum) + allCms[i] = make([]*statistics.CMSketch, 0, partitionNum) + allTopN[i] = make([]*statistics.TopN, 0, partitionNum) + } + + for _, partitionID := range partitionIDs { + h.mu.Lock() + partitionTable, ok := h.getTableByPhysicalID(is, partitionID) + h.mu.Unlock() + if !ok { + err = errors.Errorf("unknown physical ID %d in stats meta table, maybe it has been dropped", partitionID) + return + } + tableInfo := partitionTable.Meta() + var partitionStats *statistics.Table + partitionStats, err = h.TableStatsFromStorage(tableInfo, partitionID, false, 0) + if err != nil { + return + } + if partitionStats == nil { + err = errors.Errorf("[stats] error occurred when read partition-level stats of the table with tableID %d and partitionID %d", physicalID, partitionID) + return + } + globalStats.Count += partitionStats.Count + for i := 0; i < globalStats.Num; i++ { + ID := tableInfo.Columns[i].ID + if isIndex != 0 { + // If the statistics is the index stats, we should use the index ID to replace the column ID. + ID = idxID + } + hg, cms, topN := partitionStats.GetStatsInfo(ID, isIndex == 1) + allHg[i] = append(allHg[i], hg) + allCms[i] = append(allCms[i], cms) + allTopN[i] = append(allTopN[i], topN) + } + } + + // After collect all of the statistics from the partition-level stats, + // we should merge them together. + for i := 0; i < globalStats.Num; i++ { + // Merge CMSketch + globalStats.Cms[i] = allCms[i][0].Copy() + for j := uint64(1); j < partitionNum; j++ { + err = globalStats.Cms[i].MergeCMSketch(allCms[i][j]) + if err != nil { + return + } + } + + // Merge topN. We need to merge TopN before merging the histogram. + // Because after merging TopN, some numbers will be left. + // These left numbers should be inserted into the histogram. + err = errors.Errorf("TODO: The merge function of the topN structure has not been implemented yet") + if err != nil { + return + } + + // Merge histogram + err = errors.Errorf("TODO: The merge function of the histogram structure has not been implemented yet") + if err != nil { + return + } + } + return +} + func (h *Handle) getTableByPhysicalID(is infoschema.InfoSchema, physicalID int64) (table.Table, bool) { if is.SchemaMetaVersion() != h.mu.schemaVersion { h.mu.schemaVersion = is.SchemaMetaVersion() @@ -541,6 +683,7 @@ func (h *Handle) columnStatsFromStorage(reader *statsReader, row chunk.Row, tabl ErrorRate: errorRate, IsHandle: tableInfo.PKIsHandle && mysql.HasPriKeyFlag(colInfo.Flag), Flag: flag, + StatsVer: statsVer, } lastAnalyzePos.Copy(&col.LastAnalyzePos) col.Histogram.Correlation = correlation @@ -1205,3 +1348,19 @@ func (h *Handle) RefreshVars() error { defer h.mu.Unlock() return h.mu.ctx.RefreshVars(context.Background()) } + +// CheckAnalyzeVersion checks whether all the statistics versions of this table's columns and indexes are the same. +func (h *Handle) CheckAnalyzeVersion(tblInfo *model.TableInfo, physicalIDs []int64, version *int) bool { + // We simply choose one physical id to get its stats. + var tbl *statistics.Table + for _, pid := range physicalIDs { + tbl = h.GetPartitionStats(tblInfo, pid) + if !tbl.Pseudo { + break + } + } + if tbl == nil || tbl.Pseudo { + return true + } + return statistics.CheckAnalyzeVerOnTable(tbl, version) +} From c2008889d04796db21a3501d85fd07539d86408f Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Thu, 4 Feb 2021 17:55:05 +0800 Subject: [PATCH 09/10] update the test --- executor/analyze.go | 3 +-- statistics/handle/handle_test.go | 24 +++++++++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index 6405d8a1d78e7..a25cca395c9ac 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -170,8 +170,7 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { for globalStatsID, info := range globalStatsMap { globalStats, err := statsHandle.MergePartitionStats2GlobalStats(infoschema.GetInfoSchema(e.ctx), globalStatsID.tableID, info.isIndex, info.idxID) if err != nil { - logutil.Logger(ctx).Error("merge partition-level stats to global-level stats failed", zap.Error(err)) - continue + return err } for i := 0; i < globalStats.Num; i++ { hg, cms, topN := globalStats.Hg[i], globalStats.Cms[i], globalStats.TopN[i] diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 6efb69b0cff92..6467a2d93f654 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -683,40 +683,54 @@ func (s *testStatsSuite) TestBuildGlobalLevelStats(c *C) { testKit.MustExec("drop table if exists t, t1;") testKit.MustExec("set @@tidb_partition_prune_mode = 'static-only';") testKit.MustExec("create table t(a int, b int, c int) PARTITION BY HASH(a) PARTITIONS 3;") - testKit.MustExec("create table t1(a int) PARTITION BY HASH(a) PARTITIONS 2;") + testKit.MustExec("create table t1(a int);") testKit.MustExec("insert into t values(1,1,1),(3,12,3),(4,20,4),(2,7,2),(5,21,5);") testKit.MustExec("insert into t1 values(1),(3),(4),(2),(5);") testKit.MustExec("create index idx_t_ab on t(a, b);") testKit.MustExec("create index idx_t_b on t(b);") - testKit.MustExec("analyze table t;") + testKit.MustExec("analyze table t, t1;") result := testKit.MustQuery("show stats_meta where table_name = 't';").Sort() c.Assert(len(result.Rows()), Equals, 3) c.Assert(result.Rows()[0][5], Equals, "1") c.Assert(result.Rows()[1][5], Equals, "2") c.Assert(result.Rows()[2][5], Equals, "2") + result = testKit.MustQuery("show stats_histograms where table_name = 't';").Sort() + c.Assert(len(result.Rows()), Equals, 15) result = testKit.MustQuery("show stats_meta where table_name = 't1';").Sort() - c.Assert(len(result.Rows()), Equals, 0) + c.Assert(len(result.Rows()), Equals, 1) + c.Assert(result.Rows()[0][5], Equals, "5") + result = testKit.MustQuery("show stats_histograms where table_name = 't1';").Sort() + c.Assert(len(result.Rows()), Equals, 1) + // Test the 'dynamic-only' mode testKit.MustExec("set @@tidb_partition_prune_mode = 'dynamic-only';") - testKit.MustExec("analyze table t, t1;") + err := testKit.ExecToErr("analyze table t, t1;") + c.Assert(err.Error(), Equals, "TODO: The merge function of the topN structure has not been implemented yet") result = testKit.MustQuery("show stats_meta where table_name = 't'").Sort() c.Assert(len(result.Rows()), Equals, 3) c.Assert(result.Rows()[0][5], Equals, "1") c.Assert(result.Rows()[1][5], Equals, "2") c.Assert(result.Rows()[2][5], Equals, "2") + result = testKit.MustQuery("show stats_histograms where table_name = 't';").Sort() + c.Assert(len(result.Rows()), Equals, 15) result = testKit.MustQuery("show stats_meta where table_name = 't1';").Sort() c.Assert(len(result.Rows()), Equals, 2) c.Assert(result.Rows()[0][5], Equals, "3") c.Assert(result.Rows()[1][5], Equals, "2") + result = testKit.MustQuery("show stats_histograms where table_name = 't1';").Sort() + c.Assert(len(result.Rows()), Equals, 1) - testKit.MustExec("analyze table t index idx_t_ab, idx_t_b;") + err = testKit.ExecToErr("analyze table t index idx_t_ab, idx_t_b;") + c.Assert(err.Error(), Equals, "TODO: The merge function of the topN structure has not been implemented yet") result = testKit.MustQuery("show stats_meta where table_name = 't'").Sort() c.Assert(len(result.Rows()), Equals, 3) c.Assert(result.Rows()[0][5], Equals, "1") c.Assert(result.Rows()[1][5], Equals, "2") c.Assert(result.Rows()[2][5], Equals, "2") + result = testKit.MustQuery("show stats_histograms where table_name = 't';").Sort() + c.Assert(len(result.Rows()), Equals, 15) } func (s *testStatsSuite) TestExtendedStatsDefaultSwitch(c *C) { From 845f2835ca765d66ff4b0eb664aaa05a97372192 Mon Sep 17 00:00:00 2001 From: Reminiscent Date: Thu, 4 Feb 2021 19:00:25 +0800 Subject: [PATCH 10/10] make the show stats to work --- executor/show_stats.go | 29 +++++++++++++++++++++++++++++ statistics/handle/handle_test.go | 5 ++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/executor/show_stats.go b/executor/show_stats.go index 09589b0d44b8c..3de99b405b99d 100644 --- a/executor/show_stats.go +++ b/executor/show_stats.go @@ -34,6 +34,11 @@ func (e *ShowExec) fetchShowStatsMeta() error { pi := tbl.GetPartitionInfo() if pi == nil || e.ctx.GetSessionVars().UseDynamicPartitionPrune() { e.appendTableForStatsMeta(db.Name.O, tbl.Name.O, "", h.GetTableStats(tbl)) + if pi != nil { + for _, def := range pi.Definitions { + e.appendTableForStatsMeta(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)) + } + } } else { for _, def := range pi.Definitions { e.appendTableForStatsMeta(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)) @@ -67,6 +72,11 @@ func (e *ShowExec) fetchShowStatsHistogram() error { pi := tbl.GetPartitionInfo() if pi == nil || e.ctx.GetSessionVars().UseDynamicPartitionPrune() { e.appendTableForStatsHistograms(db.Name.O, tbl.Name.O, "", h.GetTableStats(tbl)) + if pi != nil { + for _, def := range pi.Definitions { + e.appendTableForStatsHistograms(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)) + } + } } else { for _, def := range pi.Definitions { e.appendTableForStatsHistograms(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)) @@ -124,6 +134,13 @@ func (e *ShowExec) fetchShowStatsBuckets() error { if err := e.appendTableForStatsBuckets(db.Name.O, tbl.Name.O, "", h.GetTableStats(tbl)); err != nil { return err } + if pi != nil { + for _, def := range pi.Definitions { + if err := e.appendTableForStatsBuckets(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)); err != nil { + return err + } + } + } } else { for _, def := range pi.Definitions { if err := e.appendTableForStatsBuckets(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)); err != nil { @@ -172,6 +189,13 @@ func (e *ShowExec) fetchShowStatsTopN() error { if err := e.appendTableForStatsTopN(db.Name.O, tbl.Name.O, "", h.GetTableStats(tbl)); err != nil { return err } + if pi != nil { + for _, def := range pi.Definitions { + if err := e.appendTableForStatsTopN(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)); err != nil { + return err + } + } + } } else { for _, def := range pi.Definitions { if err := e.appendTableForStatsTopN(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)); err != nil { @@ -291,6 +315,11 @@ func (e *ShowExec) fetchShowStatsHealthy() { pi := tbl.GetPartitionInfo() if pi == nil || e.ctx.GetSessionVars().UseDynamicPartitionPrune() { e.appendTableForStatsHealthy(db.Name.O, tbl.Name.O, "", h.GetTableStats(tbl)) + if pi != nil { + for _, def := range pi.Definitions { + e.appendTableForStatsHealthy(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)) + } + } } else { for _, def := range pi.Definitions { e.appendTableForStatsHealthy(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)) diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 6467a2d93f654..7a2efb8fa39c3 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -716,9 +716,8 @@ func (s *testStatsSuite) TestBuildGlobalLevelStats(c *C) { c.Assert(len(result.Rows()), Equals, 15) result = testKit.MustQuery("show stats_meta where table_name = 't1';").Sort() - c.Assert(len(result.Rows()), Equals, 2) - c.Assert(result.Rows()[0][5], Equals, "3") - c.Assert(result.Rows()[1][5], Equals, "2") + c.Assert(len(result.Rows()), Equals, 1) + c.Assert(result.Rows()[0][5], Equals, "5") result = testKit.MustQuery("show stats_histograms where table_name = 't1';").Sort() c.Assert(len(result.Rows()), Equals, 1)