database/tidb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

statistics: enables global-level stats to be generated in fast analyze mode (#22931)

Browse Source
This commit is contained in:
Chengpeng Yan
2021-03-01 16:40:50 +08:00
committed by GitHub
parent 6bd6df7d1e
commit 550ca8eaeb
9 changed files with 47 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
errno/errcode.go
View File

@ -991,6 +991,7 @@ const (
ErrInvalidTableSample = 8128
ErrJSONObjectKeyTooLong = 8129
ErrMultiStatementDisabled = 8130
ErrBuildGlobalLevelStatsFailed = 8131
// Error codes used by TiDB ddl package
ErrUnsupportedDDLOperation = 8200

3
errno/errname.go
View File

@ -1032,7 +1032,8 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
ErrInvalidTableSample: mysql.Message("Invalid TABLESAMPLE: %s", nil),
ErrJSONObjectKeyTooLong: mysql.Message("TiDB does not yet support JSON objects with the key length >= 65536", nil),
ErrJSONObjectKeyTooLong: mysql.Message("TiDB does not yet support JSON objects with the key length >= 65536", nil),
ErrBuildGlobalLevelStatsFailed: mysql.Message("Build global-level stats failed due to missing partition-level stats", nil),
ErrInvalidPlacementSpec: mysql.Message("Invalid placement policy '%s': %s", nil),
ErrPlacementPolicyCheck: mysql.Message("Placement policy didn't meet the constraint, reason: %s", nil),

5
errors.toml
View File

@ -1571,6 +1571,11 @@ error = '''
TiDB does not yet support JSON objects with the key length >= 65536
'''
["types:8131"]
error = '''
Build global-level stats failed due to missing partition-level stats
'''
["variable:1193"]
error = '''
Unknown system variable '%-.64s'

8
executor/analyze.go
View File

@ -171,6 +171,11 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error {
sc := e.ctx.GetSessionVars().StmtCtx
globalStats, err := statsHandle.MergePartitionStats2GlobalStats(sc, infoschema.GetInfoSchema(e.ctx), globalStatsID.tableID, info.isIndex, info.idxID)
if err != nil {
if types.ErrBuildGlobalLevelStatsFailed.Equal(err) {
// When we find some partition-level stats are missing, we need to report warning.
sc.AppendWarning(err)
continue
}
return err
}
for i := 0; i < globalStats.Num; i++ {
@ -809,8 +814,7 @@ func (e *AnalyzeFastExec) calculateEstimateSampleStep() (err error) {
sql := new(strings.Builder)
sqlexec.MustFormatSQL(sql, "select count(*) from %n.%n", dbInfo.Name.L, e.tblInfo.Name.L)
pruneMode := variable.PartitionPruneMode(e.ctx.GetSessionVars().PartitionPruneMode.Load())
if pruneMode != variable.Dynamic && e.tblInfo.ID != e.tableID.GetStatisticsID() {
if e.tblInfo.ID != e.tableID.GetStatisticsID() {
for _, definition := range e.tblInfo.Partition.Definitions {
if definition.ID == e.tableID.GetStatisticsID() {
sqlexec.MustFormatSQL(sql, " partition(%n)", definition.Name.L)

20
executor/analyze_test.go
View File

@ -473,6 +473,26 @@ func (s *testFastAnalyze) TestFastAnalyze(c *C) {
"└─IndexRangeScan 2.00 cop[tikv] table:t3, partition:p1, index:k(v) range:[3,3], keep order:false",
))
tk.MustExec(`set @@tidb_partition_prune_mode='` + string(variable.Dynamic) + `'`)
// test fast analyze in dynamic mode
tk.MustExec("drop table if exists t4;")
tk.MustExec("create table t4(a int, b int) PARTITION BY HASH(a) PARTITIONS 2;")
tk.MustExec("insert into t4 values(1,1),(3,3),(4,4),(2,2),(5,5);")
// Because the statistics of partition p1 are missing, the construction of global-level stats will fail.
tk.MustExec("analyze table t4 partition p1;")
tk.MustQuery("show warnings").Check(testkit.Rows("Warning 8131 Build global-level stats failed due to missing partition-level stats"))
// Although the global-level stats build failed, we build partition-level stats for partition p1 success.
result := tk.MustQuery("show stats_meta where table_name = 't4'").Sort()
c.Assert(len(result.Rows()), Equals, 1)
c.Assert(result.Rows()[0][5], Equals, "3")
// Now, we have the partition-level stats for partition p0. We need get the stats for partition p1. And build the global-level stats.
tk.MustExec("analyze table t4 partition p0;")
tk.MustQuery("show warnings").Check(testkit.Rows())
result = tk.MustQuery("show stats_meta where table_name = 't4'").Sort()
c.Assert(len(result.Rows()), Equals, 3)
c.Assert(result.Rows()[0][5], Equals, "5")
c.Assert(result.Rows()[1][5], Equals, "2")
c.Assert(result.Rows()[2][5], Equals, "3")
}
func (s *testSuite1) TestIssue15993(c *C) {

15
planner/core/planbuilder.go
View File

@ -1636,11 +1636,6 @@ func getPhysicalIDsAndPartitionNames(tblInfo *model.TableInfo, partitionNames []
func (b *PlanBuilder) buildAnalyzeTable(as *ast.AnalyzeTableStmt, opts map[ast.AnalyzeOptionType]uint64, version int) (Plan, error) {
p := &Analyze{Opts: opts}
pruneMode := variable.PartitionPruneMode(b.ctx.GetSessionVars().PartitionPruneMode.Load())
if len(as.PartitionNames) > 0 && pruneMode == variable.Dynamic {
logutil.BgLogger().Info("analyze partition didn't affect in dynamic-prune-mode", zap.String("partitions", as.PartitionNames[0].L))
return p, nil
}
for _, tbl := range as.TableNames {
if tbl.TableInfo.IsView() {
return nil, errors.Errorf("analyze view %s is not supported now.", tbl.Name.O)
@ -1713,11 +1708,6 @@ func (b *PlanBuilder) buildAnalyzeTable(as *ast.AnalyzeTableStmt, opts map[ast.A
func (b *PlanBuilder) buildAnalyzeIndex(as *ast.AnalyzeTableStmt, opts map[ast.AnalyzeOptionType]uint64, version int) (Plan, error) {
p := &Analyze{Opts: opts}
tblInfo := as.TableNames[0].TableInfo
pruneMode := variable.PartitionPruneMode(b.ctx.GetSessionVars().PartitionPruneMode.Load())
if len(as.PartitionNames) > 0 && pruneMode == variable.Dynamic {
logutil.BgLogger().Info("analyze partition didn't affect in dynamic-prune-mode", zap.String("table", tblInfo.Name.L), zap.String("partitions", as.PartitionNames[0].L))
return p, nil
}
physicalIDs, names, err := getPhysicalIDsAndPartitionNames(tblInfo, as.PartitionNames)
if err != nil {
return nil, err
@ -1778,11 +1768,6 @@ func (b *PlanBuilder) buildAnalyzeIndex(as *ast.AnalyzeTableStmt, opts map[ast.A
func (b *PlanBuilder) buildAnalyzeAllIndex(as *ast.AnalyzeTableStmt, opts map[ast.AnalyzeOptionType]uint64, version int) (Plan, error) {
p := &Analyze{Opts: opts}
tblInfo := as.TableNames[0].TableInfo
pruneMode := variable.PartitionPruneMode(b.ctx.GetSessionVars().PartitionPruneMode.Load())
if len(as.PartitionNames) > 0 && pruneMode == variable.Dynamic {
logutil.BgLogger().Info("analyze partition didn't affect in dynamic-prune-mode", zap.String("table", tblInfo.Name.L), zap.String("partitions", as.PartitionNames[0].L))
return p, nil
}
physicalIDs, names, err := getPhysicalIDsAndPartitionNames(tblInfo, as.PartitionNames)
if err != nil {
return nil, err

10
statistics/handle/handle.go
View File

@ -354,18 +354,22 @@ func (h *Handle) MergePartitionStats2GlobalStats(sc *stmtctx.StatementContext, i
if err != nil {
return
}
// if the err == nil && partitionStats == nil, it means we lack the partition-level stats which the physicalID is equal to partitionID.
if partitionStats == nil {
err = errors.Errorf("[stats] error occurred when read partition-level stats of the table with tableID %d and partitionID %d", physicalID, partitionID)
err = types.ErrBuildGlobalLevelStatsFailed
return
}
globalStats.Count += partitionStats.Count
for i := 0; i < globalStats.Num; i++ {
ID := tableInfo.Columns[i].ID
if isIndex != 0 {
// If the statistics is the index stats, we should use the index ID to replace the column ID.
ID = idxID
}
hg, cms, topN, fms := partitionStats.GetStatsInfo(ID, isIndex == 1)
count, hg, cms, topN, fms := partitionStats.GetStatsInfo(ID, isIndex == 1)
if i == 0 {
// In a partition, we will only update globalStats.Count once
globalStats.Count += count
}
allHg[i] = append(allHg[i], hg)
allCms[i] = append(allCms[i], cms)
allTopN[i] = append(allTopN[i], topN)

6
statistics/table.go
View File

@ -198,13 +198,13 @@ func (t *Table) ColumnByName(colName string) *Column {
}
// GetStatsInfo returns their statistics according to the ID of the column or index, including histogram, CMSketch, TopN and FMSketch.
func (t *Table) GetStatsInfo(ID int64, isIndex bool) (*Histogram, *CMSketch, *TopN, *FMSketch) {
func (t *Table) GetStatsInfo(ID int64, isIndex bool) (int64, *Histogram, *CMSketch, *TopN, *FMSketch) {
if isIndex {
idxStatsInfo := t.Indices[ID]
return idxStatsInfo.Histogram.Copy(), idxStatsInfo.CMSketch.Copy(), idxStatsInfo.TopN.Copy(), nil
return int64(idxStatsInfo.TotalRowCount()), idxStatsInfo.Histogram.Copy(), idxStatsInfo.CMSketch.Copy(), idxStatsInfo.TopN.Copy(), nil
}
colStatsInfo := t.Columns[ID]
return colStatsInfo.Histogram.Copy(), colStatsInfo.CMSketch.Copy(), colStatsInfo.TopN.Copy(), colStatsInfo.FMSketch.Copy()
return int64(colStatsInfo.TotalRowCount()), colStatsInfo.Histogram.Copy(), colStatsInfo.CMSketch.Copy(), colStatsInfo.TopN.Copy(), colStatsInfo.FMSketch.Copy()
}
type tableColumnID struct {

3
types/errors.go
View File

@ -83,4 +83,7 @@ var (
ErrWrongValue = dbterror.ClassTypes.NewStdErr(mysql.ErrTruncatedWrongValue, mysql.MySQLErrName[mysql.ErrWrongValue])
// ErrWrongValueForType is returned when the input value is in wrong format for function.
ErrWrongValueForType = dbterror.ClassTypes.NewStdErr(mysql.ErrWrongValueForType, mysql.MySQLErrName[mysql.ErrWrongValueForType])
// ErrBuildGlobalLevelStatsFailed is returned when the partition-level stats is missing and the build global-level stats fails.
// Put this error here is to prevent `import cycle not allowed`.
ErrBuildGlobalLevelStatsFailed = dbterror.ClassTypes.NewStd(mysql.ErrBuildGlobalLevelStatsFailed)
)