database/tidb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

statistics: do not skip negative and zero weight tables (#51978)

Browse Source 
close pingcap/tidb#51972
This commit is contained in:
二手掉包工程师
2024-03-21 17:11:21 +08:00
committed by GitHub
parent f6ce672e9c
commit 2fbcd9be26
7 changed files with 15 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/statistics/handle/autoanalyze/priorityqueue/BUILD.bazel
View File

@ -20,7 +20,6 @@ go_library(
"//pkg/statistics/handle/logutil",
"//pkg/statistics/handle/types",
"//pkg/statistics/handle/util",
"//pkg/util/intest",
"@org_uber_go_zap//:zap",
],
)

7
pkg/statistics/handle/autoanalyze/priorityqueue/dynamic_partitioned_table_analysis_job.go
View File

@ -114,9 +114,6 @@ func (j *DynamicPartitionedTableAnalysisJob) HasNewlyAddedIndex() bool {
func (j *DynamicPartitionedTableAnalysisJob) IsValidToAnalyze(
sctx sessionctx.Context,
) (bool, string) {
if valid, failReason := isValidWeight(j.Weight); !valid {
return false, failReason
}
// Check whether the table or partition is valid to analyze.
if len(j.Partitions) > 0 || len(j.PartitionIndexes) > 0 {
// Any partition is invalid to analyze, the whole table is invalid to analyze.
@ -156,10 +153,10 @@ func (j *DynamicPartitionedTableAnalysisJob) String() string {
"\tGlobal Table: %s\n"+
"\tGlobal TableID: %d\n"+
"\tTableStatsVer: %d\n"+
"\tChangePercentage: %.2f\n"+
"\tChangePercentage: %.6f\n"+
"\tTableSize: %.2f\n"+
"\tLastAnalysisDuration: %s\n"+
"\tWeight: %.4f\n",
"\tWeight: %.6f\n",
j.getAnalyzeType(),
strings.Join(j.Partitions, ", "),
j.PartitionIndexes,

13
pkg/statistics/handle/autoanalyze/priorityqueue/job.go
View File

@ -21,7 +21,6 @@ import (
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/statistics/handle/logutil"
statstypes "github.com/pingcap/tidb/pkg/statistics/handle/types"
"github.com/pingcap/tidb/pkg/util/intest"
"go.uber.org/zap"
)
@ -75,18 +74,6 @@ type AnalysisJob interface {
fmt.Stringer
}
// Usually, we should not put this kind of table into the queue.
// This is just a double check.
func isValidWeight(weight float64) (bool, string) {
// No need to analyze this table.
intest.Assert(weight > 0, "weight is less than or equal to 0")
if weight <= 0 {
return false, fmt.Sprintf("weight is less than or equal to 0: %.4f", weight)
}
return true, ""
}
// isValidToAnalyze checks whether the table is valid to analyze.
// It checks the last failed analysis duration and the average analysis duration.
// If the last failed analysis duration is less than 2 times the average analysis duration,

12
pkg/statistics/handle/autoanalyze/priorityqueue/job_test.go
View File

@ -39,7 +39,7 @@ func TestStringer(t *testing.T) {
ChangePercentage: 0.5,
},
},
want: "NonPartitionedTableAnalysisJob:\n\tAnalyzeType: analyzeTable\n\tIndexes: \n\tSchema: test_schema\n\tTable: test_table\n\tTableID: 1\n\tTableStatsVer: 1\n\tChangePercentage: 0.50\n\tTableSize: 0.00\n\tLastAnalysisDuration: 0s\n\tWeight: 2.0000\n",
want: "NonPartitionedTableAnalysisJob:\n\tAnalyzeType: analyzeTable\n\tIndexes: \n\tSchema: test_schema\n\tTable: test_table\n\tTableID: 1\n\tTableStatsVer: 1\n\tChangePercentage: 0.500000\n\tTableSize: 0.00\n\tLastAnalysisDuration: 0s\n\tWeight: 1.999999\n",
},
{
name: "analyze non-partitioned table index",
@ -54,7 +54,7 @@ func TestStringer(t *testing.T) {
ChangePercentage: 0.5,
},
},
want: "NonPartitionedTableAnalysisJob:\n\tAnalyzeType: analyzeIndex\n\tIndexes: idx\n\tSchema: test_schema\n\tTable: test_table\n\tTableID: 2\n\tTableStatsVer: 1\n\tChangePercentage: 0.50\n\tTableSize: 0.00\n\tLastAnalysisDuration: 0s\n\tWeight: 2.0000\n",
want: "NonPartitionedTableAnalysisJob:\n\tAnalyzeType: analyzeIndex\n\tIndexes: idx\n\tSchema: test_schema\n\tTable: test_table\n\tTableID: 2\n\tTableStatsVer: 1\n\tChangePercentage: 0.500000\n\tTableSize: 0.00\n\tLastAnalysisDuration: 0s\n\tWeight: 1.999999\n",
},
{
name: "analyze dynamic partition",
@ -69,7 +69,7 @@ func TestStringer(t *testing.T) {
ChangePercentage: 0.5,
},
},
want: "DynamicPartitionedTableAnalysisJob:\n\tAnalyzeType: analyzeDynamicPartition\n\tPartitions: p0, p1\n\tPartitionIndexes: map[]\n\tSchema: test_schema\n\tGlobal Table: test_table\n\tGlobal TableID: 3\n\tTableStatsVer: 1\n\tChangePercentage: 0.50\n\tTableSize: 0.00\n\tLastAnalysisDuration: 0s\n\tWeight: 2.0000\n",
want: "DynamicPartitionedTableAnalysisJob:\n\tAnalyzeType: analyzeDynamicPartition\n\tPartitions: p0, p1\n\tPartitionIndexes: map[]\n\tSchema: test_schema\n\tGlobal Table: test_table\n\tGlobal TableID: 3\n\tTableStatsVer: 1\n\tChangePercentage: 0.500000\n\tTableSize: 0.00\n\tLastAnalysisDuration: 0s\n\tWeight: 1.999999\n",
},
{
name: "analyze dynamic partition's indexes",
@ -86,7 +86,7 @@ func TestStringer(t *testing.T) {
ChangePercentage: 0.5,
},
},
want: "DynamicPartitionedTableAnalysisJob:\n\tAnalyzeType: analyzeDynamicPartitionIndex\n\tPartitions: \n\tPartitionIndexes: map[idx:[p0 p1]]\n\tSchema: test_schema\n\tGlobal Table: test_table\n\tGlobal TableID: 4\n\tTableStatsVer: 1\n\tChangePercentage: 0.50\n\tTableSize: 0.00\n\tLastAnalysisDuration: 0s\n\tWeight: 2.0000\n",
want: "DynamicPartitionedTableAnalysisJob:\n\tAnalyzeType: analyzeDynamicPartitionIndex\n\tPartitions: \n\tPartitionIndexes: map[idx:[p0 p1]]\n\tSchema: test_schema\n\tGlobal Table: test_table\n\tGlobal TableID: 4\n\tTableStatsVer: 1\n\tChangePercentage: 0.500000\n\tTableSize: 0.00\n\tLastAnalysisDuration: 0s\n\tWeight: 1.999999\n",
},
{
name: "analyze static partition",
@ -102,7 +102,7 @@ func TestStringer(t *testing.T) {
ChangePercentage: 0.5,
},
},
want: "StaticPartitionedTableAnalysisJob:\n\tAnalyzeType: analyzeStaticPartition\n\tIndexes: \n\tSchema: test_schema\n\tGlobalTable: test_table\n\tGlobalTableID: 5\n\tStaticPartition: p0\n\tStaticPartitionID: 6\n\tTableStatsVer: 1\n\tChangePercentage: 0.50\n\tTableSize: 0.00\n\tLastAnalysisDuration: 0s\n\tWeight: 2.0000\n",
want: "StaticPartitionedTableAnalysisJob:\n\tAnalyzeType: analyzeStaticPartition\n\tIndexes: \n\tSchema: test_schema\n\tGlobalTable: test_table\n\tGlobalTableID: 5\n\tStaticPartition: p0\n\tStaticPartitionID: 6\n\tTableStatsVer: 1\n\tChangePercentage: 0.500000\n\tTableSize: 0.00\n\tLastAnalysisDuration: 0s\n\tWeight: 1.999999\n",
},
{
name: "analyze static partition's index",
@ -119,7 +119,7 @@ func TestStringer(t *testing.T) {
ChangePercentage: 0.5,
},
},
want: "StaticPartitionedTableAnalysisJob:\n\tAnalyzeType: analyzeStaticPartitionIndex\n\tIndexes: idx\n\tSchema: test_schema\n\tGlobalTable: test_table\n\tGlobalTableID: 7\n\tStaticPartition: p0\n\tStaticPartitionID: 8\n\tTableStatsVer: 1\n\tChangePercentage: 0.50\n\tTableSize: 0.00\n\tLastAnalysisDuration: 0s\n\tWeight: 2.0000\n",
want: "StaticPartitionedTableAnalysisJob:\n\tAnalyzeType: analyzeStaticPartitionIndex\n\tIndexes: idx\n\tSchema: test_schema\n\tGlobalTable: test_table\n\tGlobalTableID: 7\n\tStaticPartition: p0\n\tStaticPartitionID: 8\n\tTableStatsVer: 1\n\tChangePercentage: 0.500000\n\tTableSize: 0.00\n\tLastAnalysisDuration: 0s\n\tWeight: 1.999999\n",
},
}
for _, tt := range tests {

8
pkg/statistics/handle/autoanalyze/priorityqueue/non_partitioned_table_analysis_job.go
View File

@ -94,10 +94,6 @@ func (j *NonPartitionedTableAnalysisJob) HasNewlyAddedIndex() bool {
func (j *NonPartitionedTableAnalysisJob) IsValidToAnalyze(
sctx sessionctx.Context,
) (bool, string) {
if valid, failReason := isValidWeight(j.Weight); !valid {
return false, failReason
}
if valid, failReason := isValidToAnalyze(
sctx,
j.TableSchema,
@ -134,10 +130,10 @@ func (j *NonPartitionedTableAnalysisJob) String() string {
"\tTable: %s\n"+
"\tTableID: %d\n"+
"\tTableStatsVer: %d\n"+
"\tChangePercentage: %.2f\n"+
"\tChangePercentage: %.6f\n"+
"\tTableSize: %.2f\n"+
"\tLastAnalysisDuration: %v\n"+
"\tWeight: %.4f\n",
"\tWeight: %.6f\n",
j.getAnalyzeType(),
strings.Join(j.Indexes, ", "),
j.TableSchema, j.TableName, j.TableID, j.TableStatsVer,

7
pkg/statistics/handle/autoanalyze/priorityqueue/static_partitioned_table_analysis_job.go
View File

@ -107,9 +107,6 @@ func (j *StaticPartitionedTableAnalysisJob) HasNewlyAddedIndex() bool {
func (j *StaticPartitionedTableAnalysisJob) IsValidToAnalyze(
sctx sessionctx.Context,
) (bool, string) {
if valid, failReason := isValidWeight(j.Weight); !valid {
return false, failReason
}
// Check whether the partition is valid to analyze.
// For static partition table we only need to check the specified static partition.
if j.StaticPartitionName != "" {
@ -149,10 +146,10 @@ func (j *StaticPartitionedTableAnalysisJob) String() string {
"\tStaticPartition: %s\n"+
"\tStaticPartitionID: %d\n"+
"\tTableStatsVer: %d\n"+
"\tChangePercentage: %.2f\n"+
"\tChangePercentage: %.6f\n"+
"\tTableSize: %.2f\n"+
"\tLastAnalysisDuration: %s\n"+
"\tWeight: %.4f\n",
"\tWeight: %.6f\n",
j.getAnalyzeType(),
strings.Join(j.Indexes, ", "),
j.TableSchema, j.GlobalTableName, j.GlobalTableID,

7
pkg/statistics/handle/autoanalyze/refresher/refresher.go
View File

@ -202,12 +202,11 @@ func (r *Refresher) RebuildTableAnalysisJobQueue() error {
// We apply a penalty to larger tables, which can potentially result in a negative weight.
// To prevent this, we filter out any negative weights. Under normal circumstances, table sizes should not be negative.
if weight <= 0 {
statslogutil.StatsLogger().Info(
"Table is not ready to analyze",
zap.String("reason", "weight is not positive"),
statslogutil.SingletonStatsSamplerLogger().Warn(
"Table gets a negative weight",
zap.Float64("weight", weight),
zap.Stringer("job", job),
)
return
}
job.SetWeight(weight)
// Push the job onto the queue.