From 99799fea4d9747c67d32be047edce65bee80c7c8 Mon Sep 17 00:00:00 2001 From: Arenatlx <314806019@qq.com> Date: Fri, 5 Jul 2024 10:38:58 +0800 Subject: [PATCH] planner: classify logical index scan into a separate file for later pkg move (#54443) ref pingcap/tidb#51664, ref pingcap/tidb#52714 --- pkg/planner/cascades/implementation_rules.go | 2 +- pkg/planner/core/BUILD.bazel | 1 + pkg/planner/core/explain.go | 23 -- pkg/planner/core/find_best_task.go | 4 +- pkg/planner/core/logical_index_scan.go | 212 +++++++++++++++++++ pkg/planner/core/logical_initialize.go | 6 - pkg/planner/core/logical_plans.go | 46 ---- pkg/planner/core/property_cols_prune.go | 13 -- pkg/planner/core/rule_build_key_info.go | 19 -- pkg/planner/core/stats.go | 26 --- 10 files changed, 216 insertions(+), 136 deletions(-) create mode 100644 pkg/planner/core/logical_index_scan.go diff --git a/pkg/planner/cascades/implementation_rules.go b/pkg/planner/cascades/implementation_rules.go index 2a53f59c66..5170b0f155 100644 --- a/pkg/planner/cascades/implementation_rules.go +++ b/pkg/planner/cascades/implementation_rules.go @@ -220,7 +220,7 @@ func (*ImplIndexScan) Match(expr *memo.GroupExpr, prop *property.PhysicalPropert // OnImplement implements ImplementationRule OnImplement interface. func (*ImplIndexScan) OnImplement(expr *memo.GroupExpr, reqProp *property.PhysicalProperty) ([]memo.Implementation, error) { logicalScan := expr.ExprNode.(*plannercore.LogicalIndexScan) - is := logicalScan.GetPhysicalIndexScan(expr.Group.Prop.Schema, expr.Group.Prop.Stats.ScaleByExpectCnt(reqProp.ExpectedCnt)) + is := plannercore.GetPhysicalIndexScan4LogicalIndexScan(logicalScan, expr.Group.Prop.Schema, expr.Group.Prop.Stats.ScaleByExpectCnt(reqProp.ExpectedCnt)) if !reqProp.IsSortItemEmpty() { is.KeepOrder = true if reqProp.SortItems[0].Desc { diff --git a/pkg/planner/core/BUILD.bazel b/pkg/planner/core/BUILD.bazel index 99dac0045b..8936ffef84 100644 --- a/pkg/planner/core/BUILD.bazel +++ b/pkg/planner/core/BUILD.bazel @@ -22,6 +22,7 @@ go_library( "indexmerge_unfinished_path.go", "initialize.go", "logical_aggregation.go", + "logical_index_scan.go", "logical_initialize.go", "logical_limit.go", "logical_lock.go", diff --git a/pkg/planner/core/explain.go b/pkg/planner/core/explain.go index 5efbe96d43..310c1ecdc8 100644 --- a/pkg/planner/core/explain.go +++ b/pkg/planner/core/explain.go @@ -1006,29 +1006,6 @@ func explainNormalizedByItems(buffer *bytes.Buffer, byItems []*util.ByItems) *by return buffer } -// ExplainInfo implements Plan interface. -func (p *LogicalIndexScan) ExplainInfo() string { - buffer := bytes.NewBufferString(p.Source.ExplainInfo()) - index := p.Index - if len(index.Columns) > 0 { - buffer.WriteString(", index:") - for i, idxCol := range index.Columns { - if tblCol := p.Source.TableInfo.Columns[idxCol.Offset]; tblCol.Hidden { - buffer.WriteString(tblCol.GeneratedExprString) - } else { - buffer.WriteString(idxCol.Name.O) - } - if i+1 < len(index.Columns) { - buffer.WriteString(", ") - } - } - } - if len(p.AccessConds) > 0 { - fmt.Fprintf(buffer, ", cond:%v", p.AccessConds) - } - return buffer.String() -} - // ExplainInfo implements Plan interface. func (p *PhysicalMemTable) ExplainInfo() string { accessObject, operatorInfo := p.AccessObject().String(), p.OperatorInfo(false) diff --git a/pkg/planner/core/find_best_task.go b/pkg/planner/core/find_best_task.go index e426ceb62e..374deb138d 100644 --- a/pkg/planner/core/find_best_task.go +++ b/pkg/planner/core/find_best_task.go @@ -2369,8 +2369,8 @@ func GetPhysicalScan4LogicalTableScan(s *LogicalTableScan, schema *expression.Sc return ts } -// GetPhysicalIndexScan returns PhysicalIndexScan for the logical IndexScan. -func (s *LogicalIndexScan) GetPhysicalIndexScan(_ *expression.Schema, stats *property.StatsInfo) *PhysicalIndexScan { +// GetPhysicalIndexScan4LogicalIndexScan returns PhysicalIndexScan for the logical IndexScan. +func GetPhysicalIndexScan4LogicalIndexScan(s *LogicalIndexScan, _ *expression.Schema, stats *property.StatsInfo) *PhysicalIndexScan { ds := s.Source is := PhysicalIndexScan{ Table: ds.TableInfo, diff --git a/pkg/planner/core/logical_index_scan.go b/pkg/planner/core/logical_index_scan.go new file mode 100644 index 0000000000..508c2d1c90 --- /dev/null +++ b/pkg/planner/core/logical_index_scan.go @@ -0,0 +1,212 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import ( + "bytes" + "fmt" + + "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/parser/model" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/planner/core/base" + "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop" + "github.com/pingcap/tidb/pkg/planner/property" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/plancodec" + "github.com/pingcap/tidb/pkg/util/ranger" +) + +// LogicalIndexScan is the logical index scan operator for TiKV. +type LogicalIndexScan struct { + logicalop.LogicalSchemaProducer + // DataSource should be read-only here. + Source *DataSource + IsDoubleRead bool + + EqCondCount int + AccessConds expression.CNFExprs + Ranges []*ranger.Range + + Index *model.IndexInfo + Columns []*model.ColumnInfo + FullIdxCols []*expression.Column + FullIdxColLens []int + IdxCols []*expression.Column + IdxColLens []int +} + +// Init initializes LogicalIndexScan. +func (is LogicalIndexScan) Init(ctx base.PlanContext, offset int) *LogicalIndexScan { + is.BaseLogicalPlan = logicalop.NewBaseLogicalPlan(ctx, plancodec.TypeIdxScan, &is, offset) + return &is +} + +// *************************** start implementation of Plan interface *************************** + +// ExplainInfo implements Plan interface. +func (is *LogicalIndexScan) ExplainInfo() string { + buffer := bytes.NewBufferString(is.Source.ExplainInfo()) + index := is.Index + if len(index.Columns) > 0 { + buffer.WriteString(", index:") + for i, idxCol := range index.Columns { + if tblCol := is.Source.TableInfo.Columns[idxCol.Offset]; tblCol.Hidden { + buffer.WriteString(tblCol.GeneratedExprString) + } else { + buffer.WriteString(idxCol.Name.O) + } + if i+1 < len(index.Columns) { + buffer.WriteString(", ") + } + } + } + if len(is.AccessConds) > 0 { + fmt.Fprintf(buffer, ", cond:%v", is.AccessConds) + } + return buffer.String() +} + +// *************************** end implementation of Plan interface *************************** + +// *************************** start implementation of logicalPlan interface *************************** + +// HashCode inherits BaseLogicalPlan.<0th> interface. + +// PredicatePushDown inherits BaseLogicalPlan.<1st> interface. + +// PruneColumns inherits BaseLogicalPlan.<2nd> interface. + +// FindBestTask inherits BaseLogicalPlan.<3rd> interface. + +// BuildKeyInfo implements base.LogicalPlan.<4th> interface. +func (is *LogicalIndexScan) BuildKeyInfo(selfSchema *expression.Schema, _ []*expression.Schema) { + selfSchema.Keys = nil + for _, path := range is.Source.PossibleAccessPaths { + if path.IsTablePath() { + continue + } + if uniqueKey, newKey := checkIndexCanBeKey(path.Index, is.Columns, selfSchema); newKey != nil { + selfSchema.Keys = append(selfSchema.Keys, newKey) + } else if uniqueKey != nil { + selfSchema.UniqueKeys = append(selfSchema.UniqueKeys, uniqueKey) + } + } + handle := is.getPKIsHandleCol(selfSchema) + if handle != nil { + selfSchema.Keys = append(selfSchema.Keys, []*expression.Column{handle}) + } +} + +// PushDownTopN inherits BaseLogicalPlan.<5th> interface. + +// DeriveTopN inherits BaseLogicalPlan.LogicalPlan.<6th> implementation. + +// PredicateSimplification inherits BaseLogicalPlan.LogicalPlan.<7th> implementation. + +// ConstantPropagation inherits BaseLogicalPlan.LogicalPlan.<8th> implementation. + +// PullUpConstantPredicates inherits BaseLogicalPlan.LogicalPlan.<9th> implementation. + +// RecursiveDeriveStats inherits BaseLogicalPlan.LogicalPlan.<10th> implementation. + +// DeriveStats implements base.LogicalPlan.<11th> interface. +func (is *LogicalIndexScan) DeriveStats(_ []*property.StatsInfo, selfSchema *expression.Schema, _ []*expression.Schema, _ [][]*expression.Column) (*property.StatsInfo, error) { + is.Source.initStats(nil) + exprCtx := is.SCtx().GetExprCtx() + for i, expr := range is.AccessConds { + is.AccessConds[i] = expression.PushDownNot(exprCtx, expr) + } + is.SetStats(is.Source.deriveStatsByFilter(is.AccessConds, nil)) + if len(is.AccessConds) == 0 { + is.Ranges = ranger.FullRange() + } + is.IdxCols, is.IdxColLens = expression.IndexInfo2PrefixCols(is.Columns, selfSchema.Columns, is.Index) + is.FullIdxCols, is.FullIdxColLens = expression.IndexInfo2Cols(is.Columns, selfSchema.Columns, is.Index) + if !is.Index.Unique && !is.Index.Primary && len(is.Index.Columns) == len(is.IdxCols) { + handleCol := is.getPKIsHandleCol(selfSchema) + if handleCol != nil && !mysql.HasUnsignedFlag(handleCol.RetType.GetFlag()) { + is.IdxCols = append(is.IdxCols, handleCol) + is.IdxColLens = append(is.IdxColLens, types.UnspecifiedLength) + } + } + return is.StatsInfo(), nil +} + +// ExtractColGroups inherits BaseLogicalPlan.LogicalPlan.<12th> implementation. + +// PreparePossibleProperties implements base.LogicalPlan.<13th> interface. +func (is *LogicalIndexScan) PreparePossibleProperties(_ *expression.Schema, _ ...[][]*expression.Column) [][]*expression.Column { + if len(is.IdxCols) == 0 { + return nil + } + result := make([][]*expression.Column, 0, is.EqCondCount+1) + for i := 0; i <= is.EqCondCount; i++ { + result = append(result, make([]*expression.Column, len(is.IdxCols)-i)) + copy(result[i], is.IdxCols[i:]) + } + return result +} + +// ExhaustPhysicalPlans inherits BaseLogicalPlan.LogicalPlan.<14th> implementation. + +// ExtractCorrelatedCols inherits BaseLogicalPlan.LogicalPlan.<15th> implementation. + +// MaxOneRow inherits BaseLogicalPlan.LogicalPlan.<16th> implementation. + +// Children inherits BaseLogicalPlan.LogicalPlan.<17th> implementation. + +// SetChildren inherits BaseLogicalPlan.LogicalPlan.<18th> implementation. + +// SetChild inherits BaseLogicalPlan.LogicalPlan.<19th> implementation. + +// RollBackTaskMap inherits BaseLogicalPlan.LogicalPlan.<20th> implementation. + +// CanPushToCop inherits BaseLogicalPlan.LogicalPlan.<21st> implementation. + +// ExtractFD inherits BaseLogicalPlan.LogicalPlan.<22nd> implementation. + +// GetBaseLogicalPlan inherits BaseLogicalPlan.LogicalPlan.<23rd> implementation. + +// ConvertOuterToInnerJoin inherits BaseLogicalPlan.LogicalPlan.<24th> implementation. + +// *************************** end implementation of logicalPlan interface *************************** + +// MatchIndexProp checks if the indexScan can match the required property. +func (is *LogicalIndexScan) MatchIndexProp(prop *property.PhysicalProperty) (match bool) { + if prop.IsSortItemEmpty() { + return true + } + if all, _ := prop.AllSameOrder(); !all { + return false + } + sctx := is.SCtx() + evalCtx := sctx.GetExprCtx().GetEvalCtx() + for i, col := range is.IdxCols { + if col.Equal(evalCtx, prop.SortItems[0].Col) { + return matchIndicesProp(sctx, is.IdxCols[i:], is.IdxColLens[i:], prop.SortItems) + } else if i >= is.EqCondCount { + break + } + } + return false +} + +func (is *LogicalIndexScan) getPKIsHandleCol(schema *expression.Schema) *expression.Column { + // We cannot use p.Source.getPKIsHandleCol() here, + // Because we may re-prune p.Columns and p.schema during the transformation. + // That will make p.Columns different from p.Source.Columns. + return getPKIsHandleColFromSchema(is.Columns, schema, is.Source.TableInfo.PKIsHandle) +} diff --git a/pkg/planner/core/logical_initialize.go b/pkg/planner/core/logical_initialize.go index f66d8f55f2..8afcd7da16 100644 --- a/pkg/planner/core/logical_initialize.go +++ b/pkg/planner/core/logical_initialize.go @@ -32,12 +32,6 @@ func (ds DataSource) Init(ctx base.PlanContext, offset int) *DataSource { return &ds } -// Init initializes LogicalIndexScan. -func (is LogicalIndexScan) Init(ctx base.PlanContext, offset int) *LogicalIndexScan { - is.BaseLogicalPlan = logicalop.NewBaseLogicalPlan(ctx, plancodec.TypeIdxScan, &is, offset) - return &is -} - // Init initializes LogicalApply. func (la LogicalApply) Init(ctx base.PlanContext, offset int) *LogicalApply { la.BaseLogicalPlan = logicalop.NewBaseLogicalPlan(ctx, plancodec.TypeApply, &la, offset) diff --git a/pkg/planner/core/logical_plans.go b/pkg/planner/core/logical_plans.go index e256383a45..e9e005ede8 100644 --- a/pkg/planner/core/logical_plans.go +++ b/pkg/planner/core/logical_plans.go @@ -1046,45 +1046,6 @@ func (ds *DataSource) ExtractCorrelatedCols() []*expression.CorrelatedColumn { return corCols } -// LogicalIndexScan is the logical index scan operator for TiKV. -type LogicalIndexScan struct { - logicalop.LogicalSchemaProducer - // DataSource should be read-only here. - Source *DataSource - IsDoubleRead bool - - EqCondCount int - AccessConds expression.CNFExprs - Ranges []*ranger.Range - - Index *model.IndexInfo - Columns []*model.ColumnInfo - FullIdxCols []*expression.Column - FullIdxColLens []int - IdxCols []*expression.Column - IdxColLens []int -} - -// MatchIndexProp checks if the indexScan can match the required property. -func (p *LogicalIndexScan) MatchIndexProp(prop *property.PhysicalProperty) (match bool) { - if prop.IsSortItemEmpty() { - return true - } - if all, _ := prop.AllSameOrder(); !all { - return false - } - sctx := p.SCtx() - evalCtx := sctx.GetExprCtx().GetEvalCtx() - for i, col := range p.IdxCols { - if col.Equal(evalCtx, prop.SortItems[0].Col) { - return matchIndicesProp(sctx, p.IdxCols[i:], p.IdxColLens[i:], prop.SortItems) - } else if i >= p.EqCondCount { - break - } - } - return false -} - // getTablePath finds the TablePath from a group of accessPaths. func getTablePath(paths []*util.AccessPath) *util.AccessPath { for _, path := range paths { @@ -1408,13 +1369,6 @@ func (ds *DataSource) getPKIsHandleCol() *expression.Column { return getPKIsHandleColFromSchema(ds.Columns, ds.Schema(), ds.TableInfo.PKIsHandle) } -func (p *LogicalIndexScan) getPKIsHandleCol(schema *expression.Schema) *expression.Column { - // We cannot use p.Source.getPKIsHandleCol() here, - // Because we may re-prune p.Columns and p.schema during the transformation. - // That will make p.Columns different from p.Source.Columns. - return getPKIsHandleColFromSchema(p.Columns, schema, p.Source.TableInfo.PKIsHandle) -} - // WindowFrame represents a window function frame. type WindowFrame struct { Type ast.FrameType diff --git a/pkg/planner/core/property_cols_prune.go b/pkg/planner/core/property_cols_prune.go index b4eaa21cdb..a3e5c435c5 100644 --- a/pkg/planner/core/property_cols_prune.go +++ b/pkg/planner/core/property_cols_prune.go @@ -56,19 +56,6 @@ func (ds *DataSource) PreparePossibleProperties(_ *expression.Schema, _ ...[][]* return result } -// PreparePossibleProperties implements base.LogicalPlan PreparePossibleProperties interface. -func (is *LogicalIndexScan) PreparePossibleProperties(_ *expression.Schema, _ ...[][]*expression.Column) [][]*expression.Column { - if len(is.IdxCols) == 0 { - return nil - } - result := make([][]*expression.Column, 0, is.EqCondCount+1) - for i := 0; i <= is.EqCondCount; i++ { - result = append(result, make([]*expression.Column, len(is.IdxCols)-i)) - copy(result[i], is.IdxCols[i:]) - } - return result -} - func getPossiblePropertyFromByItems(items []*util.ByItems) []*expression.Column { cols := make([]*expression.Column, 0, len(items)) for _, item := range items { diff --git a/pkg/planner/core/rule_build_key_info.go b/pkg/planner/core/rule_build_key_info.go index 9bf467132e..eda67f11e7 100644 --- a/pkg/planner/core/rule_build_key_info.go +++ b/pkg/planner/core/rule_build_key_info.go @@ -197,25 +197,6 @@ func (ds *DataSource) BuildKeyInfo(selfSchema *expression.Schema, _ []*expressio } } -// BuildKeyInfo implements base.LogicalPlan BuildKeyInfo interface. -func (is *LogicalIndexScan) BuildKeyInfo(selfSchema *expression.Schema, _ []*expression.Schema) { - selfSchema.Keys = nil - for _, path := range is.Source.PossibleAccessPaths { - if path.IsTablePath() { - continue - } - if uniqueKey, newKey := checkIndexCanBeKey(path.Index, is.Columns, selfSchema); newKey != nil { - selfSchema.Keys = append(selfSchema.Keys, newKey) - } else if uniqueKey != nil { - selfSchema.UniqueKeys = append(selfSchema.UniqueKeys, uniqueKey) - } - } - handle := is.getPKIsHandleCol(selfSchema) - if handle != nil { - selfSchema.Keys = append(selfSchema.Keys, []*expression.Column{handle}) - } -} - func (*buildKeySolver) name() string { return "build_keys" } diff --git a/pkg/planner/core/stats.go b/pkg/planner/core/stats.go index 8fdb73881a..05923c22e7 100644 --- a/pkg/planner/core/stats.go +++ b/pkg/planner/core/stats.go @@ -28,7 +28,6 @@ import ( "github.com/pingcap/tidb/pkg/infoschema" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/parser/model" - "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/planner/cardinality" "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/core/cost" @@ -40,10 +39,8 @@ import ( "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/statistics/asyncload" "github.com/pingcap/tidb/pkg/table" - "github.com/pingcap/tidb/pkg/types" h "github.com/pingcap/tidb/pkg/util/hint" "github.com/pingcap/tidb/pkg/util/logutil" - "github.com/pingcap/tidb/pkg/util/ranger" "go.uber.org/zap" ) @@ -475,29 +472,6 @@ func getMinSelectivityFromPaths(paths []*util.AccessPath, totalRowCount float64) return minSelectivity } -// DeriveStats implements LogicalPlan DeriveStats interface. -func (is *LogicalIndexScan) DeriveStats(_ []*property.StatsInfo, selfSchema *expression.Schema, _ []*expression.Schema, _ [][]*expression.Column) (*property.StatsInfo, error) { - is.Source.initStats(nil) - exprCtx := is.SCtx().GetExprCtx() - for i, expr := range is.AccessConds { - is.AccessConds[i] = expression.PushDownNot(exprCtx, expr) - } - is.SetStats(is.Source.deriveStatsByFilter(is.AccessConds, nil)) - if len(is.AccessConds) == 0 { - is.Ranges = ranger.FullRange() - } - is.IdxCols, is.IdxColLens = expression.IndexInfo2PrefixCols(is.Columns, selfSchema.Columns, is.Index) - is.FullIdxCols, is.FullIdxColLens = expression.IndexInfo2Cols(is.Columns, selfSchema.Columns, is.Index) - if !is.Index.Unique && !is.Index.Primary && len(is.Index.Columns) == len(is.IdxCols) { - handleCol := is.getPKIsHandleCol(selfSchema) - if handleCol != nil && !mysql.HasUnsignedFlag(handleCol.RetType.GetFlag()) { - is.IdxCols = append(is.IdxCols, handleCol) - is.IdxColLens = append(is.IdxColLens, types.UnspecifiedLength) - } - } - return is.StatsInfo(), nil -} - func deriveLimitStats(childProfile *property.StatsInfo, limitCount float64) *property.StatsInfo { stats := &property.StatsInfo{ RowCount: math.Min(limitCount, childProfile.RowCount),