diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 3cbb40545b..137df9fa30 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -37,13 +37,13 @@ import ( "go.uber.org/zap" ) -func (p *LogicalUnionScan) exhaustPhysicalPlans(prop *property.PhysicalProperty) []PhysicalPlan { +func (p *LogicalUnionScan) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { childProp := prop.Clone() us := PhysicalUnionScan{ Conditions: p.conditions, HandleCol: p.handleCol, }.Init(p.ctx, p.stats, p.blockOffset, childProp) - return []PhysicalPlan{us} + return []PhysicalPlan{us}, true } func getMaxSortPrefix(sortCols, allCols []*expression.Column) []int { @@ -1299,25 +1299,9 @@ func (p *LogicalJoin) tryToGetIndexJoin(prop *property.PhysicalProperty) (indexJ defer func() { // refine error message - if !canForced && needForced { - if hasINLMJHint && len(indexJoins) > 0 && len(prop.Items) > 0 { - containIdxMergeJoin := false - for _, idxJoin := range indexJoins { - if _, ok := idxJoin.(*PhysicalIndexMergeJoin); ok { - containIdxMergeJoin = true - break - } - } - // 1. IndexMergeJoin requires stricter conditions than Index(Hash)Join when the output order is needed. - // 2. IndexMergeJoin requires the same conditions with Index(Hash)Join when the output is unordered. - // 3. If ordered-Index(Hash)Join can be chosen but ordered-IndexMergeJoin can not be chosen, we can build a plan with an enforced sort on IndexMergeJoin. - // 4. Thus we can give up the plans here if IndexMergeJoin is nil when `hasINLMJHint` is true. Because we can make sure that an IndexMeregJoin with enforced sort will be built. - if !containIdxMergeJoin { - canForced = true - indexJoins = nil - return - } - } + // If the required property is not empty, we will enforce it and try the hint again. + // So we only need to generate warning message when the property is empty. + if !canForced && needForced && prop.IsEmpty() { // Construct warning message prefix. var errMsg string switch { @@ -1431,33 +1415,40 @@ func (p *LogicalJoin) tryToGetIndexJoin(prop *property.PhysicalProperty) (indexJ // Firstly we check the hint, if hint is figured by user, we force to choose the corresponding physical plan. // If the hint is not matched, it will get other candidates. // If the hint is not figured, we will pick all candidates. -func (p *LogicalJoin) exhaustPhysicalPlans(prop *property.PhysicalProperty) []PhysicalPlan { +func (p *LogicalJoin) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { failpoint.Inject("MockOnlyEnableIndexHashJoin", func(val failpoint.Value) { if val.(bool) { indexJoins, _ := p.tryToGetIndexJoin(prop) - failpoint.Return(indexJoins) + failpoint.Return(indexJoins, true) } }) mergeJoins := p.GetMergeJoin(prop, p.schema, p.Stats(), p.children[0].statsInfo(), p.children[1].statsInfo()) - if (p.preferJoinType & preferMergeJoin) > 0 { - return mergeJoins + if (p.preferJoinType&preferMergeJoin) > 0 && len(mergeJoins) > 0 { + return mergeJoins, true } joins := make([]PhysicalPlan, 0, 5) joins = append(joins, mergeJoins...) indexJoins, forced := p.tryToGetIndexJoin(prop) if forced { - return indexJoins + return indexJoins, true } joins = append(joins, indexJoins...) hashJoins := p.getHashJoins(prop) - if (p.preferJoinType & preferHashJoin) > 0 { - return hashJoins + if (p.preferJoinType&preferHashJoin) > 0 && len(hashJoins) > 0 { + return hashJoins, true } joins = append(joins, hashJoins...) - return joins + + if p.preferJoinType > 0 { + // If we reach here, it means we have a hint that doesn't work. + // It might be affected by the required property, so we enforce + // this property and try the hint again. + return joins, false + } + return joins, true } // TryToGetChildProp will check if this sort property can be pushed or not. @@ -1479,10 +1470,10 @@ func (p *LogicalProjection) TryToGetChildProp(prop *property.PhysicalProperty) ( return newProp, true } -func (p *LogicalProjection) exhaustPhysicalPlans(prop *property.PhysicalProperty) []PhysicalPlan { +func (p *LogicalProjection) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { newProp, ok := p.TryToGetChildProp(prop) if !ok { - return nil + return nil, true } proj := PhysicalProjection{ Exprs: p.Exprs, @@ -1490,7 +1481,7 @@ func (p *LogicalProjection) exhaustPhysicalPlans(prop *property.PhysicalProperty AvoidColumnEvaluator: p.AvoidColumnEvaluator, }.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), p.blockOffset, newProp) proj.SetSchema(p.schema) - return []PhysicalPlan{proj} + return []PhysicalPlan{proj}, true } func (lt *LogicalTopN) getPhysTopN() []PhysicalPlan { @@ -1538,11 +1529,11 @@ func MatchItems(p *property.PhysicalProperty, items []*ByItems) bool { return true } -func (lt *LogicalTopN) exhaustPhysicalPlans(prop *property.PhysicalProperty) []PhysicalPlan { +func (lt *LogicalTopN) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { if MatchItems(prop, lt.ByItems) { - return append(lt.getPhysTopN(), lt.getPhysLimits()...) + return append(lt.getPhysTopN(), lt.getPhysLimits()...), true } - return nil + return nil, true } // GetHashJoin is public for cascades planner. @@ -1550,9 +1541,9 @@ func (la *LogicalApply) GetHashJoin(prop *property.PhysicalProperty) *PhysicalHa return la.LogicalJoin.getHashJoin(prop, 1, false) } -func (la *LogicalApply) exhaustPhysicalPlans(prop *property.PhysicalProperty) []PhysicalPlan { +func (la *LogicalApply) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { if !prop.AllColsFromSchema(la.children[0].Schema()) { // for convenient, we don't pass through any prop - return nil + return nil, true } join := la.GetHashJoin(prop) apply := PhysicalApply{ @@ -1564,16 +1555,16 @@ func (la *LogicalApply) exhaustPhysicalPlans(prop *property.PhysicalProperty) [] &property.PhysicalProperty{ExpectedCnt: math.MaxFloat64, Items: prop.Items}, &property.PhysicalProperty{ExpectedCnt: math.MaxFloat64}) apply.SetSchema(la.schema) - return []PhysicalPlan{apply} + return []PhysicalPlan{apply}, true } -func (p *LogicalWindow) exhaustPhysicalPlans(prop *property.PhysicalProperty) []PhysicalPlan { +func (p *LogicalWindow) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { var byItems []property.Item byItems = append(byItems, p.PartitionBy...) byItems = append(byItems, p.OrderBy...) childProperty := &property.PhysicalProperty{ExpectedCnt: math.MaxFloat64, Items: byItems, Enforced: true} if !prop.IsPrefix(childProperty) { - return nil + return nil, true } window := PhysicalWindow{ WindowFuncDescs: p.WindowFuncDescs, @@ -1582,11 +1573,11 @@ func (p *LogicalWindow) exhaustPhysicalPlans(prop *property.PhysicalProperty) [] Frame: p.Frame, }.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), p.blockOffset, childProperty) window.SetSchema(p.Schema()) - return []PhysicalPlan{window} + return []PhysicalPlan{window}, true } // exhaustPhysicalPlans is only for implementing interface. DataSource and Dual generate task in `findBestTask` directly. -func (p *baseLogicalPlan) exhaustPhysicalPlans(_ *property.PhysicalProperty) []PhysicalPlan { +func (p *baseLogicalPlan) exhaustPhysicalPlans(_ *property.PhysicalProperty) ([]PhysicalPlan, bool) { panic("baseLogicalPlan.exhaustPhysicalPlans() should never be called.") } @@ -1748,7 +1739,7 @@ func (la *LogicalAggregation) ResetHintIfConflicted() (preferHash bool, preferSt return } -func (la *LogicalAggregation) exhaustPhysicalPlans(prop *property.PhysicalProperty) []PhysicalPlan { +func (la *LogicalAggregation) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { if la.aggHints.preferAggToCop { if !la.canPushToCop() { errMsg := "Optimizer Hint AGG_TO_COP is inapplicable" @@ -1762,37 +1753,36 @@ func (la *LogicalAggregation) exhaustPhysicalPlans(prop *property.PhysicalProper hashAggs := la.getHashAggs(prop) if hashAggs != nil && preferHash { - return hashAggs + return hashAggs, true } streamAggs := la.getStreamAggs(prop) if streamAggs != nil && preferStream { - return streamAggs + return streamAggs, true } - if streamAggs == nil && preferStream { + aggs := append(hashAggs, streamAggs...) + + if streamAggs == nil && preferStream && !prop.IsEmpty() { errMsg := "Optimizer Hint STREAM_AGG is inapplicable" warning := ErrInternal.GenWithStack(errMsg) la.ctx.GetSessionVars().StmtCtx.AppendWarning(warning) } - aggs := make([]PhysicalPlan, 0, len(hashAggs)+len(streamAggs)) - aggs = append(aggs, hashAggs...) - aggs = append(aggs, streamAggs...) - return aggs + return aggs, !(preferStream || preferHash) } -func (p *LogicalSelection) exhaustPhysicalPlans(prop *property.PhysicalProperty) []PhysicalPlan { +func (p *LogicalSelection) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { childProp := prop.Clone() sel := PhysicalSelection{ Conditions: p.Conditions, }.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), p.blockOffset, childProp) - return []PhysicalPlan{sel} + return []PhysicalPlan{sel}, true } -func (p *LogicalLimit) exhaustPhysicalPlans(prop *property.PhysicalProperty) []PhysicalPlan { +func (p *LogicalLimit) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { if !prop.IsEmpty() { - return nil + return nil, true } ret := make([]PhysicalPlan, 0, len(wholeTaskTypes)) for _, tp := range wholeTaskTypes { @@ -1803,23 +1793,23 @@ func (p *LogicalLimit) exhaustPhysicalPlans(prop *property.PhysicalProperty) []P }.Init(p.ctx, p.stats, p.blockOffset, resultProp) ret = append(ret, limit) } - return ret + return ret, true } -func (p *LogicalLock) exhaustPhysicalPlans(prop *property.PhysicalProperty) []PhysicalPlan { +func (p *LogicalLock) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { childProp := prop.Clone() lock := PhysicalLock{ Lock: p.Lock, TblID2Handle: p.tblID2Handle, PartitionedTable: p.partitionedTable, }.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), childProp) - return []PhysicalPlan{lock} + return []PhysicalPlan{lock}, true } -func (p *LogicalUnionAll) exhaustPhysicalPlans(prop *property.PhysicalProperty) []PhysicalPlan { +func (p *LogicalUnionAll) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { // TODO: UnionAll can not pass any order, but we can change it to sort merge to keep order. if !prop.IsEmpty() { - return nil + return nil, true } chReqProps := make([]*property.PhysicalProperty, 0, len(p.children)) for range p.children { @@ -1827,7 +1817,7 @@ func (p *LogicalUnionAll) exhaustPhysicalPlans(prop *property.PhysicalProperty) } ua := PhysicalUnionAll{}.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), p.blockOffset, chReqProps...) ua.SetSchema(p.Schema()) - return []PhysicalPlan{ua} + return []PhysicalPlan{ua}, true } func (ls *LogicalSort) getPhysicalSort(prop *property.PhysicalProperty) *PhysicalSort { @@ -1846,7 +1836,7 @@ func (ls *LogicalSort) getNominalSort(reqProp *property.PhysicalProperty) *Nomin return ps } -func (ls *LogicalSort) exhaustPhysicalPlans(prop *property.PhysicalProperty) []PhysicalPlan { +func (ls *LogicalSort) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { if MatchItems(prop, ls.ByItems) { ret := make([]PhysicalPlan, 0, 2) ret = append(ret, ls.getPhysicalSort(prop)) @@ -1854,15 +1844,15 @@ func (ls *LogicalSort) exhaustPhysicalPlans(prop *property.PhysicalProperty) []P if ns != nil { ret = append(ret, ns) } - return ret + return ret, true } - return nil + return nil, true } -func (p *LogicalMaxOneRow) exhaustPhysicalPlans(prop *property.PhysicalProperty) []PhysicalPlan { +func (p *LogicalMaxOneRow) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { if !prop.IsEmpty() { - return nil + return nil, true } mor := PhysicalMaxOneRow{}.Init(p.ctx, p.stats, p.blockOffset, &property.PhysicalProperty{ExpectedCnt: 2}) - return []PhysicalPlan{mor} + return []PhysicalPlan{mor}, true } diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 0aac06c699..578f226871 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -132,46 +132,9 @@ func (p *LogicalShowDDLJobs) findBestTask(prop *property.PhysicalProperty) (task return &rootTask{p: pShow}, nil } -// findBestTask implements LogicalPlan interface. -func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty) (bestTask task, err error) { - // If p is an inner plan in an IndexJoin, the IndexJoin will generate an inner plan by itself, - // and set inner child prop nil, so here we do nothing. - if prop == nil { - return nil, nil - } - // Look up the task with this prop in the task map. - // It's used to reduce double counting. - bestTask = p.getTask(prop) - if bestTask != nil { - return bestTask, nil - } - - if prop.TaskTp != property.RootTaskType { - // Currently all plan cannot totally push down. - p.storeTask(prop, invalidTask) - return invalidTask, nil - } - - bestTask = invalidTask +func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPlan, prop *property.PhysicalProperty) (task, error) { + var bestTask task = invalidTask childTasks := make([]task, 0, len(p.children)) - - // If prop.enforced is true, cols of prop as parameter in exhaustPhysicalPlans should be nil - // And reset it for enforcing task prop and storing map - oldPropCols := prop.Items - if prop.Enforced { - // First, get the bestTask without enforced prop - prop.Enforced = false - bestTask, err = p.findBestTask(prop) - if err != nil { - return nil, err - } - prop.Enforced = true - // Next, get the bestTask with enforced prop - prop.Items = []property.Item{} - } - physicalPlans := p.self.exhaustPhysicalPlans(prop) - prop.Items = oldPropCols - for _, pp := range physicalPlans { // find best child tasks firstly. childTasks = childTasks[:0] @@ -205,10 +168,84 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty) (bestTas } // get the most efficient one. - if curTask.cost() < bestTask.cost() { + if curTask.cost() < bestTask.cost() || (bestTask.invalid() && !curTask.invalid()) { bestTask = curTask } } + return bestTask, nil +} + +// findBestTask implements LogicalPlan interface. +func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty) (bestTask task, err error) { + // If p is an inner plan in an IndexJoin, the IndexJoin will generate an inner plan by itself, + // and set inner child prop nil, so here we do nothing. + if prop == nil { + return nil, nil + } + // Look up the task with this prop in the task map. + // It's used to reduce double counting. + bestTask = p.getTask(prop) + if bestTask != nil { + return bestTask, nil + } + + if prop.TaskTp != property.RootTaskType { + // Currently all plan cannot totally push down. + p.storeTask(prop, invalidTask) + return invalidTask, nil + } + + bestTask = invalidTask + // prop should be read only because its cached hashcode might be not consistent + // when it is changed. So we clone a new one for the temporary changes. + newProp := prop.Clone() + newProp.Enforced = prop.Enforced + var plansFitsProp, plansNeedEnforce []PhysicalPlan + var hintWorksWithProp bool + // Maybe the plan can satisfy the required property, + // so we try to get the task without the enforced sort first. + plansFitsProp, hintWorksWithProp = p.self.exhaustPhysicalPlans(newProp) + if !hintWorksWithProp && !newProp.IsEmpty() { + // If there is a hint in the plan and the hint cannot satisfy the property, + // we enforce this property and try to generate the PhysicalPlan again to + // make sure the hint can work. + newProp.Enforced = true + } + + if newProp.Enforced { + // Then, we use the empty property to get physicalPlans and + // try to get the task with an enforced sort. + newProp.Items = []property.Item{} + newProp.ExpectedCnt = math.MaxFloat64 + var hintCanWork bool + plansNeedEnforce, hintCanWork = p.self.exhaustPhysicalPlans(newProp) + if hintCanWork && !hintWorksWithProp { + // If the hint can work with the empty property, but cannot work with + // the required property, we give up `plansFitProp` to make sure the hint + // can work. + plansFitsProp = nil + } + if !hintCanWork && !hintWorksWithProp && !prop.Enforced { + // If the original property is not enforced and hint cannot + // work anyway, we give up `plansNeedEnforce` for efficiency, + plansNeedEnforce = nil + } + newProp.Items = prop.Items + newProp.ExpectedCnt = prop.ExpectedCnt + } + + newProp.Enforced = false + if bestTask, err = p.enumeratePhysicalPlans4Task(plansFitsProp, newProp); err != nil { + return nil, err + } + newProp.Enforced = true + curTask, err := p.enumeratePhysicalPlans4Task(plansNeedEnforce, newProp) + if err != nil { + return nil, err + } + if curTask.cost() < bestTask.cost() || (bestTask.invalid() && !curTask.invalid()) { + bestTask = curTask + } p.storeTask(prop, bestTask) return bestTask, nil diff --git a/planner/core/find_best_task_test.go b/planner/core/find_best_task_test.go new file mode 100644 index 0000000000..701376b035 --- /dev/null +++ b/planner/core/find_best_task_test.go @@ -0,0 +1,274 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import ( + "fmt" + "math" + + . "github.com/pingcap/check" + "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/planner/property" + "github.com/pingcap/tidb/sessionctx" +) + +var _ = Suite(&testFindBestTaskSuite{}) + +type testFindBestTaskSuite struct { + ctx sessionctx.Context +} + +func (s *testFindBestTaskSuite) SetUpSuite(c *C) { + s.ctx = MockContext() +} + +type mockDataSource struct { + baseLogicalPlan +} + +func (ds mockDataSource) Init(ctx sessionctx.Context) *mockDataSource { + ds.baseLogicalPlan = newBaseLogicalPlan(ctx, "mockDS", &ds, 0) + return &ds +} + +func (ds *mockDataSource) findBestTask(prop *property.PhysicalProperty) (task, error) { + // It can satisfy any of the property! + // Just use a TableDual for convenience. + p := PhysicalTableDual{}.Init(ds.ctx, &property.StatsInfo{RowCount: 1}, 0) + task := &rootTask{ + p: p, + cst: 10000, + } + return task, nil +} + +// mockLogicalPlan4Test is a LogicalPlan which is used for unit test. +// The basic assumption: +// 1. mockLogicalPlan4Test can generate tow kinds of physical plan: physicalPlan1 and +// physicalPlan2. physicalPlan1 can pass the property only when they are the same +// order; while physicalPlan2 cannot match any of the property(in other words, we can +// generate it only when then property is empty). +// 2. We have a hint for physicalPlan2. +// 3. If the property is empty, we still need to check `canGeneratePlan2` to decide +// whether it can generate physicalPlan2. +type mockLogicalPlan4Test struct { + baseLogicalPlan + // hasHintForPlan2 indicates whether this mockPlan contains hint. + // This hint is used to generate physicalPlan2. See the implementation + // of exhaustPhysicalPlans(). + hasHintForPlan2 bool + // canGeneratePlan2 indicates whether this plan can generate physicalPlan2. + canGeneratePlan2 bool + // costOverflow indicates whether this plan will generate physical plan whose cost is overflowed. + costOverflow bool +} + +func (p mockLogicalPlan4Test) Init(ctx sessionctx.Context) *mockLogicalPlan4Test { + p.baseLogicalPlan = newBaseLogicalPlan(ctx, "mockPlan", &p, 0) + return &p +} + +func (p *mockLogicalPlan4Test) getPhysicalPlan1(prop *property.PhysicalProperty) PhysicalPlan { + physicalPlan1 := mockPhysicalPlan4Test{planType: 1, costOverflow: p.costOverflow}.Init(p.ctx) + physicalPlan1.stats = &property.StatsInfo{RowCount: 1} + physicalPlan1.childrenReqProps = make([]*property.PhysicalProperty, 1) + physicalPlan1.childrenReqProps[0] = prop.Clone() + return physicalPlan1 +} + +func (p *mockLogicalPlan4Test) getPhysicalPlan2(prop *property.PhysicalProperty) PhysicalPlan { + physicalPlan2 := mockPhysicalPlan4Test{planType: 2, costOverflow: p.costOverflow}.Init(p.ctx) + physicalPlan2.stats = &property.StatsInfo{RowCount: 1} + physicalPlan2.childrenReqProps = make([]*property.PhysicalProperty, 1) + physicalPlan2.childrenReqProps[0] = property.NewPhysicalProperty(prop.TaskTp, nil, false, prop.ExpectedCnt, false) + return physicalPlan2 +} + +func (p *mockLogicalPlan4Test) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { + plan1 := make([]PhysicalPlan, 0, 1) + plan2 := make([]PhysicalPlan, 0, 1) + if prop.IsEmpty() && p.canGeneratePlan2 { + // Generate PhysicalPlan2 when the property is empty. + plan2 = append(plan2, p.getPhysicalPlan2(prop)) + if p.hasHintForPlan2 { + return plan2, true + } + } + if all, _ := prop.AllSameOrder(); all { + // Generate PhysicalPlan1 when properties are the same order. + plan1 = append(plan1, p.getPhysicalPlan1(prop)) + } + if p.hasHintForPlan2 { + // The hint cannot work. + if prop.IsEmpty() { + p.ctx.GetSessionVars().StmtCtx.AppendWarning(fmt.Errorf("the hint is inapplicable for plan2")) + } + return plan1, false + } + return append(plan1, plan2...), true +} + +type mockPhysicalPlan4Test struct { + basePhysicalPlan + // 1 or 2 for physicalPlan1 or physicalPlan2. + // See the comment of mockLogicalPlan4Test. + planType int + costOverflow bool +} + +func (p mockPhysicalPlan4Test) Init(ctx sessionctx.Context) *mockPhysicalPlan4Test { + p.basePhysicalPlan = newBasePhysicalPlan(ctx, "mockPlan", &p, 0) + return &p +} + +func (p *mockPhysicalPlan4Test) attach2Task(tasks ...task) task { + t := tasks[0].copy() + attachPlan2Task(p, t) + if p.costOverflow { + t.addCost(math.MaxFloat64) + } else { + t.addCost(1) + } + return t +} + +func (s *testFindBestTaskSuite) TestCostOverflow(c *C) { + ctx := MockContext() + // Plan Tree: mockPlan -> mockDataSource + mockPlan := mockLogicalPlan4Test{costOverflow: true}.Init(ctx) + mockDS := mockDataSource{}.Init(ctx) + mockPlan.SetChildren(mockDS) + // An empty property is enough for this test. + prop := property.NewPhysicalProperty(property.RootTaskType, nil, false, 0, false) + t, err := mockPlan.findBestTask(prop) + c.Assert(err, IsNil) + // The cost should be overflowed, but the task shouldn't be invalid. + c.Assert(t.invalid(), IsFalse) + c.Assert(t.cost(), Equals, math.MaxFloat64) +} + +func (s *testFindBestTaskSuite) TestEnforcedProperty(c *C) { + ctx := MockContext() + // PlanTree : mockLogicalPlan -> mockDataSource + mockPlan := mockLogicalPlan4Test{}.Init(ctx) + mockDS := mockDataSource{}.Init(ctx) + mockPlan.SetChildren(mockDS) + + col0 := &expression.Column{UniqueID: 1} + col1 := &expression.Column{UniqueID: 2} + // Use different order, so that mockLogicalPlan cannot generate any of the + // physical plans. + item0 := property.Item{Col: col0, Desc: false} + item1 := property.Item{Col: col1, Desc: true} + items := []property.Item{item0, item1} + + prop0 := &property.PhysicalProperty{ + Items: items, + Enforced: false, + } + // should return invalid task because no physical plan can match this property. + task, err := mockPlan.findBestTask(prop0) + c.Assert(err, IsNil) + c.Assert(task.invalid(), IsTrue) + + prop1 := &property.PhysicalProperty{ + Items: items, + Enforced: true, + } + // should return the valid task when the property is enforced. + task, err = mockPlan.findBestTask(prop1) + c.Assert(err, IsNil) + c.Assert(task.invalid(), IsFalse) +} + +func (s *testFindBestTaskSuite) TestHintCannotFitProperty(c *C) { + ctx := MockContext() + // PlanTree : mockLogicalPlan -> mockDataSource + mockPlan0 := mockLogicalPlan4Test{ + hasHintForPlan2: true, + canGeneratePlan2: true, + }.Init(ctx) + mockDS := mockDataSource{}.Init(ctx) + mockPlan0.SetChildren(mockDS) + + col0 := &expression.Column{UniqueID: 1} + item0 := property.Item{Col: col0} + items := []property.Item{item0} + // case 1, The property is not empty and enforced, should enforce a sort. + prop0 := &property.PhysicalProperty{ + Items: items, + Enforced: true, + } + task, err := mockPlan0.findBestTask(prop0) + c.Assert(err, IsNil) + c.Assert(task.invalid(), IsFalse) + _, enforcedSort := task.plan().(*PhysicalSort) + c.Assert(enforcedSort, IsTrue) + plan2 := task.plan().Children()[0] + mockPhysicalPlan, ok := plan2.(*mockPhysicalPlan4Test) + c.Assert(ok, IsTrue) + c.Assert(mockPhysicalPlan.planType, Equals, 2) + + // case 2, The property is not empty but not enforced, still need to enforce a sort + // to ensure the hint can work + prop1 := &property.PhysicalProperty{ + Items: items, + Enforced: false, + } + task, err = mockPlan0.findBestTask(prop1) + c.Assert(err, IsNil) + c.Assert(task.invalid(), IsFalse) + _, enforcedSort = task.plan().(*PhysicalSort) + c.Assert(enforcedSort, IsTrue) + plan2 = task.plan().Children()[0] + mockPhysicalPlan, ok = plan2.(*mockPhysicalPlan4Test) + c.Assert(ok, IsTrue) + c.Assert(mockPhysicalPlan.planType, Equals, 2) + + // case 3, The hint cannot work even if the property is empty, should return a warning + // and generate physicalPlan1. + prop2 := &property.PhysicalProperty{ + Items: items, + Enforced: false, + } + mockPlan1 := mockLogicalPlan4Test{ + hasHintForPlan2: true, + canGeneratePlan2: false, + }.Init(ctx) + mockPlan1.SetChildren(mockDS) + task, err = mockPlan1.findBestTask(prop2) + c.Assert(err, IsNil) + c.Assert(task.invalid(), IsFalse) + c.Assert(ctx.GetSessionVars().StmtCtx.WarningCount(), Equals, uint16(1)) + // Because physicalPlan1 can match the property, so we should get it. + mockPhysicalPlan, ok = task.plan().(*mockPhysicalPlan4Test) + c.Assert(ok, IsTrue) + c.Assert(mockPhysicalPlan.planType, Equals, 1) + + // case 4, Similar to case 3, but the property is enforced now. Ths result should be + // the same with case 3. + ctx.GetSessionVars().StmtCtx.SetWarnings(nil) + prop3 := &property.PhysicalProperty{ + Items: items, + Enforced: true, + } + task, err = mockPlan1.findBestTask(prop3) + c.Assert(err, IsNil) + c.Assert(task.invalid(), IsFalse) + c.Assert(ctx.GetSessionVars().StmtCtx.WarningCount(), Equals, uint16(1)) + // Because physicalPlan1 can match the property, so we don't need to enforce a sort. + mockPhysicalPlan, ok = task.plan().(*mockPhysicalPlan4Test) + c.Assert(ok, IsTrue) + c.Assert(mockPhysicalPlan.planType, Equals, 1) +} diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index f23a381cd0..847ab6077d 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -749,6 +749,37 @@ func (s *testIntegrationSuite) TestIssue15546(c *C) { tk.MustQuery("select * from pt, vt where pt.a = vt.a").Check(testkit.Rows("1 1 1 1")) } +func (s *testIntegrationSuite) TestHintWithRequiredProperty(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int primary key, b int, c int, key b(b))") + var input []string + var output []struct { + SQL string + Plan []string + Warnings []string + } + s.testData.GetTestCases(c, &input, &output) + for i, tt := range input { + s.testData.OnRecord(func() { + output[i].SQL = tt + output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Rows()) + warnings := tk.Se.GetSessionVars().StmtCtx.GetWarnings() + output[i].Warnings = make([]string, len(warnings)) + for j, warning := range warnings { + output[i].Warnings[j] = warning.Err.Error() + } + }) + tk.MustQuery(tt).Check(testkit.Rows(output[i].Plan...)) + warnings := tk.Se.GetSessionVars().StmtCtx.GetWarnings() + c.Assert(len(warnings), Equals, len(output[i].Warnings)) + for j, warning := range warnings { + c.Assert(output[i].Warnings[j], Equals, warning.Err.Error()) + } + } +} + func (s *testIntegrationSuite) TestIssue15813(c *C) { tk := testkit.NewTestKit(c, s.store) diff --git a/planner/core/plan.go b/planner/core/plan.go index d23d8a151d..441027ee5f 100644 --- a/planner/core/plan.go +++ b/planner/core/plan.go @@ -177,7 +177,10 @@ type LogicalPlan interface { PreparePossibleProperties(schema *expression.Schema, childrenProperties ...[][]*expression.Column) [][]*expression.Column // exhaustPhysicalPlans generates all possible plans that can match the required property. - exhaustPhysicalPlans(*property.PhysicalProperty) []PhysicalPlan + // It will return: + // 1. All possible plans that can match the required property. + // 2. Whether the SQL hint can work. Return true if there is no hint. + exhaustPhysicalPlans(*property.PhysicalProperty) (physicalPlans []PhysicalPlan, hintCanWork bool) // ExtractCorrelatedCols extracts correlated columns inside the LogicalPlan. ExtractCorrelatedCols() []*expression.CorrelatedColumn diff --git a/planner/core/testdata/integration_suite_in.json b/planner/core/testdata/integration_suite_in.json index 9c75f22a40..18c95deffd 100644 --- a/planner/core/testdata/integration_suite_in.json +++ b/planner/core/testdata/integration_suite_in.json @@ -73,6 +73,18 @@ "desc select /*+ TIDB_INLJ(t2)*/ * from t1, t2 where t1.a = t2.a and t1.b = t2.a and t1.b = t2.b" ] }, + { + "name": "TestHintWithRequiredProperty", + "cases": [ + "desc select /*+ INL_JOIN(t2) */ * from t t1, t t2 where t1.a = t2.b order by t2.a", + "desc select /*+ INL_HASH_JOIN(t2) */ * from t t1, t t2 where t1.a = t2.b order by t2.a", + "desc select /*+ INL_MERGE_JOIN(t2)*/ t1.a, t2.a from t t1, t t2 ,t t3 where t1.a = t2.a and t3.a=t2.a", + "desc select * from t t1, (select /*+ HASH_AGG() */ b, max(a) from t t2 group by b) t2 where t1.b = t2.b order by t1.b", + "desc select /*+ INL_HASH_JOIN(t2) */ distinct t2.a from t t1 join t t2 on t1.a = t2.a", + // This hint cannot work, so choose another plan. + "desc select /*+ INL_JOIN(t2) */ * from t t1, t t2 where t1.a = t2.c order by t1.a" + ] + }, { "name": "TestIndexHintWarning", "cases": [ diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index d65c621ab7..81faaa6637 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -272,6 +272,97 @@ } ] }, + { + "Name": "TestHintWithRequiredProperty", + "Cases": [ + { + "SQL": "desc select /*+ INL_JOIN(t2) */ * from t t1, t t2 where t1.a = t2.b order by t2.a", + "Plan": [ + "Sort_7 12487.50 root test.t.a:asc", + "└─IndexJoin_15 12487.50 root inner join, inner:IndexLookUp_14, outer key:test.t.a, inner key:test.t.b", + " ├─TableReader_25(Build) 10000.00 root data:TableFullScan_24", + " │ └─TableFullScan_24 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo", + " └─IndexLookUp_14(Probe) 1.25 root ", + " ├─Selection_13(Build) 1.25 cop[tikv] not(isnull(test.t.b))", + " │ └─IndexRangeScan_11 1.25 cop[tikv] table:t2, index:b(b) range: decided by [eq(test.t.b, test.t.a)], keep order:false, stats:pseudo", + " └─TableRowIDScan_12(Probe) 1.25 cop[tikv] table:t2 keep order:false, stats:pseudo" + ], + "Warnings": [] + }, + { + "SQL": "desc select /*+ INL_HASH_JOIN(t2) */ * from t t1, t t2 where t1.a = t2.b order by t2.a", + "Plan": [ + "Sort_7 12487.50 root test.t.a:asc", + "└─IndexHashJoin_23 12487.50 root inner join, inner:IndexLookUp_14, outer key:test.t.a, inner key:test.t.b", + " ├─TableReader_25(Build) 10000.00 root data:TableFullScan_24", + " │ └─TableFullScan_24 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo", + " └─IndexLookUp_14(Probe) 1.25 root ", + " ├─Selection_13(Build) 1.25 cop[tikv] not(isnull(test.t.b))", + " │ └─IndexRangeScan_11 1.25 cop[tikv] table:t2, index:b(b) range: decided by [eq(test.t.b, test.t.a)], keep order:false, stats:pseudo", + " └─TableRowIDScan_12(Probe) 1.25 cop[tikv] table:t2 keep order:false, stats:pseudo" + ], + "Warnings": [] + }, + { + "SQL": "desc select /*+ INL_MERGE_JOIN(t2)*/ t1.a, t2.a from t t1, t t2 ,t t3 where t1.a = t2.a and t3.a=t2.a", + "Plan": [ + "HashJoin_21 15625.00 root inner join, equal:[eq(test.t.a, test.t.a)]", + "├─TableReader_60(Build) 10000.00 root data:TableFullScan_59", + "│ └─TableFullScan_59 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo", + "└─IndexMergeJoin_56(Probe) 12500.00 root inner join, inner:TableReader_54, outer key:test.t.a, inner key:test.t.a", + " ├─TableReader_45(Build) 10000.00 root data:TableFullScan_44", + " │ └─TableFullScan_44 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo", + " └─TableReader_54(Probe) 1.00 root data:TableRangeScan_53", + " └─TableRangeScan_53 1.00 cop[tikv] table:t2 range: decided by [test.t.a], keep order:true, stats:pseudo" + ], + "Warnings": [] + }, + { + "SQL": "desc select * from t t1, (select /*+ HASH_AGG() */ b, max(a) from t t2 group by b) t2 where t1.b = t2.b order by t1.b", + "Plan": [ + "Sort_10 9990.00 root test.t.b:asc", + "└─Projection_12 9990.00 root test.t.a, test.t.b, test.t.c, test.t.b, Column#7", + " └─HashJoin_27 9990.00 root inner join, equal:[eq(test.t.b, test.t.b)]", + " ├─HashAgg_47(Build) 7992.00 root group by:test.t.b, funcs:max(Column#10)->Column#7, funcs:firstrow(test.t.b)->test.t.b", + " │ └─IndexReader_48 7992.00 root index:HashAgg_44", + " │ └─HashAgg_44 7992.00 cop[tikv] group by:test.t.b, funcs:max(test.t.a)->Column#10", + " │ └─IndexFullScan_37 9990.00 cop[tikv] table:t2, index:b(b) keep order:false, stats:pseudo", + " └─TableReader_51(Probe) 9990.00 root data:Selection_50", + " └─Selection_50 9990.00 cop[tikv] not(isnull(test.t.b))", + " └─TableFullScan_49 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo" + ], + "Warnings": [] + }, + { + "SQL": "desc select /*+ INL_HASH_JOIN(t2) */ distinct t2.a from t t1 join t t2 on t1.a = t2.a", + "Plan": [ + "HashAgg_8 8000.00 root group by:test.t.a, funcs:firstrow(test.t.a)->test.t.a", + "└─IndexHashJoin_20 12500.00 root inner join, inner:TableReader_13, outer key:test.t.a, inner key:test.t.a", + " ├─TableReader_22(Build) 10000.00 root data:TableFullScan_21", + " │ └─TableFullScan_21 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo", + " └─TableReader_13(Probe) 1.00 root data:TableRangeScan_12", + " └─TableRangeScan_12 1.00 cop[tikv] table:t2 range: decided by [test.t.a], keep order:false, stats:pseudo" + ], + "Warnings": [] + }, + { + "SQL": "desc select /*+ INL_JOIN(t2) */ * from t t1, t t2 where t1.a = t2.c order by t1.a", + "Plan": [ + "Sort_7 12487.50 root test.t.a:asc", + "└─HashJoin_19 12487.50 root inner join, equal:[eq(test.t.a, test.t.c)]", + " ├─TableReader_23(Build) 9990.00 root data:Selection_22", + " │ └─Selection_22 9990.00 cop[tikv] not(isnull(test.t.c))", + " │ └─TableFullScan_21 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", + " └─TableReader_25(Probe) 10000.00 root data:TableFullScan_24", + " └─TableFullScan_24 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo" + ], + "Warnings": [ + "[planner:1815]Optimizer Hint /*+ INL_JOIN(t2) */ or /*+ TIDB_INLJ(t2) */ is inapplicable", + "[planner:1815]Optimizer Hint /*+ INL_JOIN(t2) */ or /*+ TIDB_INLJ(t2) */ is inapplicable" + ] + } + ] + }, { "Name": "TestIndexHintWarning", "Cases": [ diff --git a/planner/core/testdata/plan_suite_out.json b/planner/core/testdata/plan_suite_out.json index 76fba37325..2207c84c06 100644 --- a/planner/core/testdata/plan_suite_out.json +++ b/planner/core/testdata/plan_suite_out.json @@ -1229,7 +1229,7 @@ { "SQL": "select /*+ STREAM_AGG() */ sum(t1.a) from t t1 join t t2 on t1.b = t2.b group by t1.b", "Best": "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))->Sort->Projection->StreamAgg}(test.t.b,test.t.b)->HashAgg", - "Warning": "[planner:1815]Optimizer Hint STREAM_AGG is inapplicable" + "Warning": "" }, { "SQL": "select /*+ STREAM_AGG() */ e, sum(b) from t group by e",