database/tidb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

planner/core: fix a bug of adding enforcer. (#22086)

Browse Source
This commit is contained in:
Han Fei
2021-01-05 17:01:39 +08:00
committed by GitHub
parent 029eef87ed
commit 6742ed201d
10 changed files with 130 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
planner/cascades/implementation_rules.go
View File

@ -272,7 +272,7 @@ func (r *ImplSelection) OnImplement(expr *memo.GroupExpr, reqProp *property.Phys
logicalSel := expr.ExprNode.(*plannercore.LogicalSelection)
physicalSel := plannercore.PhysicalSelection{
Conditions: logicalSel.Conditions,
}.Init(logicalSel.SCtx(), expr.Group.Prop.Stats.ScaleByExpectCnt(reqProp.ExpectedCnt), logicalSel.SelectBlockOffset(), reqProp.Clone())
}.Init(logicalSel.SCtx(), expr.Group.Prop.Stats.ScaleByExpectCnt(reqProp.ExpectedCnt), logicalSel.SelectBlockOffset(), reqProp.CloneEssentialFields())
switch expr.Group.EngineType {
case memo.EngineTiDB:
return []memo.Implementation{impl.NewTiDBSelectionImpl(physicalSel)}, nil

20
planner/core/exhaust_physical_plans.go
View File

@ -44,7 +44,7 @@ func (p *LogicalUnionScan) exhaustPhysicalPlans(prop *property.PhysicalProperty)
if prop.IsFlashProp() {
return nil, true
}
childProp := prop.Clone()
childProp := prop.CloneEssentialFields()
us := PhysicalUnionScan{
Conditions: p.conditions,
HandleCols: p.handleCols,
@ -1736,7 +1736,7 @@ func (p *LogicalJoin) tryToGetMppHashJoin(prop *property.PhysicalProperty, useBC
baseJoin.InnerChildIdx = preferredBuildIndex
childrenProps := make([]*property.PhysicalProperty, 2)
if useBCJ {
childrenProps[preferredBuildIndex] = &property.PhysicalProperty{TaskTp: property.MppTaskType, ExpectedCnt: math.MaxFloat64, PartitionTp: property.BroadcastType, Enforced: true}
childrenProps[preferredBuildIndex] = &property.PhysicalProperty{TaskTp: property.MppTaskType, ExpectedCnt: math.MaxFloat64, PartitionTp: property.BroadcastType, CanAddEnforcer: true}
expCnt := math.MaxFloat64
if prop.ExpectedCnt < p.stats.RowCount {
expCntScale := prop.ExpectedCnt / p.stats.RowCount
@ -1767,8 +1767,8 @@ func (p *LogicalJoin) tryToGetMppHashJoin(prop *property.PhysicalProperty, useBC
lkeys = chooseSubsetOfJoinKeys(lkeys, matches)
rkeys = chooseSubsetOfJoinKeys(rkeys, matches)
}
childrenProps[0] = &property.PhysicalProperty{TaskTp: property.MppTaskType, ExpectedCnt: math.MaxFloat64, PartitionTp: property.HashType, PartitionCols: lkeys, Enforced: true}
childrenProps[1] = &property.PhysicalProperty{TaskTp: property.MppTaskType, ExpectedCnt: math.MaxFloat64, PartitionTp: property.HashType, PartitionCols: rkeys, Enforced: true}
childrenProps[0] = &property.PhysicalProperty{TaskTp: property.MppTaskType, ExpectedCnt: math.MaxFloat64, PartitionTp: property.HashType, PartitionCols: lkeys, CanAddEnforcer: true}
childrenProps[1] = &property.PhysicalProperty{TaskTp: property.MppTaskType, ExpectedCnt: math.MaxFloat64, PartitionTp: property.HashType, PartitionCols: rkeys, CanAddEnforcer: true}
}
join := PhysicalHashJoin{
basePhysicalJoin: baseJoin,
@ -2074,7 +2074,7 @@ func (p *LogicalWindow) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([
var byItems []property.SortItem
byItems = append(byItems, p.PartitionBy...)
byItems = append(byItems, p.OrderBy...)
childProperty := &property.PhysicalProperty{ExpectedCnt: math.MaxFloat64, SortItems: byItems, Enforced: true}
childProperty := &property.PhysicalProperty{ExpectedCnt: math.MaxFloat64, SortItems: byItems, CanAddEnforcer: true}
if !prop.IsPrefix(childProperty) {
return nil, true
}
@ -2111,9 +2111,9 @@ func (la *LogicalAggregation) getEnforcedStreamAggs(prop *property.PhysicalPrope
allTaskTypes := prop.GetAllPossibleChildTaskTypes()
enforcedAggs := make([]PhysicalPlan, 0, len(allTaskTypes))
childProp := &property.PhysicalProperty{
ExpectedCnt: math.Max(prop.ExpectedCnt*la.inputCount/la.stats.RowCount, prop.ExpectedCnt),
Enforced: true,
SortItems: property.SortItemsFromCols(la.GetGroupByCols(), desc),
ExpectedCnt: math.Max(prop.ExpectedCnt*la.inputCount/la.stats.RowCount, prop.ExpectedCnt),
CanAddEnforcer: true,
SortItems: property.SortItemsFromCols(la.GetGroupByCols(), desc),
}
if !prop.IsPrefix(childProp) {
return enforcedAggs
@ -2314,7 +2314,7 @@ func (la *LogicalAggregation) exhaustPhysicalPlans(prop *property.PhysicalProper
}
func (p *LogicalSelection) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) {
childProp := prop.Clone()
childProp := prop.CloneEssentialFields()
sel := PhysicalSelection{
Conditions: p.Conditions,
}.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), p.blockOffset, childProp)
@ -2366,7 +2366,7 @@ func (p *LogicalLock) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([]P
if prop.IsFlashProp() {
return nil, true
}
childProp := prop.Clone()
childProp := prop.CloneEssentialFields()
lock := PhysicalLock{
Lock: p.Lock,
TblID2Handle: p.tblID2Handle,

31
planner/core/find_best_task.go
View File

@ -200,7 +200,7 @@ func (p *baseLogicalPlan) rebuildChildTasks(childTasks *[]task, pp PhysicalPlan,
return nil
}
func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPlan, prop *property.PhysicalProperty, planCounter *PlanCounterTp) (task, int64, error) {
func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPlan, prop *property.PhysicalProperty, addEnforcer bool, planCounter *PlanCounterTp) (task, int64, error) {
var bestTask task = invalidTask
var curCntPlan, cntPlan int64
childTasks := make([]task, 0, len(p.children))
@ -249,7 +249,7 @@ func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPl
}
// Enforce curTask property
if prop.Enforced {
if addEnforcer {
curTask = enforceProperty(prop, curTask, p.basePlan.ctx)
}
@ -290,6 +290,8 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun
return bestTask, 1, nil
}
canAddEnforcer := prop.CanAddEnforcer
if prop.TaskTp != property.RootTaskType && !prop.IsFlashProp() {
// Currently all plan cannot totally push down to TiKV.
p.storeTask(prop, invalidTask)
@ -300,7 +302,7 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun
cntPlan = 0
// prop should be read only because its cached hashcode might be not consistent
// when it is changed. So we clone a new one for the temporary changes.
newProp := prop.Clone()
newProp := prop.CloneEssentialFields()
var plansFitsProp, plansNeedEnforce []PhysicalPlan
var hintWorksWithProp bool
// Maybe the plan can satisfy the required property,
@ -310,10 +312,10 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun
// If there is a hint in the plan and the hint cannot satisfy the property,
// we enforce this property and try to generate the PhysicalPlan again to
// make sure the hint can work.
newProp.Enforced = true
canAddEnforcer = true
}
if newProp.Enforced {
if canAddEnforcer {
// Then, we use the empty property to get physicalPlans and
// try to get the task with an enforced sort.
newProp.SortItems = []property.SortItem{}
@ -328,7 +330,7 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun
// can work.
plansFitsProp = nil
}
if !hintCanWork && !hintWorksWithProp && !prop.Enforced {
if !hintCanWork && !hintWorksWithProp && !prop.CanAddEnforcer {
// If the original property is not enforced and hint cannot
// work anyway, we give up `plansNeedEnforce` for efficiency,
plansNeedEnforce = nil
@ -336,10 +338,9 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun
newProp = prop
}
newProp.Enforced = false
var cnt int64
var curTask task
if bestTask, cnt, err = p.enumeratePhysicalPlans4Task(plansFitsProp, newProp, planCounter); err != nil {
if bestTask, cnt, err = p.enumeratePhysicalPlans4Task(plansFitsProp, newProp, false, planCounter); err != nil {
return nil, 0, err
}
cntPlan += cnt
@ -347,8 +348,7 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun
goto END
}
newProp.Enforced = true
curTask, cnt, err = p.enumeratePhysicalPlans4Task(plansNeedEnforce, newProp, planCounter)
curTask, cnt, err = p.enumeratePhysicalPlans4Task(plansNeedEnforce, newProp, true, planCounter)
if err != nil {
return nil, 0, err
}
@ -617,15 +617,15 @@ func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter
var cnt int64
// If prop.enforced is true, the prop.cols need to be set nil for ds.findBestTask.
// Before function return, reset it for enforcing task prop and storing map<prop,task>.
oldProp := prop.Clone()
if prop.Enforced {
oldProp := prop.CloneEssentialFields()
if prop.CanAddEnforcer {
// First, get the bestTask without enforced prop
prop.Enforced = false
prop.CanAddEnforcer = false
t, cnt, err = ds.findBestTask(prop, planCounter)
if err != nil {
return nil, 0, err
}
prop.Enforced = true
prop.CanAddEnforcer = true
if t != invalidTask {
ds.storeTask(prop, t)
cntPlan = cnt
@ -641,9 +641,10 @@ func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter
if err != nil {
return
}
if prop.Enforced {
if prop.CanAddEnforcer {
prop = oldProp
t = enforceProperty(prop, t, ds.basePlan.ctx)
prop.CanAddEnforcer = true
}
ds.storeTask(prop, t)
if ds.SampleInfo != nil && !t.invalid() {

26
planner/core/find_best_task_test.go
View File

@ -84,7 +84,7 @@ func (p *mockLogicalPlan4Test) getPhysicalPlan1(prop *property.PhysicalProperty)
physicalPlan1 := mockPhysicalPlan4Test{planType: 1, costOverflow: p.costOverflow}.Init(p.ctx)
physicalPlan1.stats = &property.StatsInfo{RowCount: 1}
physicalPlan1.childrenReqProps = make([]*property.PhysicalProperty, 1)
physicalPlan1.childrenReqProps[0] = prop.Clone()
physicalPlan1.childrenReqProps[0] = prop.CloneEssentialFields()
return physicalPlan1
}
@ -175,8 +175,8 @@ func (s *testFindBestTaskSuite) TestEnforcedProperty(c *C) {
items := []property.SortItem{item0, item1}
prop0 := &property.PhysicalProperty{
SortItems: items,
Enforced: false,
SortItems: items,
CanAddEnforcer: false,
}
// should return invalid task because no physical plan can match this property.
task, _, err := mockPlan.findBestTask(prop0, &PlanCounterDisabled)
@ -184,8 +184,8 @@ func (s *testFindBestTaskSuite) TestEnforcedProperty(c *C) {
c.Assert(task.invalid(), IsTrue)
prop1 := &property.PhysicalProperty{
SortItems: items,
Enforced: true,
SortItems: items,
CanAddEnforcer: true,
}
// should return the valid task when the property is enforced.
task, _, err = mockPlan.findBestTask(prop1, &PlanCounterDisabled)
@ -208,8 +208,8 @@ func (s *testFindBestTaskSuite) TestHintCannotFitProperty(c *C) {
items := []property.SortItem{item0}
// case 1, The property is not empty and enforced, should enforce a sort.
prop0 := &property.PhysicalProperty{
SortItems: items,
Enforced: true,
SortItems: items,
CanAddEnforcer: true,
}
task, _, err := mockPlan0.findBestTask(prop0, &PlanCounterDisabled)
c.Assert(err, IsNil)
@ -224,8 +224,8 @@ func (s *testFindBestTaskSuite) TestHintCannotFitProperty(c *C) {
// case 2, The property is not empty but not enforced, still need to enforce a sort
// to ensure the hint can work
prop1 := &property.PhysicalProperty{
SortItems: items,
Enforced: false,
SortItems: items,
CanAddEnforcer: false,
}
task, _, err = mockPlan0.findBestTask(prop1, &PlanCounterDisabled)
c.Assert(err, IsNil)
@ -240,8 +240,8 @@ func (s *testFindBestTaskSuite) TestHintCannotFitProperty(c *C) {
// case 3, The hint cannot work even if the property is empty, should return a warning
// and generate physicalPlan1.
prop2 := &property.PhysicalProperty{
SortItems: items,
Enforced: false,
SortItems: items,
CanAddEnforcer: false,
}
mockPlan1 := mockLogicalPlan4Test{
hasHintForPlan2: true,
@ -261,8 +261,8 @@ func (s *testFindBestTaskSuite) TestHintCannotFitProperty(c *C) {
// the same with case 3.
ctx.GetSessionVars().StmtCtx.SetWarnings(nil)
prop3 := &property.PhysicalProperty{
SortItems: items,
Enforced: true,
SortItems: items,
CanAddEnforcer: true,
}
task, _, err = mockPlan1.findBestTask(prop3, &PlanCounterDisabled)
c.Assert(err, IsNil)

42
planner/core/physical_plan_test.go
View File

@ -559,6 +559,48 @@ func (s *testPlanSuite) TestIndexJoinUnionScan(c *C) {
}
}
func (s *testPlanSuite) TestMergeJoinUnionScan(c *C) {
defer testleak.AfterTest(c)()
store, dom, err := newStoreWithBootstrap()
c.Assert(err, IsNil)
tk := testkit.NewTestKit(c, store)
defer func() {
dom.Close()
store.Close()
}()
tk.MustExec("use test")
var input [][]string
var output []struct {
SQL []string
Plan []string
}
tk.MustExec("create table t1 (c_int int, c_str varchar(40), primary key (c_int))")
tk.MustExec("create table t2 (c_int int, c_str varchar(40), primary key (c_int))")
tk.MustExec("insert into t1 (`c_int`, `c_str`) values (11, 'keen williamson'), (10, 'gracious hermann')")
tk.MustExec("insert into t2 (`c_int`, `c_str`) values (10, 'gracious hermann')")
s.testData.GetTestCases(c, &input, &output)
for i, ts := range input {
tk.MustExec("begin")
for j, tt := range ts {
if j != len(ts)-1 {
tk.MustExec(tt)
}
s.testData.OnRecord(func() {
output[i].SQL = ts
if j == len(ts)-1 {
output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Rows())
}
})
if j == len(ts)-1 {
tk.MustQuery(tt).Check(testkit.Rows(output[i].Plan...))
}
}
tk.MustExec("rollback")
}
}
func (s *testPlanSuite) TestDoSubquery(c *C) {
defer testleak.AfterTest(c)()
store, dom, err := newStoreWithBootstrap()

2
planner/core/plan.go
View File

@ -388,7 +388,7 @@ func (p *basePhysicalPlan) cloneWithSelf(newSelf PhysicalPlan) (*basePhysicalPla
base.children = append(base.children, cloned)
}
for _, prop := range p.childrenReqProps {
base.childrenReqProps = append(base.childrenReqProps, prop.Clone())
base.childrenReqProps = append(base.childrenReqProps, prop.CloneEssentialFields())
}
return base, nil
}

6
planner/core/rule_inject_extra_projection.go
View File

@ -149,7 +149,7 @@ func InjectProjBelowAgg(aggPlan PhysicalPlan, aggFuncs []*aggregation.AggFuncDes
}
child := aggPlan.Children()[0]
prop := aggPlan.GetChildReqProps(0).Clone()
prop := aggPlan.GetChildReqProps(0).CloneEssentialFields()
proj := PhysicalProjection{
Exprs: projExprs,
AvoidColumnEvaluator: false,
@ -216,7 +216,7 @@ func InjectProjBelowSort(p PhysicalPlan, orderByItems []*util.ByItems) PhysicalP
item.Expr = newArg
}
childProp := p.GetChildReqProps(0).Clone()
childProp := p.GetChildReqProps(0).CloneEssentialFields()
bottomProj := PhysicalProjection{
Exprs: bottomProjExprs,
AvoidColumnEvaluator: false,
@ -265,7 +265,7 @@ func TurnNominalSortIntoProj(p PhysicalPlan, onlyColumn bool, orderByItems []*ut
bottomProjSchemaCols = append(bottomProjSchemaCols, newArg)
}
childProp := p.GetChildReqProps(0).Clone()
childProp := p.GetChildReqProps(0).CloneEssentialFields()
bottomProj := PhysicalProjection{
Exprs: bottomProjExprs,
AvoidColumnEvaluator: false,

11
planner/core/testdata/plan_suite_in.json vendored
View File

@ -513,6 +513,17 @@
]
]
},
{
"name": "TestMergeJoinUnionScan",
"cases": [
[
"insert into t2 values (11, 'amazing merkle')",
"insert into t2 values (12, 'amazing merkle')",
// Test Merge Join + UnionScan + TableScan.
"explain select /*+ MERGE_JOIN(t1,t2) */ * from t1, t2 where t1.c_int = t2.c_int and t1.c_int = t2.c_int order by t1.c_int, t2.c_str;"
]
]
},
{
"name": "TestSemiJoinToInner",
"cases": [

23
planner/core/testdata/plan_suite_out.json vendored
View File

@ -1393,6 +1393,29 @@
}
]
},
{
"Name": "TestMergeJoinUnionScan",
"Cases": [
{
"SQL": [
"insert into t2 values (11, 'amazing merkle')",
"insert into t2 values (12, 'amazing merkle')",
"explain select /*+ MERGE_JOIN(t1,t2) */ * from t1, t2 where t1.c_int = t2.c_int and t1.c_int = t2.c_int order by t1.c_int, t2.c_str;"
],
"Plan": [
"Sort_8 12500.00 root test.t1.c_int, test.t2.c_str",
"└─MergeJoin_11 12500.00 root inner join, left key:test.t1.c_int, test.t1.c_int, right key:test.t2.c_int, test.t2.c_int",
" ├─Sort_19(Build) 10000.00 root test.t2.c_int, test.t2.c_int",
" │ └─UnionScan_16 10000.00 root ",
" │ └─TableReader_18 10000.00 root data:TableFullScan_17",
" │ └─TableFullScan_17 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo",
" └─Sort_14(Probe) 10000.00 root test.t1.c_int, test.t1.c_int",
" └─TableReader_13 10000.00 root data:TableFullScan_12",
" └─TableFullScan_12 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
]
}
]
},
{
"Name": "TestSemiJoinToInner",
"Cases": [

23
planner/property/physical_property.go
View File

@ -66,8 +66,8 @@ type PhysicalProperty struct {
// calculated when function "HashCode()" being called.
hashcode []byte
// whether need to enforce property.
Enforced bool
// indicates that whether we are allowed to add an enforcer.
CanAddEnforcer bool
// If the partition type is hash, the data should be reshuffled by partition cols.
PartitionCols []*expression.Column
@ -79,10 +79,10 @@ type PhysicalProperty struct {
// NewPhysicalProperty builds property from columns.
func NewPhysicalProperty(taskTp TaskType, cols []*expression.Column, desc bool, expectCnt float64, enforced bool) *PhysicalProperty {
return &PhysicalProperty{
SortItems: SortItemsFromCols(cols, desc),
TaskTp: taskTp,
ExpectedCnt: expectCnt,
Enforced: enforced,
SortItems: SortItemsFromCols(cols, desc),
TaskTp: taskTp,
ExpectedCnt: expectCnt,
CanAddEnforcer: enforced,
}
}
@ -167,7 +167,7 @@ func (p *PhysicalProperty) HashCode() []byte {
}
hashcodeSize := 8 + 8 + 8 + (16+8)*len(p.SortItems) + 8
p.hashcode = make([]byte, 0, hashcodeSize)
if p.Enforced {
if p.CanAddEnforcer {
p.hashcode = codec.EncodeInt(p.hashcode, 1)
} else {
p.hashcode = codec.EncodeInt(p.hashcode, 0)
@ -196,16 +196,13 @@ func (p *PhysicalProperty) String() string {
return fmt.Sprintf("Prop{cols: %v, TaskTp: %s, expectedCount: %v}", p.SortItems, p.TaskTp, p.ExpectedCnt)
}
// Clone returns a copy of PhysicalProperty. Currently, this function is only used to build new
// required property for children plan in `exhaustPhysicalPlans`, so we don't copy `Enforced` field
// because if `Enforced` is true, the `SortItems` must be empty now, this makes `Enforced` meaningless
// for children nodes.
func (p *PhysicalProperty) Clone() *PhysicalProperty {
// CloneEssentialFields returns a copy of PhysicalProperty. We only copy the essential fields that really indicate the
// property, specifically, `CanAddEnforcer` should not be included.
func (p *PhysicalProperty) CloneEssentialFields() *PhysicalProperty {
prop := &PhysicalProperty{
SortItems: p.SortItems,
TaskTp: p.TaskTp,
ExpectedCnt: p.ExpectedCnt,
Enforced: p.Enforced,
PartitionTp: p.PartitionTp,
PartitionCols: p.PartitionCols,
}