database/tidb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #856 from pingcap/coocood/fiilter-rate

Browse Source 
optimizer: refactor optimization flow, push filter condition to table.
This commit is contained in:
Ewan Chou
2016-01-19 20:29:53 +08:00
parent 150acb9cc2 17d3fdbca2
commit 9570b487bb
12 changed files with 389 additions and 469 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
executor/builder.go
View File

@ -53,8 +53,6 @@ func (b *executorBuilder) build(p plan.Plan) Executor {
return b.buildDeallocate(v)
case *plan.Execute:
return b.buildExecute(v)
case *plan.Filter:
return b.buildFilter(v)
case *plan.IndexScan:
return b.buildIndexScan(v)
case *plan.Limit:
@ -77,15 +75,27 @@ func (b *executorBuilder) build(p plan.Plan) Executor {
}
}
func (b *executorBuilder) buildFilter(src Executor, conditions []ast.ExprNode) Executor {
if len(conditions) == 0 {
return src
}
return &FilterExec{
Src: src,
Condition: b.joinConditions(conditions),
ctx: b.ctx,
}
}
func (b *executorBuilder) buildTableScan(v *plan.TableScan) Executor {
table, _ := b.is.TableByID(v.Table.ID)
return &TableScanExec{
e := &TableScanExec{
t: table,
fields: v.Fields(),
ctx: b.ctx,
ranges: v.Ranges,
seekHandle: math.MinInt64,
}
return b.buildFilter(e, v.FilterConditions)
}
func (b *executorBuilder) buildShowDDL(v *plan.ShowDDL) Executor {
@ -136,7 +146,7 @@ func (b *executorBuilder) buildIndexScan(v *plan.IndexScan) Executor {
for i, val := range v.Ranges {
e.Ranges[i] = b.buildIndexRange(e, val)
}
return e
return b.buildFilter(e, v.FilterConditions)
}
func (b *executorBuilder) buildIndexRange(scan *IndexScanExec, v *plan.IndexRange) *IndexRangeExec {
@ -162,16 +172,6 @@ func (b *executorBuilder) joinConditions(conditions []ast.ExprNode) ast.ExprNode
return condition
}
func (b *executorBuilder) buildFilter(v *plan.Filter) Executor {
src := b.build(v.Src())
e := &FilterExec{
Src: src,
Condition: b.joinConditions(v.Conditions),
ctx: b.ctx,
}
return e
}
func (b *executorBuilder) buildSelectLock(v *plan.SelectLock) Executor {
src := b.build(v.Src())
if autocommit.ShouldAutocommit(b.ctx) {
@ -213,9 +213,6 @@ func (b *executorBuilder) buildAggregate(v *plan.Aggregate) Executor {
func (b *executorBuilder) buildSort(v *plan.Sort) Executor {
src := b.build(v.Src())
if v.Bypass && !v.ByItems[0].Desc {
return src
}
e := &SortExec{
Src: src,
ByItems: v.ByItems,

19
optimizer/optimizer.go
View File

@ -39,28 +39,11 @@ func Optimize(ctx context.Context, node ast.Node) (plan.Plan, error) {
if err != nil {
return nil, errors.Trace(err)
}
alts, err := plan.Alternatives(p)
if err != nil {
return nil, errors.Trace(err)
}
err = plan.Refine(p)
if err != nil {
return nil, errors.Trace(err)
}
bestCost := plan.EstimateCost(p)
bestPlan := p
for _, alt := range alts {
err = plan.Refine(alt)
if err != nil {
return nil, errors.Trace(err)
}
cost := plan.EstimateCost(alt)
if cost < bestCost {
bestCost = cost
bestPlan = alt
}
}
return bestPlan, nil
return p, nil
}
// Prepare prepares a raw statement parsed from parser.

100
optimizer/plan/alternatives.go
View File

@ -1,100 +0,0 @@
// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package plan
import "github.com/juju/errors"
// Alternatives returns multiple alternative plans that
// can be picked based on their cost.
func Alternatives(p Plan) ([]Plan, error) {
var plans []Plan
switch x := p.(type) {
case nil:
case *TableScan:
plans = tableScanAlternatives(x)
case WithSrcPlan:
var err error
plans, err = planWithSrcAlternatives(x)
if err != nil {
return nil, errors.Trace(err)
}
case *ShowDDL:
case *CheckTable:
case *Prepare:
case *Execute:
case *Deallocate:
default:
return nil, ErrUnsupportedType.Gen("Unknown plan %T", p)
}
return plans, nil
}
// tableScanAlternatives returns all index plans from the same table.
func tableScanAlternatives(p *TableScan) []Plan {
var alts []Plan
for _, v := range p.Table.Indices {
fullRange := &IndexRange{
LowVal: []interface{}{nil},
HighVal: []interface{}{MaxVal},
}
is := &IndexScan{
Index: v,
Table: p.Table,
Ranges: []*IndexRange{fullRange},
}
is.SetFields(p.Fields())
alts = append(alts, is)
}
return alts
}
// planWithSrcAlternatives shallow copies the WithSrcPlan,
// and sets its src to src alternatives.
func planWithSrcAlternatives(p WithSrcPlan) ([]Plan, error) {
srcs, err := Alternatives(p.Src())
if err != nil {
return nil, errors.Trace(err)
}
for i, val := range srcs {
alt := shallowCopy(p)
alt.SetSrc(val)
srcs[i] = alt
}
return srcs, nil
}
func shallowCopy(p WithSrcPlan) WithSrcPlan {
var copied WithSrcPlan
switch x := p.(type) {
case *Filter:
n := *x
copied = &n
case *SelectLock:
n := *x
copied = &n
case *SelectFields:
n := *x
copied = &n
case *Sort:
n := *x
copied = &n
case *Limit:
n := *x
copied = &n
case *Aggregate:
n := *x
copied = &n
}
return copied
}

76
optimizer/plan/cost.go
View File

@ -40,10 +40,6 @@ func (c *costEstimator) Enter(p Plan) (Plan, bool) {
// Leave implements Visitor Leave interface.
func (c *costEstimator) Leave(p Plan) (Plan, bool) {
switch v := p.(type) {
case *Filter:
v.startupCost = v.Src().StartupCost()
v.rowCount = v.Src().RowCount() * FilterRate
v.totalCost = v.Src().TotalCost()
case *IndexScan:
c.indexScan(v)
case *Limit:
@ -61,21 +57,14 @@ func (c *costEstimator) Leave(p Plan) (Plan, bool) {
v.rowCount = v.Src().RowCount()
v.totalCost = v.Src().TotalCost()
case *Sort:
if v.Bypass {
// Bypassed sort doesn't add extra cost.
v.startupCost = v.Src().StartupCost()
// Sort plan must retrieve all the rows before returns the first row.
v.startupCost = v.Src().TotalCost() + v.Src().RowCount()*SortCost
if v.limit == 0 {
v.rowCount = v.Src().RowCount()
v.totalCost = v.Src().TotalCost()
} else {
// Sort plan must retrieve all the rows before returns the first row.
v.startupCost = v.Src().TotalCost() + v.Src().RowCount()*SortCost
if v.limit == 0 {
v.rowCount = v.Src().RowCount()
} else {
v.rowCount = math.Min(v.Src().RowCount(), v.limit)
}
v.totalCost = v.startupCost + v.rowCount*RowCost
v.rowCount = math.Min(v.Src().RowCount(), v.limit)
}
v.totalCost = v.startupCost + v.rowCount*RowCost
case *TableScan:
c.tableScan(v)
}
@ -83,28 +72,9 @@ func (c *costEstimator) Leave(p Plan) (Plan, bool) {
}
func (c *costEstimator) tableScan(v *TableScan) {
var rowCount float64
if len(v.Ranges) == 1 && v.Ranges[0].LowVal == math.MinInt64 && v.Ranges[0].HighVal == math.MaxInt64 {
// full range use default row count.
rowCount = FullRangeCount
} else {
for _, v := range v.Ranges {
// for condition like 'a = 0'.
if v.LowVal == v.HighVal {
rowCount++
continue
}
// For condition like 'a < 0'.
if v.LowVal == math.MinInt64 {
rowCount += HalfRangeCount
}
// For condition like 'a > 0'.
if v.HighVal == math.MaxInt64 {
rowCount += HalfRangeCount
}
// For condition like 'a > 0 and a < 1'.
rowCount += MiddleRangeCount
}
var rowCount float64 = FullRangeCount
for _, con := range v.AccessConditions {
rowCount *= guesstimateFilterRate(con)
}
v.startupCost = 0
if v.limit == 0 {
@ -117,33 +87,9 @@ func (c *costEstimator) tableScan(v *TableScan) {
}
func (c *costEstimator) indexScan(v *IndexScan) {
var rowCount float64
if len(v.Ranges) == 1 && v.Ranges[0].LowVal[0] == nil && v.Ranges[0].HighVal[0] == MaxVal {
// full range use default row count.
rowCount = FullRangeCount
} else {
for _, v := range v.Ranges {
// for condition like 'a = 0'.
if v.IsPoint() {
rowCount++
continue
}
// For condition like 'a < 0'.
if v.LowVal[0] == nil || v.LowVal[0] == MinNotNullVal {
rowCount += HalfRangeCount
}
// For condition like 'a > 0'.
if v.HighVal[0] == MaxVal {
rowCount += HalfRangeCount
}
// For condition like 'a > 0 and a < 1'.
rowCount += MiddleRangeCount
}
// If the index has too many ranges, the row count may exceed the default row count.
// Make sure the cost is lower than full range.
if rowCount >= FullRangeCount {
rowCount = FullRangeCount - 1
}
var rowCount float64 = FullRangeCount
for _, con := range v.AccessConditions {
rowCount *= guesstimateFilterRate(con)
}
v.startupCost = 0
if v.limit == 0 {

5
optimizer/plan/explainer.go
View File

@ -40,8 +40,6 @@ func (e *explainer) Leave(in Plan) (Plan, bool) {
switch x := in.(type) {
case *CheckTable:
str = "CheckTable"
case *Filter:
str = "Filter"
case *IndexScan:
str = fmt.Sprintf("Index(%s.%s)", x.Table.Name.L, x.Index.Name.L)
case *Limit:
@ -53,9 +51,6 @@ func (e *explainer) Leave(in Plan) (Plan, bool) {
case *ShowDDL:
str = "ShowDDL"
case *Sort:
if x.Bypass {
return in, true
}
str = "Sort"
case *TableScan:
if len(x.Ranges) > 0 {

115
optimizer/plan/filterrate.go Normal file
View File

@ -0,0 +1,115 @@
// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package plan
import (
"github.com/pingcap/tidb/ast"
"github.com/pingcap/tidb/parser/opcode"
)
const (
rateFull float64 = 1
rateEqual float64 = 0.01
rateNotEqual float64 = 0.99
rateBetween float64 = 0.1
rateGreaterOrLess float64 = 0.33
rateIsFalse float64 = 0.1
rateIsNull float64 = 0.1
rateLike float64 = 0.1
)
// guesstimateFilterRate guesstimates the filter rate for an expression.
// For example: a table has 100 rows, after filter expression 'a between 0 and 9',
// 10 rows returned, then the filter rate is '0.1'.
// It only depends on the expression type, not the expression value.
// The expr parameter should contain only one column name.
func guesstimateFilterRate(expr ast.ExprNode) float64 {
switch x := expr.(type) {
case *ast.BetweenExpr:
return rateBetween
case *ast.BinaryOperationExpr:
return guesstimateBinop(x)
case *ast.ColumnNameExpr:
return rateFull
case *ast.IsNullExpr:
return guesstimateIsNull(x)
case *ast.IsTruthExpr:
return guesstimateIsTrue(x)
case *ast.ParenthesesExpr:
return guesstimateFilterRate(x.Expr)
case *ast.PatternInExpr:
return guesstimatePatternIn(x)
case *ast.PatternLikeExpr:
return guesstimatePatternLike(x)
}
return rateFull
}
func guesstimateBinop(expr *ast.BinaryOperationExpr) float64 {
switch expr.Op {
case opcode.AndAnd:
// P(A and B) = P(A) * P(B)
return guesstimateFilterRate(expr.L) * guesstimateFilterRate(expr.R)
case opcode.OrOr:
// P(A or B) = P(A) + P(B) – P(A and B)
rateL := guesstimateFilterRate(expr.L)
rateR := guesstimateFilterRate(expr.R)
return rateL + rateR - rateL*rateR
case opcode.EQ:
return rateEqual
case opcode.GT, opcode.GE, opcode.LT, opcode.LE:
return rateGreaterOrLess
case opcode.NE:
return rateNotEqual
}
return rateFull
}
func guesstimateIsNull(expr *ast.IsNullExpr) float64 {
if expr.Not {
return rateFull - rateIsNull
}
return rateIsNull
}
func guesstimateIsTrue(expr *ast.IsTruthExpr) float64 {
if expr.True == 0 {
if expr.Not {
return rateFull - rateIsFalse
}
return rateIsFalse
}
if expr.Not {
return rateIsFalse + rateIsNull
}
return rateFull - rateIsFalse - rateIsNull
}
func guesstimatePatternIn(expr *ast.PatternInExpr) float64 {
if len(expr.List) > 0 {
rate := rateEqual * float64(len(expr.List))
if expr.Not {
return rateFull - rate
}
return rate
}
return rateFull
}
func guesstimatePatternLike(expr *ast.PatternLikeExpr) float64 {
if expr.Not {
return rateFull - rateLike
}
return rateLike
}

132
optimizer/plan/plan_test.go
View File

@ -15,7 +15,6 @@ package plan
import (
"fmt"
"strings"
"testing"
. "github.com/pingcap/check"
@ -184,6 +183,10 @@ func (s *testPlanSuite) TestRangeBuilder(c *C) {
exprStr: `(a < 0 OR a > 3) AND (a < 1 OR a > 4)`,
resultStr: `[[-inf 0) (4 +inf]]`,
},
{
exprStr: `a > NULL`,
resultStr: `[]`,
},
}
for _, ca := range cases {
@ -198,63 +201,35 @@ func (s *testPlanSuite) TestRangeBuilder(c *C) {
}
}
func (s *testPlanSuite) TestBuilder(c *C) {
func (s *testPlanSuite) TestFilterRate(c *C) {
cases := []struct {
sqlStr string
planStr string
expr string
rate float64
}{
{
sqlStr: "select 1",
planStr: "Fields",
},
{
sqlStr: "select a from t",
planStr: "Table(t)->Fields",
},
{
sqlStr: "select a from t where a = 1",
planStr: "Table(t)->Filter->Fields",
},
{
sqlStr: "select a from t where a = 1 order by a",
planStr: "Table(t)->Filter->Fields->Sort",
},
{
sqlStr: "select a from t where a = 1 order by a limit 1",
planStr: "Table(t)->Filter->Fields->Sort->Limit",
},
{
sqlStr: "select a from t where a = 1 limit 1",
planStr: "Table(t)->Filter->Fields->Limit",
},
{
sqlStr: "select a from t where a = 1 limit 1 for update",
planStr: "Table(t)->Filter->Lock->Fields->Limit",
},
{
sqlStr: "admin show ddl",
planStr: "ShowDDL",
},
{
sqlStr: "admin check table t",
planStr: "CheckTable",
},
{expr: "a = 1", rate: rateEqual},
{expr: "a > 1", rate: rateGreaterOrLess},
{expr: "a between 1 and 100", rate: rateBetween},
{expr: "a is null", rate: rateIsNull},
{expr: "a is not null", rate: rateFull - rateIsNull},
{expr: "a is true", rate: rateFull - rateIsNull - rateIsFalse},
{expr: "a is not true", rate: rateIsNull + rateIsFalse},
{expr: "a is false", rate: rateIsFalse},
{expr: "a is not false", rate: rateFull - rateIsFalse},
{expr: "a like 'a'", rate: rateLike},
{expr: "a not like 'a'", rate: rateFull - rateLike},
{expr: "a in (1, 2, 3)", rate: rateEqual * 3},
{expr: "a not in (1, 2, 3)", rate: rateFull - rateEqual*3},
{expr: "a > 1 and a < 9", rate: float64(rateGreaterOrLess) * float64(rateGreaterOrLess)},
{expr: "a = 1 or a = 2", rate: rateEqual + rateEqual - rateEqual*rateEqual},
{expr: "a != 1", rate: rateNotEqual},
}
var stmt ast.StmtNode
for _, ca := range cases {
s, err := parser.ParseOneStmt(ca.sqlStr, "", "")
c.Assert(err, IsNil, Commentf("for expr %s", ca.sqlStr))
if strings.HasPrefix(ca.sqlStr, "select") {
stmt = s.(*ast.SelectStmt)
} else if strings.HasPrefix(ca.sqlStr, "admin") {
stmt = s.(*ast.AdminStmt)
}
mockResolve(stmt)
p, err := BuildPlan(stmt)
c.Assert(err, IsNil)
explainStr, err := Explain(p)
c.Assert(err, IsNil)
c.Assert(explainStr, Equals, ca.planStr, Commentf("for expr %s", ca.sqlStr))
sql := "select 1 from dual where " + ca.expr
s, err := parser.ParseOneStmt(sql, "", "")
c.Assert(err, IsNil, Commentf("for expr %s", ca.expr))
stmt := s.(*ast.SelectStmt)
rate := guesstimateFilterRate(stmt.Where)
c.Assert(rate, Equals, ca.rate, Commentf("for expr %s", ca.expr))
}
}
@ -273,66 +248,63 @@ func (s *testPlanSuite) TestBestPlan(c *C) {
},
{
sql: "select * from t where b = 1 order by a",
best: "Index(t.b)->Filter->Fields->Sort",
best: "Index(t.b)->Fields->Sort",
},
{
sql: "select * from t where (a between 1 and 2) and (b = 3)",
best: "Index(t.b)->Filter->Fields",
best: "Index(t.b)->Fields",
},
{
sql: "select * from t where a > 0 order by b limit 100",
best: "Index(t.b)->Filter->Fields->Limit",
best: "Index(t.b)->Fields->Limit",
},
{
sql: "select * from t where d = 0",
best: "Table(t)->Filter->Fields",
best: "Table(t)->Fields",
},
{
sql: "select * from t where c = 0 and d = 0",
best: "Index(t.c_d)->Filter->Fields",
best: "Index(t.c_d)->Fields",
},
{
sql: "select * from t where b like 'abc%'",
best: "Index(t.b)->Filter->Fields",
best: "Index(t.b)->Fields",
},
{
sql: "select * from t where d",
best: "Table(t)->Filter->Fields",
best: "Table(t)->Fields",
},
{
sql: "select * from t where a is null",
best: "Range(t)->Filter->Fields",
best: "Range(t)->Fields",
},
{
sql: "select a from t where a = 1 limit 1 for update",
best: "Range(t)->Lock->Fields->Limit",
},
{
sql: "admin show ddl",
best: "ShowDDL",
},
{
sql: "admin check table t",
best: "CheckTable",
},
}
for _, ca := range cases {
comment := Commentf("for %s", ca.sql)
s, err := parser.ParseOneStmt(ca.sql, "", "")
stmt, err := parser.ParseOneStmt(ca.sql, "", "")
c.Assert(err, IsNil, comment)
stmt := s.(*ast.SelectStmt)
ast.SetFlag(stmt)
mockResolve(stmt)
p, err := BuildPlan(stmt)
c.Assert(err, IsNil)
alts, err := Alternatives(p)
c.Assert(err, IsNil)
err = Refine(p)
explainStr, err := Explain(p)
c.Assert(err, IsNil)
bestCost := EstimateCost(p)
bestPlan := p
for _, alt := range alts {
c.Assert(Refine(alt), IsNil)
cost := EstimateCost(alt)
if cost < bestCost {
bestCost = cost
bestPlan = alt
}
}
explainStr, err := Explain(bestPlan)
c.Assert(err, IsNil)
c.Assert(explainStr, Equals, ca.best, Commentf("for %s cost %v", ca.sql, bestCost))
c.Assert(explainStr, Equals, ca.best, Commentf("for %s cost %v", ca.sql, EstimateCost(p)))
}
}

201
optimizer/plan/planbuilder.go
View File

@ -14,8 +14,6 @@
package plan
import (
"math"
"github.com/juju/errors"
"github.com/pingcap/tidb/ast"
"github.com/pingcap/tidb/infoschema"
@ -137,16 +135,10 @@ func (b *planBuilder) buildSelect(sel *ast.SelectStmt) Plan {
}
var p Plan
if sel.From != nil {
p = b.buildJoin(sel.From.TableRefs)
p = b.buildJoin(sel)
if b.err != nil {
return nil
}
if sel.Where != nil {
p = b.buildFilter(p, sel.Where)
if b.err != nil {
return nil
}
}
if sel.LockTp != ast.SelectLockNone {
p = b.buildSelectLock(p, sel.LockTp)
if b.err != nil {
@ -168,14 +160,8 @@ func (b *planBuilder) buildSelect(sel *ast.SelectStmt) Plan {
if b.err != nil {
return nil
}
if sel.Where != nil {
p = b.buildFilter(p, sel.Where)
if b.err != nil {
return nil
}
}
}
if sel.OrderBy != nil {
if sel.OrderBy != nil && !matchOrder(p, sel.OrderBy.Items) {
p = b.buildSort(p, sel.OrderBy.Items)
if b.err != nil {
return nil
@ -190,8 +176,12 @@ func (b *planBuilder) buildSelect(sel *ast.SelectStmt) Plan {
return p
}
func (b *planBuilder) buildJoin(from *ast.Join) Plan {
// Only support single table for now.
func (b *planBuilder) buildJoin(sel *ast.SelectStmt) Plan {
from := sel.From.TableRefs
if from.Right != nil {
b.err = ErrUnsupportedType.Gen("Only support single table for now.")
return nil
}
ts, ok := from.Left.(*ast.TableSource)
if !ok {
b.err = ErrUnsupportedType.Gen("Unsupported type %T", from.Left)
@ -202,38 +192,99 @@ func (b *planBuilder) buildJoin(from *ast.Join) Plan {
b.err = ErrUnsupportedType.Gen("Unsupported type %T", ts.Source)
return nil
}
conditions := splitWhere(sel.Where)
candidates := b.buildAllAccessMethodsPlan(tn, conditions)
var bestPlan Plan
var lowestCost float64
for _, v := range candidates {
cost := EstimateCost(b.buildPseudoSelectPlan(v, sel))
if bestPlan == nil {
bestPlan = v
lowestCost = cost
}
if cost < lowestCost {
bestPlan = v
lowestCost = cost
}
}
return bestPlan
}
func (b *planBuilder) buildAllAccessMethodsPlan(tn *ast.TableName, conditions []ast.ExprNode) []Plan {
var candidates []Plan
p := b.buildTableScanPlan(tn, conditions)
candidates = append(candidates, p)
for _, index := range tn.TableInfo.Indices {
ip := b.buildIndexScanPlan(index, tn, conditions)
candidates = append(candidates, ip)
}
return candidates
}
func (b *planBuilder) buildTableScanPlan(tn *ast.TableName, conditions []ast.ExprNode) Plan {
p := &TableScan{
Table: tn.TableInfo,
Ranges: []TableRange{{math.MinInt64, math.MaxInt64}},
Table: tn.TableInfo,
}
p.SetFields(tn.GetResultFields())
var pkName model.CIStr
if p.Table.PKIsHandle {
for _, colInfo := range p.Table.Columns {
if mysql.HasPriKeyFlag(colInfo.Flag) {
pkName = colInfo.Name
}
}
}
for _, con := range conditions {
if pkName.L != "" {
checker := conditionChecker{tableName: tn.TableInfo.Name, pkName: pkName}
if checker.check(con) {
p.AccessConditions = append(p.AccessConditions, con)
} else {
p.FilterConditions = append(p.FilterConditions, con)
}
} else {
p.FilterConditions = append(p.FilterConditions, con)
}
}
return p
}
// splitWhere split a where expression to a list of AND conditions.
func (b *planBuilder) splitWhere(where ast.ExprNode) []ast.ExprNode {
var conditions []ast.ExprNode
switch x := where.(type) {
case *ast.BinaryOperationExpr:
if x.Op == opcode.AndAnd {
conditions = append(conditions, x.L)
conditions = append(conditions, b.splitWhere(x.R)...)
func (b *planBuilder) buildIndexScanPlan(index *model.IndexInfo, tn *ast.TableName, conditions []ast.ExprNode) Plan {
ip := &IndexScan{Table: tn.TableInfo, Index: index}
ip.SetFields(tn.GetResultFields())
// Only use first column as access condition for cost estimation,
// In executor, we can try to use second index column to build index range.
checker := conditionChecker{tableName: tn.TableInfo.Name, idx: index, columnOffset: 0}
for _, con := range conditions {
if checker.check(con) {
ip.AccessConditions = append(ip.AccessConditions, con)
} else {
conditions = append(conditions, x)
ip.FilterConditions = append(ip.FilterConditions, con)
}
default:
conditions = append(conditions, where)
}
return conditions
return ip
}
func (b *planBuilder) buildFilter(src Plan, where ast.ExprNode) *Filter {
filter := &Filter{
Conditions: b.splitWhere(where),
// buildPseudoSelectPlan pre-builds more complete plans that may affect total cost.
func (b *planBuilder) buildPseudoSelectPlan(p Plan, sel *ast.SelectStmt) Plan {
if sel.OrderBy == nil {
return p
}
filter.SetSrc(src)
filter.SetFields(src.Fields())
return filter
if sel.GroupBy != nil {
return p
}
if !matchOrder(p, sel.OrderBy.Items) {
np := &Sort{ByItems: sel.OrderBy.Items}
np.SetSrc(p)
p = np
}
if sel.Limit != nil {
np := &Limit{Offset: sel.Limit.Offset, Count: sel.Limit.Count}
np.SetSrc(p)
np.SetLimit(0)
p = np
}
return p
}
func (b *planBuilder) buildSelectLock(src Plan, lock ast.SelectLockType) *SelectLock {
@ -355,3 +406,77 @@ func buildResultField(tableName, name string, tp byte, size int) *ast.ResultFiel
Expr: expr,
}
}
// matchOrder checks if the plan has the same ordering as items.
func matchOrder(p Plan, items []*ast.ByItem) bool {
switch x := p.(type) {
case *Aggregate:
return false
case *IndexScan:
if len(items) > len(x.Index.Columns) {
return false
}
for i, item := range items {
if item.Desc {
return false
}
var rf *ast.ResultField
switch y := item.Expr.(type) {
case *ast.ColumnNameExpr:
rf = y.Refer
case *ast.PositionExpr:
rf = y.Refer
default:
return false
}
if rf.Table.Name.L != x.Table.Name.L || rf.Column.Name.L != x.Index.Columns[i].Name.L {
return false
}
}
return true
case *TableScan:
if len(items) != 1 || !x.Table.PKIsHandle {
return false
}
if items[0].Desc {
return false
}
var refer *ast.ResultField
switch x := items[0].Expr.(type) {
case *ast.ColumnNameExpr:
refer = x.Refer
case *ast.PositionExpr:
refer = x.Refer
default:
return false
}
if mysql.HasPriKeyFlag(refer.Column.Flag) {
return true
}
return false
case *Sort:
// Sort plan should not be checked here as there should only be one sort plan in a plan tree.
return false
case WithSrcPlan:
return matchOrder(x.Src(), items)
}
return true
}
// splitWhere split a where expression to a list of AND conditions.
func splitWhere(where ast.ExprNode) []ast.ExprNode {
var conditions []ast.ExprNode
switch x := where.(type) {
case nil:
case *ast.BinaryOperationExpr:
if x.Op == opcode.AndAnd {
conditions = append(conditions, x.L)
conditions = append(conditions, splitWhere(x.R)...)
} else {
conditions = append(conditions, x)
}
default:
conditions = append(conditions, where)
}
return conditions
}

51
optimizer/plan/plans.go
View File

@ -31,6 +31,12 @@ type TableScan struct {
Table *model.TableInfo
Desc bool
Ranges []TableRange
// AccessConditions can be used to build index range.
AccessConditions []ast.ExprNode
// FilterConditions can be used to filter result.
FilterConditions []ast.ExprNode
}
// Accept implements Plan Accept interface.
@ -118,6 +124,12 @@ type IndexScan struct {
// Desc indicates whether the index should be scanned in descending order.
Desc bool
// AccessConditions can be used to build index range.
AccessConditions []ast.ExprNode
// FilterConditions can be used to filter result.
FilterConditions []ast.ExprNode
}
// Accept implements Plan Accept interface.
@ -126,38 +138,6 @@ func (p *IndexScan) Accept(v Visitor) (Plan, bool) {
return v.Leave(np)
}
// Filter represents a filter plan.
type Filter struct {
planWithSrc
// Originally the WHERE or ON condition is parsed into a single expression,
// but after we converted to CNF(Conjunctive normal form), it can be
// split into a list of AND conditions.
Conditions []ast.ExprNode
}
// Accept implements Plan Accept interface.
func (p *Filter) Accept(v Visitor) (Plan, bool) {
np, skip := v.Enter(p)
if skip {
v.Leave(np)
}
p = np.(*Filter)
var ok bool
p.src, ok = p.src.Accept(v)
if !ok {
return p, false
}
return v.Leave(p)
}
// SetLimit implements Plan SetLimit interface.
func (p *Filter) SetLimit(limit float64) {
p.limit = limit
// We assume 50% of the src row is filtered out.
p.src.SetLimit(limit * 2)
}
// SelectLock represents a select lock plan.
type SelectLock struct {
planWithSrc
@ -221,10 +201,6 @@ type Sort struct {
planWithSrc
ByItems []*ast.ByItem
// If the source is already in the same order, the sort process can be by passed.
// It depends on the Src plan, so if the Src plan has been modified, Bypass needs
// to be recalculated.
Bypass bool
}
// Accept implements Plan Accept interface.
@ -247,9 +223,6 @@ func (p *Sort) Accept(v Visitor) (Plan, bool) {
// Bypass has to be determined before this get called.
func (p *Sort) SetLimit(limit float64) {
p.limit = limit
if p.Bypass {
p.src.SetLimit(limit)
}
}
// Limit represents offset and limit plan.

3
optimizer/plan/range.go
View File

@ -163,6 +163,9 @@ func (r *rangeBuilder) buildFromBinop(x *ast.BinaryOperationExpr) []rangePoint {
value = x.R.GetValue()
op = x.Op
}
if value == nil {
return nil
}
switch op {
case opcode.EQ:
startPoint := rangePoint{value: value, start: true}

123
optimizer/plan/refiner.go
View File

@ -14,15 +14,15 @@
package plan
import (
"math"
"github.com/pingcap/tidb/ast"
"github.com/pingcap/tidb/model"
"github.com/pingcap/tidb/mysql"
"github.com/pingcap/tidb/parser/opcode"
"github.com/pingcap/tidb/util/types"
)
// Refine tries to build index range, bypass sort, set limit for source plan.
// It prepares the plan for cost estimation.
// Refine tries to build index or table range.
func Refine(p Plan) error {
r := refiner{}
p.Accept(&r)
@ -30,22 +30,10 @@ func Refine(p Plan) error {
}
type refiner struct {
conditions []ast.ExprNode
// store scan plan for sort to use.
indexScan *IndexScan
tableScan *TableScan
err error
err error
}
func (r *refiner) Enter(in Plan) (Plan, bool) {
switch x := in.(type) {
case *Filter:
r.conditions = x.Conditions
case *IndexScan:
r.indexScan = x
case *TableScan:
r.tableScan = x
}
return in, false
}
@ -55,66 +43,12 @@ func (r *refiner) Leave(in Plan) (Plan, bool) {
r.buildIndexRange(x)
case *Limit:
x.SetLimit(0)
case *Sort:
r.sortBypass(x)
case *TableScan:
r.buildTableRange(x)
}
return in, r.err == nil
}
func (r *refiner) sortBypass(p *Sort) {
if r.indexScan != nil {
idx := r.indexScan.Index
if len(p.ByItems) > len(idx.Columns) {
return
}
var desc bool
for i, val := range p.ByItems {
if val.Desc {
desc = true
}
cn, ok := val.Expr.(*ast.ColumnNameExpr)
if !ok {
return
}
if r.indexScan.Table.Name.L != cn.Refer.Table.Name.L {
return
}
indexColumn := idx.Columns[i]
if indexColumn.Name.L != cn.Refer.Column.Name.L {
return
}
}
if desc {
// TODO: support desc when index reverse iterator is supported.
r.indexScan.Desc = true
return
}
p.Bypass = true
} else if r.tableScan != nil {
if len(p.ByItems) != 1 {
return
}
byItem := p.ByItems[0]
if byItem.Desc {
// TODO: support desc when table reverse iterator is supported.
return
}
cn, ok := byItem.Expr.(*ast.ColumnNameExpr)
if !ok {
return
}
if !mysql.HasPriKeyFlag(cn.Refer.Column.Flag) {
return
}
if !cn.Refer.Table.PKIsHandle {
return
}
p.Bypass = true
}
}
var fullRange = []rangePoint{
{start: true},
{value: MaxVal},
@ -122,50 +56,25 @@ var fullRange = []rangePoint{
func (r *refiner) buildIndexRange(p *IndexScan) {
rb := rangeBuilder{}
for i := 0; i < len(p.Index.Columns); i++ {
checker := conditionChecker{idx: p.Index, tableName: p.Table.Name, columnOffset: i}
rangePoints := fullRange
var columnUsed bool
for _, cond := range r.conditions {
if checker.check(cond) {
rangePoints = rb.intersection(rangePoints, rb.build(cond))
columnUsed = true
}
}
if !columnUsed {
// For multi-column index, if the prefix column is not used, following columns
// can not be used.
break
}
if i == 0 {
// Build index range from the first column.
p.Ranges = rb.buildIndexRanges(rangePoints)
} else {
// range built from following columns should be appended to previous ranges.
p.Ranges = rb.appendIndexRanges(p.Ranges, rangePoints)
}
rangePoints := fullRange
for _, cond := range p.AccessConditions {
rangePoints = rb.intersection(rangePoints, rb.build(cond))
}
p.Ranges = rb.buildIndexRanges(rangePoints)
// TODO: build index range for second column.
r.err = rb.err
return
}
func (r *refiner) buildTableRange(p *TableScan) {
var pkHandleColumn *model.ColumnInfo
for _, colInfo := range p.Table.Columns {
if mysql.HasPriKeyFlag(colInfo.Flag) && p.Table.PKIsHandle {
pkHandleColumn = colInfo
}
}
if pkHandleColumn == nil {
if len(p.AccessConditions) == 0 {
p.Ranges = []TableRange{{math.MinInt64, math.MaxInt64}}
return
}
rb := rangeBuilder{}
rangePoints := fullRange
checker := conditionChecker{pkName: pkHandleColumn.Name, tableName: p.Table.Name}
for _, cond := range r.conditions {
if checker.check(cond) {
rangePoints = rb.intersection(rangePoints, rb.build(cond))
}
for _, cond := range p.AccessConditions {
rangePoints = rb.intersection(rangePoints, rb.build(cond))
}
p.Ranges = rb.buildTableRanges(rangePoints)
r.err = rb.err
@ -264,6 +173,8 @@ func (c *conditionChecker) checkColumnExpr(expr ast.ExprNode) bool {
if c.pkName.L != "" {
return c.pkName.L == cn.Refer.Column.Name.L
}
return cn.Refer.Column.Name.L == c.idx.Columns[c.columnOffset].Name.L
if c.idx != nil {
return cn.Refer.Column.Name.L == c.idx.Columns[c.columnOffset].Name.L
}
return true
}

2
session_test.go
View File

@ -1366,7 +1366,7 @@ func (s *testSessionSuite) TestMultiColumnIndex(c *C) {
mustExecSQL(c, se, "insert into t values (1, 5)")
sql := "select c1 from t where c1 in (1) and c2 < 10"
expectedExplain := "Index(t.idx_c1_c2)->Filter->Fields"
expectedExplain := "Index(t.idx_c1_c2)->Fields"
checkPlan(c, se, sql, expectedExplain)
mustExecMatch(c, se, sql, [][]interface{}{{1}})