database/tidb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

planner: change more conditions that are always false to dual (#59199)

Browse Source 
close pingcap/tidb#51446
This commit is contained in:
Weizhen Wang
2025-02-07 19:21:34 +08:00
committed by GitHub
parent c60c8415a3
commit fa0bc8e5bc
16 changed files with 143 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/executor/prepared_test.go
View File

@ -55,7 +55,7 @@ func TestPreparedNullParam(t *testing.T) {
ps := []*util.ProcessInfo{tkProcess}
tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps})
tk.MustQuery(fmt.Sprintf("explain for connection %d", tkProcess.ID)).Check(testkit.Rows(
"TableDual_5 0.00 root rows:0"))
"TableDual_6 0.00 root rows:0"))
}
}

16
pkg/expression/util.go
View File

@ -2162,3 +2162,19 @@ func binaryDurationWithMS(pos int, paramValues []byte,
pos += 4
return pos, fmt.Sprintf("%s.%06d", dur, microSecond)
}
// IsConstNull is used to check whether the expression is a constant null expression.
// For example, `1 > NULL` is a constant null expression.
// Now we just assume that the first argrument is a column,
// the second argument is a constant null.
func IsConstNull(expr Expression) bool {
if e, ok := expr.(*ScalarFunction); ok {
switch e.FuncName.L {
case ast.LT, ast.LE, ast.GT, ast.GE, ast.EQ, ast.NE:
if constExpr, ok := e.GetArgs()[1].(*Constant); ok && constExpr.Value.IsNull() && constExpr.DeferredExpr == nil {
return true
}
}
}
return false
}

3
pkg/planner/core/casetest/rule/BUILD.bazel
View File

@ -4,6 +4,7 @@ go_test(
name = "rule_test",
timeout = "short",
srcs = [
"dual_test.go",
"main_test.go",
"rule_derive_topn_from_window_test.go",
"rule_inject_extra_projection_test.go",
@ -13,7 +14,7 @@ go_test(
],
data = glob(["testdata/**"]),
flaky = True,
shard_count = 8,
shard_count = 9,
deps = [
"//pkg/domain",
"//pkg/expression",

47
pkg/planner/core/casetest/rule/dual_test.go Normal file
View File

@ -0,0 +1,47 @@
// Copyright 2023 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rule
import (
"testing"
"github.com/pingcap/tidb/pkg/testkit"
)
func TestDual(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("CREATE TABLE t (id INT PRIMARY KEY AUTO_INCREMENT,d INT);")
tk.MustQuery("explain select a from (select d as a from t where d = 0) k where k.a = 5").Check(testkit.Rows(
"TableDual_8 0.00 root rows:0"))
tk.MustQuery("select a from (select d as a from t where d = 0) k where k.a = 5").Check(testkit.Rows())
tk.MustQuery("explain select a from (select 1+2 as a from t where d = 0) k where k.a = 5").Check(testkit.Rows(
"Projection_8 0.00 root 3->Column#3",
"└─TableDual_9 0.00 root rows:0"))
tk.MustQuery("select a from (select 1+2 as a from t where d = 0) k where k.a = 5").Check(testkit.Rows())
tk.MustQuery("explain select * from t where d != null;").Check(testkit.Rows(
"TableDual_6 0.00 root rows:0"))
tk.MustQuery("explain select * from t where d > null;").Check(testkit.Rows(
"TableDual_6 0.00 root rows:0"))
tk.MustQuery("explain select * from t where d >= null;").Check(testkit.Rows(
"TableDual_6 0.00 root rows:0"))
tk.MustQuery("explain select * from t where d < null;").Check(testkit.Rows(
"TableDual_6 0.00 root rows:0"))
tk.MustQuery("explain select * from t where d <= null;").Check(testkit.Rows(
"TableDual_6 0.00 root rows:0"))
tk.MustQuery("explain select * from t where d = null;").Check(testkit.Rows(
"TableDual_6 0.00 root rows:0"))
}

17
pkg/planner/core/casetest/rule/testdata/outer2inner_out.json vendored
View File

@ -161,14 +161,7 @@
{
"SQL": "select * from t3 as t1 left join t3 as t2 on t1.c3 = t2.c3 where t2.b3 != NULL; -- self join",
"Plan": [
"Projection 0.00 root test.t3.a3, test.t3.b3, test.t3.c3, test.t3.a3, test.t3.b3, test.t3.c3",
"└─HashJoin 0.00 root inner join, equal:[eq(test.t3.c3, test.t3.c3)]",
" ├─TableReader(Build) 0.00 root data:Selection",
" │ └─Selection 0.00 cop[tikv] ne(test.t3.b3, NULL), not(isnull(test.t3.c3))",
" │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo",
" └─TableReader(Probe) 9990.00 root data:Selection",
" └─Selection 9990.00 cop[tikv] not(isnull(test.t3.c3))",
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
"TableDual 0.00 root rows:0"
]
},
{
@ -434,13 +427,7 @@
{
"SQL": "select * from t3 as t1 left join t3 as t2 on t1.c3 = t2.c3 where t1.b3 != NULL -- negative case with self join",
"Plan": [
"HashJoin 0.00 root left outer join, left side:TableReader, equal:[eq(test.t3.c3, test.t3.c3)]",
"├─TableReader(Build) 0.00 root data:Selection",
"│ └─Selection 0.00 cop[tikv] ne(test.t3.b3, NULL)",
"│ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
"└─TableReader(Probe) 9990.00 root data:Selection",
" └─Selection 9990.00 cop[tikv] not(isnull(test.t3.c3))",
" └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo"
"TableDual 0.00 root rows:0"
]
},
{

1
pkg/planner/core/issuetest/main_test.go
View File

@ -30,6 +30,7 @@ func TestMain(m *testing.M) {
testsetup.SetupForCommonTest()
flag.Parse()
opts := []goleak.Option{
goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"),
goleak.IgnoreTopFunction("github.com/golang/glog.(*fileSink).flushDaemon"),
goleak.IgnoreTopFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"),
goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"),

2
pkg/planner/core/logical_plans_test.go
View File

@ -330,7 +330,7 @@ func TestAntiSemiJoinConstFalse(t *testing.T) {
}{
{
sql: "select a from t t1 where not exists (select a from t t2 where t1.a = t2.a and t2.b = 1 and t2.b = 2)",
best: "Join{DataScan(t1)->DataScan(t2)}(test.t.a,test.t.a)->Projection",
best: "Join{DataScan(t1)->Dual}(test.t.a,test.t.a)->Projection",
joinType: "anti semi join",
},
}

1
pkg/planner/core/operator/logicalop/BUILD.bazel
View File

@ -4,6 +4,7 @@ go_library(
name = "logicalop",
srcs = [
"base_logical_plan.go",
"expression_util.go",
"hash64_equals_generated.go",
"logical_aggregation.go",
"logical_apply.go",

52
pkg/planner/core/operator/logicalop/expression_util.go Normal file
View File

@ -0,0 +1,52 @@
// Copyright 2025 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package logicalop
import (
"github.com/pingcap/tidb/pkg/expression"
"github.com/pingcap/tidb/pkg/planner/core/base"
)
// Conds2TableDual builds a LogicalTableDual if cond is constant false or null.
func Conds2TableDual(p base.LogicalPlan, conds []expression.Expression) base.LogicalPlan {
for _, cond := range conds {
if expression.IsConstNull(cond) {
if expression.MaybeOverOptimized4PlanCache(p.SCtx().GetExprCtx(), conds) {
return nil
}
dual := LogicalTableDual{}.Init(p.SCtx(), p.QueryBlockOffset())
dual.SetSchema(p.Schema())
return dual
}
}
if len(conds) != 1 {
return nil
}
con, ok := conds[0].(*expression.Constant)
if !ok {
return nil
}
sc := p.SCtx().GetSessionVars().StmtCtx
if expression.MaybeOverOptimized4PlanCache(p.SCtx().GetExprCtx(), []expression.Expression{con}) {
return nil
}
if isTrue, err := con.Value.ToBool(sc.TypeCtxOrDefault()); (err == nil && isTrue == 0) || con.Value.IsNull() {
dual := LogicalTableDual{}.Init(p.SCtx(), p.QueryBlockOffset())
dual.SetSchema(p.Schema())
return dual
}
return nil
}

5
pkg/planner/core/operator/logicalop/logical_datasource.go
View File

@ -155,6 +155,11 @@ func (ds *DataSource) PredicatePushDown(predicates []expression.Expression, opt
// TODO: remove it to the place building logical plan
predicates = utilfuncp.AddPrefix4ShardIndexes(ds, ds.SCtx(), predicates)
ds.AllConds = predicates
dual := Conds2TableDual(ds, ds.AllConds)
if dual != nil {
AppendTableDualTraceStep(ds, dual, predicates, opt)
return nil, dual
}
ds.PushedDownConds, predicates = expression.PushDownExprs(util.GetPushDownCtx(ds.SCtx()), predicates, kv.UnSpecified)
appendDataSourcePredicatePushDownTraceStep(ds, opt)
return predicates, ds

21
pkg/planner/core/operator/logicalop/logical_join.go
View File

@ -1871,27 +1871,6 @@ func deriveNotNullExpr(ctx base.PlanContext, expr expression.Expression, schema
return nil
}
// Conds2TableDual builds a LogicalTableDual if cond is constant false or null.
func Conds2TableDual(p base.LogicalPlan, conds []expression.Expression) base.LogicalPlan {
if len(conds) != 1 {
return nil
}
con, ok := conds[0].(*expression.Constant)
if !ok {
return nil
}
sc := p.SCtx().GetSessionVars().StmtCtx
if expression.MaybeOverOptimized4PlanCache(p.SCtx().GetExprCtx(), []expression.Expression{con}) {
return nil
}
if isTrue, err := con.Value.ToBool(sc.TypeCtxOrDefault()); (err == nil && isTrue == 0) || con.Value.IsNull() {
dual := LogicalTableDual{}.Init(p.SCtx(), p.QueryBlockOffset())
dual.SetSchema(p.Schema())
return dual
}
return nil
}
// BuildLogicalJoinSchema builds the schema for join operator.
func BuildLogicalJoinSchema(joinType JoinType, join base.LogicalPlan) *expression.Schema {
leftSchema := join.Children()[0].Schema()

4
pkg/planner/core/testdata/plan_suite_unexported_out.json vendored
View File

@ -67,8 +67,8 @@
"Join{DataScan(a)->DataScan(b)}(test.t.a,test.t.a)->Aggr(count(1))->Projection",
"DataScan(t)->Projection->Projection",
"DataScan(t)->Projection->Projection",
"DataScan(t)->Projection->Projection",
"DataScan(t)->Projection->Projection",
"Dual->Projection->Projection",
"Dual->Projection->Projection",
"Join{DataScan(ta)->DataScan(tb)}(test.t.d,test.t.b)(test.t.a,test.t.c)->Projection",
"Join{DataScan(t1)->DataScan(t2)}(test.t.a,test.t.b)(test.t.d,test.t.d)->Projection",
"Join{DataScan(ta)->DataScan(tb)}(test.t.d,test.t.d)->Projection",

24
tests/integrationtest/r/executor/executor.result
View File

@ -2765,47 +2765,35 @@ create table t (c1 year(4), c2 int, key(c1));
insert into t values(2001, 1);
explain format = 'brief' select t1.c1, t2.c1 from t as t1 inner join t as t2 on t1.c1 = t2.c1 where t1.c1 != NULL;
id estRows task access object operator info
MergeJoin 0.00 root inner join, left key:executor__executor.t.c1, right key:executor__executor.t.c1
├─TableDual(Build) 0.00 root rows:0
└─TableDual(Probe) 0.00 root rows:0
TableDual 0.00 root rows:0
select t1.c1, t2.c1 from t as t1 inner join t as t2 on t1.c1 = t2.c1 where t1.c1 != NULL;
c1 c1
explain format = 'brief' select * from t as t1 inner join t as t2 on t1.c1 = t2.c1 where t1.c1 != NULL;
id estRows task access object operator info
MergeJoin 0.00 root inner join, left key:executor__executor.t.c1, right key:executor__executor.t.c1
├─TableDual(Build) 0.00 root rows:0
└─TableDual(Probe) 0.00 root rows:0
TableDual 0.00 root rows:0
select * from t as t1 inner join t as t2 on t1.c1 = t2.c1 where t1.c1 != NULL;
c1 c2 c1 c2
explain format = 'brief' select count(*) from t as t1 inner join t as t2 on t1.c1 = t2.c1 where t1.c1 != NULL;
id estRows task access object operator info
StreamAgg 1.00 root funcs:count(1)->Column#7
└─MergeJoin 0.00 root inner join, left key:executor__executor.t.c1, right key:executor__executor.t.c1
├─TableDual(Build) 0.00 root rows:0
└─TableDual(Probe) 0.00 root rows:0
└─TableDual 0.00 root rows:0
select count(*) from t as t1 inner join t as t2 on t1.c1 = t2.c1 where t1.c1 != NULL;
count(*)
0
explain format = 'brief' select t1.c1, t2.c1 from t as t1 left join t as t2 on t1.c1 = t2.c1 where t1.c1 != NULL;
id estRows task access object operator info
MergeJoin 0.00 root left outer join, left side:TableDual, left key:executor__executor.t.c1, right key:executor__executor.t.c1
├─TableDual(Build) 0.00 root rows:0
└─TableDual(Probe) 0.00 root rows:0
TableDual 0.00 root rows:0
select t1.c1, t2.c1 from t as t1 left join t as t2 on t1.c1 = t2.c1 where t1.c1 != NULL;
c1 c1
explain format = 'brief' select * from t as t1 left join t as t2 on t1.c1 = t2.c1 where t1.c1 != NULL;
id estRows task access object operator info
MergeJoin 0.00 root left outer join, left side:TableDual, left key:executor__executor.t.c1, right key:executor__executor.t.c1
├─TableDual(Build) 0.00 root rows:0
└─TableDual(Probe) 0.00 root rows:0
TableDual 0.00 root rows:0
select * from t as t1 left join t as t2 on t1.c1 = t2.c1 where t1.c1 != NULL;
c1 c2 c1 c2
explain format = 'brief' select count(*) from t as t1 left join t as t2 on t1.c1 = t2.c1 where t1.c1 != NULL;
id estRows task access object operator info
StreamAgg 1.00 root funcs:count(1)->Column#7
└─MergeJoin 0.00 root left outer join, left side:TableDual, left key:executor__executor.t.c1, right key:executor__executor.t.c1
├─TableDual(Build) 0.00 root rows:0
└─TableDual(Probe) 0.00 root rows:0
└─TableDual 0.00 root rows:0
select count(*) from t as t1 left join t as t2 on t1.c1 = t2.c1 where t1.c1 != NULL;
count(*)
0

2
tests/integrationtest/r/planner/core/casetest/index/index.result
View File

@ -134,7 +134,7 @@ Selection_5 0.00 root json_overlaps(planner__core__casetest__index__index.t2.a,
└─TableRowIDScan_8(Probe) 0.00 cop[tikv] table:t2 keep order:false, stats:pseudo
explain select /*+ use_index_merge(t2, idx2, idx) */ * from t2 where 1 member of (a) and c=1 and c=2; -- 6: AND index merge from multi complicated mv indexes (empty range);
id estRows task access object operator info
TableDual_5 0.00 root rows:0
TableDual_6 0.00 root rows:0
drop table if exists t;
create table t(a int, b int, c int, unique index(a), unique index(b), primary key(c));
explain format = 'brief' select /*+ USE_INDEX_MERGE(t, a, b) */ * from t where a = 1 or b = 2;

8
tests/integrationtest/r/planner/core/casetest/predicate_simplification.result
View File

@ -122,14 +122,10 @@ Selection 12.80 root gt(Column#4, 100)
└─TableFullScan 10000.00 cop[tikv] table:ts keep order:false, stats:pseudo
explain format = 'brief' select f from t where f <> NULL and f in (1,2,3) -- Special case of NULL with no simplification.;
id estRows task access object operator info
TableReader 0.00 root data:Selection
└─Selection 0.00 cop[tikv] in(planner__core__casetest__predicate_simplification.t.f, 1, 2, 3), ne(planner__core__casetest__predicate_simplification.t.f, NULL)
└─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo
TableDual 0.00 root rows:0
explain format = 'brief' select f from t where f != NULL and f in (NULL,2,3) -- Special case of NULL with no simplification.;
id estRows task access object operator info
TableReader 0.00 root data:Selection
└─Selection 0.00 cop[tikv] in(planner__core__casetest__predicate_simplification.t.f, NULL, 2, 3), ne(planner__core__casetest__predicate_simplification.t.f, NULL)
└─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo
TableDual 0.00 root rows:0
drop table if exists dt;
drop table if exists it;
CREATE TABLE `dt` (

25
tests/integrationtest/r/tpch.result
View File

@ -710,20 +710,8 @@ and n_name = 'MOZAMBIQUE'
order by
value desc;
id estRows task access object operator info
Projection 1283496.34 root tpch50.partsupp.ps_partkey, Column#35->Column#58
└─Sort 1283496.34 root Column#35:desc
└─Selection 1283496.34 root gt(Column#35, NULL)
└─HashAgg 1604370.43 root group by:Column#60, funcs:sum(Column#59)->Column#35, funcs:firstrow(Column#60)->tpch50.partsupp.ps_partkey
└─Projection 1604370.43 root mul(tpch50.partsupp.ps_supplycost, cast(tpch50.partsupp.ps_availqty, decimal(10,0) BINARY))->Column#59, tpch50.partsupp.ps_partkey->Column#60
└─HashJoin 1604370.43 root inner join, equal:[eq(tpch50.supplier.s_suppkey, tpch50.partsupp.ps_suppkey)]
├─HashJoin(Build) 19999.44 root inner join, equal:[eq(tpch50.nation.n_nationkey, tpch50.supplier.s_nationkey)]
│ ├─TableReader(Build) 1.00 root data:Selection
│ │ └─Selection 1.00 cop[tikv] eq(tpch50.nation.n_name, "MOZAMBIQUE")
│ │ └─TableFullScan 25.00 cop[tikv] table:nation keep order:false
│ └─TableReader(Probe) 500000.00 root data:TableFullScan
│ └─TableFullScan 500000.00 cop[tikv] table:supplier keep order:false
└─TableReader(Probe) 40000000.00 root data:TableFullScan
└─TableFullScan 40000000.00 cop[tikv] table:partsupp keep order:false
Projection 0.00 root tpch50.partsupp.ps_partkey, Column#35->Column#58
└─TableDual 0.00 root rows:0
/*
Q12 Shipping Modes and Order Priority Query
This query determines whether selecting less expensive modes of shipping is negatively affecting the critical-priority
@ -1292,10 +1280,5 @@ id estRows task access object operator info
Sort 1.00 root Column#31
└─Projection 1.00 root Column#31, Column#32, Column#33
└─HashAgg 1.00 root group by:Column#36, funcs:count(1)->Column#32, funcs:sum(Column#35)->Column#33, funcs:firstrow(Column#36)->Column#31
└─Projection 0.64 root tpch50.customer.c_acctbal->Column#35, substring(tpch50.customer.c_phone, 1, 2)->Column#36
└─HashJoin 0.64 root anti semi join, left side:TableReader, equal:[eq(tpch50.customer.c_custkey, tpch50.orders.o_custkey)]
├─TableReader(Build) 0.80 root data:Selection
│ └─Selection 0.80 cop[tikv] gt(tpch50.customer.c_acctbal, NULL), in(substring(tpch50.customer.c_phone, 1, 2), "20", "40", "22", "30", "39", "42", "21")
│ └─TableFullScan 7500000.00 cop[tikv] table:customer keep order:false
└─TableReader(Probe) 75000000.00 root data:TableFullScan
└─TableFullScan 75000000.00 cop[tikv] table:orders keep order:false
└─Projection 0.00 root tpch50.customer.c_acctbal->Column#35, substring(tpch50.customer.c_phone, 1, 2)->Column#36
└─TableDual 0.00 root rows:0