database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](planner) infer predicate could generate predicates in another scope (#13691)

Browse Source 
* [fix](planner) infer predicate could generate predicates in another scope
This commit is contained in:
morrySnow
2022-11-01 09:03:41 +08:00
committed by GitHub
parent d2c5c1af3b
commit b27714542d
2 changed files with 197 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
View File

@ -239,6 +239,59 @@ public class Analyzer {
return globalState.autoBroadcastJoinThreshold;
}
private static class InferPredicateState {
// map from two table tuple ids to JoinOperator between two tables.
// NOTE: first tupleId's position in front of the second tupleId.
public final Map<Pair<TupleId, TupleId>, JoinOperator> anyTwoTalesJoinOperator = Maps.newHashMap();
// slotEqSlotExpr: Record existing and infer equivalent connections
private final List<Expr> onSlotEqSlotExpr = new ArrayList<>();
// slotEqSlotDeDuplication: De-Duplication for slotEqSlotExpr
private final Set<Pair<Expr, Expr>> onSlotEqSlotDeDuplication = Sets.newHashSet();
// slotToLiteralExpr: Record existing and infer expr which slot and literal are equal
private final List<Expr> onSlotToLiteralExpr = new ArrayList<>();
// slotToLiteralDeDuplication: De-Duplication for slotToLiteralExpr
private final Set<Pair<Expr, Expr>> onSlotToLiteralDeDuplication = Sets.newHashSet();
// inExpr: Recoud existing and infer expr which in predicate
private final List<Expr> onInExpr = new ArrayList<>();
// inExprDeDuplication: De-Duplication for inExpr
private final Set<Expr> onInDeDuplication = Sets.newHashSet();
// isNullExpr: Record existing and infer not null predicate
private final List<Expr> onIsNullExpr = new ArrayList<>();
//isNullDeDuplication: De-Duplication for isNullExpr
private final Set<Expr> onIsNullDeDuplication = Sets.newHashSet();
// slotToLiteralDeDuplication: De-Duplication for slotToLiteralExpr. Contain on and where.
private final Set<Pair<Expr, Expr>> globalSlotToLiteralDeDuplication = Sets.newHashSet();
// inExprDeDuplication: De-Duplication for inExpr. Contain on and where
private final Set<Expr> globalInDeDuplication = Sets.newHashSet();
public InferPredicateState() {
}
public InferPredicateState(InferPredicateState that) {
anyTwoTalesJoinOperator.putAll(that.anyTwoTalesJoinOperator);
onSlotEqSlotExpr.addAll(that.onSlotEqSlotExpr);
onSlotEqSlotDeDuplication.addAll(that.onSlotEqSlotDeDuplication);
onSlotToLiteralExpr.addAll(that.onSlotToLiteralExpr);
onSlotToLiteralDeDuplication.addAll(that.onSlotToLiteralDeDuplication);
onInExpr.addAll(that.onInExpr);
onInDeDuplication.addAll(that.onInDeDuplication);
onIsNullExpr.addAll(that.onIsNullExpr);
onIsNullDeDuplication.addAll(that.onIsNullDeDuplication);
globalSlotToLiteralDeDuplication.addAll(that.globalSlotToLiteralDeDuplication);
globalInDeDuplication.addAll(that.globalInDeDuplication);
}
}
// state shared between all objects of an Analyzer tree
// TODO: Many maps here contain properties about tuples, e.g., whether
// a tuple is outer/semi joined, etc. Remove the maps in favor of making
@ -322,40 +375,6 @@ public class Analyzer {
// TODO chenhao16, to save conjuncts, which children are constant
public final Map<TupleId, Set<Expr>> constantConjunct = Maps.newHashMap();
// map from two table tuple ids to JoinOperator between two tables.
// NOTE: first tupleId's position in front of the second tupleId.
public final Map<Pair<TupleId, TupleId>, JoinOperator> anyTwoTalesJoinOperator = Maps.newHashMap();
// slotEqSlotExpr: Record existing and infer equivalent connections
private final List<Expr> onSlotEqSlotExpr = new ArrayList<>();
// slotEqSlotDeDuplication: De-Duplication for slotEqSlotExpr
private final Set<Pair<Expr, Expr>> onSlotEqSlotDeDuplication = Sets.newHashSet();
// slotToLiteralExpr: Record existing and infer expr which slot and literal are equal
private final List<Expr> onSlotToLiteralExpr = new ArrayList<>();
// slotToLiteralDeDuplication: De-Duplication for slotToLiteralExpr
private final Set<Pair<Expr, Expr>> onSlotToLiteralDeDuplication = Sets.newHashSet();
// inExpr: Recoud existing and infer expr which in predicate
private final List<Expr> onInExpr = new ArrayList<>();
// inExprDeDuplication: De-Duplication for inExpr
private final Set<Expr> onInDeDuplication = Sets.newHashSet();
// isNullExpr: Record existing and infer not null predicate
private final List<Expr> onIsNullExpr = new ArrayList<>();
//isNullDeDuplication: De-Duplication for isNullExpr
private final Set<Expr> onIsNullDeDuplication = Sets.newHashSet();
// slotToLiteralDeDuplication: De-Duplication for slotToLiteralExpr. Contain on and where.
private final Set<Pair<Expr, Expr>> globalSlotToLiteralDeDuplication = Sets.newHashSet();
// inExprDeDuplication: De-Duplication for inExpr. Contain on and where
private final Set<Expr> globalInDeDuplication = Sets.newHashSet();
// map from slot id to the analyzer/block in which it was registered
private final Map<SlotId, Analyzer> blockBySlot = Maps.newHashMap();
@ -426,6 +445,8 @@ public class Analyzer {
private final GlobalState globalState;
private final InferPredicateState inferPredicateState;
// An analyzer stores analysis state for a single select block. A select block can be
// a top level select statement, or an inline view select block.
// ancestors contains the Analyzers of the enclosing select blocks of 'this'
@ -460,6 +481,7 @@ public class Analyzer {
public Analyzer(Env env, ConnectContext context) {
ancestors = Lists.newArrayList();
globalState = new GlobalState(env, context);
inferPredicateState = new InferPredicateState();
}
/**
@ -469,7 +491,7 @@ public class Analyzer {
* @param parentAnalyzer the analyzer of the enclosing select block
*/
public Analyzer(Analyzer parentAnalyzer) {
this(parentAnalyzer, parentAnalyzer.globalState);
this(parentAnalyzer, parentAnalyzer.globalState, parentAnalyzer.inferPredicateState);
if (parentAnalyzer.isSubquery) {
this.isSubquery = true;
}
@ -478,10 +500,11 @@ public class Analyzer {
/**
* Analyzer constructor for nested select block with the specified global state.
*/
private Analyzer(Analyzer parentAnalyzer, GlobalState globalState) {
private Analyzer(Analyzer parentAnalyzer, GlobalState globalState, InferPredicateState inferPredicateState) {
ancestors = Lists.newArrayList(parentAnalyzer);
ancestors.addAll(parentAnalyzer.ancestors);
this.globalState = globalState;
this.inferPredicateState = new InferPredicateState(inferPredicateState);
}
/**
@ -490,7 +513,7 @@ public class Analyzer {
*/
public static Analyzer createWithNewGlobalState(Analyzer parentAnalyzer) {
GlobalState globalState = new GlobalState(parentAnalyzer.globalState.env, parentAnalyzer.getContext());
return new Analyzer(parentAnalyzer, globalState);
return new Analyzer(parentAnalyzer, globalState, new InferPredicateState());
}
public void setIsExplain() {
@ -1038,47 +1061,47 @@ public class Analyzer {
if (joinOperator == null) {
joinOperator = JoinOperator.INNER_JOIN;
}
globalState.anyTwoTalesJoinOperator.put(tids, joinOperator);
inferPredicateState.anyTwoTalesJoinOperator.put(tids, joinOperator);
}
public void registerOnSlotEqSlotExpr(Expr expr) {
globalState.onSlotEqSlotExpr.add(expr);
inferPredicateState.onSlotEqSlotExpr.add(expr);
}
public void registerOnSlotEqSlotDeDuplication(Pair<Expr, Expr> pair) {
globalState.onSlotEqSlotDeDuplication.add(pair);
inferPredicateState.onSlotEqSlotDeDuplication.add(pair);
}
public void registerOnSlotToLiteralExpr(Expr expr) {
globalState.onSlotToLiteralExpr.add(expr);
inferPredicateState.onSlotToLiteralExpr.add(expr);
}
public void registerOnSlotToLiteralDeDuplication(Pair<Expr, Expr> pair) {
globalState.onSlotToLiteralDeDuplication.add(pair);
inferPredicateState.onSlotToLiteralDeDuplication.add(pair);
}
public void registerInExpr(Expr expr) {
globalState.onInExpr.add(expr);
inferPredicateState.onInExpr.add(expr);
}
public void registerInDeDuplication(Expr expr) {
globalState.onInDeDuplication.add(expr);
inferPredicateState.onInDeDuplication.add(expr);
}
public void registerOnIsNullExpr(Expr expr) {
globalState.onIsNullExpr.add(expr);
inferPredicateState.onIsNullExpr.add(expr);
}
public void registerOnIsNullDeDuplication(Expr expr) {
globalState.onIsNullDeDuplication.add(expr);
inferPredicateState.onIsNullDeDuplication.add(expr);
}
public void registerGlobalSlotToLiteralDeDuplication(Pair<Expr, Expr> pair) {
globalState.globalSlotToLiteralDeDuplication.add(pair);
inferPredicateState.globalSlotToLiteralDeDuplication.add(pair);
}
public void registerGlobalInDeDuplication(Expr expr) {
globalState.globalInDeDuplication.add(expr);
inferPredicateState.globalInDeDuplication.add(expr);
}
public void registerConjunct(Expr e, TupleId tupleId) throws AnalysisException {
@ -1417,11 +1440,11 @@ public class Analyzer {
* Return JoinOperator between two tables
*/
public JoinOperator getAnyTwoTablesJoinOp(Pair<TupleId, TupleId> tids) {
return globalState.anyTwoTalesJoinOperator.get(tids);
return inferPredicateState.anyTwoTalesJoinOperator.get(tids);
}
public boolean isContainTupleIds(Pair<TupleId, TupleId> tids) {
return globalState.anyTwoTalesJoinOperator.containsKey(tids);
return inferPredicateState.anyTwoTalesJoinOperator.containsKey(tids);
}
public boolean isWhereClauseConjunct(Expr e) {
@ -1518,43 +1541,43 @@ public class Analyzer {
}
public List<Expr> getOnSlotEqSlotExpr() {
return new ArrayList<>(globalState.onSlotEqSlotExpr);
return new ArrayList<>(inferPredicateState.onSlotEqSlotExpr);
}
public Set<Pair<Expr, Expr>> getOnSlotEqSlotDeDuplication() {
return Sets.newHashSet(globalState.onSlotEqSlotDeDuplication);
return Sets.newHashSet(inferPredicateState.onSlotEqSlotDeDuplication);
}
public List<Expr> getOnSlotToLiteralExpr() {
return new ArrayList<>(globalState.onSlotToLiteralExpr);
return new ArrayList<>(inferPredicateState.onSlotToLiteralExpr);
}
public Set<Pair<Expr, Expr>> getOnSlotToLiteralDeDuplication() {
return Sets.newHashSet(globalState.onSlotToLiteralDeDuplication);
return Sets.newHashSet(inferPredicateState.onSlotToLiteralDeDuplication);
}
public List<Expr> getInExpr() {
return new ArrayList<>(globalState.onInExpr);
return new ArrayList<>(inferPredicateState.onInExpr);
}
public Set<Expr> getInDeDuplication() {
return Sets.newHashSet(globalState.onInDeDuplication);
return Sets.newHashSet(inferPredicateState.onInDeDuplication);
}
public List<Expr> getOnIsNullExpr() {
return new ArrayList<>(globalState.onIsNullExpr);
return new ArrayList<>(inferPredicateState.onIsNullExpr);
}
public Set<Expr> getOnIsNullDeDuplication() {
return Sets.newHashSet(globalState.onIsNullDeDuplication);
return Sets.newHashSet(inferPredicateState.onIsNullDeDuplication);
}
public Set<Pair<Expr, Expr>> getGlobalSlotToLiteralDeDuplication() {
return Sets.newHashSet(globalState.globalSlotToLiteralDeDuplication);
return Sets.newHashSet(inferPredicateState.globalSlotToLiteralDeDuplication);
}
public Set<Expr> getGlobalInDeDuplication() {
return Sets.newHashSet(globalState.globalInDeDuplication);
return Sets.newHashSet(inferPredicateState.globalInDeDuplication);
}
/**

114
regression-test/suites/correctness/test_infer_predicate.groovy Normal file
View File

@ -0,0 +1,114 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_infer_predicate") {
sql """ DROP TABLE IF EXISTS infer_predicate_t1 """
sql """ DROP TABLE IF EXISTS infer_predicate_t2 """
sql """ DROP VIEW IF EXISTS infer_predicate_v1 """
sql """ DROP TABLE IF EXISTS infer_predicate_t4 """
sql """ DROP TABLE IF EXISTS infer_predicate_t5 """
sql"""
CREATE TABLE infer_predicate_t2 (
a varchar(1) NULL COMMENT "",
x varchar(1) NULL COMMENT "",
c varchar(1) NULL COMMENT ""
) ENGINE=OLAP
UNIQUE KEY(a)
DISTRIBUTED BY HASH(a) BUCKETS 8
PROPERTIES (
"replication_allocation" = "tag.location.default:1",
"in_memory" = "false",
"storage_format" = "V2"
);
"""
sql"""
CREATE TABLE infer_predicate_t5 (
x varchar(5) NULL COMMENT "",
d varchar(3) NULL COMMENT ""
) ENGINE=OLAP
UNIQUE KEY(x)
DISTRIBUTED BY HASH(x) BUCKETS 8
PROPERTIES (
"replication_allocation" = "tag.location.default:1",
"in_memory" = "false",
"storage_format" = "V2"
);
"""
sql"""
CREATE TABLE infer_predicate_t4 (
a varchar(1) NULL COMMENT "",
e varchar(1) NULL COMMENT ""
) ENGINE=OLAP
UNIQUE KEY(a)
DISTRIBUTED BY HASH(a) BUCKETS 8
PROPERTIES (
"replication_allocation" = "tag.location.default:1",
"in_memory" = "false",
"storage_format" = "V2"
);
"""
sql"""
CREATE TABLE infer_predicate_t1 (
k1 varchar(20) NULL COMMENT "",
dt date NULL COMMENT ""
) ENGINE=OLAP
UNIQUE KEY(k1)
DISTRIBUTED BY HASH(k1) BUCKETS 6
PROPERTIES (
"replication_allocation" = "tag.location.default:1",
"in_memory" = "false",
"storage_format" = "V2"
);
"""
sql"""
CREATE VIEW infer_predicate_v1
AS
SELECT k1
FROM infer_predicate_t1
WHERE dt = (
SELECT max(dt)
FROM infer_predicate_t1
);
"""
try {
sql """
select
f1.k1
from (
select
infer_predicate_t2.c as k1
from infer_predicate_t5
inner join infer_predicate_t2 on infer_predicate_t2.x=infer_predicate_t5.x
inner join infer_predicate_t4 on infer_predicate_t2.a=infer_predicate_t4.a and infer_predicate_t2.x in ('ZCR', 'ZDR')
) f1
left join infer_predicate_v1 on f1.k1=infer_predicate_v1.k1;
"""
} finally {
sql """ DROP TABLE IF EXISTS infer_predicate_t1 """
sql """ DROP TABLE IF EXISTS infer_predicate_t2 """
sql """ DROP VIEW IF EXISTS infer_predicate_v1 """
sql """ DROP TABLE IF EXISTS infer_predicate_t4 """
sql """ DROP TABLE IF EXISTS infer_predicate_t5 """
}
}