database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](fe)fix anti join bug (#15955)

Browse Source 
* [fix](fe)fix anti join bug

* fix fe ut
This commit is contained in:
starocean999
2023-01-17 20:25:00 +08:00
committed by GitHub
parent d2a8b3fc1e
commit e2d145cf5d
7 changed files with 177 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
View File

@ -360,7 +360,9 @@ public class Analyzer {
// corresponding value could be an empty list. There is no entry for non-outer joins.
public final Map<TupleId, List<ExprId>> conjunctsByOjClause = Maps.newHashMap();
public final Map<TupleId, List<ExprId>> conjunctsByAntiJoinClause = Maps.newHashMap();
public final Map<TupleId, List<ExprId>> conjunctsByAntiJoinNullAwareClause = Maps.newHashMap();
public final Map<TupleId, List<ExprId>> conjunctsBySemiAntiJoinNoNullAwareClause = Maps.newHashMap();
// map from registered conjunct to its containing outer join On clause (represented
// by its right-hand side table ref); only conjuncts that can only be correctly
@ -1391,10 +1393,27 @@ public class Analyzer {
* Return all unassigned conjuncts of the anti join referenced by
* right-hand side table ref.
*/
public List<Expr> getUnassignedAntiJoinConjuncts(TableRef ref) {
Preconditions.checkState(ref.getJoinOp().isAntiJoin());
public List<Expr> getUnassignedAntiJoinNullAwareConjuncts(TableRef ref) {
Preconditions.checkState(ref.getJoinOp().isAntiJoinNullAware());
List<Expr> result = Lists.newArrayList();
List<ExprId> candidates = globalState.conjunctsByAntiJoinClause.get(ref.getId());
List<ExprId> candidates = globalState.conjunctsByAntiJoinNullAwareClause.get(ref.getId());
if (candidates == null) {
return result;
}
for (ExprId conjunctId : candidates) {
if (!globalState.assignedConjuncts.contains(conjunctId)) {
Expr e = globalState.conjuncts.get(conjunctId);
Preconditions.checkState(e != null);
result.add(e);
}
}
return result;
}
public List<Expr> getUnassignedSemiAntiJoinNoNullAwareConjuncts(TableRef ref) {
Preconditions.checkState(ref.getJoinOp().isSemiOrAntiJoinNoNullAware());
List<Expr> result = Lists.newArrayList();
List<ExprId> candidates = globalState.conjunctsBySemiAntiJoinNoNullAwareClause.get(ref.getId());
if (candidates == null) {
return result;
}
@ -1484,6 +1503,30 @@ public class Analyzer {
return (tblRef.getJoinOp().isAntiJoin()) ? tblRef : null;
}
public boolean isAntiJoinedNullAwareConjunct(Expr e) {
return getAntiJoinNullAwareRef(e) != null;
}
private TableRef getAntiJoinNullAwareRef(Expr e) {
TableRef tblRef = globalState.sjClauseByConjunct.get(e.getId());
if (tblRef == null) {
return null;
}
return (tblRef.getJoinOp().isAntiJoinNullAware()) ? tblRef : null;
}
public boolean isAntiJoinedNoNullAwareConjunct(Expr e) {
return getAntiJoinNoNullAwareRef(e) != null;
}
public TableRef getAntiJoinNoNullAwareRef(Expr e) {
TableRef tblRef = globalState.sjClauseByConjunct.get(e.getId());
if (tblRef == null) {
return null;
}
return (tblRef.getJoinOp().isAntiJoinNoNullAware()) ? tblRef : null;
}
public boolean isFullOuterJoined(TupleId tid) {
return globalState.fullOuterJoinedTupleIds.containsKey(tid);
}
@ -1701,11 +1744,14 @@ public class Analyzer {
globalState.ojClauseByConjunct.put(conjunct.getId(), rhsRef);
ojConjuncts.add(conjunct.getId());
}
if (rhsRef.getJoinOp().isSemiJoin()) {
if (rhsRef.getJoinOp().isSemiAntiJoin()) {
globalState.sjClauseByConjunct.put(conjunct.getId(), rhsRef);
if (rhsRef.getJoinOp().isAntiJoin()) {
globalState.conjunctsByAntiJoinClause.computeIfAbsent(rhsRef.getId(), k -> Lists.newArrayList())
.add(conjunct.getId());
if (rhsRef.getJoinOp().isAntiJoinNullAware()) {
globalState.conjunctsByAntiJoinNullAwareClause.computeIfAbsent(rhsRef.getId(),
k -> Lists.newArrayList()).add(conjunct.getId());
} else {
globalState.conjunctsBySemiAntiJoinNoNullAwareClause.computeIfAbsent(rhsRef.getId(),
k -> Lists.newArrayList()).add(conjunct.getId());
}
}
if (rhsRef.getJoinOp().isInnerJoin()) {
@ -1725,7 +1771,7 @@ public class Analyzer {
*/
private void markConstantConjunct(Expr conjunct, boolean fromHavingClause)
throws AnalysisException {
if (!conjunct.isConstant() || isOjConjunct(conjunct) || isAntiJoinedConjunct(conjunct)) {
if (!conjunct.isConstant() || isOjConjunct(conjunct)) {
return;
}
if ((!fromHavingClause && !hasEmptySpjResultSet)
@ -1742,24 +1788,25 @@ public class Analyzer {
conjunct.analyze(this);
}
final Expr newConjunct = conjunct.getResultValue();
if (newConjunct instanceof BoolLiteral) {
final BoolLiteral value = (BoolLiteral) newConjunct;
if (!value.getValue()) {
if (fromHavingClause) {
hasEmptyResultSet = true;
if (newConjunct instanceof BoolLiteral || newConjunct instanceof NullLiteral) {
boolean evalResult = true;
if (newConjunct instanceof BoolLiteral) {
evalResult = ((BoolLiteral) newConjunct).getValue();
} else {
evalResult = false;
}
if (fromHavingClause) {
hasEmptyResultSet = !evalResult;
} else {
if (isAntiJoinedNoNullAwareConjunct(conjunct)) {
hasEmptySpjResultSet = evalResult;
} else {
hasEmptySpjResultSet = true;
hasEmptySpjResultSet = !evalResult;
}
}
markConjunctAssigned(conjunct);
}
if (newConjunct instanceof NullLiteral) {
if (fromHavingClause) {
hasEmptyResultSet = true;
} else {
hasEmptySpjResultSet = true;
if (hasEmptyResultSet || hasEmptySpjResultSet) {
markConjunctAssigned(conjunct);
}
markConjunctAssigned(conjunct);
}
} catch (AnalysisException ex) {
throw new AnalysisException("Error evaluating \"" + conjunct.toSql() + "\"", ex);

13
fe/fe-core/src/main/java/org/apache/doris/analysis/JoinOperator.java
View File

@ -71,6 +71,19 @@ public enum JoinOperator {
|| this == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN;
}
public boolean isSemiOrAntiJoinNoNullAware() {
return this == JoinOperator.LEFT_SEMI_JOIN || this == JoinOperator.LEFT_ANTI_JOIN
|| this == JoinOperator.RIGHT_SEMI_JOIN || this == JoinOperator.RIGHT_ANTI_JOIN;
}
public boolean isAntiJoinNullAware() {
return this == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN;
}
public boolean isAntiJoinNoNullAware() {
return this == JoinOperator.LEFT_ANTI_JOIN || this == JoinOperator.RIGHT_ANTI_JOIN;
}
public boolean isLeftSemiJoin() {
return this.thriftJoinOp == TJoinOp.LEFT_SEMI_JOIN;
}

9
fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
View File

@ -2070,11 +2070,10 @@ public class SingleNodePlanner {
// Also assign conjuncts from On clause. All remaining unassigned conjuncts
// that can be evaluated by this join are assigned in createSelectPlan().
ojConjuncts = analyzer.getUnassignedOjConjuncts(innerRef);
} else if (innerRef.getJoinOp().isAntiJoin()) {
ojConjuncts = analyzer.getUnassignedAntiJoinConjuncts(innerRef);
} else if (innerRef.getJoinOp().isSemiJoin()) {
final List<TupleId> tupleIds = innerRef.getAllTupleIds();
ojConjuncts = analyzer.getUnassignedConjuncts(tupleIds, false);
} else if (innerRef.getJoinOp().isAntiJoinNullAware()) {
ojConjuncts = analyzer.getUnassignedAntiJoinNullAwareConjuncts(innerRef);
} else if (innerRef.getJoinOp().isSemiOrAntiJoinNoNullAware()) {
ojConjuncts = analyzer.getUnassignedSemiAntiJoinNoNullAwareConjuncts(innerRef);
}
analyzer.markConjunctsAssigned(ojConjuncts);
if (eqJoinConjuncts.isEmpty()) {

2
fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java
View File

@ -405,7 +405,7 @@ public class PlannerTest extends TestWithFeService {
compare.accept("select * from db1.tbl2 where k1 = 2.0", "select * from db1.tbl2 where k1 = 2");
compare.accept("select * from db1.tbl2 where k1 = 2.1", "select * from db1.tbl2 where 2 = 2.1");
compare.accept("select * from db1.tbl2 where k1 != 2.0", "select * from db1.tbl2 where k1 != 2");
compare.accept("select * from db1.tbl2 where k1 != 2.1", "select * from db1.tbl2");
compare.accept("select * from db1.tbl2 where k1 != 2.1", "select * from db1.tbl2 where TRUE");
compare.accept("select * from db1.tbl2 where k1 <= 2.0", "select * from db1.tbl2 where k1 <= 2");
compare.accept("select * from db1.tbl2 where k1 <= 2.1", "select * from db1.tbl2 where k1 <= 2");
compare.accept("select * from db1.tbl2 where k1 >= 2.0", "select * from db1.tbl2 where k1 >= 2");

2
fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
View File

@ -1025,7 +1025,7 @@ public class QueryPlanTest extends TestWithFeService {
String explainString = getSQLPlanOrErrorMsg("explain " + sql);
Assert.assertTrue(explainString.contains("PLAN FRAGMENT"));
Assert.assertTrue(explainString.contains("NESTED LOOP JOIN"));
Assert.assertTrue(!explainString.contains("PREDICATES"));
Assert.assertTrue(!explainString.contains("PREDICATES") || explainString.contains("PREDICATES: TRUE"));
}
@Test

80
regression-test/data/query_p0/join/test_join.out
View File

@ -2805,3 +2805,83 @@ false true true false false
5 2.200 null \N 2019-09-09T00:00 8.9 2 \N 2 \N \N 8.9
5 2.200 null \N 2019-09-09T00:00 8.9 3 \N null 2019-09-09 \N 8.9
-- !sql --
\N
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
-- !sql --
-- !sql --
-- !sql --
\N
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
-- !sql --
\N
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
-- !sql --
-- !sql --
-- !sql --
\N
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

8
regression-test/suites/query_p0/join/test_join.groovy
View File

@ -1227,7 +1227,15 @@ suite("test_join", "query,p0") {
sql"""drop table ${table_3}"""
sql"""drop table ${table_4}"""
qt_sql """select k1 from baseall left semi join test on true order by k1;"""
qt_sql """select k1 from baseall left semi join test on false order by k1;"""
qt_sql """select k1 from baseall left anti join test on true order by k1;"""
qt_sql """select k1 from baseall left anti join test on false order by k1;"""
qt_sql """select k1 from test right semi join baseall on true order by k1;"""
qt_sql """select k1 from test right semi join baseall on false order by k1;"""
qt_sql """select k1 from test right anti join baseall on true order by k1;"""
qt_sql """select k1 from test right anti join baseall on false order by k1;"""
// test bucket shuffle join, github issue #6171
sql"""create database if not exists test_issue_6171"""