database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[bug](vectorized) fix bug of tuple is null null side do not set (#12012)

Browse Source
This commit is contained in:
HappenLee
2022-08-24 16:19:43 +08:00
committed by GitHub
parent 8b4f693ad5
commit d87ab69ead
9 changed files with 63 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
View File

@ -289,6 +289,11 @@ public class Analyzer {
// to the last Join clause (represented by its rhs table ref) that outer-joined it
private final Map<TupleId, TableRef> outerJoinedTupleIds = Maps.newHashMap();
// set of left side and right side of tuple id to mark null side in vec
// exec engine
private final Set<TupleId> outerLeftSideJoinTupleIds = Sets.newHashSet();
private final Set<TupleId> outerRightSideJoinTupleIds = Sets.newHashSet();
// Map of registered conjunct to the last full outer join (represented by its
// rhs table ref) that outer joined it.
public final Map<ExprId, TableRef> fullOuterJoinedConjuncts = Maps.newHashMap();
@ -1001,6 +1006,16 @@ public class Analyzer {
}
}
public void registerOuterJoinedRightSideTids(List<TupleId> tids) {
globalState.outerRightSideJoinTupleIds.addAll(tids);
}
public void registerOuterJoinedLeftSideTids(List<TupleId> tids) {
globalState.outerLeftSideJoinTupleIds.addAll(tids);
}
/**
* Register the given tuple id as being the invisible side of a semi-join.
*/
@ -2233,6 +2248,14 @@ public class Analyzer {
return globalState.outerJoinedTupleIds.containsKey(tid);
}
public boolean isOuterJoinedLeftSide(TupleId tid) {
return globalState.outerLeftSideJoinTupleIds.contains(tid);
}
public boolean isOuterJoinedRightSide(TupleId tid) {
return globalState.outerRightSideJoinTupleIds.contains(tid);
}
public boolean isInlineView(TupleId tid) {
return globalState.inlineViewTupleIds.contains(tid);
}

7
fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
View File

@ -28,6 +28,7 @@ import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.rewrite.ExprRewriter;
import org.apache.doris.thrift.TNullSide;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@ -364,7 +365,11 @@ public class InlineViewRef extends TableRef {
if (!requiresNullWrapping(analyzer, smap.getRhs().get(i), nullSMap)) {
continue;
}
params.add(new TupleIsNullPredicate(materializedTupleIds));
if (analyzer.isOuterJoinedLeftSide(materializedTupleIds.get(0))) {
params.add(new TupleIsNullPredicate(materializedTupleIds, TNullSide.LEFT));
} else {
params.add(new TupleIsNullPredicate(materializedTupleIds, TNullSide.RIGHT));
}
params.add(NullLiteral.create(smap.getRhs().get(i).getType()));
params.add(smap.getRhs().get(i));
Expr ifExpr = new FunctionCallExpr("if", params);

2
fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
View File

@ -490,10 +490,12 @@ public class TableRef implements ParseNode, Writable {
if (joinOp == JoinOperator.LEFT_OUTER_JOIN
|| joinOp == JoinOperator.FULL_OUTER_JOIN) {
analyzer.registerOuterJoinedTids(getId().asList(), this);
analyzer.registerOuterJoinedRightSideTids(getId().asList());
}
if (joinOp == JoinOperator.RIGHT_OUTER_JOIN
|| joinOp == JoinOperator.FULL_OUTER_JOIN) {
analyzer.registerOuterJoinedTids(leftTblRef.getAllTableRefIds(), this);
analyzer.registerOuterJoinedLeftSideTids(leftTblRef.getAllTableRefIds());
}
// register the tuple ids of a full outer join
if (joinOp == JoinOperator.FULL_OUTER_JOIN) {

62
fe/fe-core/src/main/java/org/apache/doris/analysis/TupleIsNullPredicate.java
View File

@ -43,14 +43,13 @@ import java.util.Objects;
*/
public class TupleIsNullPredicate extends Predicate {
private List<TupleId> tupleIds = Lists.newArrayList();
// Only effective in vectorized exec engine to mark null side,
// can set null in origin exec engine
private TNullSide nullSide = null;
public TupleIsNullPredicate(List<TupleId> tupleIds) {
Preconditions.checkState(tupleIds != null && !tupleIds.isEmpty());
public TupleIsNullPredicate(List<TupleId> tupleIds, TNullSide nullSide) {
Preconditions.checkState(tupleIds != null && (!tupleIds.isEmpty() || nullSide != null));
this.tupleIds.addAll(tupleIds);
}
public TupleIsNullPredicate(TNullSide nullSide) {
this.nullSide = nullSide;
}
@ -131,7 +130,7 @@ public class TupleIsNullPredicate extends Predicate {
* Returns a new list with the nullable exprs.
*/
public static List<Expr> wrapExprs(List<Expr> inputExprs,
List<TupleId> tids, Analyzer analyzer) throws UserException {
List<TupleId> tids, TNullSide nullSide, Analyzer analyzer) throws UserException {
// Assert that all tids are materialized.
for (TupleId tid : tids) {
TupleDescriptor tupleDesc = analyzer.getTupleDesc(tid);
@ -140,42 +139,23 @@ public class TupleIsNullPredicate extends Predicate {
// Perform the wrapping.
List<Expr> result = Lists.newArrayListWithCapacity(inputExprs.size());
for (Expr e : inputExprs) {
result.add(wrapExpr(e, tids, analyzer));
result.add(wrapExpr(e, tids, nullSide, analyzer));
}
return result;
}
/**
* Makes each input expr nullable, if necessary, by wrapping it as follows:
* IF(TupleIsNull(nullSide), NULL, expr)
* <p>
* The given inputExprs are expected to be bound
* by null side tuple id once fully substituted against base tables. However, inputExprs may not yet
* be fully substituted at this point.
* <p>
* Returns a new list with the nullable exprs. only use in vectorized exec engine
*/
public static List<Expr> wrapExprs(List<Expr> inputExprs,
TNullSide nullSide, Analyzer analyzer) throws UserException {
// Perform the wrapping.
List<Expr> result = Lists.newArrayListWithCapacity(inputExprs.size());
for (Expr e : inputExprs) {
result.add(wrapExpr(e, nullSide, analyzer));
}
return result;
}
/**
* Returns a new analyzed conditional expr 'IF(TupleIsNull(tids), NULL, expr)',
* if required to make expr nullable. Otherwise, returns expr.
*/
public static Expr wrapExpr(Expr expr, List<TupleId> tids, Analyzer analyzer)
public static Expr wrapExpr(Expr expr, List<TupleId> tids, TNullSide nullSide, Analyzer analyzer)
throws UserException {
if (!requiresNullWrapping(expr, analyzer)) {
return expr;
}
List<Expr> params = Lists.newArrayList();
params.add(new TupleIsNullPredicate(tids));
params.add(new TupleIsNullPredicate(tids, nullSide));
params.add(new NullLiteral());
params.add(expr);
Expr ifExpr = new FunctionCallExpr("if", params);
@ -192,32 +172,6 @@ public class TupleIsNullPredicate extends Predicate {
return ifExpr;
}
/**
* Returns a new analyzed conditional expr 'IF(TupleIsNull(nullSide), NULL, expr)',
* if required to make expr nullable. Otherwise, returns expr. only use in vectorized exec engine
*/
public static Expr wrapExpr(Expr expr, TNullSide nullSide, Analyzer analyzer)
throws UserException {
if (!requiresNullWrapping(expr, analyzer)) {
return expr;
}
List<Expr> params = Lists.newArrayList();
params.add(new TupleIsNullPredicate(nullSide));
params.add(new NullLiteral());
params.add(expr);
Expr ifExpr = new FunctionCallExpr("if", params);
ifExpr.analyzeNoThrow(analyzer);
// The type of function which is different from the type of expr will return the incorrect result in query.
// Example:
// the type of expr is date
// the type of function is int
// So, the upper fragment will receive a int value instead of date while the result expr is date.
// If there is no cast function, the result of query will be incorrect.
if (expr.getType().getPrimitiveType() != ifExpr.getType().getPrimitiveType()) {
ifExpr = ifExpr.uncheckedCastTo(expr.getType());
}
return ifExpr;
}
/**
* Returns true if the given expr evaluates to a non-NULL value if all its contained

6
fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
View File

@ -505,8 +505,8 @@ public class HashJoinNode extends PlanNode {
// Then: add tuple is null in left child columns
if (leftNullable && getChild(0).tblRefIds.size() == 1 && analyzer.isInlineView(getChild(0).tblRefIds.get(0))) {
List<Expr> tupleIsNullLhs = TupleIsNullPredicate
.wrapExprs(vSrcToOutputSMap.getLhs().subList(0, leftNullableNumber), TNullSide.LEFT,
analyzer);
.wrapExprs(vSrcToOutputSMap.getLhs().subList(0, leftNullableNumber), new ArrayList<>(),
TNullSide.LEFT, analyzer);
tupleIsNullLhs
.addAll(vSrcToOutputSMap.getLhs().subList(leftNullableNumber, vSrcToOutputSMap.getLhs().size()));
vSrcToOutputSMap.updateLhsExprs(tupleIsNullLhs);
@ -519,7 +519,7 @@ public class HashJoinNode extends PlanNode {
int rightBeginIndex = vSrcToOutputSMap.size() - rightNullableNumber;
List<Expr> tupleIsNullLhs = TupleIsNullPredicate
.wrapExprs(vSrcToOutputSMap.getLhs().subList(rightBeginIndex, vSrcToOutputSMap.size()),
TNullSide.RIGHT, analyzer);
new ArrayList<>(), TNullSide.RIGHT, analyzer);
List<Expr> newLhsList = Lists.newArrayList();
if (rightBeginIndex > 0) {
newLhsList.addAll(vSrcToOutputSMap.getLhs().subList(0, rightBeginIndex));

13
fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
View File

@ -65,6 +65,7 @@ import org.apache.doris.common.Reference;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.VectorizedUtil;
import org.apache.doris.planner.external.ExternalFileScanNode;
import org.apache.doris.thrift.TNullSide;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
@ -1393,8 +1394,14 @@ public class SingleNodePlanner {
//set outputSmap to substitute literal in outputExpr
unionNode.setWithoutTupleIsNullOutputSmap(inlineViewRef.getSmap());
if (analyzer.isOuterJoined(inlineViewRef.getId())) {
List<Expr> nullableRhs = TupleIsNullPredicate.wrapExprs(
inlineViewRef.getSmap().getRhs(), unionNode.getTupleIds(), analyzer);
List<Expr> nullableRhs;
if (analyzer.isOuterJoinedLeftSide(inlineViewRef.getId())) {
nullableRhs = TupleIsNullPredicate.wrapExprs(inlineViewRef.getSmap().getRhs(),
unionNode.getTupleIds(), TNullSide.LEFT, analyzer);
} else {
nullableRhs = TupleIsNullPredicate.wrapExprs(inlineViewRef.getSmap().getRhs(),
unionNode.getTupleIds(), TNullSide.RIGHT, analyzer);
}
unionNode.setOutputSmap(new ExprSubstitutionMap(inlineViewRef.getSmap().getLhs(), nullableRhs));
}
return unionNode;
@ -1422,7 +1429,7 @@ public class SingleNodePlanner {
// because the rhs exprs must first be resolved against the physical output of
// 'planRoot' to correctly determine whether wrapping is necessary.
List<Expr> nullableRhs = TupleIsNullPredicate.wrapExprs(
outputSmap.getRhs(), rootNode.getTupleIds(), analyzer);
outputSmap.getRhs(), rootNode.getTupleIds(), null, analyzer);
outputSmap = new ExprSubstitutionMap(outputSmap.getLhs(), nullableRhs);
}
// Set output smap of rootNode *before* creating a SelectNode for proper resolution.

2
fe/fe-core/src/test/java/org/apache/doris/analysis/TupleIsNullPredicateTest.java
View File

@ -30,7 +30,7 @@ public class TupleIsNullPredicateTest {
List<TupleId> tupleIds = Lists.newArrayList();
tupleIds.add(new TupleId(20));
tupleIds.add(new TupleId(21));
TupleIsNullPredicate tupleIsNullPredicate = new TupleIsNullPredicate(tupleIds);
TupleIsNullPredicate tupleIsNullPredicate = new TupleIsNullPredicate(tupleIds, null);
Assert.assertFalse(tupleIsNullPredicate.isBoundByTupleIds(Lists.newArrayList(new TupleId(1))));
}
}