database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](nereids)support stats estimation for is-null predicate (#24764)

Browse Source 
1. condition order: filter/hashCondition/otherCondition,
2. update regression out
3. remove tpch_sf500 shape case(covered by tpch sf1000)
4. implement is-null stats estimation
5. update ssb shape
This commit is contained in:
minghong
2023-09-27 10:04:35 +08:00
committed by GitHub
parent b9928a017b
commit a8f312794e
183 changed files with 947 additions and 3455 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
View File

@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.GreaterThan;
import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.LessThan;
import org.apache.doris.nereids.trees.expressions.LessThanEqual;
import org.apache.doris.nereids.trees.expressions.Like;
@ -386,6 +387,22 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
return statisticsBuilder.build();
}
@Override
public Statistics visitIsNull(IsNull isNull, EstimationContext context) {
ColumnStatistic childStats = ExpressionEstimation.estimate(isNull.child(), context.statistics);
if (childStats.isUnKnown()) {
return new StatisticsBuilder(context.statistics).build();
}
double outputRowCount = childStats.numNulls;
ColumnStatisticBuilder colBuilder = new ColumnStatisticBuilder(childStats);
// do not modify ndv/min/max to make is-not-null work
colBuilder.setCount(outputRowCount).setNumNulls(outputRowCount);
StatisticsBuilder builder = new StatisticsBuilder(context.statistics);
builder.putColumnStatistics(isNull.child(), colBuilder.build());
// TODO we do not call updateRowCountOnly() to make is-not-null work. this need refactor
return builder.build();
}
static class EstimationContext {
private final Statistics statistics;

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java
View File

@ -90,7 +90,7 @@ public class InPredicate extends Expression {
@Override
public String toSql() {
return compareExpr.toSql() + " IN " + options.stream()
.map(Expression::toSql)
.map(Expression::toSql).sorted()
.collect(Collectors.joining(", ", "(", ")"));
}

12
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java
View File

@ -37,6 +37,7 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Physical filter plan.
@ -136,9 +137,14 @@ public class PhysicalFilter<CHILD_TYPE extends Plan> extends PhysicalUnary<CHILD
@Override
public String shapeInfo() {
StringBuilder builder = new StringBuilder();
builder.append("filter(");
conjuncts.forEach(conjunct -> builder.append(conjunct.shapeInfo()));
builder.append(")");
builder.append("filter");
builder.append(
conjuncts.stream().map(conjunct -> conjunct.shapeInfo())
.sorted()
.collect(Collectors.joining(" and ", "(", ")")));
// List<String> strConjuncts = Lists.newArrayList();
// conjuncts.forEach(conjunct -> strConjuncts.add(conjunct.shapeInfo()));
// builder.append(strConjuncts.stream().sorted().collect(Collectors.joining(" and ", "(", ")")));
return builder.toString();
}

37
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
View File

@ -48,7 +48,6 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@ -274,43 +273,15 @@ public class PhysicalHashJoin<
return pushedDown;
}
private class ExprComparator implements Comparator<Expression> {
@Override
public int compare(Expression e1, Expression e2) {
List<ExprId> ids1 = e1.getInputSlotExprIds()
.stream().sorted(Comparator.comparing(ExprId::asInt))
.collect(Collectors.toList());
List<ExprId> ids2 = e2.getInputSlotExprIds()
.stream().sorted(Comparator.comparing(ExprId::asInt))
.collect(Collectors.toList());
if (ids1.size() > ids2.size()) {
return 1;
} else if (ids1.size() < ids2.size()) {
return -1;
} else {
for (int i = 0; i < ids1.size(); i++) {
if (ids1.get(i).asInt() > ids2.get(i).asInt()) {
return 1;
} else if (ids1.get(i).asInt() < ids2.get(i).asInt()) {
return -1;
}
}
return 0;
}
}
}
@Override
public String shapeInfo() {
StringBuilder builder = new StringBuilder();
builder.append("hashJoin[").append(joinType).append("]");
// print sorted hash conjuncts for plan check
hashJoinConjuncts.stream().sorted(new ExprComparator()).forEach(expr -> {
builder.append(expr.shapeInfo());
});
otherJoinConjuncts.stream().sorted(new ExprComparator()).forEach(expr -> {
builder.append(expr.shapeInfo());
});
builder.append(hashJoinConjuncts.stream().map(conjunct -> conjunct.shapeInfo())
.sorted().collect(Collectors.joining(" and ", " hashCondition=(", ")")));
builder.append(otherJoinConjuncts.stream().map(cond -> cond.shapeInfo())
.sorted().collect(Collectors.joining(" and ", "otherCondition=(", ")")));
return builder.toString();
}