database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](nereids) PushdownExpressionsInHashCondition contains duplicate column and WindowExpression miss column stats (#17624)

Browse Source 
tpcds: q47 and q57
1. PushdownExpressionsInHashCondition:project contains duplicate column
2. WindowExpression stats caclucate: miss column stats
This commit is contained in:
minghong
2023-03-10 16:08:43 +08:00
committed by GitHub
parent 365c8eed7e
commit 9ae5ec4dc5
2 changed files with 21 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownExpressionsInHashCondition.java
View File

@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
@ -33,7 +34,6 @@ import org.apache.doris.nereids.util.JoinUtils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@ -42,6 +42,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
/**
* push down expression which is not slot reference
@ -70,7 +71,7 @@ public class PushdownExpressionsInHashCondition extends OneRewriteRuleFactory {
.then(join -> {
List<List<Expression>> exprsOfHashConjuncts =
Lists.newArrayList(Lists.newArrayList(), Lists.newArrayList());
Map<Expression, Alias> exprMap = Maps.newHashMap();
Map<Expression, NamedExpression> exprMap = Maps.newHashMap();
join.getHashJoinConjuncts().forEach(conjunct -> {
Preconditions.checkArgument(conjunct instanceof EqualTo);
// sometimes: t1 join t2 on t2.a + 1 = t1.a + 2, so check the situation, but actually it
@ -79,8 +80,13 @@ public class PushdownExpressionsInHashCondition extends OneRewriteRuleFactory {
(EqualTo) conjunct, join.left().getOutputSet());
exprsOfHashConjuncts.get(0).add(conjunct.child(0));
exprsOfHashConjuncts.get(1).add(conjunct.child(1));
conjunct.children().forEach(expr ->
exprMap.put(expr, new Alias(expr, "expr_" + expr.toSql())));
conjunct.children().forEach(expr -> {
if ((expr instanceof SlotReference)) {
exprMap.put(expr, (SlotReference) expr);
} else {
exprMap.put(expr, new Alias(expr, "expr_" + expr.toSql()));
}
});
});
Iterator<List<Expression>> iter = exprsOfHashConjuncts.iterator();
return join.withHashJoinConjunctsAndChildren(
@ -90,10 +96,16 @@ public class PushdownExpressionsInHashCondition extends OneRewriteRuleFactory {
.collect(ImmutableList.toImmutableList())))
.collect(ImmutableList.toImmutableList()),
join.children().stream().map(
plan -> new LogicalProject<>(new Builder<NamedExpression>()
.addAll(iter.next().stream().map(exprMap::get)
.collect(ImmutableList.toImmutableList()))
.addAll(getOutput(plan, join)).build(), plan))
plan -> {
Set<NamedExpression> projectSet = Sets.newHashSet();
projectSet.addAll(iter.next().stream().map(exprMap::get)
.collect(Collectors.toList()));
projectSet.addAll(getOutput(plan, join));
List<NamedExpression> projectList = projectSet.stream()
.collect(ImmutableList.toImmutableList());
return new LogicalProject<>(projectList, plan);
}
)
.collect(ImmutableList.toImmutableList()));
}).toRule(RuleType.PUSHDOWN_EXPRESSIONS_IN_HASH_CONDITIONS);
}

1
fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
View File

@ -666,6 +666,7 @@ public class StatsCalculator extends DefaultPlanVisitor<StatsDeriveResult, Void>
}
return Pair.of(expr.toSlot().getExprId(), value);
}).collect(Collectors.toMap(Pair::key, Pair::value));
columnStatisticMap.putAll(childColumnStats);
return new StatsDeriveResult(stats.getRowCount(), columnStatisticMap);
}
}