database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feat](Nereids) push down predicates with multi columns through LogicalWindow and LogicalPartitionTopN (#36828) (#36981)

Browse Source 
cherry-pick #36828 to branch-2.1

The requirement for predicate pushdown through the window operator is
that the partition by slots of the window contains all slots in the
predicate. The original implementation of doris only allows predicate
pushdown with one slot. This PR relaxes this restriction and allows for
predicate pushdown with multiple slots. The same applies to the
predicate pushdown of the LogicalPartitionTopN operator. The following
sql is an example.

select
	*
from
	(
	select
		row_number() over(partition by id, value1 order by value1) as num,
		id,
		value1
	from
		push_down_multi_column_predicate_through_window_t ) t
where
	abs(id + value1)<4
	and num <= 2;


Co-authored-by: feiniaofeiafei <moailing@selectdb.com>
This commit is contained in:
feiniaofeiafei
2024-07-04 21:00:08 +08:00
committed by GitHub
parent c8978fc9d1
commit 4e4f3d204e
4 changed files with 89 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughPartitionTopN.java
View File

@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSet.Builder;
import java.util.HashSet;
import java.util.Set;
/**
@ -62,19 +63,17 @@ public class PushDownFilterThroughPartitionTopN extends OneRewriteRuleFactory {
// PushdownFilterThroughWindow
Builder<Expression> bottomConjunctsBuilder = ImmutableSet.builder();
Builder<Expression> upperConjunctsBuilder = ImmutableSet.builder();
for (Expression expr : filter.getConjuncts()) {
boolean pushed = false;
Set<Slot> exprInputSlots = expr.getInputSlots();
for (Expression partitionKey : partitionTopN.getPartitionKeys()) {
if (partitionKey instanceof SlotReference
&& exprInputSlots.size() == 1
&& partitionKey.getInputSlots().containsAll(exprInputSlots)) {
bottomConjunctsBuilder.add(expr);
pushed = true;
break;
}
Set<SlotReference> partitionKeySlots = new HashSet<>();
for (Expression partitionKey : partitionTopN.getPartitionKeys()) {
if (partitionKey instanceof SlotReference) {
partitionKeySlots.add((SlotReference) partitionKey);
}
if (!pushed) {
}
for (Expression expr : filter.getConjuncts()) {
Set<Slot> exprInputSlots = expr.getInputSlots();
if (partitionKeySlots.containsAll(exprInputSlots)) {
bottomConjunctsBuilder.add(expr);
} else {
upperConjunctsBuilder.add(expr);
}
}

15
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughWindow.java
View File

@ -64,18 +64,9 @@ public class PushDownFilterThroughWindow extends OneRewriteRuleFactory {
Set<Expression> bottomConjuncts = Sets.newHashSet();
Set<Expression> upperConjuncts = Sets.newHashSet();
for (Expression expr : filter.getConjuncts()) {
boolean pushed = false;
for (Expression partitionKey : commonPartitionKeys) {
// partitionKey is a single slot reference,
// we want to push expressions which have only one input slot, and the input slot is used as
// partition key in all windowExpressions.
if (partitionKey.getInputSlots().containsAll(expr.getInputSlots())) {
bottomConjuncts.add(expr);
pushed = true;
break;
}
}
if (!pushed) {
if (commonPartitionKeys.containsAll(expr.getInputSlots())) {
bottomConjuncts.add(expr);
} else {
upperConjuncts.add(expr);
}
}