database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Improvement](nereids) Support join derivation when mv rewrite (#29609)

Browse Source 
materialized view def is as following:
>            select l_linenumber, o_custkey
>           from orders
>            left join lineitem on lineitem.L_ORDERKEY = orders.O_ORDERKEY
>            where o_custkey = 1;

when query is as following, it can be rewritten by mv above
it requires that query has reject null filters on the join right input, 
current supported filter are  "=", "<", "<=", ">", ">=", "<=>" 
>            select IFNULL(orders.O_CUSTKEY, 0) as custkey_not_null,
>           case when l_linenumber in (1,2,3) then l_linenumber else o_custkey end as case_when
>            from orders
>            inner join lineitem on orders.O_ORDERKEY = lineitem.L_ORDERKEY
>            where o_custkey = 1 and l_linenumber > 0;
This commit is contained in:
seawinde
2024-01-09 15:32:19 +08:00
committed by yiguolei
parent 34fe5ee38b
commit fda001b6d3
11 changed files with 237 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View File

@ -41,6 +41,8 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.plans.JoinType;
@ -50,6 +52,7 @@ import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.TypeUtils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
@ -149,7 +152,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
queryStructInfo.addPredicates(pulledUpExpressions);
}
SplitPredicate compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo,
queryToViewSlotMapping);
queryToViewSlotMapping, comparisonResult, cascadesContext);
// Can not compensate, bail out
if (compensatePredicates.isEmpty()) {
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
@ -189,16 +192,15 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
if (rewrittenPlan == null) {
continue;
}
// checkout the output logical properties is the same with query
if (!checkOutput(queryPlan, rewrittenPlan, materializationContext)) {
continue;
}
// run rbo job on mv rewritten plan
CascadesContext rewrittenPlanContext = CascadesContext.initContext(
cascadesContext.getStatementContext(), rewrittenPlan,
cascadesContext.getCurrentJobContext().getRequiredProperties());
Rewriter.getWholeTreeRewriter(rewrittenPlanContext).execute();
rewrittenPlan = rewrittenPlanContext.getRewritePlan();
if (!checkOutput(queryPlan, rewrittenPlan, materializationContext)) {
continue;
}
// check the partitions used by rewritten plan is valid or not
Set<Long> invalidPartitionsQueryUsed =
calcInvalidPartitions(rewrittenPlan, materializationContext, cascadesContext);
@ -213,7 +215,6 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
.collect(Collectors.toSet()))));
continue;
}
materializationContext.setSuccess(true);
recordIfRewritten(queryPlan, materializationContext);
rewriteResults.add(rewrittenPlan);
}
@ -315,20 +316,28 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
List<Expression> rewrittenExpressions = new ArrayList<>();
for (int index = 0; index < sourceShuttledExpressions.size(); index++) {
Expression expressionToRewrite = sourceShuttledExpressions.get(index);
if (expressionToRewrite instanceof Literal) {
rewrittenExpressions.add(expressionToRewrite);
Expression expressionShuttledToRewrite = sourceShuttledExpressions.get(index);
if (expressionShuttledToRewrite instanceof Literal) {
rewrittenExpressions.add(expressionShuttledToRewrite);
continue;
}
final Set<Object> slotsToRewrite = expressionToRewrite.collectToSet(
expression -> expression instanceof Slot);
Expression replacedExpression = ExpressionUtils.replace(expressionToRewrite,
final Set<Object> slotsToRewrite =
expressionShuttledToRewrite.collectToSet(expression -> expression instanceof Slot);
Expression replacedExpression = ExpressionUtils.replace(expressionShuttledToRewrite,
targetToTargetReplacementMapping);
if (replacedExpression.anyMatch(slotsToRewrite::contains)) {
// if contains any slot to rewrite, which means can not be rewritten by target, bail out
return ImmutableList.of();
}
Expression sourceExpression = sourceExpressionsToWrite.get(index);
if (sourceExpression instanceof NamedExpression
&& replacedExpression.nullable() != sourceExpression.nullable()) {
// if enable join eliminate, query maybe inner join and mv maybe outer join.
// If the slot is at null generate side, the nullable maybe different between query and view
// So need to force to consistent.
replacedExpression = sourceExpression.nullable()
? new Nullable(replacedExpression) : new NonNullable(replacedExpression);
}
if (sourceExpression instanceof NamedExpression) {
NamedExpression sourceNamedExpression = (NamedExpression) sourceExpression;
replacedExpression = new Alias(sourceNamedExpression.getExprId(), replacedExpression,
@ -358,30 +367,87 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
* For another example as following:
* predicate a = b in mv, and a = b and c = d in query, the compensatory predicate is c = d
*/
protected SplitPredicate predicatesCompensate(StructInfo queryStructInfo, StructInfo viewStructInfo,
SlotMapping queryToViewSlotMapping) {
protected SplitPredicate predicatesCompensate(
StructInfo queryStructInfo,
StructInfo viewStructInfo,
SlotMapping queryToViewSlotMapping,
ComparisonResult comparisonResult,
CascadesContext cascadesContext
) {
// viewEquivalenceClass to query based
SlotMapping viewToQuerySlotMapping = queryToViewSlotMapping.inverse();
final Set<Expression> equalCompensateConjunctions = compensateEquivalence(
queryStructInfo,
viewStructInfo,
viewToQuerySlotMapping,
comparisonResult);
// range compensate
final Set<Expression> rangeCompensatePredicates = compensateRangePredicate(
queryStructInfo,
viewStructInfo,
viewToQuerySlotMapping,
comparisonResult);
// residual compensate
final Set<Expression> residualCompensatePredicates = compensateResidualPredicate(
queryStructInfo,
viewStructInfo,
viewToQuerySlotMapping,
comparisonResult);
// if the join type in query and mv plan is different, we should check and add filter on mv to make
// the mv join type is accord with query
Set<Set<Slot>> requireNoNullableViewSlot = comparisonResult.getViewNoNullableSlot();
// check query is use the null reject slot which view comparison need
if (!requireNoNullableViewSlot.isEmpty()) {
Set<Expression> queryPulledUpPredicates = queryStructInfo.getPredicates().getPulledUpPredicates();
Set<Expression> nullRejectPredicates = ExpressionUtils.inferNotNull(queryPulledUpPredicates,
cascadesContext);
if (nullRejectPredicates.isEmpty() || queryPulledUpPredicates.containsAll(nullRejectPredicates)) {
// query has not null reject predicates, so return
return SplitPredicate.invalid();
}
Set<Expression> queryUsedNeedRejectNullSlotsViewBased = nullRejectPredicates.stream()
.map(expression -> TypeUtils.isNotNull(expression).orElse(null))
.filter(Objects::nonNull)
.map(expr -> ExpressionUtils.replace((Expression) expr,
queryToViewSlotMapping.toSlotReferenceMap()))
.collect(Collectors.toSet());
if (requireNoNullableViewSlot.stream().anyMatch(
set -> Sets.intersection(set, queryUsedNeedRejectNullSlotsViewBased).isEmpty())) {
return SplitPredicate.invalid();
}
}
return SplitPredicate.of(ExpressionUtils.and(equalCompensateConjunctions),
rangeCompensatePredicates.isEmpty() ? BooleanLiteral.of(true)
: ExpressionUtils.and(rangeCompensatePredicates),
residualCompensatePredicates.isEmpty() ? BooleanLiteral.of(true)
: ExpressionUtils.and(residualCompensatePredicates));
}
protected Set<Expression> compensateEquivalence(StructInfo queryStructInfo,
StructInfo viewStructInfo,
SlotMapping viewToQuerySlotMapping,
ComparisonResult comparisonResult) {
EquivalenceClass queryEquivalenceClass = queryStructInfo.getEquivalenceClass();
EquivalenceClass viewEquivalenceClass = viewStructInfo.getEquivalenceClass();
// viewEquivalenceClass to query based
Map<SlotReference, SlotReference> viewToQuerySlotMapping = queryToViewSlotMapping.inverse()
.toSlotReferenceMap();
EquivalenceClass viewEquivalenceClassQueryBased = viewEquivalenceClass.permute(viewToQuerySlotMapping);
Map<SlotReference, SlotReference> viewToQuerySlotMap = viewToQuerySlotMapping.toSlotReferenceMap();
EquivalenceClass viewEquivalenceClassQueryBased = viewEquivalenceClass.permute(viewToQuerySlotMap);
if (viewEquivalenceClassQueryBased == null) {
return SplitPredicate.empty();
return ImmutableSet.of();
}
final List<Expression> equalCompensateConjunctions = new ArrayList<>();
final Set<Expression> equalCompensateConjunctions = new HashSet<>();
if (queryEquivalenceClass.isEmpty() && viewEquivalenceClass.isEmpty()) {
equalCompensateConjunctions.add(BooleanLiteral.of(true));
}
if (queryEquivalenceClass.isEmpty() && !viewEquivalenceClass.isEmpty()) {
return SplitPredicate.empty();
if (queryEquivalenceClass.isEmpty()
&& !viewEquivalenceClass.isEmpty()) {
return ImmutableSet.of();
}
EquivalenceClassSetMapping queryToViewEquivalenceMapping = EquivalenceClassSetMapping.generate(
queryEquivalenceClass, viewEquivalenceClassQueryBased);
EquivalenceClassSetMapping queryToViewEquivalenceMapping =
EquivalenceClassSetMapping.generate(queryEquivalenceClass, viewEquivalenceClassQueryBased);
// can not map all target equivalence class, can not compensate
if (queryToViewEquivalenceMapping.getEquivalenceClassSetMap().size()
< viewEquivalenceClass.getEquivalenceSetList().size()) {
return SplitPredicate.empty();
return ImmutableSet.of();
}
// do equal compensate
Set<Set<SlotReference>> mappedQueryEquivalenceSet =
@ -410,49 +476,57 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
}
}
);
// TODO range predicates and residual predicates compensate, Simplify implementation.
return equalCompensateConjunctions;
}
protected Set<Expression> compensateResidualPredicate(StructInfo queryStructInfo,
StructInfo viewStructInfo,
SlotMapping viewToQuerySlotMapping,
ComparisonResult comparisonResult) {
SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate();
SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate();
// range compensate
List<Expression> rangeCompensate = new ArrayList<>();
Expression queryRangePredicate = querySplitPredicate.getRangePredicate();
Expression viewRangePredicate = viewSplitPredicate.getRangePredicate();
Expression viewRangePredicateQueryBased = ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping);
Set<Expression> queryRangeSet = Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate));
Set<Expression> viewRangeQueryBasedSet = Sets.newHashSet(
ExpressionUtils.extractConjunction(viewRangePredicateQueryBased));
// query range predicate can not contain all view range predicate when view have range predicate, bail out
if (!viewRangePredicateQueryBased.equals(BooleanLiteral.TRUE) && !queryRangeSet.containsAll(
viewRangeQueryBasedSet)) {
return SplitPredicate.empty();
}
queryRangeSet.removeAll(viewRangeQueryBasedSet);
rangeCompensate.addAll(queryRangeSet);
// residual compensate
List<Expression> residualCompensate = new ArrayList<>();
Expression queryResidualPredicate = querySplitPredicate.getResidualPredicate();
Expression viewResidualPredicate = viewSplitPredicate.getResidualPredicate();
Expression viewResidualPredicateQueryBased =
ExpressionUtils.replace(viewResidualPredicate, viewToQuerySlotMapping);
ExpressionUtils.replace(viewResidualPredicate, viewToQuerySlotMapping.toSlotReferenceMap());
Set<Expression> queryResidualSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(queryResidualPredicate));
Set<Expression> viewResidualQueryBasedSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(viewResidualPredicateQueryBased));
// query residual predicate can not contain all view residual predicate when view have residual predicate,
// bail out
if (!viewResidualPredicateQueryBased.equals(BooleanLiteral.TRUE) && !queryResidualSet.containsAll(
viewResidualQueryBasedSet)) {
return SplitPredicate.empty();
if (!viewResidualPredicateQueryBased.equals(BooleanLiteral.TRUE)
&& !queryResidualSet.containsAll(viewResidualQueryBasedSet)) {
return ImmutableSet.of();
}
queryResidualSet.removeAll(viewResidualQueryBasedSet);
residualCompensate.addAll(queryResidualSet);
return queryResidualSet;
}
return SplitPredicate.of(ExpressionUtils.and(equalCompensateConjunctions),
rangeCompensate.isEmpty() ? BooleanLiteral.of(true) : ExpressionUtils.and(rangeCompensate),
residualCompensate.isEmpty() ? BooleanLiteral.of(true) : ExpressionUtils.and(residualCompensate));
protected Set<Expression> compensateRangePredicate(StructInfo queryStructInfo,
StructInfo viewStructInfo,
SlotMapping viewToQuerySlotMapping,
ComparisonResult comparisonResult) {
// TODO range predicates and residual predicates compensate, Simplify implementation.
SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate();
SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate();
Expression queryRangePredicate = querySplitPredicate.getRangePredicate();
Expression viewRangePredicate = viewSplitPredicate.getRangePredicate();
Expression viewRangePredicateQueryBased =
ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping.toSlotReferenceMap());
Set<Expression> queryRangeSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate));
Set<Expression> viewRangeQueryBasedSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(viewRangePredicateQueryBased));
// query range predicate can not contain all view range predicate when view have range predicate, bail out
if (!viewRangePredicateQueryBased.equals(BooleanLiteral.TRUE)
&& !queryRangeSet.containsAll(viewRangeQueryBasedSet)) {
return ImmutableSet.of();
}
queryRangeSet.removeAll(viewRangeQueryBasedSet);
return queryRangeSet;
}
/**
@ -508,6 +582,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
}
protected void recordIfRewritten(Plan plan, MaterializationContext context) {
context.setSuccess(true);
if (plan.getGroupExpression().isPresent()) {
context.addMatchedGroup(plan.getGroupExpression().get().getOwnerGroup().getGroupId());
}

4
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
View File

@ -51,7 +51,7 @@ public class Predicates {
return predicates;
}
public Set<? extends Expression> getPulledUpPredicates() {
public Set<Expression> getPulledUpPredicates() {
return pulledUpPredicates;
}
@ -103,7 +103,7 @@ public class Predicates {
return residualPredicate.orElse(BooleanLiteral.TRUE);
}
public static SplitPredicate empty() {
public static SplitPredicate invalid() {
return new SplitPredicate(null, null, null);
}

3
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java
View File

@ -66,6 +66,9 @@ public class PredicatesSplitter {
if (leftArgOnlyContainsColumnRef && rightArgOnlyContainsColumnRef) {
equalPredicates.add(comparisonPredicate);
return null;
} else if ((leftArgOnlyContainsColumnRef && rightArg instanceof Literal)
|| (rightArgOnlyContainsColumnRef && leftArg instanceof Literal)) {
rangePredicates.add(comparisonPredicate);
} else {
residualPredicates.add(comparisonPredicate);
}

10
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NonNullable.java
View File

@ -24,6 +24,10 @@ import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
import org.apache.doris.nereids.types.DataType;
import com.google.common.base.Preconditions;
import java.util.List;
/**
* change nullable input col to non_nullable col
*/
@ -39,4 +43,10 @@ public class NonNullable extends ScalarFunction implements UnaryExpression, Cust
return FunctionSignature.ret(dataType).args(dataType);
}
@Override
public Expression withChildren(List<Expression> children) {
Preconditions.checkArgument(children.size() == 1,
"the child expression of NonNullable should be only one");
return new NonNullable(children.get(0));
}
}

10
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Nullable.java
View File

@ -24,6 +24,10 @@ import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
import org.apache.doris.nereids.types.DataType;
import com.google.common.base.Preconditions;
import java.util.List;
/**
* change non_nullable input col to nullable col
*/
@ -39,4 +43,10 @@ public class Nullable extends ScalarFunction implements UnaryExpression, CustomS
return FunctionSignature.ret(dataType).args(dataType);
}
@Override
public Expression withChildren(List<Expression> children) {
Preconditions.checkArgument(children.size() == 1,
"the child expression of NonNullable should be only one");
return new Nullable(children.get(0));
}
}