database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](mtmv) Fix some pr to 21, prs are (#39041)(#38958)(#39541) (#39678)

Browse Source 
## Proposed changes

pr: https://github.com/apache/doris/pull/39041
commitId: 22562985

pr: https://github.com/apache/doris/pull/38958
commitId: c365cb64

pr: https://github.com/apache/doris/pull/39541
commitId: 89bb669c
This commit is contained in:
seawinde
2024-08-22 10:27:55 +08:00
committed by GitHub
parent d36ba97eae
commit 021982fc71
20 changed files with 1306 additions and 390 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
View File

@ -190,7 +190,7 @@ public class MTMV extends OlapTable {
this.relation = relation;
if (!Env.isCheckpointThread()) {
try {
this.cache = MTMVCache.from(this, MTMVPlanUtil.createMTMVContext(this));
this.cache = MTMVCache.from(this, MTMVPlanUtil.createMTMVContext(this), true);
} catch (Throwable e) {
this.cache = null;
LOG.warn("generate cache failed", e);
@ -277,7 +277,7 @@ public class MTMV extends OlapTable {
writeMvLock();
try {
if (cache == null) {
this.cache = MTMVCache.from(this, connectionContext);
this.cache = MTMVCache.from(this, connectionContext, true);
}
} finally {
writeMvUnlock();

13
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
View File

@ -79,7 +79,7 @@ public class MTMVCache {
return structInfo;
}
public static MTMVCache from(MTMV mtmv, ConnectContext connectContext) {
public static MTMVCache from(MTMV mtmv, ConnectContext connectContext, boolean needCost) {
LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(mtmv.getQuerySql());
StatementContext mvSqlStatementContext = new StatementContext(connectContext,
new OriginStatement(mtmv.getQuerySql(), 0));
@ -89,7 +89,13 @@ public class MTMVCache {
}
// Can not convert to table sink, because use the same column from different table when self join
// the out slot is wrong
planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN);
if (needCost) {
// Only in mv rewrite, we need plan with eliminated cost which is used for mv chosen
planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN);
} else {
// No need cost for performance
planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN);
}
Plan originPlan = planner.getCascadesContext().getRewritePlan();
// Eliminate result sink because sink operator is useless in query rewrite by materialized view
// and the top sort can also be removed
@ -111,7 +117,8 @@ public class MTMVCache {
Optional<StructInfo> structInfoOptional = MaterializationContext.constructStructInfo(mvPlan, originPlan,
planner.getCascadesContext(),
new BitSet());
return new MTMVCache(mvPlan, originPlan, planner.getCascadesContext().getMemo().getRoot().getStatistics(),
return new MTMVCache(mvPlan, originPlan, needCost
? planner.getCascadesContext().getMemo().getRoot().getStatistics() : null,
structInfoOptional.orElseGet(() -> null));
}
}

38
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
View File

@ -214,7 +214,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
LogicalAggregate<Plan> queryAggregate = queryTopPlanAndAggPair.value();
List<Expression> queryGroupByExpressions = queryAggregate.getGroupByExpressions();
// handle the scene that query top plan not use the group by in query bottom aggregate
if (queryGroupByExpressions.size() != queryTopPlanGroupBySet.size()) {
if (needCompensateGroupBy(queryTopPlanGroupBySet, queryGroupByExpressions)) {
for (Expression expression : queryGroupByExpressions) {
if (queryTopPlanGroupBySet.contains(expression)) {
continue;
@ -263,6 +263,42 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
return new LogicalAggregate<>(finalGroupExpressions, finalOutputExpressions, tempRewritedPlan);
}
/**
* handle the scene that query top plan not use the group by in query bottom aggregate
* If mv is select o_orderdate from orders group by o_orderdate;
* query is select 1 from orders group by o_orderdate.
* Or mv is select o_orderdate from orders group by o_orderdate
* query is select o_orderdate from orders group by o_orderdate, o_orderkey;
* if the slot which query top project use can not cover the slot which query bottom aggregate group by slot
* should compensate group by to make sure the data is right.
* For example:
* mv is select o_orderdate from orders group by o_orderdate;
* query is select o_orderdate from orders group by o_orderdate, o_orderkey;
*
* @param queryGroupByExpressions query bottom aggregate group by is o_orderdate, o_orderkey
* @param queryTopProject query top project is o_orderdate
* @return need to compensate group by if true or not need
*
*/
private static boolean needCompensateGroupBy(Set<? extends Expression> queryTopProject,
List<Expression> queryGroupByExpressions) {
Set<Expression> queryGroupByExpressionSet = new HashSet<>(queryGroupByExpressions);
if (queryGroupByExpressionSet.size() != queryTopProject.size()) {
return true;
}
Set<NamedExpression> queryTopPlanGroupByUseNamedExpressions = new HashSet<>();
Set<NamedExpression> queryGroupByUseNamedExpressions = new HashSet<>();
for (Expression expr : queryTopProject) {
queryTopPlanGroupByUseNamedExpressions.addAll(expr.collect(NamedExpression.class::isInstance));
}
for (Expression expr : queryGroupByExpressionSet) {
queryGroupByUseNamedExpressions.addAll(expr.collect(NamedExpression.class::isInstance));
}
// if the slots query top project use can not cover the slots which query bottom aggregate use
// Should compensate.
return !queryTopPlanGroupByUseNamedExpressions.containsAll(queryGroupByUseNamedExpressions);
}
/**
* Try to rewrite query expression by view, contains both group by dimension and aggregate function
*/

13
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectScanRule.java
View File

@ -19,8 +19,9 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.collect.ImmutableList;
@ -37,9 +38,11 @@ public class MaterializedViewFilterProjectScanRule extends MaterializedViewScanR
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
logicalFilter(logicalProject(logicalOlapScan())).thenApplyMultiNoThrow(ctx -> {
LogicalFilter<LogicalProject<LogicalOlapScan>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_PROJECT_SCAN));
logicalFilter(logicalProject(any().when(LogicalCatalogRelation.class::isInstance)))
.thenApplyMultiNoThrow(
ctx -> {
LogicalFilter<LogicalProject<Plan>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_PROJECT_SCAN));
}
}

7
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterScanRule.java
View File

@ -19,8 +19,9 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import com.google.common.collect.ImmutableList;
@ -36,8 +37,8 @@ public class MaterializedViewFilterScanRule extends MaterializedViewScanRule {
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
logicalFilter(logicalOlapScan()).thenApplyMultiNoThrow(ctx -> {
LogicalFilter<LogicalOlapScan> root = ctx.root;
logicalFilter(any().when(LogicalCatalogRelation.class::isInstance)).thenApplyMultiNoThrow(ctx -> {
LogicalFilter<Plan> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_SCAN));
}

13
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterScanRule.java
View File

@ -19,8 +19,9 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.collect.ImmutableList;
@ -37,9 +38,11 @@ public class MaterializedViewProjectFilterScanRule extends MaterializedViewScanR
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
logicalProject(logicalFilter(logicalOlapScan())).thenApplyMultiNoThrow(ctx -> {
LogicalProject<LogicalFilter<LogicalOlapScan>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_FILTER_SCAN));
logicalProject(logicalFilter(any().when(LogicalCatalogRelation.class::isInstance)))
.thenApplyMultiNoThrow(
ctx -> {
LogicalProject<LogicalFilter<Plan>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_FILTER_SCAN));
}
}

7
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectScanRule.java
View File

@ -19,7 +19,8 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.collect.ImmutableList;
@ -36,8 +37,8 @@ public class MaterializedViewProjectScanRule extends MaterializedViewScanRule {
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
logicalProject(logicalOlapScan()).thenApplyMultiNoThrow(ctx -> {
LogicalProject<LogicalOlapScan> root = ctx.root;
logicalProject(any().when(LogicalCatalogRelation.class::isInstance)).thenApplyMultiNoThrow(ctx -> {
LogicalProject<Plan> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_SCAN));
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java
View File

@ -75,7 +75,7 @@ public class TableCollector extends DefaultPlanVisitor<Plan, TableCollectorConte
}
// Make sure use only one connection context when in query to avoid ConnectionContext.get() wrong
MTMVCache expandedMv = MTMVCache.from(mtmv, context.getConnectContext() == null
? MTMVPlanUtil.createMTMVContext(mtmv) : context.getConnectContext());
? MTMVPlanUtil.createMTMVContext(mtmv) : context.getConnectContext(), false);
expandedMv.getLogicalPlan().accept(this, context);
}