database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](mtmv)Fix slot desc wrong in query rewrite by materialized view when query is complex (#34904)

Browse Source
This commit is contained in:
seawinde
2024-05-17 11:39:39 +08:00
committed by yiguolei
parent 5b72dd1217
commit cc11e50200
19 changed files with 920 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
View File

@ -40,6 +40,7 @@ import org.apache.doris.mtmv.MTMVRefreshSnapshot;
import org.apache.doris.mtmv.MTMVRelation;
import org.apache.doris.mtmv.MTMVStatus;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@ -267,12 +268,15 @@ public class MTMV extends OlapTable {
}
}
public MTMVCache getOrGenerateCache() throws AnalysisException {
/**
* Called when in query, Should use one connection context in query
*/
public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws AnalysisException {
if (cache == null) {
writeMvLock();
try {
if (cache == null) {
this.cache = MTMVCache.from(this, MTMVPlanUtil.createMTMVContext(this));
this.cache = MTMVCache.from(this, connectionContext);
}
} finally {
writeMvUnlock();

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View File

@ -323,9 +323,9 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
continue;
}
recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext);
rewriteResults.add(rewrittenPlan);
// if rewrite successfully, try to regenerate mv scan because it maybe used again
materializationContext.tryReGenerateMvScanPlan(cascadesContext);
rewriteResults.add(rewrittenPlan);
}
return rewriteResults;
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
View File

@ -63,6 +63,9 @@ public class InitMaterializationContextHook implements PlannerHook {
}
Plan rewritePlan = cascadesContext.getRewritePlan();
TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), true);
// Keep use one connection context when in query, if new connect context,
// the ConnectionContext.get() will change
collectorContext.setConnectContext(cascadesContext.getConnectContext());
rewritePlan.accept(TableCollector.INSTANCE, collectorContext);
Set<TableIf> collectedTables = collectorContext.getCollectedTables();
if (collectedTables.isEmpty()) {
@ -80,7 +83,7 @@ public class InitMaterializationContextHook implements PlannerHook {
for (MTMV materializedView : availableMTMVs) {
MTMVCache mtmvCache = null;
try {
mtmvCache = materializedView.getOrGenerateCache();
mtmvCache = materializedView.getOrGenerateCache(cascadesContext.getConnectContext());
} catch (AnalysisException e) {
LOG.warn("MaterializationContext init mv cache generate fail", e);
}

13
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
View File

@ -134,15 +134,14 @@ public abstract class MaterializationContext {
* Try to generate scan plan for materialization
* if MaterializationContext is already rewritten successfully, then should generate new scan plan in later
* query rewrite, because one plan may hit the materialized view repeatedly and the mv scan output
* should be different
* should be different.
* This method should be called when query rewrite successfully
*/
public void tryReGenerateMvScanPlan(CascadesContext cascadesContext) {
if (!this.matchedSuccessGroups.isEmpty()) {
this.mvScanPlan = doGenerateMvPlan(cascadesContext);
// mv output expression shuttle, this will be used to expression rewrite
this.mvExprToMvScanExprMapping = ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions,
this.mvScanPlan.getExpressions());
}
this.mvScanPlan = doGenerateMvPlan(cascadesContext);
// mv output expression shuttle, this will be used to expression rewrite
this.mvExprToMvScanExprMapping = ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions,
this.mvScanPlan.getOutput());
}
public void addSlotMappingToCache(RelationMapping relationMapping, SlotMapping slotMapping) {

7
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
View File

@ -184,12 +184,13 @@ public class MaterializedViewUtils {
LogicalOlapScan mvScan = new LogicalOlapScan(
cascadesContext.getStatementContext().getNextRelationId(),
materializedView,
ImmutableList.of(materializedView.getQualifiedDbName()),
materializedView.getFullQualifiers(),
ImmutableList.of(),
materializedView.getBaseIndexId(),
PreAggStatus.on(),
// this must be empty, or it will be used to sample
ImmutableList.of(),
ImmutableList.of(),
Optional.empty());
mvScan = mvScan.withMaterializedIndexSelected(PreAggStatus.on(), materializedView.getBaseIndexId());
List<NamedExpression> mvProjects = mvScan.getOutput().stream().map(NamedExpression.class::cast)
.collect(Collectors.toList());
return new LogicalProject<Plan>(mvProjects, mvScan);

14
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java
View File

@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext;
import org.apache.doris.qe.ConnectContext;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -72,7 +73,9 @@ public class TableCollector extends DefaultPlanVisitor<Plan, TableCollectorConte
if (!context.isExpand()) {
return;
}
MTMVCache expandedMv = MTMVCache.from(mtmv, MTMVPlanUtil.createMTMVContext(mtmv));
// Make sure use only one connection context when in query to avoid ConnectionContext.get() wrong
MTMVCache expandedMv = MTMVCache.from(mtmv, context.getConnectContext() == null
? MTMVPlanUtil.createMTMVContext(mtmv) : context.getConnectContext());
expandedMv.getLogicalPlan().accept(this, context);
}
@ -85,6 +88,7 @@ public class TableCollector extends DefaultPlanVisitor<Plan, TableCollectorConte
private final Set<TableType> targetTableTypes;
// if expand the mv or not
private final boolean expand;
private ConnectContext connectContext;
public TableCollectorContext(Set<TableType> targetTableTypes, boolean expand) {
this.targetTableTypes = targetTableTypes;
@ -102,5 +106,13 @@ public class TableCollector extends DefaultPlanVisitor<Plan, TableCollectorConte
public boolean isExpand() {
return expand;
}
public ConnectContext getConnectContext() {
return connectContext;
}
public void setConnectContext(ConnectContext connectContext) {
this.connectContext = connectContext;
}
}
}