database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

pick some pr from to branch21 #38115 #38008 #37929 (#38940)

Browse Source 
## Proposed changes

pr: https://github.com/apache/doris/pull/38115
commitId: 2b29288c

pr: https://github.com/apache/doris/pull/38008
commitId: c6b924da

pr: https://github.com/apache/doris/pull/37929
commitId: d44fcdc5
This commit is contained in:
seawinde
2024-08-07 10:19:41 +08:00
committed by GitHub
parent 2543b569bb
commit 2b1aa05370
45 changed files with 2841 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
View File

@ -273,8 +273,9 @@ public class CreateMaterializedViewStmt extends DdlStmt {
Expr selectListItemExpr = selectListItem.getExpr();
selectListItemExpr.setDisableTableName(true);
if (!(selectListItemExpr instanceof SlotRef) && !(selectListItemExpr instanceof FunctionCallExpr)
&& !(selectListItemExpr instanceof ArithmeticExpr)) {
Expr realItem = selectListItemExpr.unwrapExpr(false);
if (!(realItem instanceof SlotRef) && !(realItem instanceof FunctionCallExpr)
&& !(realItem instanceof ArithmeticExpr)) {
throw new AnalysisException("The materialized view only support the single column or function expr. "
+ "Error column: " + selectListItemExpr.toSql());
}

14
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
View File

@ -47,6 +47,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.BiPredicate;
/**
* when do some operation, do something about cache
@ -76,13 +77,17 @@ public class MTMVRelationManager implements MTMVHookService {
* @param ctx
* @return
*/
public Set<MTMV> getAvailableMTMVs(List<BaseTableInfo> tableInfos, ConnectContext ctx) {
public Set<MTMV> getAvailableMTMVs(List<BaseTableInfo> tableInfos, ConnectContext ctx,
boolean forceConsistent, BiPredicate<ConnectContext, MTMV> predicate) {
Set<MTMV> res = Sets.newLinkedHashSet();
Set<BaseTableInfo> mvInfos = getMTMVInfos(tableInfos);
for (BaseTableInfo tableInfo : mvInfos) {
try {
MTMV mtmv = (MTMV) MTMVUtil.getTable(tableInfo);
if (isMVPartitionValid(mtmv, ctx)) {
if (predicate.test(ctx, mtmv)) {
continue;
}
if (isMVPartitionValid(mtmv, ctx, forceConsistent)) {
res.add(mtmv);
}
} catch (AnalysisException e) {
@ -94,9 +99,10 @@ public class MTMVRelationManager implements MTMVHookService {
}
@VisibleForTesting
public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) {
public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean forceConsistent) {
long currentTimeMillis = System.currentTimeMillis();
return !CollectionUtils
.isEmpty(MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv, ctx, System.currentTimeMillis()));
.isEmpty(MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMillis, forceConsistent));
}
private Set<BaseTableInfo> getMTMVInfos(List<BaseTableInfo> tableInfos) {

13
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
View File

@ -45,18 +45,9 @@ public class MTMVRewriteUtil {
* @return
*/
public static Collection<Partition> getMTMVCanRewritePartitions(MTMV mtmv, ConnectContext ctx,
long currentTimeMills) {
long currentTimeMills, boolean forceConsistent) {
List<Partition> res = Lists.newArrayList();
Collection<Partition> allPartitions = mtmv.getPartitions();
// check session variable if enable rewrite
if (!ctx.getSessionVariable().isEnableMaterializedViewRewrite()) {
return res;
}
if (MTMVUtil.mtmvContainsExternalTable(mtmv) && !ctx.getSessionVariable()
.isMaterializedViewRewriteEnableContainExternalTable()) {
return res;
}
MTMVRelation mtmvRelation = mtmv.getRelation();
if (mtmvRelation == null) {
return res;
@ -71,7 +62,7 @@ public class MTMVRewriteUtil {
long gracePeriodMills = mtmv.getGracePeriod();
for (Partition partition : allPartitions) {
if (gracePeriodMills > 0 && currentTimeMills <= (partition.getVisibleVersionTime()
+ gracePeriodMills)) {
+ gracePeriodMills) && !forceConsistent) {
res.add(partition);
continue;
}

4
fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
View File

@ -122,7 +122,7 @@ public class CascadesContext implements ScheduleContext {
private final Optional<CTEId> currentTree;
private final Optional<CascadesContext> parent;
private final List<MaterializationContext> materializationContexts;
private final Set<MaterializationContext> materializationContexts;
private boolean isLeadingJoin = false;
private boolean isLeadingDisableJoinReorder = false;
@ -160,7 +160,7 @@ public class CascadesContext implements ScheduleContext {
this.currentJobContext = new JobContext(this, requireProperties, Double.MAX_VALUE);
this.subqueryExprIsAnalyzed = new HashMap<>();
this.runtimeFilterContext = new RuntimeFilterContext(getConnectContext().getSessionVariable());
this.materializationContexts = new ArrayList<>();
this.materializationContexts = new HashSet<>();
if (statementContext.getConnectContext() != null) {
ConnectContext connectContext = statementContext.getConnectContext();
SessionVariable sessionVariable = connectContext.getSessionVariable();

11
fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
View File

@ -92,7 +92,6 @@ public class NereidsPlanner extends Planner {
// The cost of optimized plan
private double cost = 0;
private LogicalPlanAdapter logicalPlanAdapter;
private List<PlannerHook> hooks = new ArrayList<>();
public NereidsPlanner(StatementContext statementContext) {
this.statementContext = statementContext;
@ -274,7 +273,7 @@ public class NereidsPlanner extends Planner {
LOG.debug("Start analyze plan");
}
keepOrShowPlanProcess(showPlanProcess, () -> cascadesContext.newAnalyzer().analyze());
getHooks().forEach(hook -> hook.afterAnalyze(this));
this.statementContext.getPlannerHooks().forEach(hook -> hook.afterAnalyze(this));
NereidsTracer.logImportantTime("EndAnalyzePlan");
if (LOG.isDebugEnabled()) {
LOG.debug("End analyze plan");
@ -640,14 +639,6 @@ public class NereidsPlanner extends Planner {
return logicalPlanAdapter;
}
public List<PlannerHook> getHooks() {
return hooks;
}
public void addHook(PlannerHook hook) {
this.hooks.add(hook);
}
private String getTimeMetricString(Function<SummaryProfile, String> profileSupplier) {
return getProfile(summaryProfile -> {
String metricString = profileSupplier.apply(summaryProfile);

10
fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
View File

@ -163,6 +163,8 @@ public class StatementContext implements Closeable {
private FormatOptions formatOptions = FormatOptions.getDefault();
private List<PlannerHook> plannerHooks = new ArrayList<>();
public StatementContext() {
this(ConnectContext.get(), null, 0);
}
@ -488,6 +490,14 @@ public class StatementContext implements Closeable {
return formatOptions;
}
public List<PlannerHook> getPlannerHooks() {
return plannerHooks;
}
public void addPlannerHook(PlannerHook plannerHook) {
this.plannerHooks.add(plannerHook);
}
private static class CloseableResource implements Closeable {
public final String resourceName;
public final String threadName;

7
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
View File

@ -22,7 +22,6 @@ import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.JobType;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
@ -102,10 +101,6 @@ public class OptimizeGroupExpressionJob extends Job {
}
private List<Rule> getMvRules() {
ConnectContext connectContext = context.getCascadesContext().getConnectContext();
if (connectContext.getSessionVariable().isEnableMaterializedViewRewrite()) {
return getRuleSet().getMaterializedViewRules();
}
return ImmutableList.of();
return getRuleSet().getMaterializedViewRules();
}
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.nereids.jobs.executor;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.jobs.rewrite.RewriteJob;
import org.apache.doris.nereids.rules.analysis.AddInitMaterializationHook;
import org.apache.doris.nereids.rules.analysis.AdjustAggregateNullableForEmptySet;
import org.apache.doris.nereids.rules.analysis.AnalyzeCTE;
import org.apache.doris.nereids.rules.analysis.BindExpression;
@ -123,6 +124,7 @@ public class Analyzer extends AbstractBatchJobExecutor {
bottomUp(new BindExpression()),
topDown(new BindSink()),
bottomUp(new CheckAfterBind()),
bottomUp(new AddInitMaterializationHook()),
bottomUp(
new ProjectToGlobalAggregate(),
// this rule check's the logicalProject node's isDistinct property

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View File

@ -33,6 +33,8 @@ public enum RuleType {
BINDING_INSERT_HIVE_TABLE(RuleTypeClass.REWRITE),
BINDING_INSERT_ICEBERG_TABLE(RuleTypeClass.REWRITE),
BINDING_INSERT_TARGET_TABLE(RuleTypeClass.REWRITE),
INIT_MATERIALIZATION_HOOK_FOR_FILE_SINK(RuleTypeClass.REWRITE),
INIT_MATERIALIZATION_HOOK_FOR_TABLE_SINK(RuleTypeClass.REWRITE),
BINDING_INSERT_FILE(RuleTypeClass.REWRITE),
BINDING_ONE_ROW_RELATION_SLOT(RuleTypeClass.REWRITE),
BINDING_RELATION(RuleTypeClass.REWRITE),

54
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AddInitMaterializationHook.java Normal file
View File

@ -0,0 +1,54 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.analysis;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.exploration.mv.InitConsistentMaterializationContextHook;
import org.apache.doris.nereids.trees.plans.logical.LogicalTableSink;
import com.google.common.collect.ImmutableList;
import java.util.List;
/**
* Add init materialization hook for table sink and file sink
* */
public class AddInitMaterializationHook implements AnalysisRuleFactory {
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
RuleType.INIT_MATERIALIZATION_HOOK_FOR_FILE_SINK.build(logicalFileSink()
.thenApply(ctx -> {
if (ctx.connectContext.getSessionVariable().isEnableDmlMaterializedViewRewrite()) {
ctx.statementContext.addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE);
}
return ctx.root;
})),
RuleType.INIT_MATERIALIZATION_HOOK_FOR_TABLE_SINK.build(
any().when(LogicalTableSink.class::isInstance)
.thenApply(ctx -> {
if (ctx.connectContext.getSessionVariable().isEnableDmlMaterializedViewRewrite()) {
ctx.statementContext.addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE);
}
return ctx.root;
}))
);
}
}

3
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
View File

@ -119,7 +119,6 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
queryTopPlan,
materializationContext.getShuttledExprToScanExprMapping(),
viewToQuerySlotMapping,
true,
queryStructInfo.getTableBitSet());
boolean isRewrittenQueryExpressionValid = true;
if (!rewrittenQueryExpressions.isEmpty()) {
@ -356,7 +355,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
viewGroupExpressionQueryBased
);
}
if (queryGroupShuttledExpression.equals(viewShuttledExpressionQueryBasedToGroupByExpressionMap.values())) {
if (queryGroupShuttledExpression.equals(viewShuttledExpressionQueryBasedToGroupByExpressionMap.keySet())) {
// return true, if equals directly
return true;
}

1
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
View File

@ -49,7 +49,6 @@ public abstract class AbstractMaterializedViewJoinRule extends AbstractMateriali
queryStructInfo.getTopPlan(),
materializationContext.getShuttledExprToScanExprMapping(),
targetToSourceMapping,
true,
queryStructInfo.getTableBitSet()
);
// Can not rewrite, bail out

145
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View File

@ -44,10 +44,12 @@ import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt;
import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
@ -56,6 +58,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.types.VariantType;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.TypeUtils;
import org.apache.doris.qe.SessionVariable;
@ -114,7 +117,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
continue;
}
// check mv plan is valid or not
if (!isMaterializationValid(cascadesContext, context)) {
if (!isMaterializationValid(queryPlan, cascadesContext, context)) {
continue;
}
// get query struct infos according to the view strut info, if valid query struct infos is empty, bail out
@ -238,7 +241,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
// Try to rewrite compensate predicates by using mv scan
List<Expression> rewriteCompensatePredicates = rewriteExpression(compensatePredicates.toList(),
queryPlan, materializationContext.getShuttledExprToScanExprMapping(),
viewToQuerySlotMapping, true, queryStructInfo.getTableBitSet());
viewToQuerySlotMapping, queryStructInfo.getTableBitSet());
if (rewriteCompensatePredicates.isEmpty()) {
materializationContext.recordFailReason(queryStructInfo,
"Rewrite compensate predicate by view fail",
@ -429,7 +432,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
.collect(Collectors.toSet());
Collection<Partition> mvValidPartitions = MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv,
cascadesContext.getConnectContext(), System.currentTimeMillis());
cascadesContext.getConnectContext(), System.currentTimeMillis(), false);
Set<String> mvValidPartitionNameSet = new HashSet<>();
Set<String> mvValidBaseTablePartitionNameSet = new HashSet<>();
Set<String> mvValidHasDataRelatedBaseTableNameSet = new HashSet<>();
@ -499,33 +502,20 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
* @param sourcePlan the source plan witch the source expression belong to
* @param targetExpressionMapping target expression mapping, if finding the expression in key set of the mapping
* then use the corresponding value of mapping to replace it
* @param targetExpressionNeedSourceBased if targetExpressionNeedSourceBased is true,
* we should make the target expression map key to source based,
* Note: the key expression in targetExpressionMapping should be shuttled. with the method
* ExpressionUtils.shuttleExpressionWithLineage.
* example as following:
* source target
* project(slot 1, 2) project(slot 3, 2, 1)
* scan(table) scan(table)
* then
* transform source to:
* project(slot 2, 1)
* target
*/
protected List<Expression> rewriteExpression(List<? extends Expression> sourceExpressionsToWrite, Plan sourcePlan,
ExpressionMapping targetExpressionMapping, SlotMapping targetToSourceMapping,
boolean targetExpressionNeedSourceBased, BitSet sourcePlanBitSet) {
ExpressionMapping targetExpressionMapping, SlotMapping targetToSourceMapping, BitSet sourcePlanBitSet) {
// Firstly, rewrite the target expression using source with inverse mapping
// then try to use the target expression to represent the query. if any of source expressions
// can not be represented by target expressions, return null.
// generate target to target replacement expression mapping, and change target expression to source based
List<? extends Expression> sourceShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage(
sourceExpressionsToWrite, sourcePlan, sourcePlanBitSet);
ExpressionMapping expressionMappingKeySourceBased = targetExpressionNeedSourceBased
? targetExpressionMapping.keyPermute(targetToSourceMapping) : targetExpressionMapping;
ExpressionMapping expressionMappingKeySourceBased = targetExpressionMapping.keyPermute(targetToSourceMapping);
// target to target replacement expression mapping, because mv is 1:1 so get first element
List<Map<Expression, Expression>> flattenExpressionMap = expressionMappingKeySourceBased.flattenMap();
Map<? extends Expression, ? extends Expression> targetToTargetReplacementMapping = flattenExpressionMap.get(0);
Map<Expression, Expression> targetToTargetReplacementMappingQueryBased =
flattenExpressionMap.get(0);
List<Expression> rewrittenExpressions = new ArrayList<>();
for (Expression expressionShuttledToRewrite : sourceShuttledExpressions) {
@ -535,8 +525,13 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
}
final Set<Object> slotsToRewrite =
expressionShuttledToRewrite.collectToSet(expression -> expression instanceof Slot);
final Set<SlotReference> variants =
expressionShuttledToRewrite.collectToSet(expression -> expression instanceof SlotReference
&& ((SlotReference) expression).getDataType() instanceof VariantType);
extendMappingByVariant(variants, targetToTargetReplacementMappingQueryBased);
Expression replacedExpression = ExpressionUtils.replace(expressionShuttledToRewrite,
targetToTargetReplacementMapping);
targetToTargetReplacementMappingQueryBased);
if (replacedExpression.anyMatch(slotsToRewrite::contains)) {
// if contains any slot to rewrite, which means can not be rewritten by target, bail out
return ImmutableList.of();
@ -546,6 +541,94 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
return rewrittenExpressions;
}
/**
* if query contains variant slot reference, extend the expression mapping for rewrte
* such as targetToTargetReplacementMappingQueryBased is
* id#0 -> id#8
* type#1 -> type#9
* payload#4 -> payload#10
* query variants is payload['issue']['number']#20
* then we can add payload['issue']['number']#20 -> element_at(element_at(payload#10, 'issue'), 'number')
* to targetToTargetReplacementMappingQueryBased
* */
private void extendMappingByVariant(Set<SlotReference> queryVariants,
Map<Expression, Expression> targetToTargetReplacementMappingQueryBased) {
if (queryVariants.isEmpty()) {
return;
}
Map<List<String>, Expression> viewNameToExprMap = new HashMap<>();
for (Map.Entry<Expression, Expression> targetExpressionEntry :
targetToTargetReplacementMappingQueryBased.entrySet()) {
if (targetExpressionEntry.getKey() instanceof SlotReference
&& ((SlotReference) targetExpressionEntry.getKey()).getDataType() instanceof VariantType) {
SlotReference targetSlotReference = (SlotReference) targetExpressionEntry.getKey();
List<String> nameIdentifier = new ArrayList<>(targetSlotReference.getQualifier());
nameIdentifier.add(targetSlotReference.getName());
nameIdentifier.addAll(targetSlotReference.getSubPath());
viewNameToExprMap.put(nameIdentifier, targetExpressionEntry.getValue());
}
}
if (viewNameToExprMap.isEmpty()) {
return;
}
Map<List<String>, SlotReference> queryNameAndExpressionMap = new HashMap<>();
for (SlotReference slotReference : queryVariants) {
List<String> nameIdentifier = new ArrayList<>(slotReference.getQualifier());
nameIdentifier.add(slotReference.getName());
nameIdentifier.addAll(slotReference.getSubPath());
queryNameAndExpressionMap.put(nameIdentifier, slotReference);
}
for (Map.Entry<List<String>, ? extends Expression> queryNameEntry : queryNameAndExpressionMap.entrySet()) {
Expression minExpr = null;
List<String> minCompensateName = null;
for (Map.Entry<List<String>, Expression> entry : viewNameToExprMap.entrySet()) {
if (!containsAllWithOrder(queryNameEntry.getKey(), entry.getKey())) {
continue;
}
List<String> removedQueryName = new ArrayList<>(queryNameEntry.getKey());
removedQueryName.removeAll(entry.getKey());
if (minCompensateName == null) {
minCompensateName = removedQueryName;
minExpr = entry.getValue();
}
if (removedQueryName.size() < minCompensateName.size()) {
minCompensateName = removedQueryName;
minExpr = entry.getValue();
}
}
if (minExpr != null) {
targetToTargetReplacementMappingQueryBased.put(queryNameEntry.getValue(),
constructElementAt(minExpr, minCompensateName));
}
}
}
private static Expression constructElementAt(Expression target, List<String> atList) {
Expression elementAt = target;
for (String at : atList) {
elementAt = new ElementAt(elementAt, new VarcharLiteral(at));
}
return elementAt;
}
// source names is contain all target with order or not
private static boolean containsAllWithOrder(List<String> sourceNames, List<String> targetNames) {
if (sourceNames.size() < targetNames.size()) {
return false;
}
for (int index = 0; index < targetNames.size(); index++) {
String sourceName = sourceNames.get(index);
String targetName = targetNames.get(index);
if (sourceName == null || targetName == null) {
return false;
}
if (!sourceName.equals(targetName)) {
return false;
}
}
return true;
}
/**
* Normalize expression with query, keep the consistency of exprId and nullable props with
* query
@ -731,8 +814,9 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
}
// check mv plan is valid or not, this can use cache for performance
private boolean isMaterializationValid(CascadesContext cascadesContext, MaterializationContext context) {
long materializationId = context.getMaterializationQualifier().hashCode();
private boolean isMaterializationValid(Plan queryPlan, CascadesContext cascadesContext,
MaterializationContext context) {
long materializationId = context.generateMaterializationIdentifier().hashCode();
Boolean cachedCheckResult = cascadesContext.getMemo().materializationHasChecked(this.getClass(),
materializationId);
if (cachedCheckResult == null) {
@ -742,6 +826,11 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
context.recordFailReason(context.getStructInfo(),
"View struct info is invalid", () -> String.format("view plan is %s",
context.getStructInfo().getOriginalPlan().treeString()));
// tmp to location question
LOG.debug(String.format("View struct info is invalid, mv identifier is %s, query plan is %s,"
+ "view plan is %s",
context.generateMaterializationIdentifier(), queryPlan.treeString(),
context.getStructInfo().getTopPlan().treeString()));
cascadesContext.getMemo().recordMaterializationCheckResult(this.getClass(), materializationId,
false);
return false;
@ -753,12 +842,20 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
context.recordFailReason(context.getStructInfo(),
"View struct info is invalid", () -> String.format("view plan is %s",
context.getStructInfo().getOriginalPlan().treeString()));
LOG.debug(String.format("View struct info is invalid, mv identifier is %s, query plan is %s,"
+ "view plan is %s",
context.generateMaterializationIdentifier(), queryPlan.treeString(),
context.getStructInfo().getTopPlan().treeString()));
return false;
}
if (!context.getStructInfo().isValid()) {
context.recordFailReason(context.getStructInfo(),
"View struct info is invalid", () -> String.format("view plan is %s",
"View original struct info is invalid", () -> String.format("view plan is %s",
context.getStructInfo().getOriginalPlan().treeString()));
LOG.debug(String.format("View struct info is invalid, mv identifier is %s, query plan is %s,"
+ "view plan is %s",
context.generateMaterializationIdentifier(), queryPlan.treeString(),
context.getStructInfo().getTopPlan().treeString()));
return false;
}
return true;

20
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java
View File

@ -50,7 +50,6 @@ public class AsyncMaterializationContext extends MaterializationContext {
private static final Logger LOG = LogManager.getLogger(AsyncMaterializationContext.class);
private final MTMV mtmv;
private List<String> materializationQualifier;
/**
* MaterializationContext, this contains necessary info for query rewriting by mv
@ -72,11 +71,11 @@ public class AsyncMaterializationContext extends MaterializationContext {
}
@Override
List<String> getMaterializationQualifier() {
if (this.materializationQualifier == null) {
this.materializationQualifier = this.mtmv.getFullQualifiers();
List<String> generateMaterializationIdentifier() {
if (super.identifier == null) {
super.identifier = MaterializationContext.generateMaterializationIdentifier(mtmv, null);
}
return this.materializationQualifier;
return super.identifier;
}
@Override
@ -92,7 +91,7 @@ public class AsyncMaterializationContext extends MaterializationContext {
}
}
failReasonBuilder.append("\n").append("]");
return Utils.toSqlString("MaterializationContext[" + getMaterializationQualifier() + "]",
return Utils.toSqlString("MaterializationContext[" + generateMaterializationIdentifier() + "]",
"rewriteSuccess", this.success,
"failReason", failReasonBuilder.toString());
}
@ -104,7 +103,7 @@ public class AsyncMaterializationContext extends MaterializationContext {
mtmvCache = mtmv.getOrGenerateCache(cascadesContext.getConnectContext());
} catch (AnalysisException e) {
LOG.warn(String.format("get mv plan statistics fail, materialization qualifier is %s",
getMaterializationQualifier()), e);
generateMaterializationIdentifier()), e);
return Optional.empty();
}
RelationId relationId = null;
@ -120,7 +119,12 @@ public class AsyncMaterializationContext extends MaterializationContext {
if (!(relation instanceof PhysicalCatalogRelation)) {
return false;
}
return ((PhysicalCatalogRelation) relation).getTable() instanceof MTMV;
if (!(((PhysicalCatalogRelation) relation).getTable() instanceof MTMV)) {
return false;
}
return ((PhysicalCatalogRelation) relation).getTable().getFullQualifiers().equals(
this.generateMaterializationIdentifier()
);
}
public Plan getScanPlan() {

61
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java Normal file
View File

@ -0,0 +1,61 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVUtil;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.PlannerHook;
import com.google.common.annotations.VisibleForTesting;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* If enable query rewrite with mv in dml, should init consistent materialization context after analyze
*/
public class InitConsistentMaterializationContextHook extends InitMaterializationContextHook implements PlannerHook {
public static final InitConsistentMaterializationContextHook INSTANCE =
new InitConsistentMaterializationContextHook();
@VisibleForTesting
@Override
public void initMaterializationContext(CascadesContext cascadesContext) {
if (!cascadesContext.getConnectContext().getSessionVariable().isEnableDmlMaterializedViewRewrite()) {
return;
}
super.doInitMaterializationContext(cascadesContext);
}
protected Set<MTMV> getAvailableMTMVs(Set<TableIf> usedTables, CascadesContext cascadesContext) {
List<BaseTableInfo> usedBaseTables =
usedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList());
return Env.getCurrentEnv().getMtmvService().getRelationManager()
.getAvailableMTMVs(usedBaseTables, cascadesContext.getConnectContext(),
true, ((connectContext, mtmv) -> {
return MTMVUtil.mtmvContainsExternalTable(mtmv) && (!connectContext.getSessionVariable()
.isEnableDmlMaterializedViewRewriteWhenBaseTableUnawareness());
}));
}
}

42
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVCache;
import org.apache.doris.mtmv.MTMVUtil;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.PlannerHook;
@ -35,6 +36,7 @@ import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
import java.util.Set;
@ -53,14 +55,19 @@ public class InitMaterializationContextHook implements PlannerHook {
initMaterializationContext(planner.getCascadesContext());
}
/**
* init materialization context
*/
@VisibleForTesting
public void initMaterializationContext(CascadesContext cascadesContext) {
if (!cascadesContext.getConnectContext().getSessionVariable().isEnableMaterializedViewRewrite()) {
return;
}
doInitMaterializationContext(cascadesContext);
}
/**
* Init materialization context
* @param cascadesContext current cascadesContext in the planner
*/
protected void doInitMaterializationContext(CascadesContext cascadesContext) {
TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), true);
try {
Plan rewritePlan = cascadesContext.getRewritePlan();
@ -77,15 +84,33 @@ public class InitMaterializationContextHook implements PlannerHook {
if (collectedTables.isEmpty()) {
return;
}
// Create async materialization context
for (MaterializationContext context : createAsyncMaterializationContext(cascadesContext,
collectorContext.getCollectedTables())) {
cascadesContext.addMaterializationContext(context);
}
}
protected Set<MTMV> getAvailableMTMVs(Set<TableIf> usedTables, CascadesContext cascadesContext) {
List<BaseTableInfo> usedBaseTables =
collectedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList());
Set<MTMV> availableMTMVs = Env.getCurrentEnv().getMtmvService().getRelationManager()
.getAvailableMTMVs(usedBaseTables, cascadesContext.getConnectContext());
usedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList());
return Env.getCurrentEnv().getMtmvService().getRelationManager()
.getAvailableMTMVs(usedBaseTables, cascadesContext.getConnectContext(),
false, ((connectContext, mtmv) -> {
return MTMVUtil.mtmvContainsExternalTable(mtmv) && (!connectContext.getSessionVariable()
.isEnableMaterializedViewRewriteWhenBaseTableUnawareness());
}));
}
private List<MaterializationContext> createAsyncMaterializationContext(CascadesContext cascadesContext,
Set<TableIf> usedTables) {
Set<MTMV> availableMTMVs = getAvailableMTMVs(usedTables, cascadesContext);
if (availableMTMVs.isEmpty()) {
LOG.debug(String.format("Enable materialized view rewrite but availableMTMVs is empty, current queryId "
+ "is %s", cascadesContext.getConnectContext().getQueryIdentifier()));
return;
return ImmutableList.of();
}
List<MaterializationContext> asyncMaterializationContext = new ArrayList<>();
for (MTMV materializedView : availableMTMVs) {
MTMVCache mtmvCache = null;
try {
@ -100,7 +125,7 @@ public class InitMaterializationContextHook implements PlannerHook {
BitSet tableBitSetInCurrentCascadesContext = new BitSet();
mvStructInfo.getRelations().forEach(relation -> tableBitSetInCurrentCascadesContext.set(
cascadesContext.getStatementContext().getTableId(relation.getTable()).asInt()));
cascadesContext.addMaterializationContext(new AsyncMaterializationContext(materializedView,
asyncMaterializationContext.add(new AsyncMaterializationContext(materializedView,
mtmvCache.getLogicalPlan(), mtmvCache.getOriginalPlan(), ImmutableList.of(),
ImmutableList.of(), cascadesContext,
mtmvCache.getStructInfo().withTableBitSet(tableBitSetInCurrentCascadesContext)));
@ -109,5 +134,6 @@ public class InitMaterializationContextHook implements PlannerHook {
cascadesContext.getConnectContext().getQueryIdentifier()), e);
}
}
return asyncMaterializationContext;
}
}

37
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
View File

@ -18,7 +18,9 @@
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.Id;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext;
@ -40,6 +42,7 @@ import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.Statistics;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Multimap;
import org.apache.logging.log4j.LogManager;
@ -98,6 +101,7 @@ public abstract class MaterializationContext {
// The key is the query belonged group expression objectId, the value is the fail reasons because
// for one materialization query may be multi when nested materialized view.
protected final Multimap<ObjectId, Pair<String, String>> failReason = HashMultimap.create();
protected List<String> identifier;
/**
* MaterializationContext, this contains necessary info for query rewriting by materialization
@ -209,9 +213,22 @@ public abstract class MaterializationContext {
abstract Plan doGenerateScanPlan(CascadesContext cascadesContext);
/**
* Get materialization unique qualifier which identify it
* Get materialization unique identifier which identify it
*/
abstract List<String> getMaterializationQualifier();
abstract List<String> generateMaterializationIdentifier();
/**
* Common method for generating materialization identifier
*/
public static List<String> generateMaterializationIdentifier(OlapTable olapTable, String indexName) {
return indexName == null
? ImmutableList.of(olapTable.getDatabase().getCatalog().getName(),
ClusterNamespace.getNameFromFullName(olapTable.getDatabase().getFullName()),
olapTable.getName())
: ImmutableList.of(olapTable.getDatabase().getCatalog().getName(),
ClusterNamespace.getNameFromFullName(olapTable.getDatabase().getFullName()),
olapTable.getName(), indexName);
}
/**
* Get String info which is used for to string
@ -344,7 +361,7 @@ public abstract class MaterializationContext {
public Void visitPhysicalRelation(PhysicalRelation physicalRelation, Void context) {
for (MaterializationContext rewrittenContext : rewrittenSuccessMaterializationSet) {
if (rewrittenContext.isFinalChosen(physicalRelation)) {
chosenMaterializationQualifiers.add(rewrittenContext.getMaterializationQualifier());
chosenMaterializationQualifiers.add(rewrittenContext.generateMaterializationIdentifier());
}
}
return null;
@ -357,18 +374,18 @@ public abstract class MaterializationContext {
builder.append("\nMaterializedViewRewriteSuccessAndChose:\n");
if (!chosenMaterializationQualifiers.isEmpty()) {
chosenMaterializationQualifiers.forEach(materializationQualifier ->
builder.append(generateQualifierName(materializationQualifier)).append(", \n"));
builder.append(generateIdentifierName(materializationQualifier)).append(", \n"));
}
// rewrite success but not chosen
builder.append("\nMaterializedViewRewriteSuccessButNotChose:\n");
Set<List<String>> rewriteSuccessButNotChoseQualifiers = rewrittenSuccessMaterializationSet.stream()
.map(MaterializationContext::getMaterializationQualifier)
.map(MaterializationContext::generateMaterializationIdentifier)
.filter(materializationQualifier -> !chosenMaterializationQualifiers.contains(materializationQualifier))
.collect(Collectors.toSet());
if (!rewriteSuccessButNotChoseQualifiers.isEmpty()) {
builder.append(" Names: ");
rewriteSuccessButNotChoseQualifiers.forEach(materializationQualifier ->
builder.append(generateQualifierName(materializationQualifier)).append(", "));
builder.append(generateIdentifierName(materializationQualifier)).append(", "));
}
// rewrite fail
builder.append("\nMaterializedViewRewriteFail:");
@ -377,7 +394,7 @@ public abstract class MaterializationContext {
Set<String> failReasonSet =
ctx.getFailReason().values().stream().map(Pair::key).collect(ImmutableSet.toImmutableSet());
builder.append("\n")
.append(" Name: ").append(generateQualifierName(ctx.getMaterializationQualifier()))
.append(" Name: ").append(generateIdentifierName(ctx.generateMaterializationIdentifier()))
.append("\n")
.append(" FailSummary: ").append(String.join(", ", failReasonSet));
}
@ -385,7 +402,7 @@ public abstract class MaterializationContext {
return builder.toString();
}
private static String generateQualifierName(List<String> qualifiers) {
private static String generateIdentifierName(List<String> qualifiers) {
return String.join("#", qualifiers);
}
@ -398,11 +415,11 @@ public abstract class MaterializationContext {
return false;
}
MaterializationContext context = (MaterializationContext) o;
return getMaterializationQualifier().equals(context.getMaterializationQualifier());
return generateMaterializationIdentifier().equals(context.generateMaterializationIdentifier());
}
@Override
public int hashCode() {
return Objects.hash(getMaterializationQualifier());
return Objects.hash(generateMaterializationIdentifier());
}
}

1
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java
View File

@ -50,7 +50,6 @@ public abstract class MaterializedViewScanRule extends AbstractMaterializedViewR
queryStructInfo.getTopPlan(),
materializationContext.getShuttledExprToScanExprMapping(),
targetToSourceMapping,
true,
queryStructInfo.getTableBitSet()
);
// Can not rewrite, bail out

22
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java
View File

@ -19,10 +19,15 @@ package org.apache.doris.nereids.rules.exploration.mv.mapping;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import javax.annotation.Nullable;
@ -41,13 +46,24 @@ public abstract class Mapping {
public final RelationId relationId;
public final CatalogRelation belongedRelation;
// Generate eagerly, will be used to generate slot mapping
private final Map<String, Slot> slotNameToSlotMap = new HashMap<>();
private final Map<List<String>, Slot> slotNameToSlotMap = new HashMap<>();
/**
* Construct relation and slot map
*/
public MappedRelation(RelationId relationId, CatalogRelation belongedRelation) {
this.relationId = relationId;
this.belongedRelation = belongedRelation;
for (Slot slot : belongedRelation.getOutput()) {
slotNameToSlotMap.put(slot.getName(), slot);
if (slot instanceof SlotReference) {
// variant slot
List<String> slotNames = new ArrayList<>();
slotNames.add(slot.getName());
slotNames.addAll(((SlotReference) slot).getSubPath());
slotNameToSlotMap.put(slotNames, slot);
} else {
slotNameToSlotMap.put(ImmutableList.of(slot.getName()), slot);
}
}
}
@ -63,7 +79,7 @@ public abstract class Mapping {
return belongedRelation;
}
public Map<String, Slot> getSlotNameToSlotMap() {
public Map<List<String>, Slot> getSlotNameToSlotMap() {
return slotNameToSlotMap;
}

21
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.exploration.mv.mapping;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.types.VariantType;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
@ -26,6 +27,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
@ -68,20 +70,29 @@ public class SlotMapping extends Mapping {
BiMap<MappedRelation, MappedRelation> mappedRelationMap = relationMapping.getMappedRelationMap();
for (Map.Entry<MappedRelation, MappedRelation> mappedRelationEntry : mappedRelationMap.entrySet()) {
MappedRelation sourceRelation = mappedRelationEntry.getKey();
Map<String, Slot> sourceSlotNameToSlotMap = sourceRelation.getSlotNameToSlotMap();
Map<List<String>, Slot> sourceSlotNameToSlotMap = sourceRelation.getSlotNameToSlotMap();
MappedRelation targetRelation = mappedRelationEntry.getValue();
Map<String, Slot> targetSlotNameSlotMap = targetRelation.getSlotNameToSlotMap();
Map<List<String>, Slot> targetSlotNameSlotMap = targetRelation.getSlotNameToSlotMap();
for (String sourceSlotName : sourceSlotNameToSlotMap.keySet()) {
for (List<String> sourceSlotName : sourceSlotNameToSlotMap.keySet()) {
Slot sourceSlot = sourceSlotNameToSlotMap.get(sourceSlotName);
Slot targetSlot = targetSlotNameSlotMap.get(sourceSlotName);
// source slot can not map from target, bail out
if (targetSlot == null) {
if (targetSlot == null && !(((SlotReference) sourceSlot).getDataType() instanceof VariantType)) {
LOG.warn(String.format("SlotMapping generate is null, source relation is %s, "
+ "target relation is %s", sourceRelation, targetRelation));
return null;
}
Slot sourceSlot = sourceSlotNameToSlotMap.get(sourceSlotName);
if (targetSlot == null) {
// if variant, though can not map slot from query to view, but we maybe derive slot from query
// variant self, such as query slot to view slot mapping is payload#4 -> payload#10
// and query has a variant which is payload['issue']['number']#20, this can not get from view.
// in this scene, we can derive
// payload['issue']['number']#20 -> element_at(element_at(payload#10, 'issue'), 'number') mapping
// in expression rewrite
continue;
}
relationSlotMap.put(MappedSlot.of(sourceSlot,
sourceRelation.getBelongedRelation()),
MappedSlot.of(targetSlot, targetRelation.getBelongedRelation()));

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
View File

@ -83,7 +83,7 @@ public class ExplainCommand extends Command implements NoForward {
executor.setParsedStmt(logicalPlanAdapter);
NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext());
if (ctx.getSessionVariable().isEnableMaterializedViewRewrite()) {
planner.addHook(InitMaterializationContextHook.INSTANCE);
ctx.getStatementContext().addPlannerHook(InitMaterializationContextHook.INSTANCE);
}
planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift());
executor.setPlanner(planner);

4
fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
View File

@ -41,7 +41,6 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.Config;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.UserException;
import org.apache.doris.nereids.PlannerHook;
import org.apache.doris.qe.CommonResultSet;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ResultSet;
@ -657,7 +656,4 @@ public class OriginalPlanner extends Planner {
ResultSet resultSet = new CommonResultSet(metadata, Collections.singletonList(data));
return Optional.of(resultSet);
}
@Override
public void addHook(PlannerHook hook) {}
}

4
fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java
View File

@ -23,7 +23,6 @@ import org.apache.doris.analysis.StatementBase;
import org.apache.doris.common.UserException;
import org.apache.doris.common.profile.PlanTreeBuilder;
import org.apache.doris.common.profile.PlanTreePrinter;
import org.apache.doris.nereids.PlannerHook;
import org.apache.doris.qe.ResultSet;
import org.apache.doris.thrift.TQueryOptions;
@ -129,7 +128,4 @@ public abstract class Planner {
public abstract List<RuntimeFilter> getRuntimeFilters();
public abstract Optional<ResultSet> handleQueryInFe(StatementBase parsedStmt);
public abstract void addHook(PlannerHook hook);
}

42
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -533,10 +533,16 @@ public class SessionVariable implements Serializable, Writable {
public static final String ENABLE_MATERIALIZED_VIEW_REWRITE
= "enable_materialized_view_rewrite";
public static final String ENABLE_DML_MATERIALIZED_VIEW_REWRITE
= "enable_dml_materialized_view_rewrite";
public static final String ENABLE_DML_MATERIALIZED_VIEW_REWRITE_WHEN_BASE_TABLE_UNAWARENESS
= "enable_dml_materialized_view_rewrite_when_base_table_unawareness";
public static final String ALLOW_MODIFY_MATERIALIZED_VIEW_DATA
= "allow_modify_materialized_view_data";
public static final String MATERIALIZED_VIEW_REWRITE_ENABLE_CONTAIN_EXTERNAL_TABLE
public static final String ENABLE_MATERIALIZED_VIEW_REWRITE_WHEN_BASE_TABLE_UNAWARENESS
= "materialized_view_rewrite_enable_contain_external_table";
public static final String MATERIALIZED_VIEW_REWRITE_SUCCESS_CANDIDATE_NUM
@ -1740,16 +1746,27 @@ public class SessionVariable implements Serializable, Writable {
"Is it allowed to modify the data of the materialized view"})
public boolean allowModifyMaterializedViewData = false;
@VariableMgr.VarAttr(name = MATERIALIZED_VIEW_REWRITE_ENABLE_CONTAIN_EXTERNAL_TABLE, needForward = true,
description = {"基于结构信息的透明改写，是否使用包含外表的物化视图",
"Whether to use a materialized view that contains the foreign table "
+ "when using rewriting based on struct info"})
public boolean materializedViewRewriteEnableContainExternalTable = false;
@VariableMgr.VarAttr(name = ENABLE_MATERIALIZED_VIEW_REWRITE_WHEN_BASE_TABLE_UNAWARENESS,
needForward = true,
description = {"查询时，当物化视图存在无法实时感知数据的外表时，是否开启基于结构信息的物化视图透明改写",
""})
public boolean enableMaterializedViewRewriteWhenBaseTableUnawareness = false;
@VariableMgr.VarAttr(name = MATERIALIZED_VIEW_REWRITE_SUCCESS_CANDIDATE_NUM, needForward = true,
description = {"异步物化视图透明改写成功的结果集合，允许参与到CBO候选的最大数量",
"The max candidate num which participate in CBO when using asynchronous materialized views"})
public int materializedViewRewriteSuccessCandidateNum = 3;
@VariableMgr.VarAttr(name = ENABLE_DML_MATERIALIZED_VIEW_REWRITE, needForward = true,
description = {"DML 时, 是否开启基于结构信息的物化视图透明改写",
"Whether to enable materialized view rewriting based on struct info"})
public boolean enableDmlMaterializedViewRewrite = true;
@VariableMgr.VarAttr(name = ENABLE_DML_MATERIALIZED_VIEW_REWRITE_WHEN_BASE_TABLE_UNAWARENESS,
needForward = true,
description = {"DML 时，当物化视图存在无法实时感知数据的外表时，是否开启基于结构信息的物化视图透明改写",
""})
public boolean enableDmlMaterializedViewRewriteWhenBaseTableUnawareness = false;
@VariableMgr.VarAttr(name = MATERIALIZED_VIEW_RELATION_MAPPING_MAX_COUNT, needForward = true,
description = {"透明改写过程中，relation mapping最大允许数量，如果超过，进行截取",
"During transparent rewriting, relation mapping specifies the maximum allowed number. "
@ -3954,12 +3971,21 @@ public class SessionVariable implements Serializable, Writable {
this.enableMaterializedViewRewrite = enableMaterializedViewRewrite;
}
public boolean isEnableDmlMaterializedViewRewrite() {
return enableDmlMaterializedViewRewrite;
}
public boolean isEnableDmlMaterializedViewRewriteWhenBaseTableUnawareness() {
return enableDmlMaterializedViewRewriteWhenBaseTableUnawareness;
}
public boolean isAllowModifyMaterializedViewData() {
return allowModifyMaterializedViewData;
}
public boolean isMaterializedViewRewriteEnableContainExternalTable() {
return materializedViewRewriteEnableContainExternalTable;
public boolean isEnableMaterializedViewRewriteWhenBaseTableUnawareness() {
return enableMaterializedViewRewriteWhenBaseTableUnawareness;
}
public int getMaterializedViewRewriteSuccessCandidateNum() {

2
fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
View File

@ -750,7 +750,7 @@ public class StmtExecutor {
syncJournalIfNeeded();
planner = new NereidsPlanner(statementContext);
if (context.getSessionVariable().isEnableMaterializedViewRewrite()) {
planner.addHook(InitMaterializationContextHook.INSTANCE);
statementContext.addPlannerHook(InitMaterializationContextHook.INSTANCE);
}
try {
planner.plan(parsedStmt, context.getSessionVariable().toThrift());