[improvement](mtmv) Improve the performance for query rewritting by materialized view (#31886)
- Limit the number of times for the query rewritting to the group - Remove the unnecessary log and explain detail info in query
This commit is contained in:
seawinde
@ -884,14 +884,14 @@ public class Column implements Writable, GsonPostProcessable {
|
||||
&& Objects.equals(realDefaultValue, other.realDefaultValue)
|
||||
&& clusterKeyId == other.clusterKeyId;
|
||||
|
||||
if (!ok) {
|
||||
LOG.info("this column: name {} default value {} aggregationType {} isAggregationTypeImplicit {} "
|
||||
if (!ok && LOG.isDebugEnabled()) {
|
||||
LOG.debug("this column: name {} default value {} aggregationType {} isAggregationTypeImplicit {} "
|
||||
+ "isKey {}, isAllowNull {}, datatype {}, strlen {}, precision {}, scale {}, visible {} "
|
||||
+ "children {}, realDefaultValue {}, clusterKeyId {}",
|
||||
name, getDefaultValue(), aggregationType, isAggregationTypeImplicit, isKey, isAllowNull,
|
||||
getDataType(), getStrLen(), getPrecision(), getScale(), visible, children, realDefaultValue,
|
||||
clusterKeyId);
|
||||
LOG.info("other column: name {} default value {} aggregationType {} isAggregationTypeImplicit {} "
|
||||
LOG.debug("other column: name {} default value {} aggregationType {} isAggregationTypeImplicit {} "
|
||||
+ "isKey {}, isAllowNull {}, datatype {}, strlen {}, precision {}, scale {}, visible {}, "
|
||||
+ "children {}, realDefaultValue {}, clusterKeyId {}",
|
||||
other.name, other.getDefaultValue(), other.aggregationType, other.isAggregationTypeImplicit,
|
||||
|
||||
@ -433,7 +433,7 @@ public class NereidsPlanner extends Planner {
|
||||
+ "\n\n========== OPTIMIZED PLAN ==========\n"
|
||||
+ optimizedPlan.treeString()
|
||||
+ "\n\n========== MATERIALIZATIONS ==========\n"
|
||||
+ MaterializationContext.toString(cascadesContext.getMaterializationContexts());
|
||||
+ MaterializationContext.toDetailString(cascadesContext.getMaterializationContexts());
|
||||
break;
|
||||
case ALL_PLAN:
|
||||
plan = "========== PARSED PLAN "
|
||||
|
||||
@ -103,16 +103,16 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
|
||||
// get view and query aggregate and top plan correspondingly
|
||||
Pair<Plan, LogicalAggregate<Plan>> viewTopPlanAndAggPair = splitToTopPlanAndAggregate(viewStructInfo);
|
||||
if (viewTopPlanAndAggPair == null) {
|
||||
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("Split view to top plan and agg fail, view doesn't not contain aggregate",
|
||||
String.format("view plan = %s\n", viewStructInfo.getOriginalPlan().treeString())));
|
||||
materializationContext.recordFailReason(queryStructInfo,
|
||||
"Split view to top plan and agg fail, view doesn't not contain aggregate",
|
||||
() -> String.format("view plan = %s\n", viewStructInfo.getOriginalPlan().treeString()));
|
||||
return null;
|
||||
}
|
||||
Pair<Plan, LogicalAggregate<Plan>> queryTopPlanAndAggPair = splitToTopPlanAndAggregate(queryStructInfo);
|
||||
if (queryTopPlanAndAggPair == null) {
|
||||
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("Split query to top plan and agg fail",
|
||||
String.format("query plan = %s\n", queryStructInfo.getOriginalPlan().treeString())));
|
||||
materializationContext.recordFailReason(queryStructInfo,
|
||||
"Split query to top plan and agg fail",
|
||||
() -> String.format("query plan = %s\n", queryStructInfo.getOriginalPlan().treeString()));
|
||||
return null;
|
||||
}
|
||||
// Firstly,if group by expression between query and view is equals, try to rewrite expression directly
|
||||
@ -131,13 +131,13 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
|
||||
|
||||
}
|
||||
// if fails, record the reason and then try to roll up aggregate function
|
||||
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("Can not rewrite expression when no roll up",
|
||||
String.format("expressionToWrite = %s,\n mvExprToMvScanExprMapping = %s,\n"
|
||||
+ "viewToQuerySlotMapping = %s",
|
||||
queryTopPlan.getOutput(),
|
||||
materializationContext.getMvExprToMvScanExprMapping(),
|
||||
viewToQuerySlotMapping)));
|
||||
materializationContext.recordFailReason(queryStructInfo,
|
||||
"Can not rewrite expression when no roll up",
|
||||
() -> String.format("expressionToWrite = %s,\n mvExprToMvScanExprMapping = %s,\n"
|
||||
+ "viewToQuerySlotMapping = %s",
|
||||
queryTopPlan.getOutput(),
|
||||
materializationContext.getMvExprToMvScanExprMapping(),
|
||||
viewToQuerySlotMapping));
|
||||
}
|
||||
// if view is scalar aggregate but query is not. Or if query is scalar aggregate but view is not
|
||||
// Should not rewrite
|
||||
@ -145,12 +145,12 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
|
||||
List<Expression> viewGroupByExpressions = viewTopPlanAndAggPair.value().getGroupByExpressions();
|
||||
if ((queryGroupByExpressions.isEmpty() && !viewGroupByExpressions.isEmpty())
|
||||
|| (!queryGroupByExpressions.isEmpty() && viewGroupByExpressions.isEmpty())) {
|
||||
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("only one the of query or view is scalar aggregate and "
|
||||
+ "can not rewrite expression meanwhile",
|
||||
String.format("query aggregate = %s,\n view aggregate = %s,\n",
|
||||
queryTopPlanAndAggPair.value().treeString(),
|
||||
viewTopPlanAndAggPair.value().treeString())));
|
||||
materializationContext.recordFailReason(queryStructInfo,
|
||||
"only one the of query or view is scalar aggregate and "
|
||||
+ "can not rewrite expression meanwhile",
|
||||
() -> String.format("query aggregate = %s,\n view aggregate = %s,\n",
|
||||
queryTopPlanAndAggPair.value().treeString(),
|
||||
viewTopPlanAndAggPair.value().treeString()));
|
||||
return null;
|
||||
}
|
||||
// try to roll up.
|
||||
@ -178,10 +178,10 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
|
||||
Expression rollupedExpression = queryFunctionShuttled.accept(AGGREGATE_EXPRESSION_REWRITER,
|
||||
context);
|
||||
if (!context.isValid()) {
|
||||
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("Query function roll up fail",
|
||||
String.format("queryFunctionShuttled = %s,\n mvExprToMvScanExprQueryBased = %s",
|
||||
queryFunctionShuttled, mvExprToMvScanExprQueryBased)));
|
||||
materializationContext.recordFailReason(queryStructInfo,
|
||||
"Query function roll up fail",
|
||||
() -> String.format("queryFunctionShuttled = %s,\n mvExprToMvScanExprQueryBased = %s",
|
||||
queryFunctionShuttled, mvExprToMvScanExprQueryBased));
|
||||
return null;
|
||||
}
|
||||
finalOutputExpressions.add(new Alias(rollupedExpression));
|
||||
@ -196,10 +196,10 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
|
||||
context);
|
||||
if (!context.isValid()) {
|
||||
// group expr can not rewrite by view
|
||||
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("View dimensions doesn't not cover the query dimensions",
|
||||
String.format("mvExprToMvScanExprQueryBased is %s,\n queryGroupShuttledExpr is %s",
|
||||
mvExprToMvScanExprQueryBased, queryGroupShuttledExpr)));
|
||||
materializationContext.recordFailReason(queryStructInfo,
|
||||
"View dimensions doesn't not cover the query dimensions",
|
||||
() -> String.format("mvExprToMvScanExprQueryBased is %s,\n queryGroupShuttledExpr is %s",
|
||||
mvExprToMvScanExprQueryBased, queryGroupShuttledExpr));
|
||||
return null;
|
||||
}
|
||||
NamedExpression groupByExpression = rewrittenGroupByExpression instanceof NamedExpression
|
||||
|
||||
@ -17,7 +17,6 @@
|
||||
|
||||
package org.apache.doris.nereids.rules.exploration.mv;
|
||||
|
||||
import org.apache.doris.common.Pair;
|
||||
import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext;
|
||||
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
|
||||
import org.apache.doris.nereids.trees.expressions.Alias;
|
||||
@ -52,21 +51,14 @@ public abstract class AbstractMaterializedViewJoinRule extends AbstractMateriali
|
||||
);
|
||||
// Can not rewrite, bail out
|
||||
if (expressionsRewritten.isEmpty()) {
|
||||
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("Rewrite expressions by view in join fail",
|
||||
String.format("expressionToRewritten is %s,\n mvExprToMvScanExprMapping is %s,\n"
|
||||
+ "targetToSourceMapping = %s",
|
||||
queryStructInfo.getExpressions(),
|
||||
materializationContext.getMvExprToMvScanExprMapping(),
|
||||
targetToSourceMapping)));
|
||||
materializationContext.recordFailReason(queryStructInfo,
|
||||
"Rewrite expressions by view in join fail",
|
||||
() -> String.format("expressionToRewritten is %s,\n mvExprToMvScanExprMapping is %s,\n"
|
||||
+ "targetToSourceMapping = %s", queryStructInfo.getExpressions(),
|
||||
materializationContext.getMvExprToMvScanExprMapping(),
|
||||
targetToSourceMapping));
|
||||
return null;
|
||||
}
|
||||
// record the group id in materializationContext, and when rewrite again in
|
||||
// the same group, bail out quickly.
|
||||
if (queryStructInfo.getOriginalPlan().getGroupExpression().isPresent()) {
|
||||
materializationContext.addMatchedGroup(
|
||||
queryStructInfo.getOriginalPlan().getGroupExpression().get().getOwnerGroup().getGroupId());
|
||||
}
|
||||
return new LogicalProject<>(
|
||||
expressionsRewritten.stream()
|
||||
.map(expression -> expression instanceof NamedExpression ? expression : new Alias(expression))
|
||||
|
||||
@ -22,13 +22,11 @@ import org.apache.doris.catalog.Partition;
|
||||
import org.apache.doris.catalog.PartitionInfo;
|
||||
import org.apache.doris.catalog.PartitionType;
|
||||
import org.apache.doris.catalog.TableIf;
|
||||
import org.apache.doris.common.Pair;
|
||||
import org.apache.doris.mtmv.BaseTableInfo;
|
||||
import org.apache.doris.mtmv.MTMVPartitionInfo;
|
||||
import org.apache.doris.mtmv.MTMVRewriteUtil;
|
||||
import org.apache.doris.nereids.CascadesContext;
|
||||
import org.apache.doris.nereids.jobs.executor.Rewriter;
|
||||
import org.apache.doris.nereids.memo.GroupExpression;
|
||||
import org.apache.doris.nereids.rules.exploration.ExplorationRuleFactory;
|
||||
import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
|
||||
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
|
||||
@ -45,7 +43,6 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
|
||||
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
|
||||
import org.apache.doris.nereids.trees.expressions.literal.Literal;
|
||||
import org.apache.doris.nereids.trees.plans.JoinType;
|
||||
import org.apache.doris.nereids.trees.plans.ObjectId;
|
||||
import org.apache.doris.nereids.trees.plans.Plan;
|
||||
import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
|
||||
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
|
||||
@ -94,10 +91,13 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
|
||||
// TODO Just support only one query struct info, support multi later.
|
||||
StructInfo queryStructInfo = queryStructInfos.get(0);
|
||||
try {
|
||||
rewrittenPlans.addAll(doRewrite(queryStructInfo, cascadesContext, context));
|
||||
if (rewrittenPlans.size() < cascadesContext.getConnectContext()
|
||||
.getSessionVariable().getMaterializedViewRewriteSuccessCandidateNum()) {
|
||||
rewrittenPlans.addAll(doRewrite(queryStructInfo, cascadesContext, context));
|
||||
}
|
||||
} catch (Exception exception) {
|
||||
context.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("Materialized view rule exec fail", exception.toString()));
|
||||
context.recordFailReason(queryStructInfo,
|
||||
"Materialized view rule exec fail", exception::toString);
|
||||
}
|
||||
}
|
||||
return rewrittenPlans;
|
||||
@ -117,10 +117,9 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
|
||||
StructInfo queryStructInfo = queryStructInfos.get(0);
|
||||
if (!checkPattern(queryStructInfo)) {
|
||||
cascadesContext.getMaterializationContexts().forEach(ctx ->
|
||||
ctx.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("Query struct info is invalid",
|
||||
String.format("queryPlan is %s", queryPlan.treeString())))
|
||||
);
|
||||
ctx.recordFailReason(queryStructInfo, "Query struct info is invalid",
|
||||
() -> String.format("queryPlan is %s", queryPlan.treeString())
|
||||
));
|
||||
return validQueryStructInfos;
|
||||
}
|
||||
validQueryStructInfos.add(queryStructInfo);
|
||||
@ -138,64 +137,70 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
|
||||
materializationContext.getMvPlan(), cascadesContext);
|
||||
if (viewStructInfos.size() > 1) {
|
||||
// view struct info should only have one
|
||||
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("The num of view struct info is more then one",
|
||||
String.format("mv plan is %s", materializationContext.getMvPlan().treeString())));
|
||||
materializationContext.recordFailReason(queryStructInfo,
|
||||
"The num of view struct info is more then one",
|
||||
() -> String.format("mv plan is %s", materializationContext.getMvPlan().treeString()));
|
||||
return rewriteResults;
|
||||
}
|
||||
StructInfo viewStructInfo = viewStructInfos.get(0);
|
||||
if (!checkPattern(viewStructInfo)) {
|
||||
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("View struct info is invalid",
|
||||
String.format(", view plan is %s", viewStructInfo.getOriginalPlan().treeString())));
|
||||
materializationContext.recordFailReason(queryStructInfo,
|
||||
"View struct info is invalid",
|
||||
() -> String.format(", view plan is %s", viewStructInfo.getOriginalPlan().treeString()));
|
||||
return rewriteResults;
|
||||
}
|
||||
MatchMode matchMode = decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations());
|
||||
if (MatchMode.COMPLETE != matchMode) {
|
||||
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("Match mode is invalid", String.format("matchMode is %s", matchMode)));
|
||||
materializationContext.recordFailReason(queryStructInfo, "Match mode is invalid",
|
||||
() -> String.format("matchMode is %s", matchMode));
|
||||
return rewriteResults;
|
||||
}
|
||||
List<RelationMapping> queryToViewTableMappings = RelationMapping.generate(queryStructInfo.getRelations(),
|
||||
viewStructInfo.getRelations());
|
||||
// if any relation in query and view can not map, bail out.
|
||||
if (queryToViewTableMappings == null) {
|
||||
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("Query to view table mapping is null", ""));
|
||||
materializationContext.recordFailReason(queryStructInfo,
|
||||
"Query to view table mapping is null", () -> "");
|
||||
return rewriteResults;
|
||||
}
|
||||
for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) {
|
||||
SlotMapping queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping);
|
||||
if (queryToViewSlotMapping == null) {
|
||||
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("Query to view slot mapping is null", ""));
|
||||
materializationContext.recordFailReason(queryStructInfo,
|
||||
"Query to view slot mapping is null", () -> "");
|
||||
continue;
|
||||
}
|
||||
SlotMapping viewToQuerySlotMapping = queryToViewSlotMapping.inverse();
|
||||
// check the column used in query is in mv or not
|
||||
if (!checkColumnUsedValid(queryStructInfo, viewStructInfo, queryToViewSlotMapping)) {
|
||||
materializationContext.recordFailReason(queryStructInfo,
|
||||
"The columns used by query are not in view",
|
||||
() -> String.format("query struct info is %s, view struct info is %s",
|
||||
queryStructInfo.getTopPlan().treeString(),
|
||||
viewStructInfo.getTopPlan().treeString()));
|
||||
continue;
|
||||
}
|
||||
LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from(
|
||||
queryToViewTableMapping, queryToViewSlotMapping, queryStructInfo, viewStructInfo);
|
||||
ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo,
|
||||
compatibilityContext);
|
||||
if (comparisonResult.isInvalid()) {
|
||||
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("The graph logic between query and view is not consistent",
|
||||
comparisonResult.getErrorMessage()));
|
||||
materializationContext.recordFailReason(queryStructInfo,
|
||||
"The graph logic between query and view is not consistent",
|
||||
comparisonResult::getErrorMessage);
|
||||
continue;
|
||||
}
|
||||
SplitPredicate compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo,
|
||||
viewToQuerySlotMapping, comparisonResult, cascadesContext);
|
||||
// Can not compensate, bail out
|
||||
if (compensatePredicates.isInvalid()) {
|
||||
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("Predicate compensate fail",
|
||||
String.format("query predicates = %s,\n query equivalenceClass = %s, \n"
|
||||
+ "view predicates = %s,\n query equivalenceClass = %s\n"
|
||||
+ "comparisonResult = %s ",
|
||||
queryStructInfo.getPredicates(),
|
||||
queryStructInfo.getEquivalenceClass(),
|
||||
viewStructInfo.getPredicates(),
|
||||
viewStructInfo.getEquivalenceClass(),
|
||||
comparisonResult)));
|
||||
materializationContext.recordFailReason(queryStructInfo,
|
||||
"Predicate compensate fail",
|
||||
() -> String.format("query predicates = %s,\n query equivalenceClass = %s, \n"
|
||||
+ "view predicates = %s,\n query equivalenceClass = %s\n"
|
||||
+ "comparisonResult = %s ", queryStructInfo.getPredicates(),
|
||||
queryStructInfo.getEquivalenceClass(), viewStructInfo.getPredicates(),
|
||||
viewStructInfo.getEquivalenceClass(), comparisonResult));
|
||||
continue;
|
||||
}
|
||||
Plan rewrittenPlan;
|
||||
@ -209,13 +214,12 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
|
||||
originalPlan, materializationContext.getMvExprToMvScanExprMapping(),
|
||||
viewToQuerySlotMapping, true);
|
||||
if (rewriteCompensatePredicates.isEmpty()) {
|
||||
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("Rewrite compensate predicate by view fail", String.format(
|
||||
"compensatePredicates = %s,\n mvExprToMvScanExprMapping = %s,\n"
|
||||
materializationContext.recordFailReason(queryStructInfo,
|
||||
"Rewrite compensate predicate by view fail",
|
||||
() -> String.format("compensatePredicates = %s,\n mvExprToMvScanExprMapping = %s,\n"
|
||||
+ "viewToQuerySlotMapping = %s",
|
||||
compensatePredicates,
|
||||
materializationContext.getMvExprToMvScanExprMapping(),
|
||||
viewToQuerySlotMapping)));
|
||||
compensatePredicates, materializationContext.getMvExprToMvScanExprMapping(),
|
||||
viewToQuerySlotMapping));
|
||||
continue;
|
||||
}
|
||||
rewrittenPlan = new LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan);
|
||||
@ -226,37 +230,55 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
|
||||
if (rewrittenPlan == null) {
|
||||
continue;
|
||||
}
|
||||
rewrittenPlan = rewriteByRules(cascadesContext, rewrittenPlan, originalPlan);
|
||||
if (!isOutputValid(originalPlan, rewrittenPlan)) {
|
||||
ObjectId planObjId = originalPlan.getGroupExpression().map(GroupExpression::getId)
|
||||
.orElseGet(() -> new ObjectId(-1));
|
||||
materializationContext.recordFailReason(planObjId, Pair.of(
|
||||
final Plan finalRewrittenPlan = rewriteByRules(cascadesContext, rewrittenPlan, originalPlan);
|
||||
if (!isOutputValid(originalPlan, finalRewrittenPlan)) {
|
||||
materializationContext.recordFailReason(queryStructInfo,
|
||||
"RewrittenPlan output logical properties is different with target group",
|
||||
String.format("planOutput logical properties = %s,\n"
|
||||
+ "groupOutput logical properties = %s", rewrittenPlan.getLogicalProperties(),
|
||||
originalPlan.getLogicalProperties())));
|
||||
() -> String.format("planOutput logical"
|
||||
+ " properties = %s,\n groupOutput logical properties = %s",
|
||||
finalRewrittenPlan.getLogicalProperties(), originalPlan.getLogicalProperties()));
|
||||
continue;
|
||||
}
|
||||
// check the partitions used by rewritten plan is valid or not
|
||||
Set<Long> invalidPartitionsQueryUsed =
|
||||
calcInvalidPartitions(rewrittenPlan, materializationContext, cascadesContext);
|
||||
calcInvalidPartitions(finalRewrittenPlan, materializationContext, cascadesContext);
|
||||
if (!invalidPartitionsQueryUsed.isEmpty()) {
|
||||
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
|
||||
Pair.of("Check partition query used validation fail",
|
||||
String.format("the partition used by query is invalid by materialized view,"
|
||||
+ "invalid partition info query used is %s",
|
||||
materializationContext.getMTMV().getPartitions().stream()
|
||||
.filter(partition ->
|
||||
invalidPartitionsQueryUsed.contains(partition.getId()))
|
||||
.collect(Collectors.toSet()))));
|
||||
materializationContext.recordFailReason(queryStructInfo,
|
||||
"Check partition query used validation fail",
|
||||
() -> String.format("the partition used by query is invalid by materialized view,"
|
||||
+ "invalid partition info query used is %s",
|
||||
materializationContext.getMTMV().getPartitions().stream()
|
||||
.filter(partition ->
|
||||
invalidPartitionsQueryUsed.contains(partition.getId()))
|
||||
.collect(Collectors.toSet())));
|
||||
continue;
|
||||
}
|
||||
recordIfRewritten(originalPlan, materializationContext);
|
||||
rewriteResults.add(rewrittenPlan);
|
||||
rewriteResults.add(finalRewrittenPlan);
|
||||
}
|
||||
return rewriteResults;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the column used by query is in materialized view output or not
|
||||
*/
|
||||
protected boolean checkColumnUsedValid(StructInfo queryInfo, StructInfo mvInfo,
|
||||
SlotMapping queryToViewSlotMapping) {
|
||||
Set<ExprId> queryUsedSlotSetViewBased = ExpressionUtils.shuttleExpressionWithLineage(
|
||||
queryInfo.getTopPlan().getOutput(), queryInfo.getTopPlan()).stream()
|
||||
.flatMap(expr -> ExpressionUtils.replace(expr, queryToViewSlotMapping.toSlotReferenceMap())
|
||||
.collectToSet(each -> each instanceof Slot).stream())
|
||||
.map(each -> ((Slot) each).getExprId())
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
Set<ExprId> viewUsedSlotSet = ExpressionUtils.shuttleExpressionWithLineage(mvInfo.getTopPlan().getOutput(),
|
||||
mvInfo.getTopPlan()).stream()
|
||||
.flatMap(expr -> expr.collectToSet(each -> each instanceof Slot).stream())
|
||||
.map(each -> ((Slot) each).getExprId())
|
||||
.collect(Collectors.toSet());
|
||||
return viewUsedSlotSet.containsAll(queryUsedSlotSetViewBased);
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrite by rules and try to make output is the same after optimize by rules
|
||||
*/
|
||||
@ -324,7 +346,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
|
||||
}
|
||||
// get mv valid partitions
|
||||
Set<Long> mvDataValidPartitionIdSet = MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv,
|
||||
cascadesContext.getConnectContext(), System.currentTimeMillis()).stream()
|
||||
cascadesContext.getConnectContext(), System.currentTimeMillis()).stream()
|
||||
.map(Partition::getId)
|
||||
.collect(Collectors.toSet());
|
||||
Set<Long> queryUsedPartitionIdSet = rewrittenPlan.collectToList(node -> node instanceof LogicalOlapScan
|
||||
@ -521,8 +543,8 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
|
||||
.collect(Collectors.toSet());
|
||||
// query pulledUp predicates should have null reject predicates and contains any require noNullable slot
|
||||
return !queryPulledUpPredicates.containsAll(nullRejectPredicates)
|
||||
&& requireNoNullableViewSlot.stream().noneMatch(
|
||||
set -> Sets.intersection(set, queryUsedNeedRejectNullSlotsViewBased).isEmpty());
|
||||
&& requireNoNullableViewSlot.stream().noneMatch(set ->
|
||||
Sets.intersection(set, queryUsedNeedRejectNullSlotsViewBased).isEmpty());
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -95,7 +95,7 @@ public class InitMaterializationContextHook implements PlannerHook {
|
||||
// todo should force keep consistency to mv sql plan output
|
||||
Plan projectScan = new LogicalProject<Plan>(mvProjects, mvScan);
|
||||
cascadesContext.addMaterializationContext(
|
||||
MaterializationContext.fromMaterializedView(materializedView, projectScan));
|
||||
MaterializationContext.fromMaterializedView(materializedView, projectScan, cascadesContext));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@ -17,15 +17,18 @@
|
||||
|
||||
package org.apache.doris.nereids.rules.exploration.mv;
|
||||
|
||||
import org.apache.doris.analysis.StatementBase;
|
||||
import org.apache.doris.catalog.MTMV;
|
||||
import org.apache.doris.catalog.Table;
|
||||
import org.apache.doris.common.AnalysisException;
|
||||
import org.apache.doris.common.Pair;
|
||||
import org.apache.doris.mtmv.MTMVCache;
|
||||
import org.apache.doris.nereids.CascadesContext;
|
||||
import org.apache.doris.nereids.memo.GroupId;
|
||||
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
|
||||
import org.apache.doris.nereids.trees.plans.ObjectId;
|
||||
import org.apache.doris.nereids.trees.plans.Plan;
|
||||
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
|
||||
import org.apache.doris.nereids.util.ExpressionUtils;
|
||||
import org.apache.doris.nereids.util.Utils;
|
||||
|
||||
@ -38,6 +41,7 @@ import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
@ -64,16 +68,20 @@ public class MaterializationContext {
|
||||
// if rewrite by mv fail, record the reason, if success the failReason should be empty.
|
||||
// The key is the query belonged group expression objectId, the value is the fail reason
|
||||
private final Map<ObjectId, Pair<String, String>> failReason = new HashMap<>();
|
||||
private boolean enableRecordFailureDetail = false;
|
||||
|
||||
/**
|
||||
* MaterializationContext, this contains necessary info for query rewriting by mv
|
||||
*/
|
||||
public MaterializationContext(MTMV mtmv, Plan mvScanPlan, List<Table> baseTables, List<Table> baseViews) {
|
||||
public MaterializationContext(MTMV mtmv, Plan mvScanPlan, List<Table> baseTables, List<Table> baseViews,
|
||||
CascadesContext cascadesContext) {
|
||||
this.mtmv = mtmv;
|
||||
this.mvScanPlan = mvScanPlan;
|
||||
this.baseTables = baseTables;
|
||||
this.baseViews = baseViews;
|
||||
|
||||
StatementBase parsedStatement = cascadesContext.getStatementContext().getParsedStatement();
|
||||
this.enableRecordFailureDetail = parsedStatement != null && parsedStatement.isExplain()
|
||||
&& ExplainLevel.MEMO_PLAN == parsedStatement.getExplainOptions().getExplainLevel();
|
||||
MTMVCache mtmvCache = null;
|
||||
try {
|
||||
mtmvCache = mtmv.getOrGenerateCache();
|
||||
@ -94,10 +102,6 @@ public class MaterializationContext {
|
||||
this.mvPlan = mtmvCache.getLogicalPlan();
|
||||
}
|
||||
|
||||
public Set<GroupId> getMatchedGroups() {
|
||||
return matchedGroups;
|
||||
}
|
||||
|
||||
public boolean alreadyRewrite(GroupId groupId) {
|
||||
return this.matchedGroups.contains(groupId);
|
||||
}
|
||||
@ -138,6 +142,10 @@ public class MaterializationContext {
|
||||
return failReason;
|
||||
}
|
||||
|
||||
public boolean isEnableRecordFailureDetail() {
|
||||
return enableRecordFailureDetail;
|
||||
}
|
||||
|
||||
public void setSuccess(boolean success) {
|
||||
this.success = success;
|
||||
this.failReason.clear();
|
||||
@ -146,13 +154,18 @@ public class MaterializationContext {
|
||||
/**
|
||||
* recordFailReason
|
||||
*/
|
||||
public void recordFailReason(ObjectId objectId, Pair<String, String> summaryAndReason) {
|
||||
public void recordFailReason(StructInfo structInfo, String summary, Supplier<String> failureReasonSupplier) {
|
||||
// record it's rewritten
|
||||
if (structInfo.getTopPlan().getGroupExpression().isPresent()) {
|
||||
this.addMatchedGroup(structInfo.getTopPlan().getGroupExpression().get().getOwnerGroup().getGroupId());
|
||||
}
|
||||
// once success, do not record the fail reason
|
||||
if (this.success) {
|
||||
return;
|
||||
}
|
||||
this.success = false;
|
||||
this.failReason.put(objectId, summaryAndReason);
|
||||
this.failReason.put(structInfo.getOriginalPlanId(),
|
||||
Pair.of(summary, this.isEnableRecordFailureDetail() ? failureReasonSupplier.get() : ""));
|
||||
}
|
||||
|
||||
public boolean isSuccess() {
|
||||
@ -178,7 +191,7 @@ public class MaterializationContext {
|
||||
/**
|
||||
* toString, this contains summary and detail info.
|
||||
*/
|
||||
public static String toString(List<MaterializationContext> materializationContexts) {
|
||||
public static String toDetailString(List<MaterializationContext> materializationContexts) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("materializationContexts:").append("\n");
|
||||
for (MaterializationContext ctx : materializationContexts) {
|
||||
@ -200,6 +213,22 @@ public class MaterializationContext {
|
||||
.collect(Collectors.toSet());
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("\nMaterializedView");
|
||||
// rewrite success and chosen
|
||||
builder.append("\nMaterializedViewRewriteSuccessAndChose:\n");
|
||||
if (!materializationChosenNameSet.isEmpty()) {
|
||||
builder.append(" Names: ").append(String.join(", ", materializationChosenNameSet));
|
||||
}
|
||||
// rewrite success but not chosen
|
||||
builder.append("\nMaterializedViewRewriteSuccessButNotChose:\n");
|
||||
Set<String> rewriteSuccessButNotChoseNameSet = materializationContexts.stream()
|
||||
.filter(materializationContext -> materializationContext.isSuccess()
|
||||
&& !materializationChosenNameSet.contains(materializationContext.getMTMV().getName()))
|
||||
.map(materializationContext -> materializationContext.getMTMV().getName())
|
||||
.collect(Collectors.toSet());
|
||||
if (!rewriteSuccessButNotChoseNameSet.isEmpty()) {
|
||||
builder.append(" Names: ").append(String.join(", ", rewriteSuccessButNotChoseNameSet));
|
||||
}
|
||||
// rewrite fail
|
||||
builder.append("\nMaterializedViewRewriteFail:");
|
||||
for (MaterializationContext ctx : materializationContexts) {
|
||||
if (!ctx.isSuccess()) {
|
||||
@ -211,25 +240,15 @@ public class MaterializationContext {
|
||||
.append(" FailSummary: ").append(String.join(", ", failReasonSet));
|
||||
}
|
||||
}
|
||||
builder.append("\nMaterializedViewRewriteSuccessButNotChose:\n");
|
||||
builder.append(" Names: ").append(materializationContexts.stream()
|
||||
.filter(materializationContext -> materializationContext.isSuccess()
|
||||
&& !materializationChosenNameSet.contains(materializationContext.getMTMV().getName()))
|
||||
.map(materializationContext -> materializationContext.getMTMV().getName())
|
||||
.collect(Collectors.joining(", ")));
|
||||
builder.append("\nMaterializedViewRewriteSuccessAndChose:\n");
|
||||
builder.append(" Names: ").append(String.join(", ", materializationChosenNameSet));
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* MaterializationContext fromMaterializedView
|
||||
*/
|
||||
public static MaterializationContext fromMaterializedView(MTMV materializedView, Plan mvScanPlan) {
|
||||
return new MaterializationContext(
|
||||
materializedView,
|
||||
mvScanPlan,
|
||||
ImmutableList.of(),
|
||||
ImmutableList.of());
|
||||
public static MaterializationContext fromMaterializedView(MTMV materializedView, Plan mvScanPlan,
|
||||
CascadesContext cascadesContext) {
|
||||
return new MaterializationContext(materializedView, mvScanPlan, ImmutableList.of(), ImmutableList.of(),
|
||||
cascadesContext);
|
||||
}
|
||||
}
|
||||
|
||||
@ -498,6 +498,9 @@ public class SessionVariable implements Serializable, Writable {
|
||||
public static final String MATERIALIZED_VIEW_REWRITE_ENABLE_CONTAIN_EXTERNAL_TABLE
|
||||
= "materialized_view_rewrite_enable_contain_external_table";
|
||||
|
||||
public static final String MATERIALIZED_VIEW_REWRITE_SUCCESS_CANDIDATE_NUM
|
||||
= "materialized_view_rewrite_success_candidate_num";
|
||||
|
||||
public static final String CREATE_TABLE_PARTITION_MAX_NUM
|
||||
= "create_table_partition_max_num";
|
||||
|
||||
@ -1582,9 +1585,13 @@ public class SessionVariable implements Serializable, Writable {
|
||||
|
||||
@VariableMgr.VarAttr(name = MATERIALIZED_VIEW_REWRITE_ENABLE_CONTAIN_EXTERNAL_TABLE, needForward = true,
|
||||
description = {"基于结构信息的透明改写，是否使用包含外表的物化视图",
|
||||
"whether to use a materialized view that contains the foreign table "
|
||||
"Whether to use a materialized view that contains the foreign table "
|
||||
+ "when using rewriting based on struct info"})
|
||||
public boolean materializedViewRewriteEnableContainExternalTable = false;
|
||||
@VariableMgr.VarAttr(name = MATERIALIZED_VIEW_REWRITE_SUCCESS_CANDIDATE_NUM, needForward = true,
|
||||
description = {"异步物化视图透明改写成功的结果集合，允许参与到CBO候选的最大数量",
|
||||
"The max candidate num which participate in CBO when using asynchronous materialized views"})
|
||||
public int materializedViewRewriteSuccessCandidateNum = 3;
|
||||
|
||||
@VariableMgr.VarAttr(name = CREATE_TABLE_PARTITION_MAX_NUM, needForward = true,
|
||||
description = {"建表时创建分区的最大数量",
|
||||
@ -3432,6 +3439,10 @@ public class SessionVariable implements Serializable, Writable {
|
||||
return materializedViewRewriteEnableContainExternalTable;
|
||||
}
|
||||
|
||||
public int getMaterializedViewRewriteSuccessCandidateNum() {
|
||||
return materializedViewRewriteSuccessCandidateNum;
|
||||
}
|
||||
|
||||
public int getCreateTablePartitionMaxNum() {
|
||||
return createTablePartitionMaxNum;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user