database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](nereids) Fix data wrong using mv rewrite and ignore case when getting mv related partition table (#28699)

Browse Source 
1. Fix data wrong using mv rewrite
2. Ignore case when getting mv related partition table
3. Enable infer expression column name without alias when create mv
This commit is contained in:
seawinde
2023-12-20 17:59:46 +08:00
committed by GitHub
parent 2b2d3d0eb1
commit 9a5ec43f05
21 changed files with 310 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
View File

@ -35,7 +35,6 @@ import org.apache.doris.mtmv.MTMVRefreshInfo;
import org.apache.doris.mtmv.MTMVRelation;
import org.apache.doris.mtmv.MTMVStatus;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
@ -199,13 +198,12 @@ public class MTMV extends OlapTable {
return Sets.newHashSet(split);
}
// this should use the same connectContext with query, to use the same session variable
public MTMVCache getOrGenerateCache(ConnectContext parent) throws AnalysisException {
public MTMVCache getOrGenerateCache() throws AnalysisException {
if (cache == null) {
writeMvLock();
try {
if (cache == null) {
this.cache = MTMVCache.from(this, parent);
this.cache = MTMVCache.from(this, MTMVPlanUtil.createMTMVContext(this));
}
} finally {
writeMvUnlock();

3
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
View File

@ -63,7 +63,8 @@ public class MTMVCache {
public static MTMVCache from(MTMV mtmv, ConnectContext connectContext) {
LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(mtmv.getQuerySql());
// TODO: connect context set current db when create mv by use database
// this will be removed in the future when support join derivation
connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES, ELIMINATE_OUTER_JOIN");
StatementContext mvSqlStatementContext = new StatementContext(connectContext,
new OriginStatement(mtmv.getQuerySql(), 0));
NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext);

16
fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java
View File

@ -93,6 +93,22 @@ public class PatternDescriptor<INPUT_TYPE extends Plan> {
return new PatternMatcher<>(pattern, defaultPromise, matchedAction);
}
/**
* Apply rule to return multi result, catch exception to make sure no influence on other rule
*/
public <OUTPUT_TYPE extends Plan> PatternMatcher<INPUT_TYPE, OUTPUT_TYPE> thenApplyMultiNoThrow(
MatchedMultiAction<INPUT_TYPE, OUTPUT_TYPE> matchedMultiAction) {
MatchedMultiAction<INPUT_TYPE, OUTPUT_TYPE> adaptMatchedMultiAction = ctx -> {
try {
return matchedMultiAction.apply(ctx);
} catch (Exception ex) {
LOG.warn("nereids apply rule failed, because {}", ex.getMessage(), ex);
return null;
}
};
return new PatternMatcher<>(pattern, defaultPromise, adaptMatchedMultiAction);
}
public Pattern<INPUT_TYPE> getPattern() {
return pattern;
}

67
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View File

@ -84,7 +84,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
List<MaterializationContext> materializationContexts = cascadesContext.getMaterializationContexts();
List<Plan> rewriteResults = new ArrayList<>();
if (materializationContexts.isEmpty()) {
logger.info(currentClassName + " materializationContexts is empty so return");
logger.debug(currentClassName + " materializationContexts is empty so return");
return rewriteResults;
}
@ -92,7 +92,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
// TODO Just Check query queryPlan firstly, support multi later.
StructInfo queryStructInfo = queryStructInfos.get(0);
if (!checkPattern(queryStructInfo)) {
logger.info(currentClassName + " queryStructInfo is not valid so return");
logger.debug(currentClassName + " queryStructInfo is not valid so return");
return rewriteResults;
}
@ -101,42 +101,42 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
if (queryPlan.getGroupExpression().isPresent()
&& materializationContext.alreadyRewrite(
queryPlan.getGroupExpression().get().getOwnerGroup().getGroupId())) {
logger.info(currentClassName + " this group is already rewritten so skip");
logger.debug(currentClassName + " this group is already rewritten so skip");
continue;
}
MTMV mtmv = materializationContext.getMTMV();
MTMVCache mtmvCache = getCacheFromMTMV(mtmv, cascadesContext);
MTMVCache mtmvCache = getCacheFromMTMV(mtmv);
if (mtmvCache == null) {
logger.info(currentClassName + " mv cache is null so return");
logger.warn(currentClassName + " mv cache is null so return");
return rewriteResults;
}
List<StructInfo> viewStructInfos = extractStructInfo(mtmvCache.getLogicalPlan(), cascadesContext);
if (viewStructInfos.size() > 1) {
// view struct info should only have one
logger.info(currentClassName + " the num of view struct info is more then one so return");
logger.warn(currentClassName + " the num of view struct info is more then one so return");
return rewriteResults;
}
StructInfo viewStructInfo = viewStructInfos.get(0);
if (!checkPattern(viewStructInfo)) {
logger.info(currentClassName + " viewStructInfo is not valid so return");
logger.debug(currentClassName + " viewStructInfo is not valid so return");
continue;
}
MatchMode matchMode = decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations());
if (MatchMode.COMPLETE != matchMode) {
logger.info(currentClassName + " match mode is not complete so return");
logger.debug(currentClassName + " match mode is not complete so return");
continue;
}
List<RelationMapping> queryToViewTableMappings =
RelationMapping.generate(queryStructInfo.getRelations(), viewStructInfo.getRelations());
// if any relation in query and view can not map, bail out.
if (queryToViewTableMappings == null) {
logger.info(currentClassName + " query to view table mapping null so return");
logger.warn(currentClassName + " query to view table mapping null so return");
return rewriteResults;
}
for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) {
SlotMapping queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping);
if (queryToViewSlotMapping == null) {
logger.info(currentClassName + " query to view slot mapping null so continue");
logger.warn(currentClassName + " query to view slot mapping null so continue");
continue;
}
LogicalCompatibilityContext compatibilityContext =
@ -145,7 +145,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
List<Expression> pulledUpExpressions = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo,
compatibilityContext);
if (pulledUpExpressions == null) {
logger.info(currentClassName + " graph logical is not equals so continue");
logger.debug(currentClassName + " graph logical is not equals so continue");
continue;
}
// set pulled up expression to queryStructInfo predicates and update related predicates
@ -156,13 +156,13 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
queryToViewSlotMapping);
// Can not compensate, bail out
if (compensatePredicates.isEmpty()) {
logger.info(currentClassName + " predicate compensate fail so continue");
logger.debug(currentClassName + " predicate compensate fail so continue");
continue;
}
Plan rewritedPlan;
Plan rewrittenPlan;
Plan mvScan = materializationContext.getMvScanPlan();
if (compensatePredicates.isAlwaysTrue()) {
rewritedPlan = mvScan;
rewrittenPlan = mvScan;
} else {
// Try to rewrite compensate predicates by using mv scan
List<Expression> rewriteCompensatePredicates = rewriteExpression(
@ -172,39 +172,52 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
queryToViewSlotMapping,
true);
if (rewriteCompensatePredicates.isEmpty()) {
logger.info(currentClassName + " compensate predicate rewrite by view fail so continue");
logger.debug(currentClassName + " compensate predicate rewrite by view fail so continue");
continue;
}
rewritedPlan = new LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan);
rewrittenPlan = new LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan);
}
// Rewrite query by view
rewritedPlan = rewriteQueryByView(matchMode,
rewrittenPlan = rewriteQueryByView(matchMode,
queryStructInfo,
viewStructInfo,
queryToViewSlotMapping,
rewritedPlan,
rewrittenPlan,
materializationContext);
if (rewritedPlan == null) {
logger.info(currentClassName + " rewrite query by view fail so continue");
if (rewrittenPlan == null) {
logger.debug(currentClassName + " rewrite query by view fail so continue");
continue;
}
if (!checkPartitionIsValid(queryStructInfo, materializationContext, cascadesContext)) {
logger.info(currentClassName + " check partition validation fail so continue");
logger.debug(currentClassName + " check partition validation fail so continue");
continue;
}
if (!checkOutput(queryPlan, rewrittenPlan)) {
logger.debug(currentClassName + " check output validation fail so continue");
continue;
}
// run rbo job on mv rewritten plan
CascadesContext rewrittenPlanContext =
CascadesContext.initContext(cascadesContext.getStatementContext(), rewritedPlan,
CascadesContext.initContext(cascadesContext.getStatementContext(), rewrittenPlan,
cascadesContext.getCurrentJobContext().getRequiredProperties());
Rewriter.getWholeTreeRewriter(cascadesContext).execute();
rewritedPlan = rewrittenPlanContext.getRewritePlan();
logger.info(currentClassName + "rewrite by materialized view success");
rewriteResults.add(rewritedPlan);
rewrittenPlan = rewrittenPlanContext.getRewritePlan();
logger.debug(currentClassName + "rewrite by materialized view success");
rewriteResults.add(rewrittenPlan);
}
}
return rewriteResults;
}
protected boolean checkOutput(Plan sourcePlan, Plan rewrittenPlan) {
if (sourcePlan.getGroupExpression().isPresent() && !rewrittenPlan.getLogicalProperties().equals(
sourcePlan.getGroupExpression().get().getOwnerGroup().getLogicalProperties())) {
logger.error("rewrittenPlan output logical properties is not same with target group");
return false;
}
return true;
}
/**
* Partition will be pruned in query then add the record the partitions to select partitions on
* catalog relation.
@ -276,10 +289,10 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
&& relatedTalbeValidSet.containsAll(relatedTableSelectedPartitionToCheck);
}
private MTMVCache getCacheFromMTMV(MTMV mtmv, CascadesContext cascadesContext) {
private MTMVCache getCacheFromMTMV(MTMV mtmv) {
MTMVCache cache;
try {
cache = mtmv.getOrGenerateCache(cascadesContext.getConnectContext());
cache = mtmv.getOrGenerateCache();
} catch (AnalysisException analysisException) {
logger.warn(this.getClass().getSimpleName() + " get mtmv cache analysisException", analysisException);
return null;

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
View File

@ -67,7 +67,7 @@ public class MaterializationContext {
MTMVCache mtmvCache = null;
try {
mtmvCache = mtmv.getOrGenerateCache(cascadesContext.getConnectContext());
mtmvCache = mtmv.getOrGenerateCache();
} catch (AnalysisException e) {
LOG.warn("MaterializationContext init mv cache generate fail", e);
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java
View File

@ -36,7 +36,7 @@ public class MaterializedViewAggregateRule extends AbstractMaterializedViewAggre
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
logicalAggregate(any()).thenApplyMulti(ctx -> {
logicalAggregate(any()).thenApplyMultiNoThrow(ctx -> {
LogicalAggregate<Plan> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_ONLY_AGGREGATE));

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java
View File

@ -35,7 +35,7 @@ public class MaterializedViewProjectAggregateRule extends AbstractMaterializedVi
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
logicalProject(logicalAggregate(any())).thenApplyMulti(ctx -> {
logicalProject(logicalAggregate(any())).thenApplyMultiNoThrow(ctx -> {
LogicalProject<LogicalAggregate<Plan>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_AGGREGATE));

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java
View File

@ -37,7 +37,7 @@ public class MaterializedViewProjectJoinRule extends AbstractMaterializedViewJoi
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
logicalProject(logicalJoin(any(), any())).thenApplyMulti(ctx -> {
logicalProject(logicalJoin(any(), any())).thenApplyMultiNoThrow(ctx -> {
LogicalProject<LogicalJoin<Plan, Plan>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_JOIN));

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
View File

@ -63,7 +63,7 @@ public class MaterializedViewUtils {
Slot columnExpr = null;
// get column slot
for (Slot outputSlot : outputExpressions) {
if (outputSlot.getName().equals(column)) {
if (outputSlot.getName().equalsIgnoreCase(column)) {
columnExpr = outputSlot;
break;
}

7
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
View File

@ -23,9 +23,10 @@ import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
/**
@ -34,7 +35,7 @@ import java.util.stream.Collectors;
public class Predicates {
// Predicates that can be pulled up
private final List<Expression> pulledUpPredicates = new ArrayList<>();
private final Set<Expression> pulledUpPredicates = new HashSet<>();
private Predicates() {
}
@ -49,7 +50,7 @@ public class Predicates {
return predicates;
}
public List<? extends Expression> getPulledUpPredicates() {
public Set<? extends Expression> getPulledUpPredicates() {
return pulledUpPredicates;
}

33
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java
View File

@ -20,17 +20,16 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
import org.apache.doris.nereids.trees.expressions.EqualPredicate;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
import org.apache.doris.nereids.util.ExpressionUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/**
* Split the expression to equal, range and residual predicate.
@ -39,27 +38,26 @@ import java.util.List;
*/
public class PredicatesSplitter {
private final List<Expression> equalPredicates = new ArrayList<>();
private final List<Expression> rangePredicates = new ArrayList<>();
private final List<Expression> residualPredicates = new ArrayList<>();
private final Set<Expression> equalPredicates = new HashSet<>();
private final Set<Expression> rangePredicates = new HashSet<>();
private final Set<Expression> residualPredicates = new HashSet<>();
private final List<Expression> conjunctExpressions;
private final PredicateExtract instance = new PredicateExtract();
public PredicatesSplitter(Expression target) {
this.conjunctExpressions = ExpressionUtils.extractConjunction(target);
PredicateExtract instance = new PredicateExtract();
for (Expression expression : conjunctExpressions) {
expression.accept(instance, expression);
expression.accept(instance, null);
}
}
/**
* PredicateExtract
* extract to equal, range, residual predicate set
*/
public class PredicateExtract extends DefaultExpressionVisitor<Void, Expression> {
public class PredicateExtract extends DefaultExpressionVisitor<Void, Void> {
@Override
public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Expression sourceExpression) {
public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Void context) {
Expression leftArg = comparisonPredicate.getArgument(0);
Expression rightArg = comparisonPredicate.getArgument(1);
boolean leftArgOnlyContainsColumnRef = containOnlyColumnRef(leftArg, true);
@ -81,12 +79,9 @@ public class PredicatesSplitter {
}
@Override
public Void visitCompoundPredicate(CompoundPredicate compoundPredicate, Expression context) {
if (compoundPredicate instanceof Or) {
residualPredicates.add(compoundPredicate);
return null;
}
return super.visitCompoundPredicate(compoundPredicate, context);
public Void visit(Expression expr, Void context) {
residualPredicates.add(expr);
return null;
}
}
@ -98,7 +93,7 @@ public class PredicatesSplitter {
}
private static boolean containOnlyColumnRef(Expression expression, boolean allowCast) {
if (expression instanceof SlotReference && ((SlotReference) expression).isColumnFromTable()) {
if (expression instanceof SlotReference && expression.isColumnFromTable()) {
return true;
}
if (allowCast && expression instanceof Cast) {

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
View File

@ -162,7 +162,7 @@ public class StructInfo {
private void predicatesDerive() {
// construct equivalenceClass according to equals predicates
List<Expression> shuttledExpression = ExpressionUtils.shuttleExpressionWithLineage(
this.predicates.getPulledUpPredicates(), originalPlan).stream()
new ArrayList<>(this.predicates.getPulledUpPredicates()), originalPlan).stream()
.map(Expression.class::cast)
.collect(Collectors.toList());
SplitPredicate splitPredicate = Predicates.splitPredicates(ExpressionUtils.and(shuttledExpression));

6
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
View File

@ -43,6 +43,7 @@ import org.apache.doris.mtmv.MTMVRelation;
import org.apache.doris.mtmv.MTMVUtil;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.analyzer.UnboundResultSink;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
@ -54,6 +55,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
import org.apache.doris.nereids.trees.plans.visitor.NondeterministicFunctionCollector;
import org.apache.doris.nereids.trees.plans.visitor.TableCollector;
import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext;
@ -199,7 +201,9 @@ public class CreateMTMVInfo {
public void analyzeQuery(ConnectContext ctx) {
// create table as select
NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext());
Plan plan = planner.plan(logicalQuery, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN);
// this is for expression column name infer when not use alias
LogicalSink<Plan> logicalSink = new UnboundResultSink<>(logicalQuery);
Plan plan = planner.plan(logicalSink, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN);
if (plan.anyMatch(node -> node instanceof OneRowRelation)) {
throw new AnalysisException("at least contain one table");
}