diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionGenTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionGenTable.java index c4181fa991..748564e9ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionGenTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionGenTable.java @@ -22,7 +22,7 @@ import org.apache.doris.tablefunction.TableValuedFunctionIf; import java.util.List; public class FunctionGenTable extends Table { - private TableValuedFunctionIf tvf; + private final TableValuedFunctionIf tvf; public FunctionGenTable(long id, String tableName, TableType type, List fullSchema, TableValuedFunctionIf tvf) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CTEContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CTEContext.java index d00684a70e..6bc6787070 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CTEContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CTEContext.java @@ -43,14 +43,14 @@ public class CTEContext { /* build head CTEContext */ public CTEContext() { - this(null, null, CTEId.DEFAULT); + this(CTEId.DEFAULT, null, null); } /** * CTEContext */ - public CTEContext(@Nullable LogicalSubQueryAlias parsedPlan, - @Nullable CTEContext previousCteContext, CTEId cteId) { + public CTEContext(CTEId cteId, @Nullable LogicalSubQueryAlias parsedPlan, + @Nullable CTEContext previousCteContext) { if ((parsedPlan == null && previousCteContext != null) || (parsedPlan != null && previousCteContext == null)) { throw new AnalysisException("Only first CteContext can contains null cte plan or previousCteContext"); } @@ -78,7 +78,7 @@ public class CTEContext { /** * Get for CTE reuse. */ - public Optional getReuse(String cteName) { + public Optional getAnalyzedCTEPlan(String cteName) { if (!findCTEContext(cteName).isPresent()) { return Optional.empty(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index 8f1f68d159..404f9680c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -26,10 +26,8 @@ import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.jobs.Job; import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.executor.Analyzer; -import org.apache.doris.nereids.jobs.rewrite.CustomRewriteJob; import org.apache.doris.nereids.jobs.rewrite.RewriteBottomUpJob; import org.apache.doris.nereids.jobs.rewrite.RewriteTopDownJob; -import org.apache.doris.nereids.jobs.rewrite.RootPlanTreeRewriteJob.RootRewriteJobContext; import org.apache.doris.nereids.jobs.scheduler.JobPool; import org.apache.doris.nereids.jobs.scheduler.JobScheduler; import org.apache.doris.nereids.jobs.scheduler.JobStack; @@ -39,22 +37,21 @@ import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.processor.post.RuntimeFilterContext; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleFactory; import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.analysis.BindRelation.CustomTableResolver; import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; -import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.statistics.ColumnStatistic; @@ -69,10 +66,10 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.Stack; -import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import java.util.function.Function; import javax.annotation.Nullable; @@ -84,12 +81,11 @@ public class CascadesContext implements ScheduleContext { // in analyze/rewrite stage, the plan will storage in this field private Plan plan; - private Optional currentRootRewriteJobContext; // in optimize stage, the plan will storage in the memo private Memo memo; private final StatementContext statementContext; - private CTEContext cteContext; + private final CTEContext cteContext; private final RuleSet ruleSet; private final JobPool jobPool; private final JobScheduler jobScheduler; @@ -103,19 +99,9 @@ public class CascadesContext implements ScheduleContext { private boolean isRewriteRoot; private volatile boolean isTimeout = false; - private Map> cteIdToConsumers = new HashMap<>(); - private Map> cteIdToCTEClosure = new HashMap<>(); - private Map> cteIdToProjects = new HashMap<>(); - private Map> consumerIdToFilters = new HashMap<>(); - private Map> cteIdToConsumerUnderProjects = new HashMap<>(); - - // Used to update consumer's stats - private Map, Group>>> cteIdToConsumerGroup = new HashMap<>(); - - public CascadesContext(Plan plan, Memo memo, StatementContext statementContext, - PhysicalProperties requestProperties) { - this(plan, memo, statementContext, new CTEContext(), requestProperties); - } + // current process subtree, represent outer plan if empty + private final Optional currentTree; + private final Optional parent; /** * Constructor of OptimizerContext. @@ -123,55 +109,76 @@ public class CascadesContext implements ScheduleContext { * @param memo {@link Memo} reference * @param statementContext {@link StatementContext} reference */ - public CascadesContext(Plan plan, Memo memo, StatementContext statementContext, + private CascadesContext(Optional parent, Optional currentTree, + StatementContext statementContext, Plan plan, Memo memo, CTEContext cteContext, PhysicalProperties requireProperties) { - this.plan = plan; + this.parent = Objects.requireNonNull(parent, "parent should not null"); + this.currentTree = Objects.requireNonNull(currentTree, "currentTree should not null"); + this.statementContext = Objects.requireNonNull(statementContext, "statementContext should not null"); + this.plan = Objects.requireNonNull(plan, "plan should not null"); this.memo = memo; - this.statementContext = statementContext; + this.cteContext = Objects.requireNonNull(cteContext, "cteContext should not null"); this.ruleSet = new RuleSet(); this.jobPool = new JobStack(); this.jobScheduler = new SimpleJobScheduler(); this.currentJobContext = new JobContext(this, requireProperties, Double.MAX_VALUE); this.subqueryExprIsAnalyzed = new HashMap<>(); this.runtimeFilterContext = new RuntimeFilterContext(getConnectContext().getSessionVariable()); - this.cteContext = cteContext; - } - - public static CascadesContext newRewriteContext(StatementContext statementContext, - Plan initPlan, PhysicalProperties requireProperties) { - return new CascadesContext(initPlan, null, statementContext, requireProperties); - } - - public static CascadesContext newRewriteContext(StatementContext statementContext, - Plan initPlan, CTEContext cteContext) { - return newRewriteContext(statementContext, initPlan, cteContext, PhysicalProperties.ANY); - } - - public static CascadesContext newRewriteContext(StatementContext statementContext, - Plan initPlan, CTEContext cteContext, PhysicalProperties requireProperties) { - return new CascadesContext(initPlan, null, statementContext, cteContext, requireProperties); } /** - * New rewrite context. + * init a brand-new context to process whole tree */ - public static CascadesContext newRewriteContext(CascadesContext context, Plan plan) { - return newRewriteContext(context, plan, PhysicalProperties.ANY); + public static CascadesContext initContext(StatementContext statementContext, + Plan initPlan, PhysicalProperties requireProperties) { + return newContext(Optional.empty(), Optional.empty(), statementContext, + initPlan, new CTEContext(), requireProperties); + } + + /** + * use for analyze cte. we must pass CteContext from outer since we need to get right scope of cte + */ + public static CascadesContext newContextWithCteContext(CascadesContext cascadesContext, + Plan initPlan, CTEContext cteContext) { + return newContext(Optional.of(cascadesContext), Optional.empty(), + cascadesContext.getStatementContext(), initPlan, cteContext, PhysicalProperties.ANY); + } + + public static CascadesContext newCurrentTreeContext(CascadesContext context) { + return CascadesContext.newContext(context.getParent(), context.getCurrentTree(), context.getStatementContext(), + context.getRewritePlan(), context.getCteContext(), + context.getCurrentJobContext().getRequiredProperties()); } /** * New rewrite context copy from current context, used in cbo rewriter. */ - public static CascadesContext newRewriteContext(CascadesContext context, + public static CascadesContext newSubtreeContext(Optional subtree, CascadesContext context, Plan plan, PhysicalProperties requireProperties) { - CascadesContext cascadesContext = CascadesContext.newRewriteContext( - context.getStatementContext(), plan, context.getCteContext(), requireProperties); - cascadesContext.cteIdToConsumers = context.cteIdToConsumers; - cascadesContext.cteIdToProjects = context.cteIdToProjects; - cascadesContext.cteContext = context.cteContext; - cascadesContext.cteIdToCTEClosure = context.cteIdToCTEClosure; - cascadesContext.consumerIdToFilters = context.consumerIdToFilters; - return cascadesContext; + return CascadesContext.newContext(Optional.of(context), subtree, context.getStatementContext(), + plan, context.getCteContext(), requireProperties); + } + + private static CascadesContext newContext(Optional parent, Optional subtree, + StatementContext statementContext, Plan initPlan, + CTEContext cteContext, PhysicalProperties requireProperties) { + return new CascadesContext(parent, subtree, statementContext, initPlan, null, cteContext, requireProperties); + } + + public CascadesContext getRoot() { + CascadesContext root = this; + while (root.getParent().isPresent()) { + root = root.getParent().get(); + } + return root; + } + + public Optional getParent() { + return parent; + } + + public Optional getCurrentTree() { + return currentTree; } public synchronized void setIsTimeout(boolean isTimeout) { @@ -194,10 +201,6 @@ public class CascadesContext implements ScheduleContext { return new Analyzer(this, customTableResolver); } - public Analyzer newCustomAnalyzer(Optional customTableResolver) { - return new Analyzer(this, customTableResolver); - } - @Override public void pushJob(Job job) { jobPool.push(job); @@ -257,15 +260,6 @@ public class CascadesContext implements ScheduleContext { this.plan = plan; } - public Optional getCurrentRootRewriteJobContext() { - return currentRootRewriteJobContext; - } - - public void setCurrentRootRewriteJobContext( - RootRewriteJobContext currentRootRewriteJobContext) { - this.currentRootRewriteJobContext = Optional.ofNullable(currentRootRewriteJobContext); - } - public void setSubqueryExprIsAnalyzed(SubqueryExpr subqueryExpr, boolean isAnalyzed) { subqueryExprIsAnalyzed.put(subqueryExpr, isAnalyzed); } @@ -282,41 +276,14 @@ public class CascadesContext implements ScheduleContext { return execute(new RewriteBottomUpJob(memo.getRoot(), currentJobContext, ImmutableList.copyOf(rules))); } - public CascadesContext bottomUpRewrite(Rule... rules) { - return bottomUpRewrite(ImmutableList.copyOf(rules)); - } - - public CascadesContext bottomUpRewrite(List rules) { - return execute(new RewriteBottomUpJob(memo.getRoot(), rules, currentJobContext)); - } - public CascadesContext topDownRewrite(RuleFactory... rules) { return execute(new RewriteTopDownJob(memo.getRoot(), currentJobContext, ImmutableList.copyOf(rules))); } - public CascadesContext topDownRewrite(Rule... rules) { - return topDownRewrite(ImmutableList.copyOf(rules)); - } - - public CascadesContext topDownRewrite(List rules) { - return execute(new RewriteTopDownJob(memo.getRoot(), rules, currentJobContext)); - } - - public CascadesContext topDownRewrite(CustomRewriter customRewriter) { - CustomRewriteJob customRewriteJob = new CustomRewriteJob(() -> customRewriter, RuleType.TEST_REWRITE); - customRewriteJob.execute(currentJobContext); - toMemo(); - return this; - } - public CTEContext getCteContext() { return cteContext; } - public void setCteContext(CTEContext cteContext) { - this.cteContext = cteContext; - } - public void setIsRewriteRoot(boolean isRewriteRoot) { this.isRewriteRoot = isRewriteRoot; } @@ -347,9 +314,8 @@ public class CascadesContext implements ScheduleContext { if (statementContext == null) { return defaultValue; } - T cacheResult = statementContext.getOrRegisterCache(cacheName, + return statementContext.getOrRegisterCache(cacheName, () -> variableSupplier.apply(connectContext.getSessionVariable())); - return cacheResult; } private CascadesContext execute(Job job) { @@ -392,9 +358,9 @@ public class CascadesContext implements ScheduleContext { Set unboundRelations = new HashSet<>(); logicalPlan.foreach(p -> { if (p instanceof LogicalFilter) { - unboundRelations.addAll(extractUnboundRelationFromFilter((LogicalFilter) p)); + unboundRelations.addAll(extractUnboundRelationFromFilter((LogicalFilter) p)); } else if (p instanceof LogicalCTE) { - unboundRelations.addAll(extractUnboundRelationFromCTE((LogicalCTE) p)); + unboundRelations.addAll(extractUnboundRelationFromCTE((LogicalCTE) p)); } else { unboundRelations.addAll(p.collect(UnboundRelation.class::isInstance)); } @@ -402,7 +368,7 @@ public class CascadesContext implements ScheduleContext { return unboundRelations; } - private Set extractUnboundRelationFromFilter(LogicalFilter filter) { + private Set extractUnboundRelationFromFilter(LogicalFilter filter) { Set subqueryExprs = filter.getPredicate() .collect(SubqueryExpr.class::isInstance); Set relations = new HashSet<>(); @@ -413,7 +379,7 @@ public class CascadesContext implements ScheduleContext { return relations; } - private Set extractUnboundRelationFromCTE(LogicalCTE cte) { + private Set extractUnboundRelationFromCTE(LogicalCTE cte) { List> subQueryAliases = cte.getAliasQueries(); Set relations = new HashSet<>(); for (LogicalSubQueryAlias subQueryAlias : subQueryAliases) { @@ -463,7 +429,7 @@ public class CascadesContext implements ScheduleContext { CascadesContext cascadesContext; - private Stack locked = new Stack<>(); + private final Stack
locked = new Stack<>(); /** * Try to acquire read locks on tables, throw runtime exception once the acquiring for read lock failed. @@ -491,93 +457,49 @@ public class CascadesContext implements ScheduleContext { } } - public void putCTEIdToCTEClosure(CTEId cteId, Callable cteClosure) { - this.cteIdToCTEClosure.put(cteId, cteClosure); - } - - public void putAllCTEIdToCTEClosure(Map> cteConsumers) { - this.cteIdToCTEClosure.putAll(cteConsumers); - } - public void putCTEIdToConsumer(LogicalCTEConsumer cteConsumer) { - Set consumers = - this.cteIdToConsumers.computeIfAbsent(cteConsumer.getCteId(), k -> new HashSet<>()); + Set consumers = this.statementContext.getCteIdToConsumers() + .computeIfAbsent(cteConsumer.getCteId(), k -> new HashSet<>()); consumers.add(cteConsumer); } - public void putAllCTEIdToConsumer(Map> cteConsumers) { - this.cteIdToConsumers.putAll(cteConsumers); - } - - public void putCTEIdToProject(CTEId cteId, Expression p) { - Set projects = this.cteIdToProjects.computeIfAbsent(cteId, k -> new HashSet<>()); + public void putCTEIdToProject(CTEId cteId, NamedExpression p) { + Set projects = this.statementContext.getCteIdToProjects() + .computeIfAbsent(cteId, k -> new HashSet<>()); projects.add(p); } - public Set getProjectForProducer(CTEId cteId) { - return this.cteIdToProjects.get(cteId); - } - - /** - * Fork for rewritten child tree of CTEProducer. - */ - public CascadesContext forkForCTEProducer(Plan plan) { - CascadesContext cascadesContext = new CascadesContext(plan, memo, statementContext, PhysicalProperties.ANY); - cascadesContext.cteIdToConsumers = cteIdToConsumers; - cascadesContext.cteIdToProjects = cteIdToProjects; - cascadesContext.cteContext = cteContext; - cascadesContext.cteIdToCTEClosure = cteIdToCTEClosure; - cascadesContext.consumerIdToFilters = consumerIdToFilters; - return cascadesContext; - } - - public int cteReferencedCount(CTEId cteId) { - Set cteConsumer = cteIdToConsumers.get(cteId); - if (cteConsumer == null) { - return 0; - } - return cteIdToConsumers.get(cteId).size(); + public Set getProjectForProducer(CTEId cteId) { + return this.statementContext.getCteIdToProjects().get(cteId); } public Map> getCteIdToConsumers() { - return cteIdToConsumers; + return this.statementContext.getCteIdToConsumers(); } - public Map> getCteIdToCTEClosure() { - return cteIdToCTEClosure; - } - - public LogicalPlan findCTEPlanForInline(CTEId cteId) { - try { - return cteIdToCTEClosure.get(cteId).call(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public void putConsumerIdToFilter(int id, Expression filter) { - Set filters = this.consumerIdToFilters.computeIfAbsent(id, k -> new HashSet<>()); + public void putConsumerIdToFilter(RelationId id, Expression filter) { + Set filters = this.getConsumerIdToFilters().computeIfAbsent(id, k -> new HashSet<>()); filters.add(filter); } - public Map> getConsumerIdToFilters() { - return consumerIdToFilters; + public Map> getConsumerIdToFilters() { + return this.statementContext.getConsumerIdToFilters(); } public void markConsumerUnderProject(LogicalCTEConsumer cteConsumer) { - Set consumerIds = - this.cteIdToConsumerUnderProjects.computeIfAbsent(cteConsumer.getCteId(), k -> new HashSet<>()); - consumerIds.add(cteConsumer.getConsumerId()); + Set consumerIds = this.statementContext.getCteIdToConsumerUnderProjects() + .computeIfAbsent(cteConsumer.getCteId(), k -> new HashSet<>()); + consumerIds.add(cteConsumer.getRelationId()); } public boolean couldPruneColumnOnProducer(CTEId cteId) { - Set consumerIds = this.cteIdToConsumerUnderProjects.get(cteId); - return consumerIds.size() == this.cteIdToConsumers.get(cteId).size(); + Set consumerIds = this.statementContext.getCteIdToConsumerUnderProjects().get(cteId); + return consumerIds.size() == this.statementContext.getCteIdToConsumers().get(cteId).size(); } public void addCTEConsumerGroup(CTEId cteId, Group g, Map producerSlotToConsumerSlot) { List, Group>> consumerGroups = - this.cteIdToConsumerGroup.computeIfAbsent(cteId, k -> new ArrayList<>()); + this.statementContext.getCteIdToConsumerGroup().computeIfAbsent(cteId, k -> new ArrayList<>()); consumerGroups.add(Pair.of(producerSlotToConsumerSlot, g)); } @@ -585,7 +507,7 @@ public class CascadesContext implements ScheduleContext { * Update CTE consumer group as producer's stats update */ public void updateConsumerStats(CTEId cteId, Statistics statistics) { - List, Group>> consumerGroups = this.cteIdToConsumerGroup.get(cteId); + List, Group>> consumerGroups = this.statementContext.getCteIdToConsumerGroup().get(cteId); for (Pair, Group> p : consumerGroups) { Map producerSlotToConsumerSlot = p.first; Statistics updatedConsumerStats = new Statistics(statistics); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 25064c1c68..02df5e9fa4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -268,7 +268,7 @@ public class NereidsPlanner extends Planner { } private void initCascadesContext(LogicalPlan plan, PhysicalProperties requireProperties) { - cascadesContext = CascadesContext.newRewriteContext(statementContext, plan, requireProperties); + cascadesContext = CascadesContext.initContext(statementContext, plan, requireProperties); if (statementContext.getConnectContext().getTables() != null) { cascadesContext.setTables(statementContext.getConnectContext().getTables()); } @@ -283,7 +283,7 @@ public class NereidsPlanner extends Planner { * Logical plan rewrite based on a series of heuristic rules. */ private void rewrite() { - new Rewriter(cascadesContext).execute(); + Rewriter.getWholeTreeRewriter(cascadesContext).execute(); NereidsTracer.logImportantTime("EndRewritePlan"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index a6e73b32fd..20aac89dca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -19,10 +19,18 @@ package org.apache.doris.nereids; import org.apache.doris.analysis.StatementBase; import org.apache.doris.common.IdGenerator; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.rules.analysis.ColumnAliasGenerator; import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.ObjectId; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.OriginStatement; @@ -30,7 +38,8 @@ import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.collect.Maps; -import java.util.HashSet; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import javax.annotation.concurrent.GuardedBy; @@ -42,29 +51,31 @@ public class StatementContext { private ConnectContext connectContext; + @GuardedBy("this") + private final Map> contextCacheMap = Maps.newLinkedHashMap(); + private OriginStatement originStatement; + // NOTICE: we set the plan parsed by DorisParser to parsedStatement and if the plan is command, create a + // LogicalPlanAdapter with the logical plan in the command. + private StatementBase parsedStatement; + private ColumnAliasGenerator columnAliasGenerator; private int maxNAryInnerJoin = 0; - private boolean isDpHyp = false; private boolean isOtherJoinReorder = false; private final IdGenerator exprIdGenerator = ExprId.createGenerator(); - private final IdGenerator objectIdGenerator = ObjectId.createGenerator(); - + private final IdGenerator relationIdGenerator = RelationId.createGenerator(); private final IdGenerator cteIdGenerator = CTEId.createGenerator(); - @GuardedBy("this") - private final Map> contextCacheMap = Maps.newLinkedHashMap(); - - // NOTICE: we set the plan parsed by DorisParser to parsedStatement and if the plan is command, create a - // LogicalPlanAdapter with the logical plan in the command. - private StatementBase parsedStatement; - - private Set columnNames; - - private ColumnAliasGenerator columnAliasGenerator; + private final Map> cteIdToConsumers = new HashMap<>(); + private final Map> cteIdToProjects = new HashMap<>(); + private final Map> consumerIdToFilters = new HashMap<>(); + private final Map> cteIdToConsumerUnderProjects = new HashMap<>(); + // Used to update consumer's stats + private final Map, Group>>> cteIdToConsumerGroup = new HashMap<>(); + private final Map rewrittenCtePlan = new HashMap<>(); public StatementContext() { this.connectContext = ConnectContext.get(); @@ -91,7 +102,7 @@ public class StatementContext { return originStatement; } - public void setMaxNArayInnerJoin(int maxNAryInnerJoin) { + public void setMaxNAryInnerJoin(int maxNAryInnerJoin) { if (maxNAryInnerJoin > this.maxNAryInnerJoin) { this.maxNAryInnerJoin = maxNAryInnerJoin; } @@ -129,6 +140,10 @@ public class StatementContext { return objectIdGenerator.getNextId(); } + public RelationId getNextRelationId() { + return relationIdGenerator.getNextId(); + } + public void setParsedStatement(StatementBase parsedStatement) { this.parsedStatement = parsedStatement; } @@ -143,17 +158,9 @@ public class StatementContext { return supplier.get(); } - public Set getColumnNames() { - return columnNames == null ? new HashSet<>() : columnNames; - } - - public void setColumnNames(Set columnNames) { - this.columnNames = columnNames; - } - public ColumnAliasGenerator getColumnAliasGenerator() { return columnAliasGenerator == null - ? columnAliasGenerator = new ColumnAliasGenerator(this) + ? columnAliasGenerator = new ColumnAliasGenerator() : columnAliasGenerator; } @@ -164,4 +171,28 @@ public class StatementContext { public StatementBase getParsedStatement() { return parsedStatement; } + + public Map> getCteIdToConsumers() { + return cteIdToConsumers; + } + + public Map> getCteIdToProjects() { + return cteIdToProjects; + } + + public Map> getConsumerIdToFilters() { + return consumerIdToFilters; + } + + public Map> getCteIdToConsumerUnderProjects() { + return cteIdToConsumerUnderProjects; + } + + public Map, Group>>> getCteIdToConsumerGroup() { + return cteIdToConsumerGroup; + } + + public Map getRewrittenCtePlan() { + return rewrittenCtePlan; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java index 3f349c9224..dd04e9625a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java @@ -24,11 +24,11 @@ import org.apache.doris.nereids.properties.UnboundLogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf; +import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -36,37 +36,30 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import java.util.List; -import java.util.Objects; import java.util.Optional; /** * A relation that contains only one row consist of some constant expressions. * e.g. select 100, 'value' */ -public class UnboundOneRowRelation extends LogicalLeaf implements Unbound, OneRowRelation { +public class UnboundOneRowRelation extends LogicalRelation implements Unbound, OneRowRelation { - private final ObjectId id; private final List projects; - public UnboundOneRowRelation(ObjectId id, List projects) { - this(id, projects, Optional.empty(), Optional.empty()); + public UnboundOneRowRelation(RelationId relationId, List projects) { + this(relationId, projects, Optional.empty(), Optional.empty()); } - private UnboundOneRowRelation(ObjectId id, + private UnboundOneRowRelation(RelationId id, List projects, Optional groupExpression, Optional logicalProperties) { - super(PlanType.LOGICAL_UNBOUND_ONE_ROW_RELATION, groupExpression, logicalProperties); + super(id, PlanType.LOGICAL_UNBOUND_ONE_ROW_RELATION, groupExpression, logicalProperties); Preconditions.checkArgument(projects.stream().noneMatch(p -> p.containsType(Slot.class)), "OneRowRelation can not contains any slot"); - this.id = id; this.projects = ImmutableList.copyOf(projects); } - public ObjectId getId() { - return id; - } - @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitUnboundOneRowRelation(this, context); @@ -84,13 +77,14 @@ public class UnboundOneRowRelation extends LogicalLeaf implements Unbound, OneRo @Override public Plan withGroupExpression(Optional groupExpression) { - return new UnboundOneRowRelation(id, projects, groupExpression, Optional.of(logicalPropertiesSupplier.get())); + return new UnboundOneRowRelation(relationId, projects, + groupExpression, Optional.of(logicalPropertiesSupplier.get())); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new UnboundOneRowRelation(id, projects, groupExpression, logicalProperties); + return new UnboundOneRowRelation(relationId, projects, groupExpression, logicalProperties); } @Override @@ -106,28 +100,8 @@ public class UnboundOneRowRelation extends LogicalLeaf implements Unbound, OneRo @Override public String toString() { return Utils.toSqlString("UnboundOneRowRelation", - "relationId", id, + "relationId", relationId, "projects", projects ); } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - UnboundOneRowRelation that = (UnboundOneRowRelation) o; - return Objects.equals(id, that.id) && Objects.equals(projects, that.projects); - } - - @Override - public int hashCode() { - return Objects.hash(id, projects); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index 1f820398d9..6d625f7068 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -17,16 +17,15 @@ package org.apache.doris.nereids.analyzer; -import org.apache.doris.catalog.Table; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.UnboundLogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -50,20 +49,20 @@ public class UnboundRelation extends LogicalRelation implements Unbound { private final boolean isTempPart; private final List hints; - public UnboundRelation(ObjectId id, List nameParts) { + public UnboundRelation(RelationId id, List nameParts) { this(id, nameParts, Optional.empty(), Optional.empty(), ImmutableList.of(), false, ImmutableList.of()); } - public UnboundRelation(ObjectId id, List nameParts, List partNames, boolean isTempPart) { + public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart) { this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, ImmutableList.of()); } - public UnboundRelation(ObjectId id, List nameParts, List partNames, boolean isTempPart, + public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart, List hints) { this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, hints); } - public UnboundRelation(ObjectId id, List nameParts, Optional groupExpression, + public UnboundRelation(RelationId id, List nameParts, Optional groupExpression, Optional logicalProperties, List partNames, boolean isTempPart, List hints) { super(id, PlanType.LOGICAL_UNBOUND_RELATION, groupExpression, logicalProperties); @@ -73,11 +72,6 @@ public class UnboundRelation extends LogicalRelation implements Unbound { this.hints = ImmutableList.copyOf(Objects.requireNonNull(hints, "hints should not be null.")); } - @Override - public Table getTable() { - throw new UnsupportedOperationException("unbound relation cannot get table"); - } - public List getNameParts() { return nameParts; } @@ -94,14 +88,15 @@ public class UnboundRelation extends LogicalRelation implements Unbound { @Override public Plan withGroupExpression(Optional groupExpression) { - return new UnboundRelation(id, nameParts, groupExpression, Optional.of(getLogicalProperties()), partNames, - isTempPart, hints); + return new UnboundRelation(relationId, nameParts, + groupExpression, Optional.of(getLogicalProperties()), + partNames, isTempPart, hints); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new UnboundRelation(id, nameParts, groupExpression, logicalProperties, partNames, + return new UnboundRelation(relationId, nameParts, groupExpression, logicalProperties, partNames, isTempPart, hints); } @@ -113,7 +108,7 @@ public class UnboundRelation extends LogicalRelation implements Unbound { @Override public String toString() { List args = Lists.newArrayList( - "id", id, + "id", relationId, "nameParts", StringUtils.join(nameParts, ".") ); if (CollectionUtils.isNotEmpty(hints)) { @@ -133,30 +128,6 @@ public class UnboundRelation extends LogicalRelation implements Unbound { throw new UnsupportedOperationException(this.getClass().getSimpleName() + " don't support getExpression()"); } - public ObjectId getId() { - return id; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - UnboundRelation that = (UnboundRelation) o; - return id.equals(that.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - public List getPartNames() { return partNames; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java index 2116c63c29..83c9d4fa28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java @@ -25,11 +25,11 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.TVFProperties; import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.TVFRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf; +import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -38,20 +38,18 @@ import java.util.Objects; import java.util.Optional; /** UnboundTVFRelation */ -public class UnboundTVFRelation extends LogicalLeaf implements TVFRelation, Unbound { +public class UnboundTVFRelation extends LogicalRelation implements TVFRelation, Unbound { - private final ObjectId id; private final String functionName; private final TVFProperties properties; - public UnboundTVFRelation(ObjectId id, String functionName, TVFProperties properties) { + public UnboundTVFRelation(RelationId id, String functionName, TVFProperties properties) { this(id, functionName, properties, Optional.empty(), Optional.empty()); } - public UnboundTVFRelation(ObjectId id, String functionName, TVFProperties properties, + public UnboundTVFRelation(RelationId id, String functionName, TVFProperties properties, Optional groupExpression, Optional logicalProperties) { - super(PlanType.LOGICAL_UNBOUND_TVF_RELATION, groupExpression, logicalProperties); - this.id = id; + super(id, PlanType.LOGICAL_UNBOUND_TVF_RELATION, groupExpression, logicalProperties); this.functionName = Objects.requireNonNull(functionName, "functionName can not be null"); this.properties = Objects.requireNonNull(properties, "properties can not be null"); } @@ -64,10 +62,6 @@ public class UnboundTVFRelation extends LogicalLeaf implements TVFRelation, Unbo return properties; } - public ObjectId getId() { - return id; - } - @Override public TableValuedFunction getFunction() { throw new UnboundException("getFunction"); @@ -95,14 +89,14 @@ public class UnboundTVFRelation extends LogicalLeaf implements TVFRelation, Unbo @Override public Plan withGroupExpression(Optional groupExpression) { - return new UnboundTVFRelation(id, functionName, properties, groupExpression, + return new UnboundTVFRelation(relationId, functionName, properties, groupExpression, Optional.of(getLogicalProperties())); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new UnboundTVFRelation(id, functionName, properties, groupExpression, logicalProperties); + return new UnboundTVFRelation(relationId, functionName, properties, groupExpression, logicalProperties); } @Override @@ -112,24 +106,4 @@ public class UnboundTVFRelation extends LogicalLeaf implements TVFRelation, Unbo "arguments", properties ); } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - UnboundTVFRelation that = (UnboundTVFRelation) o; - return functionName.equals(that.functionName) && properties.equals(that.properties) && id.equals(that.id); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), functionName, properties, id); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 56e4139357..1f5f90efc1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -411,7 +411,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor runtimeFilterGenerator.getTargetOnScanNode(fileScan.getId()).forEach( + runtimeFilterGenerator -> runtimeFilterGenerator.getTargetOnScanNode(fileScan.getRelationId()).forEach( expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, finalScanNode, context) ) ); @@ -453,7 +453,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor runtimeFilterGenerator.getTargetOnScanNode(esScan.getId()).forEach( + runtimeFilterGenerator -> runtimeFilterGenerator.getTargetOnScanNode(esScan.getRelationId()).forEach( expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, esScanNode, context) ) ); @@ -475,7 +475,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor runtimeFilterGenerator.getTargetOnScanNode(jdbcScan.getId()).forEach( + runtimeFilterGenerator -> runtimeFilterGenerator.getTargetOnScanNode(jdbcScan.getRelationId()).forEach( expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, jdbcScanNode, context) ) ); @@ -541,8 +541,9 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor runtimeFilterTranslator.getTargetOnScanNode(olapScan.getId()).forEach( - expr -> runtimeFilterTranslator.translateRuntimeFilterTarget(expr, olapScanNode, context) + runtimeFilterTranslator -> runtimeFilterTranslator.getTargetOnScanNode(olapScan.getRelationId()) + .forEach(expr -> runtimeFilterTranslator.translateRuntimeFilterTarget( + expr, olapScanNode, context) ) ); // TODO: we need to remove all finalizeForNereids @@ -599,8 +600,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor runtimeFilterGenerator.getTargetOnScanNode(schemaScan.getId()).forEach( - expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, scanNode, context) + runtimeFilterGenerator -> runtimeFilterGenerator.getTargetOnScanNode(schemaScan.getRelationId()) + .forEach(expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, scanNode, context) ) ); scanNode.finalizeForNereids(); @@ -614,14 +615,14 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor slots = tvfRelation.getLogicalProperties().getOutput(); - TupleDescriptor tupleDescriptor = generateTupleDesc(slots, tvfRelation.getTable(), context); + TupleDescriptor tupleDescriptor = generateTupleDesc(slots, tvfRelation.getFunction().getTable(), context); TableValuedFunctionIf catalogFunction = tvfRelation.getFunction().getCatalogFunction(); ScanNode scanNode = catalogFunction.getScanNode(context.nextPlanNodeId(), tupleDescriptor); Utils.execWithUncheckedException(scanNode::init); context.getRuntimeTranslator().ifPresent( - runtimeFilterGenerator -> runtimeFilterGenerator.getTargetOnScanNode(tvfRelation.getId()).forEach( - expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, scanNode, context) + runtimeFilterGenerator -> runtimeFilterGenerator.getTargetOnScanNode(tvfRelation.getRelationId()) + .forEach(expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, scanNode, context) ) ); Utils.execWithUncheckedException(scanNode::finalizeForNereids); @@ -820,7 +821,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor outputs = cteProducer.getProjects().stream() + List outputs = cteProducer.getOutput().stream() .map(e -> ExpressionTranslator.translate(e, context)) .collect(Collectors.toList()); - cteProduce.setOutputExprs(outputs); - context.getCteProduceFragments().put(cteId, cteProduce); + multiCastPlanFragment.setOutputExprs(outputs); + context.getCteProduceFragments().put(cteId, multiCastPlanFragment); context.getCteProduceMap().put(cteId, cteProducer); if (context.getRuntimeTranslator().isPresent()) { context.getRuntimeTranslator().get().getContext().getCteProduceMap().put(cteId, cteProducer); } - context.getPlanFragments().add(cteProduce); + context.getPlanFragments().add(multiCastPlanFragment); return child; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java index bbb9997f2e..6980f05fa5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java @@ -28,7 +28,7 @@ import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.ObjectId; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin; import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; import org.apache.doris.planner.HashJoinNode; @@ -68,7 +68,7 @@ public class RuntimeFilterTranslator { return context; } - public List getTargetOnScanNode(ObjectId id) { + public List getTargetOnScanNode(RelationId id) { return context.getTargetOnOlapScanNodeMap().getOrDefault(id, Collections.emptyList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java index 7f089d14b1..96afa6b24a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java @@ -20,6 +20,7 @@ package org.apache.doris.nereids.jobs.executor; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.jobs.rewrite.RewriteJob; import org.apache.doris.nereids.rules.analysis.AdjustAggregateNullableForEmptySet; +import org.apache.doris.nereids.rules.analysis.AnalyzeCTE; import org.apache.doris.nereids.rules.analysis.BindExpression; import org.apache.doris.nereids.rules.analysis.BindInsertTargetTable; import org.apache.doris.nereids.rules.analysis.BindRelation; @@ -31,7 +32,6 @@ import org.apache.doris.nereids.rules.analysis.FillUpMissingSlots; import org.apache.doris.nereids.rules.analysis.NormalizeRepeat; import org.apache.doris.nereids.rules.analysis.ProjectToGlobalAggregate; import org.apache.doris.nereids.rules.analysis.ProjectWithDistinctToAggregate; -import org.apache.doris.nereids.rules.analysis.RegisterCTE; import org.apache.doris.nereids.rules.analysis.ReplaceExpressionByChildOutput; import org.apache.doris.nereids.rules.analysis.ResolveOrdinalInOrderByAndGroupBy; import org.apache.doris.nereids.rules.analysis.SubqueryToApply; @@ -49,9 +49,7 @@ public class Analyzer extends AbstractBatchJobExecutor { public static final List DEFAULT_ANALYZE_JOBS = buildAnalyzeJobs(Optional.empty()); - private Optional customTableResolver; - - private List jobs; + private final List jobs; /** * Execute the analysis job with scope. @@ -63,7 +61,7 @@ public class Analyzer extends AbstractBatchJobExecutor { public Analyzer(CascadesContext cascadesContext, Optional customTableResolver) { super(cascadesContext); - this.customTableResolver = Objects.requireNonNull(customTableResolver, "customTableResolver cannot be null"); + Objects.requireNonNull(customTableResolver, "customTableResolver cannot be null"); this.jobs = !customTableResolver.isPresent() ? DEFAULT_ANALYZE_JOBS : buildAnalyzeJobs(customTableResolver); } @@ -81,21 +79,15 @@ public class Analyzer extends AbstractBatchJobExecutor { private static List buildAnalyzeJobs(Optional customTableResolver) { return jobs( - topDown( - new RegisterCTE() - ), + topDown(new AnalyzeCTE()), bottomUp( - new BindRelation(customTableResolver.orElse(null)), + new BindRelation(customTableResolver), new CheckPolicy(), new UserAuthentication(), new BindExpression() ), - topDown( - new BindInsertTargetTable() - ), - bottomUp( - new CheckBound() - ), + topDown(new BindInsertTargetTable()), + bottomUp(new CheckBound()), bottomUp( new ProjectToGlobalAggregate(), // this rule check's the logicalProject node's isDistinct property diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index a1008ff945..18fdccb221 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -18,11 +18,11 @@ package org.apache.doris.nereids.jobs.executor; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.rewrite.CostBasedRewriteJob; import org.apache.doris.nereids.jobs.rewrite.RewriteJob; import org.apache.doris.nereids.processor.pre.EliminateLogicalSelectHint; import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.analysis.AddDefaultLimit; import org.apache.doris.nereids.rules.analysis.AdjustAggregateNullableForEmptySet; import org.apache.doris.nereids.rules.analysis.AvgDistinctToSumDivCount; import org.apache.doris.nereids.rules.analysis.CheckAfterRewrite; @@ -31,12 +31,12 @@ import org.apache.doris.nereids.rules.expression.CheckLegalityAfterRewrite; import org.apache.doris.nereids.rules.expression.ExpressionNormalization; import org.apache.doris.nereids.rules.expression.ExpressionOptimization; import org.apache.doris.nereids.rules.expression.ExpressionRewrite; +import org.apache.doris.nereids.rules.rewrite.AddDefaultLimit; import org.apache.doris.nereids.rules.rewrite.AdjustConjunctsReturnType; import org.apache.doris.nereids.rules.rewrite.AdjustNullable; import org.apache.doris.nereids.rules.rewrite.AggScalarSubQueryToWindowFunction; import org.apache.doris.nereids.rules.rewrite.BuildAggForUnion; -import org.apache.doris.nereids.rules.rewrite.BuildCTEAnchorAndCTEProducer; -import org.apache.doris.nereids.rules.rewrite.CTEProducerRewrite; +import org.apache.doris.nereids.rules.rewrite.CTEInline; import org.apache.doris.nereids.rules.rewrite.CheckAndStandardizeWindowFunctionAndFrame; import org.apache.doris.nereids.rules.rewrite.CheckDataTypes; import org.apache.doris.nereids.rules.rewrite.CheckMatchExpression; @@ -64,7 +64,6 @@ import org.apache.doris.nereids.rules.rewrite.InferFilterNotNull; import org.apache.doris.nereids.rules.rewrite.InferJoinNotNull; import org.apache.doris.nereids.rules.rewrite.InferPredicates; import org.apache.doris.nereids.rules.rewrite.InferSetOperatorDistinct; -import org.apache.doris.nereids.rules.rewrite.InlineCTE; import org.apache.doris.nereids.rules.rewrite.MergeFilters; import org.apache.doris.nereids.rules.rewrite.MergeOneRowRelationIntoUnion; import org.apache.doris.nereids.rules.rewrite.MergeProjects; @@ -74,6 +73,7 @@ import org.apache.doris.nereids.rules.rewrite.NormalizeSort; import org.apache.doris.nereids.rules.rewrite.PruneFileScanPartition; import org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition; import org.apache.doris.nereids.rules.rewrite.PruneOlapScanTablet; +import org.apache.doris.nereids.rules.rewrite.PullUpCteAnchor; import org.apache.doris.nereids.rules.rewrite.PushFilterInsideJoin; import org.apache.doris.nereids.rules.rewrite.PushProjectIntoOneRowRelation; import org.apache.doris.nereids.rules.rewrite.PushProjectThroughUnion; @@ -82,6 +82,7 @@ import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughWindow; import org.apache.doris.nereids.rules.rewrite.PushdownLimit; import org.apache.doris.nereids.rules.rewrite.PushdownTopNThroughWindow; import org.apache.doris.nereids.rules.rewrite.ReorderJoin; +import org.apache.doris.nereids.rules.rewrite.RewriteCteChildren; import org.apache.doris.nereids.rules.rewrite.SemiJoinCommute; import org.apache.doris.nereids.rules.rewrite.SimplifyAggGroupBy; import org.apache.doris.nereids.rules.rewrite.SplitLimit; @@ -96,15 +97,14 @@ import org.apache.doris.nereids.rules.rewrite.mv.SelectMaterializedIndexWithAggr import org.apache.doris.nereids.rules.rewrite.mv.SelectMaterializedIndexWithoutAggregate; import java.util.List; +import java.util.stream.Collectors; /** - * Apply rules to optimize logical plan. + * Apply rules to rewrite logical plan. */ public class Rewriter extends AbstractBatchJobExecutor { - public static final List REWRITE_JOBS = jobs( - bottomUp(new InlineCTE()), - custom(RuleType.ADD_DEFAULT_LIMIT, AddDefaultLimit::new), + private static final List CTE_CHILDREN_REWRITE_JOBS = jobs( topic("Plan Normalization", topDown( new EliminateOrderByConstant(), @@ -128,6 +128,7 @@ public class Rewriter extends AbstractBatchJobExecutor { new ExtractSingleTableExpressionFromDisjunction() ) ), + // subquery unnesting relay on ExpressionNormalization to extract common factor expression topic("Subquery unnesting", costBased( custom(RuleType.AGG_SCALAR_SUBQUERY_TO_WINDOW_FUNCTION, @@ -277,11 +278,6 @@ public class Rewriter extends AbstractBatchJobExecutor { bottomUp(RuleSet.PUSH_DOWN_FILTERS), custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT, EliminateUnnecessaryProject::new) ), - topic("Match expression check", - topDown( - new CheckMatchExpression() - ) - ), // this rule batch must keep at the end of rewrite to do some plan check topic("Final rewrite and check", custom(RuleType.ENSURE_PROJECT_ON_TOP_JOIN, EnsureProjectOnTopJoin::new), @@ -290,30 +286,74 @@ public class Rewriter extends AbstractBatchJobExecutor { new MergeProjects() ), custom(RuleType.ADJUST_CONJUNCTS_RETURN_TYPE, AdjustConjunctsReturnType::new), - custom(RuleType.ADJUST_NULLABLE, AdjustNullable::new), bottomUp( new ExpressionRewrite(CheckLegalityAfterRewrite.INSTANCE), + new CheckMatchExpression(), new CheckAfterRewrite() - )), - topic("MATERIALIZED CTE", topDown( + ) + ), + topic("Push project and filter on cte consumer to cte producer", + topDown( new CollectFilterAboveConsumer(), - new CollectProjectAboveConsumer(), - new BuildCTEAnchorAndCTEProducer()), - topDown(new CTEProducerRewrite())) + new CollectProjectAboveConsumer() + ) + ) ); + private static final List WHOLE_TREE_REWRITE_JOBS + = getWholeTreeRewriteJobs(true); + + private static final List WHOLE_TREE_REWRITE_JOBS_WITHOUT_COST_BASED + = getWholeTreeRewriteJobs(false); + private final List rewriteJobs; - public Rewriter(CascadesContext cascadesContext) { - super(cascadesContext); - this.rewriteJobs = REWRITE_JOBS; - } - - public Rewriter(CascadesContext cascadesContext, List rewriteJobs) { + private Rewriter(CascadesContext cascadesContext, List rewriteJobs) { super(cascadesContext); this.rewriteJobs = rewriteJobs; } + public static Rewriter getWholeTreeRewriterWithoutCostBasedJobs(CascadesContext cascadesContext) { + return new Rewriter(cascadesContext, WHOLE_TREE_REWRITE_JOBS_WITHOUT_COST_BASED); + } + + public static Rewriter getWholeTreeRewriter(CascadesContext cascadesContext) { + return new Rewriter(cascadesContext, WHOLE_TREE_REWRITE_JOBS); + } + + public static Rewriter getCteChildrenRewriter(CascadesContext cascadesContext, List jobs) { + return new Rewriter(cascadesContext, jobs); + } + + public static Rewriter getWholeTreeRewriterWithCustomJobs(CascadesContext cascadesContext, List jobs) { + return new Rewriter(cascadesContext, getWholeTreeRewriteJobs(jobs)); + } + + private static List getWholeTreeRewriteJobs(boolean withCostBased) { + List withoutCostBased = Rewriter.CTE_CHILDREN_REWRITE_JOBS.stream() + .filter(j -> !(j instanceof CostBasedRewriteJob)) + .collect(Collectors.toList()); + return getWholeTreeRewriteJobs(withCostBased ? CTE_CHILDREN_REWRITE_JOBS : withoutCostBased); + } + + private static List getWholeTreeRewriteJobs(List jobs) { + return jobs( + topic("cte inline and pull up all cte anchor", + custom(RuleType.PULL_UP_CTE_ANCHOR, PullUpCteAnchor::new), + custom(RuleType.CTE_INLINE, CTEInline::new) + ), + topic("process limit session variables", + custom(RuleType.ADD_DEFAULT_LIMIT, AddDefaultLimit::new) + ), + topic("rewrite cte sub-tree", + custom(RuleType.REWRITE_CTE_CHILDREN, () -> new RewriteCteChildren(jobs)) + ), + topic("whole plan check", + custom(RuleType.ADJUST_NULLABLE, AdjustNullable::new) + ) + ); + } + @Override public List getJobs() { return rewriteJobs; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CostBasedRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CostBasedRewriteJob.java index 72f5187cfd..2e5132f4dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CostBasedRewriteJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CostBasedRewriteJob.java @@ -24,6 +24,8 @@ import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.executor.Optimizer; import org.apache.doris.nereids.jobs.executor.Rewriter; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -48,37 +50,29 @@ public class CostBasedRewriteJob implements RewriteJob { @Override public void execute(JobContext jobContext) { - CascadesContext cascadesContext = jobContext.getCascadesContext(); - CascadesContext skipCboRuleCtx = CascadesContext.newRewriteContext( - cascadesContext, cascadesContext.getRewritePlan(), - cascadesContext.getCurrentJobContext().getRequiredProperties()); - CascadesContext applyCboRuleCtx = CascadesContext.newRewriteContext( - cascadesContext, cascadesContext.getRewritePlan(), - cascadesContext.getCurrentJobContext().getRequiredProperties()); + CascadesContext currentCtx = jobContext.getCascadesContext(); + CascadesContext skipCboRuleCtx = CascadesContext.newCurrentTreeContext(currentCtx); + CascadesContext applyCboRuleCtx = CascadesContext.newCurrentTreeContext(currentCtx); // execute cbo rule on one candidate - new Rewriter(applyCboRuleCtx, rewriteJobs).execute(); + Rewriter.getCteChildrenRewriter(applyCboRuleCtx, rewriteJobs).execute(); if (skipCboRuleCtx.getRewritePlan().deepEquals(applyCboRuleCtx.getRewritePlan())) { // this means rewrite do not do anything return; } - // Do rewrite on 2 candidates - new Rewriter(skipCboRuleCtx, jobContext.getRemainJobs()).execute(); - new Rewriter(applyCboRuleCtx, jobContext.getRemainJobs()).execute(); - // Do optimize on 2 candidates - new Optimizer(skipCboRuleCtx).execute(); - new Optimizer(applyCboRuleCtx).execute(); - Optional> skipCboRuleCost = skipCboRuleCtx.getMemo().getRoot() - .getLowestCostPlan(skipCboRuleCtx.getCurrentJobContext().getRequiredProperties()); - Optional> appliedCboRuleCost = applyCboRuleCtx.getMemo().getRoot() - .getLowestCostPlan(applyCboRuleCtx.getCurrentJobContext().getRequiredProperties()); + + // compare two candidates + Optional> skipCboRuleCost = getCost(currentCtx, skipCboRuleCtx, jobContext); + Optional> appliedCboRuleCost = getCost(currentCtx, applyCboRuleCtx, jobContext); // If one of them optimize failed, just return if (!skipCboRuleCost.isPresent() || !appliedCboRuleCost.isPresent()) { - LOG.warn("Cbo rewrite execute failed"); + LOG.warn("Cbo rewrite execute failed on sql: {}, jobs are {}, plan is {}.", + currentCtx.getStatementContext().getOriginStatement().originStmt, + rewriteJobs, currentCtx.getRewritePlan()); return; } // If the candidate applied cbo rule is better, replace the original plan with it. if (appliedCboRuleCost.get().first.getValue() < skipCboRuleCost.get().first.getValue()) { - cascadesContext.setRewritePlan(applyCboRuleCtx.getRewritePlan()); + currentCtx.setRewritePlan(applyCboRuleCtx.getRewritePlan()); } } @@ -87,4 +81,27 @@ public class CostBasedRewriteJob implements RewriteJob { // TODO: currently, we do not support execute it more than once. return true; } + + private Optional> getCost(CascadesContext currentCtx, + CascadesContext cboCtx, JobContext jobContext) { + // Do subtree rewrite + Rewriter.getCteChildrenRewriter(cboCtx, jobContext.getRemainJobs()).execute(); + CascadesContext rootCtx = currentCtx.getRoot(); + if (rootCtx.getRewritePlan() instanceof LogicalCTEAnchor) { + // set subtree rewrite cache + currentCtx.getStatementContext().getRewrittenCtePlan() + .put(currentCtx.getCurrentTree().orElse(null), (LogicalPlan) cboCtx.getRewritePlan()); + // Do Whole tree rewrite + CascadesContext rootCtxCopy = CascadesContext.newCurrentTreeContext(rootCtx); + Rewriter.getWholeTreeRewriterWithoutCostBasedJobs(rootCtxCopy).execute(); + // Do optimize + new Optimizer(rootCtxCopy).execute(); + return rootCtxCopy.getMemo().getRoot().getLowestCostPlan( + rootCtxCopy.getCurrentJobContext().getRequiredProperties()); + } else { + new Optimizer(cboCtx).execute(); + return cboCtx.getMemo().getRoot().getLowestCostPlan( + cboCtx.getCurrentJobContext().getRequiredProperties()); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java index 36d168f5c1..4f623e5450 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.JobType; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; import java.util.List; import java.util.Objects; @@ -102,7 +103,10 @@ public class PlanTreeRewriteBottomUpJob extends PlanTreeRewriteJob { Plan child = children.get(i); RewriteJobContext childRewriteJobContext = new RewriteJobContext( child, clearedStateContext, i, false); - pushJob(new PlanTreeRewriteBottomUpJob(childRewriteJobContext, context, rules)); + // NOTICE: this relay on pull up cte anchor + if (!(rewriteJobContext.plan instanceof LogicalCTEAnchor)) { + pushJob(new PlanTreeRewriteBottomUpJob(childRewriteJobContext, context, rules)); + } } } @@ -142,7 +146,10 @@ public class PlanTreeRewriteBottomUpJob extends PlanTreeRewriteJob { // we should transform this new plan nodes too. RewriteJobContext childRewriteJobContext = new RewriteJobContext( child, rewriteJobContext, i, false); - pushJob(new PlanTreeRewriteBottomUpJob(childRewriteJobContext, context, rules)); + // NOTICE: this relay on pull up cte anchor + if (!(rewriteJobContext.plan instanceof LogicalCTEAnchor)) { + pushJob(new PlanTreeRewriteBottomUpJob(childRewriteJobContext, context, rules)); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java index 3cb554d45a..d8dba41b37 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.JobType; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; import java.util.List; import java.util.Objects; @@ -59,7 +60,10 @@ public class PlanTreeRewriteTopDownJob extends PlanTreeRewriteJob { for (int i = children.size() - 1; i >= 0; i--) { RewriteJobContext childRewriteJobContext = new RewriteJobContext( children.get(i), newRewriteJobContext, i, false); - pushJob(new PlanTreeRewriteTopDownJob(childRewriteJobContext, context, rules)); + // NOTICE: this relay on pull up cte anchor + if (!(rewriteJobContext.plan instanceof LogicalCTEAnchor)) { + pushJob(new PlanTreeRewriteTopDownJob(childRewriteJobContext, context, rules)); + } } } else { // All the children part are already visited. Just link the children plan to the current node. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java index 8547972cce..23db791e9b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java @@ -50,8 +50,6 @@ public class RootPlanTreeRewriteJob implements RewriteJob { context.getScheduleContext().pushJob(rewriteJob); cascadesContext.getJobScheduler().executeJobPool(cascadesContext); - - cascadesContext.setCurrentRootRewriteJobContext(null); } @Override @@ -72,7 +70,6 @@ public class RootPlanTreeRewriteJob implements RewriteJob { RootRewriteJobContext(Plan plan, boolean childrenVisited, JobContext jobContext) { super(plan, null, -1, childrenVisited); this.jobContext = Objects.requireNonNull(jobContext, "jobContext cannot be null"); - jobContext.getCascadesContext().setCurrentRootRewriteJobContext(this); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java index 643ffe819c..07984ae165 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java @@ -19,9 +19,6 @@ package org.apache.doris.nereids.memo; import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; -import org.apache.doris.nereids.CTEContext; -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.cost.Cost; import org.apache.doris.nereids.cost.CostCalculator; import org.apache.doris.nereids.metrics.EventChannel; @@ -263,17 +260,6 @@ public class Memo { return planWithChildren.withGroupExpression(groupExpression); } - /** - * Utility function to create a new {@link CascadesContext} with this Memo. - */ - public CascadesContext newCascadesContext(StatementContext statementContext) { - return new CascadesContext(null, this, statementContext, PhysicalProperties.ANY); - } - - public CascadesContext newCascadesContext(StatementContext statementContext, CTEContext cteContext) { - return new CascadesContext(null, this, statementContext, cteContext, PhysicalProperties.ANY); - } - /** * init memo by a first plan. * @param plan first plan diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index bb1181e008..696ac16f6f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -161,6 +161,7 @@ import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.OrderExpression; import org.apache.doris.nereids.trees.expressions.Regexp; import org.apache.doris.nereids.trees.expressions.ScalarSubquery; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.Subtract; import org.apache.doris.nereids.trees.expressions.TVFProperties; import org.apache.doris.nereids.trees.expressions.TimestampArithmetic; @@ -244,7 +245,6 @@ import org.apache.doris.nereids.trees.plans.logical.UsingJoin; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.coercion.CharacterType; import org.apache.doris.nereids.util.ExpressionUtils; -import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.policy.FilterType; import org.apache.doris.policy.PolicyTypeEnum; import org.apache.doris.qe.ConnectContext; @@ -335,7 +335,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { @Override public LogicalPlan visitUpdate(UpdateContext ctx) { LogicalPlan query = withCheckPolicy(new UnboundRelation( - RelationUtil.newRelationId(), visitMultipartIdentifier(ctx.tableName))); + StatementScopeIdGenerator.newRelationId(), visitMultipartIdentifier(ctx.tableName))); query = withTableAlias(query, ctx.tableAlias()); if (ctx.fromClause() != null) { query = withRelations(query, ctx.fromClause().relation()); @@ -354,7 +354,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { List tableName = visitMultipartIdentifier(ctx.tableName); List partitions = ctx.partition == null ? ImmutableList.of() : visitIdentifierList(ctx.partition); LogicalPlan query = withTableAlias(withCheckPolicy( - new UnboundRelation(RelationUtil.newRelationId(), tableName)), ctx.tableAlias()); + new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableName)), ctx.tableAlias()); if (ctx.USING() != null) { query = withRelations(query, ctx.relation()); } @@ -582,7 +582,8 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { } LogicalPlan checkedRelation = withCheckPolicy( - new UnboundRelation(RelationUtil.newRelationId(), tableId, partitionNames, isTempPart, relationHints)); + new UnboundRelation(StatementScopeIdGenerator.newRelationId(), + tableId, partitionNames, isTempPart, relationHints)); LogicalPlan plan = withTableAlias(checkedRelation, ctx.tableAlias()); for (LateralViewContext lateralViewContext : ctx.lateralView()) { plan = withGenerate(plan, lateralViewContext); @@ -613,7 +614,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { String value = parseTVFPropertyItem(argument.value); map.put(key, value); } - LogicalPlan relation = new UnboundTVFRelation(RelationUtil.newRelationId(), + LogicalPlan relation = new UnboundTVFRelation(StatementScopeIdGenerator.newRelationId(), functionName, new TVFProperties(map.build())); return withTableAlias(relation, ctx.tableAlias()); }); @@ -1488,7 +1489,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { return ParserUtils.withOrigin(selectCtx, () -> { // fromClause does not exists. List projects = getNamedExpressions(selectCtx.namedExpressionSeq()); - return new UnboundOneRowRelation(RelationUtil.newRelationId(), projects); + return new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(), projects); }); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java index 76fb3311fe..5aab5ab5dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java @@ -26,8 +26,8 @@ import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; @@ -63,7 +63,7 @@ public class RuntimeFilterContext { private final Map> joinToTargetExprId = Maps.newHashMap(); // olap scan node that contains target of a runtime filter. - private final Map> targetOnOlapScanNodeMap = Maps.newHashMap(); + private final Map> targetOnOlapScanNodeMap = Maps.newHashMap(); private final List legacyFilters = Lists.newArrayList(); @@ -157,7 +157,7 @@ public class RuntimeFilterContext { } } - public void setTargetsOnScanNode(ObjectId id, Slot slot) { + public void setTargetsOnScanNode(RelationId id, Slot slot) { this.targetOnOlapScanNodeMap.computeIfAbsent(id, k -> Lists.newArrayList()).add(slot); } @@ -186,7 +186,7 @@ public class RuntimeFilterContext { return targetExprIdToFilter; } - public Map> getTargetOnOlapScanNodeMap() { + public Map> getTargetOnOlapScanNodeMap() { return targetOnOlapScanNodeMap; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java index 537b156b81..1e5575e4ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java @@ -203,7 +203,7 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { ImmutableList.of(bitmapContains.child(1)), type, i, join, isNot, -1L); ctx.addJoinToTargetMap(join, olapScanSlot.getExprId()); ctx.setTargetExprIdToFilter(olapScanSlot.getExprId(), filter); - ctx.setTargetsOnScanNode(aliasTransferMap.get(targetSlot).first.getId(), + ctx.setTargetsOnScanNode(aliasTransferMap.get(targetSlot).first.getRelationId(), olapScanSlot); join.addBitmapRuntimeFilterCondition(bitmapRuntimeFilterCondition); } @@ -322,7 +322,7 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { equalTo.right(), ImmutableList.of(olapScanSlot), type, exprOrder, join, buildSideNdv); ctx.addJoinToTargetMap(join, olapScanSlot.getExprId()); ctx.setTargetExprIdToFilter(olapScanSlot.getExprId(), filter); - ctx.setTargetsOnScanNode(aliasTransferMap.get(unwrappedSlot).first.getId(), olapScanSlot); + ctx.setTargetsOnScanNode(aliasTransferMap.get(unwrappedSlot).first.getRelationId(), olapScanSlot); } } @@ -369,7 +369,7 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { } targetList.add(olapScanSlot); ctx.addJoinToTargetMap(join, olapScanSlot.getExprId()); - ctx.setTargetsOnScanNode(aliasTransferMap.get(origSlot).first.getId(), olapScanSlot); + ctx.setTargetsOnScanNode(aliasTransferMap.get(origSlot).first.getRelationId(), olapScanSlot); } } if (!targetList.isEmpty()) { @@ -612,7 +612,7 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { PhysicalOlapScan scan = entry.getValue(); targetList.add(targetSlot); ctx.addJoinToTargetMap(join, targetSlot.getExprId()); - ctx.setTargetsOnScanNode(scan.getId(), targetSlot); + ctx.setTargetsOnScanNode(scan.getRelationId(), targetSlot); } // build multi-target runtime filter // since always on different join, set the expr_order as 0 diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java index 6534bc715e..eb31e89eb9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java @@ -131,7 +131,7 @@ public class RuntimeFilterPruner extends PlanPostProcessor { @Override public PhysicalRelation visitPhysicalScan(PhysicalRelation scan, CascadesContext context) { RuntimeFilterContext rfCtx = context.getRuntimeFilterContext(); - List slots = rfCtx.getTargetOnOlapScanNodeMap().get(scan.getId()); + List slots = rfCtx.getTargetOnOlapScanNodeMap().get(scan.getRelationId()); if (slots != null) { for (Slot slot : slots) { //if this scan node is the target of any effective RF, it is effective source diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index 98e42eb311..010618276e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -42,8 +42,8 @@ import org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTranspose import org.apache.doris.nereids.rules.implementation.AggregateStrategies; import org.apache.doris.nereids.rules.implementation.LogicalAssertNumRowsToPhysicalAssertNumRows; import org.apache.doris.nereids.rules.implementation.LogicalCTEAnchorToPhysicalCTEAnchor; -import org.apache.doris.nereids.rules.implementation.LogicalCTEConsumeToPhysicalCTEConsume; -import org.apache.doris.nereids.rules.implementation.LogicalCTEProduceToPhysicalCTEProduce; +import org.apache.doris.nereids.rules.implementation.LogicalCTEConsumerToPhysicalCTEConsumer; +import org.apache.doris.nereids.rules.implementation.LogicalCTEProducerToPhysicalCTEProducer; import org.apache.doris.nereids.rules.implementation.LogicalEmptyRelationToPhysicalEmptyRelation; import org.apache.doris.nereids.rules.implementation.LogicalEsScanToPhysicalEsScan; import org.apache.doris.nereids.rules.implementation.LogicalExceptToPhysicalExcept; @@ -76,8 +76,6 @@ import org.apache.doris.nereids.rules.rewrite.MergeProjects; import org.apache.doris.nereids.rules.rewrite.PushdownAliasThroughJoin; import org.apache.doris.nereids.rules.rewrite.PushdownExpressionsInHashCondition; import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughAggregation; -import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughCTE; -import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughCTEAnchor; import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughJoin; import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughProject; import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughRepeat; @@ -85,8 +83,6 @@ import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughSetOperation; import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughSort; import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughWindow; import org.apache.doris.nereids.rules.rewrite.PushdownJoinOtherCondition; -import org.apache.doris.nereids.rules.rewrite.PushdownProjectThroughCTE; -import org.apache.doris.nereids.rules.rewrite.PushdownProjectThroughCTEAnchor; import org.apache.doris.nereids.rules.rewrite.PushdownProjectThroughLimit; import com.google.common.collect.ImmutableList; @@ -133,15 +129,11 @@ public class RuleSet { new MergeFilters(), new MergeGenerates(), new MergeLimits(), - new PushdownFilterThroughCTE(), - new PushdownProjectThroughCTE(), - new PushdownFilterThroughCTEAnchor(), - new PushdownProjectThroughCTEAnchor(), new PushdownAliasThroughJoin()); public static final List IMPLEMENTATION_RULES = planRuleFactories() - .add(new LogicalCTEProduceToPhysicalCTEProduce()) - .add(new LogicalCTEConsumeToPhysicalCTEConsume()) + .add(new LogicalCTEProducerToPhysicalCTEProducer()) + .add(new LogicalCTEConsumerToPhysicalCTEConsumer()) .add(new LogicalCTEAnchorToPhysicalCTEAnchor()) .add(new LogicalRepeatToPhysicalRepeat()) .add(new LogicalFilterToPhysicalFilter()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index c2ac8567cb..3052408ac8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -70,7 +70,7 @@ public enum RuleType { PROJECT_TO_GLOBAL_AGGREGATE(RuleTypeClass.REWRITE), PROJECT_WITH_DISTINCT_TO_AGGREGATE(RuleTypeClass.REWRITE), AVG_DISTINCT_TO_SUM_DIV_COUNT(RuleTypeClass.REWRITE), - REGISTER_CTE(RuleTypeClass.REWRITE), + ANALYZE_CTE(RuleTypeClass.REWRITE), RELATION_AUTHENTICATION(RuleTypeClass.VALIDATION), ADJUST_NULLABLE_FOR_PROJECT_SLOT(RuleTypeClass.REWRITE), @@ -235,15 +235,12 @@ public enum RuleType { // ensure having project on the top join ENSURE_PROJECT_ON_TOP_JOIN(RuleTypeClass.REWRITE), - BUILD_CTE_ANCHOR_AND_CTE_PRODUCER(RuleTypeClass.REWRITE), + PULL_UP_CTE_ANCHOR(RuleTypeClass.REWRITE), + CTE_INLINE(RuleTypeClass.REWRITE), + REWRITE_CTE_CHILDREN(RuleTypeClass.REWRITE), COLLECT_FILTER_ON_CONSUMER(RuleTypeClass.REWRITE), COLLECT_PROJECT_ABOVE_CONSUMER(RuleTypeClass.REWRITE), - COLLECT_PROJECT_ABOVE_FILTER_CONSUMER(RuleTypeClass.REWRITE), - CTE_PRODUCER_REWRITE(RuleTypeClass.REWRITE), - PUSH_DOWN_PROJECT_THROUGH_CTE(RuleTypeClass.REWRITE), - PUSH_DOWN_PROJECT_THROUGH_CTE_ANCHOR(RuleTypeClass.REWRITE), - INLINE_CTE(RuleTypeClass.REWRITE), REWRITE_SENTINEL(RuleTypeClass.REWRITE), // exploration rules @@ -287,8 +284,8 @@ public enum RuleType { LOGICAL_JOIN_TO_NESTED_LOOP_JOIN_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_PROJECT_TO_PHYSICAL_PROJECT_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_FILTER_TO_PHYSICAL_FILTER_RULE(RuleTypeClass.IMPLEMENTATION), - LOGICAL_CTE_PRODUCE_TO_PHYSICAL_CTE_PRODUCER_RULE(RuleTypeClass.IMPLEMENTATION), - LOGICAL_CTE_CONSUME_TO_PHYSICAL_CTE_CONSUMER_RULE(RuleTypeClass.IMPLEMENTATION), + LOGICAL_CTE_PRODUCER_TO_PHYSICAL_CTE_PRODUCER_RULE(RuleTypeClass.IMPLEMENTATION), + LOGICAL_CTE_CONSUMER_TO_PHYSICAL_CTE_CONSUMER_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_CTE_ANCHOR_TO_PHYSICAL_CTE_ANCHOR_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_SORT_TO_PHYSICAL_QUICK_SORT_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_TOP_N_TO_PHYSICAL_TOP_N_RULE(RuleTypeClass.IMPLEMENTATION), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java new file mode 100644 index 0000000000..65d9607a1c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTE.java @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.analysis; + +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.CTEContext; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; + +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Register CTE, includes checking columnAliases, checking CTE name, analyzing each CTE and store the + * analyzed logicalPlan of CTE's query in CTEContext; + * A LogicalProject node will be added to the root of the initial logicalPlan if there exist columnAliases. + * Node LogicalCTE will be eliminated after registering. + */ +public class AnalyzeCTE extends OneAnalysisRuleFactory { + + @Override + public Rule build() { + return logicalCTE().thenApply(ctx -> { + LogicalCTE logicalCTE = ctx.root; + + // step 1. analyzed all cte plan + Pair>> result = analyzeCte(logicalCTE, ctx.cascadesContext); + CascadesContext outerCascadesCtx = CascadesContext.newContextWithCteContext( + ctx.cascadesContext, logicalCTE.child(), result.first); + outerCascadesCtx.newAnalyzer().analyze(); + Plan root = outerCascadesCtx.getRewritePlan(); + // should construct anchor from back to front, because the cte behind depends on the front + for (int i = result.second.size() - 1; i >= 0; i--) { + root = new LogicalCTEAnchor<>(result.second.get(i).getCteId(), result.second.get(i), root); + } + return root; + }).toRule(RuleType.ANALYZE_CTE); + } + + /** + * register and store CTEs in CTEContext + */ + private Pair>> analyzeCte( + LogicalCTE logicalCTE, CascadesContext cascadesContext) { + CTEContext outerCteCtx = cascadesContext.getCteContext(); + List> aliasQueries = logicalCTE.getAliasQueries(); + List> cteProducerPlans = new ArrayList<>(); + for (LogicalSubQueryAlias aliasQuery : aliasQueries) { + String cteName = aliasQuery.getAlias(); + if (outerCteCtx.containsCTE(cteName)) { + throw new AnalysisException("CTE name [" + cteName + "] cannot be used more than once."); + } + + // we should use a chain to ensure visible of cte + CTEContext innerCteCtx = outerCteCtx; + + LogicalPlan parsedCtePlan = (LogicalPlan) aliasQuery.child(); + CascadesContext innerCascadesCtx = CascadesContext.newContextWithCteContext( + cascadesContext, parsedCtePlan, innerCteCtx); + innerCascadesCtx.newAnalyzer().analyze(); + LogicalPlan analyzedCtePlan = (LogicalPlan) innerCascadesCtx.getRewritePlan(); + checkColumnAlias(aliasQuery, analyzedCtePlan.getOutput()); + CTEId cteId = cascadesContext.getStatementContext().getNextCTEId(); + LogicalSubQueryAlias logicalSubQueryAlias = + aliasQuery.withChildren(ImmutableList.of(analyzedCtePlan)); + outerCteCtx = new CTEContext(cteId, logicalSubQueryAlias, outerCteCtx); + outerCteCtx.setAnalyzedPlan(logicalSubQueryAlias); + cteProducerPlans.add(new LogicalCTEProducer<>(cteId, logicalSubQueryAlias)); + } + return Pair.of(outerCteCtx, cteProducerPlans); + } + + /** + * check columnAliases' size and name + */ + private void checkColumnAlias(LogicalSubQueryAlias aliasQuery, List outputSlots) { + if (aliasQuery.getColumnAliases().isPresent()) { + List columnAlias = aliasQuery.getColumnAliases().get(); + // if the size of columnAlias is smaller than outputSlots' size, we will replace the corresponding number + // of front slots with columnAlias. + if (columnAlias.size() > outputSlots.size()) { + throw new AnalysisException("CTE [" + aliasQuery.getAlias() + "] returns " + + columnAlias.size() + " columns, but " + outputSlots.size() + " labels were specified." + + " The number of column labels must be smaller or equal to the number of returned columns."); + } + + Set names = new HashSet<>(); + // column alias cannot be used more than once + columnAlias.forEach(alias -> { + if (names.contains(alias.toLowerCase())) { + throw new AnalysisException("Duplicated CTE column alias:" + + " [" + alias.toLowerCase() + "] in CTE [" + aliasQuery.getAlias() + "]"); + } + names.add(alias); + }); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index bde3f2c8ef..9cb49f1c76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -484,7 +484,7 @@ public class BindExpression implements AnalysisRuleFactory { .map(project -> bindSlot(project, ImmutableList.of(), ctx.cascadesContext)) .map(project -> bindFunction(project, ctx.cascadesContext)) .collect(Collectors.toList()); - return new LogicalOneRowRelation(projects); + return new LogicalOneRowRelation(oneRowRelation.getRelationId(), projects); }) ), RuleType.BINDING_SET_OPERATION_SLOT.build( @@ -508,27 +508,18 @@ public class BindExpression implements AnalysisRuleFactory { } // we need to do cast before set operation, because we maybe use these slot to do shuffle // so, we must cast it before shuffle to get correct hash code. - List> castExpressions = setOperation.collectCastExpressions(); + List> childrenProjections = setOperation.collectChildrenProjections(); ImmutableList.Builder newChildren = ImmutableList.builder(); - for (int i = 0; i < castExpressions.size(); i++) { - if (castExpressions.stream().allMatch(SlotReference.class::isInstance)) { + for (int i = 0; i < childrenProjections.size(); i++) { + if (childrenProjections.stream().allMatch(SlotReference.class::isInstance)) { newChildren.add(setOperation.child(i)); } else { - List projections = castExpressions.get(i).stream() - .map(e -> { - if (e instanceof SlotReference) { - return (SlotReference) e; - } else { - return new Alias(e, e.toSql()); - } - }).collect(ImmutableList.toImmutableList()); - LogicalProject logicalProject = new LogicalProject<>(projections, - setOperation.child(i)); - newChildren.add(logicalProject); + newChildren.add(new LogicalProject<>(childrenProjections.get(i), setOperation.child(i))); } } - List newOutputs = setOperation.buildNewOutputs(castExpressions.get(0)); - return setOperation.withNewOutputs(newOutputs).withChildren(newChildren.build()); + setOperation = (LogicalSetOperation) setOperation.withChildren(newChildren.build()); + List newOutputs = setOperation.buildNewOutputs(); + return setOperation.withNewOutputs(newOutputs); }) ), RuleType.BINDING_GENERATE_SLOT.build( @@ -618,7 +609,6 @@ public class BindExpression implements AnalysisRuleFactory { .collect(Collectors.toList()); } - @SuppressWarnings("unchecked") private E bindSlot(E expr, Plan input, CascadesContext cascadesContext) { return bindSlot(expr, input, cascadesContext, true, true); } @@ -700,7 +690,7 @@ public class BindExpression implements AnalysisRuleFactory { if (!(function instanceof TableValuedFunction)) { throw new AnalysisException(function.toSql() + " is not a TableValuedFunction"); } - return new LogicalTVFRelation(unboundTVFRelation.getId(), (TableValuedFunction) function); + return new LogicalTVFRelation(unboundTVFRelation.getRelationId(), (TableValuedFunction) function); } private void checkSameNameSlot(List childOutputs, String subQueryAlias) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index e23078ce54..a479833ecb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -64,20 +64,19 @@ import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.Optional; import java.util.function.Function; -import javax.annotation.Nullable; /** * Rule to bind relations in query plan. */ public class BindRelation extends OneAnalysisRuleFactory { - private CustomTableResolver customTableResolver; + private final Optional customTableResolver; public BindRelation() { - this(null); + this(Optional.empty()); } - public BindRelation(@Nullable CustomTableResolver customTableResolver) { + public BindRelation(Optional customTableResolver) { this.customTableResolver = customTableResolver; } @@ -122,13 +121,10 @@ public class BindRelation extends OneAnalysisRuleFactory { // check if it is a CTE's name CTEContext cteContext = cascadesContext.getCteContext().findCTEContext(tableName).orElse(null); if (cteContext != null) { - Optional analyzedCte = cteContext.getReuse(tableName); + Optional analyzedCte = cteContext.getAnalyzedCTEPlan(tableName); if (analyzedCte.isPresent()) { - LogicalCTEConsumer logicalCTEConsumer = - new LogicalCTEConsumer(Optional.empty(), Optional.empty(), - analyzedCte.get(), cteContext.getCteId(), tableName); - cascadesContext.putCTEIdToConsumer(logicalCTEConsumer); - return logicalCTEConsumer; + return new LogicalCTEConsumer(unboundRelation.getRelationId(), + cteContext.getCteId(), tableName, analyzedCte.get()); } } List tableQualifier = RelationUtil.getQualifierName(cascadesContext.getConnectContext(), @@ -137,12 +133,14 @@ public class BindRelation extends OneAnalysisRuleFactory { if (cascadesContext.getTables() != null) { table = cascadesContext.getTableByName(tableName); } - if (customTableResolver != null) { - table = customTableResolver.apply(tableQualifier); - } if (table == null) { - // In some cases even if we have already called the "cascadesContext.getTableByName", - // it also gets the null. So, we just check it in the catalog again for safety. + if (customTableResolver.isPresent()) { + table = customTableResolver.get().apply(tableQualifier); + } + } + // In some cases even if we have already called the "cascadesContext.getTableByName", + // it also gets the null. So, we just check it in the catalog again for safety. + if (table == null) { table = RelationUtil.getTable(tableQualifier, cascadesContext.getConnectContext().getEnv()); } @@ -154,9 +152,11 @@ public class BindRelation extends OneAnalysisRuleFactory { List tableQualifier = RelationUtil.getQualifierName(cascadesContext.getConnectContext(), unboundRelation.getNameParts()); TableIf table = null; - if (customTableResolver != null) { - table = customTableResolver.apply(tableQualifier); + if (customTableResolver.isPresent()) { + table = customTableResolver.get().apply(tableQualifier); } + // In some cases even if we have already called the "cascadesContext.getTableByName", + // it also gets the null. So, we just check it in the catalog again for safety. if (table == null) { table = RelationUtil.getTable(tableQualifier, cascadesContext.getConnectContext().getEnv()); } @@ -167,10 +167,10 @@ public class BindRelation extends OneAnalysisRuleFactory { LogicalOlapScan scan; List partIds = getPartitionIds(table, unboundRelation); if (!CollectionUtils.isEmpty(partIds)) { - scan = new LogicalOlapScan(RelationUtil.newRelationId(), + scan = new LogicalOlapScan(unboundRelation.getRelationId(), (OlapTable) table, ImmutableList.of(tableQualifier.get(1)), partIds, unboundRelation.getHints()); } else { - scan = new LogicalOlapScan(RelationUtil.newRelationId(), + scan = new LogicalOlapScan(unboundRelation.getRelationId(), (OlapTable) table, ImmutableList.of(tableQualifier.get(1)), unboundRelation.getHints()); } if (!Util.showHiddenColumns() && scan.getTable().hasDeleteSign() @@ -212,15 +212,16 @@ public class BindRelation extends OneAnalysisRuleFactory { return new LogicalSubQueryAlias<>(tableQualifier, hiveViewPlan); } } - return new LogicalFileScan(RelationUtil.newRelationId(), + return new LogicalFileScan(unboundRelation.getRelationId(), (HMSExternalTable) table, ImmutableList.of(dbName)); case SCHEMA: - return new LogicalSchemaScan(RelationUtil.newRelationId(), table, ImmutableList.of(dbName)); + return new LogicalSchemaScan(unboundRelation.getRelationId(), + table, ImmutableList.of(dbName)); case JDBC_EXTERNAL_TABLE: case JDBC: - return new LogicalJdbcScan(RelationUtil.newRelationId(), table, ImmutableList.of(dbName)); + return new LogicalJdbcScan(unboundRelation.getRelationId(), table, ImmutableList.of(dbName)); case ES_EXTERNAL_TABLE: - return new LogicalEsScan(RelationUtil.newRelationId(), + return new LogicalEsScan(unboundRelation.getRelationId(), (EsExternalTable) table, ImmutableList.of(dbName)); default: throw new AnalysisException("Unsupported tableType:" + table.getType()); @@ -241,7 +242,7 @@ public class BindRelation extends OneAnalysisRuleFactory { private Plan parseAndAnalyzeView(String viewSql, CascadesContext parentContext) { LogicalPlan parsedViewPlan = new NereidsParser().parseSingle(viewSql); - CascadesContext viewContext = CascadesContext.newRewriteContext( + CascadesContext viewContext = CascadesContext.initContext( parentContext.getStatementContext(), parsedViewPlan, PhysicalProperties.ANY); viewContext.newAnalyzer().analyze(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java index 7e7c1e55dd..a30cdfcbdd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java @@ -105,10 +105,14 @@ public class CheckAfterRewrite extends OneAnalysisRuleFactory { .collect(Collectors.toSet()); notFromChildren = removeValidSlotsNotFromChildren(notFromChildren, childrenOutput); if (!notFromChildren.isEmpty()) { - throw new AnalysisException(String.format("Input slot(s) not in child's output: %s in plan: %s", + throw new AnalysisException(String.format("Input slot(s) not in child's output: %s in plan: %s," + + " child output is: %s", StringUtils.join(notFromChildren.stream() - .map(ExpressionTrait::toSql) - .collect(Collectors.toSet()), ", "), plan)); + .map(ExpressionTrait::toString) + .collect(Collectors.toSet()), ", "), plan, + plan.children().stream() + .flatMap(child -> child.getOutput().stream()) + .collect(Collectors.toSet()))); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ColumnAliasGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ColumnAliasGenerator.java index 75e5e0cc5e..2c3fa9cf25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ColumnAliasGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ColumnAliasGenerator.java @@ -21,9 +21,6 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.common.AliasGenerator; -import org.apache.doris.nereids.StatementContext; - -import com.google.common.base.Preconditions; /** * Generate the table name required in the rewrite process. @@ -31,9 +28,7 @@ import com.google.common.base.Preconditions; public class ColumnAliasGenerator extends AliasGenerator { private static final String DEFAULT_COL_ALIAS_PREFIX = "$c$"; - public ColumnAliasGenerator(StatementContext statementContext) { - Preconditions.checkNotNull(statementContext); + public ColumnAliasGenerator() { aliasPrefix = DEFAULT_COL_ALIAS_PREFIX; - usedAliases.addAll(statementContext.getColumnNames()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/LogicalSubQueryAliasToLogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/LogicalSubQueryAliasToLogicalProject.java index b8a0566efb..d37e57a02b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/LogicalSubQueryAliasToLogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/LogicalSubQueryAliasToLogicalProject.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; @@ -29,7 +30,7 @@ import com.google.common.collect.ImmutableList; *

* TODO: refactor group merge strategy to support the feature above */ -public class LogicalSubQueryAliasToLogicalProject extends OneAnalysisRuleFactory { +public class LogicalSubQueryAliasToLogicalProject extends OneRewriteRuleFactory { @Override public Rule build() { return RuleType.LOGICAL_SUB_QUERY_ALIAS_TO_LOGICAL_PROJECT.build( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/RegisterCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/RegisterCTE.java deleted file mode 100644 index 6f90d0544a..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/RegisterCTE.java +++ /dev/null @@ -1,127 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.analysis; - -import org.apache.doris.nereids.CTEContext; -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.expressions.CTEId; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; - -import com.google.common.collect.ImmutableList; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.Callable; - -/** - * Register CTE, includes checking columnAliases, checking CTE name, analyzing each CTE and store the - * analyzed logicalPlan of CTE's query in CTEContext; - * A LogicalProject node will be added to the root of the initial logicalPlan if there exist columnAliases. - * Node LogicalCTE will be eliminated after registering. - */ -public class RegisterCTE extends OneAnalysisRuleFactory { - - @Override - public Rule build() { - return logicalCTE().whenNot(LogicalCTE::isRegistered).thenApply(ctx -> { - LogicalCTE logicalCTE = ctx.root; - List> analyzedCTE = register(logicalCTE, ctx.cascadesContext); - return new LogicalCTE<>(analyzedCTE, logicalCTE.child(), true, - logicalCTE.getCteNameToId()); - }).toRule(RuleType.REGISTER_CTE); - } - - /** - * register and store CTEs in CTEContext - */ - private List> register(LogicalCTE logicalCTE, - CascadesContext cascadesContext) { - CTEContext cteCtx = cascadesContext.getCteContext(); - List> aliasQueries = logicalCTE.getAliasQueries(); - List> analyzedCTE = new ArrayList<>(); - for (LogicalSubQueryAlias aliasQuery : aliasQueries) { - String cteName = aliasQuery.getAlias(); - if (cteCtx.containsCTE(cteName)) { - throw new AnalysisException("CTE name [" + cteName + "] cannot be used more than once."); - } - - // we should use a chain to ensure visible of cte - CTEContext localCteContext = cteCtx; - - LogicalPlan parsedPlan = (LogicalPlan) aliasQuery.child(); - CascadesContext localCascadesContext = CascadesContext.newRewriteContext( - cascadesContext.getStatementContext(), parsedPlan, localCteContext); - localCascadesContext.newAnalyzer().analyze(); - LogicalPlan analyzedCteBody = (LogicalPlan) localCascadesContext.getRewritePlan(); - cascadesContext.putAllCTEIdToConsumer(localCascadesContext.getCteIdToConsumers()); - cascadesContext.putAllCTEIdToCTEClosure(localCascadesContext.getCteIdToCTEClosure()); - if (aliasQuery.getColumnAliases().isPresent()) { - checkColumnAlias(aliasQuery, analyzedCteBody.getOutput()); - } - CTEId cteId = logicalCTE.findCTEId(aliasQuery.getAlias()); - cteCtx = new CTEContext(aliasQuery, localCteContext, cteId); - - LogicalSubQueryAlias logicalSubQueryAlias = - aliasQuery.withChildren(ImmutableList.of(analyzedCteBody)); - cteCtx.setAnalyzedPlan(logicalSubQueryAlias); - Callable cteClosure = () -> { - CascadesContext localCascadesContextInClosure = CascadesContext.newRewriteContext( - cascadesContext.getStatementContext(), aliasQuery, localCteContext); - localCascadesContextInClosure.newAnalyzer().analyze(); - return (LogicalPlan) localCascadesContextInClosure.getRewritePlan(); - }; - cascadesContext.putCTEIdToCTEClosure(cteId, cteClosure); - analyzedCTE.add(logicalSubQueryAlias); - } - cascadesContext.setCteContext(cteCtx); - return analyzedCTE; - } - - /** - * check columnAliases' size and name - */ - private void checkColumnAlias(LogicalSubQueryAlias aliasQuery, List outputSlots) { - List columnAlias = aliasQuery.getColumnAliases().get(); - // if the size of columnAlias is smaller than outputSlots' size, we will replace the corresponding number - // of front slots with columnAlias. - if (columnAlias.size() > outputSlots.size()) { - throw new AnalysisException("CTE [" + aliasQuery.getAlias() + "] returns " + columnAlias.size() - + " columns, but " + outputSlots.size() + " labels were specified. The number of column labels must " - + "be smaller or equal to the number of returned columns."); - } - - Set names = new HashSet<>(); - // column alias cannot be used more than once - columnAlias.forEach(alias -> { - if (names.contains(alias.toLowerCase())) { - throw new AnalysisException("Duplicated CTE column alias: [" + alias.toLowerCase() - + "] in CTE [" + aliasQuery.getAlias() + "]"); - } - names.add(alias); - }); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java index f64ee4a6b7..e5f13007bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java @@ -30,7 +30,6 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; import java.util.List; @@ -70,15 +69,10 @@ class SlotBinder extends SubExprAnalyzer { public Expression visitUnboundAlias(UnboundAlias unboundAlias, CascadesContext context) { Expression child = unboundAlias.child().accept(this, context); if (unboundAlias.getAlias().isPresent()) { - collectColumnNames(unboundAlias.getAlias().get()); return new Alias(child, unboundAlias.getAlias().get()); - } - if (child instanceof NamedExpression) { - collectColumnNames(((NamedExpression) child).getName()); + } else if (child instanceof NamedExpression) { return new Alias(child, ((NamedExpression) child).getName()); } else { - // TODO: resolve aliases - collectColumnNames(child.toSql()); return new Alias(child, child.toSql()); } } @@ -223,11 +217,4 @@ class SlotBinder extends SubExprAnalyzer { + StringUtils.join(nameParts, ".")); }).collect(Collectors.toList()); } - - private void collectColumnNames(String columnName) { - Preconditions.checkNotNull(getCascadesContext()); - if (!getCascadesContext().getStatementContext().getColumnNames().add(columnName)) { - throw new AnalysisException("Collect column name failed, columnName : " + columnName); - } - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java index 20e695109d..9b323b5ba4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java @@ -168,11 +168,11 @@ class SubExprAnalyzer extends DefaultExpressionRewriter { } private AnalyzedResult analyzeSubquery(SubqueryExpr expr) { - CascadesContext subqueryContext = CascadesContext.newRewriteContext(cascadesContext, expr.getQueryPlan()); + CascadesContext subqueryContext = CascadesContext.newContextWithCteContext( + cascadesContext, expr.getQueryPlan(), cascadesContext.getCteContext()); Scope subqueryScope = genScopeWithSubquery(expr); subqueryContext.setOuterScope(subqueryScope); subqueryContext.newAnalyzer().analyze(); - cascadesContext.putAllCTEIdToConsumer(subqueryContext.getCteIdToConsumers()); return new AnalyzedResult((LogicalPlan) subqueryContext.getRewritePlan(), subqueryScope.getCorrelatedSlots()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/UserAuthentication.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/UserAuthentication.java index b88108cbc9..3c97e90476 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/UserAuthentication.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/UserAuthentication.java @@ -23,50 +23,38 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.qe.ConnectContext; -import com.google.common.collect.Sets; - -import java.util.Set; - /** * Check whether a user is permitted to scan specific tables. */ public class UserAuthentication extends OneAnalysisRuleFactory { - Set> relationsToCheck = Sets.newHashSet(LogicalOlapScan.class, LogicalEsScan.class, - LogicalFileScan.class, LogicalSchemaScan.class); @Override public Rule build() { return logicalRelation() - .thenApply(ctx -> checkPermission(ctx.root, ctx.connectContext)) + .when(CatalogRelation.class::isInstance) + .thenApply(ctx -> checkPermission((CatalogRelation) ctx.root, ctx.connectContext)) .toRule(RuleType.RELATION_AUTHENTICATION); } - private Plan checkPermission(LogicalRelation relation, ConnectContext connectContext) { + private Plan checkPermission(CatalogRelation relation, ConnectContext connectContext) { // do not check priv when replaying dump file if (connectContext.getSessionVariable().isPlayNereidsDump()) { - return relation; + return null; } - if (relationsToCheck.contains(relation.getClass())) { - String dbName = - !relation.getQualifier().isEmpty() ? relation.getQualifier().get(0) : null; - String tableName = relation.getTable().getName(); - if (!connectContext.getEnv().getAccessManager().checkTblPriv(connectContext, dbName, - tableName, PrivPredicate.SELECT)) { - String message = ErrorCode.ERR_TABLEACCESS_DENIED_ERROR.formatErrorMsg("SELECT", - ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(), - dbName + ": " + tableName); - throw new AnalysisException(message); - } + String dbName = relation.getDatabase().getFullName(); + String tableName = relation.getTable().getName(); + if (!connectContext.getEnv().getAccessManager().checkTblPriv(connectContext, dbName, + tableName, PrivPredicate.SELECT)) { + String message = ErrorCode.ERR_TABLEACCESS_DENIED_ERROR.formatErrorMsg("SELECT", + ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(), + dbName + ": " + tableName); + throw new AnalysisException(message); } - return relation; + return null; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java index 83aaa362b2..676c079e03 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java @@ -111,7 +111,7 @@ public class ExpressionRewrite implements RewriteRuleFactory { if (projects.equals(newProjects)) { return oneRowRelation; } - return new LogicalOneRowRelation(newProjects); + return new LogicalOneRowRelation(oneRowRelation.getRelationId(), newProjects); }).toRule(RuleType.REWRITE_ONE_ROW_RELATION_EXPRESSION); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalCTEConsumeToPhysicalCTEConsume.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalCTEConsumerToPhysicalCTEConsumer.java similarity index 73% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalCTEConsumeToPhysicalCTEConsume.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalCTEConsumerToPhysicalCTEConsumer.java index 041ec32f15..a17aa0e832 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalCTEConsumeToPhysicalCTEConsume.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalCTEConsumerToPhysicalCTEConsumer.java @@ -24,15 +24,16 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer; /** * Implementation rule that convert logical CTE consumer to physical CTE consumer. */ -public class LogicalCTEConsumeToPhysicalCTEConsume extends OneImplementationRuleFactory { +public class LogicalCTEConsumerToPhysicalCTEConsumer extends OneImplementationRuleFactory { @Override public Rule build() { return logicalCTEConsumer().then(cte -> new PhysicalCTEConsumer( - cte.getCteId(), - cte.getConsumerToProducerOutputMap(), - cte.getProducerToConsumerOutputMap(), - cte.getLogicalProperties() - ) - ).toRule(RuleType.LOGICAL_CTE_CONSUME_TO_PHYSICAL_CTE_CONSUMER_RULE); + cte.getRelationId(), + cte.getCteId(), + cte.getConsumerToProducerOutputMap(), + cte.getProducerToConsumerOutputMap(), + cte.getLogicalProperties() + ) + ).toRule(RuleType.LOGICAL_CTE_CONSUMER_TO_PHYSICAL_CTE_CONSUMER_RULE); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalCTEProduceToPhysicalCTEProduce.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalCTEProducerToPhysicalCTEProducer.java similarity index 86% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalCTEProduceToPhysicalCTEProduce.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalCTEProducerToPhysicalCTEProducer.java index beeab98ed4..bcc48f8d8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalCTEProduceToPhysicalCTEProduce.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalCTEProducerToPhysicalCTEProducer.java @@ -24,14 +24,13 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; /** * Implementation rule that convert logical CTE producer to physical CTE producer. */ -public class LogicalCTEProduceToPhysicalCTEProduce extends OneImplementationRuleFactory { +public class LogicalCTEProducerToPhysicalCTEProducer extends OneImplementationRuleFactory { @Override public Rule build() { - return logicalCTEProducer().then(cte -> new PhysicalCTEProducer( + return logicalCTEProducer().then(cte -> new PhysicalCTEProducer<>( cte.getCteId(), - cte.getProjects(), cte.getLogicalProperties(), cte.child()) - ).toRule(RuleType.LOGICAL_CTE_PRODUCE_TO_PHYSICAL_CTE_PRODUCER_RULE); + ).toRule(RuleType.LOGICAL_CTE_PRODUCER_TO_PHYSICAL_CTE_PRODUCER_RULE); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEmptyRelationToPhysicalEmptyRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEmptyRelationToPhysicalEmptyRelation.java index b48d07a4a7..c3513ed5ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEmptyRelationToPhysicalEmptyRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEmptyRelationToPhysicalEmptyRelation.java @@ -28,7 +28,8 @@ public class LogicalEmptyRelationToPhysicalEmptyRelation extends OneImplementati @Override public Rule build() { return logicalEmptyRelation() - .then(relation -> new PhysicalEmptyRelation(relation.getProjects(), relation.getLogicalProperties())) + .then(relation -> new PhysicalEmptyRelation(relation.getRelationId(), + relation.getProjects(), relation.getLogicalProperties())) .toRule(RuleType.LOGICAL_EMPTY_RELATION_TO_PHYSICAL_EMPTY_RELATION_RULE); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEsScanToPhysicalEsScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEsScanToPhysicalEsScan.java index 74ad0cc356..1e4db2eba1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEsScanToPhysicalEsScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEsScanToPhysicalEsScan.java @@ -32,7 +32,7 @@ public class LogicalEsScanToPhysicalEsScan extends OneImplementationRuleFactory public Rule build() { return logicalEsScan().then(esScan -> new PhysicalEsScan( - esScan.getId(), + esScan.getRelationId(), esScan.getTable(), esScan.getQualifier(), DistributionSpecAny.INSTANCE, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java index 08e9e9e503..c78d8174ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java @@ -32,7 +32,7 @@ public class LogicalFileScanToPhysicalFileScan extends OneImplementationRuleFact public Rule build() { return logicalFileScan().then(fileScan -> new PhysicalFileScan( - fileScan.getId(), + fileScan.getRelationId(), fileScan.getTable(), fileScan.getQualifier(), DistributionSpecAny.INSTANCE, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJdbcScanToPhysicalJdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJdbcScanToPhysicalJdbcScan.java index 3bff1699ba..99120c2b2e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJdbcScanToPhysicalJdbcScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJdbcScanToPhysicalJdbcScan.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.rules.implementation; -import org.apache.doris.nereids.properties.DistributionSpecAny; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan; @@ -32,10 +31,9 @@ public class LogicalJdbcScanToPhysicalJdbcScan extends OneImplementationRuleFact public Rule build() { return logicalJdbcScan().then(jdbcScan -> new PhysicalJdbcScan( - jdbcScan.getId(), + jdbcScan.getRelationId(), jdbcScan.getTable(), jdbcScan.getQualifier(), - DistributionSpecAny.INSTANCE, Optional.empty(), jdbcScan.getLogicalProperties()) ).toRule(RuleType.LOGICAL_JDBC_SCAN_TO_PHYSICAL_JDBC_SCAN_RULE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java index 12399722af..b578247d06 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java @@ -50,7 +50,7 @@ public class LogicalOlapScanToPhysicalOlapScan extends OneImplementationRuleFact public Rule build() { return logicalOlapScan().then(olapScan -> new PhysicalOlapScan( - olapScan.getId(), + olapScan.getRelationId(), olapScan.getTable(), olapScan.getQualifier(), olapScan.getSelectedIndexId(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOneRowRelationToPhysicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOneRowRelationToPhysicalOneRowRelation.java index 0d007d6984..62cd2b705b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOneRowRelationToPhysicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOneRowRelationToPhysicalOneRowRelation.java @@ -28,7 +28,7 @@ public class LogicalOneRowRelationToPhysicalOneRowRelation extends OneImplementa @Override public Rule build() { return logicalOneRowRelation() - .then(relation -> new PhysicalOneRowRelation( + .then(relation -> new PhysicalOneRowRelation(relation.getRelationId(), relation.getProjects(), relation.getLogicalProperties())) .toRule(RuleType.LOGICAL_ONE_ROW_RELATION_TO_PHYSICAL_ONE_ROW_RELATION); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalSchemaScanToPhysicalSchemaScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalSchemaScanToPhysicalSchemaScan.java index cb4832067a..df459fcd45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalSchemaScanToPhysicalSchemaScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalSchemaScanToPhysicalSchemaScan.java @@ -30,7 +30,7 @@ public class LogicalSchemaScanToPhysicalSchemaScan extends OneImplementationRule @Override public Rule build() { return logicalSchemaScan().then(scan -> - new PhysicalSchemaScan(scan.getId(), + new PhysicalSchemaScan(scan.getRelationId(), scan.getTable(), scan.getQualifier(), Optional.empty(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalTVFRelationToPhysicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalTVFRelationToPhysicalTVFRelation.java index 6b3099f83c..4828f77d5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalTVFRelationToPhysicalTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalTVFRelationToPhysicalTVFRelation.java @@ -28,7 +28,7 @@ public class LogicalTVFRelationToPhysicalTVFRelation extends OneImplementationRu @Override public Rule build() { return logicalTVFRelation() - .then(relation -> new PhysicalTVFRelation(relation.getId(), + .then(relation -> new PhysicalTVFRelation(relation.getRelationId(), relation.getFunction(), relation.getLogicalProperties())) .toRule(RuleType.LOGICAL_TVF_RELATION_TO_PHYSICAL_TVF_RELATION); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AddDefaultLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AddDefaultLimit.java similarity index 72% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AddDefaultLimit.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AddDefaultLimit.java index 47cb8469dd..99904a70ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AddDefaultLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AddDefaultLimit.java @@ -15,16 +15,16 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.rules.analysis; +package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.analyzer.UnboundOlapTableSink; import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.trees.plans.LimitPhase; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; @@ -53,27 +53,35 @@ public class AddDefaultLimit extends DefaultPlanRewriter imple return plan; } + // should add limit under anchor to keep optimize opportunity @Override - public LogicalPlan visitLogicalLimit(LogicalLimit limit, StatementContext context) { - return limit; - } - - @Override - public LogicalPlan visitLogicalCTE(LogicalCTE cte, StatementContext context) { - Plan child = cte.child().accept(this, context); - return ((LogicalPlan) cte.withChildren(child)); + public Plan visitLogicalCTEAnchor(LogicalCTEAnchor cteAnchor, + StatementContext context) { + return cteAnchor.withChildren(cteAnchor.child(0), cteAnchor.child(1)); } // we should keep that sink node is the top node of the plan tree. // currently, it's one of the olap table sink and file sink. @Override - public LogicalPlan visitUnboundOlapTableSink(UnboundOlapTableSink sink, StatementContext context) { - Plan child = sink.child().accept(this, context); - return ((LogicalPlan) sink.withChildren(child)); + public Plan visitLogicalOlapTableSink(LogicalOlapTableSink olapTableSink, + StatementContext context) { + Plan child = olapTableSink.child().accept(this, context); + return olapTableSink.withChildren(child); } @Override - public LogicalPlan visitLogicalSort(LogicalSort sort, StatementContext context) { + public Plan visitLogicalFileSink(LogicalFileSink fileSink, StatementContext context) { + Plan child = fileSink.child().accept(this, context); + return fileSink.withChildren(child); + } + + @Override + public Plan visitLogicalLimit(LogicalLimit limit, StatementContext context) { + return limit; + } + + @Override + public Plan visitLogicalSort(LogicalSort sort, StatementContext context) { ConnectContext ctx = context.getConnectContext(); if (ctx != null) { long defaultLimit = ctx.getSessionVariable().defaultOrderByLimit; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java index d708d6226c..eb907dcafe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.OrderExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; @@ -30,15 +31,18 @@ import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; @@ -47,7 +51,9 @@ import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -57,70 +63,68 @@ import java.util.stream.Collectors; * because some rule could change output's nullable. * So, we need add a rule to adjust all expression's nullable attribute after rewrite. */ -public class AdjustNullable extends DefaultPlanRewriter implements CustomRewriter { +public class AdjustNullable extends DefaultPlanRewriter> implements CustomRewriter { @Override public Plan rewriteRoot(Plan plan, JobContext jobContext) { - return plan.accept(this, null); + return plan.accept(this, Maps.newHashMap()); } @Override - public Plan visit(Plan plan, Void context) { - LogicalPlan logicalPlan = (LogicalPlan) super.visit(plan, context); - return logicalPlan.recomputeLogicalProperties(); + public Plan visit(Plan plan, Map replaceMap) { + LogicalPlan logicalPlan = (LogicalPlan) super.visit(plan, replaceMap); + logicalPlan = logicalPlan.recomputeLogicalProperties(); + logicalPlan.getOutputSet().forEach(s -> replaceMap.put(s.getExprId(), s)); + return logicalPlan; } @Override - public Plan visitLogicalAggregate(LogicalAggregate aggregate, Void context) { - aggregate = (LogicalAggregate) super.visit(aggregate, context); - Map exprIdSlotMap = collectChildrenOutputMap(aggregate); + public Plan visitLogicalAggregate(LogicalAggregate aggregate, Map replaceMap) { + aggregate = (LogicalAggregate) super.visit(aggregate, replaceMap); List newOutputs - = updateExpressions(aggregate.getOutputExpressions(), exprIdSlotMap); + = updateExpressions(aggregate.getOutputExpressions(), replaceMap); List newGroupExpressions - = updateExpressions(aggregate.getGroupByExpressions(), exprIdSlotMap); + = updateExpressions(aggregate.getGroupByExpressions(), replaceMap); + newOutputs.forEach(o -> replaceMap.put(o.getExprId(), o.toSlot())); return aggregate.withGroupByAndOutput(newGroupExpressions, newOutputs); } @Override - public Plan visitLogicalFilter(LogicalFilter filter, Void context) { - filter = (LogicalFilter) super.visit(filter, context); - Map exprIdSlotMap = collectChildrenOutputMap(filter); - Set conjuncts = updateExpressions(filter.getConjuncts(), exprIdSlotMap); + public Plan visitLogicalFilter(LogicalFilter filter, Map replaceMap) { + filter = (LogicalFilter) super.visit(filter, replaceMap); + Set conjuncts = updateExpressions(filter.getConjuncts(), replaceMap); return filter.withConjuncts(conjuncts).recomputeLogicalProperties(); } @Override - public Plan visitLogicalGenerate(LogicalGenerate generate, Void context) { - generate = (LogicalGenerate) super.visit(generate, context); - Map exprIdSlotMap = collectChildrenOutputMap(generate); - List newGenerators = updateExpressions(generate.getGenerators(), exprIdSlotMap); - return generate.withGenerators(newGenerators).recomputeLogicalProperties(); + public Plan visitLogicalGenerate(LogicalGenerate generate, Map replaceMap) { + generate = (LogicalGenerate) super.visit(generate, replaceMap); + List newGenerators = updateExpressions(generate.getGenerators(), replaceMap); + Plan newGenerate = generate.withGenerators(newGenerators).recomputeLogicalProperties(); + newGenerate.getOutputSet().forEach(o -> replaceMap.put(o.getExprId(), o)); + return newGenerate; } @Override - public Plan visitLogicalJoin(LogicalJoin join, Void context) { - join = (LogicalJoin) super.visit(join, context); - Map exprIdSlotMap = collectChildrenOutputMap(join); - List hashConjuncts = updateExpressions(join.getHashJoinConjuncts(), exprIdSlotMap); - // because other join compute on join's output on be, so we need to change slot to join's output - exprIdSlotMap = join.getOutputSet().stream() - .collect(Collectors.toMap(NamedExpression::getExprId, s -> s)); - List otherConjuncts = updateExpressions(join.getOtherJoinConjuncts(), exprIdSlotMap); + public Plan visitLogicalJoin(LogicalJoin join, Map replaceMap) { + join = (LogicalJoin) super.visit(join, replaceMap); + List hashConjuncts = updateExpressions(join.getHashJoinConjuncts(), replaceMap); + join.getOutputSet().forEach(o -> replaceMap.put(o.getExprId(), o)); + List otherConjuncts = updateExpressions(join.getOtherJoinConjuncts(), replaceMap); return join.withJoinConjuncts(hashConjuncts, otherConjuncts).recomputeLogicalProperties(); } @Override - public Plan visitLogicalProject(LogicalProject project, Void context) { - project = (LogicalProject) super.visit(project, context); - Map exprIdSlotMap = collectChildrenOutputMap(project); - List newProjects = updateExpressions(project.getProjects(), exprIdSlotMap); + public Plan visitLogicalProject(LogicalProject project, Map replaceMap) { + project = (LogicalProject) super.visit(project, replaceMap); + List newProjects = updateExpressions(project.getProjects(), replaceMap); + newProjects.forEach(p -> replaceMap.put(p.getExprId(), p.toSlot())); return project.withProjects(newProjects); } @Override - public Plan visitLogicalRepeat(LogicalRepeat repeat, Void context) { - repeat = (LogicalRepeat) super.visit(repeat, context); - Map exprIdSlotMap = collectChildrenOutputMap(repeat); + public Plan visitLogicalRepeat(LogicalRepeat repeat, Map replaceMap) { + repeat = (LogicalRepeat) super.visit(repeat, replaceMap); Set flattenGroupingSetExpr = ImmutableSet.copyOf( ExpressionUtils.flatExpressions(repeat.getGroupingSets())); List newOutputs = Lists.newArrayList(); @@ -128,15 +132,16 @@ public class AdjustNullable extends DefaultPlanRewriter implements CustomR if (flattenGroupingSetExpr.contains(output)) { newOutputs.add(output); } else { - newOutputs.add(updateExpression(output, exprIdSlotMap)); + newOutputs.add(updateExpression(output, replaceMap)); } } + newOutputs.forEach(o -> replaceMap.put(o.getExprId(), o.toSlot())); return repeat.withGroupSetsAndOutput(repeat.getGroupingSets(), newOutputs).recomputeLogicalProperties(); } @Override - public Plan visitLogicalSetOperation(LogicalSetOperation setOperation, Void context) { - setOperation = (LogicalSetOperation) super.visit(setOperation, context); + public Plan visitLogicalSetOperation(LogicalSetOperation setOperation, Map replaceMap) { + setOperation = (LogicalSetOperation) super.visit(setOperation, replaceMap); if (setOperation.children().isEmpty()) { return setOperation; } @@ -150,6 +155,16 @@ public class AdjustNullable extends DefaultPlanRewriter implements CustomR } } } + if (setOperation instanceof LogicalUnion) { + LogicalUnion logicalUnion = (LogicalUnion) setOperation; + for (List constantExprs : logicalUnion.getConstantExprsList()) { + for (int j = 0; j < constantExprs.size(); j++) { + if (constantExprs.get(j).nullable()) { + inputNullable.set(j, true); + } + } + } + } List outputs = setOperation.getOutputs(); List newOutputs = Lists.newArrayListWithCapacity(outputs.size()); for (int i = 0; i < inputNullable.size(); i++) { @@ -160,48 +175,71 @@ public class AdjustNullable extends DefaultPlanRewriter implements CustomR } newOutputs.add(ne instanceof Alias ? (NamedExpression) ne.withChildren(slot) : slot); } + newOutputs.forEach(o -> replaceMap.put(o.getExprId(), o.toSlot())); return setOperation.withNewOutputs(newOutputs).recomputeLogicalProperties(); } @Override - public Plan visitLogicalSort(LogicalSort sort, Void context) { - sort = (LogicalSort) super.visit(sort, context); - Map exprIdSlotMap = collectChildrenOutputMap(sort); + public Plan visitLogicalSort(LogicalSort sort, Map replaceMap) { + sort = (LogicalSort) super.visit(sort, replaceMap); List newKeys = sort.getOrderKeys().stream() - .map(old -> old.withExpression(updateExpression(old.getExpr(), exprIdSlotMap))) + .map(old -> old.withExpression(updateExpression(old.getExpr(), replaceMap))) .collect(ImmutableList.toImmutableList()); return sort.withOrderKeys(newKeys).recomputeLogicalProperties(); } @Override - public Plan visitLogicalTopN(LogicalTopN topN, Void context) { - topN = (LogicalTopN) super.visit(topN, context); - Map exprIdSlotMap = collectChildrenOutputMap(topN); + public Plan visitLogicalTopN(LogicalTopN topN, Map replaceMap) { + topN = (LogicalTopN) super.visit(topN, replaceMap); List newKeys = topN.getOrderKeys().stream() - .map(old -> old.withExpression(updateExpression(old.getExpr(), exprIdSlotMap))) + .map(old -> old.withExpression(updateExpression(old.getExpr(), replaceMap))) .collect(ImmutableList.toImmutableList()); return topN.withOrderKeys(newKeys).recomputeLogicalProperties(); } @Override - public Plan visitLogicalWindow(LogicalWindow window, Void context) { - window = (LogicalWindow) super.visit(window, context); - Map exprIdSlotMap = collectChildrenOutputMap(window); + public Plan visitLogicalWindow(LogicalWindow window, Map replaceMap) { + window = (LogicalWindow) super.visit(window, replaceMap); List windowExpressions = - updateExpressions(window.getWindowExpressions(), exprIdSlotMap); + updateExpressions(window.getWindowExpressions(), replaceMap); + windowExpressions.forEach(w -> replaceMap.put(w.getExprId(), w.toSlot())); return window.withExpression(windowExpressions, window.child()); } - private T updateExpression(T input, Map exprIdSlotMap) { - return (T) input.rewriteDownShortCircuit(e -> e.accept(SlotReferenceReplacer.INSTANCE, exprIdSlotMap)); + @Override + public Plan visitLogicalPartitionTopN(LogicalPartitionTopN partitionTopN, + Map replaceMap) { + partitionTopN = (LogicalPartitionTopN) super.visit(partitionTopN, replaceMap); + List partitionKeys = updateExpressions(partitionTopN.getPartitionKeys(), replaceMap); + List orderKeys = updateExpressions(partitionTopN.getOrderKeys(), replaceMap); + return partitionTopN.withPartitionKeysAndOrderKeys(partitionKeys, orderKeys); } - private List updateExpressions(List inputs, Map exprIdSlotMap) { - return inputs.stream().map(i -> updateExpression(i, exprIdSlotMap)).collect(ImmutableList.toImmutableList()); + @Override + public Plan visitLogicalCTEConsumer(LogicalCTEConsumer cteConsumer, Map replaceMap) { + Map consumerToProducerOutputMap = new LinkedHashMap<>(); + Map producerToConsumerOutputMap = new LinkedHashMap<>(); + for (Slot producerOutputSlot : cteConsumer.getConsumerToProducerOutputMap().values()) { + Slot newProducerOutputSlot = updateExpression(producerOutputSlot, replaceMap); + Slot newConsumerOutputSlot = cteConsumer.getProducerToConsumerOutputMap().get(producerOutputSlot) + .withNullable(newProducerOutputSlot.nullable()); + producerToConsumerOutputMap.put(newProducerOutputSlot, newConsumerOutputSlot); + consumerToProducerOutputMap.put(newConsumerOutputSlot, newProducerOutputSlot); + replaceMap.put(newConsumerOutputSlot.getExprId(), newConsumerOutputSlot); + } + return cteConsumer.withTwoMaps(consumerToProducerOutputMap, producerToConsumerOutputMap); } - private Set updateExpressions(Set inputs, Map exprIdSlotMap) { - return inputs.stream().map(i -> updateExpression(i, exprIdSlotMap)).collect(ImmutableSet.toImmutableSet()); + private T updateExpression(T input, Map replaceMap) { + return (T) input.rewriteDownShortCircuit(e -> e.accept(SlotReferenceReplacer.INSTANCE, replaceMap)); + } + + private List updateExpressions(List inputs, Map replaceMap) { + return inputs.stream().map(i -> updateExpression(i, replaceMap)).collect(ImmutableList.toImmutableList()); + } + + private Set updateExpressions(Set inputs, Map replaceMap) { + return inputs.stream().map(i -> updateExpression(i, replaceMap)).collect(ImmutableSet.toImmutableSet()); } private Map collectChildrenOutputMap(LogicalPlan plan) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AggScalarSubQueryToWindowFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AggScalarSubQueryToWindowFunction.java index d0e5742487..9f588d4c5d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AggScalarSubQueryToWindowFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AggScalarSubQueryToWindowFunction.java @@ -32,6 +32,7 @@ import org.apache.doris.nereids.trees.expressions.functions.window.SupportWindow import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -248,11 +249,11 @@ public class AggScalarSubQueryToWindowFunction extends DefaultPlanRewriter correlatedSlots) { - List outerTables = outerPlans.stream().filter(LogicalRelation.class::isInstance) - .map(LogicalRelation.class::cast) + List outerTables = outerPlans.stream().filter(CatalogRelation.class::isInstance) + .map(CatalogRelation.class::cast) .collect(Collectors.toList()); - List innerTables = innerPlans.stream().filter(LogicalRelation.class::isInstance) - .map(LogicalRelation.class::cast) + List innerTables = innerPlans.stream().filter(CatalogRelation.class::isInstance) + .map(CatalogRelation.class::cast) .collect(Collectors.toList()); List outerIds = outerTables.stream().map(node -> node.getTable().getId()).collect(Collectors.toList()); @@ -273,15 +274,16 @@ public class AggScalarSubQueryToWindowFunction extends DefaultPlanRewriter correlatedRelationOutput = outerTables.stream() .filter(node -> outerIds.contains(node.getTable().getId())) + .map(LogicalRelation.class::cast) .map(LogicalRelation::getOutputExprIdSet).flatMap(Collection::stream).collect(Collectors.toSet()); return ExpressionUtils.collect(correlatedSlots, NamedExpression.class::isInstance).stream() .map(NamedExpression.class::cast) .allMatch(e -> correlatedRelationOutput.contains(e.getExprId())); } - private void createSlotMapping(List outerTables, List innerTables) { - for (LogicalRelation outerTable : outerTables) { - for (LogicalRelation innerTable : innerTables) { + private void createSlotMapping(List outerTables, List innerTables) { + for (CatalogRelation outerTable : outerTables) { + for (CatalogRelation innerTable : innerTables) { if (innerTable.getTable().getId() == outerTable.getTable().getId()) { for (Slot innerSlot : innerTable.getOutput()) { for (Slot outerSlot : outerTable.getOutput()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/BuildCTEAnchorAndCTEProducer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/BuildCTEAnchorAndCTEProducer.java deleted file mode 100644 index 11700ecb77..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/BuildCTEAnchorAndCTEProducer.java +++ /dev/null @@ -1,67 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.expressions.CTEId; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; -import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; -import org.apache.doris.qe.ConnectContext; - -/** - * BuildCTEAnchorAndCTEProducer. - */ -public class BuildCTEAnchorAndCTEProducer extends OneRewriteRuleFactory { - - @Override - public Rule build() { - return logicalCTE().thenApply(ctx -> { - return rewrite(ctx.root, ctx.cascadesContext); - }).toRule(RuleType.BUILD_CTE_ANCHOR_AND_CTE_PRODUCER); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - private LogicalPlan rewrite(LogicalPlan p, CascadesContext cascadesContext) { - if (!(p instanceof LogicalCTE)) { - return p; - } - LogicalCTE logicalCTE = (LogicalCTE) p; - LogicalPlan child = (LogicalPlan) logicalCTE.child(); - if (!(child instanceof LogicalEmptyRelation)) { - for (int i = logicalCTE.getAliasQueries().size() - 1; i >= 0; i--) { - LogicalSubQueryAlias s = (LogicalSubQueryAlias) logicalCTE.getAliasQueries().get(i); - CTEId id = logicalCTE.findCTEId(s.getAlias()); - if (cascadesContext.cteReferencedCount(id) - <= ConnectContext.get().getSessionVariable().inlineCTEReferencedThreshold - || !ConnectContext.get().getSessionVariable().getEnablePipelineEngine()) { - continue; - } - LogicalCTEProducer logicalCTEProducer = new LogicalCTEProducer( - rewrite((LogicalPlan) s.child(), cascadesContext), id); - child = new LogicalCTEAnchor(logicalCTEProducer, child, id); - } - } - return child; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEInline.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEInline.java new file mode 100644 index 0000000000..46216ec043 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEInline.java @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.trees.copier.DeepCopierContext; +import org.apache.doris.nereids.trees.copier.LogicalPlanDeepCopier; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import java.util.List; + +/** + * pull up LogicalCteAnchor to the top of plan to avoid CteAnchor break other rewrite rules pattern + * The front producer may depend on the back producer in {@code List>} + * After this rule, we normalize all CteAnchor in plan, all CteAnchor under CteProducer should pull out + * and put all of them to the top of plan depends on dependency tree of them. + */ +public class CTEInline extends DefaultPlanRewriter> implements CustomRewriter { + + @Override + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + Plan root = plan.accept(this, null); + // collect cte id to consumer + root.foreach(p -> { + if (p instanceof LogicalCTEConsumer) { + jobContext.getCascadesContext().putCTEIdToConsumer(((LogicalCTEConsumer) p)); + } + }); + return root; + } + + @Override + public Plan visitLogicalCTEAnchor(LogicalCTEAnchor cteAnchor, + LogicalCTEProducer producer) { + if (producer != null) { + // process upper anchor + List children = cteAnchor.children().stream() + .map(c -> c.accept(this, producer)) + .collect(ImmutableList.toImmutableList()); + return cteAnchor.withChildren(children); + } else { + // process this anchor + List consumers = cteAnchor.child(1).collectToList(p -> { + if (p instanceof LogicalCTEConsumer) { + return ((LogicalCTEConsumer) p).getCteId().equals(cteAnchor.getCteId()); + } + return false; + }); + if (ConnectContext.get().getSessionVariable().getEnablePipelineEngine() + && ConnectContext.get().getSessionVariable().enableCTEMaterialize + && consumers.size() > ConnectContext.get().getSessionVariable().inlineCTEReferencedThreshold) { + // not inline + Plan right = cteAnchor.right().accept(this, null); + return cteAnchor.withChildren(cteAnchor.left(), right); + } else { + // should inline + Plan root = cteAnchor.right().accept(this, (LogicalCTEProducer) cteAnchor.left()); + // process child + return root.accept(this, null); + } + } + } + + @Override + public Plan visitLogicalCTEConsumer(LogicalCTEConsumer cteConsumer, LogicalCTEProducer producer) { + if (producer != null && cteConsumer.getCteId().equals(producer.getCteId())) { + DeepCopierContext deepCopierContext = new DeepCopierContext(); + Plan inlinedPlan = LogicalPlanDeepCopier.INSTANCE + .deepCopy((LogicalPlan) producer.child(), deepCopierContext); + List projects = Lists.newArrayList(); + for (Slot consumerSlot : cteConsumer.getOutput()) { + Slot producerSlot = cteConsumer.getProducerSlot(consumerSlot); + ExprId inlineExprId = deepCopierContext.exprIdReplaceMap.get(producerSlot.getExprId()); + Alias alias = new Alias(consumerSlot.getExprId(), producerSlot.withExprId(inlineExprId), + consumerSlot.getName()); + projects.add(alias); + } + return new LogicalProject<>(projects, inlinedPlan); + } + return cteConsumer; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEProducerRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEProducerRewrite.java deleted file mode 100644 index ee197c429c..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEProducerRewrite.java +++ /dev/null @@ -1,122 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.jobs.executor.Rewriter; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.expressions.CTEId; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.nereids.util.ExpressionUtils; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import org.apache.commons.collections.CollectionUtils; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Map.Entry; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Rewrite CTE Producer recursively. - */ -public class CTEProducerRewrite extends OneRewriteRuleFactory { - - @Override - public Rule build() { - return logicalCTEProducer().when(p -> !p.isRewritten()).thenApply(ctx -> { - LogicalCTEProducer cteProducer = ctx.root; - Set projects = ctx.cascadesContext.getProjectForProducer(cteProducer.getCteId()); - LogicalPlan child = tryToConstructFilter(ctx.cascadesContext, cteProducer.getCteId(), - (LogicalPlan) ctx.root.child()); - if (CollectionUtils.isNotEmpty(projects) - && ctx.cascadesContext.couldPruneColumnOnProducer(cteProducer.getCteId())) { - child = new LogicalProject(ImmutableList.copyOf(projects), child); - } - CascadesContext rewrittenCtx = ctx.cascadesContext.forkForCTEProducer(child); - Rewriter rewriter = new Rewriter(rewrittenCtx); - rewriter.execute(); - return cteProducer.withChildrenAndProjects(ImmutableList.of(rewrittenCtx.getRewritePlan()), - new ArrayList<>(child.getOutput()), true); - }).toRule(RuleType.CTE_PRODUCER_REWRITE); - } - - /* - * An expression can only be pushed down if it has filter expressions on all consumers that reference the slot. - * For example, let's assume a producer has two consumers, consumer1 and consumer2: - * - * filter(a > 5 and b < 1) -> consumer1 - * filter(a < 8) -> consumer2 - * - * In this case, the only expression that can be pushed down to the producer is filter(a > 5 or a < 8). - */ - private LogicalPlan tryToConstructFilter(CascadesContext cascadesContext, CTEId cteId, LogicalPlan child) { - Set consumerIds = cascadesContext.getCteIdToConsumers().get(cteId).stream() - .map(LogicalCTEConsumer::getConsumerId) - .collect(Collectors.toSet()); - Set> filtersAboveEachConsumer = cascadesContext.getConsumerIdToFilters().entrySet().stream() - .filter(kv -> consumerIds.contains(kv.getKey())) - .map(Entry::getValue) - .collect(Collectors.toSet()); - Set someone = filtersAboveEachConsumer.stream().findFirst().orElse(null); - if (someone == null) { - return child; - } - int filterSize = cascadesContext.getCteIdToConsumers().get(cteId).size(); - Set filter = new HashSet<>(); - for (Expression f : someone) { - int matchCount = 1; - Set slots = f.collect(e -> e instanceof SlotReference); - Set mightBeJoined = new HashSet<>(); - for (Set another : filtersAboveEachConsumer) { - if (another.equals(someone)) { - continue; - } - Set matched = new HashSet<>(); - for (Expression e : another) { - Set otherSlots = e.collect(ae -> ae instanceof SlotReference); - if (otherSlots.equals(slots)) { - matched.add(e); - } - } - if (!matched.isEmpty()) { - matchCount++; - } - mightBeJoined.addAll(matched); - } - if (matchCount >= filterSize) { - mightBeJoined.add(f); - filter.add(ExpressionUtils.or(mightBeJoined)); - } - } - if (!filter.isEmpty()) { - return new LogicalFilter(ImmutableSet.of(ExpressionUtils.and(filter)), child); - } - return child; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckMatchExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckMatchExpression.java index 3b04059a48..dbfe480baf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckMatchExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckMatchExpression.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Match; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import java.util.List; @@ -41,15 +42,15 @@ public class CheckMatchExpression extends OneRewriteRuleFactory { .toRule(RuleType.CHECK_MATCH_EXPRESSION); } - private LogicalFilter checkChildren(LogicalFilter filter) { + private Plan checkChildren(LogicalFilter filter) { List expressions = filter.getExpressions(); for (Expression expr : expressions) { if (expr instanceof Match) { Match matchExpression = (Match) expr; if (!(matchExpression.left() instanceof SlotReference) || !(matchExpression.right() instanceof Literal)) { - throw new AnalysisException(String.format( - "Only support match left operand is SlotRef, right operand is Literal")); + throw new AnalysisException(String.format("Only support match left operand is SlotRef," + + " right operand is Literal. But meet expression %s", matchExpression)); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectFilterAboveConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectFilterAboveConsumer.java index 6646a59c2a..4c4249b69c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectFilterAboveConsumer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectFilterAboveConsumer.java @@ -40,11 +40,11 @@ public class CollectFilterAboveConsumer extends OneRewriteRuleFactory { for (Expression expr : exprs) { Expression rewrittenExpr = expr.rewriteUp(e -> { if (e instanceof Slot) { - return cteConsumer.findProducerSlot((Slot) e); + return cteConsumer.getProducerSlot((Slot) e); } return e; }); - ctx.cascadesContext.putConsumerIdToFilter(cteConsumer.getConsumerId(), rewrittenExpr); + ctx.cascadesContext.putConsumerIdToFilter(cteConsumer.getRelationId(), rewrittenExpr); } return ctx.root; }).toRule(RuleType.COLLECT_FILTER_ON_CONSUMER); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectProjectAboveConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectProjectAboveConsumer.java index e7f85ca79e..43f8231cbc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectProjectAboveConsumer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectProjectAboveConsumer.java @@ -48,8 +48,8 @@ public class CollectProjectAboveConsumer implements RewriteRuleFactory { collectProject(ctx.cascadesContext, namedExpressions, cteConsumer); return ctx.root; })), - RuleType.COLLECT_PROJECT_ABOVE_FILTER_CONSUMER.build(logicalProject(logicalFilter(logicalCTEConsumer())) - .thenApply(ctx -> { + RuleType.COLLECT_PROJECT_ABOVE_FILTER_CONSUMER + .build(logicalProject(logicalFilter(logicalCTEConsumer())).thenApply(ctx -> { LogicalProject> project = ctx.root; LogicalFilter filter = project.child(); Set filterSlots = filter.getInputSlots(); @@ -72,7 +72,7 @@ public class CollectProjectAboveConsumer implements RewriteRuleFactory { if (!(node instanceof Slot)) { return; } - Slot slot = cteConsumer.findProducerSlot((Slot) node); + Slot slot = cteConsumer.getProducerSlot((Slot) node); ctx.putCTEIdToProject(cteConsumer.getCteId(), slot); ctx.markConsumerUnderProject(cteConsumer); }); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java index 7f3205d8f2..2fc17a717d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -161,10 +162,15 @@ public class ColumnPruning extends DefaultPlanRewriter implements } @Override - public Plan visitLogicalOlapTableSink(LogicalOlapTableSink olapTableSink, PruneContext context) { + public Plan visitLogicalOlapTableSink(LogicalOlapTableSink olapTableSink, PruneContext context) { return skipPruneThisAndFirstLevelChildren(olapTableSink); } + @Override + public Plan visitLogicalFileSink(LogicalFileSink fileSink, PruneContext context) { + return skipPruneThisAndFirstLevelChildren(fileSink); + } + // the backend not support filter(project(agg)), so we can not prune the key set in the agg, // only prune the agg functions here @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java index e3d3fb48e1..5f5d8b39a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -36,11 +37,13 @@ public class EliminateFilter extends OneRewriteRuleFactory { public Rule build() { return logicalFilter() .when(filter -> filter.getConjuncts().stream().anyMatch(BooleanLiteral.class::isInstance)) - .then(filter -> { + .thenApply(ctx -> { + LogicalFilter filter = ctx.root; Set newConjuncts = Sets.newHashSetWithExpectedSize(filter.getConjuncts().size()); for (Expression expression : filter.getConjuncts()) { if (expression == BooleanLiteral.FALSE) { - return new LogicalEmptyRelation(filter.getOutput()); + return new LogicalEmptyRelation(ctx.statementContext.getNextRelationId(), + filter.getOutput()); } else if (expression != BooleanLiteral.TRUE) { newConjuncts.add(expression); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateLimit.java index 0190086256..9cc19e47d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateLimit.java @@ -29,7 +29,8 @@ public class EliminateLimit extends OneRewriteRuleFactory { public Rule build() { return logicalLimit() .when(limit -> limit.getLimit() == 0) - .then(limit -> new LogicalEmptyRelation(limit.getOutput())) + .thenApply(ctx -> new LogicalEmptyRelation(ctx.statementContext.getNextRelationId(), + ctx.root.getOutput())) .toRule(RuleType.ELIMINATE_LIMIT); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProject.java index 9b0a0de522..adb355f94d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProject.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.annotation.DependsRules; import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -56,7 +57,7 @@ public class EliminateUnnecessaryProject implements CustomRewriter { private Plan rewriteProject(LogicalProject project, boolean outputSavePoint) { if (project.child() instanceof LogicalEmptyRelation) { // eliminate unnecessary project - return new LogicalEmptyRelation(project.getProjects()); + return new LogicalEmptyRelation(StatementScopeIdGenerator.newRelationId(), project.getProjects()); } else if (project.canEliminate() && outputSavePoint && project.getOutputSet().equals(project.child().getOutputSet())) { // eliminate unnecessary project diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InApplyToJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InApplyToJoin.java index 261316e83e..64d8defcfe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InApplyToJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InApplyToJoin.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; @@ -93,7 +94,9 @@ public class InApplyToJoin extends OneRewriteRuleFactory { //in-predicate to equal Expression predicate; Expression left = ((InSubquery) apply.getSubqueryExpr()).getCompareExpr(); - Expression right = apply.getSubqueryExpr().getSubqueryOutput(); + // TODO: trick here, because when deep copy logical plan the apply right child + // is not same with query plan in subquery expr, since the scan node copy twice + Expression right = apply.getSubqueryExpr().getSubqueryOutput((LogicalPlan) apply.right()); if (apply.isCorrelated()) { predicate = ExpressionUtils.and(new EqualTo(left, right), apply.getCorrelationFilter().get()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InlineCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InlineCTE.java deleted file mode 100644 index a50f42a2c0..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InlineCTE.java +++ /dev/null @@ -1,73 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.expressions.Alias; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.qe.ConnectContext; - -import java.util.ArrayList; -import java.util.List; - -/** - * A CTEConsumer would be converted to a inlined plan if corresponding CTE referenced less than or - * equal inline_cte_referenced_threshold (it's a session variable, by default is 1). - */ -public class InlineCTE extends OneRewriteRuleFactory { - - private static final int INLINE_CTE_REFERENCED_THRESHOLD = 1; - - @Override - public Rule build() { - return logicalCTEConsumer().thenApply(ctx -> { - LogicalCTEConsumer cteConsumer = ctx.root; - int refCount = ctx.cascadesContext.cteReferencedCount(cteConsumer.getCteId()); - /* - * Current we only implement CTE Materialize on pipeline engine and only materialize those CTE whose - * refCount > NereidsRewriter.INLINE_CTE_REFERENCED_THRESHOLD. - */ - if (ConnectContext.get().getSessionVariable().getEnablePipelineEngine() - && ConnectContext.get().getSessionVariable().enableCTEMaterialize - && refCount > INLINE_CTE_REFERENCED_THRESHOLD) { - return cteConsumer; - } - LogicalPlan inlinedPlan = ctx.cascadesContext.findCTEPlanForInline(cteConsumer.getCteId()); - List inlinedPlanOutput = inlinedPlan.getOutput(); - List cteConsumerOutput = cteConsumer.getOutput(); - List projects = new ArrayList<>(); - for (Slot inlineSlot : inlinedPlanOutput) { - String name = inlineSlot.getName(); - for (Slot consumerSlot : cteConsumerOutput) { - if (consumerSlot.getName().equals(name)) { - Alias alias = new Alias(consumerSlot.getExprId(), inlineSlot, name); - projects.add(alias); - break; - } - } - } - return new LogicalProject<>(projects, - inlinedPlan); - }).toRule(RuleType.INLINE_CTE); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpCteAnchor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpCteAnchor.java new file mode 100644 index 0000000000..027a20b3db --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpCteAnchor.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; + +import com.google.common.collect.Lists; + +import java.util.List; + +/** + * pull up LogicalCteAnchor to the top of plan to avoid CteAnchor break other rewrite rules pattern + * The front producer may depend on the back producer in {@code List>} + * After this rule, we normalize all CteAnchor in plan, all CteAnchor under CteProducer should pull out + * and put all of them to the top of plan depends on dependency tree of them. + */ +public class PullUpCteAnchor extends DefaultPlanRewriter>> implements CustomRewriter { + + @Override + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + List> producers = Lists.newArrayList(); + return rewriteRoot(plan, producers); + } + + private Plan rewriteRoot(Plan plan, List> producers) { + Plan root = plan.accept(this, producers); + for (LogicalCTEProducer producer : producers) { + root = new LogicalCTEAnchor<>(producer.getCteId(), producer, root); + } + return root; + } + + @Override + public Plan visitLogicalCTEAnchor(LogicalCTEAnchor cteAnchor, + List> producers) { + // 1. process child side + Plan root = cteAnchor.child(1).accept(this, producers); + // 2. process producers side, need to collect all producer + cteAnchor.child(0).accept(this, producers); + return root; + } + + @Override + public LogicalCTEProducer visitLogicalCTEProducer(LogicalCTEProducer cteProducer, + List> producers) { + List> childProducers = Lists.newArrayList(); + Plan child = cteProducer.child().accept(this, childProducers); + LogicalCTEProducer newProducer = (LogicalCTEProducer) cteProducer.withChildren(child); + // because current producer relay on it child's producers, so add current producer first. + producers.add(newProducer); + producers.addAll(childProducers); + return newProducer; + } + + // we should keep that sink node is the top node of the plan tree. + // currently, it's one of the olap table sink and file sink. + @Override + public Plan visitLogicalOlapTableSink(LogicalOlapTableSink olapTableSink, + List> producers) { + return olapTableSink.withChildren(rewriteRoot(olapTableSink.child(), producers)); + } + + @Override + public Plan visitLogicalFileSink(LogicalFileSink fileSink, + List> producers) { + return fileSink.withChildren(rewriteRoot(fileSink.child(), producers)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughAggregation.java index e1f4457067..217834c4bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughAggregation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughAggregation.java @@ -83,9 +83,12 @@ public class PushdownFilterThroughAggregation extends OneRewriteRuleFactory { */ public static Set getCanPushDownSlots(LogicalAggregate aggregate) { Set canPushDownSlots = new HashSet<>(); - if (aggregate.hasRepeat()) { + if (aggregate.getSourceRepeat().isPresent()) { // When there is a repeat, the push-down condition is consistent with the repeat - canPushDownSlots.addAll(aggregate.getSourceRepeat().get().getCommonGroupingSetExpressions()); + aggregate.getSourceRepeat().get().getCommonGroupingSetExpressions().stream() + .filter(Slot.class::isInstance) + .map(Slot.class::cast) + .forEach(canPushDownSlots::add); } else { for (Expression groupByExpression : aggregate.getGroupByExpressions()) { if (groupByExpression instanceof Slot) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughCTE.java deleted file mode 100644 index 47dc5468bc..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughCTE.java +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; - -/** - * Push filter through CTE. - */ -public class PushdownFilterThroughCTE extends OneRewriteRuleFactory { - - @Override - public Rule build() { - return logicalFilter(logicalCTE()).thenApply(ctx -> { - LogicalFilter> filter = ctx.root; - LogicalCTE anchor = filter.child(); - return anchor.withChildren(filter.withChildren(anchor.child())); - }).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_CTE); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughCTEAnchor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughCTEAnchor.java deleted file mode 100644 index 2942e3654e..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughCTEAnchor.java +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; - -/** - * Push filter through CTEAnchor. - */ -public class PushdownFilterThroughCTEAnchor extends OneRewriteRuleFactory { - - @Override - public Rule build() { - return logicalFilter(logicalCTEAnchor()).thenApply(ctx -> { - LogicalFilter> filter = ctx.root; - LogicalCTEAnchor anchor = filter.child(); - return anchor.withChildren(anchor.left(), filter.withChildren((Plan) anchor.right())); - }).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_CTE_ANCHOR); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindow.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindow.java index 61ba62f8ac..0addaeac0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindow.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindow.java @@ -31,6 +31,7 @@ import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; @@ -77,7 +78,8 @@ public class PushdownFilterThroughWindow extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalFilter(logicalWindow()).then(filter -> { + return logicalFilter(logicalWindow()).thenApply(ctx -> { + LogicalFilter> filter = ctx.root; LogicalWindow window = filter.child(); // We have already done such optimization rule, so just ignore it. @@ -117,7 +119,7 @@ public class PushdownFilterThroughWindow extends OneRewriteRuleFactory { limitVal--; } if (limitVal < 0) { - return new LogicalEmptyRelation(filter.getOutput()); + return new LogicalEmptyRelation(ctx.statementContext.getNextRelationId(), filter.getOutput()); } if (hasPartitionLimit) { partitionLimit = Math.min(partitionLimit, limitVal); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownLimit.java index 8bc140467d..48e2665121 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownLimit.java @@ -20,6 +20,7 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.UnaryNode; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Limit; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; @@ -134,7 +135,8 @@ public class PushdownLimit implements RewriteRuleFactory { }).toRule(RuleType.PUSH_LIMIT_INTO_SORT), logicalLimit(logicalOneRowRelation()) .then(limit -> limit.getLimit() > 0 && limit.getOffset() == 0 - ? limit.child() : new LogicalEmptyRelation(limit.child().getOutput())) + ? limit.child() : new LogicalEmptyRelation(StatementScopeIdGenerator.newRelationId(), + limit.child().getOutput())) .toRule(RuleType.ELIMINATE_LIMIT_ON_ONE_ROW_RELATION), logicalLimit(logicalEmptyRelation()) .then(UnaryNode::child) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownProjectThroughCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownProjectThroughCTE.java deleted file mode 100644 index 9db0b5adc4..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownProjectThroughCTE.java +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; - -/** - * Push project through CTE. - */ -public class PushdownProjectThroughCTE extends OneRewriteRuleFactory { - - @Override - public Rule build() { - return logicalProject(logicalCTE()).thenApply(ctx -> { - LogicalProject> project = ctx.root; - LogicalCTE anchor = project.child(); - return anchor.withChildren(project.withChildren(anchor.child())); - }).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_CTE); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownProjectThroughCTEAnchor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownProjectThroughCTEAnchor.java deleted file mode 100644 index d92499043d..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownProjectThroughCTEAnchor.java +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; - -/** - * Push project through CTEAnchor. - */ -public class PushdownProjectThroughCTEAnchor extends OneRewriteRuleFactory { - - @Override - public Rule build() { - return logicalProject(logicalCTEAnchor()).thenApply(ctx -> { - LogicalProject> project = ctx.root; - LogicalCTEAnchor anchor = project.child(); - return anchor.withChildren(anchor.child(0), project.withChildren(anchor.child(1))); - }).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_CTE_ANCHOR); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java index 5e14405191..a9e685b1c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java @@ -87,7 +87,7 @@ public class ReorderJoin extends OneRewriteRuleFactory { Plan plan = joinToMultiJoin(filter, planToHintType); Preconditions.checkState(plan instanceof MultiJoin); MultiJoin multiJoin = (MultiJoin) plan; - ctx.statementContext.setMaxNArayInnerJoin(multiJoin.children().size()); + ctx.statementContext.setMaxNAryInnerJoin(multiJoin.children().size()); Plan after = multiJoinToJoin(multiJoin, planToHintType); return after; }).toRule(RuleType.REORDER_JOIN); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java new file mode 100644 index 0000000000..d88ef62e31 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java @@ -0,0 +1,189 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.annotation.DependsRules; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.jobs.executor.Rewriter; +import org.apache.doris.nereids.jobs.rewrite.RewriteJob; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.apache.commons.collections.CollectionUtils; + +import java.util.HashSet; +import java.util.List; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * rewrite CteAnchor consumer side and producer side recursively， all CteAnchor must at top of the plan + */ +@DependsRules({PullUpCteAnchor.class, CTEInline.class}) +public class RewriteCteChildren extends DefaultPlanRewriter implements CustomRewriter { + + private final List jobs; + + public RewriteCteChildren(List jobs) { + this.jobs = jobs; + } + + @Override + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + return plan.accept(this, jobContext.getCascadesContext()); + } + + @Override + public Plan visit(Plan plan, CascadesContext context) { + Rewriter.getCteChildrenRewriter(context, jobs).execute(); + return context.getRewritePlan(); + } + + @Override + public Plan visitLogicalCTEAnchor(LogicalCTEAnchor cteAnchor, + CascadesContext cascadesContext) { + LogicalPlan outer; + if (cascadesContext.getStatementContext().getRewrittenCtePlan().containsKey(null)) { + outer = cascadesContext.getStatementContext().getRewrittenCtePlan().get(null); + } else { + CascadesContext outerCascadesCtx = CascadesContext.newSubtreeContext( + Optional.empty(), cascadesContext, cteAnchor.child(1), + cascadesContext.getCurrentJobContext().getRequiredProperties()); + outer = (LogicalPlan) cteAnchor.child(1).accept(this, outerCascadesCtx); + cascadesContext.getStatementContext().getRewrittenCtePlan().put(null, outer); + } + boolean reserveAnchor = outer.anyMatch(p -> { + if (p instanceof LogicalCTEConsumer) { + LogicalCTEConsumer logicalCTEConsumer = (LogicalCTEConsumer) p; + return logicalCTEConsumer.getCteId().equals(cteAnchor.getCteId()); + } + return false; + }); + if (!reserveAnchor) { + return outer; + } + Plan producer = cteAnchor.child(0).accept(this, cascadesContext); + return cteAnchor.withChildren(producer, outer); + } + + @Override + public Plan visitLogicalCTEProducer(LogicalCTEProducer cteProducer, + CascadesContext cascadesContext) { + LogicalPlan child; + if (cascadesContext.getStatementContext().getRewrittenCtePlan().containsKey(cteProducer.getCteId())) { + child = cascadesContext.getStatementContext().getRewrittenCtePlan().get(cteProducer.getCteId()); + } else { + child = (LogicalPlan) cteProducer.child(); + child = tryToConstructFilter(cascadesContext, cteProducer.getCteId(), child); + Set projects = cascadesContext.getProjectForProducer(cteProducer.getCteId()); + if (CollectionUtils.isNotEmpty(projects) + && cascadesContext.couldPruneColumnOnProducer(cteProducer.getCteId())) { + child = new LogicalProject<>(ImmutableList.copyOf(projects), child); + child = pushPlanUnderAnchor(child); + } + CascadesContext rewrittenCtx = CascadesContext.newSubtreeContext( + Optional.of(cteProducer.getCteId()), cascadesContext, child, PhysicalProperties.ANY); + child = (LogicalPlan) child.accept(this, rewrittenCtx); + cascadesContext.getStatementContext().getRewrittenCtePlan().put(cteProducer.getCteId(), child); + } + return cteProducer.withChildren(child); + } + + private LogicalPlan pushPlanUnderAnchor(LogicalPlan plan) { + if (plan.child(0) instanceof LogicalCTEAnchor) { + LogicalPlan child = (LogicalPlan) plan.withChildren(plan.child(0).child(1)); + return (LogicalPlan) plan.child(0).withChildren( + plan.child(0).child(0), pushPlanUnderAnchor(child)); + } + return plan; + } + + /* + * An expression can only be pushed down if it has filter expressions on all consumers that reference the slot. + * For example, let's assume a producer has two consumers, consumer1 and consumer2: + * + * filter(a > 5 and b < 1) -> consumer1 + * filter(a < 8) -> consumer2 + * + * In this case, the only expression that can be pushed down to the producer is filter(a > 5 or a < 8). + */ + private LogicalPlan tryToConstructFilter(CascadesContext cascadesContext, CTEId cteId, LogicalPlan child) { + Set consumerIds = cascadesContext.getCteIdToConsumers().get(cteId).stream() + .map(LogicalCTEConsumer::getRelationId) + .collect(Collectors.toSet()); + Set> filtersAboveEachConsumer = cascadesContext.getConsumerIdToFilters().entrySet().stream() + .filter(kv -> consumerIds.contains(kv.getKey())) + .map(Entry::getValue) + .collect(Collectors.toSet()); + Set someone = filtersAboveEachConsumer.stream().findFirst().orElse(null); + if (someone == null) { + return child; + } + int filterSize = cascadesContext.getCteIdToConsumers().get(cteId).size(); + Set conjuncts = new HashSet<>(); + for (Expression f : someone) { + int matchCount = 1; + Set slots = f.collect(e -> e instanceof SlotReference); + Set mightBeJoined = new HashSet<>(); + for (Set another : filtersAboveEachConsumer) { + if (another.equals(someone)) { + continue; + } + Set matched = new HashSet<>(); + for (Expression e : another) { + Set otherSlots = e.collect(ae -> ae instanceof SlotReference); + if (otherSlots.equals(slots)) { + matched.add(e); + } + } + if (!matched.isEmpty()) { + matchCount++; + } + mightBeJoined.addAll(matched); + } + if (matchCount >= filterSize) { + mightBeJoined.add(f); + conjuncts.add(ExpressionUtils.or(mightBeJoined)); + } + } + if (!conjuncts.isEmpty()) { + LogicalPlan filter = new LogicalFilter<>(ImmutableSet.of(ExpressionUtils.and(conjuncts)), child); + return pushPlanUnderAnchor(filter); + } + return child; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index 570df7a807..a52a22d92b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -39,6 +39,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunctio import org.apache.doris.nereids.trees.expressions.functions.window.Rank; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.algebra.EmptyRelation; import org.apache.doris.nereids.trees.plans.algebra.Filter; import org.apache.doris.nereids.trees.plans.algebra.Generate; @@ -46,7 +47,6 @@ import org.apache.doris.nereids.trees.plans.algebra.Limit; import org.apache.doris.nereids.trees.plans.algebra.PartitionTopN; import org.apache.doris.nereids.trees.plans.algebra.Project; import org.apache.doris.nereids.trees.plans.algebra.Repeat; -import org.apache.doris.nereids.trees.plans.algebra.Scan; import org.apache.doris.nereids.trees.plans.algebra.SetOperation; import org.apache.doris.nereids.trees.plans.algebra.TopN; import org.apache.doris.nereids.trees.plans.algebra.Union; @@ -214,7 +214,7 @@ public class StatsCalculator extends DefaultPlanVisitor { // For unit test only public static void estimate(GroupExpression groupExpression, CascadesContext context) { StatsCalculator statsCalculator = new StatsCalculator(groupExpression, false, - new HashMap<>(), false, Collections.EMPTY_MAP, context); + new HashMap<>(), false, Collections.emptyMap(), context); statsCalculator.estimate(); } @@ -287,18 +287,18 @@ public class StatsCalculator extends DefaultPlanVisitor { @Override public Statistics visitLogicalOlapScan(LogicalOlapScan olapScan, Void context) { - return computeScan(olapScan); + return computeCatalogRelation(olapScan); } @Override public Statistics visitLogicalSchemaScan(LogicalSchemaScan schemaScan, Void context) { - return computeScan(schemaScan); + return computeCatalogRelation(schemaScan); } @Override public Statistics visitLogicalFileScan(LogicalFileScan fileScan, Void context) { fileScan.getExpressions(); - return computeScan(fileScan); + return computeCatalogRelation(fileScan); } @Override @@ -309,13 +309,13 @@ public class StatsCalculator extends DefaultPlanVisitor { @Override public Statistics visitLogicalJdbcScan(LogicalJdbcScan jdbcScan, Void context) { jdbcScan.getExpressions(); - return computeScan(jdbcScan); + return computeCatalogRelation(jdbcScan); } @Override public Statistics visitLogicalEsScan(LogicalEsScan esScan, Void context) { esScan.getExpressions(); - return computeScan(esScan); + return computeCatalogRelation(esScan); } @Override @@ -419,17 +419,17 @@ public class StatsCalculator extends DefaultPlanVisitor { @Override public Statistics visitPhysicalOlapScan(PhysicalOlapScan olapScan, Void context) { - return computeScan(olapScan); + return computeCatalogRelation(olapScan); } @Override public Statistics visitPhysicalSchemaScan(PhysicalSchemaScan schemaScan, Void context) { - return computeScan(schemaScan); + return computeCatalogRelation(schemaScan); } @Override public Statistics visitPhysicalFileScan(PhysicalFileScan fileScan, Void context) { - return computeScan(fileScan); + return computeCatalogRelation(fileScan); } @Override @@ -445,12 +445,12 @@ public class StatsCalculator extends DefaultPlanVisitor { @Override public Statistics visitPhysicalJdbcScan(PhysicalJdbcScan jdbcScan, Void context) { - return computeScan(jdbcScan); + return computeCatalogRelation(jdbcScan); } @Override public Statistics visitPhysicalEsScan(PhysicalEsScan esScan, Void context) { - return computeScan(esScan); + return computeCatalogRelation(esScan); } @Override @@ -585,12 +585,12 @@ public class StatsCalculator extends DefaultPlanVisitor { // TODO: 1. Subtract the pruned partition // 2. Consider the influence of runtime filter // 3. Get NDV and column data size from StatisticManger, StatisticManager doesn't support it now. - private Statistics computeScan(Scan scan) { - Set slotSet = scan.getOutput().stream().filter(SlotReference.class::isInstance) + private Statistics computeCatalogRelation(CatalogRelation catalogRelation) { + Set slotSet = catalogRelation.getOutput().stream().filter(SlotReference.class::isInstance) .map(s -> (SlotReference) s).collect(Collectors.toSet()); Map columnStatisticMap = new HashMap<>(); - TableIf table = scan.getTable(); - double rowCount = scan.getTable().estimatedRowCount(); + TableIf table = catalogRelation.getTable(); + double rowCount = catalogRelation.getTable().estimatedRowCount(); for (SlotReference slotReference : slotSet) { String colName = slotReference.getName(); if (colName == null) { @@ -1022,7 +1022,7 @@ public class StatsCalculator extends DefaultPlanVisitor { Preconditions.checkArgument(prodStats != null, String.format("Stats for CTE: %s not found", cteId)); Statistics consumerStats = new Statistics(prodStats.getRowCount(), new HashMap<>()); for (Slot slot : cteConsumer.getOutput()) { - Slot prodSlot = cteConsumer.findProducerSlot(slot); + Slot prodSlot = cteConsumer.getProducerSlot(slot); ColumnStatistic colStats = prodStats.columnStatistics().get(prodSlot); if (colStats == null) { continue; @@ -1058,7 +1058,7 @@ public class StatsCalculator extends DefaultPlanVisitor { Preconditions.checkArgument(prodStats != null, String.format("Stats for CTE: %s not found", cteId)); Statistics consumerStats = new Statistics(prodStats.getRowCount(), new HashMap<>()); for (Slot slot : cteConsumer.getOutput()) { - Slot prodSlot = cteConsumer.findProducerSlot(slot); + Slot prodSlot = cteConsumer.getProducerSlot(slot); ColumnStatistic colStats = prodStats.columnStatistics().get(prodSlot); if (colStats == null) { continue; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/DeepCopierContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/DeepCopierContext.java new file mode 100644 index 0000000000..c4772fb7e8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/DeepCopierContext.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.copier; + +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; + +import com.google.common.collect.Maps; + +import java.util.Map; + +/** + * context info used in LogicalPlan deep copy + */ +public class DeepCopierContext { + /** + * the original SlotReference to new SlotReference map + */ + public final Map exprIdReplaceMap = Maps.newHashMap(); + /** + * because LogicalApply keep original plan in itself and its right child in the meantime + * so, we must use exact same output (same ExprIds) relations between the two plan tree + * to ensure they keep same after deep copy + */ + private final Map relationReplaceMap = Maps.newHashMap(); + + public void putRelation(RelationId relationId, LogicalRelation newRelation) { + relationReplaceMap.put(relationId, newRelation); + } + + public Map getRelationReplaceMap() { + return relationReplaceMap; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/ExpressionDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/ExpressionDeepCopier.java new file mode 100644 index 0000000000..0038b9e4cc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/ExpressionDeepCopier.java @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.copier; + +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Exists; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.InSubquery; +import org.apache.doris.nereids.trees.expressions.ListQuery; +import org.apache.doris.nereids.trees.expressions.ScalarSubquery; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * deep copy expression, generate new expr id for SlotReference and Alias. + */ +public class ExpressionDeepCopier extends DefaultExpressionRewriter { + + public static ExpressionDeepCopier INSTANCE = new ExpressionDeepCopier(); + + public Expression deepCopy(Expression expression, DeepCopierContext context) { + return expression.accept(this, context); + } + + @Override + public Expression visitAlias(Alias alias, DeepCopierContext context) { + Expression child = alias.child().accept(this, context); + Map exprIdReplaceMap = context.exprIdReplaceMap; + Alias newOne; + if (exprIdReplaceMap.containsKey(alias.getExprId())) { + // NOTICE: because we do not do normalize agg, so we could get same Alias in more than one place + // so, if we already copy this Alias once, we must use the existed ExprId for this Alias. + newOne = new Alias(exprIdReplaceMap.get(alias.getExprId()), child, alias.getName()); + } else { + newOne = new Alias(child, alias.getName()); + exprIdReplaceMap.put(alias.getExprId(), newOne.getExprId()); + } + return newOne; + } + + @Override + public Expression visitSlotReference(SlotReference slotReference, DeepCopierContext context) { + Map exprIdReplaceMap = context.exprIdReplaceMap; + if (exprIdReplaceMap.containsKey(slotReference.getExprId())) { + ExprId newExprId = exprIdReplaceMap.get(slotReference.getExprId()); + return slotReference.withExprId(newExprId); + } else { + SlotReference newOne = new SlotReference(slotReference.getName(), slotReference.getDataType(), + slotReference.nullable(), slotReference.getQualifier()); + exprIdReplaceMap.put(slotReference.getExprId(), newOne.getExprId()); + return newOne; + } + } + + @Override + public Expression visitExistsSubquery(Exists exists, DeepCopierContext context) { + LogicalPlan logicalPlan = LogicalPlanDeepCopier.INSTANCE.deepCopy(exists.getQueryPlan(), context); + List correlateSlots = exists.getCorrelateSlots().stream() + .map(s -> (Slot) s.accept(this, context)) + .collect(Collectors.toList()); + Optional typeCoercionExpr = exists.getTypeCoercionExpr() + .map(c -> c.accept(this, context)); + return new Exists(logicalPlan, correlateSlots, typeCoercionExpr, exists.isNot()); + } + + @Override + public Expression visitListQuery(ListQuery listQuery, DeepCopierContext context) { + LogicalPlan logicalPlan = LogicalPlanDeepCopier.INSTANCE.deepCopy(listQuery.getQueryPlan(), context); + List correlateSlots = listQuery.getCorrelateSlots().stream() + .map(s -> (Slot) s.accept(this, context)) + .collect(Collectors.toList()); + Optional typeCoercionExpr = listQuery.getTypeCoercionExpr() + .map(c -> c.accept(this, context)); + return new ListQuery(logicalPlan, correlateSlots, typeCoercionExpr); + } + + @Override + public Expression visitInSubquery(InSubquery in, DeepCopierContext context) { + Expression compareExpr = in.getCompareExpr().accept(this, context); + List correlateSlots = in.getCorrelateSlots().stream() + .map(s -> (Slot) s.accept(this, context)) + .collect(Collectors.toList()); + Optional typeCoercionExpr = in.getTypeCoercionExpr() + .map(c -> c.accept(this, context)); + ListQuery listQuery = (ListQuery) in.getListQuery().accept(this, context); + return new InSubquery(compareExpr, listQuery, correlateSlots, typeCoercionExpr, in.isNot()); + } + + @Override + public Expression visitScalarSubquery(ScalarSubquery scalar, DeepCopierContext context) { + LogicalPlan logicalPlan = LogicalPlanDeepCopier.INSTANCE.deepCopy(scalar.getQueryPlan(), context); + List correlateSlots = scalar.getCorrelateSlots().stream() + .map(s -> (Slot) s.accept(this, context)) + .collect(Collectors.toList()); + Optional typeCoercionExpr = scalar.getTypeCoercionExpr() + .map(c -> c.accept(this, context)); + return new ScalarSubquery(logicalPlan, correlateSlots, typeCoercionExpr); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java new file mode 100644 index 0000000000..9da4690390 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java @@ -0,0 +1,429 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.copier; + +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.expressions.SubqueryExpr; +import org.apache.doris.nereids.trees.expressions.functions.Function; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalApply; +import org.apache.doris.nereids.trees.plans.logical.LogicalAssertNumRows; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; +import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; +import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; +import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalSort; +import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * deep copy a plan + */ +public class LogicalPlanDeepCopier extends DefaultPlanRewriter { + + public static LogicalPlanDeepCopier INSTANCE = new LogicalPlanDeepCopier(); + + public LogicalPlan deepCopy(LogicalPlan plan, DeepCopierContext context) { + return (LogicalPlan) plan.accept(this, context); + } + + @Override + public Plan visitLogicalEmptyRelation(LogicalEmptyRelation emptyRelation, DeepCopierContext context) { + List newProjects = emptyRelation.getProjects().stream() + .map(p -> (NamedExpression) ExpressionDeepCopier.INSTANCE.deepCopy(p, context)) + .collect(ImmutableList.toImmutableList()); + return new LogicalEmptyRelation(StatementScopeIdGenerator.newRelationId(), newProjects); + } + + @Override + public Plan visitLogicalOneRowRelation(LogicalOneRowRelation oneRowRelation, DeepCopierContext context) { + List newProjects = oneRowRelation.getProjects().stream() + .map(p -> (NamedExpression) ExpressionDeepCopier.INSTANCE.deepCopy(p, context)) + .collect(ImmutableList.toImmutableList()); + return new LogicalOneRowRelation(StatementScopeIdGenerator.newRelationId(), newProjects); + } + + @Override + public Plan visitLogicalApply(LogicalApply apply, DeepCopierContext context) { + Plan left = apply.left().accept(this, context); + Plan right = apply.right().accept(this, context); + List correlationSlot = apply.getCorrelationSlot().stream() + .map(s -> ExpressionDeepCopier.INSTANCE.deepCopy(s, context)) + .collect(ImmutableList.toImmutableList()); + SubqueryExpr subqueryExpr = (SubqueryExpr) ExpressionDeepCopier.INSTANCE + .deepCopy(apply.getSubqueryExpr(), context); + Optional correlationFilter = apply.getCorrelationFilter() + .map(f -> ExpressionDeepCopier.INSTANCE.deepCopy(f, context)); + Optional markJoinSlotReference = apply.getMarkJoinSlotReference() + .map(m -> (MarkJoinSlotReference) ExpressionDeepCopier.INSTANCE.deepCopy(m, context)); + Optional subCorrespondingConjunct = apply.getSubCorrespondingConjunct() + .map(c -> ExpressionDeepCopier.INSTANCE.deepCopy(c, context)); + return new LogicalApply<>(correlationSlot, subqueryExpr, correlationFilter, + markJoinSlotReference, subCorrespondingConjunct, apply.isNeedAddSubOutputToProjects(), left, right); + } + + @Override + public Plan visitLogicalAggregate(LogicalAggregate aggregate, DeepCopierContext context) { + Plan child = aggregate.child().accept(this, context); + List groupByExpressions = aggregate.getGroupByExpressions().stream() + .map(k -> ExpressionDeepCopier.INSTANCE.deepCopy(k, context)) + .collect(ImmutableList.toImmutableList()); + List outputExpressions = aggregate.getOutputExpressions().stream() + .map(o -> (NamedExpression) ExpressionDeepCopier.INSTANCE.deepCopy(o, context)) + .collect(ImmutableList.toImmutableList()); + return new LogicalAggregate<>(groupByExpressions, outputExpressions, child); + } + + @Override + public Plan visitLogicalRepeat(LogicalRepeat repeat, DeepCopierContext context) { + Plan child = repeat.child().accept(this, context); + List> groupingSets = repeat.getGroupingSets().stream() + .map(l -> l.stream() + .map(e -> ExpressionDeepCopier.INSTANCE.deepCopy(e, context)) + .collect(ImmutableList.toImmutableList())) + .collect(ImmutableList.toImmutableList()); + List outputExpressions = repeat.getOutputExpressions().stream() + .map(e -> (NamedExpression) ExpressionDeepCopier.INSTANCE.deepCopy(e, context)) + .collect(ImmutableList.toImmutableList()); + return new LogicalRepeat<>(groupingSets, outputExpressions, child); + } + + @Override + public Plan visitLogicalFilter(LogicalFilter filter, DeepCopierContext context) { + Plan child = filter.child().accept(this, context); + Set conjuncts = filter.getConjuncts().stream() + .map(p -> ExpressionDeepCopier.INSTANCE.deepCopy(p, context)) + .collect(ImmutableSet.toImmutableSet()); + return new LogicalFilter<>(conjuncts, child); + } + + @Override + public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, DeepCopierContext context) { + if (context.getRelationReplaceMap().containsKey(olapScan.getRelationId())) { + return context.getRelationReplaceMap().get(olapScan.getRelationId()); + } + LogicalOlapScan newOlapScan; + if (olapScan.getManuallySpecifiedPartitions().isEmpty()) { + newOlapScan = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), + olapScan.getTable(), olapScan.getQualifier(), olapScan.getHints()); + } else { + newOlapScan = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), + olapScan.getTable(), olapScan.getQualifier(), + olapScan.getManuallySpecifiedPartitions(), olapScan.getHints()); + } + newOlapScan.getOutput(); + context.putRelation(olapScan.getRelationId(), newOlapScan); + updateReplaceMapWithOutput(olapScan, newOlapScan, context.exprIdReplaceMap); + return newOlapScan; + } + + @Override + public Plan visitLogicalSchemaScan(LogicalSchemaScan schemaScan, DeepCopierContext context) { + if (context.getRelationReplaceMap().containsKey(schemaScan.getRelationId())) { + return context.getRelationReplaceMap().get(schemaScan.getRelationId()); + } + LogicalSchemaScan newSchemaScan = new LogicalSchemaScan(StatementScopeIdGenerator.newRelationId(), + schemaScan.getTable(), schemaScan.getQualifier()); + updateReplaceMapWithOutput(schemaScan, newSchemaScan, context.exprIdReplaceMap); + context.putRelation(schemaScan.getRelationId(), newSchemaScan); + return newSchemaScan; + } + + @Override + public Plan visitLogicalFileScan(LogicalFileScan fileScan, DeepCopierContext context) { + if (context.getRelationReplaceMap().containsKey(fileScan.getRelationId())) { + return context.getRelationReplaceMap().get(fileScan.getRelationId()); + } + LogicalFileScan newFileScan = new LogicalFileScan(StatementScopeIdGenerator.newRelationId(), + fileScan.getTable(), fileScan.getQualifier()); + updateReplaceMapWithOutput(fileScan, newFileScan, context.exprIdReplaceMap); + context.putRelation(fileScan.getRelationId(), newFileScan); + Set conjuncts = fileScan.getConjuncts().stream() + .map(p -> ExpressionDeepCopier.INSTANCE.deepCopy(p, context)) + .collect(ImmutableSet.toImmutableSet()); + return newFileScan.withConjuncts(conjuncts); + } + + @Override + public Plan visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, DeepCopierContext context) { + if (context.getRelationReplaceMap().containsKey(tvfRelation.getRelationId())) { + return context.getRelationReplaceMap().get(tvfRelation.getRelationId()); + } + LogicalTVFRelation newTVFRelation = new LogicalTVFRelation(StatementScopeIdGenerator.newRelationId(), + tvfRelation.getFunction()); + updateReplaceMapWithOutput(newTVFRelation, tvfRelation, context.exprIdReplaceMap); + context.putRelation(tvfRelation.getRelationId(), newTVFRelation); + return newTVFRelation; + } + + @Override + public Plan visitLogicalJdbcScan(LogicalJdbcScan jdbcScan, DeepCopierContext context) { + if (context.getRelationReplaceMap().containsKey(jdbcScan.getRelationId())) { + return context.getRelationReplaceMap().get(jdbcScan.getRelationId()); + } + LogicalJdbcScan newJdbcScan = new LogicalJdbcScan(StatementScopeIdGenerator.newRelationId(), + jdbcScan.getTable(), jdbcScan.getQualifier()); + updateReplaceMapWithOutput(jdbcScan, newJdbcScan, context.exprIdReplaceMap); + context.putRelation(jdbcScan.getRelationId(), newJdbcScan); + return newJdbcScan; + } + + @Override + public Plan visitLogicalEsScan(LogicalEsScan esScan, DeepCopierContext context) { + if (context.getRelationReplaceMap().containsKey(esScan.getRelationId())) { + return context.getRelationReplaceMap().get(esScan.getRelationId()); + } + LogicalEsScan newEsScan = new LogicalEsScan(StatementScopeIdGenerator.newRelationId(), + esScan.getTable(), esScan.getQualifier()); + updateReplaceMapWithOutput(esScan, newEsScan, context.exprIdReplaceMap); + context.putRelation(esScan.getRelationId(), newEsScan); + return newEsScan; + } + + @Override + public Plan visitLogicalProject(LogicalProject project, DeepCopierContext context) { + Plan child = project.child().accept(this, context); + List newProjects = project.getProjects().stream() + .map(p -> (NamedExpression) ExpressionDeepCopier.INSTANCE.deepCopy(p, context)) + .collect(ImmutableList.toImmutableList()); + return new LogicalProject<>(newProjects, child); + } + + @Override + public Plan visitLogicalSort(LogicalSort sort, DeepCopierContext context) { + Plan child = sort.child().accept(this, context); + List orderKeys = sort.getOrderKeys().stream() + .map(o -> new OrderKey(ExpressionDeepCopier.INSTANCE.deepCopy(o.getExpr(), context), + o.isAsc(), o.isNullFirst())) + .collect(ImmutableList.toImmutableList()); + return new LogicalSort<>(orderKeys, child); + } + + @Override + public Plan visitLogicalTopN(LogicalTopN topN, DeepCopierContext context) { + Plan child = topN.child().accept(this, context); + List orderKeys = topN.getOrderKeys().stream() + .map(o -> new OrderKey(ExpressionDeepCopier.INSTANCE.deepCopy(o.getExpr(), context), + o.isAsc(), o.isNullFirst())) + .collect(ImmutableList.toImmutableList()); + return new LogicalTopN<>(orderKeys, topN.getLimit(), topN.getOffset(), child); + } + + @Override + public Plan visitLogicalPartitionTopN(LogicalPartitionTopN partitionTopN, + DeepCopierContext context) { + Plan child = partitionTopN.child().accept(this, context); + List partitionKeys = partitionTopN.getPartitionKeys().stream() + .map(p -> ExpressionDeepCopier.INSTANCE.deepCopy(p, context)) + .collect(ImmutableList.toImmutableList()); + List orderKeys = partitionTopN.getOrderKeys().stream() + .map(o -> (OrderExpression) ExpressionDeepCopier.INSTANCE.deepCopy(o, context)) + .collect(ImmutableList.toImmutableList()); + return new LogicalPartitionTopN<>(partitionTopN.getFunction(), partitionKeys, orderKeys, + partitionTopN.hasGlobalLimit(), partitionTopN.getPartitionLimit(), child); + } + + @Override + public Plan visitLogicalLimit(LogicalLimit limit, DeepCopierContext context) { + Plan child = limit.child().accept(this, context); + return new LogicalLimit<>(limit.getLimit(), limit.getOffset(), limit.getPhase(), child); + } + + @Override + public Plan visitLogicalJoin(LogicalJoin join, DeepCopierContext context) { + List children = join.children().stream() + .map(c -> c.accept(this, context)) + .collect(ImmutableList.toImmutableList()); + List otherJoinConjuncts = join.getOtherJoinConjuncts().stream() + .map(c -> ExpressionDeepCopier.INSTANCE.deepCopy(c, context)) + .collect(ImmutableList.toImmutableList()); + List hashJoinConjuncts = join.getHashJoinConjuncts().stream() + .map(c -> ExpressionDeepCopier.INSTANCE.deepCopy(c, context)) + .collect(ImmutableList.toImmutableList()); + return new LogicalJoin<>(join.getJoinType(), hashJoinConjuncts, otherJoinConjuncts, + join.getHint(), join.getMarkJoinSlotReference(), children); + } + + @Override + public Plan visitLogicalAssertNumRows(LogicalAssertNumRows assertNumRows, + DeepCopierContext context) { + Plan child = assertNumRows.child().accept(this, context); + return new LogicalAssertNumRows<>(assertNumRows.getAssertNumRowsElement(), child); + } + + @Override + public Plan visitLogicalHaving(LogicalHaving having, DeepCopierContext context) { + Plan child = having.child().accept(this, context); + Set conjuncts = having.getConjuncts().stream() + .map(p -> ExpressionDeepCopier.INSTANCE.deepCopy(p, context)) + .collect(ImmutableSet.toImmutableSet()); + return new LogicalHaving<>(conjuncts, child); + } + + @Override + public Plan visitLogicalUnion(LogicalUnion union, DeepCopierContext context) { + List children = union.children().stream() + .map(c -> c.accept(this, context)) + .collect(ImmutableList.toImmutableList()); + List> constantExprsList = union.getConstantExprsList().stream() + .map(l -> l.stream() + .map(e -> (NamedExpression) ExpressionDeepCopier.INSTANCE.deepCopy(e, context)) + .collect(ImmutableList.toImmutableList())) + .collect(ImmutableList.toImmutableList()); + List outputs = union.getOutputs().stream() + .map(o -> (NamedExpression) ExpressionDeepCopier.INSTANCE.deepCopy(o, context)) + .collect(ImmutableList.toImmutableList()); + return new LogicalUnion(union.getQualifier(), outputs, constantExprsList, union.hasPushedFilter(), children); + } + + @Override + public Plan visitLogicalExcept(LogicalExcept except, DeepCopierContext context) { + List children = except.children().stream() + .map(c -> c.accept(this, context)) + .collect(ImmutableList.toImmutableList()); + List outputs = except.getOutputs().stream() + .map(o -> (NamedExpression) ExpressionDeepCopier.INSTANCE.deepCopy(o, context)) + .collect(ImmutableList.toImmutableList()); + return new LogicalExcept(except.getQualifier(), outputs, children); + } + + @Override + public Plan visitLogicalIntersect(LogicalIntersect intersect, DeepCopierContext context) { + List children = intersect.children().stream() + .map(c -> c.accept(this, context)) + .collect(ImmutableList.toImmutableList()); + List outputs = intersect.getOutputs().stream() + .map(o -> (NamedExpression) ExpressionDeepCopier.INSTANCE.deepCopy(o, context)) + .collect(ImmutableList.toImmutableList()); + return new LogicalIntersect(intersect.getQualifier(), outputs, children); + } + + @Override + public Plan visitLogicalGenerate(LogicalGenerate generate, DeepCopierContext context) { + Plan child = generate.child().accept(this, context); + List generators = generate.getGenerators().stream() + .map(g -> (Function) ExpressionDeepCopier.INSTANCE.deepCopy(g, context)) + .collect(ImmutableList.toImmutableList()); + List generatorOutput = generate.getGeneratorOutput().stream() + .map(o -> (Slot) ExpressionDeepCopier.INSTANCE.deepCopy(o, context)) + .collect(ImmutableList.toImmutableList()); + return new LogicalGenerate<>(generators, generatorOutput, child); + } + + @Override + public Plan visitLogicalWindow(LogicalWindow window, DeepCopierContext context) { + Plan child = window.child().accept(this, context); + List windowExpressions = window.getWindowExpressions().stream() + .map(w -> (NamedExpression) ExpressionDeepCopier.INSTANCE.deepCopy(w, context)) + .collect(ImmutableList.toImmutableList()); + return new LogicalWindow<>(windowExpressions, child); + } + + @Override + public Plan visitLogicalOlapTableSink(LogicalOlapTableSink olapTableSink, + DeepCopierContext context) { + Plan child = olapTableSink.child().accept(this, context); + return new LogicalOlapTableSink<>(olapTableSink.getDatabase(), olapTableSink.getTargetTable(), + olapTableSink.getCols(), olapTableSink.getPartitionIds(), child); + } + + @Override + public Plan visitLogicalFileSink(LogicalFileSink fileSink, DeepCopierContext context) { + Plan child = fileSink.child().accept(this, context); + return fileSink.withChildren(child); + } + + @Override + public Plan visitLogicalCTEProducer(LogicalCTEProducer cteProducer, DeepCopierContext context) { + throw new AnalysisException("plan deep copier could not copy CTEProducer."); + } + + @Override + public Plan visitLogicalCTEConsumer(LogicalCTEConsumer cteConsumer, DeepCopierContext context) { + if (context.getRelationReplaceMap().containsKey(cteConsumer.getRelationId())) { + return context.getRelationReplaceMap().get(cteConsumer.getRelationId()); + } + Map consumerToProducerOutputMap = new LinkedHashMap<>(); + Map producerToConsumerOutputMap = new LinkedHashMap<>(); + for (Slot consumerOutput : cteConsumer.getOutput()) { + Slot newOutput = (Slot) ExpressionDeepCopier.INSTANCE.deepCopy(consumerOutput, context); + consumerToProducerOutputMap.put(newOutput, cteConsumer.getProducerSlot(consumerOutput)); + producerToConsumerOutputMap.put(cteConsumer.getProducerSlot(consumerOutput), newOutput); + } + LogicalCTEConsumer newCTEConsumer = new LogicalCTEConsumer( + StatementScopeIdGenerator.newRelationId(), + cteConsumer.getCteId(), cteConsumer.getName(), + consumerToProducerOutputMap, producerToConsumerOutputMap); + context.putRelation(cteConsumer.getRelationId(), newCTEConsumer); + return newCTEConsumer; + } + + @Override + public Plan visitLogicalCTEAnchor(LogicalCTEAnchor cteAnchor, + DeepCopierContext context) { + throw new AnalysisException("plan deep copier could not copy CTEAnchor."); + } + + private void updateReplaceMapWithOutput(Plan oldPlan, Plan newPlan, Map replaceMap) { + List oldOutput = oldPlan.getOutput(); + List newOutput = newPlan.getOutput(); + for (int i = 0; i < newOutput.size(); i++) { + replaceMap.put(oldOutput.get(i).getExprId(), newOutput.get(i).getExprId()); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java index 28762addd7..5fba8c5687 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java @@ -34,11 +34,12 @@ import java.util.Optional; * Exists subquery expression. */ public class Exists extends SubqueryExpr implements LeafExpression { + private final boolean isNot; public Exists(LogicalPlan subquery, boolean isNot) { super(Objects.requireNonNull(subquery, "subquery can not be null")); - this.isNot = Objects.requireNonNull(isNot, "isNot can not be null"); + this.isNot = isNot; } public Exists(LogicalPlan subquery, List correlateSlots, boolean isNot) { @@ -52,7 +53,7 @@ public class Exists extends SubqueryExpr implements LeafExpression { super(Objects.requireNonNull(subquery, "subquery can not be null"), Objects.requireNonNull(correlateSlots, "subquery can not be null"), typeCoercionExpr); - this.isNot = Objects.requireNonNull(isNot, "isNot can not be null"); + this.isNot = isNot; } public boolean isNot() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/MarkJoinSlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/MarkJoinSlotReference.java index ff841bdeb2..099e64eb5d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/MarkJoinSlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/MarkJoinSlotReference.java @@ -20,6 +20,8 @@ package org.apache.doris.nereids.trees.expressions; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BooleanType; +import com.google.common.collect.ImmutableList; + /** * A special type of column that will be generated to replace the subquery when unnesting the subquery of MarkJoin. */ @@ -36,6 +38,11 @@ public class MarkJoinSlotReference extends SlotReference implements SlotNotFromC this.existsHasAgg = existsHasAgg; } + public MarkJoinSlotReference(ExprId exprId, String name, boolean existsHasAgg) { + super(exprId, name, BooleanType.INSTANCE, false, ImmutableList.of()); + this.existsHasAgg = existsHasAgg; + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitMarkJoinReference(this, context); @@ -61,4 +68,9 @@ public class MarkJoinSlotReference extends SlotReference implements SlotNotFromC public boolean isExistsHasAgg() { return existsHasAgg; } + + @Override + public MarkJoinSlotReference withExprId(ExprId exprId) { + return new MarkJoinSlotReference(exprId, name, existsHasAgg); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Slot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Slot.java index 3a3b498c51..c5dc57018f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Slot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Slot.java @@ -35,11 +35,15 @@ public abstract class Slot extends NamedExpression implements LeafExpression { throw new RuntimeException("Do not implement"); } - public Slot withQualifier(List qualifiers) { + public Slot withQualifier(List qualifier) { throw new RuntimeException("Do not implement"); } public Slot withName(String name) { throw new RuntimeException("Do not implement"); } + + public Slot withExprId(ExprId exprId) { + throw new RuntimeException("Do not implement"); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index 1c1452b399..b76cc77a51 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -33,13 +33,11 @@ import javax.annotation.Nullable; * Reference to slot in expression. */ public class SlotReference extends Slot { - private final ExprId exprId; - // TODO: we should distinguish the name is alias or column name, and the column name should contains - // `cluster:db`.`table`.`column` - private final String name; - private final DataType dataType; - private final boolean nullable; - private final List qualifier; + protected final ExprId exprId; + protected final String name; + protected final DataType dataType; + protected final boolean nullable; + protected final List qualifier; private final Column column; public SlotReference(String name, DataType dataType) { @@ -182,6 +180,7 @@ public class SlotReference extends Slot { return this; } + @Override public SlotReference withNullable(boolean newNullable) { if (this.nullable == newNullable) { return this; @@ -190,16 +189,21 @@ public class SlotReference extends Slot { } @Override - public SlotReference withQualifier(List qualifiers) { - return new SlotReference(exprId, name, dataType, nullable, qualifiers, column); + public SlotReference withQualifier(List qualifier) { + return new SlotReference(exprId, name, dataType, nullable, qualifier, column); + } + + @Override + public SlotReference withName(String name) { + return new SlotReference(exprId, name, dataType, nullable, qualifier, column); + } + + @Override + public SlotReference withExprId(ExprId exprId) { + return new SlotReference(exprId, name, dataType, nullable, qualifier, column); } public boolean isVisible() { return column == null || column.isVisible(); } - - @Override - public Slot withName(String name) { - return new SlotReference(exprId, name, dataType, nullable, qualifier, column); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java index ed9fceeb11..f5aed7e582 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.trees.plans.ObjectId; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.qe.ConnectContext; import com.google.common.annotations.VisibleForTesting; @@ -47,6 +48,14 @@ public class StatementScopeIdGenerator { return ConnectContext.get().getStatementContext().getNextObjectId(); } + public static RelationId newRelationId() { + // this branch is for test only + if (ConnectContext.get() == null || ConnectContext.get().getStatementContext() == null) { + return statementContext.getNextRelationId(); + } + return ConnectContext.get().getStatementContext().getNextRelationId(); + } + public static CTEId newCTEId() { // this branch is for test only if (ConnectContext.get() == null || ConnectContext.get().getStatementContext() == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java index b783918209..44ea635d56 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java @@ -62,6 +62,10 @@ public abstract class SubqueryExpr extends Expression { return typeCoercionExpr.orElseGet(() -> queryPlan.getOutput().get(0)); } + public Expression getSubqueryOutput(LogicalPlan queryPlan) { + return typeCoercionExpr.orElseGet(() -> queryPlan.getOutput().get(0)); + } + @Override public DataType getDataType() throws UnboundException { throw new UnboundException("getDataType"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/VirtualSlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/VirtualSlotReference.java index 76906d2139..eb0829bf0e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/VirtualSlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/VirtualSlotReference.java @@ -119,4 +119,30 @@ public class VirtualSlotReference extends SlotReference implements SlotNotFromCh public boolean nullable() { return false; } + + public VirtualSlotReference withNullable(boolean newNullable) { + if (this.nullable == newNullable) { + return this; + } + return new VirtualSlotReference(exprId, name, dataType, newNullable, qualifier, + originExpression, computeLongValueMethod); + } + + @Override + public VirtualSlotReference withQualifier(List qualifier) { + return new VirtualSlotReference(exprId, name, dataType, nullable, qualifier, + originExpression, computeLongValueMethod); + } + + @Override + public VirtualSlotReference withName(String name) { + return new VirtualSlotReference(exprId, name, dataType, nullable, qualifier, + originExpression, computeLongValueMethod); + } + + @Override + public VirtualSlotReference withExprId(ExprId exprId) { + return new VirtualSlotReference(exprId, name, dataType, nullable, qualifier, + originExpression, computeLongValueMethod); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java index ff203c9b2f..eae1e25201 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java @@ -41,7 +41,8 @@ import java.util.stream.Collectors; /** TableValuedFunction */ public abstract class TableValuedFunction extends BoundFunction implements UnaryExpression, CustomSignature { - protected final Supplier catalogFunctionCache = Suppliers.memoize(() -> toCatalogFunction()); + + protected final Supplier catalogFunctionCache = Suppliers.memoize(this::toCatalogFunction); protected final Supplier tableCache = Suppliers.memoize(() -> { try { return catalogFunctionCache.get().getTable(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/ObjectId.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/ObjectId.java index a540afd0ce..c1f58361b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/ObjectId.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/ObjectId.java @@ -63,6 +63,6 @@ public class ObjectId extends Id { @Override public String toString() { - return "RelationId#" + id; + return "ObjectId#" + id; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java index 05bebf7403..abefbce492 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java @@ -75,7 +75,7 @@ public interface Plan extends TreeNode { /** * Get extra plans. */ - default List extraPlans() { + default List extraPlans() { return ImmutableList.of(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/RelationId.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/RelationId.java new file mode 100644 index 0000000000..45e2ab7ee0 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/RelationId.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans; + +import org.apache.doris.common.Id; +import org.apache.doris.common.IdGenerator; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; + +import java.util.Objects; + +/** + * relation id + */ +public class RelationId extends Id { + + public RelationId(int id) { + super(id); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RelationId relationId = (RelationId) o; + return id == relationId.id; + } + + /** + * Should be only called by {@link StatementScopeIdGenerator}. + */ + public static IdGenerator createGenerator() { + return new IdGenerator() { + @Override + public RelationId getNextId() { + return new RelationId(nextId++); + } + }; + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public String toString() { + return "RelationId#" + id; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/CatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/CatalogRelation.java index cef0b3fce4..f009c41334 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/CatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/CatalogRelation.java @@ -18,12 +18,13 @@ package org.apache.doris.nereids.trees.plans.algebra; import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.exceptions.AnalysisException; /** CatalogRelation */ public interface CatalogRelation extends Relation { - Table getTable(); + + TableIf getTable(); Database getDatabase() throws AnalysisException; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/OlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/OlapScan.java index d0074c4119..d5fe7c2341 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/OlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/OlapScan.java @@ -22,7 +22,8 @@ import org.apache.doris.catalog.OlapTable; import java.util.List; /** OlapScan */ -public interface OlapScan extends Scan { +public interface OlapScan { + OlapTable getTable(); long getSelectedIndexId(); @@ -39,10 +40,10 @@ public interface OlapScan extends Scan { } OlapTable olapTable = getTable(); - Integer selectTabletNumInPartitions = getSelectedPartitionIds().stream() - .map(partitionId -> olapTable.getPartition(partitionId)) + int selectTabletNumInPartitions = getSelectedPartitionIds().stream() + .map(olapTable::getPartition) .map(partition -> partition.getDistributionInfo().getBucketNum()) - .reduce((b1, b2) -> b1 + b2) + .reduce(Integer::sum) .orElse(0); if (selectTabletNumInPartitions > 0) { return selectTabletNumInPartitions; @@ -52,7 +53,7 @@ public interface OlapScan extends Scan { return olapTable.getAllPartitions() .stream() .map(partition -> partition.getDistributionInfo().getBucketNum()) - .reduce((b1, b2) -> b1 + b2) + .reduce(Integer::sum) .orElse(0); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Relation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Relation.java index f0f0ab844e..1c95e33ff6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Relation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Relation.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.algebra; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.RelationId; import java.util.List; @@ -25,5 +26,8 @@ import java.util.List; * Relation base interface */ public interface Relation { + + RelationId getRelationId(); + List getOutput(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Scan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Scan.java deleted file mode 100644 index 5f0baf2232..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Scan.java +++ /dev/null @@ -1,27 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.algebra; - -import org.apache.doris.catalog.TableIf; - -/** - * Common interface for logical/physical scan. - */ -public interface Scan extends Relation { - TableIf getTable(); -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java index c5b2b4aeea..9ef669af1c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java @@ -35,7 +35,7 @@ import java.util.function.Supplier; */ public abstract class AbstractLogicalPlan extends AbstractPlan implements LogicalPlan, Explainable { - private Supplier hasUnboundExpressions = () -> super.hasUnboundExpression(); + private final Supplier hasUnboundExpressions = super::hasUnboundExpression; public AbstractLogicalPlan(PlanType type, Plan... children) { super(type, children); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index e77f47e2b5..1068798f67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -60,7 +60,7 @@ public class LogicalAggregate private final List outputExpressions; // When there are grouping sets/rollup/cube, LogicalAgg is generated by LogicalRepeat. - private final Optional sourceRepeat; + private final Optional> sourceRepeat; private final boolean normalized; private final boolean ordinalIsResolved; @@ -105,7 +105,7 @@ public class LogicalAggregate public LogicalAggregate( List groupByExpressions, List outputExpressions, - Optional sourceRepeat, + Optional> sourceRepeat, CHILD_TYPE child) { this(groupByExpressions, outputExpressions, false, sourceRepeat, child); } @@ -114,7 +114,7 @@ public class LogicalAggregate List groupByExpressions, List outputExpressions, boolean normalized, - Optional sourceRepeat, + Optional> sourceRepeat, CHILD_TYPE child) { this(groupByExpressions, outputExpressions, normalized, false, false, false, sourceRepeat, Optional.empty(), Optional.empty(), child); @@ -130,7 +130,7 @@ public class LogicalAggregate boolean ordinalIsResolved, boolean generated, boolean hasPushed, - Optional sourceRepeat, + Optional> sourceRepeat, Optional groupExpression, Optional logicalProperties, CHILD_TYPE child) { @@ -156,14 +156,10 @@ public class LogicalAggregate return outputExpressions.stream().map(ExpressionTrait::toSql).collect(Collectors.joining(", ")); } - public Optional getSourceRepeat() { + public Optional> getSourceRepeat() { return sourceRepeat; } - public boolean hasRepeat() { - return sourceRepeat.isPresent(); - } - public boolean isDistinct() { return outputExpressions.equals(groupByExpressions); } @@ -223,7 +219,7 @@ public class LogicalAggregate if (o == null || getClass() != o.getClass()) { return false; } - LogicalAggregate that = (LogicalAggregate) o; + LogicalAggregate that = (LogicalAggregate) o; return Objects.equals(groupByExpressions, that.groupByExpressions) && Objects.equals(outputExpressions, that.outputExpressions) && normalized == that.normalized diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java index 9b2727faab..e21726ad7c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java @@ -19,7 +19,6 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; @@ -29,11 +28,8 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -44,35 +40,14 @@ public class LogicalCTE extends LogicalUnary> aliasQueries; - private final Map cteNameToId; - - private final boolean registered; - public LogicalCTE(List> aliasQueries, CHILD_TYPE child) { - this(aliasQueries, Optional.empty(), Optional.empty(), child, false, null); - } - - public LogicalCTE(List> aliasQueries, CHILD_TYPE child, boolean registered, - Map cteNameToId) { - this(aliasQueries, Optional.empty(), Optional.empty(), child, registered, - cteNameToId); + this(aliasQueries, Optional.empty(), Optional.empty(), child); } public LogicalCTE(List> aliasQueries, Optional groupExpression, - Optional logicalProperties, CHILD_TYPE child, - boolean registered, Map cteNameToId) { + Optional logicalProperties, CHILD_TYPE child) { super(PlanType.LOGICAL_CTE, groupExpression, logicalProperties, child); this.aliasQueries = ImmutableList.copyOf(Objects.requireNonNull(aliasQueries, "aliasQueries can not be null")); - this.registered = registered; - this.cteNameToId = cteNameToId == null ? ImmutableMap.copyOf(initCTEId()) : cteNameToId; - } - - private Map initCTEId() { - Map subQueryAliasToUniqueId = new HashMap<>(); - for (LogicalSubQueryAlias subQueryAlias : aliasQueries) { - subQueryAliasToUniqueId.put(subQueryAlias.getAlias(), subQueryAlias.getCteId()); - } - return subQueryAliasToUniqueId; } public List> getAliasQueries() { @@ -80,8 +55,8 @@ public class LogicalCTE extends LogicalUnary extraPlans() { - return (List) aliasQueries; + public List extraPlans() { + return aliasQueries; } /** @@ -126,7 +101,7 @@ public class LogicalCTE extends LogicalUnary children) { Preconditions.checkArgument(aliasQueries.size() > 0); - return new LogicalCTE<>(aliasQueries, children.get(0), registered, cteNameToId); + return new LogicalCTE<>(aliasQueries, children.get(0)); } @Override @@ -141,30 +116,13 @@ public class LogicalCTE extends LogicalUnary withGroupExpression(Optional groupExpression) { - return new LogicalCTE<>(aliasQueries, groupExpression, Optional.of(getLogicalProperties()), child(), - registered, cteNameToId); + return new LogicalCTE<>(aliasQueries, groupExpression, Optional.of(getLogicalProperties()), child()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { Preconditions.checkArgument(aliasQueries.size() > 0); - return new LogicalCTE<>(aliasQueries, groupExpression, logicalProperties, children.get(0), - registered, cteNameToId); - } - - public boolean isRegistered() { - return registered; - } - - public CTEId findCTEId(String subQueryAlias) { - CTEId id = cteNameToId.get(subQueryAlias); - Preconditions.checkArgument(id != null, "Cannot find id for sub-query : %s", - subQueryAlias); - return id; - } - - public Map getCteNameToId() { - return cteNameToId; + return new LogicalCTE<>(aliasQueries, groupExpression, logicalProperties, children.get(0)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEAnchor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEAnchor.java index 605554efc1..c7b6bb5158 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEAnchor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEAnchor.java @@ -40,21 +40,19 @@ public class LogicalCTEAnchor groupExpression, - Optional logicalProperties, - LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild, CTEId cteId) { + public LogicalCTEAnchor(CTEId cteId, Optional groupExpression, + Optional logicalProperties, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { super(PlanType.LOGICAL_CTE_ANCHOR, groupExpression, logicalProperties, leftChild, rightChild); this.cteId = cteId; } @Override public Plan withChildren(List children) { - return new LogicalCTEAnchor<>(groupExpression, Optional.of(getLogicalProperties()), - children.get(0), children.get(1), cteId); + return new LogicalCTEAnchor<>(cteId, children.get(0), children.get(1)); } @Override @@ -69,13 +67,13 @@ public class LogicalCTEAnchor groupExpression) { - return new LogicalCTEAnchor<>(groupExpression, Optional.of(getLogicalProperties()), left(), right(), cteId); + return new LogicalCTEAnchor<>(cteId, groupExpression, Optional.of(getLogicalProperties()), left(), right()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalCTEAnchor<>(groupExpression, logicalProperties, children.get(0), children.get(1), cteId); + return new LogicalCTEAnchor<>(cteId, groupExpression, logicalProperties, children.get(0), children.get(1)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java index 33e688645d..86f58ce4c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java @@ -20,12 +20,11 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.CTEId; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -35,59 +34,67 @@ import com.google.common.collect.ImmutableList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; /** * LogicalCTEConsumer */ -public class LogicalCTEConsumer extends LogicalLeaf { - - private final CTEId cteId; - - private final Map consumerToProducerOutputMap = new LinkedHashMap<>(); - - private final Map producerToConsumerOutputMap = new LinkedHashMap<>(); - - private final int consumerId; +public class LogicalCTEConsumer extends LogicalRelation { private final String name; + private final CTEId cteId; + private final Map consumerToProducerOutputMap; + private final Map producerToConsumerOutputMap; /** * Logical CTE consumer. */ - public LogicalCTEConsumer(Optional groupExpression, - Optional logicalProperties, LogicalPlan childPlan, CTEId cteId, String name) { - super(PlanType.LOGICAL_CTE_RELATION, groupExpression, logicalProperties); - this.cteId = cteId; - this.name = name; - initProducerToConsumerOutputMap(childPlan); - for (Map.Entry entry : producerToConsumerOutputMap.entrySet()) { - this.consumerToProducerOutputMap.put(entry.getValue(), entry.getKey()); - } - this.consumerId = StatementScopeIdGenerator.newCTEId().asInt(); + public LogicalCTEConsumer(RelationId relationId, CTEId cteId, String name, + Map consumerToProducerOutputMap, Map producerToConsumerOutputMap) { + super(relationId, PlanType.LOGICAL_CTE_RELATION, Optional.empty(), Optional.empty()); + this.cteId = Objects.requireNonNull(cteId, "cteId should not null"); + this.name = Objects.requireNonNull(name, "name should not null"); + this.consumerToProducerOutputMap = Objects.requireNonNull(consumerToProducerOutputMap, + "consumerToProducerOutputMap should not null"); + this.producerToConsumerOutputMap = Objects.requireNonNull(producerToConsumerOutputMap, + "producerToConsumerOutputMap should not null"); } /** * Logical CTE consumer. */ - public LogicalCTEConsumer(Optional groupExpression, - Optional logicalProperties, CTEId cteId, - Map consumerToProducerOutputMap, - Map producerToConsumerOutputMap, int consumerId, String name) { - super(PlanType.LOGICAL_CTE_RELATION, groupExpression, logicalProperties); - this.cteId = cteId; - this.consumerToProducerOutputMap.putAll(consumerToProducerOutputMap); - this.producerToConsumerOutputMap.putAll(producerToConsumerOutputMap); - this.consumerId = consumerId; - this.name = name; + public LogicalCTEConsumer(RelationId relationId, CTEId cteId, String name, LogicalPlan producerPlan) { + super(relationId, PlanType.LOGICAL_CTE_RELATION, Optional.empty(), Optional.empty()); + this.cteId = Objects.requireNonNull(cteId, "cteId should not null"); + this.name = Objects.requireNonNull(name, "name should not null"); + this.consumerToProducerOutputMap = new LinkedHashMap<>(); + this.producerToConsumerOutputMap = new LinkedHashMap<>(); + initOutputMaps(producerPlan); } - private void initProducerToConsumerOutputMap(LogicalPlan childPlan) { + /** + * Logical CTE consumer. + */ + public LogicalCTEConsumer(RelationId relationId, CTEId cteId, String name, + Map consumerToProducerOutputMap, Map producerToConsumerOutputMap, + Optional groupExpression, Optional logicalProperties) { + super(relationId, PlanType.LOGICAL_CTE_RELATION, groupExpression, logicalProperties); + this.cteId = Objects.requireNonNull(cteId, "cteId should not null"); + this.name = Objects.requireNonNull(name, "name should not null"); + this.consumerToProducerOutputMap = Objects.requireNonNull(consumerToProducerOutputMap, + "consumerToProducerOutputMap should not null"); + this.producerToConsumerOutputMap = Objects.requireNonNull(producerToConsumerOutputMap, + "producerToConsumerOutputMap should not null"); + } + + private void initOutputMaps(LogicalPlan childPlan) { List producerOutput = childPlan.getOutput(); for (Slot producerOutputSlot : producerOutput) { Slot consumerSlot = new SlotReference(producerOutputSlot.getName(), producerOutputSlot.getDataType(), producerOutputSlot.nullable(), ImmutableList.of(name)); producerToConsumerOutputMap.put(producerOutputSlot, consumerSlot); + consumerToProducerOutputMap.put(consumerSlot, producerOutputSlot); } } @@ -104,26 +111,25 @@ public class LogicalCTEConsumer extends LogicalLeaf { return visitor.visitLogicalCTEConsumer(this, context); } - @Override - public List getExpressions() { - return ImmutableList.of(); + public Plan withTwoMaps(Map consumerToProducerOutputMap, Map producerToConsumerOutputMap) { + return new LogicalCTEConsumer(relationId, cteId, name, + consumerToProducerOutputMap, producerToConsumerOutputMap, + Optional.empty(), Optional.empty()); } @Override public Plan withGroupExpression(Optional groupExpression) { - return new LogicalCTEConsumer(groupExpression, Optional.of(getLogicalProperties()), cteId, - consumerToProducerOutputMap, - producerToConsumerOutputMap, - consumerId, name); + return new LogicalCTEConsumer(relationId, cteId, name, + consumerToProducerOutputMap, producerToConsumerOutputMap, + groupExpression, Optional.of(getLogicalProperties())); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalCTEConsumer(groupExpression, logicalProperties, cteId, - consumerToProducerOutputMap, - producerToConsumerOutputMap, - consumerId, name); + return new LogicalCTEConsumer(relationId, cteId, name, + consumerToProducerOutputMap, producerToConsumerOutputMap, + groupExpression, logicalProperties); } @Override @@ -135,31 +141,11 @@ public class LogicalCTEConsumer extends LogicalLeaf { return cteId; } - @Override - public int hashCode() { - return consumerId; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - return this.consumerId == ((LogicalCTEConsumer) o).consumerId; - } - - public int getConsumerId() { - return consumerId; - } - public String getName() { return name; } - public Slot findProducerSlot(Slot consumerSlot) { + public Slot getProducerSlot(Slot consumerSlot) { Slot slot = consumerToProducerOutputMap.get(consumerSlot); Preconditions.checkArgument(slot != null, String.format("Required producer" + "slot for :%s doesn't exist", consumerSlot)); @@ -170,6 +156,7 @@ public class LogicalCTEConsumer extends LogicalLeaf { public String toString() { return Utils.toSqlString("LogicalCteConsumer[" + id.asInt() + "]", "cteId", cteId, - "consumerId", consumerId); + "relationId", relationId, + "name", name); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEProducer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEProducer.java index b6123edaca..089f1ec9fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEProducer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEProducer.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -40,45 +41,30 @@ public class LogicalCTEProducer extends LogicalUnary projects; - - private final boolean rewritten; - - public LogicalCTEProducer(CHILD_TYPE child, CTEId cteId) { + public LogicalCTEProducer(CTEId cteId, CHILD_TYPE child) { super(PlanType.LOGICAL_CTE_PRODUCER, child); this.cteId = cteId; - this.projects = ImmutableList.of(); - this.rewritten = false; } - public LogicalCTEProducer(Optional groupExpression, - Optional logicalProperties, CHILD_TYPE child, CTEId cteId, - List projects, boolean rewritten) { + public LogicalCTEProducer(CTEId cteId, Optional groupExpression, + Optional logicalProperties, CHILD_TYPE child) { super(PlanType.LOGICAL_CTE_PRODUCER, groupExpression, logicalProperties, child); this.cteId = cteId; - this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, - "projects should not null")); - this.rewritten = rewritten; } public CTEId getCteId() { return cteId; } - public List getProjects() { - return projects; - } - @Override public Plan withChildren(List children) { Preconditions.checkArgument(children.size() == 1); - return new LogicalCTEProducer<>(groupExpression, Optional.of(getLogicalProperties()), children.get(0), - cteId, projects, rewritten); + return new LogicalCTEProducer<>(cteId, children.get(0)); } - public Plan withChildrenAndProjects(List children, List projects, boolean rewritten) { - return new LogicalCTEProducer<>(groupExpression, Optional.of(getLogicalProperties()), children.get(0), - cteId, projects, rewritten); + @Override + public List getExpressions() { + return ImmutableList.of(); } @Override @@ -86,41 +72,26 @@ public class LogicalCTEProducer extends LogicalUnary getExpressions() { - return child().getExpressions(); - } - @Override public Plan withGroupExpression(Optional groupExpression) { - return new LogicalCTEProducer<>(groupExpression, Optional.of(getLogicalProperties()), child(), cteId, - projects, rewritten); + return new LogicalCTEProducer<>(cteId, groupExpression, Optional.of(getLogicalProperties()), child()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalCTEProducer<>(groupExpression, logicalProperties, children.get(0), cteId, - projects, rewritten); + return new LogicalCTEProducer<>(cteId, groupExpression, logicalProperties, children.get(0)); } @Override public List computeOutput() { - return child().computeOutput(); + return child().getOutput(); } @Override public String toString() { - return String.format("LOGICAL_CTE_PRODUCER#%d", cteId.asInt()); - } - - public boolean isRewritten() { - return rewritten; - } - - @Override - public int hashCode() { - return Objects.hash(cteId, projects, rewritten); + return Utils.toSqlString("LogicalCteProducer[" + id.asInt() + "]", + "cteId", cteId); } @Override @@ -131,13 +102,15 @@ public class LogicalCTEProducer extends LogicalUnary that = (LogicalCTEProducer) o; + return Objects.equals(cteId, that.cteId); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), cteId); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java new file mode 100644 index 0000000000..6150e9047a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.logical; + +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.util.Utils; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * abstract class catalog relation for logical relation + */ +public abstract class LogicalCatalogRelation extends LogicalRelation implements CatalogRelation { + + protected final TableIf table; + protected final ImmutableList qualifier; + + public LogicalCatalogRelation(RelationId relationId, PlanType type, TableIf table, List qualifier) { + super(relationId, type); + this.table = Objects.requireNonNull(table, "table can not be null"); + this.qualifier = ImmutableList.copyOf(Objects.requireNonNull(qualifier, "qualifier can not be null")); + } + + public LogicalCatalogRelation(RelationId relationId, PlanType type, TableIf table, List qualifier, + Optional groupExpression, Optional logicalProperties) { + super(relationId, type, groupExpression, logicalProperties); + this.table = Objects.requireNonNull(table, "table can not be null"); + this.qualifier = ImmutableList.copyOf(Objects.requireNonNull(qualifier, "qualifier can not be null")); + } + + @Override + public TableIf getTable() { + return table; + } + + @Override + public Database getDatabase() throws AnalysisException { + Preconditions.checkArgument(!qualifier.isEmpty()); + return Env.getCurrentInternalCatalog().getDbOrException(qualifier.get(0), + s -> new AnalysisException("Database [" + qualifier.get(0) + "] does not exist.")); + } + + @Override + public List computeOutput() { + return table.getBaseSchema() + .stream() + .map(col -> SlotReference.fromColumn(col, qualified())) + .collect(ImmutableList.toImmutableList()); + } + + public List getQualifier() { + return qualifier; + } + + /** + * Full qualified name parts, i.e., concat qualifier and name into a list. + */ + public List qualified() { + return Utils.qualifiedNameParts(qualifier, table.getName()); + } + + /** + * Full qualified table name, concat qualifier and name with `.` as separator. + */ + public String qualifiedName() { + return Utils.qualifiedName(qualifier, table.getName()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java index 3484b95983..32e7e1144a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java @@ -19,11 +19,11 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.EmptyRelation; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -39,17 +39,17 @@ import java.util.Optional; * e.g. * select * from tbl limit 0 */ -public class LogicalEmptyRelation extends LogicalLeaf implements EmptyRelation, OutputPrunable { +public class LogicalEmptyRelation extends LogicalRelation implements EmptyRelation, OutputPrunable { private final List projects; - public LogicalEmptyRelation(List projects) { - this(projects, Optional.empty(), Optional.empty()); + public LogicalEmptyRelation(RelationId relationId, List projects) { + this(relationId, projects, Optional.empty(), Optional.empty()); } - public LogicalEmptyRelation(List projects, Optional groupExpression, - Optional logicalProperties) { - super(PlanType.LOGICAL_ONE_ROW_RELATION, groupExpression, logicalProperties); + public LogicalEmptyRelation(RelationId relationId, List projects, + Optional groupExpression, Optional logicalProperties) { + super(relationId, PlanType.LOGICAL_ONE_ROW_RELATION, groupExpression, logicalProperties); this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, "projects can not be null")); } @@ -63,24 +63,20 @@ public class LogicalEmptyRelation extends LogicalLeaf implements EmptyRelation, return projects; } - @Override - public List getExpressions() { - return ImmutableList.of(); - } - public LogicalEmptyRelation withProjects(List projects) { - return new LogicalEmptyRelation(projects, Optional.empty(), Optional.empty()); + return new LogicalEmptyRelation(relationId, projects, Optional.empty(), Optional.empty()); } @Override public Plan withGroupExpression(Optional groupExpression) { - return new LogicalEmptyRelation(projects, groupExpression, Optional.of(logicalPropertiesSupplier.get())); + return new LogicalEmptyRelation(relationId, projects, + groupExpression, Optional.of(logicalPropertiesSupplier.get())); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalEmptyRelation(projects, groupExpression, logicalProperties); + return new LogicalEmptyRelation(relationId, projects, groupExpression, logicalProperties); } @Override @@ -97,26 +93,6 @@ public class LogicalEmptyRelation extends LogicalLeaf implements EmptyRelation, ); } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - LogicalEmptyRelation that = (LogicalEmptyRelation) o; - return Objects.equals(projects, that.projects); - } - - @Override - public int hashCode() { - return Objects.hash(projects); - } - @Override public List getOutputs() { return projects; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java index 012348938b..d34f47266f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java @@ -20,9 +20,9 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.catalog.external.ExternalTable; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -34,19 +34,18 @@ import java.util.Optional; /** * Logical scan for external es catalog. */ -public class LogicalEsScan extends LogicalRelation { +public class LogicalEsScan extends LogicalCatalogRelation { /** * Constructor for LogicalEsScan. */ - public LogicalEsScan(ObjectId id, ExternalTable table, List qualifier, + public LogicalEsScan(RelationId id, ExternalTable table, List qualifier, Optional groupExpression, Optional logicalProperties) { - super(id, PlanType.LOGICAL_ES_SCAN, table, qualifier, - groupExpression, logicalProperties); + super(id, PlanType.LOGICAL_ES_SCAN, table, qualifier, groupExpression, logicalProperties); } - public LogicalEsScan(ObjectId id, ExternalTable table, List qualifier) { + public LogicalEsScan(RelationId id, ExternalTable table, List qualifier) { this(id, table, qualifier, Optional.empty(), Optional.empty()); } @@ -66,14 +65,14 @@ public class LogicalEsScan extends LogicalRelation { @Override public LogicalEsScan withGroupExpression(Optional groupExpression) { - return new LogicalEsScan(id, (ExternalTable) table, qualifier, groupExpression, + return new LogicalEsScan(relationId, (ExternalTable) table, qualifier, groupExpression, Optional.of(getLogicalProperties())); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalEsScan(id, (ExternalTable) table, qualifier, groupExpression, logicalProperties); + return new LogicalEsScan(relationId, (ExternalTable) table, qualifier, groupExpression, logicalProperties); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java index 52d66de129..af5dcaa0f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java @@ -21,9 +21,9 @@ import org.apache.doris.catalog.external.ExternalTable; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -31,67 +31,63 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Sets; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.Set; /** * Logical file scan for external catalog. */ -public class LogicalFileScan extends LogicalRelation { +public class LogicalFileScan extends LogicalCatalogRelation { + // TODO remove this conjuncts. private final Set conjuncts; /** * Constructor for LogicalFileScan. */ - public LogicalFileScan(ObjectId id, ExternalTable table, List qualifier, - Optional groupExpression, - Optional logicalProperties, - Set conjuncts) { + public LogicalFileScan(RelationId id, ExternalTable table, List qualifier, + Optional groupExpression, Optional logicalProperties, + Set conjuncts) { super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, groupExpression, logicalProperties); this.conjuncts = conjuncts; } - public LogicalFileScan(ObjectId id, ExternalTable table, List qualifier) { + public LogicalFileScan(RelationId id, ExternalTable table, List qualifier) { this(id, table, qualifier, Optional.empty(), Optional.empty(), Sets.newHashSet()); } @Override public ExternalTable getTable() { - Preconditions.checkArgument(table instanceof ExternalTable); + Preconditions.checkArgument(table instanceof ExternalTable, + "LogicalFileScan's table must be ExternalTable, but table is " + table.getClass().getSimpleName()); return (ExternalTable) table; } @Override public String toString() { return Utils.toSqlString("LogicalFileScan", - "qualified", qualifiedName(), - "output", getOutput() + "qualified", qualifiedName(), + "output", getOutput() ); } - @Override - public boolean equals(Object o) { - return super.equals(o) && Objects.equals(conjuncts, ((LogicalFileScan) o).conjuncts); - } - @Override public LogicalFileScan withGroupExpression(Optional groupExpression) { - return new LogicalFileScan(id, (ExternalTable) table, qualifier, groupExpression, + return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, groupExpression, Optional.of(getLogicalProperties()), conjuncts); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalFileScan(id, (ExternalTable) table, qualifier, groupExpression, logicalProperties, conjuncts); + return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, + groupExpression, logicalProperties, conjuncts); } public LogicalFileScan withConjuncts(Set conjuncts) { - return new LogicalFileScan(id, (ExternalTable) table, qualifier, groupExpression, - Optional.of(getLogicalProperties()), conjuncts); + return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, groupExpression, + Optional.of(getLogicalProperties()), conjuncts); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java index 6943062e7d..a181755543 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java @@ -67,7 +67,7 @@ public class LogicalFilter extends LogicalUnary extraPlans() { + public List extraPlans() { return conjuncts.stream().map(Expression::children).flatMap(Collection::stream).flatMap(m -> { if (m instanceof SubqueryExpr) { return Stream.of(new LogicalSubQueryAlias<>(m.toSql(), ((SubqueryExpr) m).getQueryPlan())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java index e8778b13d7..c35879f969 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import java.util.List; import java.util.Objects; @@ -79,8 +80,16 @@ public class LogicalGenerate extends LogicalUnary withGenerators(List generators) { - return new LogicalGenerate<>(generators, generatorOutput, + Preconditions.checkArgument(generators.size() == generatorOutput.size()); + List newGeneratorOutput = Lists.newArrayList(); + for (int i = 0; i < generators.size(); i++) { + newGeneratorOutput.add(generatorOutput.get(i).withNullable(generators.get(i).nullable())); + } + return new LogicalGenerate<>(generators, newGeneratorOutput, Optional.empty(), Optional.of(getLogicalProperties()), child()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJdbcScan.java index ee12dc521a..6a3d7b28e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJdbcScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJdbcScan.java @@ -22,9 +22,9 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.external.ExternalTable; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -36,19 +36,19 @@ import java.util.Optional; /** * Logical scan for external jdbc catalog and jdbc table. */ -public class LogicalJdbcScan extends LogicalRelation { +public class LogicalJdbcScan extends LogicalCatalogRelation { /** * Constructor for LogicalJdbcScan. */ - public LogicalJdbcScan(ObjectId id, TableIf table, List qualifier, + public LogicalJdbcScan(RelationId id, TableIf table, List qualifier, Optional groupExpression, Optional logicalProperties) { super(id, PlanType.LOGICAL_JDBC_SCAN, table, qualifier, groupExpression, logicalProperties); } - public LogicalJdbcScan(ObjectId id, TableIf table, List qualifier) { + public LogicalJdbcScan(RelationId id, TableIf table, List qualifier) { this(id, table, qualifier, Optional.empty(), Optional.empty()); } @@ -69,19 +69,18 @@ public class LogicalJdbcScan extends LogicalRelation { @Override public LogicalJdbcScan withGroupExpression(Optional groupExpression) { - return new LogicalJdbcScan(id, table, qualifier, groupExpression, + return new LogicalJdbcScan(relationId, table, qualifier, groupExpression, Optional.of(getLogicalProperties())); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalJdbcScan(id, table, qualifier, groupExpression, logicalProperties); + return new LogicalJdbcScan(relationId, table, qualifier, groupExpression, logicalProperties); } @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitLogicalJdbcScan(this, context); } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index ffd69a94e5..6d58ba1f71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -18,22 +18,18 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.common.util.Util; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.rules.rewrite.mv.AbstractSelectMaterializedIndexRule; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.PreAggStatus; -import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.OlapScan; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -55,7 +51,7 @@ import java.util.stream.Stream; /** * Logical OlapScan. */ -public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, OlapScan { +public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan { /////////////////////////////////////////////////////////////////////////// // Members for materialized index. @@ -77,10 +73,10 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, private final PreAggStatus preAggStatus; /** - * When the Slotreference is generated through fromColumn, + * When the SlotReference is generated through fromColumn, * the exprId will be generated incrementally, * causing the slotId of the base to change when the output is recalculated. - * This structure is responsible for storing the generated Slotreference + * This structure is responsible for storing the generated SlotReference */ private final Map cacheSlotWithSlotName; @@ -110,45 +106,40 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, /////////////////////////////////////////////////////////////////////////// private final List hints; - public LogicalOlapScan(ObjectId id, OlapTable table) { + public LogicalOlapScan(RelationId id, OlapTable table) { this(id, table, ImmutableList.of()); } - public LogicalOlapScan(ObjectId id, OlapTable table, List qualifier) { + public LogicalOlapScan(RelationId id, OlapTable table, List qualifier) { this(id, table, qualifier, Optional.empty(), Optional.empty(), table.getPartitionIds(), false, ImmutableList.of(), -1, false, PreAggStatus.on(), ImmutableList.of(), ImmutableList.of(), Maps.newHashMap()); } - public LogicalOlapScan(ObjectId id, OlapTable table, List qualifier, List hints) { + public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, List hints) { this(id, table, qualifier, Optional.empty(), Optional.empty(), table.getPartitionIds(), false, ImmutableList.of(), -1, false, PreAggStatus.on(), ImmutableList.of(), hints, Maps.newHashMap()); } - public LogicalOlapScan(ObjectId id, OlapTable table, List qualifier, List specifiedPartitions, + public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, List specifiedPartitions, List hints) { this(id, table, qualifier, Optional.empty(), Optional.empty(), + // must use specifiedPartitions here for prune partition by sql like 'select * from t partition p1' specifiedPartitions, false, ImmutableList.of(), -1, false, PreAggStatus.on(), specifiedPartitions, hints, Maps.newHashMap()); } - public LogicalOlapScan(ObjectId id, Table table, List qualifier) { - this(id, table, qualifier, Optional.empty(), Optional.empty(), - ((OlapTable) table).getPartitionIds(), false, ImmutableList.of(), - -1, false, PreAggStatus.on(), ImmutableList.of(), ImmutableList.of(), Maps.newHashMap()); - } - /** * Constructor for LogicalOlapScan. */ - public LogicalOlapScan(ObjectId id, Table table, List qualifier, + public LogicalOlapScan(RelationId id, Table table, List qualifier, Optional groupExpression, Optional logicalProperties, List selectedPartitionIds, boolean partitionPruned, List selectedTabletIds, long selectedIndexId, boolean indexSelected, - PreAggStatus preAggStatus, List partitions, + PreAggStatus preAggStatus, List specifiedPartitions, List hints, Map cacheSlotWithSlotName) { super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier, @@ -159,10 +150,11 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, this.selectedIndexId = selectedIndexId <= 0 ? getTable().getBaseIndexId() : selectedIndexId; this.indexSelected = indexSelected; this.preAggStatus = preAggStatus; - this.manuallySpecifiedPartitions = ImmutableList.copyOf(partitions); + this.manuallySpecifiedPartitions = ImmutableList.copyOf(specifiedPartitions); this.selectedPartitionIds = selectedPartitionIds.stream() - .filter(partitionId -> this.getTable().getPartition(partitionId).hasData()).collect( - Collectors.toList()); + .filter(partitionId -> this.getTable().getPartition(partitionId) != null) + .filter(partitionId -> this.getTable().getPartition(partitionId).hasData()) + .collect(Collectors.toList()); this.hints = Objects.requireNonNull(hints, "hints can not be null"); this.cacheSlotWithSlotName = Objects.requireNonNull(cacheSlotWithSlotName, "mvNameToSlot can not be null"); } @@ -177,13 +169,6 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, return (OlapTable) table; } - @Override - public Database getDatabase() throws AnalysisException { - Preconditions.checkArgument(!qualifier.isEmpty()); - return Env.getCurrentInternalCatalog().getDbOrException(qualifier.get(0), - s -> new AnalysisException("Database [" + qualifier.get(0) + "] does not exist.")); - } - @Override public String toString() { return Utils.toSqlString("LogicalOlapScan", @@ -199,31 +184,31 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, if (this == o) { return true; } - if (o == null || getClass() != o.getClass() || !super.equals(o)) { + if (o == null || getClass() != o.getClass()) { return false; } - return Objects.equals(id, ((LogicalOlapScan) o).id) - && Objects.equals(selectedPartitionIds, ((LogicalOlapScan) o).selectedPartitionIds) - && Objects.equals(partitionPruned, ((LogicalOlapScan) o).partitionPruned) - && Objects.equals(selectedIndexId, ((LogicalOlapScan) o).selectedIndexId) - && Objects.equals(indexSelected, ((LogicalOlapScan) o).indexSelected) - && Objects.equals(selectedTabletIds, ((LogicalOlapScan) o).selectedTabletIds) - && Objects.equals(hints, ((LogicalOlapScan) o).hints) - && Objects.equals(cacheSlotWithSlotName, ((LogicalOlapScan) o).cacheSlotWithSlotName); + if (!super.equals(o)) { + return false; + } + LogicalOlapScan that = (LogicalOlapScan) o; + return selectedIndexId == that.selectedIndexId && indexSelected == that.indexSelected + && partitionPruned == that.partitionPruned && Objects.equals(preAggStatus, that.preAggStatus) + && Objects.equals(selectedTabletIds, that.selectedTabletIds) + && Objects.equals(manuallySpecifiedPartitions, that.manuallySpecifiedPartitions) + && Objects.equals(selectedPartitionIds, that.selectedPartitionIds) + && Objects.equals(hints, that.hints); } @Override public int hashCode() { - return Objects.hash(id, - selectedPartitionIds, partitionPruned, - selectedIndexId, indexSelected, - selectedTabletIds, - hints); + return Objects.hash(super.hashCode(), selectedIndexId, indexSelected, preAggStatus, cacheSlotWithSlotName, + selectedTabletIds, partitionPruned, manuallySpecifiedPartitions, selectedPartitionIds, hints); } @Override public LogicalOlapScan withGroupExpression(Optional groupExpression) { - return new LogicalOlapScan(id, (Table) table, qualifier, groupExpression, Optional.of(getLogicalProperties()), + return new LogicalOlapScan(relationId, (Table) table, qualifier, + groupExpression, Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName); @@ -232,34 +217,38 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalOlapScan(id, (Table) table, qualifier, groupExpression, logicalProperties, + return new LogicalOlapScan(relationId, (Table) table, qualifier, groupExpression, logicalProperties, selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName); } public LogicalOlapScan withSelectedPartitionIds(List selectedPartitionIds) { - return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), + return new LogicalOlapScan(relationId, (Table) table, qualifier, + Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, true, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName); } public LogicalOlapScan withMaterializedIndexSelected(PreAggStatus preAgg, long indexId) { - return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), + return new LogicalOlapScan(relationId, (Table) table, qualifier, + Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, indexId, true, preAgg, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName); } public LogicalOlapScan withSelectedTabletIds(List selectedTabletIds) { - return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), + return new LogicalOlapScan(relationId, (Table) table, qualifier, + Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName); } public LogicalOlapScan withPreAggStatus(PreAggStatus preAggStatus) { - return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), + return new LogicalOlapScan(relationId, (Table) table, qualifier, + Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java index 575c13b75b..8a03ae0882 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -40,18 +41,17 @@ import java.util.Optional; * A relation that contains only one row consist of some constant expressions. * e.g. select 100, 'value' */ -public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation, OutputPrunable { +public class LogicalOneRowRelation extends LogicalRelation implements OneRowRelation, OutputPrunable { private final List projects; - public LogicalOneRowRelation(List projects) { - this(projects, Optional.empty(), Optional.empty()); + public LogicalOneRowRelation(RelationId relationId, List projects) { + this(relationId, projects, Optional.empty(), Optional.empty()); } - private LogicalOneRowRelation(List projects, - Optional groupExpression, - Optional logicalProperties) { - super(PlanType.LOGICAL_ONE_ROW_RELATION, groupExpression, logicalProperties); + private LogicalOneRowRelation(RelationId relationId, List projects, + Optional groupExpression, Optional logicalProperties) { + super(relationId, PlanType.LOGICAL_ONE_ROW_RELATION, groupExpression, logicalProperties); Preconditions.checkArgument(projects.stream().noneMatch(p -> p.containsType(Slot.class)), "OneRowRelation can not contains any slot"); Preconditions.checkArgument(projects.stream().noneMatch(p -> p.containsType(AggregateFunction.class)), @@ -76,13 +76,13 @@ public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation @Override public Plan withGroupExpression(Optional groupExpression) { - return new LogicalOneRowRelation(projects, groupExpression, Optional.of(getLogicalProperties())); + return new LogicalOneRowRelation(relationId, projects, groupExpression, Optional.of(getLogicalProperties())); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalOneRowRelation(projects, groupExpression, logicalProperties); + return new LogicalOneRowRelation(relationId, projects, groupExpression, logicalProperties); } @Override @@ -92,13 +92,6 @@ public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation .collect(ImmutableList.toImmutableList()); } - @Override - public String toString() { - return Utils.toSqlString("LogicalOneRowRelation", - "projects", projects - ); - } - @Override public boolean equals(Object o) { if (this == o) { @@ -116,11 +109,18 @@ public class LogicalOneRowRelation extends LogicalLeaf implements OneRowRelation @Override public int hashCode() { - return Objects.hash(projects); + return Objects.hash(super.hashCode(), projects); + } + + @Override + public String toString() { + return Utils.toSqlString("LogicalOneRowRelation", + "projects", projects + ); } public LogicalOneRowRelation withProjects(List namedExpressions) { - return new LogicalOneRowRelation(namedExpressions, Optional.empty(), Optional.empty()); + return new LogicalOneRowRelation(relationId, namedExpressions, Optional.empty(), Optional.empty()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPartitionTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPartitionTopN.java index 3c6da5534a..003b273727 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPartitionTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPartitionTopN.java @@ -175,6 +175,11 @@ public class LogicalPartitionTopN extends LogicalUnary< .build(); } + public LogicalPartitionTopN withPartitionKeysAndOrderKeys( + List partitionKeys, List orderKeys) { + return new LogicalPartitionTopN<>(function, partitionKeys, orderKeys, hasGlobalLimit, partitionLimit, child()); + } + @Override public LogicalPartitionTopN withChildren(List children) { Preconditions.checkArgument(children.size() == 1); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRelation.java index fdf5b79999..8b1a3e5987 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRelation.java @@ -17,18 +17,13 @@ package org.apache.doris.nereids.trees.plans.logical; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.PlanType; -import org.apache.doris.nereids.trees.plans.algebra.Scan; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.algebra.Relation; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; -import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; import org.json.JSONObject; @@ -40,42 +35,19 @@ import java.util.Optional; /** * Logical relation plan. */ -public abstract class LogicalRelation extends LogicalLeaf implements Scan { +public abstract class LogicalRelation extends LogicalLeaf implements Relation { - protected final ObjectId id; - protected final TableIf table; - protected final ImmutableList qualifier; + protected final RelationId relationId; - public LogicalRelation(ObjectId id, PlanType type, TableIf table, List qualifier) { - this(id, type, table, qualifier, Optional.empty(), Optional.empty()); + public LogicalRelation(RelationId relationId, PlanType type) { + this(relationId, type, Optional.empty(), Optional.empty()); } - public LogicalRelation(ObjectId id, PlanType type, Optional groupExpression, - Optional logicalProperties) { - this(id, type, new OlapTable(), ImmutableList.of(), groupExpression, logicalProperties); - } - - /** - * Constructor for LogicalRelationPlan. - * - * @param table Doris table - * @param qualifier qualified relation name - */ - public LogicalRelation(ObjectId id, PlanType type, TableIf table, List qualifier, + public LogicalRelation(RelationId relationId, PlanType type, Optional groupExpression, Optional logicalProperties) { super(type, groupExpression, logicalProperties); - this.id = id; - this.table = Objects.requireNonNull(table, "table can not be null"); - this.qualifier = ImmutableList.copyOf(Objects.requireNonNull(qualifier, "qualifier can not be null")); - } + this.relationId = relationId; - @Override - public TableIf getTable() { - return table; - } - - public List getQualifier() { - return qualifier; } @Override @@ -87,22 +59,12 @@ public abstract class LogicalRelation extends LogicalLeaf implements Scan { return false; } LogicalRelation that = (LogicalRelation) o; - return this.id.equals(that.getId()) - && Objects.equals(this.table.getId(), that.table.getId()) - && Objects.equals(this.qualifier, that.qualifier); + return this.relationId.equals(that.getRelationId()); } @Override public int hashCode() { - return Objects.hash(id); - } - - @Override - public List computeOutput() { - return table.getBaseSchema() - .stream() - .map(col -> SlotReference.fromColumn(col, qualified())) - .collect(ImmutableList.toImmutableList()); + return Objects.hash(relationId); } @Override @@ -115,31 +77,15 @@ public abstract class LogicalRelation extends LogicalLeaf implements Scan { return ImmutableList.of(); } - /** - * Full qualified name parts, i.e., concat qualifier and name into a list. - */ - public List qualified() { - return Utils.qualifiedNameParts(qualifier, table.getName()); - } - - /** - * Full qualified table name, concat qualifier and name with `.` as separator. - */ - public String qualifiedName() { - return Utils.qualifiedName(qualifier, table.getName()); - } - - public ObjectId getId() { - return id; + public RelationId getRelationId() { + return relationId; } @Override public JSONObject toJson() { JSONObject logicalRelation = super.toJson(); JSONObject properties = new JSONObject(); - properties.put("ObjectId", id.toString()); - properties.put("Table", table.toString()); - properties.put("Qualifier", qualifier.toString()); + properties.put("RelationId", relationId.toString()); logicalRelation.put("Properties", properties); return logicalRelation; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSchemaScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSchemaScan.java index 1b0e51d17b..24cabfc04e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSchemaScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSchemaScan.java @@ -21,10 +21,9 @@ import org.apache.doris.catalog.SchemaTable; import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; -import org.apache.doris.nereids.trees.plans.algebra.Scan; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -34,13 +33,13 @@ import java.util.Optional; /** * LogicalSchemaScan. */ -public class LogicalSchemaScan extends LogicalRelation implements Scan { +public class LogicalSchemaScan extends LogicalCatalogRelation { - public LogicalSchemaScan(ObjectId id, TableIf table, List qualifier) { + public LogicalSchemaScan(RelationId id, TableIf table, List qualifier) { super(id, PlanType.LOGICAL_SCHEMA_SCAN, table, qualifier); } - public LogicalSchemaScan(ObjectId id, TableIf table, List qualifier, + public LogicalSchemaScan(RelationId id, TableIf table, List qualifier, Optional groupExpression, Optional logicalProperties) { super(id, PlanType.LOGICAL_SCHEMA_SCAN, table, qualifier, groupExpression, logicalProperties); } @@ -57,23 +56,14 @@ public class LogicalSchemaScan extends LogicalRelation implements Scan { @Override public Plan withGroupExpression(Optional groupExpression) { - return new LogicalSchemaScan(id, table, qualifier, groupExpression, Optional.of(getLogicalProperties())); + return new LogicalSchemaScan(relationId, table, qualifier, + groupExpression, Optional.of(getLogicalProperties())); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalSchemaScan(id, table, qualifier, groupExpression, logicalProperties); - } - - @Override - public boolean equals(Object o) { - return super.equals(o); - } - - @Override - public int hashCode() { - return super.hashCode(); + return new LogicalSchemaScan(relationId, table, qualifier, groupExpression, logicalProperties); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java index 308cd2a9e9..72a4996ece 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.UnboundLogicalProperties; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -103,25 +104,17 @@ public abstract class LogicalSetOperation extends AbstractLogicalPlan implements .collect(ImmutableList.toImmutableList()); } - public List> collectCastExpressions() { - return castCommonDataTypeOutputs(resetNullableForLeftOutputs()); + public List> collectChildrenProjections() { + return castCommonDataTypeOutputs(); } /** * Generate new output for SetOperation. */ - public List buildNewOutputs(List leftCastExpressions) { + public List buildNewOutputs() { ImmutableList.Builder newOutputs = new Builder<>(); - for (Expression expression : leftCastExpressions) { - if (expression instanceof Cast) { - Cast cast = ((Cast) expression); - newOutputs.add(new SlotReference( - cast.child().toSql(), expression.getDataType(), - cast.child().nullable())); - } else if (expression instanceof Slot) { - Slot slot = ((Slot) expression); - newOutputs.add(new SlotReference(slot.toSql(), slot.getDataType(), slot.nullable())); - } + for (Slot slot : resetNullableForLeftOutputs()) { + newOutputs.add(new SlotReference(slot.toSql(), slot.getDataType(), slot.nullable())); } return newOutputs.build(); } @@ -140,12 +133,12 @@ public abstract class LogicalSetOperation extends AbstractLogicalPlan implements return ImmutableList.copyOf(resetNullableForLeftOutputs); } - private List> castCommonDataTypeOutputs(List resetNullableForLeftOutputs) { - List newLeftOutputs = new ArrayList<>(); - List newRightOutputs = new ArrayList<>(); + private List> castCommonDataTypeOutputs() { + List newLeftOutputs = new ArrayList<>(); + List newRightOutputs = new ArrayList<>(); // Ensure that the output types of the left and right children are consistent and expand upward. - for (int i = 0; i < resetNullableForLeftOutputs.size(); ++i) { - Slot left = resetNullableForLeftOutputs.get(i); + for (int i = 0; i < child(0).getOutput().size(); ++i) { + Slot left = child(0).getOutput().get(i); Slot right = child(1).getOutput().get(i); DataType compatibleType = DataType.fromCatalogType(Type.getAssignmentCompatibleType( left.getDataType().toCatalogDataType(), @@ -153,11 +146,17 @@ public abstract class LogicalSetOperation extends AbstractLogicalPlan implements false)); Expression newLeft = TypeCoercionUtils.castIfNotSameType(left, compatibleType); Expression newRight = TypeCoercionUtils.castIfNotSameType(right, compatibleType); - newLeftOutputs.add(newLeft); - newRightOutputs.add(newRight); + if (newLeft instanceof Cast) { + newLeft = new Alias(newLeft, left.getName()); + } + if (newRight instanceof Cast) { + newRight = new Alias(newRight, right.getName()); + } + newLeftOutputs.add((NamedExpression) newLeft); + newRightOutputs.add((NamedExpression) newRight); } - List> resultExpressions = new ArrayList<>(); + List> resultExpressions = new ArrayList<>(); resultExpressions.add(newLeftOutputs); resultExpressions.add(newRightOutputs); return ImmutableList.copyOf(resultExpressions); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java index 53bbee0002..ef8588b1fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java @@ -19,10 +19,8 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; @@ -46,8 +44,6 @@ public class LogicalSubQueryAlias extends LogicalUnary< private final List qualifier; private final Optional> columnAliases; - private final CTEId cteId; - public LogicalSubQueryAlias(String tableAlias, CHILD_TYPE child) { this(ImmutableList.of(tableAlias), Optional.empty(), Optional.empty(), Optional.empty(), child); } @@ -70,24 +66,12 @@ public class LogicalSubQueryAlias extends LogicalUnary< super(PlanType.LOGICAL_SUBQUERY_ALIAS, groupExpression, logicalProperties, child); this.qualifier = ImmutableList.copyOf(Objects.requireNonNull(qualifier, "qualifier is null")); this.columnAliases = columnAliases; - this.cteId = cteId(); - } - - public LogicalSubQueryAlias(List qualifier, Optional> columnAliases, - Optional groupExpression, - Optional logicalProperties, CHILD_TYPE child, CTEId cteId) { - super(PlanType.LOGICAL_SUBQUERY_ALIAS, groupExpression, logicalProperties, child); - this.qualifier = ImmutableList.copyOf(Objects.requireNonNull(qualifier)); - this.columnAliases = columnAliases; - this.cteId = cteId; } @Override public List computeOutput() { List childOutput = child().getOutput(); - List columnAliases = this.columnAliases.isPresent() - ? this.columnAliases.get() - : ImmutableList.of(); + List columnAliases = this.columnAliases.orElseGet(ImmutableList::of); ImmutableList.Builder currentOutput = ImmutableList.builder(); for (int i = 0; i < childOutput.size(); i++) { Slot originSlot = childOutput.get(i); @@ -115,15 +99,12 @@ public class LogicalSubQueryAlias extends LogicalUnary< @Override public String toString() { - if (columnAliases.isPresent()) { - return Utils.toSqlString("LogicalSubQueryAlias", - "qualifier", qualifier, - "columnAliases", StringUtils.join(columnAliases.get(), ",") - ); - } - return Utils.toSqlString("LogicalSubQueryAlias", + return columnAliases.map(strings -> Utils.toSqlString("LogicalSubQueryAlias", + "qualifier", qualifier, + "columnAliases", StringUtils.join(strings, ",") + )).orElseGet(() -> Utils.toSqlString("LogicalSubQueryAlias", "qualifier", qualifier - ); + )); } @Override @@ -172,12 +153,4 @@ public class LogicalSubQueryAlias extends LogicalUnary< return new LogicalSubQueryAlias<>(qualifier, columnAliases, groupExpression, logicalProperties, children.get(0)); } - - public CTEId cteId() { - return StatementScopeIdGenerator.newCTEId(); - } - - public CTEId getCteId() { - return cteId; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java index e068048d28..6c5f554b2f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java @@ -19,10 +19,12 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.TVFRelation; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -38,41 +40,64 @@ public class LogicalTVFRelation extends LogicalRelation implements TVFRelation { private final TableValuedFunction function; - public LogicalTVFRelation(ObjectId id, TableValuedFunction function) { - super(id, PlanType.LOGICAL_TVF_RELATION, - Objects.requireNonNull(function, "table valued function can not be null").getTable(), - ImmutableList.of()); + public LogicalTVFRelation(RelationId id, TableValuedFunction function) { + super(id, PlanType.LOGICAL_TVF_RELATION); this.function = function; } - public LogicalTVFRelation(ObjectId id, TableValuedFunction function, Optional groupExpression, + public LogicalTVFRelation(RelationId id, TableValuedFunction function, Optional groupExpression, Optional logicalProperties) { - super(id, PlanType.LOGICAL_TVF_RELATION, - Objects.requireNonNull(function, "table valued function can not be null").getTable(), - ImmutableList.of(), groupExpression, logicalProperties); + super(id, PlanType.LOGICAL_TVF_RELATION, groupExpression, logicalProperties); this.function = function; } @Override public LogicalTVFRelation withGroupExpression(Optional groupExpression) { - return new LogicalTVFRelation(id, function, groupExpression, Optional.of(getLogicalProperties())); + return new LogicalTVFRelation(relationId, function, groupExpression, Optional.of(getLogicalProperties())); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalTVFRelation(id, function, groupExpression, logicalProperties); + return new LogicalTVFRelation(relationId, function, groupExpression, logicalProperties); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + LogicalTVFRelation that = (LogicalTVFRelation) o; + return Objects.equals(function, that.function); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), function); } @Override public String toString() { return Utils.toSqlString("LogicalTVFRelation", - "qualified", qualifiedName(), "output", getOutput(), "function", function.toSql() ); } + @Override + public List computeOutput() { + return function.getTable().getBaseSchema() + .stream() + .map(col -> SlotReference.fromColumn(col, ImmutableList.of())) + .collect(ImmutableList.toImmutableList()); + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitLogicalTVFRelation(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEAnchor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEAnchor.java index d85a73d1b2..5ec0829c22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEAnchor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEAnchor.java @@ -45,22 +45,18 @@ public class PhysicalCTEAnchor< private final CTEId cteId; public PhysicalCTEAnchor(CTEId cteId, LogicalProperties logicalProperties, - LEFT_CHILD_TYPE leftChild, - RIGHT_CHILD_TYPE rightChild) { + LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { this(cteId, Optional.empty(), logicalProperties, leftChild, rightChild); } public PhysicalCTEAnchor(CTEId cteId, Optional groupExpression, - LogicalProperties logicalProperties, - LEFT_CHILD_TYPE leftChild, - RIGHT_CHILD_TYPE rightChild) { + LogicalProperties logicalProperties, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { this(cteId, groupExpression, logicalProperties, null, null, leftChild, rightChild); } public PhysicalCTEAnchor(CTEId cteId, Optional groupExpression, - LogicalProperties logicalProperties, PhysicalProperties physicalProperties, - Statistics statistics, LEFT_CHILD_TYPE leftChild, - RIGHT_CHILD_TYPE rightChild) { + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, + Statistics statistics, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { super(PlanType.PHYSICAL_CTE_ANCHOR, groupExpression, logicalProperties, physicalProperties, statistics, leftChild, rightChild); this.cteId = cteId; @@ -108,7 +104,7 @@ public class PhysicalCTEAnchor< public PhysicalCTEAnchor withChildren(List children) { Preconditions.checkArgument(children.size() == 2); return new PhysicalCTEAnchor<>(cteId, groupExpression, getLogicalProperties(), physicalProperties, - statistics, children.get(0), children.get(1)); + statistics, children.get(0), children.get(1)); } @Override @@ -126,13 +122,14 @@ public class PhysicalCTEAnchor< @Override public PhysicalCTEAnchor withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, - Statistics statistics) { + Statistics statistics) { return new PhysicalCTEAnchor<>(cteId, groupExpression, getLogicalProperties(), physicalProperties, - statistics, child(0), child(1)); + statistics, child(0), child(1)); } @Override public String shapeInfo() { - return Utils.toSqlString("CteAnchor[cteId=", cteId, "]"); + return Utils.toSqlString("PhysicalCteAnchor", + "cteId", cteId); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEConsumer.java index 91c0f2a403..e6450a94e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEConsumer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEConsumer.java @@ -17,23 +17,19 @@ package org.apache.doris.nereids.trees.plans.physical; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.nereids.exceptions.TransformException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.CTEId; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; -import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.nereids.util.Utils; import org.apache.doris.statistics.Statistics; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.List; @@ -53,48 +49,35 @@ public class PhysicalCTEConsumer extends PhysicalRelation { /** * Constructor */ - public PhysicalCTEConsumer(CTEId cteId, Map consumerToProducerSlotMap, - Map producerToConsumerSlotMap, - LogicalProperties logicalProperties) { - this(cteId, consumerToProducerSlotMap, producerToConsumerSlotMap, + public PhysicalCTEConsumer(RelationId relationId, CTEId cteId, Map consumerToProducerSlotMap, + Map producerToConsumerSlotMap, LogicalProperties logicalProperties) { + this(relationId, cteId, consumerToProducerSlotMap, producerToConsumerSlotMap, Optional.empty(), logicalProperties); } /** * Constructor */ - public PhysicalCTEConsumer(CTEId cteId, Map consumerToProducerSlotMap, - Map producerToConsumerSlotMap, - Optional groupExpression, - LogicalProperties logicalProperties) { - this(cteId, consumerToProducerSlotMap, producerToConsumerSlotMap, groupExpression, logicalProperties, - null, null); + public PhysicalCTEConsumer(RelationId relationId, CTEId cteId, + Map consumerToProducerSlotMap, Map producerToConsumerSlotMap, + Optional groupExpression, LogicalProperties logicalProperties) { + this(relationId, cteId, consumerToProducerSlotMap, producerToConsumerSlotMap, + groupExpression, logicalProperties, null, null); } /** * Constructor */ - public PhysicalCTEConsumer(CTEId cteId, Map consumerToProducerSlotMap, - Map producerToConsumerSlotMap, - Optional groupExpression, - LogicalProperties logicalProperties, - PhysicalProperties physicalProperties, - Statistics statistics) { - super(RelationUtil.newRelationId(), PlanType.PHYSICAL_CTE_CONSUME, ImmutableList.of(), groupExpression, + public PhysicalCTEConsumer(RelationId relationId, CTEId cteId, Map consumerToProducerSlotMap, + Map producerToConsumerSlotMap, Optional groupExpression, + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { + super(relationId, PlanType.PHYSICAL_CTE_CONSUME, groupExpression, logicalProperties, physicalProperties, statistics); this.cteId = cteId; - this.consumerToProducerSlotMap = ImmutableMap.copyOf(consumerToProducerSlotMap); - this.producerToConsumerSlotMap = ImmutableMap.copyOf(producerToConsumerSlotMap); - } - - @Override - public OlapTable getTable() { - throw new TransformException("should not reach here"); - } - - @Override - public List getQualifier() { - throw new TransformException("should not reach here"); + this.consumerToProducerSlotMap = ImmutableMap.copyOf(Objects.requireNonNull( + consumerToProducerSlotMap, "consumerToProducerSlotMap should not null")); + this.producerToConsumerSlotMap = ImmutableMap.copyOf(Objects.requireNonNull( + producerToConsumerSlotMap, "consumerToProducerSlotMap should not null")); } public CTEId getCteId() { @@ -105,35 +88,17 @@ public class PhysicalCTEConsumer extends PhysicalRelation { return producerToConsumerSlotMap; } - @Override - public List getExpressions() { - return ImmutableList.of(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - - if (!super.equals(o)) { - return false; - } - - PhysicalCTEConsumer that = (PhysicalCTEConsumer) o; - return Objects.equals(cteId, that.cteId) - && Objects.equals(producerToConsumerSlotMap, that.producerToConsumerSlotMap) - && Objects.equals(consumerToProducerSlotMap, that.consumerToProducerSlotMap); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), cteId, producerToConsumerSlotMap, consumerToProducerSlotMap); + public Slot getProducerSlot(Slot consumerSlot) { + Slot slot = consumerToProducerSlotMap.get(consumerSlot); + Preconditions.checkArgument(slot != null, String.format( + "Required producer slot for %s doesn't exist", consumerSlot)); + return slot; } @Override public String toString() { - return Utils.toSqlString("PhysicalCTEConsumer", "cteId", cteId); + return Utils.toSqlString("PhysicalCTEConsumer[" + id.asInt() + "]", + "cteId", cteId); } @Override @@ -141,42 +106,32 @@ public class PhysicalCTEConsumer extends PhysicalRelation { return visitor.visitPhysicalCTEConsumer(this, context); } - @Override - public PhysicalCTEConsumer withChildren(List children) { - Preconditions.checkArgument(children.isEmpty()); - return new PhysicalCTEConsumer(cteId, consumerToProducerSlotMap, producerToConsumerSlotMap, - getLogicalProperties()); - } - @Override public PhysicalCTEConsumer withGroupExpression(Optional groupExpression) { - return new PhysicalCTEConsumer(cteId, consumerToProducerSlotMap, producerToConsumerSlotMap, + return new PhysicalCTEConsumer(relationId, cteId, + consumerToProducerSlotMap, producerToConsumerSlotMap, groupExpression, getLogicalProperties()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalCTEConsumer(cteId, consumerToProducerSlotMap, producerToConsumerSlotMap, + return new PhysicalCTEConsumer(relationId, cteId, + consumerToProducerSlotMap, producerToConsumerSlotMap, groupExpression, logicalProperties.get()); } @Override public PhysicalCTEConsumer withPhysicalPropertiesAndStats( PhysicalProperties physicalProperties, Statistics statistics) { - return new PhysicalCTEConsumer(cteId, consumerToProducerSlotMap, producerToConsumerSlotMap, + return new PhysicalCTEConsumer(relationId, cteId, + consumerToProducerSlotMap, producerToConsumerSlotMap, groupExpression, getLogicalProperties(), physicalProperties, statistics); } @Override public String shapeInfo() { - return Utils.toSqlString("CteConsumer[cteId=", cteId, "]"); - } - - public Slot findProducerSlot(Slot consumerSlot) { - Slot slot = consumerToProducerSlotMap.get(consumerSlot); - Preconditions.checkArgument(slot != null, String.format("Required producer" - + "slot for :%s doesn't exist", consumerSlot)); - return slot; + return Utils.toSqlString("PhysicalCteConsumer", + "cteId", cteId); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEProducer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEProducer.java index bf148b17ac..787649519d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEProducer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEProducer.java @@ -22,7 +22,6 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; @@ -42,35 +41,27 @@ import java.util.Optional; public class PhysicalCTEProducer extends PhysicalUnary { private final CTEId cteId; - private final List projects; - public PhysicalCTEProducer(CTEId cteId, List projects, - LogicalProperties logicalProperties, CHILD_TYPE child) { - this(cteId, projects, Optional.empty(), logicalProperties, child); + public PhysicalCTEProducer(CTEId cteId, LogicalProperties logicalProperties, CHILD_TYPE child) { + this(cteId, Optional.empty(), logicalProperties, child); } - public PhysicalCTEProducer(CTEId cteId, List projects, - Optional groupExpression, + public PhysicalCTEProducer(CTEId cteId, Optional groupExpression, LogicalProperties logicalProperties, CHILD_TYPE child) { - this(cteId, projects, groupExpression, logicalProperties, null, null, child); + this(cteId, groupExpression, logicalProperties, null, null, child); } - public PhysicalCTEProducer(CTEId cteId, List projects, Optional groupExpression, - LogicalProperties logicalProperties, PhysicalProperties physicalProperties, - Statistics statistics, CHILD_TYPE child) { + public PhysicalCTEProducer(CTEId cteId, Optional groupExpression, + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, + Statistics statistics, CHILD_TYPE child) { super(PlanType.PHYSICAL_CTE_PRODUCE, groupExpression, logicalProperties, physicalProperties, statistics, child); this.cteId = cteId; - this.projects = ImmutableList.copyOf(projects); } public CTEId getCteId() { return cteId; } - public List getProjects() { - return projects; - } - @Override public List getExpressions() { return ImmutableList.of(); @@ -97,7 +88,8 @@ public class PhysicalCTEProducer extends PhysicalUnary< @Override public String toString() { - return Utils.toSqlString("PhysicalCTEProducer", "cteId", cteId); + return Utils.toSqlString("PhysicalCTEProducer[" + id.asInt() + "]", + "cteId", cteId); } @Override @@ -108,29 +100,31 @@ public class PhysicalCTEProducer extends PhysicalUnary< @Override public PhysicalCTEProducer withChildren(List children) { Preconditions.checkArgument(children.size() == 1); - return new PhysicalCTEProducer<>(cteId, projects, getLogicalProperties(), children.get(0)); + return new PhysicalCTEProducer<>(cteId, groupExpression, + getLogicalProperties(), physicalProperties, statistics, children.get(0)); } @Override public PhysicalCTEProducer withGroupExpression(Optional groupExpression) { - return new PhysicalCTEProducer<>(cteId, projects, groupExpression, getLogicalProperties(), child()); + return new PhysicalCTEProducer<>(cteId, groupExpression, getLogicalProperties(), child()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalCTEProducer<>(cteId, projects, groupExpression, logicalProperties.get(), children.get(0)); + return new PhysicalCTEProducer<>(cteId, groupExpression, logicalProperties.get(), children.get(0)); } @Override public PhysicalCTEProducer withPhysicalPropertiesAndStats( PhysicalProperties physicalProperties, Statistics statistics) { - return new PhysicalCTEProducer<>(cteId, projects, groupExpression, getLogicalProperties(), physicalProperties, + return new PhysicalCTEProducer<>(cteId, groupExpression, getLogicalProperties(), physicalProperties, statistics, child()); } @Override public String shapeInfo() { - return Utils.toSqlString("CteProducer[cteId=", cteId, "]"); + return Utils.toSqlString("PhysicalCteProducer", + "cteId", cteId); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java new file mode 100644 index 0000000000..b25a7c530a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCatalogRelation.java @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.physical; + +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.util.Utils; +import org.apache.doris.statistics.Statistics; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * relation generated from TableIf + */ +public abstract class PhysicalCatalogRelation extends PhysicalRelation implements CatalogRelation { + + protected final TableIf table; + protected final ImmutableList qualifier; + + /** + * Constructor for PhysicalCatalogRelation. + * + * @param table Doris table + * @param qualifier qualified relation name + */ + public PhysicalCatalogRelation(RelationId relationId, PlanType type, TableIf table, List qualifier, + Optional groupExpression, LogicalProperties logicalProperties) { + super(relationId, type, groupExpression, logicalProperties); + this.table = Objects.requireNonNull(table, "table can not be null"); + this.qualifier = ImmutableList.copyOf(Objects.requireNonNull(qualifier, "qualifier can not be null")); + } + + /** + * Constructor for PhysicalCatalogRelation. + * + * @param table Doris table + * @param qualifier qualified relation name + */ + public PhysicalCatalogRelation(RelationId relationId, PlanType type, TableIf table, List qualifier, + Optional groupExpression, LogicalProperties logicalProperties, + PhysicalProperties physicalProperties, + Statistics statistics) { + super(relationId, type, groupExpression, logicalProperties, physicalProperties, statistics); + this.table = Objects.requireNonNull(table, "table can not be null"); + this.qualifier = ImmutableList.copyOf(Objects.requireNonNull(qualifier, "qualifier can not be null")); + } + + @Override + public TableIf getTable() { + return table; + } + + @Override + public Database getDatabase() throws AnalysisException { + Preconditions.checkArgument(!qualifier.isEmpty()); + return Env.getCurrentInternalCatalog().getDbOrException(qualifier.get(0), + s -> new AnalysisException("Database [" + qualifier.get(0) + "] does not exist.")); + } + + @Override + public List computeOutput() { + return table.getBaseSchema() + .stream() + .map(col -> SlotReference.fromColumn(col, qualified())) + .collect(ImmutableList.toImmutableList()); + } + + public List getQualifier() { + return qualifier; + } + + /** + * Full qualified name parts, i.e., concat qualifier and name into a list. + */ + public List qualified() { + return Utils.qualifiedNameParts(qualifier, table.getName()); + } + + /** + * Full qualified table name, concat qualifier and name with `.` as separator. + */ + public String qualifiedName() { + return Utils.qualifiedName(qualifier, table.getName()); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEmptyRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEmptyRelation.java index d0865b83b8..2a9c344bd4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEmptyRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEmptyRelation.java @@ -20,11 +20,11 @@ package org.apache.doris.nereids.trees.plans.physical; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.EmptyRelation; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -41,19 +41,21 @@ import java.util.Optional; * e.g. * select * from tbl limit 0 */ -public class PhysicalEmptyRelation extends PhysicalLeaf implements EmptyRelation { +public class PhysicalEmptyRelation extends PhysicalRelation implements EmptyRelation { private final List projects; - public PhysicalEmptyRelation(List projects, LogicalProperties logicalProperties) { - this(projects, Optional.empty(), logicalProperties, null, null); + public PhysicalEmptyRelation(RelationId relationId, List projects, + LogicalProperties logicalProperties) { + this(relationId, projects, Optional.empty(), logicalProperties, null, null); } - public PhysicalEmptyRelation(List projects, Optional groupExpression, + public PhysicalEmptyRelation(RelationId relationId, List projects, + Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { - super(PlanType.PHYSICAL_EMPTY_RELATION, groupExpression, logicalProperties, physicalProperties, - statistics); + super(relationId, PlanType.PHYSICAL_EMPTY_RELATION, groupExpression, + logicalProperties, physicalProperties, statistics); this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, "projects can not be null")); } @@ -62,21 +64,16 @@ public class PhysicalEmptyRelation extends PhysicalLeaf implements EmptyRelation return visitor.visitPhysicalEmptyRelation(this, context); } - @Override - public List getExpressions() { - return ImmutableList.of(); - } - @Override public Plan withGroupExpression(Optional groupExpression) { - return new PhysicalEmptyRelation(projects, groupExpression, + return new PhysicalEmptyRelation(relationId, projects, groupExpression, logicalPropertiesSupplier.get(), physicalProperties, statistics); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalEmptyRelation(projects, groupExpression, + return new PhysicalEmptyRelation(relationId, projects, groupExpression, logicalProperties.get(), physicalProperties, statistics); } @@ -94,26 +91,6 @@ public class PhysicalEmptyRelation extends PhysicalLeaf implements EmptyRelation ); } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - PhysicalEmptyRelation that = (PhysicalEmptyRelation) o; - return Objects.equals(projects, that.projects); - } - - @Override - public int hashCode() { - return Objects.hash(projects); - } - @Override public List getProjects() { return projects; @@ -122,7 +99,7 @@ public class PhysicalEmptyRelation extends PhysicalLeaf implements EmptyRelation @Override public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { - return new PhysicalEmptyRelation(projects, Optional.empty(), + return new PhysicalEmptyRelation(relationId, projects, Optional.empty(), logicalPropertiesSupplier.get(), physicalProperties, statistics); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java index 2dcc0bd0cc..4ee49ca198 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java @@ -22,46 +22,41 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import org.apache.doris.statistics.Statistics; import java.util.List; -import java.util.Objects; import java.util.Optional; /** * Physical es scan for external catalog. */ -public class PhysicalEsScan extends PhysicalRelation { +public class PhysicalEsScan extends PhysicalCatalogRelation { - private final ExternalTable table; private final DistributionSpec distributionSpec; /** * Constructor for PhysicalEsScan. */ - public PhysicalEsScan(ObjectId id, ExternalTable table, List qualifier, - DistributionSpec distributionSpec, Optional groupExpression, - LogicalProperties logicalProperties) { - super(id, PlanType.PHYSICAL_ES_SCAN, qualifier, groupExpression, logicalProperties); - this.table = table; + public PhysicalEsScan(RelationId id, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties) { + super(id, PlanType.PHYSICAL_ES_SCAN, table, qualifier, groupExpression, logicalProperties); this.distributionSpec = distributionSpec; } /** * Constructor for PhysicalEsScan. */ - public PhysicalEsScan(ObjectId id, ExternalTable table, List qualifier, - DistributionSpec distributionSpec, Optional groupExpression, - LogicalProperties logicalProperties, PhysicalProperties physicalProperties, - Statistics statistics) { - super(id, PlanType.PHYSICAL_ES_SCAN, qualifier, groupExpression, logicalProperties, + public PhysicalEsScan(RelationId id, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { + super(id, PlanType.PHYSICAL_ES_SCAN, table, qualifier, groupExpression, logicalProperties, physicalProperties, statistics); - this.table = table; this.distributionSpec = distributionSpec; } @@ -74,23 +69,6 @@ public class PhysicalEsScan extends PhysicalRelation { ); } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass() || !super.equals(o)) { - return false; - } - PhysicalEsScan that = ((PhysicalEsScan) o); - return Objects.equals(table, that.table); - } - - @Override - public int hashCode() { - return Objects.hash(id, table); - } - @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitPhysicalEsScan(this, context); @@ -98,24 +76,26 @@ public class PhysicalEsScan extends PhysicalRelation { @Override public PhysicalEsScan withGroupExpression(Optional groupExpression) { - return new PhysicalEsScan(id, table, qualifier, distributionSpec, groupExpression, getLogicalProperties()); + return new PhysicalEsScan(relationId, getTable(), qualifier, distributionSpec, + groupExpression, getLogicalProperties()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalEsScan(id, table, qualifier, distributionSpec, groupExpression, logicalProperties.get()); + return new PhysicalEsScan(relationId, getTable(), qualifier, distributionSpec, + groupExpression, logicalProperties.get()); } @Override public ExternalTable getTable() { - return table; + return (ExternalTable) table; } @Override public PhysicalEsScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statsDeriveResult) { - return new PhysicalEsScan(id, table, qualifier, distributionSpec, groupExpression, getLogicalProperties(), - physicalProperties, statsDeriveResult); + return new PhysicalEsScan(relationId, getTable(), qualifier, distributionSpec, + groupExpression, getLogicalProperties(), physicalProperties, statsDeriveResult); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java index f9584c0c46..b5af33d107 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java @@ -23,35 +23,32 @@ import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import org.apache.doris.statistics.Statistics; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.Set; /** * Physical file scan for external catalog. */ -public class PhysicalFileScan extends PhysicalRelation { +public class PhysicalFileScan extends PhysicalCatalogRelation { - private final ExternalTable table; private final DistributionSpec distributionSpec; private final Set conjuncts; /** * Constructor for PhysicalFileScan. */ - public PhysicalFileScan(ObjectId id, ExternalTable table, List qualifier, - DistributionSpec distributionSpec, Optional groupExpression, - LogicalProperties logicalProperties, Set conjuncts) { - super(id, PlanType.PHYSICAL_FILE_SCAN, qualifier, groupExpression, logicalProperties); - this.table = table; + public PhysicalFileScan(RelationId id, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, Set conjuncts) { + super(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, groupExpression, logicalProperties); this.distributionSpec = distributionSpec; this.conjuncts = conjuncts; } @@ -59,13 +56,12 @@ public class PhysicalFileScan extends PhysicalRelation { /** * Constructor for PhysicalFileScan. */ - public PhysicalFileScan(ObjectId id, ExternalTable table, List qualifier, - DistributionSpec distributionSpec, Optional groupExpression, - LogicalProperties logicalProperties, PhysicalProperties physicalProperties, - Statistics statistics, Set conjuncts) { - super(id, PlanType.PHYSICAL_FILE_SCAN, qualifier, groupExpression, logicalProperties, + public PhysicalFileScan(RelationId id, ExternalTable table, List qualifier, + DistributionSpec distributionSpec, Optional groupExpression, + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, + Statistics statistics, Set conjuncts) { + super(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, groupExpression, logicalProperties, physicalProperties, statistics); - this.table = table; this.distributionSpec = distributionSpec; this.conjuncts = conjuncts; } @@ -79,23 +75,6 @@ public class PhysicalFileScan extends PhysicalRelation { ); } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass() || !super.equals(o)) { - return false; - } - PhysicalFileScan that = ((PhysicalFileScan) o); - return Objects.equals(table, that.table); - } - - @Override - public int hashCode() { - return Objects.hash(id, table); - } - @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitPhysicalFileScan(this, context); @@ -103,27 +82,27 @@ public class PhysicalFileScan extends PhysicalRelation { @Override public PhysicalFileScan withGroupExpression(Optional groupExpression) { - return new PhysicalFileScan(id, table, qualifier, distributionSpec, + return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, groupExpression, getLogicalProperties(), conjuncts); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalFileScan(id, table, qualifier, distributionSpec, + return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, groupExpression, logicalProperties.get(), conjuncts); } @Override public ExternalTable getTable() { - return table; + return (ExternalTable) table; } @Override public PhysicalFileScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { - return new PhysicalFileScan(id, table, qualifier, distributionSpec, groupExpression, getLogicalProperties(), - physicalProperties, statistics, conjuncts); + return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, + groupExpression, getLogicalProperties(), physicalProperties, statistics, conjuncts); } public Set getConjuncts() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalJdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalJdbcScan.java index dd98dd7452..f5a68e45fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalJdbcScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalJdbcScan.java @@ -19,50 +19,39 @@ package org.apache.doris.nereids.trees.plans.physical; import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import org.apache.doris.statistics.Statistics; import java.util.List; -import java.util.Objects; import java.util.Optional; /** * Physical jdbc scan for external catalog. */ -public class PhysicalJdbcScan extends PhysicalRelation { - - private final TableIf table; - private final DistributionSpec distributionSpec; +public class PhysicalJdbcScan extends PhysicalCatalogRelation { /** * Constructor for PhysicalJdbcScan. */ - public PhysicalJdbcScan(ObjectId id, TableIf table, List qualifier, - DistributionSpec distributionSpec, Optional groupExpression, - LogicalProperties logicalProperties) { - super(id, PlanType.PHYSICAL_JDBC_SCAN, qualifier, groupExpression, logicalProperties); - this.table = table; - this.distributionSpec = distributionSpec; + public PhysicalJdbcScan(RelationId id, TableIf table, List qualifier, + Optional groupExpression, LogicalProperties logicalProperties) { + super(id, PlanType.PHYSICAL_JDBC_SCAN, table, qualifier, groupExpression, logicalProperties); } /** * Constructor for PhysicalJdbcScan. */ - public PhysicalJdbcScan(ObjectId id, TableIf table, List qualifier, - DistributionSpec distributionSpec, Optional groupExpression, - LogicalProperties logicalProperties, PhysicalProperties physicalProperties, - Statistics statistics) { - super(id, PlanType.PHYSICAL_JDBC_SCAN, qualifier, groupExpression, logicalProperties, - physicalProperties, statistics); - this.table = table; - this.distributionSpec = distributionSpec; + public PhysicalJdbcScan(RelationId id, TableIf table, List qualifier, + Optional groupExpression, + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { + super(id, PlanType.PHYSICAL_JDBC_SCAN, table, qualifier, groupExpression, + logicalProperties, physicalProperties, statistics); } @Override @@ -74,23 +63,6 @@ public class PhysicalJdbcScan extends PhysicalRelation { ); } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass() || !super.equals(o)) { - return false; - } - PhysicalJdbcScan that = ((PhysicalJdbcScan) o); - return Objects.equals(table, that.table); - } - - @Override - public int hashCode() { - return Objects.hash(id, table); - } - @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitPhysicalJdbcScan(this, context); @@ -98,24 +70,19 @@ public class PhysicalJdbcScan extends PhysicalRelation { @Override public PhysicalJdbcScan withGroupExpression(Optional groupExpression) { - return new PhysicalJdbcScan(id, table, qualifier, distributionSpec, groupExpression, getLogicalProperties()); + return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression, getLogicalProperties()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalJdbcScan(id, table, qualifier, distributionSpec, groupExpression, logicalProperties.get()); - } - - @Override - public TableIf getTable() { - return table; + return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression, logicalProperties.get()); } @Override public PhysicalJdbcScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { - return new PhysicalJdbcScan(id, table, qualifier, distributionSpec, groupExpression, getLogicalProperties(), - physicalProperties, statistics); + return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression, + getLogicalProperties(), physicalProperties, statistics); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java index 87823d9bc7..b98d30f4e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java @@ -24,10 +24,10 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.AbstractPlan; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.PreAggStatus; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.OlapScan; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -43,23 +43,21 @@ import java.util.Optional; /** * Physical olap scan plan. */ -public class PhysicalOlapScan extends PhysicalRelation implements OlapScan { +public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapScan { public static final String DEFERRED_MATERIALIZED_SLOTS = "deferred_materialized_slots"; - private final OlapTable olapTable; private final DistributionSpec distributionSpec; private final long selectedIndexId; private final ImmutableList selectedTabletIds; private final ImmutableList selectedPartitionIds; private final PreAggStatus preAggStatus; - private final List baseOutputs; /** * Constructor for PhysicalOlapScan. */ - public PhysicalOlapScan(ObjectId id, OlapTable olapTable, List qualifier, long selectedIndexId, + public PhysicalOlapScan(RelationId id, OlapTable olapTable, List qualifier, long selectedIndexId, List selectedTabletIds, List selectedPartitionIds, DistributionSpec distributionSpec, PreAggStatus preAggStatus, List baseOutputs, Optional groupExpression, LogicalProperties logicalProperties) { @@ -70,14 +68,13 @@ public class PhysicalOlapScan extends PhysicalRelation implements OlapScan { /** * Constructor for PhysicalOlapScan. */ - public PhysicalOlapScan(ObjectId id, OlapTable olapTable, List qualifier, long selectedIndexId, + public PhysicalOlapScan(RelationId id, OlapTable olapTable, List qualifier, long selectedIndexId, List selectedTabletIds, List selectedPartitionIds, DistributionSpec distributionSpec, PreAggStatus preAggStatus, List baseOutputs, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { - super(id, PlanType.PHYSICAL_OLAP_SCAN, qualifier, groupExpression, logicalProperties, physicalProperties, - statistics); - this.olapTable = olapTable; + super(id, PlanType.PHYSICAL_OLAP_SCAN, olapTable, qualifier, + groupExpression, logicalProperties, physicalProperties, statistics); this.selectedIndexId = selectedIndexId; this.selectedTabletIds = ImmutableList.copyOf(selectedTabletIds); this.selectedPartitionIds = ImmutableList.copyOf(selectedPartitionIds); @@ -102,7 +99,7 @@ public class PhysicalOlapScan extends PhysicalRelation implements OlapScan { @Override public OlapTable getTable() { - return olapTable; + return (OlapTable) table; } public DistributionSpec getDistributionSpec() { @@ -119,8 +116,8 @@ public class PhysicalOlapScan extends PhysicalRelation implements OlapScan { @Override public String toString() { - return Utils.toSqlString("PhysicalOlapScan[" + id.asInt() + "]" + getGroupIdAsString(), - "qualified", Utils.qualifiedName(qualifier, olapTable.getName()), + return Utils.toSqlString("PhysicalOlapScan[" + relationId.asInt() + "]" + getGroupIdAsString(), + "qualified", Utils.qualifiedName(qualifier, table.getName()), "stats", statistics, "fr", getMutableState(AbstractPlan.FRAGMENT_ID) ); } @@ -130,19 +127,25 @@ public class PhysicalOlapScan extends PhysicalRelation implements OlapScan { if (this == o) { return true; } - if (o == null || getClass() != o.getClass() || !super.equals(o)) { + if (o == null || getClass() != o.getClass()) { return false; } - PhysicalOlapScan that = ((PhysicalOlapScan) o); - return Objects.equals(selectedIndexId, that.selectedIndexId) - && Objects.equals(selectedTabletIds, that.selectedPartitionIds) - && Objects.equals(selectedPartitionIds, that.selectedPartitionIds) - && Objects.equals(olapTable, that.olapTable); + if (!super.equals(o)) { + return false; + } + PhysicalOlapScan olapScan = (PhysicalOlapScan) o; + return selectedIndexId == olapScan.selectedIndexId && Objects.equals(distributionSpec, + olapScan.distributionSpec) && Objects.equals(selectedTabletIds, olapScan.selectedTabletIds) + && Objects.equals(selectedPartitionIds, olapScan.selectedPartitionIds) + && Objects.equals(preAggStatus, olapScan.preAggStatus) && Objects.equals(baseOutputs, + olapScan.baseOutputs); } @Override public int hashCode() { - return Objects.hash(id, selectedIndexId, selectedPartitionIds, selectedTabletIds, olapTable); + return Objects.hash(super.hashCode(), distributionSpec, selectedIndexId, selectedTabletIds, + selectedPartitionIds, + preAggStatus, baseOutputs); } @Override @@ -152,7 +155,7 @@ public class PhysicalOlapScan extends PhysicalRelation implements OlapScan { @Override public PhysicalOlapScan withGroupExpression(Optional groupExpression) { - return new PhysicalOlapScan(id, olapTable, qualifier, selectedIndexId, selectedTabletIds, + return new PhysicalOlapScan(relationId, getTable(), qualifier, selectedIndexId, selectedTabletIds, selectedPartitionIds, distributionSpec, preAggStatus, baseOutputs, groupExpression, getLogicalProperties()); } @@ -160,7 +163,7 @@ public class PhysicalOlapScan extends PhysicalRelation implements OlapScan { @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalOlapScan(id, olapTable, qualifier, selectedIndexId, selectedTabletIds, + return new PhysicalOlapScan(relationId, getTable(), qualifier, selectedIndexId, selectedTabletIds, selectedPartitionIds, distributionSpec, preAggStatus, baseOutputs, groupExpression, logicalProperties.get()); } @@ -168,23 +171,21 @@ public class PhysicalOlapScan extends PhysicalRelation implements OlapScan { @Override public PhysicalOlapScan withPhysicalPropertiesAndStats( PhysicalProperties physicalProperties, Statistics statistics) { - return new PhysicalOlapScan(id, olapTable, qualifier, selectedIndexId, selectedTabletIds, + return new PhysicalOlapScan(relationId, getTable(), qualifier, selectedIndexId, selectedTabletIds, selectedPartitionIds, distributionSpec, preAggStatus, baseOutputs, groupExpression, getLogicalProperties(), physicalProperties, statistics); } @Override public String shapeInfo() { - StringBuilder builder = new StringBuilder(); - builder.append(this.getClass().getSimpleName()).append("[").append(olapTable.getName()).append("]"); - return builder.toString(); + return this.getClass().getSimpleName() + "[" + table.getName() + "]"; } @Override public JSONObject toJson() { JSONObject olapScan = super.toJson(); JSONObject properties = new JSONObject(); - properties.put("OlapTable", olapTable.toString()); + properties.put("OlapTable", table.toString()); properties.put("DistributionSpec", distributionSpec.toString()); properties.put("SelectedIndexId", selectedIndexId); properties.put("SelectedTabletIds", selectedTabletIds.toString()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java index 00328f739e..8ed08a642a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -39,20 +40,21 @@ import java.util.Optional; * A physical relation that contains only one row consist of some constant expressions. * e.g. select 100, 'value' */ -public class PhysicalOneRowRelation extends PhysicalLeaf implements OneRowRelation { +public class PhysicalOneRowRelation extends PhysicalRelation implements OneRowRelation { private final List projects; - public PhysicalOneRowRelation(List projects, LogicalProperties logicalProperties) { - this(projects, Optional.empty(), logicalProperties, null, null); + public PhysicalOneRowRelation(RelationId relationId, List projects, + LogicalProperties logicalProperties) { + this(relationId, projects, Optional.empty(), logicalProperties, null, null); } - private PhysicalOneRowRelation(List projects, + private PhysicalOneRowRelation(RelationId relationId, List projects, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { - super(PlanType.PHYSICAL_ONE_ROW_RELATION, groupExpression, logicalProperties, physicalProperties, - statistics); + super(relationId, PlanType.PHYSICAL_ONE_ROW_RELATION, groupExpression, + logicalProperties, physicalProperties, statistics); this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, "projects can not be null")); } @@ -73,24 +75,17 @@ public class PhysicalOneRowRelation extends PhysicalLeaf implements OneRowRelati @Override public Plan withGroupExpression(Optional groupExpression) { - return new PhysicalOneRowRelation(projects, groupExpression, + return new PhysicalOneRowRelation(relationId, projects, groupExpression, logicalPropertiesSupplier.get(), physicalProperties, statistics); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalOneRowRelation(projects, groupExpression, + return new PhysicalOneRowRelation(relationId, projects, groupExpression, logicalProperties.get(), physicalProperties, statistics); } - @Override - public String toString() { - return Utils.toSqlString("PhysicalOneRowRelation[" + id.asInt() + "]" + getGroupIdAsString(), - "expressions", projects - ); - } - @Override public boolean equals(Object o) { if (this == o) { @@ -99,19 +94,29 @@ public class PhysicalOneRowRelation extends PhysicalLeaf implements OneRowRelati if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { + return false; + } PhysicalOneRowRelation that = (PhysicalOneRowRelation) o; return Objects.equals(projects, that.projects); } @Override public int hashCode() { - return Objects.hash(projects); + return Objects.hash(super.hashCode(), projects); + } + + @Override + public String toString() { + return Utils.toSqlString("PhysicalOneRowRelation[" + id.asInt() + "]" + getGroupIdAsString(), + "expressions", projects + ); } @Override public PhysicalOneRowRelation withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { - return new PhysicalOneRowRelation(projects, groupExpression, + return new PhysicalOneRowRelation(relationId, projects, groupExpression, logicalPropertiesSupplier.get(), physicalProperties, statistics); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRelation.java index d5d47756d5..49b42bb02a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRelation.java @@ -21,9 +21,9 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.PlanType; -import org.apache.doris.nereids.trees.plans.algebra.Scan; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.algebra.Relation; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.statistics.Statistics; @@ -36,34 +36,27 @@ import java.util.Optional; /** * Abstract class for all physical scan plan. */ -public abstract class PhysicalRelation extends PhysicalLeaf implements Scan { +public abstract class PhysicalRelation extends PhysicalLeaf implements Relation { - protected final ObjectId id; - protected final ImmutableList qualifier; + protected final RelationId relationId; /** * Constructor for PhysicalRelation. */ - public PhysicalRelation(ObjectId id, PlanType type, List qualifier, + public PhysicalRelation(RelationId relationId, PlanType type, Optional groupExpression, LogicalProperties logicalProperties) { super(type, groupExpression, logicalProperties); - this.id = id; - this.qualifier = ImmutableList.copyOf(Objects.requireNonNull(qualifier, "qualifier can not be null")); + this.relationId = relationId; } /** * Constructor for PhysicalRelation. */ - public PhysicalRelation(ObjectId id, PlanType type, List qualifier, + public PhysicalRelation(RelationId relationId, PlanType type, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { super(type, groupExpression, logicalProperties, physicalProperties, statistics); - this.id = id; - this.qualifier = ImmutableList.copyOf(Objects.requireNonNull(qualifier, "qualifier can not be null")); - } - - public List getQualifier() { - return qualifier; + this.relationId = relationId; } @Override @@ -75,12 +68,12 @@ public abstract class PhysicalRelation extends PhysicalLeaf implements Scan { return false; } PhysicalRelation that = (PhysicalRelation) o; - return this.id.equals(that.id) && Objects.equals(qualifier, that.qualifier); + return Objects.equals(relationId, that.relationId); } @Override public int hashCode() { - return Objects.hash(qualifier); + return Objects.hash(relationId); } @Override @@ -93,7 +86,7 @@ public abstract class PhysicalRelation extends PhysicalLeaf implements Scan { return ImmutableList.of(); } - public ObjectId getId() { - return id; + public RelationId getRelationId() { + return relationId; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSchemaScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSchemaScan.java index 44e85777c0..ea7c008c80 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSchemaScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSchemaScan.java @@ -17,14 +17,13 @@ package org.apache.doris.nereids.trees.plans.physical; -import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.SchemaTable; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; -import org.apache.doris.nereids.trees.plans.algebra.Scan; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import org.apache.doris.statistics.Statistics; @@ -35,27 +34,23 @@ import java.util.Optional; /** * PhysicalSchemaScan. */ -public class PhysicalSchemaScan extends PhysicalRelation implements Scan { +public class PhysicalSchemaScan extends PhysicalCatalogRelation { - private final Table table; - - public PhysicalSchemaScan(ObjectId id, Table table, List qualifier, + public PhysicalSchemaScan(RelationId id, SchemaTable table, List qualifier, Optional groupExpression, LogicalProperties logicalProperties) { - super(id, PlanType.PHYSICAL_SCHEMA_SCAN, qualifier, groupExpression, logicalProperties); - this.table = table; + super(id, PlanType.PHYSICAL_SCHEMA_SCAN, table, qualifier, groupExpression, logicalProperties); } - public PhysicalSchemaScan(ObjectId id, Table table, List qualifier, + public PhysicalSchemaScan(RelationId id, SchemaTable table, List qualifier, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { - super(id, PlanType.PHYSICAL_SCHEMA_SCAN, qualifier, groupExpression, logicalProperties, physicalProperties, - statistics); - this.table = table; + super(id, PlanType.PHYSICAL_SCHEMA_SCAN, table, qualifier, groupExpression, + logicalProperties, physicalProperties, statistics); } @Override - public Table getTable() { - return table; + public SchemaTable getTable() { + return (SchemaTable) table; } @Override @@ -65,32 +60,22 @@ public class PhysicalSchemaScan extends PhysicalRelation implements Scan { @Override public Plan withGroupExpression(Optional groupExpression) { - return new PhysicalSchemaScan(id, table, qualifier, groupExpression, getLogicalProperties(), physicalProperties, - statistics); + return new PhysicalSchemaScan(relationId, getTable(), qualifier, + groupExpression, getLogicalProperties(), physicalProperties, statistics); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalSchemaScan(id, table, qualifier, groupExpression, logicalProperties.get(), - physicalProperties, statistics); + return new PhysicalSchemaScan(relationId, getTable(), qualifier, + groupExpression, logicalProperties.get(), physicalProperties, statistics); } @Override public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { - return new PhysicalSchemaScan(id, table, qualifier, groupExpression, getLogicalProperties(), physicalProperties, - statistics); - } - - @Override - public boolean equals(Object o) { - return super.equals(o); - } - - @Override - public int hashCode() { - return super.hashCode(); + return new PhysicalSchemaScan(relationId, getTable(), qualifier, + groupExpression, getLogicalProperties(), physicalProperties, statistics); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java index 094f5d75cd..7a9550adc3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.trees.plans.physical; -import org.apache.doris.catalog.Table; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; @@ -38,23 +37,23 @@ import java.util.Objects; import java.util.Optional; /** PhysicalStorageLayerAggregate */ -public class PhysicalStorageLayerAggregate extends PhysicalRelation { +public class PhysicalStorageLayerAggregate extends PhysicalCatalogRelation { - private final PhysicalRelation relation; + private final PhysicalCatalogRelation relation; private final PushDownAggOp aggOp; - public PhysicalStorageLayerAggregate(PhysicalRelation relation, PushDownAggOp aggOp) { - super(relation.getId(), relation.getType(), relation.getQualifier(), + public PhysicalStorageLayerAggregate(PhysicalCatalogRelation relation, PushDownAggOp aggOp) { + super(relation.getRelationId(), relation.getType(), relation.getTable(), relation.getQualifier(), Optional.empty(), relation.getLogicalProperties()); this.relation = Objects.requireNonNull(relation, "relation cannot be null"); this.aggOp = Objects.requireNonNull(aggOp, "aggOp cannot be null"); } - public PhysicalStorageLayerAggregate(PhysicalRelation relation, PushDownAggOp aggOp, + public PhysicalStorageLayerAggregate(PhysicalCatalogRelation relation, PushDownAggOp aggOp, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { - super(relation.getId(), relation.getType(), relation.getQualifier(), groupExpression, - logicalProperties, physicalProperties, statistics); + super(relation.getRelationId(), relation.getType(), relation.getTable(), relation.getQualifier(), + groupExpression, logicalProperties, physicalProperties, statistics); this.relation = Objects.requireNonNull(relation, "relation cannot be null"); this.aggOp = Objects.requireNonNull(aggOp, "aggOp cannot be null"); } @@ -67,39 +66,14 @@ public class PhysicalStorageLayerAggregate extends PhysicalRelation { return aggOp; } - @Override - public Table getTable() { - return (Table) relation.getTable(); - } - @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitPhysicalStorageLayerAggregate(this, context); } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - PhysicalStorageLayerAggregate that = (PhysicalStorageLayerAggregate) o; - return Objects.equals(relation, that.relation) && aggOp == that.aggOp; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), relation, aggOp); - } - @Override public String toString() { - return Utils.toSqlString("PhysicalStorageLayerAggregate[" + id.asInt() + "]" + getGroupIdAsString(), + return Utils.toSqlString("PhysicalStorageLayerAggregate[" + relationId.asInt() + "]" + getGroupIdAsString(), "pushDownAggOp", aggOp, "relation", relation, "stats", statistics @@ -112,8 +86,8 @@ public class PhysicalStorageLayerAggregate extends PhysicalRelation { @Override public PhysicalStorageLayerAggregate withGroupExpression(Optional groupExpression) { - return new PhysicalStorageLayerAggregate(relation, aggOp, groupExpression, getLogicalProperties(), - physicalProperties, statistics); + return new PhysicalStorageLayerAggregate(relation, aggOp, groupExpression, + getLogicalProperties(), physicalProperties, statistics); } @Override @@ -127,7 +101,7 @@ public class PhysicalStorageLayerAggregate extends PhysicalRelation { public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalStorageLayerAggregate( - (PhysicalRelation) relation.withPhysicalPropertiesAndStats(null, statistics), + (PhysicalCatalogRelation) relation.withPhysicalPropertiesAndStats(null, statistics), aggOp, groupExpression, getLogicalProperties(), physicalProperties, statistics); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java index dfe5ad1168..955ea2f45d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java @@ -17,14 +17,15 @@ package org.apache.doris.nereids.trees.plans.physical; -import org.apache.doris.catalog.FunctionGenTable; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.TVFRelation; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -41,54 +42,75 @@ public class PhysicalTVFRelation extends PhysicalRelation implements TVFRelation private final TableValuedFunction function; - public PhysicalTVFRelation(ObjectId id, TableValuedFunction function, LogicalProperties logicalProperties) { - super(id, PlanType.PHYSICAL_TVF_RELATION, - ImmutableList.of(), Optional.empty(), logicalProperties); + public PhysicalTVFRelation(RelationId id, TableValuedFunction function, LogicalProperties logicalProperties) { + super(id, PlanType.PHYSICAL_TVF_RELATION, Optional.empty(), logicalProperties); this.function = Objects.requireNonNull(function, "function can not be null"); } - public PhysicalTVFRelation(ObjectId id, TableValuedFunction function, Optional groupExpression, - LogicalProperties logicalProperties, PhysicalProperties physicalProperties, - Statistics statistics) { - super(id, PlanType.PHYSICAL_TVF_RELATION, ImmutableList.of(), groupExpression, logicalProperties, - physicalProperties, statistics); + public PhysicalTVFRelation(RelationId id, TableValuedFunction function, Optional groupExpression, + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { + super(id, PlanType.PHYSICAL_TVF_RELATION, groupExpression, + logicalProperties, physicalProperties, statistics); this.function = Objects.requireNonNull(function, "function can not be null"); } @Override public PhysicalTVFRelation withGroupExpression(Optional groupExpression) { - return new PhysicalTVFRelation(id, function, groupExpression, getLogicalProperties(), + return new PhysicalTVFRelation(relationId, function, groupExpression, getLogicalProperties(), physicalProperties, statistics); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalTVFRelation(id, function, groupExpression, + return new PhysicalTVFRelation(relationId, function, groupExpression, logicalProperties.get(), physicalProperties, statistics); } @Override public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { - return new PhysicalTVFRelation(id, function, Optional.empty(), + return new PhysicalTVFRelation(relationId, function, Optional.empty(), getLogicalProperties(), physicalProperties, statistics); } @Override - public FunctionGenTable getTable() { - return function.getTable(); + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + PhysicalTVFRelation that = (PhysicalTVFRelation) o; + return Objects.equals(function, that.function); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), function); } @Override public String toString() { return Utils.toSqlString("PhysicalTVFRelation", - "qualified", Utils.qualifiedName(qualifier, getTable().getName()), + "qualified", Utils.qualifiedName(ImmutableList.of(), function.getTable().getName()), "output", getOutput(), "function", function.toSql() ); } + @Override + public List computeOutput() { + return function.getTable().getBaseSchema() + .stream() + .map(col -> SlotReference.fromColumn(col, ImmutableList.of())) + .collect(ImmutableList.toImmutableList()); + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitPhysicalTVFRelation(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/RelationUtil.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/RelationUtil.java index 767e7624da..dc401a349c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/RelationUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/RelationUtil.java @@ -23,12 +23,9 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.Pair; import org.apache.doris.datasource.CatalogIf; -import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.qe.ConnectContext; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import java.util.List; @@ -38,28 +35,6 @@ import java.util.List; */ public class RelationUtil { - // for test only - private static StatementContext statementContext = new StatementContext(); - - public static ObjectId newRelationId() { - // this branch is for test only - if (ConnectContext.get() == null || ConnectContext.get().getStatementContext() == null) { - return statementContext.getNextObjectId(); - } - return ConnectContext.get().getStatementContext().getNextObjectId(); - } - - /** - * Reset Id Generator - */ - @VisibleForTesting - public static void clear() throws Exception { - if (ConnectContext.get() != null) { - ConnectContext.get().setStatementContext(new StatementContext()); - } - statementContext = new StatementContext(); - } - /** * get table qualifier */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 7d5bfb4965..b56a50c9f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1021,19 +1021,13 @@ public class SessionVariable implements Serializable, Writable { needForward = false) public int externalTableAnalyzePartNum = -1; - @VariableMgr.VarAttr( - name = INLINE_CTE_REFERENCED_THRESHOLD - ) + @VariableMgr.VarAttr(name = INLINE_CTE_REFERENCED_THRESHOLD) public int inlineCTEReferencedThreshold = 1; - @VariableMgr.VarAttr( - name = ENABLE_CTE_MATERIALIZE - ) + @VariableMgr.VarAttr(name = ENABLE_CTE_MATERIALIZE) public boolean enableCTEMaterialize = true; - @VariableMgr.VarAttr( - name = IGNORE_COMPLEX_TYPE_COLUMN - ) + @VariableMgr.VarAttr(name = IGNORE_COMPLEX_TYPE_COLUMN) public boolean ignoreColumnWithComplexType = false; @VariableMgr.VarAttr(name = ENABLE_STRONG_CONSISTENCY, description = {"用以开启强一致读。Doris 默认支持同一个会话内的" diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/datasets/tpch/TPCHTestBase.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/datasets/tpch/TPCHTestBase.java index a0c9b77f7e..2f37aaf03e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/datasets/tpch/TPCHTestBase.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/datasets/tpch/TPCHTestBase.java @@ -28,7 +28,7 @@ public abstract class TPCHTestBase extends AnalyzeCheckTestBase { // This deriveStats job is responsible for retrieving statistics from the aforementioned // internal table. FeConstants.disableInternalSchemaDb = false; - new InternalSchemaInitializer().run(); + new InternalSchemaInitializer().start(); createDatabase("tpch"); connectContext.setDatabase("default_cluster:tpch"); TPCHUtils.createTables(this); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java index d59e72e860..be5a14343d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.expressions.GreaterThan; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.PreAggStatus; @@ -33,7 +34,6 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.PlanNode; @@ -64,7 +64,7 @@ public class PhysicalPlanTranslatorTest { t1Output.add(col2); t1Output.add(col3); LogicalProperties t1Properties = new LogicalProperties(() -> t1Output); - PhysicalOlapScan scan = new PhysicalOlapScan(RelationUtil.newRelationId(), t1, qualifier, t1.getBaseIndexId(), + PhysicalOlapScan scan = new PhysicalOlapScan(StatementScopeIdGenerator.newRelationId(), t1, qualifier, t1.getBaseIndexId(), Collections.emptyList(), Collections.emptyList(), null, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), t1Properties); Literal t1FilterRight = new IntegerLiteral(1); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java index 8eb03d5b65..6f1c2d9d95 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java @@ -27,17 +27,17 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -70,7 +70,7 @@ public class RewriteTopDownJobTest { @Test public void testSimplestScene() { - Plan leaf = new UnboundRelation(RelationUtil.newRelationId(), Lists.newArrayList("test")); + Plan leaf = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), Lists.newArrayList("test")); LogicalProject project = new LogicalProject<>(ImmutableList.of( new SlotReference("name", StringType.INSTANCE, true, ImmutableList.of("test"))), leaf @@ -101,15 +101,15 @@ public class RewriteTopDownJobTest { }); } - private static class LogicalBoundRelation extends LogicalRelation { + private static class LogicalBoundRelation extends LogicalCatalogRelation { public LogicalBoundRelation(TableIf table, List qualifier) { - super(RelationUtil.newRelationId(), PlanType.LOGICAL_BOUND_RELATION, table, qualifier); + super(StatementScopeIdGenerator.newRelationId(), PlanType.LOGICAL_BOUND_RELATION, table, qualifier); } public LogicalBoundRelation(TableIf table, List qualifier, Optional groupExpression, Optional logicalProperties) { - super(RelationUtil.newRelationId(), PlanType.LOGICAL_BOUND_RELATION, table, qualifier, + super(StatementScopeIdGenerator.newRelationId(), PlanType.LOGICAL_BOUND_RELATION, table, qualifier, groupExpression, logicalProperties); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java index e4c031e4a8..c6c60c815b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; import org.apache.doris.nereids.trees.plans.Plan; @@ -32,7 +33,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.qe.ConnectContext; import org.apache.doris.statistics.Statistics; @@ -81,7 +81,7 @@ public class DeriveStatsJobTest { }}; OlapTable table1 = PlanConstructor.newOlapTable(tableId1, "t1", 0); - return (LogicalOlapScan) new LogicalOlapScan(RelationUtil.newRelationId(), table1, + return (LogicalOlapScan) new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), table1, Collections.emptyList()).withGroupExprLogicalPropChildren(Optional.empty(), Optional.of(new LogicalProperties(() -> ImmutableList.of(slot1))), ImmutableList.of()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java index 43b1c48761..3f55ba9579 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.properties.UnboundLogicalProperties; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.FakePlan; import org.apache.doris.nereids.trees.plans.GroupPlan; @@ -46,7 +47,6 @@ import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; @@ -178,7 +178,7 @@ class MemoTest implements MemoPatternMatchSupported { @Test public void initByOneLevelPlan() { OlapTable table = PlanConstructor.newOlapTable(0, "a", 1); - LogicalOlapScan scan = new LogicalOlapScan(RelationUtil.newRelationId(), table); + LogicalOlapScan scan = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), table); PlanChecker.from(connectContext, scan) .checkGroupNum(1) @@ -204,7 +204,7 @@ class MemoTest implements MemoPatternMatchSupported { @Test public void initByJoinSameUnboundTable() { - UnboundRelation scanA = new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("a")); + UnboundRelation scanA = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("a")); // when unboundRelation contains id, the case is illegal. LogicalJoin topJoin = new LogicalJoin<>(JoinType.INNER_JOIN, scanA, scanA); @@ -215,8 +215,8 @@ class MemoTest implements MemoPatternMatchSupported { @Test public void initByJoinSameLogicalTable() { OlapTable tableA = PlanConstructor.newOlapTable(0, "a", 1); - LogicalOlapScan scanA = new LogicalOlapScan(RelationUtil.newRelationId(), tableA); - LogicalOlapScan scanA1 = new LogicalOlapScan(RelationUtil.newRelationId(), tableA); + LogicalOlapScan scanA = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), tableA); + LogicalOlapScan scanA1 = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), tableA); LogicalJoin topJoin = new LogicalJoin<>(JoinType.INNER_JOIN, scanA, scanA1); @@ -234,8 +234,8 @@ class MemoTest implements MemoPatternMatchSupported { public void initByTwoLevelJoinPlan() { OlapTable tableA = PlanConstructor.newOlapTable(0, "a", 1); OlapTable tableB = PlanConstructor.newOlapTable(0, "b", 1); - LogicalOlapScan scanA = new LogicalOlapScan(RelationUtil.newRelationId(), tableA); - LogicalOlapScan scanB = new LogicalOlapScan(RelationUtil.newRelationId(), tableB); + LogicalOlapScan scanA = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), tableA); + LogicalOlapScan scanB = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), tableB); LogicalJoin topJoin = new LogicalJoin<>(JoinType.INNER_JOIN, scanA, scanB); @@ -274,10 +274,10 @@ class MemoTest implements MemoPatternMatchSupported { OlapTable tableB = PlanConstructor.newOlapTable(0, "b", 1); OlapTable tableC = PlanConstructor.newOlapTable(0, "c", 1); OlapTable tableD = PlanConstructor.newOlapTable(0, "d", 1); - LogicalOlapScan scanA = new LogicalOlapScan(RelationUtil.newRelationId(), tableA); - LogicalOlapScan scanB = new LogicalOlapScan(RelationUtil.newRelationId(), tableB); - LogicalOlapScan scanC = new LogicalOlapScan(RelationUtil.newRelationId(), tableC); - LogicalOlapScan scanD = new LogicalOlapScan(RelationUtil.newRelationId(), tableD); + LogicalOlapScan scanA = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), tableA); + LogicalOlapScan scanB = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), tableB); + LogicalOlapScan scanC = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), tableC); + LogicalOlapScan scanD = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), tableD); LogicalJoin leftJoin = new LogicalJoin<>(JoinType.CROSS_JOIN, scanA, scanB); LogicalJoin rightJoin = new LogicalJoin<>(JoinType.CROSS_JOIN, scanC, scanD); @@ -307,9 +307,9 @@ class MemoTest implements MemoPatternMatchSupported { */ @Test public void a2a() { - UnboundRelation student = new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("student")); + UnboundRelation student = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("student")); PlanChecker.from(connectContext, student) - .applyBottomUp( + .applyBottomUpInMemo( unboundRelation().then(scan -> scan) ) .checkGroupNum(1) @@ -323,10 +323,10 @@ class MemoTest implements MemoPatternMatchSupported { */ @Test public void a2b() { - LogicalOlapScan student = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); - PlanChecker.from(connectContext, new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("student"))) - .applyBottomUp( + PlanChecker.from(connectContext, new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("student"))) + .applyBottomUpInMemo( unboundRelation().then(scan -> student) ) .checkGroupNum(1) @@ -340,13 +340,13 @@ class MemoTest implements MemoPatternMatchSupported { */ @Test public void a2newA() { - LogicalOlapScan student = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); PlanChecker.from(connectContext, student) - .applyBottomUp( + .applyBottomUpInMemo( logicalOlapScan() .when(scan -> student == scan) - .then(scan -> new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student)) + .then(scan -> new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student)) ) .checkGroupNum(1) .matchesFromRoot(logicalOlapScan().when(student::equals)); @@ -361,11 +361,11 @@ class MemoTest implements MemoPatternMatchSupported { */ @Test public void a2bc() { - LogicalOlapScan student = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalLimit limit = new LogicalLimit<>(1, 0, LimitPhase.ORIGIN, student); - PlanChecker.from(connectContext, new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("student"))) - .applyBottomUp( + PlanChecker.from(connectContext, new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("student"))) + .applyBottomUpInMemo( unboundRelation().then(unboundRelation -> limit.withChildren(student)) ) .checkGroupNum(2) @@ -396,11 +396,11 @@ class MemoTest implements MemoPatternMatchSupported { public void a2ba() { // invalid case Assertions.assertThrows(IllegalStateException.class, () -> { - UnboundRelation student = new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("student")); + UnboundRelation student = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("student")); LogicalLimit limit = new LogicalLimit<>(1, 0, LimitPhase.ORIGIN, student); PlanChecker.from(connectContext, student) - .applyBottomUp( + .applyBottomUpInMemo( unboundRelation().then(limit::withChildren) ) .checkGroupNum(2) @@ -412,17 +412,17 @@ class MemoTest implements MemoPatternMatchSupported { }); // use relation id to divide different unbound relation. - UnboundRelation a = new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("student")); + UnboundRelation a = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("student")); - UnboundRelation a2 = new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("student")); + UnboundRelation a2 = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("student")); LogicalLimit limit = new LogicalLimit<>(1, 0, LimitPhase.ORIGIN, a2); PlanChecker.from(connectContext, a) .setMaxInvokeTimesPerRule(1000) - .applyBottomUp( + .applyBottomUpInMemo( unboundRelation() - .when(unboundRelation -> unboundRelation.getId().equals(a.getId())) + .when(unboundRelation -> unboundRelation.getRelationId().equals(a.getRelationId())) .then(unboundRelation -> limit.withChildren( - new UnboundRelation(a2.getId(), unboundRelation.getNameParts())))) + new UnboundRelation(a2.getRelationId(), unboundRelation.getNameParts())))) .checkGroupNum(2) .matchesFromRoot( logicalLimit( @@ -454,9 +454,9 @@ class MemoTest implements MemoPatternMatchSupported { /*@Test() public void a2ab() { Assertions.assertThrows(IllegalStateException.class, () -> { - UnboundRelation student = new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("student")); + UnboundRelation student = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("student")); LogicalLimit limit = new LogicalLimit<>(1, 0, student); - LogicalOlapScan boundStudent = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan boundStudent = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(connectContext, limit); PlanChecker.from(cascadesContext) @@ -479,12 +479,12 @@ class MemoTest implements MemoPatternMatchSupported { */ @Test public void a2bcd() { - LogicalOlapScan scan = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan scan = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalLimit limit5 = new LogicalLimit<>(5, 0, LimitPhase.ORIGIN, scan); LogicalLimit> limit10 = new LogicalLimit<>(10, 0, LimitPhase.ORIGIN, limit5); PlanChecker.from(connectContext, limit10) - .applyBottomUp( + .applyBottomUpInMemo( unboundRelation().then(r -> limit10) ) .checkGroupNum(3) @@ -507,11 +507,11 @@ class MemoTest implements MemoPatternMatchSupported { */ @Test public void ab2a() { - LogicalOlapScan student = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalLimit limit10 = new LogicalLimit<>(10, 0, LimitPhase.ORIGIN, student); PlanChecker.from(connectContext, limit10) - .applyBottomUp( + .applyBottomUpInMemo( logicalLimit().when(limit10::equals).then(limit -> limit) ) .checkGroupNum(2) @@ -531,11 +531,11 @@ class MemoTest implements MemoPatternMatchSupported { */ @Test public void ab2NewA() { - LogicalOlapScan student = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalLimit limit10 = new LogicalLimit<>(10, 0, LimitPhase.ORIGIN, student); PlanChecker.from(connectContext, limit10) - .applyBottomUp( + .applyBottomUpInMemo( logicalLimit().when(limit10::equals).then(limit -> limit.withChildren(limit.child())) ) .checkGroupNum(2) @@ -555,11 +555,11 @@ class MemoTest implements MemoPatternMatchSupported { */ @Test public void ab2GroupB() { - LogicalOlapScan student = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalLimit limit10 = new LogicalLimit<>(10, 0, LimitPhase.ORIGIN, student); PlanChecker.from(connectContext, limit10) - .applyBottomUp( + .applyBottomUpInMemo( logicalLimit().when(limit10::equals).then(limit -> limit.child()) ) .checkGroupNum(1) @@ -577,11 +577,11 @@ class MemoTest implements MemoPatternMatchSupported { */ @Test public void ab2PlanB() { - LogicalOlapScan student = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalLimit limit10 = new LogicalLimit<>(10, 0, LimitPhase.ORIGIN, student); PlanChecker.from(connectContext, limit10) - .applyBottomUp( + .applyBottomUpInMemo( logicalLimit(logicalOlapScan()).when(limit10::equals).then(limit -> limit.child()) ) .checkGroupNum(1) @@ -595,16 +595,16 @@ class MemoTest implements MemoPatternMatchSupported { * * limit(10) * | -> logicalOlapScan(student) - * UnboundRelation(RelationUtil.newRelationId(), student) + * UnboundRelation(StatementScopeIdGenerator.newRelationId(), student) */ @Test public void ab2c() { - UnboundRelation relation = new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("student")); + UnboundRelation relation = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("student")); LogicalLimit limit10 = new LogicalLimit<>(10, 0, LimitPhase.ORIGIN, relation); - LogicalOlapScan student = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); PlanChecker.from(connectContext, limit10) - .applyBottomUp( + .applyBottomUpInMemo( logicalLimit(unboundRelation()).then(limit -> student) ) .checkGroupNum(1) @@ -622,14 +622,14 @@ class MemoTest implements MemoPatternMatchSupported { */ @Test public void ab2cd() { - UnboundRelation relation = new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("student")); + UnboundRelation relation = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("student")); LogicalLimit limit10 = new LogicalLimit<>(10, 0, LimitPhase.ORIGIN, relation); - LogicalOlapScan student = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalLimit limit5 = new LogicalLimit<>(5, 0, LimitPhase.ORIGIN, student); PlanChecker.from(connectContext, limit10) - .applyBottomUp( + .applyBottomUpInMemo( logicalLimit(unboundRelation()).then(limit -> limit5) ) .checkGroupNum(2) @@ -650,12 +650,12 @@ class MemoTest implements MemoPatternMatchSupported { */ @Test public void ab2cb() { - LogicalOlapScan student = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalLimit limit10 = new LogicalLimit<>(10, 0, LimitPhase.ORIGIN, student); LogicalLimit limit5 = new LogicalLimit<>(5, 0, LimitPhase.ORIGIN, student); PlanChecker.from(connectContext, limit10) - .applyBottomUp( + .applyBottomUpInMemo( logicalLimit().when(limit10::equals).then(limit -> limit5) ) .checkGroupNum(2) @@ -681,14 +681,14 @@ class MemoTest implements MemoPatternMatchSupported { public void ab2NewANewB() { Assertions.assertThrowsExactly(IllegalStateException.class, () -> { - LogicalOlapScan student = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalLimit limit10 = new LogicalLimit<>(10, 0, LimitPhase.ORIGIN, student); PlanChecker.from(connectContext, limit10) .setMaxInvokeTimesPerRule(1000) - .applyBottomUp( + .applyBottomUpInMemo( logicalLimit().when(limit10::equals).then(limit -> limit.withChildren( - new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student) + new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student) )) ); }); @@ -706,13 +706,13 @@ class MemoTest implements MemoPatternMatchSupported { @Test public void ab2ba() { Assertions.assertThrowsExactly(IllegalStateException.class, () -> { - UnboundRelation student = new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("student")); + UnboundRelation student = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("student")); LogicalLimit limit5 = new LogicalLimit<>(5, 0, LimitPhase.ORIGIN, student); LogicalLimit> limit10 = new LogicalLimit<>(10, 0, LimitPhase.ORIGIN, limit5); PlanChecker.from(connectContext, limit10) - .applyBottomUp( + .applyBottomUpInMemo( logicalLimit(logicalLimit(unboundRelation())).when(limit10::equals).then(l -> l.child().withChildren( l @@ -733,15 +733,15 @@ class MemoTest implements MemoPatternMatchSupported { */ @Test public void ab2cde() { - UnboundRelation student = new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("student")); + UnboundRelation student = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("student")); LogicalLimit limit3 = new LogicalLimit<>(3, 0, LimitPhase.ORIGIN, student); - LogicalOlapScan scan = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan scan = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalLimit limit5 = new LogicalLimit<>(5, 0, LimitPhase.ORIGIN, scan); LogicalLimit> limit10 = new LogicalLimit<>(10, 0, LimitPhase.ORIGIN, limit5); PlanChecker.from(connectContext, limit3) - .applyBottomUp( + .applyBottomUpInMemo( logicalLimit(unboundRelation()).then(l -> limit10) ) .checkGroupNum(3) @@ -765,13 +765,13 @@ class MemoTest implements MemoPatternMatchSupported { */ @Test public void abc2bac() { - UnboundRelation student = new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("student")); + UnboundRelation student = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("student")); LogicalLimit limit5 = new LogicalLimit<>(5, 0, LimitPhase.ORIGIN, student); LogicalLimit> limit10 = new LogicalLimit<>(10, 0, LimitPhase.ORIGIN, limit5); PlanChecker.from(connectContext, limit10) - .applyBottomUp( + .applyBottomUpInMemo( logicalLimit(logicalLimit(unboundRelation())).when(limit10::equals).then(l -> // limit 5 l.child().withChildren( @@ -804,13 +804,13 @@ class MemoTest implements MemoPatternMatchSupported { */ @Test public void abc2bc() { - UnboundRelation student = new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("student")); + UnboundRelation student = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("student")); LogicalLimit limit5 = new LogicalLimit<>(5, 0, LimitPhase.ORIGIN, student); LogicalLimit> limit10 = new LogicalLimit<>(10, 0, LimitPhase.ORIGIN, limit5); PlanChecker.from(connectContext, limit10) - .applyBottomUp( + .applyBottomUpInMemo( logicalLimit(logicalLimit(unboundRelation())).then(l -> // limit 10 l.withChildren( @@ -829,13 +829,13 @@ class MemoTest implements MemoPatternMatchSupported { @Test public void testRewriteBottomPlanToOnePlan() { - LogicalOlapScan student = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalLimit limit = new LogicalLimit<>(1, 0, LimitPhase.ORIGIN, student); - LogicalOlapScan score = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.score); + LogicalOlapScan score = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.score); PlanChecker.from(connectContext, limit) - .applyBottomUp( + .applyBottomUpInMemo( logicalOlapScan().when(scan -> Objects.equals(student, scan)).then(scan -> score) ) .checkGroupNum(2) @@ -848,14 +848,14 @@ class MemoTest implements MemoPatternMatchSupported { @Test public void testRewriteBottomPlanToMultiPlan() { - LogicalOlapScan student = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalLimit limit10 = new LogicalLimit<>(10, 0, LimitPhase.ORIGIN, student); - LogicalOlapScan score = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.score); + LogicalOlapScan score = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.score); LogicalLimit limit1 = new LogicalLimit<>(1, 0, LimitPhase.ORIGIN, score); PlanChecker.from(connectContext, limit10) - .applyBottomUp( + .applyBottomUpInMemo( logicalOlapScan().when(scan -> Objects.equals(student, scan)).then(scan -> limit1) ) .checkGroupNum(3) @@ -870,15 +870,15 @@ class MemoTest implements MemoPatternMatchSupported { @Test public void testRewriteUnboundPlanToBound() { - UnboundRelation unboundTable = new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("score")); - LogicalOlapScan boundTable = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.score); + UnboundRelation unboundTable = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("score")); + LogicalOlapScan boundTable = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.score); PlanChecker.from(connectContext, unboundTable) .checkMemo(memo -> { LogicalProperties logicalProperties = memo.getRoot().getLogicalProperties(); Assertions.assertTrue(logicalProperties instanceof UnboundLogicalProperties); }) - .applyBottomUp(unboundRelation().then(unboundRelation -> boundTable)) + .applyBottomUpInMemo(unboundRelation().then(unboundRelation -> boundTable)) .checkGroupNum(1) .checkMemo(memo -> { LogicalProperties logicalProperties = memo.getRoot().getLogicalProperties(); @@ -892,10 +892,10 @@ class MemoTest implements MemoPatternMatchSupported { @Test public void testRecomputeLogicalProperties() { - UnboundRelation unboundTable = new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("score")); + UnboundRelation unboundTable = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("score")); LogicalLimit unboundLimit = new LogicalLimit<>(1, 0, LimitPhase.ORIGIN, unboundTable); - LogicalOlapScan boundTable = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.score); + LogicalOlapScan boundTable = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.score); LogicalLimit boundLimit = unboundLimit.withChildren(ImmutableList.of(boundTable)); PlanChecker.from(connectContext, unboundLimit) @@ -903,8 +903,8 @@ class MemoTest implements MemoPatternMatchSupported { LogicalProperties logicalProperties = memo.getRoot().getLogicalProperties(); Assertions.assertTrue(logicalProperties instanceof UnboundLogicalProperties); }) - .applyBottomUp(unboundRelation().then(unboundRelation -> boundTable)) - .applyBottomUp( + .applyBottomUpInMemo(unboundRelation().then(unboundRelation -> boundTable)) + .applyBottomUpInMemo( logicalPlan() .when(plan -> plan.canBind() && !(plan instanceof LeafPlan)) .then(LogicalPlan::recomputeLogicalProperties) @@ -924,11 +924,11 @@ class MemoTest implements MemoPatternMatchSupported { @Test public void testEliminateRootWithChildGroupInTwoLevels() { - LogicalOlapScan scan = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.score); + LogicalOlapScan scan = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.score); LogicalLimit limit = new LogicalLimit<>(1, 0, LimitPhase.ORIGIN, scan); PlanChecker.from(connectContext, limit) - .applyBottomUp(logicalLimit().then(LogicalLimit::child)) + .applyBottomUpInMemo(logicalLimit().then(LogicalLimit::child)) .checkGroupNum(1) .checkGroupExpressionNum(1) .checkFirstRootLogicalPlan(scan); @@ -936,11 +936,11 @@ class MemoTest implements MemoPatternMatchSupported { @Test public void testEliminateRootWithChildPlanInTwoLevels() { - LogicalOlapScan scan = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.score); + LogicalOlapScan scan = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.score); LogicalLimit limit = new LogicalLimit<>(1, 0, LimitPhase.ORIGIN, scan); PlanChecker.from(connectContext, limit) - .applyBottomUp(logicalLimit(any()).then(LogicalLimit::child)) + .applyBottomUpInMemo(logicalLimit(any()).then(LogicalLimit::child)) .checkGroupNum(1) .checkGroupExpressionNum(1) .checkFirstRootLogicalPlan(scan); @@ -948,19 +948,19 @@ class MemoTest implements MemoPatternMatchSupported { @Test public void testEliminateTwoLevelsToOnePlan() { - LogicalOlapScan score = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.score); + LogicalOlapScan score = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.score); LogicalLimit limit = new LogicalLimit<>(1, 0, LimitPhase.ORIGIN, score); - LogicalOlapScan student = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); PlanChecker.from(connectContext, limit) - .applyBottomUp(logicalLimit(any()).then(l -> student)) + .applyBottomUpInMemo(logicalLimit(any()).then(l -> student)) .checkGroupNum(1) .checkGroupExpressionNum(1) .checkFirstRootLogicalPlan(student); PlanChecker.from(connectContext, limit) - .applyBottomUp(logicalLimit(group()).then(l -> student)) + .applyBottomUpInMemo(logicalLimit(group()).then(l -> student)) .checkGroupNum(1) .checkGroupExpressionNum(1) .checkFirstRootLogicalPlan(student); @@ -968,14 +968,14 @@ class MemoTest implements MemoPatternMatchSupported { @Test public void testEliminateTwoLevelsToTwoPlans() { - LogicalOlapScan score = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.score); + LogicalOlapScan score = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.score); LogicalLimit limit1 = new LogicalLimit<>(1, 0, LimitPhase.ORIGIN, score); - LogicalOlapScan student = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalLimit limit10 = new LogicalLimit<>(10, 0, LimitPhase.ORIGIN, student); PlanChecker.from(connectContext, limit1) - .applyBottomUp(logicalLimit(any()).when(limit1::equals).then(l -> limit10)) + .applyBottomUpInMemo(logicalLimit(any()).when(limit1::equals).then(l -> limit10)) .checkGroupNum(2) .checkGroupExpressionNum(2) .matchesFromRoot( @@ -985,7 +985,7 @@ class MemoTest implements MemoPatternMatchSupported { ); PlanChecker.from(connectContext, limit1) - .applyBottomUp(logicalLimit(group()).when(limit1::equals).then(l -> limit10)) + .applyBottomUpInMemo(logicalLimit(group()).when(limit1::equals).then(l -> limit10)) .checkGroupNum(2) .checkGroupExpressionNum(2) .matchesFromRoot( @@ -1001,11 +1001,11 @@ class MemoTest implements MemoPatternMatchSupported { .analyze(new LogicalLimit<>(10, 0, LimitPhase.ORIGIN, new LogicalJoin<>(JoinType.LEFT_OUTER_JOIN, ImmutableList.of(new EqualTo(new UnboundSlot("sid"), new UnboundSlot("id"))), - new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.score), - new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student) + new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.score), + new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student) ) )) - .applyTopDown( + .applyTopDownInMemo( logicalLimit(logicalJoin()).then(limit -> { LogicalJoin join = limit.child(); switch (join.getJoinType()) { @@ -1056,7 +1056,7 @@ class MemoTest implements MemoPatternMatchSupported { */ @Test public void testRewriteMiddlePlans() { - UnboundRelation unboundRelation = new UnboundRelation(RelationUtil.newRelationId(), Lists.newArrayList("test")); + UnboundRelation unboundRelation = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), Lists.newArrayList("test")); LogicalProject insideProject = new LogicalProject<>( ImmutableList.of(new SlotReference("name", StringType.INSTANCE, true, ImmutableList.of("test"))), unboundRelation @@ -1113,7 +1113,7 @@ class MemoTest implements MemoPatternMatchSupported { */ @Test public void testEliminateRootWithChildPlanThreeLevels() { - UnboundRelation unboundRelation = new UnboundRelation(RelationUtil.newRelationId(), Lists.newArrayList("test")); + UnboundRelation unboundRelation = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), Lists.newArrayList("test")); LogicalProject insideProject = new LogicalProject<>( ImmutableList.of(new SlotReference("inside", StringType.INSTANCE, true, ImmutableList.of("test"))), unboundRelation diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java index 47454b8a7b..6c110d0b15 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.rules.RulePromise; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; @@ -32,7 +33,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.types.StringType; -import org.apache.doris.nereids.util.RelationUtil; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -48,7 +48,7 @@ public class GroupExpressionMatchingTest { public void testLeafNode() { Pattern pattern = new Pattern<>(PlanType.LOGICAL_UNBOUND_RELATION); - Memo memo = new Memo(new UnboundRelation(RelationUtil.newRelationId(), Lists.newArrayList("test"))); + Memo memo = new Memo(new UnboundRelation(StatementScopeIdGenerator.newRelationId(), Lists.newArrayList("test"))); GroupExpressionMatching groupExpressionMatching = new GroupExpressionMatching(pattern, memo.getRoot().getLogicalExpression()); @@ -65,13 +65,13 @@ public class GroupExpressionMatchingTest { Pattern pattern = new Pattern<>(PlanType.LOGICAL_PROJECT, new Pattern<>(PlanType.LOGICAL_UNBOUND_RELATION)); - Plan leaf = new UnboundRelation(RelationUtil.newRelationId(), Lists.newArrayList("test")); + Plan leaf = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), Lists.newArrayList("test")); LogicalProject root = new LogicalProject(ImmutableList .of(new SlotReference("name", StringType.INSTANCE, true, ImmutableList.of("test"))), leaf); Memo memo = new Memo(root); - Plan anotherLeaf = new UnboundRelation(RelationUtil.newRelationId(), Lists.newArrayList("test2")); + Plan anotherLeaf = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), Lists.newArrayList("test2")); memo.copyIn(anotherLeaf, memo.getRoot().getLogicalExpression().child(0), false); GroupExpressionMatching groupExpressionMatching @@ -96,13 +96,13 @@ public class GroupExpressionMatchingTest { public void testDepth2WithGroup() { Pattern pattern = new Pattern<>(PlanType.LOGICAL_PROJECT, Pattern.GROUP); - Plan leaf = new UnboundRelation(RelationUtil.newRelationId(), Lists.newArrayList("test")); + Plan leaf = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), Lists.newArrayList("test")); LogicalProject root = new LogicalProject(ImmutableList .of(new SlotReference("name", StringType.INSTANCE, true, ImmutableList.of("test"))), leaf); Memo memo = new Memo(root); - Plan anotherLeaf = new UnboundRelation(RelationUtil.newRelationId(), Lists.newArrayList("test2")); + Plan anotherLeaf = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), Lists.newArrayList("test2")); memo.copyIn(anotherLeaf, memo.getRoot().getLogicalExpression().child(0), false); GroupExpressionMatching groupExpressionMatching @@ -122,7 +122,7 @@ public class GroupExpressionMatchingTest { public void testLeafAny() { Pattern pattern = Pattern.ANY; - Memo memo = new Memo(new UnboundRelation(RelationUtil.newRelationId(), Lists.newArrayList("test"))); + Memo memo = new Memo(new UnboundRelation(StatementScopeIdGenerator.newRelationId(), Lists.newArrayList("test"))); GroupExpressionMatching groupExpressionMatching = new GroupExpressionMatching(pattern, memo.getRoot().getLogicalExpression()); @@ -139,10 +139,10 @@ public class GroupExpressionMatchingTest { Plan root = new LogicalProject( ImmutableList.of(new SlotReference("name", StringType.INSTANCE, true, ImmutableList.of("test"))), - new UnboundRelation(RelationUtil.newRelationId(), Lists.newArrayList("test"))); + new UnboundRelation(StatementScopeIdGenerator.newRelationId(), Lists.newArrayList("test"))); Memo memo = new Memo(root); - Plan anotherLeaf = new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("test2")); + Plan anotherLeaf = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("test2")); memo.copyIn(anotherLeaf, memo.getRoot().getLogicalExpression().child(0), false); GroupExpressionMatching groupExpressionMatching @@ -161,8 +161,8 @@ public class GroupExpressionMatchingTest { @Test public void testInnerLogicalJoinMatch() { Plan root = new LogicalJoin(JoinType.INNER_JOIN, - new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("a")), - new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("b")) + new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("a")), + new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("b")) ); Memo memo = new Memo(root); @@ -183,8 +183,8 @@ public class GroupExpressionMatchingTest { @Test public void testInnerLogicalJoinMismatch() { Plan root = new LogicalJoin(JoinType.LEFT_OUTER_JOIN, - new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("a")), - new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("b")) + new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("a")), + new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("b")) ); Memo memo = new Memo(root); @@ -200,8 +200,8 @@ public class GroupExpressionMatchingTest { @Test public void testTopMatchButChildrenNotMatch() { Plan root = new LogicalJoin(JoinType.LEFT_OUTER_JOIN, - new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("a")), - new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("b")) + new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("a")), + new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("b")) ); Memo memo = new Memo(root); @@ -222,9 +222,9 @@ public class GroupExpressionMatchingTest { new UnboundSlot(Lists.newArrayList("b", "id")))), new LogicalJoin(JoinType.INNER_JOIN, new LogicalJoin(JoinType.LEFT_OUTER_JOIN, - new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("a")), - new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("b"))), - new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("c"))) + new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("a")), + new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("b"))), + new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("c"))) ); Pattern p1 = patterns().logicalFilter(patterns().subTree(LogicalFilter.class, LogicalJoin.class)).pattern; Iterator matchResult1 = match(root, p1); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java index 43b9c8bae1..74822d8fe5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java @@ -26,8 +26,8 @@ import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.PreAggStatus; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; @@ -75,7 +75,7 @@ public class MergeProjectPostProcessTest { t1Output.add(b); t1Output.add(c); LogicalProperties t1Properties = new LogicalProperties(() -> t1Output); - PhysicalOlapScan scan = new PhysicalOlapScan(ObjectId.createGenerator().getNextId(), t1, qualifier, 0L, + PhysicalOlapScan scan = new PhysicalOlapScan(RelationId.createGenerator().getNextId(), t1, qualifier, 0L, Collections.emptyList(), Collections.emptyList(), null, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), t1Properties); Alias x = new Alias(a, "x"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushdownFilterThroughProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushdownFilterThroughProjectTest.java index 8c0e701e99..ccbed847c3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushdownFilterThroughProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushdownFilterThroughProjectTest.java @@ -29,8 +29,8 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.Literal; -import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.PreAggStatus; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; @@ -86,7 +86,7 @@ public class PushdownFilterThroughProjectTest { t1Output.add(b); t1Output.add(c); LogicalProperties t1Properties = new LogicalProperties(() -> t1Output); - PhysicalOlapScan scan = new PhysicalOlapScan(ObjectId.createGenerator().getNextId(), t1, + PhysicalOlapScan scan = new PhysicalOlapScan(RelationId.createGenerator().getNextId(), t1, qualifier, 0L, Collections.emptyList(), Collections.emptyList(), null, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), t1Properties); Alias x = new Alias(a, "x"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/RegisterCTETest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTETest.java similarity index 57% rename from fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/RegisterCTETest.java rename to fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTETest.java index cccc56da9e..7e95409118 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/RegisterCTETest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTETest.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.common.NereidsException; -import org.apache.doris.nereids.CTEContext; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.datasets.ssb.SSBUtils; @@ -49,35 +48,45 @@ import org.junit.jupiter.api.Test; import java.util.List; -public class RegisterCTETest extends TestWithFeService implements MemoPatternMatchSupported { +public class AnalyzeCTETest extends TestWithFeService implements MemoPatternMatchSupported { private final NereidsParser parser = new NereidsParser(); - private final String sql1 = "WITH cte1 AS (SELECT s_suppkey FROM supplier WHERE s_suppkey < 5), " + private final String multiCte = "WITH cte1 AS (SELECT s_suppkey FROM supplier WHERE s_suppkey < 5), " + "cte2 AS (SELECT s_suppkey FROM cte1 WHERE s_suppkey < 3)" + "SELECT * FROM cte1, cte2"; - private final String sql2 = "WITH cte1 (skey) AS (SELECT s_suppkey, s_nation FROM supplier WHERE s_suppkey < 5), " + private final String cteWithColumnAlias = "WITH cte1 (skey) AS (SELECT s_suppkey, s_nation FROM supplier WHERE s_suppkey < 5), " + "cte2 (sk2) AS (SELECT skey FROM cte1 WHERE skey < 3)" + "SELECT * FROM cte1, cte2"; - private final String sql3 = "WITH cte1 AS (SELECT * FROM supplier), " + private final String cteConsumerInSubQuery = "WITH cte1 AS (SELECT * FROM supplier), " + "cte2 AS (SELECT * FROM supplier WHERE s_region in (\"ASIA\", \"AFRICA\"))" + "SELECT s_region, count(*) FROM cte1 GROUP BY s_region HAVING s_region in (SELECT s_region FROM cte2)"; - private final String sql4 = "WITH cte1 AS (SELECT s_suppkey AS sk FROM supplier WHERE s_suppkey < 5), " + private final String cteConsumerJoin = "WITH cte1 AS (SELECT s_suppkey AS sk FROM supplier WHERE s_suppkey < 5), " + "cte2 AS (SELECT sk FROM cte1 WHERE sk < 3)" + "SELECT * FROM cte1 JOIN cte2 ON cte1.sk = cte2.sk"; - private final String sql5 = "WITH V1 AS (SELECT s_suppkey FROM supplier), " + private final String cteReferToAnotherOne = "WITH V1 AS (SELECT s_suppkey FROM supplier), " + "V2 AS (SELECT s_suppkey FROM V1)" + "SELECT * FROM V2"; - private final String sql6 = "WITH cte1 AS (SELECT s_suppkey FROM supplier)" + private final String cteJoinSelf = "WITH cte1 AS (SELECT s_suppkey FROM supplier)" + "SELECT * FROM cte1 AS t1, cte1 AS t2"; - private final List testSql = ImmutableList.of( - sql1, sql2, sql3, sql4, sql5, sql6 + private final String cteNested = "WITH cte1 AS (" + + "WITH cte2 AS (SELECT s_suppkey FROM supplier) SELECT * FROM cte2)" + + " SELECT * FROM cte1"; + + private final String cteInTheMiddle = "SELECT * FROM (WITH cte1 AS (SELECT s_suppkey FROM supplier)" + + " SELECT * FROM cte1) a"; + + private final String cteWithDiffRelationId = "with s as (select * from supplier) select * from s as s1, s as s2"; + + private final List testSqls = ImmutableList.of( + multiCte, cteWithColumnAlias, cteConsumerInSubQuery, cteConsumerJoin, cteReferToAnotherOne, cteJoinSelf, + cteNested, cteInTheMiddle, cteWithDiffRelationId ); @Override @@ -94,13 +103,6 @@ public class RegisterCTETest extends TestWithFeService implements MemoPatternMat StatementScopeIdGenerator.clear(); } - private CTEContext getCTEContextAfterRegisterCTE(String sql) { - return PlanChecker.from(connectContext) - .analyze(sql) - .getCascadesContext() - .getCteContext(); - } - /* ******************************************************************************************** * Test CTE * ******************************************************************************************** */ @@ -114,7 +116,7 @@ public class RegisterCTETest extends TestWithFeService implements MemoPatternMat } }; - for (String sql : testSql) { + for (String sql : testSqls) { StatementScopeIdGenerator.clear(); StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); PhysicalPlan plan = new NereidsPlanner(statementContext).plan( @@ -126,97 +128,41 @@ public class RegisterCTETest extends TestWithFeService implements MemoPatternMat } } - @Test - public void testCTERegister() { - CTEContext cteContext = getCTEContextAfterRegisterCTE(sql1); - - Assertions.assertTrue(cteContext.containsCTE("cte1") - && cteContext.containsCTE("cte2")); - // LogicalPlan cte2parsedPlan = cteContext.getParsedCtePlan("cte2").get(); - // PlanChecker.from(connectContext, cte2parsedPlan) - // .matchesFromRoot( - // logicalSubQueryAlias( - // logicalProject( - // logicalFilter( - // logicalCheckPolicy( - // unboundRelation() - // ) - // ) - // ) - // ) - // ); - } - - @Test - public void testCTERegisterWithColumnAlias() { - CTEContext cteContext = getCTEContextAfterRegisterCTE(sql2); - - Assertions.assertTrue(cteContext.containsCTE("cte1") - && cteContext.containsCTE("cte2")); - - // check analyzed plan - // LogicalPlan cte1AnalyzedPlan = cteContext.getReuse("cte1").get(); - - // PlanChecker.from(connectContext, cte1AnalyzedPlan) - // .matchesFromRoot( - // logicalSubQueryAlias( - // logicalProject() - // .when(p -> p.getProjects().size() == 2 - // && p.getProjects().get(0).getName().equals("s_suppkey") - // && p.getProjects().get(0).getExprId().asInt() == 14 - // && p.getProjects().get(0).getQualifier().equals(ImmutableList.of("default_cluster:test", "supplier")) - // && p.getProjects().get(1).getName().equals("s_nation") - // && p.getProjects().get(1).getExprId().asInt() == 18 - // && p.getProjects().get(1).getQualifier().equals(ImmutableList.of("default_cluster:test", "supplier")) - // ) - // ) - // .when(a -> a.getAlias().equals("cte1")) - // .when(a -> a.getOutput().size() == 2 - // && a.getOutput().get(0).getName().equals("skey") - // && a.getOutput().get(0).getExprId().asInt() == 14 - // && a.getOutput().get(0).getQualifier().equals(ImmutableList.of("cte1")) - // && a.getOutput().get(1).getName().equals("s_nation") - // && a.getOutput().get(1).getExprId().asInt() == 18 - // && a.getOutput().get(1).getQualifier().equals(ImmutableList.of("cte1")) - // ) - // ); - } - @Test public void testCTEInHavingAndSubquery() { PlanChecker.from(connectContext) - .analyze(sql3) + .analyze(cteConsumerInSubQuery) .applyBottomUp(new PullUpProjectUnderApply()) .applyBottomUp(new UnCorrelatedApplyFilter()) .applyBottomUp(new InApplyToJoin()) .matches( - logicalCTE( - logicalFilter( - logicalProject( - logicalJoin( - logicalAggregate( - logicalCTEConsumer() - ), logicalProject( - logicalCTEConsumer()) - ) - ) + logicalFilter( + logicalProject( + logicalJoin( + logicalAggregate(), + logicalProject() + ) + ) - ) - ) + ) ); } @Test public void testCTEWithAlias() { PlanChecker.from(connectContext) - .analyze(sql4) + .analyze(cteConsumerJoin) .matchesFromRoot( - logicalCTE( - logicalProject( - logicalJoin( - logicalCTEConsumer(), - logicalCTEConsumer() + logicalCTEAnchor( + logicalCTEProducer(), + logicalCTEAnchor( + logicalCTEProducer(), + logicalProject( + logicalJoin( + logicalCTEConsumer(), + logicalCTEConsumer() + ) ) ) ) @@ -226,15 +172,82 @@ public class RegisterCTETest extends TestWithFeService implements MemoPatternMat @Test public void testCTEWithAnExistedTableOrViewName() { PlanChecker.from(connectContext) - .analyze(sql5) + .analyze(cteReferToAnotherOne) .matchesFromRoot( - logicalCTE( + logicalCTEAnchor( + logicalCTEProducer(), + logicalCTEAnchor( + logicalCTEProducer(), + logicalProject( + logicalCTEConsumer() + ) + ) + ) + ); + + } + + @Test + public void testDifferenceRelationId() { + PlanChecker.from(connectContext) + .analyze(cteWithDiffRelationId) + .matchesFromRoot( + logicalCTEAnchor( + logicalCTEProducer(), + logicalProject( + logicalJoin( + logicalSubQueryAlias( + logicalCTEConsumer() + ), + logicalSubQueryAlias( + logicalCTEConsumer() + ) + ) + ) + ) + ); + } + + @Test + public void testCteInTheMiddle() { + PlanChecker.from(connectContext) + .analyze(cteInTheMiddle) + .matchesFromRoot( + logicalProject( + logicalSubQueryAlias( + logicalCTEAnchor( + logicalCTEProducer(), + logicalProject( + logicalCTEConsumer() + ) + ) + ) + ) + + ); + } + + @Test + public void testCteNested() { + PlanChecker.from(connectContext) + .analyze(cteNested) + .matchesFromRoot( + logicalCTEAnchor( + logicalCTEProducer( + logicalSubQueryAlias( + logicalCTEAnchor( + logicalCTEProducer(), + logicalProject( + logicalCTEConsumer() + ) + ) + ) + ), logicalProject( logicalCTEConsumer() ) ) ); - } @@ -247,9 +260,8 @@ public class RegisterCTETest extends TestWithFeService implements MemoPatternMat String sql = "WITH cte1 (a1, A1) AS (SELECT * FROM supplier)" + "SELECT * FROM cte1"; - NereidsException exception = Assertions.assertThrows(NereidsException.class, () -> { - PlanChecker.from(connectContext).checkPlannerResult(sql); - }, "Not throw expected exception."); + NereidsException exception = Assertions.assertThrows(NereidsException.class, + () -> PlanChecker.from(connectContext).checkPlannerResult(sql), "Not throw expected exception."); Assertions.assertTrue(exception.getMessage().contains("Duplicated CTE column alias: [a1] in CTE [cte1]")); } @@ -259,9 +271,8 @@ public class RegisterCTETest extends TestWithFeService implements MemoPatternMat + "(SELECT s_suppkey FROM supplier)" + "SELECT * FROM cte1"; - NereidsException exception = Assertions.assertThrows(NereidsException.class, () -> { - PlanChecker.from(connectContext).checkPlannerResult(sql); - }, "Not throw expected exception."); + NereidsException exception = Assertions.assertThrows(NereidsException.class, + () -> PlanChecker.from(connectContext).checkPlannerResult(sql), "Not throw expected exception."); System.out.println(exception.getMessage()); Assertions.assertTrue(exception.getMessage().contains("CTE [cte1] returns 2 columns, " + "but 1 labels were specified.")); @@ -273,9 +284,8 @@ public class RegisterCTETest extends TestWithFeService implements MemoPatternMat + "cte2 AS (SELECT * FROM supplier)" + "SELECT * FROM cte1, cte2"; - RuntimeException exception = Assertions.assertThrows(RuntimeException.class, () -> { - PlanChecker.from(connectContext).checkPlannerResult(sql); - }, "Not throw expected exception."); + RuntimeException exception = Assertions.assertThrows(RuntimeException.class, + () -> PlanChecker.from(connectContext).checkPlannerResult(sql), "Not throw expected exception."); Assertions.assertTrue(exception.getMessage().contains("[cte2] does not exist in database")); } @@ -284,9 +294,8 @@ public class RegisterCTETest extends TestWithFeService implements MemoPatternMat String sql = "WITH cte1 AS (SELECT * FROM not_existed_table)" + "SELECT * FROM supplier"; - RuntimeException exception = Assertions.assertThrows(RuntimeException.class, () -> { - PlanChecker.from(connectContext).checkPlannerResult(sql); - }, "Not throw expected exception."); + RuntimeException exception = Assertions.assertThrows(RuntimeException.class, + () -> PlanChecker.from(connectContext).checkPlannerResult(sql), "Not throw expected exception."); Assertions.assertTrue(exception.getMessage().contains("[not_existed_table] does not exist in database")); } @@ -296,29 +305,18 @@ public class RegisterCTETest extends TestWithFeService implements MemoPatternMat + "cte1 AS (SELECT * FROM part)" + "SELECT * FROM cte1"; - AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> { - PlanChecker.from(connectContext).analyze(sql); - }, "Not throw expected exception."); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze(sql), "Not throw expected exception."); Assertions.assertTrue(exception.getMessage().contains("[cte1] cannot be used more than once")); } @Test - public void testDifferenceRelationId() { - PlanChecker.from(connectContext) - .analyze("with s as (select * from supplier) select * from s as s1, s as s2") - .matchesFromRoot( - logicalCTE( - logicalProject( - logicalJoin( - logicalSubQueryAlias( - logicalCTEConsumer() - ), - logicalSubQueryAlias( - logicalCTEConsumer() - ) - ) - ) - ) - ); + public void testCTEExceptionOfRefterCTENameNotInScope() { + String sql = "WITH cte1 AS (WITH cte2 AS (SELECT * FROM supplier) SELECT * FROM cte2)" + + "SELECT * FROM cte2"; + + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze(sql), "Not throw expected exception."); + Assertions.assertTrue(exception.getMessage().contains("Table [cte2] does not exist in database")); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java index f9550dce0b..3a99e91dd8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java @@ -27,11 +27,11 @@ import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.pattern.GeneratedPlanPatterns; import org.apache.doris.nereids.rules.RulePromise; import org.apache.doris.nereids.rules.analysis.BindRelation.CustomTableResolver; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanRewriter; -import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.ImmutableList; @@ -59,7 +59,7 @@ class BindRelationTest extends TestWithFeService implements GeneratedPlanPattern @Test void bindInCurrentDb() { connectContext.setDatabase(DEFAULT_CLUSTER_PREFIX + DB1); - Plan plan = PlanRewriter.bottomUpRewrite(new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("t")), + Plan plan = PlanRewriter.bottomUpRewrite(new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("t")), connectContext, new BindRelation()); Assertions.assertTrue(plan instanceof LogicalOlapScan); @@ -71,7 +71,7 @@ class BindRelationTest extends TestWithFeService implements GeneratedPlanPattern @Test void bindByDbQualifier() { connectContext.setDatabase(DEFAULT_CLUSTER_PREFIX + DB2); - Plan plan = PlanRewriter.bottomUpRewrite(new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("db1", "t")), + Plan plan = PlanRewriter.bottomUpRewrite(new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("db1", "t")), connectContext, new BindRelation()); Assertions.assertTrue(plan instanceof LogicalOlapScan); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindSlotReferenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindSlotReferenceTest.java index 39dad71e07..0a3334b4cf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindSlotReferenceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindSlotReferenceTest.java @@ -33,7 +33,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -51,7 +50,7 @@ class BindSlotReferenceTest { @Test public void testCannotFindSlot() { LogicalProject project = new LogicalProject<>(ImmutableList.of(new UnboundSlot("foo")), - new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student)); + new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student)); AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> PlanChecker.from(MemoTestUtils.createConnectContext()).analyze(project)); Assertions.assertEquals("unbounded object foo in PROJECT clause.", exception.getMessage()); @@ -59,8 +58,8 @@ class BindSlotReferenceTest { @Test public void testAmbiguousSlot() { - LogicalOlapScan scan1 = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); - LogicalOlapScan scan2 = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan scan1 = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); + LogicalOlapScan scan2 = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalJoin join = new LogicalJoin<>( JoinType.CROSS_JOIN, scan1, scan2); LogicalProject> project = new LogicalProject<>( @@ -79,9 +78,9 @@ class BindSlotReferenceTest { */ @Test public void testGroupByOnJoin() { - LogicalOlapScan scan1 = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan scan1 = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalSubQueryAlias sub1 = new LogicalSubQueryAlias("t1", scan1); - LogicalOlapScan scan2 = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan scan2 = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalSubQueryAlias sub2 = new LogicalSubQueryAlias("t2", scan2); LogicalJoin, LogicalSubQueryAlias> join = new LogicalJoin<>(JoinType.CROSS_JOIN, sub1, sub2); @@ -105,9 +104,9 @@ class BindSlotReferenceTest { */ @Test public void testGroupByOnJoinAmbiguous() { - LogicalOlapScan scan1 = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan scan1 = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalSubQueryAlias sub1 = new LogicalSubQueryAlias("t1", scan1); - LogicalOlapScan scan2 = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + LogicalOlapScan scan2 = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); LogicalSubQueryAlias sub2 = new LogicalSubQueryAlias("t2", scan2); LogicalJoin, LogicalSubQueryAlias> join = new LogicalJoin<>(JoinType.CROSS_JOIN, sub1, sub2); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckAnalysisTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckAnalysisTest.java index 386d8a1718..f470c0aa1f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckAnalysisTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckAnalysisTest.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.And; import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.plans.GroupPlan; @@ -53,7 +54,7 @@ public class CheckAnalysisTest { @Test public void testCheckNotWithChildrenWithErrorType() { - Plan plan = new LogicalOneRowRelation( + Plan plan = new LogicalOneRowRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of(new Alias(new Not(new IntegerLiteral(2)), "not_2"))); CheckAnalysis checkAnalysis = new CheckAnalysis(); Assertions.assertThrows(AnalysisException.class, () -> @@ -63,7 +64,7 @@ public class CheckAnalysisTest { @Test public void testUnbound() { UnboundFunction func = new UnboundFunction("now", Lists.newArrayList(new IntegerLiteral(1))); - Plan plan = new LogicalOneRowRelation( + Plan plan = new LogicalOneRowRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of(new Alias(func, "unboundFunction"))); CheckBound checkBound = new CheckBound(); Assertions.assertThrows(AnalysisException.class, () -> diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckRowPolicyTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckRowPolicyTest.java index b7bdad064a..47e672db99 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckRowPolicyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckRowPolicyTest.java @@ -34,13 +34,13 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.FeConstants; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalCheckPolicy; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; import org.apache.doris.nereids.util.PlanRewriter; -import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.system.SystemInfoService; import org.apache.doris.thrift.TStorageType; import org.apache.doris.utframe.TestWithFeService; @@ -98,7 +98,7 @@ public class CheckRowPolicyTest extends TestWithFeService { @Test public void checkUser() throws AnalysisException, org.apache.doris.common.AnalysisException { - LogicalRelation relation = new LogicalOlapScan(RelationUtil.newRelationId(), olapTable, Arrays.asList(fullDbName)); + LogicalRelation relation = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), olapTable, Arrays.asList(fullDbName)); LogicalCheckPolicy checkPolicy = new LogicalCheckPolicy<>(relation); useUser("root"); @@ -113,7 +113,7 @@ public class CheckRowPolicyTest extends TestWithFeService { @Test public void checkNoPolicy() throws org.apache.doris.common.AnalysisException { useUser(userName); - LogicalRelation relation = new LogicalOlapScan(RelationUtil.newRelationId(), olapTable, Arrays.asList(fullDbName)); + LogicalRelation relation = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), olapTable, Arrays.asList(fullDbName)); LogicalCheckPolicy checkPolicy = new LogicalCheckPolicy<>(relation); Plan plan = PlanRewriter.bottomUpRewrite(checkPolicy, connectContext, new CheckPolicy()); Assertions.assertEquals(plan, relation); @@ -122,7 +122,7 @@ public class CheckRowPolicyTest extends TestWithFeService { @Test public void checkOnePolicy() throws Exception { useUser(userName); - LogicalRelation relation = new LogicalOlapScan(RelationUtil.newRelationId(), olapTable, Arrays.asList(fullDbName)); + LogicalRelation relation = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), olapTable, Arrays.asList(fullDbName)); LogicalCheckPolicy checkPolicy = new LogicalCheckPolicy<>(relation); connectContext.getSessionVariable().setEnableNereidsPlanner(true); createPolicy("CREATE ROW POLICY " diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java index 8ee43dbfd2..9cd4ea49c6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/ExpressionRewriteTestHelper.java @@ -25,7 +25,7 @@ import org.apache.doris.nereids.rules.expression.rules.FunctionBinder; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.ObjectId; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.DataType; @@ -52,7 +52,7 @@ public abstract class ExpressionRewriteTestHelper { public ExpressionRewriteTestHelper() { CascadesContext cascadesContext = MemoTestUtils.createCascadesContext( - new UnboundRelation(new ObjectId(1), ImmutableList.of("tbl"))); + new UnboundRelation(new RelationId(1), ImmutableList.of("tbl"))); context = new ExpressionRewriteContext(cascadesContext); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java index c9f74bc7f2..4f89f77937 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java @@ -39,7 +39,7 @@ import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Interval.TimeUnit; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; -import org.apache.doris.nereids.trees.plans.ObjectId; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DoubleType; import org.apache.doris.nereids.types.IntegerType; @@ -607,7 +607,7 @@ public class FoldConstantTest extends ExpressionRewriteTestHelper { public void testFoldTypeOfNullLiteral() { String actualExpression = "append_trailing_char_if_absent(cast(version() as varchar), cast(null as varchar))"; ExpressionRewriteContext context = new ExpressionRewriteContext( - MemoTestUtils.createCascadesContext(new UnboundRelation(new ObjectId(1), ImmutableList.of("test_table")))); + MemoTestUtils.createCascadesContext(new UnboundRelation(new RelationId(1), ImmutableList.of("test_table")))); NereidsParser parser = new NereidsParser(); Expression e1 = parser.parseExpression(actualExpression); e1 = new ExpressionNormalization().rewrite(FunctionBinder.INSTANCE.rewrite(e1, context), context); @@ -616,7 +616,7 @@ public class FoldConstantTest extends ExpressionRewriteTestHelper { private void assertRewriteExpression(String actualExpression, String expectedExpression) { ExpressionRewriteContext context = new ExpressionRewriteContext( - MemoTestUtils.createCascadesContext(new UnboundRelation(new ObjectId(1), ImmutableList.of("test_table")))); + MemoTestUtils.createCascadesContext(new UnboundRelation(new RelationId(1), ImmutableList.of("test_table")))); NereidsParser parser = new NereidsParser(); Expression e1 = parser.parseExpression(actualExpression); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java index 68aad77c64..8058137b6f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java @@ -25,7 +25,7 @@ import org.apache.doris.nereids.rules.expression.rules.SimplifyRange; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.ObjectId; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.DataType; @@ -52,7 +52,7 @@ public class SimplifyRangeTest { public SimplifyRangeTest() { CascadesContext cascadesContext = MemoTestUtils.createCascadesContext( - new UnboundRelation(new ObjectId(1), ImmutableList.of("tbl"))); + new UnboundRelation(new RelationId(1), ImmutableList.of("tbl"))); context = new ExpressionRewriteContext(cascadesContext); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/implementation/LogicalWindowToPhysicalWindowTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/implementation/LogicalWindowToPhysicalWindowTest.java index 277e095b1c..c1ac714a93 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/implementation/LogicalWindowToPhysicalWindowTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/implementation/LogicalWindowToPhysicalWindowTest.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.rules.implementation.LogicalWindowToPhysicalWind import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.expressions.WindowFrame; import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; @@ -32,7 +33,6 @@ import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -50,7 +50,7 @@ public class LogicalWindowToPhysicalWindowTest implements MemoPatternMatchSuppor @BeforeAll public final void beforeAll() { - rStudent = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student, + rStudent = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student, ImmutableList.of("")); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/AggScalarSubQueryToWindowFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/AggScalarSubQueryToWindowFunctionTest.java index 7839681a43..f70161f465 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/AggScalarSubQueryToWindowFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/AggScalarSubQueryToWindowFunctionTest.java @@ -341,7 +341,7 @@ public class AggScalarSubQueryToWindowFunctionTest extends TPCHTestBase implemen System.out.printf("Test:\n%s\n\n", sql); Plan plan = PlanChecker.from(createCascadesContext(sql)) .analyze(sql) - .applyTopDown(new AggScalarSubQueryToWindowFunction()) + .customRewrite(new AggScalarSubQueryToWindowFunction()) .rewrite() .getPlan(); System.out.println(plan.treeString()); @@ -352,7 +352,7 @@ public class AggScalarSubQueryToWindowFunctionTest extends TPCHTestBase implemen System.out.printf("Test:\n%s\n\n", sql); Plan plan = PlanChecker.from(createCascadesContext(sql)) .analyze(sql) - .applyTopDown(new AggScalarSubQueryToWindowFunction()) + .customRewrite(new AggScalarSubQueryToWindowFunction()) .rewrite() .getPlan(); System.out.println(plan.treeString()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/AggregateStrategiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/AggregateStrategiesTest.java index e8419b2458..6f3bfaa7e5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/AggregateStrategiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/AggregateStrategiesTest.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateParam; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; @@ -40,7 +41,6 @@ import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -60,7 +60,7 @@ public class AggregateStrategiesTest implements MemoPatternMatchSupported { @BeforeAll public final void beforeAll() { - rStudent = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student, + rStudent = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student, ImmutableList.of("")); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/CheckAndStandardizeWindowFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/CheckAndStandardizeWindowFunctionTest.java index 1da3839339..ffe88af2fe 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/CheckAndStandardizeWindowFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/CheckAndStandardizeWindowFunctionTest.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.expressions.WindowFrame; import org.apache.doris.nereids.trees.expressions.WindowFrame.FrameBoundary; @@ -43,7 +44,6 @@ import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -66,7 +66,7 @@ public class CheckAndStandardizeWindowFunctionTest implements MemoPatternMatchSu @BeforeAll public final void beforeAll() { - rStudent = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student, ImmutableList.of()); + rStudent = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student, ImmutableList.of()); gender = rStudent.getOutput().get(1).toSlot(); age = rStudent.getOutput().get(3).toSlot(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByConstantTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByConstantTest.java index 8e06426578..2fbf2adf17 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByConstantTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByConstantTest.java @@ -31,7 +31,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Max; import org.apache.doris.nereids.trees.expressions.functions.agg.Min; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; -import org.apache.doris.nereids.trees.plans.ObjectId; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.types.IntegerType; @@ -66,7 +66,7 @@ class EliminateGroupByConstantTest implements MemoPatternMatchSupported { @Test void testIntegerLiteral() { LogicalPlan aggregate = new LogicalPlanBuilder( - new LogicalOlapScan(ObjectId.createGenerator().getNextId(), table)) + new LogicalOlapScan(RelationId.createGenerator().getNextId(), table)) .agg(ImmutableList.of(new IntegerLiteral(1), k2), ImmutableList.of(k1, k2)) .build(); @@ -82,7 +82,7 @@ class EliminateGroupByConstantTest implements MemoPatternMatchSupported { @Test void testOtherLiteral() { LogicalPlan aggregate = new LogicalPlanBuilder( - new LogicalOlapScan(ObjectId.createGenerator().getNextId(), table)) + new LogicalOlapScan(RelationId.createGenerator().getNextId(), table)) .agg(ImmutableList.of( new StringLiteral("str"), k2), ImmutableList.of( @@ -100,7 +100,7 @@ class EliminateGroupByConstantTest implements MemoPatternMatchSupported { @Test void testMixedLiteral() { LogicalPlan aggregate = new LogicalPlanBuilder( - new LogicalOlapScan(ObjectId.createGenerator().getNextId(), table)) + new LogicalOlapScan(RelationId.createGenerator().getNextId(), table)) .agg(ImmutableList.of( new StringLiteral("str"), k2, new IntegerLiteral(1), @@ -124,7 +124,7 @@ class EliminateGroupByConstantTest implements MemoPatternMatchSupported { @Test void testComplexGroupBy() { LogicalPlan aggregate = new LogicalPlanBuilder( - new LogicalOlapScan(ObjectId.createGenerator().getNextId(), table)) + new LogicalOlapScan(RelationId.createGenerator().getNextId(), table)) .agg(ImmutableList.of( new IntegerLiteral(1), new IntegerLiteral(2), @@ -147,7 +147,7 @@ class EliminateGroupByConstantTest implements MemoPatternMatchSupported { @Test void testOutOfRange() { LogicalPlan aggregate = new LogicalPlanBuilder( - new LogicalOlapScan(ObjectId.createGenerator().getNextId(), table)) + new LogicalOlapScan(RelationId.createGenerator().getNextId(), table)) .agg(ImmutableList.of( new StringLiteral("str"), k2, new IntegerLiteral(1), diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProjectTest.java index de0462f3da..548a7dd068 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProjectTest.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.types.IntegerType; @@ -59,7 +60,7 @@ class EliminateUnnecessaryProjectTest extends TestWithFeService implements MemoP .build(); PlanChecker.from(MemoTestUtils.createConnectContext(), unnecessaryProject) - .applyTopDown(new EliminateUnnecessaryProject()) + .customRewrite(new EliminateUnnecessaryProject()) .matchesFromRoot(logicalFilter(logicalProject())); } @@ -70,7 +71,7 @@ class EliminateUnnecessaryProjectTest extends TestWithFeService implements MemoP .build(); PlanChecker.from(MemoTestUtils.createConnectContext(), unnecessaryProject) - .applyTopDown(new EliminateUnnecessaryProject()) + .customRewrite(new EliminateUnnecessaryProject()) .matchesFromRoot(logicalOlapScan()); } @@ -81,19 +82,20 @@ class EliminateUnnecessaryProjectTest extends TestWithFeService implements MemoP .build(); PlanChecker.from(MemoTestUtils.createConnectContext(), necessaryProject) - .applyTopDown(new EliminateUnnecessaryProject()) + .customRewrite(new EliminateUnnecessaryProject()) .matchesFromRoot(logicalProject()); } @Test void testEliminateProjectWhenEmptyRelationChild() { - LogicalPlan unnecessaryProject = new LogicalPlanBuilder(new LogicalEmptyRelation(ImmutableList.of( - new SlotReference("k1", IntegerType.INSTANCE), - new SlotReference("k2", IntegerType.INSTANCE)))) + LogicalPlan unnecessaryProject = new LogicalPlanBuilder(new LogicalEmptyRelation(new RelationId(1), + ImmutableList.of( + new SlotReference("k1", IntegerType.INSTANCE), + new SlotReference("k2", IntegerType.INSTANCE)))) .project(ImmutableList.of(1, 0)) .build(); PlanChecker.from(MemoTestUtils.createConnectContext(), unnecessaryProject) - .applyTopDown(new EliminateUnnecessaryProject()) + .customRewrite(new EliminateUnnecessaryProject()) .matchesFromRoot(logicalEmptyRelation()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ExtractAndNormalizeWindowExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ExtractAndNormalizeWindowExpressionTest.java index 22e1dc698a..e676caa37a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ExtractAndNormalizeWindowExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ExtractAndNormalizeWindowExpressionTest.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.OrderExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; import org.apache.doris.nereids.trees.expressions.functions.window.Rank; @@ -37,7 +38,6 @@ import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -54,7 +54,7 @@ public class ExtractAndNormalizeWindowExpressionTest implements MemoPatternMatch @BeforeAll public final void beforeAll() { - rStudent = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student, ImmutableList.of()); + rStudent = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student, ImmutableList.of()); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergeProjectsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergeProjectsTest.java index 40da31a3cf..e0cd9f6223 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergeProjectsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergeProjectsTest.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.trees.expressions.Add; import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; @@ -28,7 +29,6 @@ import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -40,7 +40,7 @@ import java.util.Objects; * MergeConsecutiveProjects ut */ public class MergeProjectsTest implements MemoPatternMatchSupported { - LogicalOlapScan score = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.score); + LogicalOlapScan score = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.score); @Test public void testMergeConsecutiveProjects() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/NormalizeAggregateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/NormalizeAggregateTest.java index e6a34daef7..32f7b324f9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/NormalizeAggregateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/NormalizeAggregateTest.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Multiply; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; @@ -36,7 +37,6 @@ import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -52,7 +52,7 @@ public class NormalizeAggregateTest implements MemoPatternMatchSupported { @BeforeAll public final void beforeAll() { - rStudent = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student, + rStudent = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student, ImmutableList.of()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java index 7fd073ef07..e0244a80e3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java @@ -34,7 +34,7 @@ import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.LessThanEqual; import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; -import org.apache.doris.nereids.trees.plans.ObjectId; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.util.MemoPatternMatchSupported; @@ -130,7 +130,7 @@ class PruneOlapScanTabletTest implements MemoPatternMatchSupported { } }; - LogicalOlapScan scan = new LogicalOlapScan(ObjectId.createGenerator().getNextId(), olapTable); + LogicalOlapScan scan = new LogicalOlapScan(RelationId.createGenerator().getNextId(), olapTable); GreaterThanEqual greaterThanEqual = new GreaterThanEqual(scan.getOutput().get(0), new DateLiteral("2019-08-22")); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownDistinctThroughJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownDistinctThroughJoinTest.java index cc2e08d188..124a72e5c0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownDistinctThroughJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownDistinctThroughJoinTest.java @@ -46,7 +46,7 @@ class PushdownDistinctThroughJoinTest implements MemoPatternMatchSupported { .build(); PlanChecker.from(MemoTestUtils.createConnectContext(), plan) - .applyTopDown(new PushdownDistinctThroughJoin()) + .customRewrite(new PushdownDistinctThroughJoin()) .matches( logicalAggregate( logicalJoin( @@ -70,7 +70,7 @@ class PushdownDistinctThroughJoinTest implements MemoPatternMatchSupported { .build(); PlanChecker.from(MemoTestUtils.createConnectContext(), plan) - .applyTopDown(new PushdownDistinctThroughJoin()) + .customRewrite(new PushdownDistinctThroughJoin()) .matches( logicalAggregate( logicalJoin( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughAggregationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughAggregationTest.java index 03f584afcf..c6fe1f6420 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughAggregationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughAggregationTest.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.expressions.GreaterThan; import org.apache.doris.nereids.trees.expressions.LessThanEqual; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.functions.agg.Max; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; @@ -35,14 +36,13 @@ import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import org.junit.jupiter.api.Test; public class PushdownFilterThroughAggregationTest implements MemoPatternMatchSupported { - private final LogicalOlapScan scan = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student, + private final LogicalOlapScan scan = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student, ImmutableList.of("")); /*- diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughSortTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughSortTest.java index b2861be62c..0ccc12145a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughSortTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughSortTest.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.GreaterThan; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; @@ -29,7 +30,6 @@ import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; @@ -37,7 +37,7 @@ import org.junit.jupiter.api.Test; import java.util.stream.Collectors; public class PushdownFilterThroughSortTest implements MemoPatternMatchSupported { - private final LogicalOlapScan scan = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student, + private final LogicalOlapScan scan = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student, ImmutableList.of("")); @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindowTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindowTest.java index 8c5ac1ccf8..a53e8c6a05 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindowTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughWindowTest.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.LessThanEqual; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.expressions.WindowFrame; import org.apache.doris.nereids.trees.expressions.functions.window.RowNumber; @@ -36,7 +37,6 @@ import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; @@ -47,7 +47,7 @@ import org.junit.jupiter.api.Test; import java.util.List; public class PushdownFilterThroughWindowTest implements MemoPatternMatchSupported { - private final LogicalOlapScan scan = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student, + private final LogicalOlapScan scan = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student, ImmutableList.of("")); /*- diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownLimitTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownLimitTest.java index e19f991d8c..f85882791e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownLimitTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushdownLimitTest.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.expressions.WindowFrame; import org.apache.doris.nereids.trees.expressions.functions.window.Rank; @@ -45,7 +46,6 @@ import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.PlanFragment; import org.apache.doris.qe.ConnectContext; @@ -64,8 +64,8 @@ import java.util.function.Function; import java.util.stream.Collectors; class PushdownLimitTest extends TestWithFeService implements MemoPatternMatchSupported { - private final LogicalOlapScan scanScore = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.score); - private Plan scanStudent = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); + private final LogicalOlapScan scanScore = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.score); + private Plan scanStudent = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student); @Override protected void runBeforeAll() throws Exception { @@ -364,8 +364,8 @@ class PushdownLimitTest extends TestWithFeService implements MemoPatternMatchSup LogicalJoin join = new LogicalJoin<>( joinType, joinConditions, - new LogicalOlapScan(((LogicalOlapScan) scanScore).getId(), PlanConstructor.score), - new LogicalOlapScan(((LogicalOlapScan) scanStudent).getId(), PlanConstructor.student) + new LogicalOlapScan(((LogicalOlapScan) scanScore).getRelationId(), PlanConstructor.score), + new LogicalOlapScan(((LogicalOlapScan) scanStudent).getRelationId(), PlanConstructor.student) ); if (hasProject) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java index 3c8be1594b..4ca19dedf7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.plans.FakePlan; import org.apache.doris.nereids.trees.plans.GroupPlan; @@ -37,7 +38,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.qe.ConnectContext; import org.apache.doris.statistics.ColumnStatistic; import org.apache.doris.statistics.ColumnStatisticBuilder; @@ -252,7 +252,7 @@ public class StatsCalculatorTest { SlotReference slot1 = new SlotReference("c1", IntegerType.INSTANCE, true, qualifier); OlapTable table1 = PlanConstructor.newOlapTable(tableId1, "t1", 0); - LogicalOlapScan logicalOlapScan1 = (LogicalOlapScan) new LogicalOlapScan(RelationUtil.newRelationId(), table1, + LogicalOlapScan logicalOlapScan1 = (LogicalOlapScan) new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), table1, Collections.emptyList()).withGroupExprLogicalPropChildren(Optional.empty(), Optional.of(new LogicalProperties(() -> ImmutableList.of(slot1))), ImmutableList.of()); Group childGroup = newGroup(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java index ad9d1df91a..fa333a0f16 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java @@ -28,12 +28,12 @@ import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateParam; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; @@ -44,8 +44,6 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.util.ExpressionUtils; -import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -122,17 +120,6 @@ public class PlanEqualsTest { Assertions.assertNotEquals(unexpected, actual); } - @Test - public void testLogicalOlapScan() { - LogicalOlapScan actual = PlanConstructor.newLogicalOlapScanWithSameId(0, "table", 0); - - LogicalOlapScan expected = PlanConstructor.newLogicalOlapScanWithSameId(0, "table", 0); - Assertions.assertEquals(expected, actual); - - LogicalOlapScan unexpected = PlanConstructor.newLogicalOlapScanWithSameId(1, "table", 0); - Assertions.assertNotEquals(unexpected, actual); - } - @Test public void testLogicalProject(@Mocked Plan child) { LogicalProject actual = new LogicalProject<>( @@ -250,24 +237,25 @@ public class PlanEqualsTest { @Mocked LogicalProperties logicalProperties, @Mocked OlapTable olapTable, @Mocked DistributionSpecHash distributionSpecHash) { + List selectedTabletId = Lists.newArrayList(); for (Partition partition : olapTable.getAllPartitions()) { selectedTabletId.addAll(partition.getBaseIndex().getTabletIdsInOrder()); } - ObjectId id = RelationUtil.newRelationId(); + RelationId id = StatementScopeIdGenerator.newRelationId(); PhysicalOlapScan actual = new PhysicalOlapScan(id, olapTable, Lists.newArrayList("a"), - olapTable.getBaseIndexId(), selectedTabletId, olapTable.getPartitionIds(), distributionSpecHash, + 1L, selectedTabletId, olapTable.getPartitionIds(), distributionSpecHash, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), logicalProperties); PhysicalOlapScan expected = new PhysicalOlapScan(id, olapTable, Lists.newArrayList("a"), - olapTable.getBaseIndexId(), selectedTabletId, olapTable.getPartitionIds(), distributionSpecHash, + 1L, selectedTabletId, olapTable.getPartitionIds(), distributionSpecHash, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), logicalProperties); Assertions.assertEquals(expected, actual); PhysicalOlapScan unexpected = new PhysicalOlapScan(id, olapTable, Lists.newArrayList("b"), - olapTable.getBaseIndexId(), selectedTabletId, olapTable.getPartitionIds(), distributionSpecHash, + 12345L, selectedTabletId, olapTable.getPartitionIds(), distributionSpecHash, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), logicalProperties); Assertions.assertNotEquals(unexpected, actual); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanOutputTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanOutputTest.java index eb508f53c9..a926f970d7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanOutputTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanOutputTest.java @@ -17,20 +17,19 @@ package org.apache.doris.nereids.trees.plans; -import org.apache.doris.catalog.Table; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.util.PlanConstructor; -import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.statistics.Statistics; import com.google.common.collect.ImmutableList; @@ -58,7 +57,7 @@ public class PlanOutputTest { @Test public void testLazyComputeOutput() { // not throw exception when create new UnboundRelation - UnboundRelation relationPlan = new UnboundRelation(RelationUtil.newRelationId(), ImmutableList.of("a")); + UnboundRelation relationPlan = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of("a")); try { // throw exception when getOutput @@ -72,8 +71,8 @@ public class PlanOutputTest { @Test public void testPhysicalPlanMustHaveLogicalProperties() { Assertions.assertThrows(NullPointerException.class, () -> - new PhysicalRelation(RelationUtil.newRelationId(), - PlanType.PHYSICAL_OLAP_SCAN, ImmutableList.of("db"), Optional.empty(), null) { + new PhysicalRelation(StatementScopeIdGenerator.newRelationId(), + PlanType.PHYSICAL_OLAP_SCAN, Optional.empty(), null) { @Override public Plan withGroupExpression(Optional groupExpression) { return null; @@ -90,11 +89,6 @@ public class PlanOutputTest { Statistics statsDeriveResult) { return null; } - - @Override - public Table getTable() { - return null; - } } ); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoTestUtils.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoTestUtils.java index f5d7da7df8..8d3298f689 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoTestUtils.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoTestUtils.java @@ -81,7 +81,7 @@ public class MemoTestUtils { public static CascadesContext createCascadesContext(StatementContext statementContext, Plan initPlan) { PhysicalProperties requestProperties = NereidsPlanner.buildInitRequireProperties(); - CascadesContext cascadesContext = CascadesContext.newRewriteContext( + CascadesContext cascadesContext = CascadesContext.initContext( statementContext, initPlan, requestProperties); cascadesContext.toMemo(); MemoValidator.validateInitState(cascadesContext.getMemo(), initPlan); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index ab57a650f6..25dbe51f42 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -30,7 +30,9 @@ import org.apache.doris.nereids.jobs.cascades.DeriveStatsJob; import org.apache.doris.nereids.jobs.executor.Optimizer; import org.apache.doris.nereids.jobs.executor.Rewriter; import org.apache.doris.nereids.jobs.joinorder.JoinOrderJob; -import org.apache.doris.nereids.jobs.rewrite.CustomRewriteJob; +import org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteBottomUpJob; +import org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteTopDownJob; +import org.apache.doris.nereids.jobs.rewrite.RootPlanTreeRewriteJob; import org.apache.doris.nereids.memo.CopyInResult; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; @@ -64,12 +66,15 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.OriginStatement; import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.function.Consumer; /** @@ -79,8 +84,6 @@ public class PlanChecker { private final ConnectContext connectContext; private CascadesContext cascadesContext; - private Plan parsedPlan; - private PhysicalPlan physicalPlan; public PlanChecker(ConnectContext connectContext) { @@ -108,7 +111,7 @@ public class PlanChecker { public PlanChecker checkParse(String sql, Consumer consumer) { PlanParseChecker checker = new PlanParseChecker(sql); consumer.accept(checker); - parsedPlan = checker.parsedSupplier.get(); + checker.parsedSupplier.get(); return this; } @@ -126,7 +129,12 @@ public class PlanChecker { public PlanChecker analyze(Plan plan) { this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan); + Set originDisableRules = connectContext.getSessionVariable().getDisableNereidsRules(); + Set disableRuleWithAuth = Sets.newHashSet(originDisableRules); + disableRuleWithAuth.add(RuleType.RELATION_AUTHENTICATION.name()); + connectContext.getSessionVariable().setDisableNereidsRules(String.join(",", disableRuleWithAuth)); this.cascadesContext.newAnalyzer().analyze(); + connectContext.getSessionVariable().setDisableNereidsRules(String.join(",", originDisableRules)); this.cascadesContext.toMemo(); MemoValidator.validate(cascadesContext.getMemo()); return this; @@ -145,14 +153,12 @@ public class PlanChecker { return this; } - public PlanChecker setRewritePlanFromMemo() { - this.cascadesContext.setRewritePlan(this.cascadesContext.getMemo().copyOut()); - return this; - } - public PlanChecker customRewrite(CustomRewriter customRewriter) { - new CustomRewriteJob(() -> customRewriter, RuleType.TEST_REWRITE).execute(cascadesContext.getCurrentJobContext()); + Rewriter.getWholeTreeRewriterWithCustomJobs(cascadesContext, + ImmutableList.of(Rewriter.custom(RuleType.TEST_REWRITE, () -> customRewriter))) + .execute(); cascadesContext.toMemo(); + MemoValidator.validate(cascadesContext.getMemo()); return this; } @@ -160,14 +166,11 @@ public class PlanChecker { return applyTopDown(ruleFactory.buildRules()); } - public PlanChecker applyTopDown(CustomRewriter customRewriter) { - cascadesContext.topDownRewrite(customRewriter); - MemoValidator.validate(cascadesContext.getMemo()); - return this; - } - public PlanChecker applyTopDown(List rule) { - cascadesContext.topDownRewrite(rule); + Rewriter.getWholeTreeRewriterWithCustomJobs(cascadesContext, + ImmutableList.of(new RootPlanTreeRewriteJob(rule, PlanTreeRewriteTopDownJob::new, true))) + .execute(); + cascadesContext.toMemo(); MemoValidator.validate(cascadesContext.getMemo()); return this; } @@ -178,7 +181,7 @@ public class PlanChecker { * @param patternMatcher the rule dsl, such as: logicalOlapScan().then(olapScan -> olapScan) * @return this checker, for call chaining of follow-up check */ - public PlanChecker applyTopDown(PatternMatcher patternMatcher) { + public PlanChecker applyTopDownInMemo(PatternMatcher patternMatcher) { cascadesContext.topDownRewrite(new OneRewriteRuleFactory() { @Override public Rule build() { @@ -190,7 +193,19 @@ public class PlanChecker { } public PlanChecker applyBottomUp(RuleFactory rule) { - cascadesContext.bottomUpRewrite(rule); + Rewriter.getWholeTreeRewriterWithCustomJobs(cascadesContext, + ImmutableList.of(Rewriter.bottomUp(rule))) + .execute(); + cascadesContext.toMemo(); + MemoValidator.validate(cascadesContext.getMemo()); + return this; + } + + public PlanChecker applyBottomUp(List rule) { + Rewriter.getWholeTreeRewriterWithCustomJobs(cascadesContext, + ImmutableList.of(new RootPlanTreeRewriteJob(rule, PlanTreeRewriteBottomUpJob::new, true))) + .execute(); + cascadesContext.toMemo(); MemoValidator.validate(cascadesContext.getMemo()); return this; } @@ -201,7 +216,7 @@ public class PlanChecker { * @param patternMatcher the rule dsl, such as: logicalOlapScan().then(olapScan -> olapScan) * @return this checker, for call chaining of follow-up check */ - public PlanChecker applyBottomUp(PatternMatcher patternMatcher) { + public PlanChecker applyBottomUpInMemo(PatternMatcher patternMatcher) { cascadesContext.bottomUpRewrite(new OneRewriteRuleFactory() { @Override public Rule build() { @@ -213,7 +228,7 @@ public class PlanChecker { } public PlanChecker rewrite() { - new Rewriter(cascadesContext).execute(); + Rewriter.getWholeTreeRewriter(cascadesContext).execute(); cascadesContext.toMemo(); return this; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanConstructor.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanConstructor.java index 51948be52b..9bb4883775 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanConstructor.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanConstructor.java @@ -25,7 +25,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.Type; import org.apache.doris.common.IdGenerator; -import org.apache.doris.nereids.trees.plans.ObjectId; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.thrift.TStorageType; @@ -39,7 +39,7 @@ public class PlanConstructor { public static final OlapTable score; public static final OlapTable course; - private static final IdGenerator RELATION_ID_GENERATOR = ObjectId.createGenerator(); + private static final IdGenerator RELATION_ID_GENERATOR = RelationId.createGenerator(); static { student = new OlapTable(0L, "student", @@ -108,12 +108,14 @@ public class PlanConstructor { ImmutableList.of("db")); } - public static LogicalOlapScan newLogicalOlapScanWithSameId(long tableId, String tableName, int hashColumn) { - return new LogicalOlapScan(ObjectId.createGenerator().getNextId(), - newOlapTable(tableId, tableName, hashColumn), ImmutableList.of("db")); + public static LogicalOlapScan newLogicalOlapScanWithSameId(long tableId, String tableName, + int hashColumn, List selectedPartitions) { + return new LogicalOlapScan(RelationId.createGenerator().getNextId(), + newOlapTable(tableId, tableName, hashColumn), ImmutableList.of("db"), + selectedPartitions, ImmutableList.of()); } - public static ObjectId getNextRelationId() { + public static RelationId getNextRelationId() { return RELATION_ID_GENERATOR.getNextId(); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanRewriter.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanRewriter.java index 67f6bc43ea..50b0789be9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanRewriter.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanRewriter.java @@ -17,59 +17,29 @@ package org.apache.doris.nereids.util; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.jobs.executor.Rewriter; import org.apache.doris.nereids.memo.Memo; -import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.RuleFactory; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.OriginStatement; +import com.google.common.collect.ImmutableList; + /** * Utility to copy plan into {@link Memo} and apply rewrite rules. */ public class PlanRewriter { public static Plan bottomUpRewrite(Plan plan, ConnectContext connectContext, RuleFactory... rules) { - return bottomUpRewriteMemo(plan, connectContext, rules).copyOut(); - } - - public static Plan bottomUpRewrite(Plan plan, ConnectContext connectContext, Rule... rules) { - return bottomUpRewriteMemo(plan, connectContext, rules).copyOut(); - } - - public static Memo bottomUpRewriteMemo(Plan plan, ConnectContext connectContext, RuleFactory... rules) { - return new Memo(plan) - .newCascadesContext(new StatementContext(connectContext, new OriginStatement("", 0))) - .bottomUpRewrite(rules) - .getMemo(); - } - - public static Memo bottomUpRewriteMemo(Plan plan, ConnectContext connectContext, Rule... rules) { - return new Memo(plan) - .newCascadesContext(new StatementContext(connectContext, new OriginStatement("", 0))) - .bottomUpRewrite(rules) - .getMemo(); - } - - public static Plan topDownRewrite(Plan plan, ConnectContext connectContext, RuleFactory... rules) { - return topDownRewriteMemo(plan, connectContext, rules).copyOut(); - } - - public static Plan topDownRewrite(Plan plan, ConnectContext connectContext, Rule... rules) { - return topDownRewriteMemo(plan, connectContext, rules).copyOut(); - } - - public static Memo topDownRewriteMemo(Plan plan, ConnectContext connectContext, RuleFactory... rules) { - return new Memo(plan) - .newCascadesContext(new StatementContext(connectContext, new OriginStatement("", 0))) - .topDownRewrite(rules) - .getMemo(); - } - - public static Memo topDownRewriteMemo(Plan plan, ConnectContext connectContext, Rule... rules) { - return new Memo(plan) - .newCascadesContext(new StatementContext(connectContext, new OriginStatement("", 0))) - .topDownRewrite(rules) - .getMemo(); + CascadesContext cascadesContext = CascadesContext.initContext( + new StatementContext(connectContext, new OriginStatement("", 0)), + plan, + PhysicalProperties.GATHER); + Rewriter.getWholeTreeRewriterWithCustomJobs(cascadesContext, + ImmutableList.of(Rewriter.bottomUp(rules))).execute(); + return cascadesContext.getRewritePlan(); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index f2610738f3..91b87c00fe 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -45,6 +45,7 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.DiskInfo; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.InternalSchemaInitializer; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Table; @@ -58,6 +59,7 @@ import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.MemoTestUtils; @@ -82,6 +84,7 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -101,6 +104,7 @@ import java.util.Comparator; import java.util.ConcurrentModificationException; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; /** @@ -130,6 +134,7 @@ public abstract class TestWithFeService { connectContext = createDefaultCtx(); beforeCluster(); createDorisCluster(); + new InternalSchemaInitializer().start(); runBeforeAll(); } @@ -191,8 +196,13 @@ public abstract class TestWithFeService { } public LogicalPlan analyze(String sql) { + Set originDisableRules = connectContext.getSessionVariable().getDisableNereidsRules(); + Set disableRuleWithAuth = Sets.newHashSet(originDisableRules); + disableRuleWithAuth.add(RuleType.RELATION_AUTHENTICATION.name()); + connectContext.getSessionVariable().setDisableNereidsRules(String.join(",", disableRuleWithAuth)); CascadesContext cascadesContext = createCascadesContext(sql); cascadesContext.newAnalyzer().analyze(); + connectContext.getSessionVariable().setDisableNereidsRules(String.join(",", originDisableRules)); cascadesContext.toMemo(); return (LogicalPlan) cascadesContext.getRewritePlan(); } diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query1.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query1.out index 838ba573be..0a15a57002 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query1.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query1.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_1 -- -CteAnchor[cteId= ( CTEId#2=] ) ---CteProducer[cteId= ( CTEId#2=] ) +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) ----hashAgg[GLOBAL] ------PhysicalDistribute --------hashAgg[LOCAL] @@ -26,7 +26,7 @@ CteAnchor[cteId= ( CTEId#2=] ) ----------------hashJoin[LEFT_SEMI_JOIN](ctr1.ctr_store_sk = ctr2.ctr_store_sk)(cast(ctr_total_return as DOUBLE) > cast((avg(ctr_total_return) * 1.2) as DOUBLE)) ------------------hashJoin[INNER_JOIN](store.s_store_sk = ctr1.ctr_store_sk) --------------------PhysicalDistribute -----------------------CteConsumer[cteId= ( CTEId#2=] ) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------PhysicalDistribute ----------------------PhysicalProject ------------------------filter((cast(s_state as VARCHAR(*)) = 'SD')) @@ -37,5 +37,5 @@ CteAnchor[cteId= ( CTEId#2=] ) ------------------------hashAgg[LOCAL] --------------------------PhysicalDistribute ----------------------------PhysicalProject -------------------------------CteConsumer[cteId= ( CTEId#2=] ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query11.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query11.out index 5e4621c1bd..b97d80540e 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query11.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query11.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_11 -- -CteAnchor[cteId= ( CTEId#4=] ) ---CteProducer[cteId= ( CTEId#4=] ) +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalUnion ------PhysicalProject --------hashAgg[GLOBAL] @@ -50,17 +50,17 @@ CteAnchor[cteId= ( CTEId#4=] ) ----------------PhysicalDistribute ------------------PhysicalProject --------------------filter((t_s_firstyear.dyear = 2001)(t_s_firstyear.sale_type = 's')(t_s_firstyear.year_total > 0.00)) -----------------------CteConsumer[cteId= ( CTEId#4=] ) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------PhysicalDistribute ------------------PhysicalProject --------------------filter((t_w_firstyear.year_total > 0.00)(t_w_firstyear.sale_type = 'w')(t_w_firstyear.dyear = 2001)) -----------------------CteConsumer[cteId= ( CTEId#4=] ) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalDistribute ----------------PhysicalProject ------------------filter((t_s_secyear.sale_type = 's')(t_s_secyear.dyear = 2002)) ---------------------CteConsumer[cteId= ( CTEId#4=] ) +--------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalDistribute --------------PhysicalProject ----------------filter((t_w_secyear.dyear = 2002)(t_w_secyear.sale_type = 'w')) -------------------CteConsumer[cteId= ( CTEId#4=] ) +------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query14.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query14.out index ecf6a69c87..96dfb9508d 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query14.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query14.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_14 -- -CteAnchor[cteId= ( CTEId#8=] ) ---CteProducer[cteId= ( CTEId#8=] ) +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject ------hashJoin[INNER_JOIN](item.i_brand_id = t.brand_id)(item.i_class_id = t.class_id)(item.i_category_id = t.category_id) --------PhysicalProject @@ -53,8 +53,8 @@ CteAnchor[cteId= ( CTEId#8=] ) ----------------------PhysicalDistribute ------------------------PhysicalProject --------------------------PhysicalOlapScan[item] ---CteAnchor[cteId= ( CTEId#10=] ) -----CteProducer[cteId= ( CTEId#10=] ) +--PhysicalCteAnchor ( cteId=CTEId#1 ) +----PhysicalCteProducer ( cteId=CTEId#1 ) ------hashAgg[GLOBAL] --------PhysicalDistribute ----------hashAgg[LOCAL] @@ -104,7 +104,7 @@ CteAnchor[cteId= ( CTEId#8=] ) ----------------------------------PhysicalProject ------------------------------------hashJoin[RIGHT_SEMI_JOIN](store_sales.ss_item_sk = cross_items.ss_item_sk) --------------------------------------PhysicalDistribute -----------------------------------------CteConsumer[cteId= ( CTEId#8=] ) +----------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------------------------hashJoin[INNER_JOIN](store_sales.ss_item_sk = item.i_item_sk) ----------------------------------------PhysicalDistribute ------------------------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk) @@ -120,7 +120,7 @@ CteAnchor[cteId= ( CTEId#8=] ) --------------------------PhysicalDistribute ----------------------------PhysicalAssertNumRows ------------------------------PhysicalDistribute ---------------------------------CteConsumer[cteId= ( CTEId#10=] ) +--------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) ----------------------PhysicalProject ------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DOUBLE) > cast(average_sales as DOUBLE)) --------------------------PhysicalProject @@ -130,7 +130,7 @@ CteAnchor[cteId= ( CTEId#8=] ) ----------------------------------PhysicalProject ------------------------------------hashJoin[RIGHT_SEMI_JOIN](catalog_sales.cs_item_sk = cross_items.ss_item_sk) --------------------------------------PhysicalDistribute -----------------------------------------CteConsumer[cteId= ( CTEId#8=] ) +----------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_item_sk = item.i_item_sk) ----------------------------------------PhysicalDistribute ------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = date_dim.d_date_sk) @@ -146,7 +146,7 @@ CteAnchor[cteId= ( CTEId#8=] ) --------------------------PhysicalDistribute ----------------------------PhysicalAssertNumRows ------------------------------PhysicalDistribute ---------------------------------CteConsumer[cteId= ( CTEId#10=] ) +--------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) ----------------------PhysicalProject ------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DOUBLE) > cast(average_sales as DOUBLE)) --------------------------PhysicalProject @@ -156,7 +156,7 @@ CteAnchor[cteId= ( CTEId#8=] ) ----------------------------------PhysicalProject ------------------------------------hashJoin[RIGHT_SEMI_JOIN](web_sales.ws_item_sk = cross_items.ss_item_sk) --------------------------------------PhysicalDistribute -----------------------------------------CteConsumer[cteId= ( CTEId#8=] ) +----------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------------------------hashJoin[INNER_JOIN](web_sales.ws_item_sk = item.i_item_sk) ----------------------------------------PhysicalDistribute ------------------------------------------hashJoin[INNER_JOIN](web_sales.ws_sold_date_sk = date_dim.d_date_sk) @@ -172,5 +172,5 @@ CteAnchor[cteId= ( CTEId#8=] ) --------------------------PhysicalDistribute ----------------------------PhysicalAssertNumRows ------------------------------PhysicalDistribute ---------------------------------CteConsumer[cteId= ( CTEId#10=] ) +--------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query2.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query2.out index 13e0f2f0e9..4d4ce2d58c 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query2.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query2.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_2 -- -CteAnchor[cteId= ( CTEId#4=] ) ---CteProducer[cteId= ( CTEId#4=] ) +PhysicalCteAnchor ( cteId=CTEId#1 ) +--PhysicalCteProducer ( cteId=CTEId#1 ) ----hashAgg[GLOBAL] ------PhysicalDistribute --------hashAgg[LOCAL] @@ -27,7 +27,7 @@ CteAnchor[cteId= ( CTEId#4=] ) ----------------hashJoin[INNER_JOIN](date_dim.d_week_seq = d_week_seq2) ------------------PhysicalDistribute --------------------PhysicalProject -----------------------CteConsumer[cteId= ( CTEId#4=] ) +----------------------PhysicalCteConsumer ( cteId=CTEId#1 ) ------------------PhysicalDistribute --------------------PhysicalProject ----------------------filter((date_dim.d_year = 1999)) @@ -37,7 +37,7 @@ CteAnchor[cteId= ( CTEId#4=] ) ----------------hashJoin[INNER_JOIN](date_dim.d_week_seq = d_week_seq1) ------------------PhysicalDistribute --------------------PhysicalProject -----------------------CteConsumer[cteId= ( CTEId#4=] ) +----------------------PhysicalCteConsumer ( cteId=CTEId#1 ) ------------------PhysicalDistribute --------------------PhysicalProject ----------------------filter((date_dim.d_year = 1998)) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query23.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query23.out index 9f4a6bcf94..1deef474c9 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query23.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query23.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_23 -- -CteAnchor[cteId= ( CTEId#1=] ) ---CteProducer[cteId= ( CTEId#1=] ) +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject ------filter((cnt > 4)) --------hashAgg[GLOBAL] @@ -19,8 +19,8 @@ CteAnchor[cteId= ( CTEId#1=] ) ------------------PhysicalDistribute --------------------PhysicalProject ----------------------PhysicalOlapScan[item] ---CteAnchor[cteId= ( CTEId#4=] ) -----CteProducer[cteId= ( CTEId#4=] ) +--PhysicalCteAnchor ( cteId=CTEId#2 ) +----PhysicalCteProducer ( cteId=CTEId#2 ) ------PhysicalProject --------NestedLoopJoin[INNER_JOIN](cast(ssales as DOUBLE) > cast(((cast(95 as DECIMALV3(8, 5)) / 100.0) * tpcds_cmax) as DOUBLE)) ----------hashAgg[GLOBAL] @@ -65,7 +65,7 @@ CteAnchor[cteId= ( CTEId#1=] ) ------------------hashJoin[RIGHT_SEMI_JOIN](catalog_sales.cs_item_sk = frequent_ss_items.item_sk) --------------------PhysicalDistribute ----------------------PhysicalProject -------------------------CteConsumer[cteId= ( CTEId#1=] ) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------PhysicalDistribute ----------------------PhysicalProject ------------------------hashJoin[LEFT_SEMI_JOIN](catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk) @@ -79,12 +79,12 @@ CteAnchor[cteId= ( CTEId#1=] ) ------------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalDistribute ----------------------------PhysicalProject -------------------------------CteConsumer[cteId= ( CTEId#4=] ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) ----------------PhysicalProject ------------------hashJoin[RIGHT_SEMI_JOIN](web_sales.ws_item_sk = frequent_ss_items.item_sk) --------------------PhysicalDistribute ----------------------PhysicalProject -------------------------CteConsumer[cteId= ( CTEId#1=] ) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------PhysicalDistribute ----------------------PhysicalProject ------------------------hashJoin[LEFT_SEMI_JOIN](web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk) @@ -98,5 +98,5 @@ CteAnchor[cteId= ( CTEId#1=] ) ------------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalDistribute ----------------------------PhysicalProject -------------------------------CteConsumer[cteId= ( CTEId#4=] ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query24.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query24.out index 56bad7457e..eebe664727 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query24.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query24.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_24 -- -CteAnchor[cteId= ( CTEId#0=] ) ---CteProducer[cteId= ( CTEId#0=] ) +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject ------hashAgg[GLOBAL] --------PhysicalDistribute @@ -42,7 +42,7 @@ CteAnchor[cteId= ( CTEId#0=] ) ------------------PhysicalDistribute --------------------PhysicalProject ----------------------filter((cast(i_color as VARCHAR(*)) = 'beige')) -------------------------CteConsumer[cteId= ( CTEId#0=] ) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalDistribute --------------PhysicalAssertNumRows ----------------PhysicalProject @@ -51,5 +51,5 @@ CteAnchor[cteId= ( CTEId#0=] ) ----------------------hashAgg[LOCAL] ------------------------PhysicalDistribute --------------------------PhysicalProject -----------------------------CteConsumer[cteId= ( CTEId#0=] ) +----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query30.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query30.out index 7338d341dc..7741e0c45c 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query30.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query30.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_30 -- -CteAnchor[cteId= ( CTEId#2=] ) ---CteProducer[cteId= ( CTEId#2=] ) +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) ----hashAgg[GLOBAL] ------PhysicalDistribute --------hashAgg[LOCAL] @@ -26,7 +26,7 @@ CteAnchor[cteId= ( CTEId#2=] ) ----------hashJoin[LEFT_SEMI_JOIN](ctr1.ctr_state = ctr2.ctr_state)(cast(ctr_total_return as DOUBLE) > cast((avg(ctr_total_return) * 1.2) as DOUBLE)) ------------hashJoin[INNER_JOIN](ctr1.ctr_customer_sk = customer.c_customer_sk) --------------PhysicalDistribute -----------------CteConsumer[cteId= ( CTEId#2=] ) +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalDistribute ----------------hashJoin[INNER_JOIN](customer_address.ca_address_sk = customer.c_current_addr_sk) ------------------PhysicalProject @@ -42,5 +42,5 @@ CteAnchor[cteId= ( CTEId#2=] ) --------------------hashAgg[LOCAL] ----------------------PhysicalDistribute ------------------------PhysicalProject ---------------------------CteConsumer[cteId= ( CTEId#2=] ) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query31.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query31.out index 12a7b7db31..dd60ee2d41 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query31.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query31.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_31 -- -CteAnchor[cteId= ( CTEId#6=] ) ---CteProducer[cteId= ( CTEId#6=] ) +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject ------hashAgg[GLOBAL] --------PhysicalDistribute @@ -19,8 +19,8 @@ CteAnchor[cteId= ( CTEId#6=] ) ----------------PhysicalDistribute ------------------PhysicalProject --------------------PhysicalOlapScan[customer_address] ---CteAnchor[cteId= ( CTEId#7=] ) -----CteProducer[cteId= ( CTEId#7=] ) +--PhysicalCteAnchor ( cteId=CTEId#1 ) +----PhysicalCteProducer ( cteId=CTEId#1 ) ------PhysicalProject --------hashAgg[GLOBAL] ----------PhysicalDistribute @@ -47,30 +47,30 @@ CteAnchor[cteId= ( CTEId#6=] ) --------------PhysicalDistribute ----------------PhysicalProject ------------------filter((ws3.d_year = 2000)(ws3.d_qoy = 3)) ---------------------CteConsumer[cteId= ( CTEId#7=] ) +--------------------PhysicalCteConsumer ( cteId=CTEId#1 ) --------------PhysicalProject ----------------hashJoin[INNER_JOIN](ss2.ca_county = ss3.ca_county) ------------------PhysicalDistribute --------------------PhysicalProject ----------------------filter((ss3.d_year = 2000)(ss3.d_qoy = 3)) -------------------------CteConsumer[cteId= ( CTEId#6=] ) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN](ws1.ca_county = ws2.ca_county)(CASE WHEN (web_sales > 0.00) THEN (cast(web_sales as DECIMALV3(38, 8)) / web_sales) ELSE NULL END > CASE WHEN (store_sales > 0.00) THEN (cast(store_sales as DECIMALV3(38, 8)) / store_sales) ELSE NULL END) --------------------hashJoin[INNER_JOIN](ss1.ca_county = ws1.ca_county) ----------------------hashJoin[INNER_JOIN](ss1.ca_county = ss2.ca_county) ------------------------PhysicalDistribute --------------------------PhysicalProject ----------------------------filter((ss1.d_year = 2000)(ss1.d_qoy = 1)) -------------------------------CteConsumer[cteId= ( CTEId#6=] ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------------PhysicalDistribute --------------------------PhysicalProject ----------------------------filter((ss2.d_year = 2000)(ss2.d_qoy = 2)) -------------------------------CteConsumer[cteId= ( CTEId#6=] ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------PhysicalDistribute ------------------------PhysicalProject --------------------------filter((ws1.d_year = 2000)(ws1.d_qoy = 1)) -----------------------------CteConsumer[cteId= ( CTEId#7=] ) +----------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) --------------------PhysicalDistribute ----------------------PhysicalProject ------------------------filter((ws2.d_qoy = 2)(ws2.d_year = 2000)) ---------------------------CteConsumer[cteId= ( CTEId#7=] ) +--------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out index 48d9738a11..977a1e5769 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_39 -- -CteAnchor[cteId= ( CTEId#3=] ) ---CteProducer[cteId= ( CTEId#3=] ) +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject ------filter((CASE WHEN (mean = 0.0) THEN 0.0 ELSE (stdev / mean) END > 1.0)) --------hashAgg[GLOBAL] @@ -30,9 +30,9 @@ CteAnchor[cteId= ( CTEId#3=] ) ------------PhysicalDistribute --------------PhysicalProject ----------------filter((inv1.d_moy = 1)) -------------------CteConsumer[cteId= ( CTEId#3=] ) +------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalDistribute --------------PhysicalProject ----------------filter((inv2.d_moy = 2)) -------------------CteConsumer[cteId= ( CTEId#3=] ) +------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query4.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query4.out index 897f72cdd6..1ea2eae2bc 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query4.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query4.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_4 -- -CteAnchor[cteId= ( CTEId#6=] ) ---CteProducer[cteId= ( CTEId#6=] ) +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalUnion ------PhysicalProject --------hashAgg[GLOBAL] @@ -75,25 +75,25 @@ CteAnchor[cteId= ( CTEId#6=] ) --------------------------PhysicalDistribute ----------------------------PhysicalProject ------------------------------filter((t_s_firstyear.year_total > 0.000000)(t_s_firstyear.dyear = 1999)(t_s_firstyear.sale_type = 's')) ---------------------------------CteConsumer[cteId= ( CTEId#6=] ) +--------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------------PhysicalDistribute ----------------------------PhysicalProject ------------------------------filter((t_c_firstyear.year_total > 0.000000)(t_c_firstyear.dyear = 1999)(t_c_firstyear.sale_type = 'c')) ---------------------------------CteConsumer[cteId= ( CTEId#6=] ) +--------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------------PhysicalDistribute --------------------------PhysicalProject ----------------------------filter((t_s_secyear.sale_type = 's')(t_s_secyear.dyear = 2000)) -------------------------------CteConsumer[cteId= ( CTEId#6=] ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------PhysicalDistribute ----------------------PhysicalProject ------------------------filter((t_c_secyear.sale_type = 'c')(t_c_secyear.dyear = 2000)) ---------------------------CteConsumer[cteId= ( CTEId#6=] ) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------PhysicalDistribute ------------------PhysicalProject --------------------filter((t_w_firstyear.dyear = 1999)(t_w_firstyear.sale_type = 'w')(t_w_firstyear.year_total > 0.000000)) -----------------------CteConsumer[cteId= ( CTEId#6=] ) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalDistribute --------------PhysicalProject ----------------filter((t_w_secyear.sale_type = 'w')(t_w_secyear.dyear = 2000)) -------------------CteConsumer[cteId= ( CTEId#6=] ) +------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query47.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query47.out index 49fc532871..ed8bcb300f 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query47.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query47.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_47 -- -CteAnchor[cteId= ( CTEId#0=] ) ---CteProducer[cteId= ( CTEId#0=] ) +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject ------PhysicalWindow --------PhysicalQuickSort @@ -37,15 +37,15 @@ CteAnchor[cteId= ( CTEId#0=] ) ------------hashJoin[INNER_JOIN](s_store_name = v1_lead.s_store_name)(v1.i_category = v1_lead.i_category)(v1.i_brand = v1_lead.i_brand)(v1.s_company_name = v1_lead.s_company_name)(v1.rn = expr_(rn - 1)) --------------PhysicalDistribute ----------------PhysicalProject -------------------CteConsumer[cteId= ( CTEId#0=] ) +------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalDistribute ----------------PhysicalProject ------------------hashJoin[INNER_JOIN](s_store_name = v1_lag.s_store_name)(v1.i_category = v1_lag.i_category)(v1.i_brand = v1_lag.i_brand)(v1.s_company_name = v1_lag.s_company_name)(v1.rn = expr_(rn + 1)) --------------------PhysicalDistribute ----------------------PhysicalProject -------------------------CteConsumer[cteId= ( CTEId#0=] ) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------PhysicalDistribute ----------------------PhysicalProject ------------------------filter((CASE WHEN (avg_monthly_sales > 0.0000) THEN (abs((cast(sum_sales as DOUBLE) - cast(avg_monthly_sales as DOUBLE))) / cast(avg_monthly_sales as DOUBLE)) ELSE NULL END > 0.1)(v2.d_year = 2001)(v2.avg_monthly_sales > 0.0000)) ---------------------------CteConsumer[cteId= ( CTEId#0=] ) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query54.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query54.out index 72a23fe3e8..e6002e5a2d 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query54.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query54.out @@ -53,7 +53,7 @@ PhysicalTopN --------------------------------------------------------------------------PhysicalOlapScan[item] ----------------------------------------------------------------PhysicalDistribute ------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------filter((date_dim.d_moy = 5)(date_dim.d_year = 1998)) +--------------------------------------------------------------------filter((date_dim.d_year = 1998)(date_dim.d_moy = 5)) ----------------------------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalDistribute --------------------------------------PhysicalProject @@ -65,7 +65,7 @@ PhysicalTopN ------------------------------------------PhysicalDistribute --------------------------------------------hashAgg[LOCAL] ----------------------------------------------PhysicalProject -------------------------------------------------filter((date_dim.d_moy = 5)(date_dim.d_year = 1998)) +------------------------------------------------filter((date_dim.d_year = 1998)(date_dim.d_moy = 5)) --------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalDistribute --------------------------------PhysicalAssertNumRows diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out index 1b3a0610a4..b54f9d5abc 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_57 -- -CteAnchor[cteId= ( CTEId#0=] ) ---CteProducer[cteId= ( CTEId#0=] ) +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject ------PhysicalWindow --------PhysicalQuickSort @@ -37,15 +37,15 @@ CteAnchor[cteId= ( CTEId#0=] ) ------------hashJoin[INNER_JOIN](i_brand = v1_lead.i_brand)(v1.i_category = v1_lead.i_category)(v1.cc_name = v1_lead.cc_name)(v1.rn = expr_(rn - 1)) --------------PhysicalDistribute ----------------PhysicalProject -------------------CteConsumer[cteId= ( CTEId#0=] ) +------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalDistribute ----------------PhysicalProject ------------------hashJoin[INNER_JOIN](i_brand = v1_lag.i_brand)(v1.i_category = v1_lag.i_category)(v1.cc_name = v1_lag.cc_name)(v1.rn = expr_(rn + 1)) --------------------PhysicalDistribute ----------------------PhysicalProject -------------------------CteConsumer[cteId= ( CTEId#0=] ) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------PhysicalDistribute ----------------------PhysicalProject ------------------------filter((CASE WHEN (avg_monthly_sales > 0.0000) THEN (abs((cast(sum_sales as DOUBLE) - cast(avg_monthly_sales as DOUBLE))) / cast(avg_monthly_sales as DOUBLE)) ELSE NULL END > 0.1)(v2.d_year = 1999)(v2.avg_monthly_sales > 0.0000)) ---------------------------CteConsumer[cteId= ( CTEId#0=] ) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query59.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query59.out index 58e7e2211c..363cf914e0 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query59.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query59.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_59 -- -CteAnchor[cteId= ( CTEId#4=] ) ---CteProducer[cteId= ( CTEId#4=] ) +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject ------hashAgg[GLOBAL] --------PhysicalDistribute @@ -24,7 +24,7 @@ CteAnchor[cteId= ( CTEId#4=] ) ------------------hashJoin[INNER_JOIN](d.d_week_seq = d_week_seq2) --------------------PhysicalDistribute ----------------------PhysicalProject -------------------------CteConsumer[cteId= ( CTEId#4=] ) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------PhysicalDistribute ----------------------PhysicalProject ------------------------filter((d.d_month_seq <= 1219)(d.d_month_seq >= 1208)) @@ -38,7 +38,7 @@ CteAnchor[cteId= ( CTEId#4=] ) ------------------hashJoin[INNER_JOIN](d.d_week_seq = d_week_seq1) --------------------PhysicalDistribute ----------------------PhysicalProject -------------------------CteConsumer[cteId= ( CTEId#4=] ) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------PhysicalDistribute ----------------------PhysicalProject ------------------------filter((d.d_month_seq <= 1207)(d.d_month_seq >= 1196)) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query74.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query74.out index 661f999867..dc32f6e970 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query74.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query74.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_74 -- -CteAnchor[cteId= ( CTEId#4=] ) ---CteProducer[cteId= ( CTEId#4=] ) +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalUnion ------PhysicalProject --------hashAgg[GLOBAL] @@ -50,17 +50,17 @@ CteAnchor[cteId= ( CTEId#4=] ) ----------------PhysicalDistribute ------------------PhysicalProject --------------------filter((t_s_firstyear.year = 1999)(t_s_firstyear.sale_type = 's')(t_s_firstyear.year_total > 0.0)) -----------------------CteConsumer[cteId= ( CTEId#4=] ) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------PhysicalDistribute ------------------PhysicalProject --------------------filter((t_s_secyear.sale_type = 's')(t_s_secyear.year = 2000)) -----------------------CteConsumer[cteId= ( CTEId#4=] ) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalDistribute ----------------PhysicalProject ------------------filter((t_w_secyear.year = 2000)(t_w_secyear.sale_type = 'w')) ---------------------CteConsumer[cteId= ( CTEId#4=] ) +--------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalDistribute --------------PhysicalProject ----------------filter((t_w_firstyear.year = 1999)(t_w_firstyear.year_total > 0.0)(t_w_firstyear.sale_type = 'w')) -------------------CteConsumer[cteId= ( CTEId#4=] ) +------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query75.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query75.out index 0ad25cc5ec..09a7279785 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query75.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query75.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_75 -- -CteAnchor[cteId= ( CTEId#3=] ) ---CteProducer[cteId= ( CTEId#3=] ) +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) ----hashAgg[GLOBAL] ------PhysicalDistribute --------hashAgg[LOCAL] @@ -68,8 +68,8 @@ CteAnchor[cteId= ( CTEId#3=] ) ----------hashJoin[INNER_JOIN](curr_yr.i_brand_id = prev_yr.i_brand_id)(curr_yr.i_class_id = prev_yr.i_class_id)(curr_yr.i_category_id = prev_yr.i_category_id)(curr_yr.i_manufact_id = prev_yr.i_manufact_id)((cast(cast(sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(sales_cnt as DECIMALV3(17, 2))) < 0.900000) ------------PhysicalDistribute --------------filter((curr_yr.d_year = 1999)) -----------------CteConsumer[cteId= ( CTEId#3=] ) +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalDistribute --------------filter((prev_yr.d_year = 1998)) -----------------CteConsumer[cteId= ( CTEId#3=] ) +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query81.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query81.out index bb72d81784..11524e0f10 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query81.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query81.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_81 -- -CteAnchor[cteId= ( CTEId#2=] ) ---CteProducer[cteId= ( CTEId#2=] ) +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) ----hashAgg[GLOBAL] ------PhysicalDistribute --------hashAgg[LOCAL] @@ -26,7 +26,7 @@ CteAnchor[cteId= ( CTEId#2=] ) ----------hashJoin[LEFT_SEMI_JOIN](ctr1.ctr_state = ctr2.ctr_state)(cast(ctr_total_return as DOUBLE) > cast((avg(ctr_total_return) * 1.2) as DOUBLE)) ------------hashJoin[INNER_JOIN](ctr1.ctr_customer_sk = customer.c_customer_sk) --------------PhysicalDistribute -----------------CteConsumer[cteId= ( CTEId#2=] ) +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalDistribute ----------------hashJoin[INNER_JOIN](customer_address.ca_address_sk = customer.c_current_addr_sk) ------------------PhysicalProject @@ -42,5 +42,5 @@ CteAnchor[cteId= ( CTEId#2=] ) --------------------hashAgg[LOCAL] ----------------------PhysicalDistribute ------------------------PhysicalProject ---------------------------CteConsumer[cteId= ( CTEId#2=] ) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out index 014535c50d..410beed371 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_95 -- -CteAnchor[cteId= ( CTEId#3=] ) ---CteProducer[cteId= ( CTEId#3=] ) +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject ------hashJoin[INNER_JOIN](ws1.ws_order_number = ws2.ws_order_number)( not (ws_warehouse_sk = ws_warehouse_sk)) --------PhysicalDistribute @@ -27,7 +27,7 @@ CteAnchor[cteId= ( CTEId#3=] ) ------------------------hashJoin[INNER_JOIN](web_returns.wr_order_number = ws_wh.ws_order_number) --------------------------PhysicalDistribute ----------------------------PhysicalProject -------------------------------CteConsumer[cteId= ( CTEId#3=] ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------------PhysicalDistribute ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[web_returns] @@ -35,7 +35,7 @@ CteAnchor[cteId= ( CTEId#3=] ) ------------------------hashJoin[RIGHT_SEMI_JOIN](ws1.ws_order_number = ws_wh.ws_order_number) --------------------------PhysicalDistribute ----------------------------PhysicalProject -------------------------------CteConsumer[cteId= ( CTEId#3=] ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------------PhysicalDistribute ----------------------------hashJoin[INNER_JOIN](ws1.ws_web_site_sk = web_site.web_site_sk) ------------------------------hashJoin[INNER_JOIN](ws1.ws_ship_addr_sk = customer_address.ca_address_sk)