From 9db56201a6b4703dfce3db39178f8edbcef0f138 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Tue, 28 Feb 2023 16:02:09 +0800 Subject: [PATCH] [refactor](Nereids) Refactor rewrite framework to speed up plan (#17126) This pr refactor the rewrite framework from memo to plan tree, and speed up the analyze/rewrite stage. Changes: - abandoned memo in the analysis/rewrite stage, so that we can skip some actions, like new GroupExpression, distinct GroupExpression in the memo(high cost), update children to GroupPlan - change the most of rules to static rule, so that we can skip initialize lots of rules in Analyzer/Rewriter at every query. but some rules need context, like visitor rule, create rule at the runtime make it is easy to use, so make `custom` rule can help us to create it. - remove the `logger` field in the Job, Job are generated in large quantities at runtime, we don't need to use logger so save huge time to initialize logger. - skip some rule as far as possible, e.g. `SelectMaterializedIndexWithoutAggregate`, skip select mv if the table not exist rullup. - add some caches for frequent operation, like get Job.getDisableRules, Plan.getUnboundExpression - new bottom up rewrite rule, it can keep traverse multiple new plan which return by rules, this feature depends on `Plan.mutableState`, it is necessary to add this variable field for plan. if the plan is fully immutable, we must use withXxx to renew the plan and set the state for it, this take more runtime overhead and developing workload. another reason is we need multiple mutable state, e.g. whether is applied the rule, whether this plan is manage by the rewrite framework. the good side of mutable state is efficient, but I suggest we don't direct use mutable state in the rule as far as possible, if we need use it, please wrap the mutable state in the framework to update and release it correctly. a good example is `AppliedAwareRuleCondition`, it can update and get the state: whether this plan is applied to a rule before. - merge some rules, invoke multiple rules in one traverse - refactor the `EliminateUnnecessaryProject` by CustomRewritor, fix the problem which eliminate some Project which decided the query output order, the case is limit(project), sort(project). TODO: add trace for new rewrite framework benchmark: legacy optimizer: ``` +-----------+---------------+---------------+---------------+ | SQL ID | avg | min | max | +-----------+---------------+---------------+---------------+ | SQL 1 | 1.39 ms | 0 ms | 9 ms | | SQL 2 | 1.38 ms | 0 ms | 10 ms | | SQL 3 | 2.05 ms | 1 ms | 18 ms | | SQL 4 | 0.89 ms | 0 ms | 9 ms | | SQL 5 | 1.74 ms | 1 ms | 11 ms | | SQL 6 | 2.00 ms | 1 ms | 13 ms | | SQL 7 | 1.83 ms | 1 ms | 15 ms | | SQL 8 | 0.92 ms | 0 ms | 7 ms | | SQL 9 | 2.60 ms | 1 ms | 19 ms | | SQL 10 | 3.54 ms | 2 ms | 28 ms | | SQL 11 | 3.04 ms | 1 ms | 18 ms | | SQL 12 | 3.26 ms | 2 ms | 16 ms | | SQL 13 | 1.10 ms | 0 ms | 10 ms | | SQL 14 | 2.90 ms | 1 ms | 13 ms | | SQL 15 | 1.18 ms | 0 ms | 9 ms | | SQL 16 | 1.05 ms | 0 ms | 13 ms | | SQL 17 | 1.03 ms | 0 ms | 7 ms | | SQL 18 | 0.94 ms | 0 ms | 7 ms | | SQL 19 | 1.47 ms | 0 ms | 13 ms | | SQL 20 | 0.47 ms | 0 ms | 4 ms | | SQL 21 | 0.54 ms | 0 ms | 5 ms | | SQL 22 | 3.34 ms | 1 ms | 19 ms | | SQL 23 | 7.97 ms | 4 ms | 44 ms | | SQL 24 | 11.11 ms | 7 ms | 28 ms | | SQL 25 | 0.98 ms | 0 ms | 8 ms | | SQL 26 | 0.83 ms | 0 ms | 7 ms | | SQL 27 | 0.93 ms | 0 ms | 16 ms | | SQL 28 | 2.19 ms | 1 ms | 18 ms | | SQL 29 | 3.23 ms | 1 ms | 20 ms | | SQL 30 | 59.99 ms | 51 ms | 81 ms | | SQL 31 | 2.65 ms | 1 ms | 18 ms | | SQL 32 | 2.47 ms | 1 ms | 17 ms | | SQL 33 | 2.30 ms | 1 ms | 16 ms | | SQL 34 | 0.66 ms | 0 ms | 8 ms | | SQL 35 | 0.63 ms | 0 ms | 6 ms | | SQL 36 | 2.25 ms | 1 ms | 15 ms | | SQL 37 | 5.97 ms | 3 ms | 20 ms | | SQL 38 | 5.73 ms | 3 ms | 21 ms | | SQL 39 | 6.32 ms | 4 ms | 23 ms | | SQL 40 | 8.61 ms | 5 ms | 35 ms | | SQL 41 | 6.29 ms | 4 ms | 28 ms | | SQL 42 | 6.04 ms | 4 ms | 15 ms | | SQL 43 | 5.81 ms | 3 ms | 16 ms | +-----------+---------------+---------------+---------------+ | TOTAL AVG | 4.22 ms | 2.47 ms | 17.05 ms | | TOTAL SUM | 181.62 ms | 106 ms | 733 ms | +-----------+---------------+---------------+---------------+ ``` nereids with memo rewrite framework(old): ``` +-----------+---------------+---------------+---------------+ | SQL ID | avg | min | max | +-----------+---------------+---------------+---------------+ | SQL 1 | 3.61 ms | 1 ms | 20 ms | | SQL 2 | 3.47 ms | 2 ms | 16 ms | | SQL 3 | 3.27 ms | 1 ms | 18 ms | | SQL 4 | 2.23 ms | 1 ms | 12 ms | | SQL 5 | 3.60 ms | 1 ms | 20 ms | | SQL 6 | 2.73 ms | 1 ms | 17 ms | | SQL 7 | 3.04 ms | 1 ms | 23 ms | | SQL 8 | 3.53 ms | 2 ms | 20 ms | | SQL 9 | 3.74 ms | 2 ms | 22 ms | | SQL 10 | 3.66 ms | 2 ms | 18 ms | | SQL 11 | 3.93 ms | 2 ms | 15 ms | | SQL 12 | 4.85 ms | 2 ms | 27 ms | | SQL 13 | 4.41 ms | 2 ms | 28 ms | | SQL 14 | 5.16 ms | 2 ms | 41 ms | | SQL 15 | 4.33 ms | 2 ms | 33 ms | | SQL 16 | 4.94 ms | 2 ms | 51 ms | | SQL 17 | 3.27 ms | 1 ms | 25 ms | | SQL 18 | 2.78 ms | 1 ms | 22 ms | | SQL 19 | 3.51 ms | 1 ms | 42 ms | | SQL 20 | 1.84 ms | 1 ms | 13 ms | | SQL 21 | 3.47 ms | 1 ms | 66 ms | | SQL 22 | 5.21 ms | 2 ms | 29 ms | | SQL 23 | 5.55 ms | 3 ms | 25 ms | | SQL 24 | 4.21 ms | 2 ms | 28 ms | | SQL 25 | 3.47 ms | 1 ms | 23 ms | | SQL 26 | 3.03 ms | 2 ms | 21 ms | | SQL 27 | 3.07 ms | 1 ms | 17 ms | | SQL 28 | 4.51 ms | 3 ms | 22 ms | | SQL 29 | 4.97 ms | 3 ms | 21 ms | | SQL 30 | 11.95 ms | 8 ms | 33 ms | | SQL 31 | 3.92 ms | 2 ms | 23 ms | | SQL 32 | 3.74 ms | 2 ms | 15 ms | | SQL 33 | 3.62 ms | 2 ms | 22 ms | | SQL 34 | 4.60 ms | 1 ms | 55 ms | | SQL 35 | 3.47 ms | 2 ms | 25 ms | | SQL 36 | 3.34 ms | 2 ms | 18 ms | | SQL 37 | 4.77 ms | 2 ms | 23 ms | | SQL 38 | 4.44 ms | 2 ms | 39 ms | | SQL 39 | 4.52 ms | 2 ms | 23 ms | | SQL 40 | 5.50 ms | 3 ms | 30 ms | | SQL 41 | 5.01 ms | 2 ms | 24 ms | | SQL 42 | 4.32 ms | 2 ms | 24 ms | | SQL 43 | 4.29 ms | 2 ms | 42 ms | +-----------+---------------+---------------+---------------+ | TOTAL AVG | 4.11 ms | 1.91 ms | 26.30 ms | | TOTAL SUM | 176.88 ms | 82 ms | 1131 ms | +-----------+---------------+---------------+---------------+ ``` nereids with plan tree rewrite framework(new): ``` +-----------+---------------+---------------+---------------+ | SQL ID | avg | min | max | +-----------+---------------+---------------+---------------+ | SQL 1 | 3.21 ms | 1 ms | 18 ms | | SQL 2 | 3.99 ms | 1 ms | 76 ms | | SQL 3 | 2.93 ms | 1 ms | 21 ms | | SQL 4 | 2.13 ms | 1 ms | 21 ms | | SQL 5 | 2.43 ms | 1 ms | 30 ms | | SQL 6 | 2.08 ms | 1 ms | 11 ms | | SQL 7 | 2.03 ms | 1 ms | 11 ms | | SQL 8 | 2.27 ms | 1 ms | 22 ms | | SQL 9 | 2.42 ms | 1 ms | 16 ms | | SQL 10 | 2.65 ms | 1 ms | 14 ms | | SQL 11 | 2.78 ms | 1 ms | 14 ms | | SQL 12 | 3.09 ms | 1 ms | 19 ms | | SQL 13 | 2.33 ms | 1 ms | 13 ms | | SQL 14 | 2.66 ms | 1 ms | 16 ms | | SQL 15 | 2.34 ms | 1 ms | 15 ms | | SQL 16 | 2.04 ms | 1 ms | 30 ms | | SQL 17 | 2.09 ms | 1 ms | 17 ms | | SQL 18 | 1.87 ms | 1 ms | 15 ms | | SQL 19 | 2.21 ms | 1 ms | 50 ms | | SQL 20 | 1.32 ms | 0 ms | 12 ms | | SQL 21 | 1.63 ms | 1 ms | 11 ms | | SQL 22 | 2.75 ms | 1 ms | 30 ms | | SQL 23 | 3.44 ms | 2 ms | 17 ms | | SQL 24 | 2.01 ms | 1 ms | 14 ms | | SQL 25 | 1.58 ms | 1 ms | 11 ms | | SQL 26 | 1.53 ms | 0 ms | 13 ms | | SQL 27 | 1.62 ms | 1 ms | 12 ms | | SQL 28 | 2.90 ms | 1 ms | 21 ms | | SQL 29 | 3.04 ms | 2 ms | 17 ms | | SQL 30 | 10.54 ms | 7 ms | 49 ms | | SQL 31 | 2.61 ms | 1 ms | 21 ms | | SQL 32 | 2.42 ms | 1 ms | 14 ms | | SQL 33 | 2.13 ms | 1 ms | 14 ms | | SQL 34 | 1.69 ms | 1 ms | 14 ms | | SQL 35 | 1.87 ms | 1 ms | 15 ms | | SQL 36 | 2.37 ms | 1 ms | 21 ms | | SQL 37 | 3.06 ms | 1 ms | 15 ms | | SQL 38 | 4.09 ms | 1 ms | 31 ms | | SQL 39 | 5.81 ms | 2 ms | 43 ms | | SQL 40 | 4.55 ms | 2 ms | 34 ms | | SQL 41 | 3.49 ms | 1 ms | 20 ms | | SQL 42 | 2.75 ms | 1 ms | 26 ms | | SQL 43 | 2.81 ms | 1 ms | 14 ms | +-----------+---------------+---------------+---------------+ | TOTAL AVG | 2.78 ms | 1.19 ms | 21.35 ms | | TOTAL SUM | 119.56 ms | 51 ms | 918 ms | +-----------+---------------+---------------+---------------+ ``` --- .../apache/doris/nereids/CascadesContext.java | 93 ++++- .../apache/doris/nereids/NereidsPlanner.java | 21 +- .../org/apache/doris/nereids/PlanSource.java | 22 ++ .../nereids/analyzer/NereidsAnalyzer.java | 79 +++-- .../apache/doris/nereids/analyzer/Scope.java | 9 +- .../org/apache/doris/nereids/jobs/Job.java | 42 ++- .../apache/doris/nereids/jobs/JobContext.java | 13 +- .../apache/doris/nereids/jobs/JobType.java | 3 +- .../apache/doris/nereids/jobs/RewriteJob.java | 25 ++ .../doris/nereids/jobs/TopicRewriteJob.java | 52 +++ .../nereids/jobs/batch/AnalyzeRulesJob.java | 82 ----- ...rtApplyToJoinJob.java => ApplyToJoin.java} | 27 +- .../nereids/jobs/batch/BatchRewriteJob.java | 116 +++++++ .../nereids/jobs/batch/BatchRulesJob.java | 109 ------ ...mptySetJob.java => CascadesOptimizer.java} | 24 +- ... => CorrelateApplyToUnCorrelateApply.java} | 41 +-- ...va => EliminateUselessPlanUnderApply.java} | 25 +- .../jobs/batch/NereidsRewriteJobExecutor.java | 150 -------- .../nereids/jobs/batch/NereidsRewriter.java | 214 ++++++++++++ .../jobs/rewrite/CustomRewriteJob.java | 78 +++++ .../rewrite/PlanTreeRewriteBottomUpJob.java | 128 +++++++ .../jobs/rewrite/PlanTreeRewriteJob.java | 117 +++++++ .../rewrite/PlanTreeRewriteTopDownJob.java | 66 ++++ .../jobs/rewrite/RewriteJobContext.java | 65 ++++ .../jobs/rewrite/RootPlanTreeRewriteJob.java | 171 ++++++++++ .../jobs/rewrite/VisitorRewriteJob.java | 69 ---- .../nereids/jobs/scheduler/JobScheduler.java | 8 +- .../ScheduleContext.java} | 20 +- .../jobs/scheduler/SimpleJobScheduler.java | 11 +- .../org/apache/doris/nereids/memo/Group.java | 13 +- .../org/apache/doris/nereids/memo/Memo.java | 4 +- .../nereids/pattern/MatchingContext.java | 4 + .../doris/nereids/pattern/MemoPatterns.java | 322 ++++++++++++++++++ .../apache/doris/nereids/pattern/Pattern.java | 39 ++- .../doris/nereids/pattern/Patterns.java | 298 ---------------- .../doris/nereids/pattern/PlanPatterns.java | 314 +++++++++++++++++ .../doris/nereids/pattern/ProxyPattern.java | 45 +++ .../LogicalBinaryPatternGenerator.java | 10 +- .../LogicalLeafPatternGenerator.java | 4 +- .../LogicalUnaryPatternGenerator.java | 10 +- .../PatternDescribableProcessor.java | 36 +- .../pattern/generator/PatternGenerator.java | 49 +-- .../generator/PatternGeneratorAnalyzer.java | 10 +- .../PhysicalBinaryPatternGenerator.java | 10 +- .../PhysicalLeafPatternGenerator.java | 4 +- .../PhysicalUnaryPatternGenerator.java | 10 +- .../nereids/processor/post/Validator.java | 6 +- .../doris/nereids/rules/AppliedAwareRule.java | 111 ++++++ .../apache/doris/nereids/rules/ProxyRule.java | 45 +++ .../org/apache/doris/nereids/rules/Rule.java | 5 + .../doris/nereids/rules/RuleFactory.java | 4 +- .../apache/doris/nereids/rules/RuleType.java | 17 +- .../rules/analysis/AnalysisRuleFactory.java | 3 +- .../analysis/AvgDistinctToSumDivCount.java | 3 +- .../rules/analysis/BindExpression.java | 138 ++++---- .../nereids/rules/analysis/BindRelation.java | 54 +-- .../nereids/rules/analysis/CheckPolicy.java | 18 +- .../nereids/rules/analysis/RegisterCTE.java | 10 +- .../ReplaceExpressionByChildOutput.java | 8 +- .../ResolveOrdinalInOrderByAndGroupBy.java | 15 +- .../nereids/rules/analysis/SlotBinder.java | 11 +- .../rules/analysis/SubExprAnalyzer.java | 15 +- ...lyzeSubquery.java => SubqueryToApply.java} | 29 +- .../rules/analysis/UserAuthentication.java | 4 +- .../exploration/ExplorationRuleFactory.java | 3 +- .../rewrite/ExpressionNormalization.java | 11 +- .../expression/rewrite/ExpressionRewrite.java | 82 +++-- .../rewrite/ExpressionRewriteContext.java | 5 +- .../rewrite/ExpressionRuleExecutor.java | 24 +- .../rewrite/rules/FoldConstantRule.java | 7 - .../implementation/AggregateStrategies.java | 30 +- .../ImplementationRuleFactory.java | 3 +- .../SelectMaterializedIndexWithAggregate.java | 13 +- ...lectMaterializedIndexWithoutAggregate.java | 31 +- .../rewrite/BatchRewriteRuleFactory.java | 47 +++ .../rules/rewrite/RewriteRuleFactory.java | 3 +- ...kAndStandardizeWindowFunctionAndFrame.java | 4 +- .../rewrite/logical/EliminateAggregate.java | 4 +- .../logical/EliminateGroupByConstant.java | 9 +- .../logical/EliminateLimitUnderApply.java | 2 +- .../rewrite/logical/EliminateNotNull.java | 8 +- .../rewrite/logical/EliminateOuterJoin.java | 4 +- .../logical/EliminateSortUnderApply.java | 6 +- .../logical/EliminateUnnecessaryProject.java | 123 ++++--- .../logical/ExtractFilterFromCrossJoin.java | 4 +- ...tSingleTableExpressionFromDisjunction.java | 1 - .../logical/FindHashConditionForJoin.java | 4 + .../logical/HideOneRowRelationUnderUnion.java | 4 +- .../rewrite/logical/InferFilterNotNull.java | 7 +- .../rewrite/logical/InferJoinNotNull.java | 15 +- .../rewrite/logical/InferPredicates.java | 8 +- .../rules/rewrite/logical/MergeFilters.java | 4 - .../rules/rewrite/logical/MergeGenerates.java | 4 +- .../rules/rewrite/logical/MergeProjects.java | 4 +- .../rewrite/logical/MergeSetOperations.java | 65 ++-- .../rewrite/logical/PruneAggChildColumns.java | 4 +- .../logical/PruneFilterChildColumns.java | 5 +- .../logical/PruneJoinChildrenColumns.java | 5 +- .../logical/PruneSortChildColumns.java | 5 +- ...atedFilterUnderApplyAggregateProject.java} | 18 +- ...ject.java => PullUpProjectUnderApply.java} | 16 +- .../rewrite/logical/PushFilterInsideJoin.java | 9 +- .../PushdownFilterThroughAggregation.java | 3 +- .../logical/PushdownFilterThroughJoin.java | 10 +- .../logical/PushdownFilterThroughProject.java | 53 ++- .../logical/PushdownFilterThroughRepeat.java | 3 +- .../logical/PushdownProjectThroughLimit.java | 10 +- .../rules/rewrite/logical/ReorderJoin.java | 13 +- ... => UnCorrelatedApplyAggregateFilter.java} | 12 +- ...lter.java => UnCorrelatedApplyFilter.java} | 9 +- ...va => UnCorrelatedApplyProjectFilter.java} | 11 +- .../apache/doris/nereids/trees/TreeNode.java | 17 + .../nereids/trees/expressions/Exists.java | 8 +- .../nereids/trees/expressions/Expression.java | 10 +- .../nereids/trees/expressions/InSubquery.java | 8 +- .../nereids/trees/expressions/IsNull.java | 2 +- .../trees/expressions/SlotReference.java | 3 +- .../trees/expressions/SubqueryExpr.java | 40 +-- .../nereids/trees/plans/AbstractPlan.java | 27 +- .../doris/nereids/trees/plans/FakePlan.java | 13 + .../doris/nereids/trees/plans/Plan.java | 16 + .../doris/nereids/trees/plans/PlanType.java | 6 +- .../nereids/trees/plans/commands/Command.java | 49 ++- .../plans/commands/CreatePolicyCommand.java | 4 +- .../trees/plans/commands/ExplainCommand.java | 4 +- .../plans/logical/AbstractLogicalPlan.java | 11 +- .../trees/plans/logical/LogicalAggregate.java | 2 +- .../trees/plans/logical/LogicalLeaf.java | 2 +- .../trees/plans/logical/LogicalOlapScan.java | 3 + .../trees/plans/logical/LogicalProject.java | 4 +- .../trees/plans/logical/LogicalRepeat.java | 3 +- .../plans/logical/LogicalSetOperation.java | 16 +- .../trees/plans/logical/OutputSavePoint.java | 22 ++ .../plans/visitor/CustomRewriter.java} | 21 +- .../doris/nereids/util/ExpressionUtils.java | 14 +- .../doris/nereids/util/MutableState.java | 92 +++++ .../org/apache/doris/nereids/util/Utils.java | 1 + .../apache/doris/nereids/JoinHintTest.java | 4 +- .../datasets/ssb/SSBJoinReorderTest.java | 4 +- .../apache/doris/nereids/memo/MemoTest.java | 4 +- .../doris/nereids/metrics/EventTest.java | 5 + .../doris/nereids/parser/ParserTestBase.java | 4 +- .../pattern/GroupExpressionMatchingTest.java | 2 +- .../rules/analysis/AnalyzeSubQueryTest.java | 20 +- .../analysis/AnalyzeWhereSubqueryTest.java | 62 ++-- .../rules/analysis/BindFunctionTest.java | 4 +- .../analysis/CheckExpressionLegalityTest.java | 4 +- .../analysis/FillUpMissingSlotsTest.java | 4 +- .../rules/analysis/FunctionRegistryTest.java | 4 +- .../rules/analysis/NormalizeRepeatTest.java | 4 +- .../rules/analysis/RegisterCTETest.java | 12 +- .../ReplaceExpressionByChildOutputTest.java | 4 +- .../join/InnerJoinLAsscomProjectTest.java | 4 +- .../join/InnerJoinLAsscomTest.java | 4 +- .../join/InnerJoinLeftAssociateTest.java | 4 +- .../join/InnerJoinRightAssociateTest.java | 4 +- .../exploration/join/JoinCommuteTest.java | 4 +- .../exploration/join/JoinExchangeTest.java | 4 +- .../exploration/join/OuterJoinAssocTest.java | 19 +- .../join/OuterJoinLAsscomProjectTest.java | 4 +- .../join/OuterJoinLAsscomTest.java | 4 +- .../SemiJoinSemiJoinTransposeProjectTest.java | 4 +- .../rewrite/ExpressionRewriteTestHelper.java | 21 +- .../expression/rewrite/FoldConstantTest.java | 2 +- .../expression/rewrite/SimplifyRangeTest.java | 13 +- .../nereids/rules/mv/SelectMvIndexTest.java | 4 +- .../rules/mv/SelectRollupIndexTest.java | 4 +- .../logical/AggregateStrategiesTest.java | 4 +- ...CheckAndStandardizeWindowFunctionTest.java | 4 +- .../rewrite/logical/ColumnPruningTest.java | 4 +- .../EliminateDedupJoinConditionTest.java | 4 +- .../logical/EliminateOuterJoinTest.java | 16 +- .../EliminateUnnecessaryProjectTest.java | 22 +- ...tractAndNormalizeWindowExpressionTest.java | 4 +- .../ExtractFilterFromCrossJoinTest.java | 4 +- ...gleTableExpressionFromDisjunctionTest.java | 4 +- .../logical/InferFilterNotNullTest.java | 4 +- .../rewrite/logical/InferJoinNotNullTest.java | 12 +- .../rewrite/logical/InferPredicatesTest.java | 4 +- .../rewrite/logical/LimitPushDownTest.java | 4 +- .../LogicalWindowToPhysicalWindowTest.java | 4 +- .../rewrite/logical/MergeProjectsTest.java | 4 +- .../logical/NormalizeAggregateTest.java | 4 +- .../PhysicalStorageLayerAggregateTest.java | 4 +- .../logical/PushFilterInsideJoinTest.java | 4 +- ...ushdownExpressionsInHashConditionTest.java | 4 +- .../PushdownFilterThroughAggregationTest.java | 8 +- .../PushdownFilterThroughJoinTest.java | 4 +- .../PushdownProjectThroughLimitTest.java | 4 +- .../rewrite/logical/ReorderJoinTest.java | 4 +- .../doris/nereids/sqltest/InferTest.java | 2 +- .../doris/nereids/sqltest/SqlTestBase.java | 4 +- .../trees/expressions/SelectExceptTest.java | 4 +- .../nereids/trees/expressions/ViewTest.java | 4 +- .../nereids/trees/plans/PlanToStringTest.java | 2 +- .../doris/nereids/util/MatchingUtils.java | 12 +- ...ed.java => MemoPatternMatchSupported.java} | 4 +- .../doris/nereids/util/MemoTestUtils.java | 5 +- .../doris/nereids/util/PlanChecker.java | 16 +- .../util/PlanPatternMatchSupported.java} | 20 +- .../doris/utframe/TestWithFeService.java | 11 +- .../explain_clickbench_benchmark.groovy | 3 +- 202 files changed, 3490 insertions(+), 1652 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/PlanSource.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/RewriteJob.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/TopicRewriteJob.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeRulesJob.java rename fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/{ConvertApplyToJoinJob.java => ApplyToJoin.java} (70%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRewriteJob.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRulesJob.java rename fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/{AdjustAggregateNullableForEmptySetJob.java => CascadesOptimizer.java} (58%) rename fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/{AdjustApplyFromCorrelateToUnCorrelateJob.java => CorrelateApplyToUnCorrelateApply.java} (54%) rename fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/{EliminateSpecificPlanUnderApplyJob.java => EliminateUselessPlanUnderApply.java} (72%) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteJobContext.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/VisitorRewriteJob.java rename fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/{batch/AnalyzeSubqueryRulesJob.java => scheduler/ScheduleContext.java} (60%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MemoPatterns.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PlanPatterns.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ProxyPattern.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/AppliedAwareRule.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/ProxyRule.java rename fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/{AnalyzeSubquery.java => SubqueryToApply.java} (88%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/BatchRewriteRuleFactory.java rename fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/{ApplyPullFilterOnProjectUnderAgg.java => PullUpCorrelatedFilterUnderApplyAggregateProject.java} (82%) rename fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/{PushApplyUnderProject.java => PullUpProjectUnderApply.java} (82%) rename fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/{ApplyPullFilterOnAgg.java => UnCorrelatedApplyAggregateFilter.java} (89%) rename fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/{PushApplyUnderFilter.java => UnCorrelatedApplyFilter.java} (89%) rename fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/{EliminateFilterUnderApplyProject.java => UnCorrelatedApplyProjectFilter.java} (90%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/OutputSavePoint.java rename fe/fe-core/src/main/java/org/apache/doris/nereids/{jobs/batch/CheckAnalysisJob.java => trees/plans/visitor/CustomRewriter.java} (60%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/util/MutableState.java rename fe/fe-core/src/test/java/org/apache/doris/nereids/util/{PatternMatchSupported.java => MemoPatternMatchSupported.java} (87%) rename fe/fe-core/src/{main/java/org/apache/doris/nereids/jobs/batch/OptimizeRulesJob.java => test/java/org/apache/doris/nereids/util/PlanPatternMatchSupported.java} (66%) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index 413aa56e0a..2ed1123bb1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -26,11 +26,14 @@ import org.apache.doris.nereids.analyzer.Scope; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.jobs.Job; import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.jobs.rewrite.CustomRewriteJob; import org.apache.doris.nereids.jobs.rewrite.RewriteBottomUpJob; import org.apache.doris.nereids.jobs.rewrite.RewriteTopDownJob; +import org.apache.doris.nereids.jobs.rewrite.RootPlanTreeRewriteJob.RootRewriteJobContext; import org.apache.doris.nereids.jobs.scheduler.JobPool; import org.apache.doris.nereids.jobs.scheduler.JobScheduler; import org.apache.doris.nereids.jobs.scheduler.JobStack; +import org.apache.doris.nereids.jobs.scheduler.ScheduleContext; import org.apache.doris.nereids.jobs.scheduler.SimpleJobScheduler; import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.processor.post.RuntimeFilterContext; @@ -38,12 +41,14 @@ import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleFactory; import org.apache.doris.nereids.rules.RuleSet; +import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; @@ -57,12 +62,20 @@ import java.util.Optional; import java.util.Set; import java.util.Stack; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; /** * Context used in memo. */ -public class CascadesContext { - private final Memo memo; +public class CascadesContext implements ScheduleContext, PlanSource { + // in analyze/rewrite stage, the plan will storage in this field + private Plan plan; + + private Optional currentRootRewriteJobContext; + + // in optimize stage, the plan will storage in the memo + private Memo memo; + private final StatementContext statementContext; private CTEContext cteContext; @@ -76,8 +89,13 @@ public class CascadesContext { private List tables = null; - public CascadesContext(Memo memo, StatementContext statementContext, PhysicalProperties requestProperties) { - this(memo, statementContext, new CTEContext(), requestProperties); + private boolean isRewriteRoot; + + private Optional outerScope = Optional.empty(); + + public CascadesContext(Plan plan, Memo memo, StatementContext statementContext, + PhysicalProperties requestProperties) { + this(plan, memo, statementContext, new CTEContext(), requestProperties); } /** @@ -86,8 +104,9 @@ public class CascadesContext { * @param memo {@link Memo} reference * @param statementContext {@link StatementContext} reference */ - public CascadesContext(Memo memo, StatementContext statementContext, + public CascadesContext(Plan plan, Memo memo, StatementContext statementContext, CTEContext cteContext, PhysicalProperties requireProperties) { + this.plan = plan; this.memo = memo; this.statementContext = statementContext; this.ruleSet = new RuleSet(); @@ -99,19 +118,30 @@ public class CascadesContext { this.cteContext = cteContext; } - public static CascadesContext newContext(StatementContext statementContext, + public static CascadesContext newMemoContext(StatementContext statementContext, Plan initPlan, PhysicalProperties requireProperties) { - return new CascadesContext(new Memo(initPlan), statementContext, requireProperties); + return new CascadesContext(initPlan, new Memo(initPlan), statementContext, requireProperties); + } + + public static CascadesContext newRewriteContext(StatementContext statementContext, + Plan initPlan, PhysicalProperties requireProperties) { + return new CascadesContext(initPlan, null, statementContext, requireProperties); + } + + public static CascadesContext newRewriteContext(StatementContext statementContext, + Plan initPlan, CTEContext cteContext) { + return new CascadesContext(initPlan, null, statementContext, cteContext, PhysicalProperties.ANY); + } + + public void toMemo() { + this.memo = new Memo(plan); } public NereidsAnalyzer newAnalyzer() { return new NereidsAnalyzer(this); } - public NereidsAnalyzer newAnalyzer(Optional outerScope) { - return new NereidsAnalyzer(this, outerScope); - } - + @Override public void pushJob(Job job) { jobPool.push(job); } @@ -136,6 +166,7 @@ public class CascadesContext { this.ruleSet = ruleSet; } + @Override public JobPool getJobPool() { return jobPool; } @@ -165,6 +196,23 @@ public class CascadesContext { return this; } + public Plan getRewritePlan() { + return plan; + } + + public void setRewritePlan(Plan plan) { + this.plan = plan; + } + + public Optional getCurrentRootRewriteJobContext() { + return currentRootRewriteJobContext; + } + + public void setCurrentRootRewriteJobContext( + RootRewriteJobContext currentRootRewriteJobContext) { + this.currentRootRewriteJobContext = Optional.ofNullable(currentRootRewriteJobContext); + } + public void setSubqueryExprIsAnalyzed(SubqueryExpr subqueryExpr, boolean isAnalyzed) { subqueryExprIsAnalyzed.put(subqueryExpr, isAnalyzed); } @@ -201,6 +249,13 @@ public class CascadesContext { return execute(new RewriteTopDownJob(memo.getRoot(), rules, currentJobContext)); } + public CascadesContext topDownRewrite(CustomRewriter customRewriter) { + CustomRewriteJob customRewriteJob = new CustomRewriteJob(() -> customRewriter, RuleType.TEST_REWRITE); + customRewriteJob.execute(currentJobContext); + toMemo(); + return this; + } + public CTEContext getCteContext() { return cteContext; } @@ -209,6 +264,22 @@ public class CascadesContext { this.cteContext = cteContext; } + public void setIsRewriteRoot(boolean isRewriteRoot) { + this.isRewriteRoot = isRewriteRoot; + } + + public boolean isRewriteRoot() { + return isRewriteRoot; + } + + public Optional getOuterScope() { + return outerScope; + } + + public void setOuterScope(@Nullable Scope outerScope) { + this.outerScope = Optional.ofNullable(outerScope); + } + private CascadesContext execute(Job job) { pushJob(job); jobScheduler.executeJobPool(this); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 9d8db1b127..d19c176228 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -26,8 +26,8 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator; import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; -import org.apache.doris.nereids.jobs.batch.NereidsRewriteJobExecutor; -import org.apache.doris.nereids.jobs.batch.OptimizeRulesJob; +import org.apache.doris.nereids.jobs.batch.CascadesOptimizer; +import org.apache.doris.nereids.jobs.batch.NereidsRewriter; import org.apache.doris.nereids.jobs.cascades.DeriveStatsJob; import org.apache.doris.nereids.jobs.joinorder.JoinOrderJob; import org.apache.doris.nereids.memo.CopyInResult; @@ -156,7 +156,7 @@ public class NereidsPlanner extends Planner { // resolve column, table and function analyze(); if (explainLevel == ExplainLevel.ANALYZED_PLAN || explainLevel == ExplainLevel.ALL_PLAN) { - analyzedPlan = cascadesContext.getMemo().copyOut(false); + analyzedPlan = cascadesContext.getRewritePlan(); if (explainLevel == ExplainLevel.ANALYZED_PLAN) { return analyzedPlan; } @@ -164,11 +164,14 @@ public class NereidsPlanner extends Planner { // rule-based optimize rewrite(); if (explainLevel == ExplainLevel.REWRITTEN_PLAN || explainLevel == ExplainLevel.ALL_PLAN) { - rewrittenPlan = cascadesContext.getMemo().copyOut(false); + rewrittenPlan = cascadesContext.getRewritePlan(); if (explainLevel == ExplainLevel.REWRITTEN_PLAN) { return rewrittenPlan; } } + + initMemo(); + deriveStats(); optimize(); @@ -190,7 +193,7 @@ public class NereidsPlanner extends Planner { } private void initCascadesContext(LogicalPlan plan, PhysicalProperties requireProperties) { - cascadesContext = CascadesContext.newContext(statementContext, plan, requireProperties); + cascadesContext = CascadesContext.newRewriteContext(statementContext, plan, requireProperties); } private void analyze() { @@ -201,7 +204,11 @@ public class NereidsPlanner extends Planner { * Logical plan rewrite based on a series of heuristic rules. */ private void rewrite() { - new NereidsRewriteJobExecutor(cascadesContext).execute(); + new NereidsRewriter(cascadesContext).execute(); + } + + private void initMemo() { + cascadesContext.toMemo(); } private void deriveStats() { @@ -236,7 +243,7 @@ public class NereidsPlanner extends Planner { .getSessionVariable().getMaxTableCountUseCascadesJoinReorder()) { dpHypOptimize(); } - new OptimizeRulesJob(cascadesContext).execute(); + new CascadesOptimizer(cascadesContext).execute(); } private PhysicalPlan postProcess(PhysicalPlan physicalPlan) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanSource.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanSource.java new file mode 100644 index 0000000000..317b03d295 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanSource.java @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids; + +/** PlanSource */ +public interface PlanSource { +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java index 99c2eb7604..f470f88244 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java @@ -18,41 +18,80 @@ package org.apache.doris.nereids.analyzer; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.jobs.batch.AdjustAggregateNullableForEmptySetJob; -import org.apache.doris.nereids.jobs.batch.AnalyzeRulesJob; -import org.apache.doris.nereids.jobs.batch.AnalyzeSubqueryRulesJob; -import org.apache.doris.nereids.jobs.batch.CheckAnalysisJob; +import org.apache.doris.nereids.jobs.RewriteJob; +import org.apache.doris.nereids.jobs.batch.BatchRewriteJob; +import org.apache.doris.nereids.rules.analysis.AdjustAggregateNullableForEmptySet; +import org.apache.doris.nereids.rules.analysis.BindExpression; +import org.apache.doris.nereids.rules.analysis.BindRelation; +import org.apache.doris.nereids.rules.analysis.CheckAnalysis; +import org.apache.doris.nereids.rules.analysis.CheckPolicy; +import org.apache.doris.nereids.rules.analysis.FillUpMissingSlots; +import org.apache.doris.nereids.rules.analysis.NormalizeRepeat; +import org.apache.doris.nereids.rules.analysis.ProjectToGlobalAggregate; +import org.apache.doris.nereids.rules.analysis.ProjectWithDistinctToAggregate; +import org.apache.doris.nereids.rules.analysis.RegisterCTE; +import org.apache.doris.nereids.rules.analysis.ReplaceExpressionByChildOutput; +import org.apache.doris.nereids.rules.analysis.ResolveOrdinalInOrderByAndGroupBy; +import org.apache.doris.nereids.rules.analysis.SubqueryToApply; +import org.apache.doris.nereids.rules.analysis.UserAuthentication; +import org.apache.doris.nereids.rules.rewrite.logical.HideOneRowRelationUnderUnion; -import java.util.Objects; -import java.util.Optional; +import java.util.List; /** * Bind symbols according to metadata in the catalog, perform semantic analysis, etc. * TODO: revisit the interface after subquery analysis is supported. */ -public class NereidsAnalyzer { - - private final CascadesContext cascadesContext; - private final Optional outerScope; +public class NereidsAnalyzer extends BatchRewriteJob { + public static final List ANALYZE_JOBS = jobs( + topDown( + new RegisterCTE() + ), + bottomUp( + new BindRelation(), + new CheckPolicy(), + new UserAuthentication(), + new BindExpression(), + new ProjectToGlobalAggregate(), + // this rule check's the logicalProject node's isDisinct property + // and replace the logicalProject node with a LogicalAggregate node + // so any rule before this, if create a new logicalProject node + // should make sure isDistinct property is correctly passed around. + // please see rule BindSlotReference or BindFunction for example + new ProjectWithDistinctToAggregate(), + new ResolveOrdinalInOrderByAndGroupBy(), + new ReplaceExpressionByChildOutput(), + new HideOneRowRelationUnderUnion() + ), + topDown( + new FillUpMissingSlots(), + // We should use NormalizeRepeat to compute nullable properties for LogicalRepeat in the analysis + // stage. NormalizeRepeat will compute nullable property, add virtual slot, LogicalAggregate and + // LogicalProject for normalize. This rule depends on FillUpMissingSlots to fill up slots. + new NormalizeRepeat() + ), + bottomUp(new SubqueryToApply()), + bottomUp(new AdjustAggregateNullableForEmptySet()), + bottomUp(new CheckAnalysis()) + ); + /** + * Execute the analysis job with scope. + * @param cascadesContext planner context for execute job + */ public NereidsAnalyzer(CascadesContext cascadesContext) { - this(cascadesContext, Optional.empty()); + super(cascadesContext); } - public NereidsAnalyzer(CascadesContext cascadesContext, Optional outerScope) { - this.cascadesContext = Objects.requireNonNull(cascadesContext, "cascadesContext cannot be null"); - this.outerScope = Objects.requireNonNull(outerScope, "outerScope cannot be null"); + @Override + public List getJobs() { + return ANALYZE_JOBS; } /** * nereids analyze sql. */ public void analyze() { - new AnalyzeRulesJob(cascadesContext, outerScope).execute(); - new AnalyzeSubqueryRulesJob(cascadesContext).execute(); - new AdjustAggregateNullableForEmptySetJob(cascadesContext).execute(); - // check whether analyze result is meaningful - new CheckAnalysisJob(cascadesContext).execute(); + execute(); } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java index 5e32f1824e..dcdb79b2c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java @@ -22,11 +22,12 @@ import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.Sets; -import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; /** * The slot range required for expression analyze. @@ -57,13 +58,13 @@ public class Scope { private final List slots; private final Optional ownerSubquery; - private List correlatedSlots; + private Set correlatedSlots; public Scope(Optional outerScope, List slots, Optional subqueryExpr) { this.outerScope = outerScope; this.slots = ImmutableList.copyOf(Objects.requireNonNull(slots, "slots can not be null")); this.ownerSubquery = subqueryExpr; - this.correlatedSlots = new ArrayList<>(); + this.correlatedSlots = Sets.newLinkedHashSet(); } public Scope(List slots) { @@ -82,7 +83,7 @@ public class Scope { return ownerSubquery; } - public List getCorrelatedSlots() { + public Set getCorrelatedSlots() { return correlatedSlots; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java index c16a654490..f017dfbacb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java @@ -19,7 +19,6 @@ package org.apache.doris.nereids.jobs; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.CopyInResult; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; @@ -39,17 +38,14 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import java.util.List; -import java.util.Locale; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Function; -import java.util.stream.Collectors; /** * Abstract class for all job using for analyze and optimize query plan in Nereids. @@ -60,8 +56,6 @@ public abstract class Job implements TracerSupplier { EventChannel.getDefaultChannel() .addEnhancers(new AddCounterEventEnhancer()) .addConsumers(new LogConsumer(CounterEvent.class, EventChannel.LOG))); - public final Logger logger = LogManager.getLogger(getClass()); - protected JobType type; protected JobContext context; protected boolean once; @@ -76,12 +70,11 @@ public abstract class Job implements TracerSupplier { this.type = type; this.context = context; this.once = once; - this.disableRules = getAndCacheSessionVariable(context, "disableNereidsRules", - ImmutableSet.of(), SessionVariable::getDisableNereidsRules); + this.disableRules = getDisableRules(context); } public void pushJob(Job job) { - context.getCascadesContext().pushJob(job); + context.getScheduleContext().pushJob(job); } public RuleSet getRuleSet() { @@ -101,12 +94,21 @@ public abstract class Job implements TracerSupplier { */ public List getValidRules(GroupExpression groupExpression, List candidateRules) { return candidateRules.stream() - .filter(rule -> !disableRules.contains(rule.getRuleType().name().toUpperCase(Locale.ROOT))) - .filter(rule -> Objects.nonNull(rule) && rule.getPattern().matchRoot(groupExpression.getPlan()) - && groupExpression.notApplied(rule)).collect(Collectors.toList()); + .filter(rule -> Objects.nonNull(rule) + && !disableRules.contains(rule.getRuleType().name()) + && rule.getPattern().matchRoot(groupExpression.getPlan()) + && groupExpression.notApplied(rule)) + .collect(ImmutableList.toImmutableList()); } - public abstract void execute() throws AnalysisException; + public List getValidRules(List candidateRules) { + return candidateRules.stream() + .filter(rule -> Objects.nonNull(rule) + && !disableRules.contains(rule.getRuleType().name())) + .collect(ImmutableList.toImmutableList()); + } + + public abstract void execute(); public EventProducer getEventTracer() { throw new UnsupportedOperationException("get_event_tracer is unsupported"); @@ -146,7 +148,17 @@ public abstract class Job implements TracerSupplier { groupExpression.getOwnerGroup(), groupExpression, groupExpression.getPlan())); } - private T getAndCacheSessionVariable(JobContext context, String cacheName, + public static Set getDisableRules(JobContext context) { + return getAndCacheSessionVariable(context, "disableNereidsRules", + ImmutableSet.of(), SessionVariable::getDisableNereidsRules); + } + + public static boolean isTraceEnable(JobContext context) { + return getAndCacheSessionVariable(context, "isTraceEnable", + false, SessionVariable::isEnableNereidsTrace); + } + + private static T getAndCacheSessionVariable(JobContext context, String cacheName, T defaultValue, Function variableSupplier) { CascadesContext cascadesContext = context.getCascadesContext(); ConnectContext connectContext = cascadesContext.getConnectContext(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobContext.java index 04fb4e87a4..626b8f24cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobContext.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.jobs; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.scheduler.ScheduleContext; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.RuleType; @@ -29,21 +30,25 @@ import java.util.Map; * Context for one job in Nereids' cascades framework. */ public class JobContext { - protected final CascadesContext cascadesContext; + protected final ScheduleContext scheduleContext; protected final PhysicalProperties requiredProperties; protected double costUpperBound; protected boolean rewritten = false; protected Map ruleInvokeTimes = Maps.newLinkedHashMap(); - public JobContext(CascadesContext cascadesContext, PhysicalProperties requiredProperties, double costUpperBound) { - this.cascadesContext = cascadesContext; + public JobContext(ScheduleContext scheduleContext, PhysicalProperties requiredProperties, double costUpperBound) { + this.scheduleContext = scheduleContext; this.requiredProperties = requiredProperties; this.costUpperBound = costUpperBound; } + public ScheduleContext getScheduleContext() { + return scheduleContext; + } + public CascadesContext getCascadesContext() { - return cascadesContext; + return (CascadesContext) scheduleContext; } public PhysicalProperties getRequiredProperties() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobType.java index 03681c58ee..d7a1517af2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobType.java @@ -31,5 +31,6 @@ public enum JobType { TOP_DOWN_REWRITE, VISITOR_REWRITE, BOTTOM_UP_REWRITE, - JOIN_ORDER; + JOIN_ORDER, + LINK_PLAN; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/RewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/RewriteJob.java new file mode 100644 index 0000000000..cdaeefb086 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/RewriteJob.java @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs; + +/** RewriteJob */ +public interface RewriteJob { + void execute(JobContext jobContext); + + boolean isOnce(); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/TopicRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/TopicRewriteJob.java new file mode 100644 index 0000000000..3fb026c4fb --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/TopicRewriteJob.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.stream.Stream; + +/** TopicRewriteJob */ +public class TopicRewriteJob implements RewriteJob { + public final String topicName; + public final List jobs; + + /** constructor */ + public TopicRewriteJob(String topicName, List jobs) { + this.topicName = topicName; + this.jobs = jobs.stream() + .flatMap(job -> job instanceof TopicRewriteJob + ? ((TopicRewriteJob) job).jobs.stream() + : Stream.of(job) + ) + .collect(ImmutableList.toImmutableList()); + } + + @Override + public void execute(JobContext jobContext) { + for (RewriteJob job : jobs) { + job.execute(jobContext); + } + } + + @Override + public boolean isOnce() { + return true; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeRulesJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeRulesJob.java deleted file mode 100644 index c68e096baf..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeRulesJob.java +++ /dev/null @@ -1,82 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.jobs.batch; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.analyzer.Scope; -import org.apache.doris.nereids.rules.analysis.AvgDistinctToSumDivCount; -import org.apache.doris.nereids.rules.analysis.BindExpression; -import org.apache.doris.nereids.rules.analysis.BindRelation; -import org.apache.doris.nereids.rules.analysis.CheckPolicy; -import org.apache.doris.nereids.rules.analysis.FillUpMissingSlots; -import org.apache.doris.nereids.rules.analysis.NormalizeRepeat; -import org.apache.doris.nereids.rules.analysis.ProjectToGlobalAggregate; -import org.apache.doris.nereids.rules.analysis.ProjectWithDistinctToAggregate; -import org.apache.doris.nereids.rules.analysis.RegisterCTE; -import org.apache.doris.nereids.rules.analysis.ReplaceExpressionByChildOutput; -import org.apache.doris.nereids.rules.analysis.ResolveOrdinalInOrderByAndGroupBy; -import org.apache.doris.nereids.rules.analysis.UserAuthentication; -import org.apache.doris.nereids.rules.rewrite.logical.HideOneRowRelationUnderUnion; - -import com.google.common.collect.ImmutableList; - -import java.util.Optional; - -/** - * Execute the analysis rules. - */ -public class AnalyzeRulesJob extends BatchRulesJob { - - /** - * Execute the analysis job with scope. - * @param cascadesContext planner context for execute job - * @param scope Parse the symbolic scope of the field - */ - public AnalyzeRulesJob(CascadesContext cascadesContext, Optional scope) { - super(cascadesContext); - rulesJob.addAll(ImmutableList.of( - bottomUpBatch( - new RegisterCTE() - ), - bottomUpBatch( - new BindRelation(), - new CheckPolicy(), - new UserAuthentication(), - new BindExpression(scope), - new ProjectToGlobalAggregate(), - // this rule check's the logicalProject node's isDisinct property - // and replace the logicalProject node with a LogicalAggregate node - // so any rule before this, if create a new logicalProject node - // should make sure isDisinct property is correctly passed around. - // please see rule BindSlotReference or BindFunction for example - new ProjectWithDistinctToAggregate(), - new AvgDistinctToSumDivCount(), - new ResolveOrdinalInOrderByAndGroupBy(), - new ReplaceExpressionByChildOutput(), - new HideOneRowRelationUnderUnion() - ), - topDownBatch( - new FillUpMissingSlots(), - // We should use NormalizeRepeat to compute nullable properties for LogicalRepeat in the analysis - // stage. NormalizeRepeat will compute nullable property, add virtual slot, LogicalAggregate and - // LogicalProject for normalize. This rule depends on FillUpMissingSlots to fill up slots. - new NormalizeRepeat() - ) - )); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/ConvertApplyToJoinJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/ApplyToJoin.java similarity index 70% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/ConvertApplyToJoinJob.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/ApplyToJoin.java index d086c4fb82..a3597ac1c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/ConvertApplyToJoinJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/ApplyToJoin.java @@ -17,27 +17,28 @@ package org.apache.doris.nereids.jobs.batch; -import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.RuleFactory; +import org.apache.doris.nereids.rules.rewrite.BatchRewriteRuleFactory; import org.apache.doris.nereids.rules.rewrite.logical.ExistsApplyToJoin; import org.apache.doris.nereids.rules.rewrite.logical.InApplyToJoin; import org.apache.doris.nereids.rules.rewrite.logical.ScalarApplyToJoin; import com.google.common.collect.ImmutableList; +import java.util.List; + /** * Convert logicalApply without a correlated to a logicalJoin. */ -public class ConvertApplyToJoinJob extends BatchRulesJob { - /** - * Constructor. - */ - public ConvertApplyToJoinJob(CascadesContext cascadesContext) { - super(cascadesContext); - rulesJob.addAll(ImmutableList.of( - topDownBatch(ImmutableList.of( - new ScalarApplyToJoin(), - new InApplyToJoin(), - new ExistsApplyToJoin()) - ))); +public class ApplyToJoin implements BatchRewriteRuleFactory { + public static final List RULES = ImmutableList.of( + new ScalarApplyToJoin(), + new InApplyToJoin(), + new ExistsApplyToJoin() + ); + + @Override + public List getRuleFactories() { + return RULES; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRewriteJob.java new file mode 100644 index 0000000000..f4d5419d81 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRewriteJob.java @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.batch; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.jobs.RewriteJob; +import org.apache.doris.nereids.jobs.TopicRewriteJob; +import org.apache.doris.nereids.jobs.rewrite.CustomRewriteJob; +import org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteBottomUpJob; +import org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteTopDownJob; +import org.apache.doris.nereids.jobs.rewrite.RootPlanTreeRewriteJob; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleFactory; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; + +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; +import java.util.stream.Stream; + +/** + * Base class for executing all jobs. + * + * Each batch of rules will be uniformly executed. + */ +public abstract class BatchRewriteJob { + protected CascadesContext cascadesContext; + + public BatchRewriteJob(CascadesContext cascadesContext) { + this.cascadesContext = Objects.requireNonNull(cascadesContext, "cascadesContext can not null"); + } + + public static List jobs(RewriteJob... jobs) { + return Arrays.stream(jobs) + .flatMap(job -> job instanceof TopicRewriteJob + ? ((TopicRewriteJob) job).jobs.stream() + : Stream.of(job) + ).collect(ImmutableList.toImmutableList()); + } + + public static TopicRewriteJob topic(String topicName, RewriteJob... jobs) { + return new TopicRewriteJob(topicName, Arrays.asList(jobs)); + } + + public static RewriteJob bottomUp(String batchName, RuleFactory... ruleFactories) { + return bottomUp(Arrays.asList(ruleFactories)); + } + + public static RewriteJob bottomUp(RuleFactory... ruleFactories) { + return bottomUp(Arrays.asList(ruleFactories)); + } + + public static RewriteJob bottomUp(List ruleFactories) { + List rules = new ArrayList<>(); + for (RuleFactory ruleFactory : ruleFactories) { + rules.addAll(ruleFactory.buildRules()); + } + return new RootPlanTreeRewriteJob(rules, PlanTreeRewriteBottomUpJob::new, true); + } + + public static RewriteJob topDown(RuleFactory... ruleFactories) { + return topDown(Arrays.asList(ruleFactories)); + } + + public static RewriteJob topDown(List ruleFactories) { + return topDown(ruleFactories, true); + } + + public static RewriteJob topDown(List ruleFactories, boolean once) { + List rules = new ArrayList<>(); + for (RuleFactory ruleFactory : ruleFactories) { + rules.addAll(ruleFactory.buildRules()); + } + return new RootPlanTreeRewriteJob(rules, PlanTreeRewriteTopDownJob::new, once); + } + + public static RewriteJob custom(RuleType ruleType, Supplier planRewriter) { + return new CustomRewriteJob(planRewriter, ruleType); + } + + /** + * execute. + */ + public void execute() { + for (RewriteJob job : getJobs()) { + JobContext jobContext = cascadesContext.getCurrentJobContext(); + do { + jobContext.setRewritten(false); + job.execute(jobContext); + } while (!job.isOnce() && jobContext.isRewritten()); + } + } + + public abstract List getJobs(); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRulesJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRulesJob.java deleted file mode 100644 index 036437eff6..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRulesJob.java +++ /dev/null @@ -1,109 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.jobs.batch; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.jobs.Job; -import org.apache.doris.nereids.jobs.JobContext; -import org.apache.doris.nereids.jobs.cascades.OptimizeGroupJob; -import org.apache.doris.nereids.jobs.rewrite.RewriteBottomUpJob; -import org.apache.doris.nereids.jobs.rewrite.RewriteTopDownJob; -import org.apache.doris.nereids.jobs.rewrite.VisitorRewriteJob; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleFactory; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; - -/** - * Base class for executing all jobs. - * - * Each batch of rules will be uniformly executed. - */ -public abstract class BatchRulesJob { - protected CascadesContext cascadesContext; - protected List rulesJob = new ArrayList<>(); - - BatchRulesJob(CascadesContext cascadesContext) { - this.cascadesContext = Objects.requireNonNull(cascadesContext, "cascadesContext can not null"); - } - - protected Job bottomUpBatch(RuleFactory... ruleFactories) { - return bottomUpBatch(Arrays.asList(ruleFactories)); - } - - protected Job bottomUpBatch(List ruleFactories) { - List rules = new ArrayList<>(); - for (RuleFactory ruleFactory : ruleFactories) { - rules.addAll(ruleFactory.buildRules()); - } - return new RewriteBottomUpJob( - cascadesContext.getMemo().getRoot(), - rules, - cascadesContext.getCurrentJobContext()); - } - - protected Job topDownBatch(RuleFactory... ruleFactories) { - return topDownBatch(Arrays.asList(ruleFactories)); - } - - protected Job topDownBatch(List ruleFactories) { - List rules = new ArrayList<>(); - for (RuleFactory ruleFactory : ruleFactories) { - rules.addAll(ruleFactory.buildRules()); - } - return new RewriteTopDownJob(cascadesContext.getMemo().getRoot(), rules, - cascadesContext.getCurrentJobContext()); - } - - protected Job topDownBatch(List ruleFactories, boolean once) { - List rules = new ArrayList<>(); - for (RuleFactory ruleFactory : ruleFactories) { - rules.addAll(ruleFactory.buildRules()); - } - return new RewriteTopDownJob(cascadesContext.getMemo().getRoot(), rules, - cascadesContext.getCurrentJobContext(), once); - } - - protected Job visitorJob(RuleType ruleType, DefaultPlanRewriter planRewriter) { - return new VisitorRewriteJob(cascadesContext, planRewriter, ruleType); - } - - protected Job optimize() { - return new OptimizeGroupJob( - cascadesContext.getMemo().getRoot(), - cascadesContext.getCurrentJobContext()); - } - - /** - * execute. - */ - public void execute() { - for (Job job : rulesJob) { - do { - cascadesContext.getCurrentJobContext().setRewritten(false); - cascadesContext.pushJob(job); - cascadesContext.getJobScheduler().executeJobPool(cascadesContext); - } while (!job.isOnce() && cascadesContext.getCurrentJobContext().isRewritten()); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AdjustAggregateNullableForEmptySetJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CascadesOptimizer.java similarity index 58% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AdjustAggregateNullableForEmptySetJob.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CascadesOptimizer.java index 403f1b8ff9..9b7d15c9fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AdjustAggregateNullableForEmptySetJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CascadesOptimizer.java @@ -18,17 +18,25 @@ package org.apache.doris.nereids.jobs.batch; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.analysis.AdjustAggregateNullableForEmptySet; +import org.apache.doris.nereids.jobs.cascades.OptimizeGroupJob; -import com.google.common.collect.ImmutableList; +import java.util.Objects; /** - * Analyze subquery. + * cascade optimizer. */ -public class AdjustAggregateNullableForEmptySetJob extends BatchRulesJob { - public AdjustAggregateNullableForEmptySetJob(CascadesContext cascadesContext) { - super(cascadesContext); - rulesJob.addAll(ImmutableList.of( - bottomUpBatch(ImmutableList.of(new AdjustAggregateNullableForEmptySet())))); +public class CascadesOptimizer { + private CascadesContext cascadesContext; + + public CascadesOptimizer(CascadesContext cascadesContext) { + this.cascadesContext = Objects.requireNonNull(cascadesContext, "cascadesContext cannot be null"); + } + + public void execute() { + cascadesContext.pushJob(new OptimizeGroupJob( + cascadesContext.getMemo().getRoot(), + cascadesContext.getCurrentJobContext()) + ); + cascadesContext.getJobScheduler().executeJobPool(cascadesContext); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AdjustApplyFromCorrelateToUnCorrelateJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CorrelateApplyToUnCorrelateApply.java similarity index 54% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AdjustApplyFromCorrelateToUnCorrelateJob.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CorrelateApplyToUnCorrelateApply.java index 856d857f11..45e51ea969 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AdjustApplyFromCorrelateToUnCorrelateJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CorrelateApplyToUnCorrelateApply.java @@ -17,15 +17,18 @@ package org.apache.doris.nereids.jobs.batch; -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.rewrite.logical.ApplyPullFilterOnAgg; -import org.apache.doris.nereids.rules.rewrite.logical.ApplyPullFilterOnProjectUnderAgg; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateFilterUnderApplyProject; -import org.apache.doris.nereids.rules.rewrite.logical.PushApplyUnderFilter; -import org.apache.doris.nereids.rules.rewrite.logical.PushApplyUnderProject; +import org.apache.doris.nereids.rules.RuleFactory; +import org.apache.doris.nereids.rules.rewrite.BatchRewriteRuleFactory; +import org.apache.doris.nereids.rules.rewrite.logical.PullUpCorrelatedFilterUnderApplyAggregateProject; +import org.apache.doris.nereids.rules.rewrite.logical.PullUpProjectUnderApply; +import org.apache.doris.nereids.rules.rewrite.logical.UnCorrelatedApplyAggregateFilter; +import org.apache.doris.nereids.rules.rewrite.logical.UnCorrelatedApplyFilter; +import org.apache.doris.nereids.rules.rewrite.logical.UnCorrelatedApplyProjectFilter; import com.google.common.collect.ImmutableList; +import java.util.List; + /** * Adjust the plan in logicalApply so that there are no correlated columns in the subquery. * Adjust the positions of apply and sub query nodes and apply, @@ -34,19 +37,17 @@ import com.google.common.collect.ImmutableList; * For the project and filter on AGG, try to adjust them to apply. * For the project and filter under AGG, bring the filter under AGG and merge it with agg. */ -public class AdjustApplyFromCorrelateToUnCorrelateJob extends BatchRulesJob { - /** - * Constructor. - */ - public AdjustApplyFromCorrelateToUnCorrelateJob(CascadesContext cascadesContext) { - super(cascadesContext); - rulesJob.addAll(ImmutableList.of( - topDownBatch(ImmutableList.of( - new PushApplyUnderProject(), - new PushApplyUnderFilter(), - new EliminateFilterUnderApplyProject(), - new ApplyPullFilterOnAgg(), - new ApplyPullFilterOnProjectUnderAgg() - )))); +public class CorrelateApplyToUnCorrelateApply implements BatchRewriteRuleFactory { + public static final List RULES = ImmutableList.of( + new PullUpProjectUnderApply(), + new UnCorrelatedApplyFilter(), + new UnCorrelatedApplyProjectFilter(), + new UnCorrelatedApplyAggregateFilter(), + new PullUpCorrelatedFilterUnderApplyAggregateProject() + ); + + @Override + public List getRuleFactories() { + return RULES; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/EliminateSpecificPlanUnderApplyJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/EliminateUselessPlanUnderApply.java similarity index 72% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/EliminateSpecificPlanUnderApplyJob.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/EliminateUselessPlanUnderApply.java index 2b8f7b25e0..4878e13f25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/EliminateSpecificPlanUnderApplyJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/EliminateUselessPlanUnderApply.java @@ -17,26 +17,27 @@ package org.apache.doris.nereids.jobs.batch; -import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.RuleFactory; +import org.apache.doris.nereids.rules.rewrite.BatchRewriteRuleFactory; import org.apache.doris.nereids.rules.rewrite.logical.EliminateLimitUnderApply; import org.apache.doris.nereids.rules.rewrite.logical.EliminateSortUnderApply; import com.google.common.collect.ImmutableList; +import java.util.List; + /** * Eliminate useless operators in the subquery, including limit and sort. * Compatible with the old optimizer, the sort and limit in the subquery will not take effect, just delete it directly. */ -public class EliminateSpecificPlanUnderApplyJob extends BatchRulesJob { - /** - * Constructor. - */ - public EliminateSpecificPlanUnderApplyJob(CascadesContext cascadesContext) { - super(cascadesContext); - rulesJob.addAll(ImmutableList.of( - topDownBatch(ImmutableList.of( - new EliminateLimitUnderApply(), - new EliminateSortUnderApply() - )))); +public class EliminateUselessPlanUnderApply implements BatchRewriteRuleFactory { + public static final List RULES = ImmutableList.of( + new EliminateLimitUnderApply(), + new EliminateSortUnderApply() + ); + + @Override + public List getRuleFactories() { + return RULES; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java deleted file mode 100644 index f86fa0abb0..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java +++ /dev/null @@ -1,150 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.jobs.batch; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.jobs.Job; -import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.analysis.AdjustAggregateNullableForEmptySet; -import org.apache.doris.nereids.rules.analysis.CheckAfterRewrite; -import org.apache.doris.nereids.rules.analysis.LogicalSubQueryAliasToLogicalProject; -import org.apache.doris.nereids.rules.expression.rewrite.ExpressionNormalization; -import org.apache.doris.nereids.rules.expression.rewrite.ExpressionOptimization; -import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewrite; -import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithAggregate; -import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithoutAggregate; -import org.apache.doris.nereids.rules.rewrite.logical.AdjustNullable; -import org.apache.doris.nereids.rules.rewrite.logical.BuildAggForUnion; -import org.apache.doris.nereids.rules.rewrite.logical.CheckAndStandardizeWindowFunctionAndFrame; -import org.apache.doris.nereids.rules.rewrite.logical.ColumnPruning; -import org.apache.doris.nereids.rules.rewrite.logical.CountDistinctRewrite; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateAggregate; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateDedupJoinCondition; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateFilter; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateGroupByConstant; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateLimit; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateNotNull; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateNullAwareLeftAntiJoin; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateOrderByConstant; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateUnnecessaryProject; -import org.apache.doris.nereids.rules.rewrite.logical.ExtractAndNormalizeWindowExpression; -import org.apache.doris.nereids.rules.rewrite.logical.ExtractFilterFromCrossJoin; -import org.apache.doris.nereids.rules.rewrite.logical.ExtractSingleTableExpressionFromDisjunction; -import org.apache.doris.nereids.rules.rewrite.logical.FindHashConditionForJoin; -import org.apache.doris.nereids.rules.rewrite.logical.InferFilterNotNull; -import org.apache.doris.nereids.rules.rewrite.logical.InferJoinNotNull; -import org.apache.doris.nereids.rules.rewrite.logical.InferPredicates; -import org.apache.doris.nereids.rules.rewrite.logical.InnerToCrossJoin; -import org.apache.doris.nereids.rules.rewrite.logical.LimitPushDown; -import org.apache.doris.nereids.rules.rewrite.logical.MergeFilters; -import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects; -import org.apache.doris.nereids.rules.rewrite.logical.MergeSetOperations; -import org.apache.doris.nereids.rules.rewrite.logical.NormalizeAggregate; -import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanPartition; -import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanTablet; -import org.apache.doris.nereids.rules.rewrite.logical.PushFilterInsideJoin; -import org.apache.doris.nereids.rules.rewrite.logical.ReorderJoin; - -import com.google.common.collect.ImmutableList; - -/** - * Apply rules to optimize logical plan. - */ -public class NereidsRewriteJobExecutor extends BatchRulesJob { - - /** - * Constructor. - * - * @param cascadesContext context for applying rules. - */ - public NereidsRewriteJobExecutor(CascadesContext cascadesContext) { - super(cascadesContext); - ImmutableList jobs = new ImmutableList.Builder() - .addAll(new EliminateSpecificPlanUnderApplyJob(cascadesContext).rulesJob) - // MergeProjects depends on this rule - .add(bottomUpBatch(ImmutableList.of(new LogicalSubQueryAliasToLogicalProject()))) - // AdjustApplyFromCorrelateToUnCorrelateJob and ConvertApplyToJoinJob - // and SelectMaterializedIndexWithAggregate depends on this rule - .add(topDownBatch(ImmutableList.of(new MergeProjects()))) - .add(topDownBatch(ImmutableList.of(new ExpressionNormalization(cascadesContext.getConnectContext())))) - .add(topDownBatch(ImmutableList.of(new ExpressionOptimization()))) - .add(topDownBatch(ImmutableList.of(new ExtractSingleTableExpressionFromDisjunction()))) - /* - * Subquery unnesting. - * 1. Adjust the plan in correlated logicalApply - * so that there are no correlated columns in the subquery. - * 2. Convert logicalApply to a logicalJoin. - * TODO: group these rules to make sure the result plan is what we expected. - */ - .addAll(new AdjustApplyFromCorrelateToUnCorrelateJob(cascadesContext).rulesJob) - .addAll(new ConvertApplyToJoinJob(cascadesContext).rulesJob) - .add(bottomUpBatch(ImmutableList.of(new AdjustAggregateNullableForEmptySet()))) - .add(topDownBatch(ImmutableList.of(new EliminateGroupByConstant()))) - .add(topDownBatch(ImmutableList.of(new NormalizeAggregate()))) - .add(topDownBatch(ImmutableList.of(new ExtractAndNormalizeWindowExpression()))) - // execute NormalizeAggregate() again to resolve nested AggregateFunctions in WindowExpression, - // e.g. sum(sum(c1)) over(partition by avg(c1)) - .add(topDownBatch(ImmutableList.of(new NormalizeAggregate()))) - .add(topDownBatch(ImmutableList.of(new CheckAndStandardizeWindowFunctionAndFrame()))) - .add(topDownBatch(ImmutableList.of(new InferFilterNotNull()))) - .add(topDownBatch(ImmutableList.of(new InferJoinNotNull()))) - .add(topDownBatch(RuleSet.PUSH_DOWN_FILTERS, false)) - .add(visitorJob(RuleType.INFER_PREDICATES, new InferPredicates())) - .add(topDownBatch(ImmutableList.of(new ExtractFilterFromCrossJoin()))) - .add(topDownBatch(ImmutableList.of(new MergeFilters()))) - .add(topDownBatch(ImmutableList.of(new ReorderJoin()))) - .add(topDownBatch(ImmutableList.of(new EliminateDedupJoinCondition()))) - .add(topDownBatch(ImmutableList.of(new ColumnPruning()))) - .add(topDownBatch(RuleSet.PUSH_DOWN_FILTERS, false)) - .add(visitorJob(RuleType.INFER_PREDICATES, new InferPredicates())) - .add(topDownBatch(RuleSet.PUSH_DOWN_FILTERS, false)) - .add(visitorJob(RuleType.INFER_PREDICATES, new InferPredicates())) - .add(topDownBatch(RuleSet.PUSH_DOWN_FILTERS, false)) - .add(topDownBatch(ImmutableList.of(new PushFilterInsideJoin()))) - .add(topDownBatch(ImmutableList.of(new FindHashConditionForJoin()))) - .add(topDownBatch(RuleSet.PUSH_DOWN_FILTERS, false)) - .add(topDownBatch(ImmutableList.of(new InnerToCrossJoin()))) - .add(topDownBatch(ImmutableList.of(new EliminateNotNull()))) - .add(topDownBatch(ImmutableList.of(new EliminateLimit()))) - .add(topDownBatch(ImmutableList.of(new EliminateFilter()))) - .add(topDownBatch(ImmutableList.of(new PruneOlapScanPartition()))) - .add(topDownBatch(ImmutableList.of(new CountDistinctRewrite()))) - // we need to execute this rule at the end of rewrite - // to avoid two consecutive same project appear when we do optimization. - .add(topDownBatch(ImmutableList.of(new EliminateOrderByConstant()))) - .add(topDownBatch(ImmutableList.of(new EliminateUnnecessaryProject()))) - .add(topDownBatch(ImmutableList.of(new SelectMaterializedIndexWithAggregate()))) - .add(topDownBatch(ImmutableList.of(new SelectMaterializedIndexWithoutAggregate()))) - .add(topDownBatch(ImmutableList.of(new PruneOlapScanTablet()))) - .add(topDownBatch(ImmutableList.of(new EliminateAggregate()))) - .add(bottomUpBatch(ImmutableList.of(new MergeSetOperations()))) - .add(topDownBatch(ImmutableList.of(new LimitPushDown()))) - .add(topDownBatch(ImmutableList.of(new BuildAggForUnion()))) - .add(topDownBatch(ImmutableList.of(new EliminateNullAwareLeftAntiJoin()))) - // this rule batch must keep at the end of rewrite to do some plan check - .add(bottomUpBatch(ImmutableList.of( - new AdjustNullable(), - new ExpressionRewrite(CheckLegalityAfterRewrite.INSTANCE), - new CheckAfterRewrite())) - ) - .build(); - - rulesJob.addAll(jobs); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java new file mode 100644 index 0000000000..28571b1724 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java @@ -0,0 +1,214 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.batch; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.RewriteJob; +import org.apache.doris.nereids.rules.RuleSet; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.analysis.AdjustAggregateNullableForEmptySet; +import org.apache.doris.nereids.rules.analysis.AvgDistinctToSumDivCount; +import org.apache.doris.nereids.rules.analysis.CheckAfterRewrite; +import org.apache.doris.nereids.rules.analysis.LogicalSubQueryAliasToLogicalProject; +import org.apache.doris.nereids.rules.expression.rewrite.ExpressionNormalization; +import org.apache.doris.nereids.rules.expression.rewrite.ExpressionOptimization; +import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewrite; +import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithAggregate; +import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithoutAggregate; +import org.apache.doris.nereids.rules.rewrite.logical.AdjustNullable; +import org.apache.doris.nereids.rules.rewrite.logical.BuildAggForUnion; +import org.apache.doris.nereids.rules.rewrite.logical.CheckAndStandardizeWindowFunctionAndFrame; +import org.apache.doris.nereids.rules.rewrite.logical.ColumnPruning; +import org.apache.doris.nereids.rules.rewrite.logical.CountDistinctRewrite; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateAggregate; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateDedupJoinCondition; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateFilter; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateGroupByConstant; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateLimit; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateNotNull; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateNullAwareLeftAntiJoin; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateOrderByConstant; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateUnnecessaryProject; +import org.apache.doris.nereids.rules.rewrite.logical.ExtractAndNormalizeWindowExpression; +import org.apache.doris.nereids.rules.rewrite.logical.ExtractFilterFromCrossJoin; +import org.apache.doris.nereids.rules.rewrite.logical.ExtractSingleTableExpressionFromDisjunction; +import org.apache.doris.nereids.rules.rewrite.logical.FindHashConditionForJoin; +import org.apache.doris.nereids.rules.rewrite.logical.InferFilterNotNull; +import org.apache.doris.nereids.rules.rewrite.logical.InferJoinNotNull; +import org.apache.doris.nereids.rules.rewrite.logical.InferPredicates; +import org.apache.doris.nereids.rules.rewrite.logical.InnerToCrossJoin; +import org.apache.doris.nereids.rules.rewrite.logical.LimitPushDown; +import org.apache.doris.nereids.rules.rewrite.logical.MergeFilters; +import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects; +import org.apache.doris.nereids.rules.rewrite.logical.MergeSetOperations; +import org.apache.doris.nereids.rules.rewrite.logical.NormalizeAggregate; +import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanPartition; +import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanTablet; +import org.apache.doris.nereids.rules.rewrite.logical.PushFilterInsideJoin; +import org.apache.doris.nereids.rules.rewrite.logical.ReorderJoin; + +import java.util.List; + +/** + * Apply rules to optimize logical plan. + */ +public class NereidsRewriter extends BatchRewriteJob { + private static final List REWRITE_JOBS = jobs( + topic("Normalization", + topDown( + new EliminateOrderByConstant(), + new EliminateGroupByConstant(), + + // MergeProjects depends on this rule + new LogicalSubQueryAliasToLogicalProject(), + + // rewrite expressions, no depends + new ExpressionNormalization(), + new ExpressionOptimization(), + new AvgDistinctToSumDivCount(), + new CountDistinctRewrite(), + + new NormalizeAggregate(), + new ExtractFilterFromCrossJoin() + ), + + // ExtractSingleTableExpressionFromDisjunction conflict to InPredicateToEqualToRule + // in the ExpressionNormalization, so must invoke in another job, or else run into + // deep loop + topDown( + new ExtractSingleTableExpressionFromDisjunction() + ) + ), + + topic("Subquery unnesting", + bottomUp( + new EliminateUselessPlanUnderApply(), + + // CorrelateApplyToUnCorrelateApply and ApplyToJoin + // and SelectMaterializedIndexWithAggregate depends on this rule + new MergeProjects(), + + /* + * Subquery unnesting. + * 1. Adjust the plan in correlated logicalApply + * so that there are no correlated columns in the subquery. + * 2. Convert logicalApply to a logicalJoin. + * TODO: group these rules to make sure the result plan is what we expected. + */ + new CorrelateApplyToUnCorrelateApply(), + new ApplyToJoin() + ) + ), + + topDown( + new AdjustAggregateNullableForEmptySet() + ), + + topic("Window analysis", + topDown( + new ExtractAndNormalizeWindowExpression(), + // execute NormalizeAggregate() again to resolve nested AggregateFunctions in WindowExpression, + // e.g. sum(sum(c1)) over(partition by avg(c1)) + new NormalizeAggregate(), + new CheckAndStandardizeWindowFunctionAndFrame() + ) + ), + + topic("Rewrite join", + // infer not null filter, then push down filter, and then reorder join(cross join to inner join) + topDown( + new InferFilterNotNull(), + new InferJoinNotNull() + ), + // ReorderJoin depends PUSH_DOWN_FILTERS + // the PUSH_DOWN_FILTERS depends on lots of rules, e.g. merge project, eliminate outer, + // sometimes transform the bottom plan make some rules usable which can apply to the top plan, + // but top-down traverse can not cover this case in one iteration, so bottom-up is more + // efficient because it can find the new plans and apply transform wherever it is + bottomUp(RuleSet.PUSH_DOWN_FILTERS), + + topDown( + new MergeFilters(), + new ReorderJoin(), + new PushFilterInsideJoin(), + new FindHashConditionForJoin(), + new InnerToCrossJoin(), + new EliminateNullAwareLeftAntiJoin() + ), + topDown( + new EliminateDedupJoinCondition() + ) + ), + + topic("Column pruning and infer predicate", + topDown(new ColumnPruning()), + + custom(RuleType.INFER_PREDICATES, () -> new InferPredicates()), + + // column pruning create new project, so we should use PUSH_DOWN_FILTERS + // to change filter-project to project-filter + bottomUp(RuleSet.PUSH_DOWN_FILTERS), + + // after eliminate outer join in the PUSH_DOWN_FILTERS, we can infer more predicate and push down + custom(RuleType.INFER_PREDICATES, () -> new InferPredicates()), + + bottomUp(RuleSet.PUSH_DOWN_FILTERS), + + // after eliminate outer join, we can move some filters to join.otherJoinConjuncts, + // this can help to translate plan to backend + topDown( + new PushFilterInsideJoin() + ) + ), + + // this rule should invoke after ColumnPruning + custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT, () -> new EliminateUnnecessaryProject()), + + // we need to execute this rule at the end of rewrite + // to avoid two consecutive same project appear when we do optimization. + topic("Others optimization", topDown( + new EliminateNotNull(), + new EliminateLimit(), + new EliminateFilter(), + new PruneOlapScanPartition(), + new SelectMaterializedIndexWithAggregate(), + new SelectMaterializedIndexWithoutAggregate(), + new PruneOlapScanTablet(), + new EliminateAggregate(), + new MergeSetOperations(), + new LimitPushDown(), + new BuildAggForUnion() + )), + + // this rule batch must keep at the end of rewrite to do some plan check + topic("Final rewrite and check", bottomUp( + new AdjustNullable(), + new ExpressionRewrite(CheckLegalityAfterRewrite.INSTANCE), + new CheckAfterRewrite() + )) + ); + + public NereidsRewriter(CascadesContext cascadesContext) { + super(cascadesContext); + } + + @Override + public List getJobs() { + return REWRITE_JOBS; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java new file mode 100644 index 0000000000..a05ad215a3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.rewrite; + +import org.apache.doris.nereids.jobs.Job; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.jobs.RewriteJob; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; + +import java.util.Locale; +import java.util.Objects; +import java.util.Set; +import java.util.function.Supplier; + +/** + * Custom rewrite the plan. + */ +public class CustomRewriteJob implements RewriteJob { + private final RuleType ruleType; + + private final Supplier customRewriter; + + /** + * Constructor. + */ + public CustomRewriteJob(Supplier rewriter, RuleType ruleType) { + this.ruleType = Objects.requireNonNull(ruleType, "ruleType cannot be null"); + this.customRewriter = Objects.requireNonNull(rewriter, "customRewriter cannot be null"); + } + + @Override + public void execute(JobContext context) { + Set disableRules = Job.getDisableRules(context); + if (disableRules.contains(ruleType.name().toUpperCase(Locale.ROOT))) { + return; + } + Plan root = context.getCascadesContext().getRewritePlan(); + // COUNTER_TRACER.log(CounterEvent.of(Memo.get=-StateId(), CounterType.JOB_EXECUTION, group, logicalExpression, + // root)); + Plan rewrittenRoot = customRewriter.get().rewriteRoot(root, context); + + // don't remove this comment, it can help us to trace some bug when developing. + + // if (!root.deepEquals(rewrittenRoot)) { + // String traceBefore = root.treeString(); + // String traceAfter = root.treeString(); + // printTraceLog(ruleType, traceBefore, traceAfter); + // } + context.getCascadesContext().setRewritePlan(rewrittenRoot); + } + + @Override + public boolean isOnce() { + return false; + } + + private void printTraceLog(RuleType ruleType, String traceBefore, String traceAfter) { + System.out.println("========== " + getClass().getSimpleName() + " " + ruleType + + " ==========\nbefore:\n" + traceBefore + "\n\nafter:\n" + traceAfter + "\n"); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java new file mode 100644 index 0000000000..8e625b638d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.rewrite; + +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.jobs.JobType; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.trees.plans.Plan; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** PlanTreeRewriteBottomUpJob */ +public class PlanTreeRewriteBottomUpJob extends PlanTreeRewriteJob { + private static final String REWRITE_STATE_KEY = "rewrite_state"; + private RewriteJobContext rewriteJobContext; + private List rules; + + enum RewriteState { + ENSURE_CHILDREN_REWRITTEN, REWRITE_THIS, REWRITTEN + } + + public PlanTreeRewriteBottomUpJob(RewriteJobContext rewriteJobContext, JobContext context, List rules) { + super(JobType.TOP_DOWN_REWRITE, context); + this.rewriteJobContext = Objects.requireNonNull(rewriteJobContext, "rewriteContext cannot be null"); + this.rules = Objects.requireNonNull(rules, "rules cannot be null"); + } + + @Override + public void execute() { + // use childrenVisited to judge whether clear the state in the previous batch + boolean clearStatePhase = !rewriteJobContext.childrenVisited; + if (clearStatePhase) { + traverseClearState(); + return; + } + + Plan plan = rewriteJobContext.plan; + RewriteState state = getState(plan); + switch (state) { + case REWRITE_THIS: + rewriteThis(); + return; + case ENSURE_CHILDREN_REWRITTEN: + ensureChildrenRewritten(); + return; + case REWRITTEN: + rewriteJobContext.result = plan; + return; + default: + throw new IllegalStateException("Unknown rewrite state: " + state); + } + } + + private void traverseClearState() { + RewriteJobContext clearedStateContext = rewriteJobContext.withChildrenVisited(true); + setState(clearedStateContext.plan, RewriteState.REWRITE_THIS); + pushJob(new PlanTreeRewriteBottomUpJob(clearedStateContext, context, rules)); + + List children = clearedStateContext.plan.children(); + for (int i = children.size() - 1; i >= 0; i--) { + Plan child = children.get(i); + RewriteJobContext childRewriteJobContext = new RewriteJobContext( + child, clearedStateContext, i, false); + pushJob(new PlanTreeRewriteBottomUpJob(childRewriteJobContext, context, rules)); + } + } + + private void rewriteThis() { + Plan plan = linkChildren(rewriteJobContext.plan, rewriteJobContext.childrenContext); + RewriteResult rewriteResult = rewrite(plan, rules, rewriteJobContext); + if (rewriteResult.hasNewPlan) { + RewriteJobContext newJobContext = rewriteJobContext.withPlan(rewriteResult.plan); + RewriteState state = getState(rewriteResult.plan); + // some eliminate rule will return a rewritten plan + if (state == RewriteState.REWRITTEN) { + newJobContext.setResult(rewriteResult.plan); + return; + } + pushJob(new PlanTreeRewriteBottomUpJob(newJobContext, context, rules)); + setState(rewriteResult.plan, RewriteState.ENSURE_CHILDREN_REWRITTEN); + } else { + setState(rewriteResult.plan, RewriteState.REWRITTEN); + rewriteJobContext.setResult(rewriteResult.plan); + } + } + + private void ensureChildrenRewritten() { + Plan plan = rewriteJobContext.plan; + setState(plan, RewriteState.REWRITE_THIS); + pushJob(new PlanTreeRewriteBottomUpJob(rewriteJobContext, context, rules)); + + List children = plan.children(); + for (int i = children.size() - 1; i >= 0; i--) { + Plan child = children.get(i); + // some rule return new plan tree, which the number of new plan node > 1, + // we should transform this new plan nodes too. + RewriteJobContext childRewriteJobContext = new RewriteJobContext( + child, rewriteJobContext, i, false); + pushJob(new PlanTreeRewriteBottomUpJob(childRewriteJobContext, context, rules)); + } + } + + private static final RewriteState getState(Plan plan) { + Optional state = plan.getMutableState(REWRITE_STATE_KEY); + return state.orElse(RewriteState.ENSURE_CHILDREN_REWRITTEN); + } + + private static final void setState(Plan plan, RewriteState state) { + plan.setMutableState(REWRITE_STATE_KEY, state); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java new file mode 100644 index 0000000000..b54e6197d9 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.rewrite; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.Job; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.jobs.JobType; +import org.apache.doris.nereids.pattern.Pattern; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.trees.plans.Plan; + +import com.google.common.base.Preconditions; + +import java.util.List; + +/** PlanTreeRewriteJob */ +public abstract class PlanTreeRewriteJob extends Job { + public PlanTreeRewriteJob(JobType type, JobContext context) { + super(type, context); + } + + protected RewriteResult rewrite(Plan plan, List rules, RewriteJobContext rewriteJobContext) { + // boolean traceEnable = isTraceEnable(context); + boolean isRewriteRoot = rewriteJobContext.isRewriteRoot(); + CascadesContext cascadesContext = context.getCascadesContext(); + cascadesContext.setIsRewriteRoot(isRewriteRoot); + List validRules = getValidRules(rules); + for (Rule rule : validRules) { + Pattern pattern = (Pattern) rule.getPattern(); + if (pattern.matchPlanTree(plan)) { + List newPlans = rule.transform(plan, cascadesContext); + Preconditions.checkState(newPlans.size() == 1, + "Rewrite rule should generate one plan: " + rule.getRuleType()); + Plan newPlan = newPlans.get(0); + if (!newPlan.deepEquals(plan)) { + // don't remove this comment, it can help us to trace some bug when developing. + + // String traceBefore = null; + // if (traceEnable) { + // traceBefore = getCurrentPlanTreeString(); + // } + rewriteJobContext.result = newPlan; + context.setRewritten(true); + rule.acceptPlan(newPlan); + // if (traceEnable) { + // String traceAfter = getCurrentPlanTreeString(); + // printTraceLog(rule, traceBefore, traceAfter); + // } + return new RewriteResult(true, newPlan); + } + } + } + return new RewriteResult(false, plan); + } + + protected Plan linkChildrenAndParent(Plan plan, RewriteJobContext rewriteJobContext) { + Plan newPlan = linkChildren(plan, rewriteJobContext.childrenContext); + rewriteJobContext.setResult(newPlan); + return newPlan; + } + + protected Plan linkChildren(Plan plan, RewriteJobContext[] childrenContext) { + boolean changed = false; + Plan[] newChildren = new Plan[childrenContext.length]; + for (int i = 0; i < childrenContext.length; ++i) { + Plan result = childrenContext[i].result; + Plan oldChild = plan.child(i); + if (result != null && result != oldChild) { + newChildren[i] = result; + changed = true; + } else { + newChildren[i] = oldChild; + } + } + return changed ? plan.withChildren(newChildren) : plan; + } + + private String getCurrentPlanTreeString() { + return context.getCascadesContext() + .getCurrentRootRewriteJobContext().get() + .getNewestPlan() + .treeString(); + } + + private void printTraceLog(Rule rule, String traceBefore, String traceAfter) { + System.out.println("========== " + getClass().getSimpleName() + " " + rule.getRuleType() + + " ==========\nbefore:\n" + traceBefore + "\n\nafter:\n" + traceAfter + "\n"); + // LOGGER.info("========== {} {} ==========\nbefore:\n{}\n\nafter:\n{}\n", + // getClass().getSimpleName(), rule.getRuleType(), traceBefore, traceAfter); + } + + static class RewriteResult { + final boolean hasNewPlan; + final Plan plan; + + public RewriteResult(boolean hasNewPlan, Plan plan) { + this.hasNewPlan = hasNewPlan; + this.plan = plan; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java new file mode 100644 index 0000000000..e92dd15f27 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.rewrite; + +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.jobs.JobType; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.trees.plans.Plan; + +import java.util.List; +import java.util.Objects; + +/** PlanTreeRewriteTopDownJob */ +public class PlanTreeRewriteTopDownJob extends PlanTreeRewriteJob { + private RewriteJobContext rewriteJobContext; + private List rules; + + public PlanTreeRewriteTopDownJob(RewriteJobContext rewriteJobContext, JobContext context, List rules) { + super(JobType.TOP_DOWN_REWRITE, context); + this.rewriteJobContext = Objects.requireNonNull(rewriteJobContext, "rewriteContext cannot be null"); + this.rules = Objects.requireNonNull(rules, "rules cannot be null"); + } + + @Override + public void execute() { + if (!rewriteJobContext.childrenVisited) { + RewriteResult rewriteResult = rewrite(rewriteJobContext.plan, rules, rewriteJobContext); + if (rewriteResult.hasNewPlan) { + RewriteJobContext newContext = rewriteJobContext + .withPlanAndChildrenVisited(rewriteResult.plan, false); + pushJob(new PlanTreeRewriteTopDownJob(newContext, context, rules)); + return; + } + + RewriteJobContext newRewriteJobContext = rewriteJobContext.withChildrenVisited(true); + pushJob(new PlanTreeRewriteTopDownJob(newRewriteJobContext, context, rules)); + + List children = newRewriteJobContext.plan.children(); + for (int i = children.size() - 1; i >= 0; i--) { + RewriteJobContext childRewriteJobContext = new RewriteJobContext( + children.get(i), newRewriteJobContext, i, false); + pushJob(new PlanTreeRewriteTopDownJob(childRewriteJobContext, context, rules)); + } + } else { + Plan result = linkChildrenAndParent(rewriteJobContext.plan, rewriteJobContext); + if (rewriteJobContext.parentContext == null) { + context.getCascadesContext().setRewritePlan(result); + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteJobContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteJobContext.java new file mode 100644 index 0000000000..32af07573d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteJobContext.java @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.rewrite; + +import org.apache.doris.nereids.trees.plans.Plan; + +import javax.annotation.Nullable; + +/** RewriteJobContext */ +public class RewriteJobContext { + final boolean childrenVisited; + final RewriteJobContext parentContext; + final int childIndexInParentContext; + final Plan plan; + final RewriteJobContext[] childrenContext; + Plan result; + + /** RewriteJobContext */ + public RewriteJobContext(Plan plan, @Nullable RewriteJobContext parentContext, int childIndexInParentContext, + boolean childrenVisited) { + this.plan = plan; + this.parentContext = parentContext; + this.childIndexInParentContext = childIndexInParentContext; + this.childrenVisited = childrenVisited; + this.childrenContext = new RewriteJobContext[plan.arity()]; + if (parentContext != null) { + parentContext.childrenContext[childIndexInParentContext] = this; + } + } + + public void setResult(Plan result) { + this.result = result; + } + + public RewriteJobContext withChildrenVisited(boolean childrenVisited) { + return new RewriteJobContext(plan, parentContext, childIndexInParentContext, childrenVisited); + } + + public RewriteJobContext withPlan(Plan plan) { + return new RewriteJobContext(plan, parentContext, childIndexInParentContext, childrenVisited); + } + + public RewriteJobContext withPlanAndChildrenVisited(Plan plan, boolean childrenVisited) { + return new RewriteJobContext(plan, parentContext, childIndexInParentContext, childrenVisited); + } + + public boolean isRewriteRoot() { + return false; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java new file mode 100644 index 0000000000..06a9904799 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.rewrite; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.Job; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.jobs.JobType; +import org.apache.doris.nereids.jobs.RewriteJob; +import org.apache.doris.nereids.jobs.scheduler.JobStack; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.trees.plans.Plan; + +import java.util.List; +import java.util.Objects; + +/** RootPlanTreeRewriteJob */ +public class RootPlanTreeRewriteJob implements RewriteJob { + private final List rules; + private final RewriteJobBuilder rewriteJobBuilder; + private final boolean once; + + public RootPlanTreeRewriteJob(List rules, RewriteJobBuilder rewriteJobBuilder, boolean once) { + this.rules = Objects.requireNonNull(rules, "rules cannot be null"); + this.rewriteJobBuilder = Objects.requireNonNull(rewriteJobBuilder, "rewriteJobBuilder cannot be null"); + this.once = once; + } + + @Override + public void execute(JobContext context) { + CascadesContext cascadesContext = context.getCascadesContext(); + // get plan from the cascades context + Plan root = cascadesContext.getRewritePlan(); + // write rewritten root plan to cascades context by the RootRewriteJobContext + RootRewriteJobContext rewriteJobContext = new RootRewriteJobContext(root, false, context); + Job rewriteJob = rewriteJobBuilder.build(rewriteJobContext, context, rules); + + context.getScheduleContext().pushJob(rewriteJob); + cascadesContext.getJobScheduler().executeJobPool(cascadesContext); + + cascadesContext.setCurrentRootRewriteJobContext(null); + } + + @Override + public boolean isOnce() { + return once; + } + + /** RewriteJobBuilder */ + public interface RewriteJobBuilder { + Job build(RewriteJobContext rewriteJobContext, JobContext jobContext, List rules); + } + + /** RootRewriteJobContext */ + public static class RootRewriteJobContext extends RewriteJobContext { + private JobContext jobContext; + + RootRewriteJobContext(Plan plan, boolean childrenVisited, JobContext jobContext) { + super(plan, null, -1, childrenVisited); + this.jobContext = Objects.requireNonNull(jobContext, "jobContext cannot be null"); + jobContext.getCascadesContext().setCurrentRootRewriteJobContext(this); + } + + @Override + public boolean isRewriteRoot() { + return true; + } + + @Override + public void setResult(Plan result) { + jobContext.getCascadesContext().setRewritePlan(result); + } + + @Override + public RewriteJobContext withChildrenVisited(boolean childrenVisited) { + return new RootRewriteJobContext(plan, childrenVisited, jobContext); + } + + @Override + public RewriteJobContext withPlan(Plan plan) { + return new RootRewriteJobContext(plan, childrenVisited, jobContext); + } + + @Override + public RewriteJobContext withPlanAndChildrenVisited(Plan plan, boolean childrenVisited) { + return new RootRewriteJobContext(plan, childrenVisited, jobContext); + } + + /** linkChildren */ + public Plan getNewestPlan() { + JobStack jobStack = new JobStack(); + LinkPlanJob linkPlanJob = new LinkPlanJob( + jobContext, this, null, false, jobStack); + jobStack.push(linkPlanJob); + while (!jobStack.isEmpty()) { + Job job = jobStack.pop(); + job.execute(); + } + return linkPlanJob.result; + } + } + + /** use to assemble the rewriting plan */ + private static class LinkPlanJob extends Job { + LinkPlanJob parentJob; + RewriteJobContext rewriteJobContext; + Plan[] childrenResult; + Plan result; + boolean linked; + JobStack jobStack; + + private LinkPlanJob(JobContext context, RewriteJobContext rewriteJobContext, + LinkPlanJob parentJob, boolean linked, JobStack jobStack) { + super(JobType.LINK_PLAN, context); + this.rewriteJobContext = rewriteJobContext; + this.parentJob = parentJob; + this.linked = linked; + this.childrenResult = new Plan[rewriteJobContext.plan.arity()]; + this.jobStack = jobStack; + } + + @Override + public void execute() { + if (!linked) { + linked = true; + jobStack.push(this); + for (int i = rewriteJobContext.childrenContext.length - 1; i >= 0; i--) { + RewriteJobContext childContext = rewriteJobContext.childrenContext[i]; + if (childContext != null) { + jobStack.push(new LinkPlanJob(context, childContext, this, false, jobStack)); + } + } + } else if (rewriteJobContext.result != null) { + linkResult(rewriteJobContext.result); + } else { + Plan[] newChildren = new Plan[childrenResult.length]; + for (int i = 0; i < newChildren.length; i++) { + Plan childResult = childrenResult[i]; + if (childResult == null) { + childResult = rewriteJobContext.plan.child(i); + } + newChildren[i] = childResult; + } + linkResult(rewriteJobContext.plan.withChildren(newChildren)); + } + } + + private void linkResult(Plan result) { + if (parentJob != null) { + parentJob.childrenResult[rewriteJobContext.childIndexInParentContext] = result; + } else { + this.result = result; + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/VisitorRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/VisitorRewriteJob.java deleted file mode 100644 index 4836803be1..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/VisitorRewriteJob.java +++ /dev/null @@ -1,69 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.jobs.rewrite; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.jobs.Job; -import org.apache.doris.nereids.jobs.JobContext; -import org.apache.doris.nereids.jobs.JobType; -import org.apache.doris.nereids.memo.Group; -import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.memo.Memo; -import org.apache.doris.nereids.metrics.CounterType; -import org.apache.doris.nereids.metrics.event.CounterEvent; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; - -import java.util.Locale; -import java.util.Objects; - -/** - * Use visitor to rewrite the plan. - */ -public class VisitorRewriteJob extends Job { - private final RuleType ruleType; - private final Group group; - - private final DefaultPlanRewriter planRewriter; - - /** - * Constructor. - */ - public VisitorRewriteJob(CascadesContext cascadesContext, - DefaultPlanRewriter rewriter, RuleType ruleType) { - super(JobType.VISITOR_REWRITE, cascadesContext.getCurrentJobContext(), true); - this.ruleType = Objects.requireNonNull(ruleType, "ruleType cannot be null"); - this.group = Objects.requireNonNull(cascadesContext.getMemo().getRoot(), "group cannot be null"); - this.planRewriter = Objects.requireNonNull(rewriter, "planRewriter cannot be null"); - } - - @Override - public void execute() { - if (disableRules.contains(ruleType.name().toUpperCase(Locale.ROOT))) { - return; - } - GroupExpression logicalExpression = group.getLogicalExpression(); - Plan root = context.getCascadesContext().getMemo().copyOut(logicalExpression, true); - COUNTER_TRACER.log(CounterEvent.of(Memo.getStateId(), CounterType.JOB_EXECUTION, group, logicalExpression, - root)); - Plan rewrittenRoot = root.accept(planRewriter, context); - context.getCascadesContext().getMemo().copyIn(rewrittenRoot, group, true); - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/JobScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/JobScheduler.java index 97dcd96d52..4b26c0decf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/JobScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/JobScheduler.java @@ -17,15 +17,9 @@ package org.apache.doris.nereids.jobs.scheduler; -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.jobs.Job; - /** * Scheduler to schedule jobs in Nereids. */ public interface JobScheduler { - void executeJob(Job job, CascadesContext context); - - void executeJobPool(CascadesContext cascadesContext) throws AnalysisException; + void executeJobPool(ScheduleContext scheduleContext); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeSubqueryRulesJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/ScheduleContext.java similarity index 60% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeSubqueryRulesJob.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/ScheduleContext.java index 1e41652e2b..f7f60119f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeSubqueryRulesJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/ScheduleContext.java @@ -15,20 +15,14 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.jobs.batch; +package org.apache.doris.nereids.jobs.scheduler; -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.analysis.AnalyzeSubquery; +import org.apache.doris.nereids.jobs.Job; -import com.google.common.collect.ImmutableList; +/** ScheduleContext */ +public interface ScheduleContext { -/** - * Analyze subquery. - */ -public class AnalyzeSubqueryRulesJob extends BatchRulesJob { - public AnalyzeSubqueryRulesJob(CascadesContext cascadesContext) { - super(cascadesContext); - rulesJob.addAll(ImmutableList.of( - bottomUpBatch(ImmutableList.of(new AnalyzeSubquery())))); - } + JobPool getJobPool(); + + void pushJob(Job job); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java index 8bd104c010..a93270d48e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java @@ -17,8 +17,6 @@ package org.apache.doris.nereids.jobs.scheduler; -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.jobs.Job; /** @@ -26,13 +24,8 @@ import org.apache.doris.nereids.jobs.Job; */ public class SimpleJobScheduler implements JobScheduler { @Override - public void executeJob(Job job, CascadesContext context) { - - } - - @Override - public void executeJobPool(CascadesContext cascadesContext) throws AnalysisException { - JobPool pool = cascadesContext.getJobPool(); + public void executeJobPool(ScheduleContext scheduleContext) { + JobPool pool = scheduleContext.getJobPool(); while (!pool.isEmpty()) { Job job = pool.pop(); job.execute(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java index bd53cb86df..88e20f34e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java @@ -393,7 +393,16 @@ public class Group { @Override public String toString() { - return "Group[" + groupId + "]"; + StringBuilder str = new StringBuilder("Group[" + groupId + "]\n"); + str.append("logical expressions:\n"); + for (GroupExpression logicalExpression : logicalExpressions) { + str.append(" ").append(logicalExpression).append("\n"); + } + str.append("physical expressions:\n"); + for (GroupExpression physicalExpression : physicalExpressions) { + str.append(" ").append(physicalExpression).append("\n"); + } + return str.toString(); } /** @@ -404,7 +413,7 @@ public class Group { public String treeString() { Function toString = obj -> { if (obj instanceof Group) { - return obj.toString(); + return "Group[" + ((Group) obj).groupId + "]"; } else if (obj instanceof GroupExpression) { return ((GroupExpression) obj).getPlan().toString(); } else if (obj instanceof Pair) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java index a64db02847..304a2da025 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java @@ -216,11 +216,11 @@ public class Memo { * Utility function to create a new {@link CascadesContext} with this Memo. */ public CascadesContext newCascadesContext(StatementContext statementContext) { - return new CascadesContext(this, statementContext, PhysicalProperties.ANY); + return new CascadesContext(null, this, statementContext, PhysicalProperties.ANY); } public CascadesContext newCascadesContext(StatementContext statementContext, CTEContext cteContext) { - return new CascadesContext(this, statementContext, cteContext, PhysicalProperties.ANY); + return new CascadesContext(null, this, statementContext, cteContext, PhysicalProperties.ANY); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MatchingContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MatchingContext.java index de067a427d..e889159d63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MatchingContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MatchingContext.java @@ -49,4 +49,8 @@ public class MatchingContext { this.connectContext = cascadesContext.getConnectContext(); this.cteContext = cascadesContext.getCteContext(); } + + public boolean isRewriteRoot() { + return cascadesContext.isRewriteRoot(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MemoPatterns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MemoPatterns.java new file mode 100644 index 0000000000..fa9d191f5e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MemoPatterns.java @@ -0,0 +1,322 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.pattern; + +import org.apache.doris.nereids.trees.plans.BinaryPlan; +import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.LeafPlan; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.UnaryPlan; +import org.apache.doris.nereids.trees.plans.algebra.Aggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalBinary; +import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; +import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; +import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnary; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.trees.plans.physical.PhysicalBinary; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLeaf; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalUnary; + +import java.util.Arrays; + +/** MemoPatterns */ +public interface MemoPatterns extends Patterns { + + default PatternDescriptor group() { + return new PatternDescriptor<>(Pattern.GROUP, defaultPromise()); + } + + default PatternDescriptor multiGroup() { + return new PatternDescriptor<>(Pattern.MULTI_GROUP, defaultPromise()); + } + + /* abstract plan operator patterns */ + + /** + * create a leafPlan pattern. + */ + default PatternDescriptor leafPlan() { + return new PatternDescriptor(new TypePattern(LeafPlan.class), defaultPromise()); + } + + /** + * create a unaryPlan pattern. + */ + default PatternDescriptor> unaryPlan() { + return new PatternDescriptor(new TypePattern(UnaryPlan.class, Pattern.GROUP), defaultPromise()); + } + + /** + * create a unaryPlan pattern. + */ + default PatternDescriptor> + unaryPlan(PatternDescriptor child) { + return new PatternDescriptor(new TypePattern(UnaryPlan.class, child.pattern), defaultPromise()); + } + + /** + * create a binaryPlan pattern. + */ + default PatternDescriptor> binaryPlan() { + return new PatternDescriptor( + new TypePattern(BinaryPlan.class, Pattern.GROUP, Pattern.GROUP), + defaultPromise() + ); + } + + /** + * create a binaryPlan pattern. + */ + default + PatternDescriptor> binaryPlan( + PatternDescriptor leftChild, + PatternDescriptor rightChild) { + return new PatternDescriptor( + new TypePattern(BinaryPlan.class, leftChild.pattern, rightChild.pattern), + defaultPromise() + ); + } + + /* abstract logical plan patterns */ + + /** + * create a logicalPlan pattern. + */ + default PatternDescriptor logicalPlan() { + return new PatternDescriptor(new TypePattern(LogicalPlan.class, multiGroup().pattern), defaultPromise()); + } + + /** + * create a logicalLeaf pattern. + */ + default PatternDescriptor logicalLeaf() { + return new PatternDescriptor(new TypePattern(LogicalLeaf.class), defaultPromise()); + } + + /** + * create a logicalUnary pattern. + */ + default PatternDescriptor> logicalUnary() { + return new PatternDescriptor(new TypePattern(LogicalUnary.class, Pattern.GROUP), defaultPromise()); + } + + /** + * create a logicalUnary pattern. + */ + default PatternDescriptor> + logicalUnary(PatternDescriptor child) { + return new PatternDescriptor(new TypePattern(LogicalUnary.class, child.pattern), defaultPromise()); + } + + /** + * create a logicalBinary pattern. + */ + default PatternDescriptor> logicalBinary() { + return new PatternDescriptor( + new TypePattern(LogicalBinary.class, Pattern.GROUP, Pattern.GROUP), + defaultPromise() + ); + } + + /** + * create a logicalBinary pattern. + */ + default + PatternDescriptor> + logicalBinary( + PatternDescriptor leftChild, + PatternDescriptor rightChild) { + return new PatternDescriptor( + new TypePattern(LogicalBinary.class, leftChild.pattern, rightChild.pattern), + defaultPromise() + ); + } + + /** + * create a logicalRelation pattern. + */ + default PatternDescriptor logicalRelation() { + return new PatternDescriptor(new TypePattern(LogicalRelation.class), defaultPromise()); + } + + /** + * create a logicalSetOperation pattern. + */ + default PatternDescriptor + logicalSetOperation( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalSetOperation.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalSetOperation group. + */ + default PatternDescriptor logicalSetOperation() { + return new PatternDescriptor( + new TypePattern(LogicalSetOperation.class, multiGroup().pattern), + defaultPromise()); + } + + /** + * create a logicalUnion pattern. + */ + default PatternDescriptor + logicalUnion( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalUnion.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalUnion group. + */ + default PatternDescriptor logicalUnion() { + return new PatternDescriptor( + new TypePattern(LogicalUnion.class, multiGroup().pattern), + defaultPromise()); + } + + /** + * create a logicalExcept pattern. + */ + default PatternDescriptor + logicalExcept( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalExcept.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalExcept group. + */ + default PatternDescriptor logicalExcept() { + return new PatternDescriptor( + new TypePattern(LogicalExcept.class, multiGroup().pattern), + defaultPromise()); + } + + /** + * create a logicalUnion pattern. + */ + default PatternDescriptor + logicalIntersect( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalIntersect.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalUnion group. + */ + default PatternDescriptor logicalIntersect() { + return new PatternDescriptor( + new TypePattern(LogicalIntersect.class, multiGroup().pattern), + defaultPromise()); + } + + /* abstract physical plan patterns */ + + /** + * create a physicalLeaf pattern. + */ + default PatternDescriptor physicalLeaf() { + return new PatternDescriptor(new TypePattern(PhysicalLeaf.class), defaultPromise()); + } + + /** + * create a physicalUnary pattern. + */ + default PatternDescriptor> physicalUnary() { + return new PatternDescriptor(new TypePattern(PhysicalUnary.class, Pattern.GROUP), defaultPromise()); + } + + /** + * create a physicalUnary pattern. + */ + default PatternDescriptor> + physicalUnary(PatternDescriptor child) { + return new PatternDescriptor(new TypePattern(PhysicalUnary.class, child.pattern), defaultPromise()); + } + + /** + * create a physicalBinary pattern. + */ + default PatternDescriptor> physicalBinary() { + return new PatternDescriptor( + new TypePattern(PhysicalBinary.class, Pattern.GROUP, Pattern.GROUP), + defaultPromise() + ); + } + + /** + * create a physicalBinary pattern. + */ + default + PatternDescriptor> + physicalBinary( + PatternDescriptor leftChild, + PatternDescriptor rightChild) { + return new PatternDescriptor( + new TypePattern(PhysicalBinary.class, leftChild.pattern, rightChild.pattern), + defaultPromise() + ); + } + + /** + * create a physicalRelation pattern. + */ + default PatternDescriptor physicalRelation() { + return new PatternDescriptor(new TypePattern(PhysicalRelation.class), defaultPromise()); + } + + /** + * create a aggregate pattern. + */ + default PatternDescriptor> aggregate() { + return new PatternDescriptor(new TypePattern(Aggregate.class, Pattern.GROUP), defaultPromise()); + } + + /** + * create a aggregate pattern. + */ + default PatternDescriptor> aggregate(PatternDescriptor child) { + return new PatternDescriptor(new TypePattern(Aggregate.class, child.pattern), defaultPromise()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Pattern.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Pattern.java index 31c2713aa3..c47dcd6a72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Pattern.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Pattern.java @@ -75,7 +75,7 @@ public class Pattern * @param predicates custom matching predicate * @param children sub pattern */ - private Pattern(PatternType patternType, PlanType planType, + protected Pattern(PatternType patternType, PlanType planType, List> predicates, Pattern... children) { super(children); this.patternType = patternType; @@ -134,6 +134,35 @@ public class Pattern return patternType == PatternType.MULTI; } + /** matchPlan */ + public boolean matchPlanTree(Plan plan) { + if (!matchRoot(plan)) { + return false; + } + int childPatternNum = arity(); + if (childPatternNum != plan.arity() && childPatternNum > 0 && child(childPatternNum - 1) != MULTI) { + return false; + } + switch (patternType) { + case ANY: + case MULTI: + return matchPredicates((TYPE) plan); + default: + } + if (this instanceof SubTreePattern) { + return matchPredicates((TYPE) plan); + } + List childrenPlan = plan.children(); + for (int i = 0; i < childrenPlan.size(); i++) { + Plan child = childrenPlan.get(i); + Pattern childPattern = child(Math.min(i, childPatternNum - 1)); + if (!childPattern.matchPlanTree(child)) { + return false; + } + } + return matchPredicates((TYPE) plan); + } + /** * Return ture if current Pattern match Plan in params. * @@ -161,7 +190,13 @@ public class Pattern * @return true if all predicates matched */ public boolean matchPredicates(TYPE root) { - return predicates.stream().allMatch(predicate -> predicate.test(root)); + // use loop to speed up + for (Predicate predicate : predicates) { + if (!predicate.test(root)) { + return false; + } + } + return true; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Patterns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Patterns.java index 765c087f19..149677c617 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Patterns.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Patterns.java @@ -18,27 +18,7 @@ package org.apache.doris.nereids.pattern; import org.apache.doris.nereids.rules.RulePromise; -import org.apache.doris.nereids.trees.plans.BinaryPlan; -import org.apache.doris.nereids.trees.plans.GroupPlan; -import org.apache.doris.nereids.trees.plans.LeafPlan; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.UnaryPlan; -import org.apache.doris.nereids.trees.plans.algebra.Aggregate; -import org.apache.doris.nereids.trees.plans.logical.LogicalBinary; -import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; -import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; -import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; -import org.apache.doris.nereids.trees.plans.logical.LogicalUnary; -import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; -import org.apache.doris.nereids.trees.plans.physical.PhysicalBinary; -import org.apache.doris.nereids.trees.plans.physical.PhysicalLeaf; -import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; -import org.apache.doris.nereids.trees.plans.physical.PhysicalUnary; - -import java.util.Arrays; /** * An interface provided some PatternDescriptor. @@ -59,285 +39,7 @@ public interface Patterns { return new PatternDescriptor<>(Pattern.MULTI, defaultPromise()); } - default PatternDescriptor group() { - return new PatternDescriptor<>(Pattern.GROUP, defaultPromise()); - } - - default PatternDescriptor multiGroup() { - return new PatternDescriptor<>(Pattern.MULTI_GROUP, defaultPromise()); - } - default PatternDescriptor subTree(Class... subTreeNodeTypes) { return new PatternDescriptor<>(new SubTreePattern(subTreeNodeTypes), defaultPromise()); } - - /* abstract plan operator patterns */ - - /** - * create a leafPlan pattern. - */ - default PatternDescriptor leafPlan() { - return new PatternDescriptor(new TypePattern(LeafPlan.class), defaultPromise()); - } - - /** - * create a unaryPlan pattern. - */ - default PatternDescriptor> unaryPlan() { - return new PatternDescriptor(new TypePattern(UnaryPlan.class, Pattern.GROUP), defaultPromise()); - } - - /** - * create a unaryPlan pattern. - */ - default PatternDescriptor> - unaryPlan(PatternDescriptor child) { - return new PatternDescriptor(new TypePattern(UnaryPlan.class, child.pattern), defaultPromise()); - } - - /** - * create a binaryPlan pattern. - */ - default PatternDescriptor> binaryPlan() { - return new PatternDescriptor( - new TypePattern(BinaryPlan.class, Pattern.GROUP, Pattern.GROUP), - defaultPromise() - ); - } - - /** - * create a binaryPlan pattern. - */ - default - PatternDescriptor> binaryPlan( - PatternDescriptor leftChild, - PatternDescriptor rightChild) { - return new PatternDescriptor( - new TypePattern(BinaryPlan.class, leftChild.pattern, rightChild.pattern), - defaultPromise() - ); - } - - /* abstract logical plan patterns */ - - /** - * create a logicalPlan pattern. - */ - default PatternDescriptor logicalPlan() { - return new PatternDescriptor(new TypePattern(LogicalPlan.class, multiGroup().pattern), defaultPromise()); - } - - /** - * create a logicalLeaf pattern. - */ - default PatternDescriptor logicalLeaf() { - return new PatternDescriptor(new TypePattern(LogicalLeaf.class), defaultPromise()); - } - - /** - * create a logicalUnary pattern. - */ - default PatternDescriptor> logicalUnary() { - return new PatternDescriptor(new TypePattern(LogicalUnary.class, Pattern.GROUP), defaultPromise()); - } - - /** - * create a logicalUnary pattern. - */ - default PatternDescriptor> - logicalUnary(PatternDescriptor child) { - return new PatternDescriptor(new TypePattern(LogicalUnary.class, child.pattern), defaultPromise()); - } - - /** - * create a logicalBinary pattern. - */ - default PatternDescriptor> logicalBinary() { - return new PatternDescriptor( - new TypePattern(LogicalBinary.class, Pattern.GROUP, Pattern.GROUP), - defaultPromise() - ); - } - - /** - * create a logicalBinary pattern. - */ - default - PatternDescriptor> - logicalBinary( - PatternDescriptor leftChild, - PatternDescriptor rightChild) { - return new PatternDescriptor( - new TypePattern(LogicalBinary.class, leftChild.pattern, rightChild.pattern), - defaultPromise() - ); - } - - /** - * create a logicalRelation pattern. - */ - default PatternDescriptor logicalRelation() { - return new PatternDescriptor(new TypePattern(LogicalRelation.class), defaultPromise()); - } - - /** - * create a logicalSetOperation pattern. - */ - default PatternDescriptor - logicalSetOperation( - PatternDescriptor... children) { - return new PatternDescriptor( - new TypePattern(LogicalSetOperation.class, - Arrays.stream(children) - .map(PatternDescriptor::getPattern) - .toArray(Pattern[]::new)), - defaultPromise()); - } - - /** - * create a logicalSetOperation group. - */ - default PatternDescriptor logicalSetOperation() { - return new PatternDescriptor( - new TypePattern(LogicalSetOperation.class, multiGroup().pattern), - defaultPromise()); - } - - /** - * create a logicalUnion pattern. - */ - default PatternDescriptor - logicalUnion( - PatternDescriptor... children) { - return new PatternDescriptor( - new TypePattern(LogicalUnion.class, - Arrays.stream(children) - .map(PatternDescriptor::getPattern) - .toArray(Pattern[]::new)), - defaultPromise()); - } - - /** - * create a logicalUnion group. - */ - default PatternDescriptor logicalUnion() { - return new PatternDescriptor( - new TypePattern(LogicalUnion.class, multiGroup().pattern), - defaultPromise()); - } - - /** - * create a logicalExcept pattern. - */ - default PatternDescriptor - logicalExcept( - PatternDescriptor... children) { - return new PatternDescriptor( - new TypePattern(LogicalExcept.class, - Arrays.stream(children) - .map(PatternDescriptor::getPattern) - .toArray(Pattern[]::new)), - defaultPromise()); - } - - /** - * create a logicalExcept group. - */ - default PatternDescriptor logicalExcept() { - return new PatternDescriptor( - new TypePattern(LogicalExcept.class, multiGroup().pattern), - defaultPromise()); - } - - /** - * create a logicalUnion pattern. - */ - default PatternDescriptor - logicalIntersect( - PatternDescriptor... children) { - return new PatternDescriptor( - new TypePattern(LogicalIntersect.class, - Arrays.stream(children) - .map(PatternDescriptor::getPattern) - .toArray(Pattern[]::new)), - defaultPromise()); - } - - /** - * create a logicalUnion group. - */ - default PatternDescriptor logicalIntersect() { - return new PatternDescriptor( - new TypePattern(LogicalIntersect.class, multiGroup().pattern), - defaultPromise()); - } - - /* abstract physical plan patterns */ - - /** - * create a physicalLeaf pattern. - */ - default PatternDescriptor physicalLeaf() { - return new PatternDescriptor(new TypePattern(PhysicalLeaf.class), defaultPromise()); - } - - /** - * create a physicalUnary pattern. - */ - default PatternDescriptor> physicalUnary() { - return new PatternDescriptor(new TypePattern(PhysicalUnary.class, Pattern.GROUP), defaultPromise()); - } - - /** - * create a physicalUnary pattern. - */ - default PatternDescriptor> - physicalUnary(PatternDescriptor child) { - return new PatternDescriptor(new TypePattern(PhysicalUnary.class, child.pattern), defaultPromise()); - } - - /** - * create a physicalBinary pattern. - */ - default PatternDescriptor> physicalBinary() { - return new PatternDescriptor( - new TypePattern(PhysicalBinary.class, Pattern.GROUP, Pattern.GROUP), - defaultPromise() - ); - } - - /** - * create a physicalBinary pattern. - */ - default - PatternDescriptor> - physicalBinary( - PatternDescriptor leftChild, - PatternDescriptor rightChild) { - return new PatternDescriptor( - new TypePattern(PhysicalBinary.class, leftChild.pattern, rightChild.pattern), - defaultPromise() - ); - } - - /** - * create a physicalRelation pattern. - */ - default PatternDescriptor physicalRelation() { - return new PatternDescriptor(new TypePattern(PhysicalRelation.class), defaultPromise()); - } - - /** - * create a aggregate pattern. - */ - default PatternDescriptor> aggregate() { - return new PatternDescriptor(new TypePattern(Aggregate.class, Pattern.GROUP), defaultPromise()); - } - - /** - * create a aggregate pattern. - */ - default PatternDescriptor> aggregate(PatternDescriptor child) { - return new PatternDescriptor(new TypePattern(Aggregate.class, child.pattern), defaultPromise()); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PlanPatterns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PlanPatterns.java new file mode 100644 index 0000000000..ffe096890a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PlanPatterns.java @@ -0,0 +1,314 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.pattern; + +import org.apache.doris.nereids.trees.plans.BinaryPlan; +import org.apache.doris.nereids.trees.plans.LeafPlan; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.UnaryPlan; +import org.apache.doris.nereids.trees.plans.algebra.Aggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalBinary; +import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; +import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; +import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnary; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.trees.plans.physical.PhysicalBinary; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLeaf; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalUnary; + +import java.util.Arrays; + +/** PlanPatterns */ +public interface PlanPatterns extends Patterns { + + /* abstract plan operator patterns */ + + /** + * create a leafPlan pattern. + */ + default PatternDescriptor leafPlan() { + return new PatternDescriptor(new TypePattern(LeafPlan.class), defaultPromise()); + } + + /** + * create a unaryPlan pattern. + */ + default PatternDescriptor> unaryPlan() { + return new PatternDescriptor(new TypePattern(UnaryPlan.class, Pattern.ANY), defaultPromise()); + } + + /** + * create a unaryPlan pattern. + */ + default PatternDescriptor> + unaryPlan(PatternDescriptor child) { + return new PatternDescriptor(new TypePattern(UnaryPlan.class, child.pattern), defaultPromise()); + } + + /** + * create a binaryPlan pattern. + */ + default PatternDescriptor> binaryPlan() { + return new PatternDescriptor( + new TypePattern(BinaryPlan.class, Pattern.ANY, Pattern.ANY), + defaultPromise() + ); + } + + /** + * create a binaryPlan pattern. + */ + default + PatternDescriptor> binaryPlan( + PatternDescriptor leftChild, + PatternDescriptor rightChild) { + return new PatternDescriptor( + new TypePattern(BinaryPlan.class, leftChild.pattern, rightChild.pattern), + defaultPromise() + ); + } + + /* abstract logical plan patterns */ + + /** + * create a logicalPlan pattern. + */ + default PatternDescriptor logicalPlan() { + return new PatternDescriptor(new TypePattern(LogicalPlan.class, multi().pattern), defaultPromise()); + } + + /** + * create a logicalLeaf pattern. + */ + default PatternDescriptor logicalLeaf() { + return new PatternDescriptor(new TypePattern(LogicalLeaf.class), defaultPromise()); + } + + /** + * create a logicalUnary pattern. + */ + default PatternDescriptor> logicalUnary() { + return new PatternDescriptor(new TypePattern(LogicalUnary.class, Pattern.ANY), defaultPromise()); + } + + /** + * create a logicalUnary pattern. + */ + default PatternDescriptor> + logicalUnary(PatternDescriptor child) { + return new PatternDescriptor(new TypePattern(LogicalUnary.class, child.pattern), defaultPromise()); + } + + /** + * create a logicalBinary pattern. + */ + default PatternDescriptor> logicalBinary() { + return new PatternDescriptor( + new TypePattern(LogicalBinary.class, Pattern.ANY, Pattern.ANY), + defaultPromise() + ); + } + + /** + * create a logicalBinary pattern. + */ + default + PatternDescriptor> + logicalBinary( + PatternDescriptor leftChild, + PatternDescriptor rightChild) { + return new PatternDescriptor( + new TypePattern(LogicalBinary.class, leftChild.pattern, rightChild.pattern), + defaultPromise() + ); + } + + /** + * create a logicalRelation pattern. + */ + default PatternDescriptor logicalRelation() { + return new PatternDescriptor(new TypePattern(LogicalRelation.class), defaultPromise()); + } + + /** + * create a logicalSetOperation pattern. + */ + default PatternDescriptor + logicalSetOperation( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalSetOperation.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalSetOperation multi. + */ + default PatternDescriptor logicalSetOperation() { + return new PatternDescriptor( + new TypePattern(LogicalSetOperation.class, multi().pattern), + defaultPromise()); + } + + /** + * create a logicalUnion pattern. + */ + default PatternDescriptor + logicalUnion( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalUnion.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalUnion multi. + */ + default PatternDescriptor logicalUnion() { + return new PatternDescriptor( + new TypePattern(LogicalUnion.class, multi().pattern), + defaultPromise()); + } + + /** + * create a logicalExcept pattern. + */ + default PatternDescriptor + logicalExcept( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalExcept.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalExcept multi. + */ + default PatternDescriptor logicalExcept() { + return new PatternDescriptor( + new TypePattern(LogicalExcept.class, multi().pattern), + defaultPromise()); + } + + /** + * create a logicalUnion pattern. + */ + default PatternDescriptor + logicalIntersect( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalIntersect.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalUnion multi. + */ + default PatternDescriptor logicalIntersect() { + return new PatternDescriptor( + new TypePattern(LogicalIntersect.class, multi().pattern), + defaultPromise()); + } + + /* abstract physical plan patterns */ + + /** + * create a physicalLeaf pattern. + */ + default PatternDescriptor physicalLeaf() { + return new PatternDescriptor(new TypePattern(PhysicalLeaf.class), defaultPromise()); + } + + /** + * create a physicalUnary pattern. + */ + default PatternDescriptor> physicalUnary() { + return new PatternDescriptor(new TypePattern(PhysicalUnary.class, Pattern.ANY), defaultPromise()); + } + + /** + * create a physicalUnary pattern. + */ + default PatternDescriptor> + physicalUnary(PatternDescriptor child) { + return new PatternDescriptor(new TypePattern(PhysicalUnary.class, child.pattern), defaultPromise()); + } + + /** + * create a physicalBinary pattern. + */ + default PatternDescriptor> physicalBinary() { + return new PatternDescriptor( + new TypePattern(PhysicalBinary.class, Pattern.ANY, Pattern.ANY), + defaultPromise() + ); + } + + /** + * create a physicalBinary pattern. + */ + default + PatternDescriptor> + physicalBinary( + PatternDescriptor leftChild, + PatternDescriptor rightChild) { + return new PatternDescriptor( + new TypePattern(PhysicalBinary.class, leftChild.pattern, rightChild.pattern), + defaultPromise() + ); + } + + /** + * create a physicalRelation pattern. + */ + default PatternDescriptor physicalRelation() { + return new PatternDescriptor(new TypePattern(PhysicalRelation.class), defaultPromise()); + } + + /** + * create a aggregate pattern. + */ + default PatternDescriptor> aggregate() { + return new PatternDescriptor(new TypePattern(Aggregate.class, Pattern.ANY), defaultPromise()); + } + + /** + * create a aggregate pattern. + */ + default PatternDescriptor> aggregate(PatternDescriptor child) { + return new PatternDescriptor(new TypePattern(Aggregate.class, child.pattern), defaultPromise()); + } +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ProxyPattern.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ProxyPattern.java new file mode 100644 index 0000000000..9f6f009371 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ProxyPattern.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.pattern; + +import org.apache.doris.nereids.trees.plans.Plan; + +/** ProxyPattern */ +public class ProxyPattern extends Pattern { + protected final Pattern pattern; + + public ProxyPattern(Pattern pattern) { + super(pattern.getPlanType(), pattern.children()); + this.pattern = pattern; + } + + @Override + public boolean matchPlanTree(Plan plan) { + return pattern.matchPlanTree(plan); + } + + @Override + public boolean matchRoot(Plan plan) { + return pattern.matchRoot(plan); + } + + @Override + public boolean matchPredicates(TYPE root) { + return pattern.matchPredicates(root); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalBinaryPatternGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalBinaryPatternGenerator.java index ff47009f6c..bec3efa270 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalBinaryPatternGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalBinaryPatternGenerator.java @@ -26,13 +26,13 @@ import java.util.TreeSet; public class LogicalBinaryPatternGenerator extends PatternGenerator { public LogicalBinaryPatternGenerator(PatternGeneratorAnalyzer analyzer, - ClassDeclaration opType, Set parentClass) { - super(analyzer, opType, parentClass); + ClassDeclaration opType, Set parentClass, boolean isMemoPattern) { + super(analyzer, opType, parentClass, isMemoPattern); } @Override public String genericType() { - return "<" + opType.name + ">"; + return "<" + opType.name + "<" + childType() + ", " + childType() + ">>"; } @Override @@ -44,7 +44,9 @@ public class LogicalBinaryPatternGenerator extends PatternGenerator { public Set getImports() { Set imports = new TreeSet<>(); imports.add(opType.getFullQualifiedName()); - imports.add("org.apache.doris.nereids.trees.plans.GroupPlan"); + if (isMemoPattern) { + imports.add("org.apache.doris.nereids.trees.plans.GroupPlan"); + } imports.add("org.apache.doris.nereids.trees.plans.Plan"); enumFieldPatternInfos.stream() .map(info -> info.enumFullName) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalLeafPatternGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalLeafPatternGenerator.java index 19580c6e37..fd7b30a8e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalLeafPatternGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalLeafPatternGenerator.java @@ -26,8 +26,8 @@ import java.util.TreeSet; public class LogicalLeafPatternGenerator extends PatternGenerator { public LogicalLeafPatternGenerator(PatternGeneratorAnalyzer analyzer, - ClassDeclaration opType, Set parentClass) { - super(analyzer, opType, parentClass); + ClassDeclaration opType, Set parentClass, boolean isMemoPattern) { + super(analyzer, opType, parentClass, isMemoPattern); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalUnaryPatternGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalUnaryPatternGenerator.java index 7f23098e1e..8ecb7c14e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalUnaryPatternGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalUnaryPatternGenerator.java @@ -26,13 +26,13 @@ import java.util.TreeSet; public class LogicalUnaryPatternGenerator extends PatternGenerator { public LogicalUnaryPatternGenerator(PatternGeneratorAnalyzer analyzer, - ClassDeclaration opType, Set parentClass) { - super(analyzer, opType, parentClass); + ClassDeclaration opType, Set parentClass, boolean isMemoPattern) { + super(analyzer, opType, parentClass, isMemoPattern); } @Override public String genericType() { - return "<" + opType.name + ">"; + return "<" + opType.name + "<" + childType() + ">>"; } @Override @@ -44,7 +44,9 @@ public class LogicalUnaryPatternGenerator extends PatternGenerator { public Set getImports() { Set imports = new TreeSet<>(); imports.add(opType.getFullQualifiedName()); - imports.add("org.apache.doris.nereids.trees.plans.GroupPlan"); + if (isMemoPattern) { + imports.add("org.apache.doris.nereids.trees.plans.GroupPlan"); + } imports.add("org.apache.doris.nereids.trees.plans.Plan"); enumFieldPatternInfos.stream() .map(info -> info.enumFullName) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternDescribableProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternDescribableProcessor.java index 504f3ed1ae..42cf82e3c0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternDescribableProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternDescribableProcessor.java @@ -86,21 +86,9 @@ public class PatternDescribableProcessor extends AbstractProcessor { List asts = parseJavaFile(file); patternGeneratorAnalyzer.addAsts(asts); } - String generatePatternCode = patternGeneratorAnalyzer.generatePatterns(); - File generatePatternFile = new File(processingEnv.getFiler() - .getResource(StandardLocation.SOURCE_OUTPUT, "org.apache.doris.nereids.pattern", - "GeneratedPatterns.java").toUri()); - if (generatePatternFile.exists()) { - generatePatternFile.delete(); - } - if (!generatePatternFile.getParentFile().exists()) { - generatePatternFile.getParentFile().mkdirs(); - } - // bypass create file for processingEnv.getFiler(), compile GeneratePatterns in next compile term - try (BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(generatePatternFile))) { - bufferedWriter.write(generatePatternCode); - } + doGenerate("GeneratedMemoPatterns", "MemoPatterns", true, patternGeneratorAnalyzer); + doGenerate("GeneratedPlanPatterns", "PlanPatterns", false, patternGeneratorAnalyzer); } catch (Throwable t) { String exceptionMsg = Throwables.getStackTraceAsString(t); processingEnv.getMessager().printMessage(Kind.ERROR, @@ -109,6 +97,26 @@ public class PatternDescribableProcessor extends AbstractProcessor { return false; } + private void doGenerate(String className, String parentClassName, boolean isMemoPattern, + PatternGeneratorAnalyzer patternGeneratorAnalyzer) throws IOException { + String generatePatternCode = patternGeneratorAnalyzer.generatePatterns( + className, parentClassName, isMemoPattern); + File generatePatternFile = new File(processingEnv.getFiler() + .getResource(StandardLocation.SOURCE_OUTPUT, "org.apache.doris.nereids.pattern", + className + ".java").toUri()); + if (generatePatternFile.exists()) { + generatePatternFile.delete(); + } + if (!generatePatternFile.getParentFile().exists()) { + generatePatternFile.getParentFile().mkdirs(); + } + + // bypass create file for processingEnv.getFiler(), compile GeneratePatterns in next compile term + try (BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(generatePatternFile))) { + bufferedWriter.write(generatePatternCode); + } + } + private List findJavaFiles(List dirs) { List files = new ArrayList<>(); for (File dir : dirs) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternGenerator.java index 69d34b967d..e916e6bee0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternGenerator.java @@ -50,13 +50,16 @@ public abstract class PatternGenerator { protected final Set parentClass; protected final List enumFieldPatternInfos; protected final List generatePatterns = new ArrayList<>(); + protected final boolean isMemoPattern; /** constructor. */ - public PatternGenerator(PatternGeneratorAnalyzer analyzer, ClassDeclaration opType, Set parentClass) { + public PatternGenerator(PatternGeneratorAnalyzer analyzer, ClassDeclaration opType, + Set parentClass, boolean isMemoPattern) { this.analyzer = analyzer; this.opType = opType; this.parentClass = parentClass; this.enumFieldPatternInfos = getEnumFieldPatternInfos(); + this.isMemoPattern = isMemoPattern; } public abstract String genericType(); @@ -74,7 +77,8 @@ public abstract class PatternGenerator { } /** generate code by generators and analyzer. */ - public static String generateCode(List generators, PatternGeneratorAnalyzer analyzer) { + public static String generateCode(String className, String parentClassName, List generators, + PatternGeneratorAnalyzer analyzer, boolean isMemoPattern) { String generateCode = "// Licensed to the Apache Software Foundation (ASF) under one\n" + "// or more contributor license agreements. See the NOTICE file\n" @@ -97,11 +101,10 @@ public abstract class PatternGenerator { + "\n" + generateImports(generators) + "\n"; - - generateCode += "public interface GeneratedPatterns extends Patterns {\n"; + generateCode += "public interface " + className + " extends " + parentClassName + " {\n"; generateCode += generators.stream() .map(generator -> { - String patternMethods = generator.generate(); + String patternMethods = generator.generate(isMemoPattern); // add indent return Arrays.stream(patternMethods.split("\n")) .map(line -> " " + line + "\n") @@ -199,21 +202,25 @@ public abstract class PatternGenerator { return parts; } + protected String childType() { + return isMemoPattern ? "GroupPlan" : "Plan"; + } + /** create generator by plan's type. */ public static Optional create(PatternGeneratorAnalyzer analyzer, - ClassDeclaration opType, Set parentClass) { + ClassDeclaration opType, Set parentClass, boolean isMemoPattern) { if (parentClass.contains("org.apache.doris.nereids.trees.plans.logical.LogicalLeaf")) { - return Optional.of(new LogicalLeafPatternGenerator(analyzer, opType, parentClass)); + return Optional.of(new LogicalLeafPatternGenerator(analyzer, opType, parentClass, isMemoPattern)); } else if (parentClass.contains("org.apache.doris.nereids.trees.plans.logical.LogicalUnary")) { - return Optional.of(new LogicalUnaryPatternGenerator(analyzer, opType, parentClass)); + return Optional.of(new LogicalUnaryPatternGenerator(analyzer, opType, parentClass, isMemoPattern)); } else if (parentClass.contains("org.apache.doris.nereids.trees.plans.logical.LogicalBinary")) { - return Optional.of(new LogicalBinaryPatternGenerator(analyzer, opType, parentClass)); + return Optional.of(new LogicalBinaryPatternGenerator(analyzer, opType, parentClass, isMemoPattern)); } else if (parentClass.contains("org.apache.doris.nereids.trees.plans.physical.PhysicalLeaf")) { - return Optional.of(new PhysicalLeafPatternGenerator(analyzer, opType, parentClass)); + return Optional.of(new PhysicalLeafPatternGenerator(analyzer, opType, parentClass, isMemoPattern)); } else if (parentClass.contains("org.apache.doris.nereids.trees.plans.physical.PhysicalUnary")) { - return Optional.of(new PhysicalUnaryPatternGenerator(analyzer, opType, parentClass)); + return Optional.of(new PhysicalUnaryPatternGenerator(analyzer, opType, parentClass, isMemoPattern)); } else if (parentClass.contains("org.apache.doris.nereids.trees.plans.physical.PhysicalBinary")) { - return Optional.of(new PhysicalBinaryPatternGenerator(analyzer, opType, parentClass)); + return Optional.of(new PhysicalBinaryPatternGenerator(analyzer, opType, parentClass, isMemoPattern)); } else { return Optional.empty(); } @@ -233,21 +240,24 @@ public abstract class PatternGenerator { } /** generate some pattern method code. */ - public String generate() { + public String generate(boolean isMemoPattern) { String opClassName = opType.name; String methodName = getPatternMethodName(); - generateTypePattern(methodName, opClassName, genericType(), "", false); + generateTypePattern(methodName, opClassName, genericType(), "", false, isMemoPattern); if (childrenNum() > 0) { - generateTypePattern(methodName, opClassName, genericTypeWithChildren(), "", true); + generateTypePattern(methodName, opClassName, genericTypeWithChildren(), + "", true, isMemoPattern); } for (EnumFieldPatternInfo info : enumFieldPatternInfos) { String predicate = ".when(p -> p." + info.enumInstanceGetter + "() == " + info.enumType + "." + info.enumInstance + ")"; - generateTypePattern(info.patternName, opClassName, genericType(), predicate, false); + generateTypePattern(info.patternName, opClassName, genericType(), + predicate, false, isMemoPattern); if (childrenNum() > 0) { - generateTypePattern(info.patternName, opClassName, genericTypeWithChildren(), predicate, true); + generateTypePattern(info.patternName, opClassName, genericTypeWithChildren(), + predicate, true, isMemoPattern); } } return generatePatterns(); @@ -255,7 +265,7 @@ public abstract class PatternGenerator { /** generate a pattern method code. */ public String generateTypePattern(String patterName, String className, - String genericParam, String predicate, boolean specifyChildren) { + String genericParam, String predicate, boolean specifyChildren, boolean isMemoPattern) { int childrenNum = childrenNum(); @@ -286,7 +296,8 @@ public abstract class PatternGenerator { generatePatterns.add(pattern); return pattern; } else { - String childrenPattern = StringUtils.repeat("Pattern.GROUP", ", ", childrenNum); + String childrenPattern = StringUtils.repeat( + isMemoPattern ? "Pattern.GROUP" : "Pattern.ANY", ", ", childrenNum); if (childrenNum > 0) { childrenPattern = ", " + childrenPattern; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternGeneratorAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternGeneratorAnalyzer.java index 6c5343fccb..f4a9d12808 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternGeneratorAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternGeneratorAnalyzer.java @@ -57,10 +57,10 @@ public class PatternGeneratorAnalyzer { } /** generate pattern methods. */ - public String generatePatterns() { + public String generatePatterns(String className, String parentClassName, boolean isMemoPattern) { analyzeImport(); analyzeParentClass(); - return doGenerate(); + return doGenerate(className, parentClassName, isMemoPattern); } Optional getType(TypeDeclaration typeDeclaration, TypeType type) { @@ -73,7 +73,7 @@ public class PatternGeneratorAnalyzer { return Optional.empty(); } - private String doGenerate() { + private String doGenerate(String className, String parentClassName, boolean isMemoPattern) { Map> planClassMap = parentClassMap.entrySet().stream() .filter(kv -> kv.getValue().contains("org.apache.doris.nereids.trees.plans.Plan")) .filter(kv -> !kv.getKey().name.equals("GroupPlan")) @@ -83,7 +83,7 @@ public class PatternGeneratorAnalyzer { List generators = planClassMap.entrySet() .stream() - .map(kv -> PatternGenerator.create(this, kv.getKey(), kv.getValue())) + .map(kv -> PatternGenerator.create(this, kv.getKey(), kv.getValue(), isMemoPattern)) .filter(Optional::isPresent) .map(Optional::get) .sorted((g1, g2) -> { @@ -100,7 +100,7 @@ public class PatternGeneratorAnalyzer { }) .collect(Collectors.toList()); - return PatternGenerator.generateCode(generators, this); + return PatternGenerator.generateCode(className, parentClassName, generators, this, isMemoPattern); } private void analyzeImport() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalBinaryPatternGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalBinaryPatternGenerator.java index f148912d99..72a3155749 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalBinaryPatternGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalBinaryPatternGenerator.java @@ -26,13 +26,13 @@ import java.util.TreeSet; public class PhysicalBinaryPatternGenerator extends PatternGenerator { public PhysicalBinaryPatternGenerator(PatternGeneratorAnalyzer analyzer, - ClassDeclaration opType, Set parentClass) { - super(analyzer, opType, parentClass); + ClassDeclaration opType, Set parentClass, boolean isMemoPattern) { + super(analyzer, opType, parentClass, isMemoPattern); } @Override public String genericType() { - return "<" + opType.name + ">"; + return "<" + opType.name + "<" + childType() + ", " + childType() + ">>"; } @Override @@ -44,7 +44,9 @@ public class PhysicalBinaryPatternGenerator extends PatternGenerator { public Set getImports() { Set imports = new TreeSet<>(); imports.add(opType.getFullQualifiedName()); - imports.add("org.apache.doris.nereids.trees.plans.GroupPlan"); + if (isMemoPattern) { + imports.add("org.apache.doris.nereids.trees.plans.GroupPlan"); + } imports.add("org.apache.doris.nereids.trees.plans.Plan"); enumFieldPatternInfos.stream() .map(info -> info.enumFullName) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalLeafPatternGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalLeafPatternGenerator.java index 68c7154999..f75746b514 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalLeafPatternGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalLeafPatternGenerator.java @@ -26,8 +26,8 @@ import java.util.TreeSet; public class PhysicalLeafPatternGenerator extends PatternGenerator { public PhysicalLeafPatternGenerator(PatternGeneratorAnalyzer analyzer, - ClassDeclaration opType, Set parentClass) { - super(analyzer, opType, parentClass); + ClassDeclaration opType, Set parentClass, boolean isMemoPattern) { + super(analyzer, opType, parentClass, isMemoPattern); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalUnaryPatternGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalUnaryPatternGenerator.java index ea01341039..4254e28ee4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalUnaryPatternGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalUnaryPatternGenerator.java @@ -26,13 +26,13 @@ import java.util.TreeSet; public class PhysicalUnaryPatternGenerator extends PatternGenerator { public PhysicalUnaryPatternGenerator(PatternGeneratorAnalyzer analyzer, - ClassDeclaration opType, Set parentClass) { - super(analyzer, opType, parentClass); + ClassDeclaration opType, Set parentClass, boolean isMemoPattern) { + super(analyzer, opType, parentClass, isMemoPattern); } @Override public String genericType() { - return "<" + opType.name + ">"; + return "<" + opType.name + "<" + childType() + ">>"; } @Override @@ -44,7 +44,9 @@ public class PhysicalUnaryPatternGenerator extends PatternGenerator { public Set getImports() { Set imports = new TreeSet<>(); imports.add(opType.getFullQualifiedName()); - imports.add("org.apache.doris.nereids.trees.plans.GroupPlan"); + if (isMemoPattern) { + imports.add("org.apache.doris.nereids.trees.plans.GroupPlan"); + } imports.add("org.apache.doris.nereids.trees.plans.Plan"); enumFieldPatternInfos.stream() .map(info -> info.enumFullName) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java index adefdad45a..3bc32c8c6d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.processor.post; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.plans.Plan; @@ -56,7 +57,10 @@ public class Validator extends PlanPostProcessor { Plan child = filter.child(); // Forbidden filter-project, we must make filter-project -> project-filter. - Preconditions.checkState(!(child instanceof PhysicalProject)); + if (child instanceof PhysicalProject) { + throw new AnalysisException( + "Nereids generate a filter-project plan, but backend not support:\n" + filter.treeString()); + } // Check filter is from child output. Set childOutputSet = child.getOutputSet(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/AppliedAwareRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/AppliedAwareRule.java new file mode 100644 index 0000000000..d7f6983c3f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/AppliedAwareRule.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.exceptions.TransformException; +import org.apache.doris.nereids.pattern.Pattern; +import org.apache.doris.nereids.pattern.ProxyPattern; +import org.apache.doris.nereids.trees.plans.Plan; + +import java.util.BitSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.function.BiPredicate; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; + +/** AppliedAwareRule */ +public class AppliedAwareRule extends Rule { + private static final String APPLIED_RULES_KEY = "applied_rules"; + + private static final Supplier CREATE_APPLIED_RULES = () -> new BitSet(RuleType.SENTINEL.ordinal()); + + protected final Rule rule; + protected final RuleType ruleType; + protected int ruleTypeIndex; + + private AppliedAwareRule(Rule rule, BiPredicate matchRootPredicate) { + super(rule.getRuleType(), + new ExtendPattern(rule.getPattern(), (Predicate) (plan -> matchRootPredicate.test(rule, plan))), + rule.getRulePromise()); + this.rule = rule; + this.ruleType = rule.getRuleType(); + this.ruleTypeIndex = rule.getRuleType().ordinal(); + } + + @Override + public List transform(Plan plan, CascadesContext context) throws TransformException { + return rule.transform(plan, context); + } + + @Override + public void acceptPlan(Plan plan) { + BitSet appliedRules = plan.getOrInitMutableState(APPLIED_RULES_KEY, CREATE_APPLIED_RULES); + appliedRules.set(ruleTypeIndex); + } + + /** + * AppliedAwareRuleCondition: convert one rule to AppliedAwareRule, so that the rule can add + * some condition depends on whether this rule is applied to some plan + */ + public static class AppliedAwareRuleCondition implements Function { + public Rule apply(Rule rule) { + return new AppliedAwareRule(rule, this::condition); + } + + /** provide this method for the child class get the applied state */ + public final boolean isAppliedRule(Rule rule, Plan plan) { + Optional appliedRules = plan.getMutableState("applied_rules"); + if (!appliedRules.isPresent()) { + return false; + } + return appliedRules.get().get(rule.getRuleType().ordinal()); + } + + /** + * the default condition is whether this rule already applied to a plan, + * this means one plan only apply for a rule only once. child class can + * override this method. + */ + protected boolean condition(Rule rule, Plan plan) { + return isAppliedRule(rule, plan); + } + } + + private static class ExtendPattern extends ProxyPattern { + private final Predicate matchRootPredicate; + + public ExtendPattern(Pattern pattern, Predicate matchRootPredicate) { + super(pattern); + this.matchRootPredicate = Objects.requireNonNull(matchRootPredicate, "matchRootPredicate cannot be null"); + } + + @Override + public boolean matchPlanTree(Plan plan) { + return matchRootPredicate.test(plan) && super.matchPlanTree(plan); + } + + @Override + public boolean matchRoot(Plan plan) { + return matchRootPredicate.test(plan) && super.matchRoot(plan); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/ProxyRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/ProxyRule.java new file mode 100644 index 0000000000..7ea13662ee --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/ProxyRule.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.exceptions.TransformException; +import org.apache.doris.nereids.pattern.Pattern; +import org.apache.doris.nereids.trees.plans.Plan; + +import java.util.List; +import java.util.Objects; + +/** ProxyRule */ +public class ProxyRule extends Rule { + protected final Rule rule; + + public ProxyRule(Rule rule) { + this(rule, rule.getRuleType(), rule.getPattern(), rule.getRulePromise()); + } + + public ProxyRule(Rule rule, RuleType ruleType, Pattern pattern, RulePromise rulePromise) { + super(ruleType, pattern, rulePromise); + this.rule = Objects.requireNonNull(rule, "rule cannot be null"); + } + + @Override + public List transform(Plan node, CascadesContext context) throws TransformException { + return rule.transform(node, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rule.java index 1d54737496..25cba46002 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rule.java @@ -68,4 +68,9 @@ public abstract class Rule { } public abstract List transform(Plan node, CascadesContext context) throws TransformException; + + /** callback this function when the traverse framework accept a new plan which produce by this rule */ + public void acceptPlan(Plan plan) { + + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleFactory.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleFactory.java index d2fb2cf872..a4c3838716 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleFactory.java @@ -17,14 +17,14 @@ package org.apache.doris.nereids.rules; -import org.apache.doris.nereids.pattern.GeneratedPatterns; +import org.apache.doris.nereids.pattern.Patterns; import java.util.List; /** * interface for all rule factories for build some rules. */ -public interface RuleFactory extends GeneratedPatterns { +public interface RuleFactory extends Patterns { // need implement List buildRules(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 5409ea0799..99d33a16d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -93,7 +93,6 @@ public enum RuleType { CHECK_AND_STANDARDIZE_WINDOW_FUNCTION_AND_FRAME(RuleTypeClass.REWRITE), AGGREGATE_DISASSEMBLE(RuleTypeClass.REWRITE), DISTINCT_AGGREGATE_DISASSEMBLE(RuleTypeClass.REWRITE), - MARK_NECESSARY_PROJECT(RuleTypeClass.REWRITE), LOGICAL_SUB_QUERY_ALIAS_TO_LOGICAL_PROJECT(RuleTypeClass.REWRITE), ELIMINATE_GROUP_BY_CONSTANT(RuleTypeClass.REWRITE), ELIMINATE_ORDER_BY_CONSTANT(RuleTypeClass.REWRITE), @@ -101,16 +100,17 @@ public enum RuleType { INFER_FILTER_NOT_NULL(RuleTypeClass.REWRITE), INFER_JOIN_NOT_NULL(RuleTypeClass.REWRITE), // subquery analyze - ANALYZE_FILTER_SUBQUERY(RuleTypeClass.REWRITE), - ANALYZE_PROJECT_SUBQUERY(RuleTypeClass.REWRITE), + FILTER_SUBQUERY_TO_APPLY(RuleTypeClass.REWRITE), + PROJECT_SUBQUERY_TO_APPLY(RuleTypeClass.REWRITE), // subquery rewrite rule ELIMINATE_LIMIT_UNDER_APPLY(RuleTypeClass.REWRITE), ELIMINATE_SORT_UNDER_APPLY(RuleTypeClass.REWRITE), - PUSH_APPLY_UNDER_PROJECT(RuleTypeClass.REWRITE), - PUSH_APPLY_UNDER_FILTER(RuleTypeClass.REWRITE), - ELIMINATE_FILTER_UNDER_APPLY_PROJECT(RuleTypeClass.REWRITE), - APPLY_PULL_FILTER_ON_AGG(RuleTypeClass.REWRITE), - APPLY_PULL_FILTER_ON_PROJECT_UNDER_AGG(RuleTypeClass.REWRITE), + ELIMINATE_SORT_UNDER_APPLY_PROJECT(RuleTypeClass.REWRITE), + PULL_UP_PROJECT_UNDER_APPLY(RuleTypeClass.REWRITE), + UN_CORRELATED_APPLY_FILTER(RuleTypeClass.REWRITE), + UN_CORRELATED_APPLY_PROJECT_FILTER(RuleTypeClass.REWRITE), + UN_CORRELATED_APPLY_AGGREGATE_FILTER(RuleTypeClass.REWRITE), + PULL_UP_CORRELATED_FILTER_UNDER_APPLY_AGGREGATE_PROJECT(RuleTypeClass.REWRITE), SCALAR_APPLY_TO_JOIN(RuleTypeClass.REWRITE), IN_APPLY_TO_JOIN(RuleTypeClass.REWRITE), EXISTS_APPLY_TO_JOIN(RuleTypeClass.REWRITE), @@ -124,6 +124,7 @@ public enum RuleType { PUSHDOWN_FILTER_THROUGH_LEFT_SEMI_JOIN(RuleTypeClass.REWRITE), PUSH_FILTER_INSIDE_JOIN(RuleTypeClass.REWRITE), PUSHDOWN_FILTER_THROUGH_PROJECT(RuleTypeClass.REWRITE), + PUSHDOWN_FILTER_THROUGH_PROJECT_UNDER_LIMIT(RuleTypeClass.REWRITE), PUSHDOWN_PROJECT_THROUGH_LIMIT(RuleTypeClass.REWRITE), PUSHDOWN_FILTER_THROUGH_SET_OPERATION(RuleTypeClass.REWRITE), // column prune rules, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalysisRuleFactory.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalysisRuleFactory.java index 0938297883..10c207eb8f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalysisRuleFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalysisRuleFactory.java @@ -17,13 +17,14 @@ package org.apache.doris.nereids.rules.analysis; +import org.apache.doris.nereids.pattern.GeneratedPlanPatterns; import org.apache.doris.nereids.rules.PlanRuleFactory; import org.apache.doris.nereids.rules.RulePromise; /** * interface for all rule factories used in analysis stage. */ -public interface AnalysisRuleFactory extends PlanRuleFactory { +public interface AnalysisRuleFactory extends PlanRuleFactory, GeneratedPlanPatterns { @Override default RulePromise defaultPromise() { return RulePromise.ANALYSIS; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AvgDistinctToSumDivCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AvgDistinctToSumDivCount.java index f72e4bc5cd..360462d48c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AvgDistinctToSumDivCount.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AvgDistinctToSumDivCount.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Divide; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -40,7 +41,7 @@ import java.util.Map; * * change avg( distinct a ) into sum( distinct a ) / count( distinct a ) if there are more than 1 distinct arguments */ -public class AvgDistinctToSumDivCount extends OneAnalysisRuleFactory { +public class AvgDistinctToSumDivCount extends OneRewriteRuleFactory { @Override public Rule build() { return RuleType.AVG_DISTINCT_TO_SUM_DIV_COUNT.build( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index fc7e38ea00..8ceecd86db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.analyzer.UnboundTVFRelation; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.rules.AppliedAwareRule.AppliedAwareRuleCondition; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.expression.rewrite.rules.TypeCoercion; @@ -47,9 +48,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunctio import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinType; -import org.apache.doris.nereids.trees.plans.LeafPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; @@ -61,7 +60,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; @@ -80,7 +78,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; @@ -93,13 +90,8 @@ import java.util.stream.Stream; @SuppressWarnings("OptionalUsedAsFieldOrParameterType") public class BindExpression implements AnalysisRuleFactory { - private final Optional outerScope; - - public BindExpression(Optional outerScope) { - this.outerScope = Objects.requireNonNull(outerScope, "outerScope cannot be null"); - } - - private Scope toScope(List slots) { + private Scope toScope(CascadesContext cascadesContext, List slots) { + Optional outerScope = cascadesContext.getOuterScope(); if (outerScope.isPresent()) { return new Scope(outerScope, slots, outerScope.get().getSubquery()); } else { @@ -109,10 +101,29 @@ public class BindExpression implements AnalysisRuleFactory { @Override public List buildRules() { + /* + * some rules not only depends on the condition Plan::canBind, for example, + * BINDING_FILTER_SLOT need transform 'filter(unix_timestamp() > 100)' to + * 'filter(unix_timestamp() > cast(100 as int))'. there is no any unbound expression + * in the filter, so the Plan::canBind return false. + * + * we need `isAppliedRule` to judge whether a plan is applied to a rule, so need convert + * the normal rule to `AppliedAwareRule` to read and write the mutable state. + */ + AppliedAwareRuleCondition ruleCondition = new AppliedAwareRuleCondition() { + @Override + protected boolean condition(Rule rule, Plan plan) { + if (!rule.getPattern().matchRoot(plan)) { + return false; + } + return plan.canBind() || (plan.bound() && !isAppliedRule(rule, plan)); + } + }; + return ImmutableList.of( RuleType.BINDING_PROJECT_SLOT.build( - logicalProject().when(Plan::canBind).thenApply(ctx -> { - LogicalProject project = ctx.root; + logicalProject().thenApply(ctx -> { + LogicalProject project = ctx.root; List boundProjections = bindSlot(project.getProjects(), project.children(), ctx.cascadesContext); List boundExceptions = bindSlot(project.getExcepts(), project.children(), @@ -125,8 +136,8 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_FILTER_SLOT.build( - logicalFilter().when(Plan::canBind).thenApply(ctx -> { - LogicalFilter filter = ctx.root; + logicalFilter().thenApply(ctx -> { + LogicalFilter filter = ctx.root; Set boundConjuncts = filter.getConjuncts().stream() .map(expr -> bindSlot(expr, filter.children(), ctx.cascadesContext)) .map(expr -> bindFunction(expr, ctx.cascadesContext)) @@ -137,7 +148,7 @@ public class BindExpression implements AnalysisRuleFactory { RuleType.BINDING_USING_JOIN_SLOT.build( usingJoin().thenApply(ctx -> { - UsingJoin using = ctx.root; + UsingJoin using = ctx.root; LogicalJoin lj = new LogicalJoin<>(using.getJoinType() == JoinType.CROSS_JOIN ? JoinType.INNER_JOIN : using.getJoinType(), using.getHashJoinConjuncts(), @@ -151,7 +162,7 @@ public class BindExpression implements AnalysisRuleFactory { // the most right slot is matched with priority. Collections.reverse(leftOutput); List leftSlots = new ArrayList<>(); - Scope scope = toScope(leftOutput.stream() + Scope scope = toScope(ctx.cascadesContext, leftOutput.stream() .filter(s -> !slotNames.contains(s.getName())) .peek(s -> slotNames.add(s.getName())) .collect(Collectors.toList())); @@ -160,7 +171,7 @@ public class BindExpression implements AnalysisRuleFactory { leftSlots.add(expression); } slotNames.clear(); - scope = toScope(lj.right().getOutput().stream() + scope = toScope(ctx.cascadesContext, lj.right().getOutput().stream() .filter(s -> !slotNames.contains(s.getName())) .peek(s -> slotNames.add(s.getName())) .collect(Collectors.toList())); @@ -178,8 +189,8 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_JOIN_SLOT.build( - logicalJoin().when(Plan::canBind).thenApply(ctx -> { - LogicalJoin join = ctx.root; + logicalJoin().thenApply(ctx -> { + LogicalJoin join = ctx.root; List cond = join.getOtherJoinConjuncts().stream() .map(expr -> bindSlot(expr, join.children(), ctx.cascadesContext)) .map(expr -> bindFunction(expr, ctx.cascadesContext)) @@ -193,8 +204,8 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_AGGREGATE_SLOT.build( - logicalAggregate().when(Plan::canBind).thenApply(ctx -> { - LogicalAggregate agg = ctx.root; + logicalAggregate().thenApply(ctx -> { + LogicalAggregate agg = ctx.root; List output = agg.getOutputExpressions().stream() .map(expr -> bindSlot(expr, agg.children(), ctx.cascadesContext)) .map(expr -> bindFunction(expr, ctx.cascadesContext)) @@ -291,9 +302,10 @@ public class BindExpression implements AnalysisRuleFactory { boundSlots.addAll(outputSlots); SlotBinder binder = new SlotBinder( - toScope(Lists.newArrayList(boundSlots)), ctx.cascadesContext); + toScope(ctx.cascadesContext, ImmutableList.copyOf(boundSlots)), ctx.cascadesContext); SlotBinder childBinder = new SlotBinder( - toScope(new ArrayList<>(agg.child().getOutputSet())), ctx.cascadesContext); + toScope(ctx.cascadesContext, ImmutableList.copyOf(agg.child().getOutputSet())), + ctx.cascadesContext); List groupBy = replacedGroupBy.stream() .map(expression -> { @@ -323,8 +335,8 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_REPEAT_SLOT.build( - logicalRepeat().when(Plan::canBind).thenApply(ctx -> { - LogicalRepeat repeat = ctx.root; + logicalRepeat().thenApply(ctx -> { + LogicalRepeat repeat = ctx.root; List output = repeat.getOutputExpressions().stream() .map(expr -> bindSlot(expr, repeat.children(), ctx.cascadesContext)) .map(expr -> bindFunction(expr, ctx.cascadesContext)) @@ -368,35 +380,35 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_SORT_SLOT.build( - logicalSort(aggregate()).when(Plan::canBind).thenApply(ctx -> { - LogicalSort> sort = ctx.root; - Aggregate aggregate = sort.child(); + logicalSort(aggregate()).thenApply(ctx -> { + LogicalSort> sort = ctx.root; + Aggregate aggregate = sort.child(); return bindSort(sort, aggregate, ctx.cascadesContext); }) ), RuleType.BINDING_SORT_SLOT.build( - logicalSort(logicalHaving(aggregate())).when(Plan::canBind).thenApply(ctx -> { - LogicalSort>> sort = ctx.root; - Aggregate aggregate = sort.child().child(); + logicalSort(logicalHaving(aggregate())).thenApply(ctx -> { + LogicalSort>> sort = ctx.root; + Aggregate aggregate = sort.child().child(); return bindSort(sort, aggregate, ctx.cascadesContext); }) ), RuleType.BINDING_SORT_SLOT.build( - logicalSort(logicalHaving(logicalProject())).when(Plan::canBind).thenApply(ctx -> { - LogicalSort>> sort = ctx.root; - LogicalProject project = sort.child().child(); + logicalSort(logicalHaving(logicalProject())).thenApply(ctx -> { + LogicalSort>> sort = ctx.root; + LogicalProject project = sort.child().child(); return bindSort(sort, project, ctx.cascadesContext); }) ), RuleType.BINDING_SORT_SLOT.build( - logicalSort(logicalProject()).when(Plan::canBind).thenApply(ctx -> { - LogicalSort> sort = ctx.root; - LogicalProject project = sort.child(); + logicalSort(logicalProject()).thenApply(ctx -> { + LogicalSort> sort = ctx.root; + LogicalProject project = sort.child(); return bindSort(sort, project, ctx.cascadesContext); }) ), RuleType.BINDING_SORT_SET_OPERATION_SLOT.build( - logicalSort(logicalSetOperation()).when(Plan::canBind).thenApply(ctx -> { + logicalSort(logicalSetOperation()).thenApply(ctx -> { LogicalSort sort = ctx.root; List sortItemList = sort.getOrderKeys() .stream() @@ -409,8 +421,8 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_HAVING_SLOT.build( - logicalHaving(aggregate()).when(Plan::canBind).thenApply(ctx -> { - LogicalHaving> having = ctx.root; + logicalHaving(aggregate()).thenApply(ctx -> { + LogicalHaving> having = ctx.root; Plan childPlan = having.child(); Set boundConjuncts = having.getConjuncts().stream() .map(expr -> { @@ -423,7 +435,7 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_HAVING_SLOT.build( - logicalHaving(any()).when(Plan::canBind).thenApply(ctx -> { + logicalHaving(any()).thenApply(ctx -> { LogicalHaving having = ctx.root; Plan childPlan = having.child(); Set boundConjuncts = having.getConjuncts().stream() @@ -449,7 +461,7 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_SET_OPERATION_SLOT.build( - logicalSetOperation().when(Plan::canBind).then(setOperation -> { + logicalSetOperation().then(setOperation -> { // check whether the left and right child output columns are the same if (setOperation.child(0).getOutput().size() != setOperation.child(1).getOutput().size()) { throw new AnalysisException("Operands have unequal number of columns:\n" @@ -472,8 +484,8 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_GENERATE_SLOT.build( - logicalGenerate().when(Plan::canBind).thenApply(ctx -> { - LogicalGenerate generate = ctx.root; + logicalGenerate().thenApply(ctx -> { + LogicalGenerate generate = ctx.root; List boundSlotGenerators = bindSlot(generate.getGenerators(), generate.children(), ctx.cascadesContext); List boundFunctionGenerators = boundSlotGenerators.stream() @@ -497,16 +509,8 @@ public class BindExpression implements AnalysisRuleFactory { UnboundTVFRelation relation = ctx.root; return bindTableValuedFunction(relation, ctx.statementContext); }) - ), - - // when child update, we need update current plan's logical properties, - // since we use cache to avoid compute more than once. - RuleType.BINDING_NON_LEAF_LOGICAL_PLAN.build( - logicalPlan() - .when(plan -> plan.canBind() && !(plan instanceof LeafPlan)) - .then(LogicalPlan::recomputeLogicalProperties) ) - ); + ).stream().map(ruleCondition).collect(ImmutableList.toImmutableList()); } private Plan bindSort(LogicalSort sort, Plan plan, CascadesContext ctx) { @@ -528,8 +532,8 @@ public class BindExpression implements AnalysisRuleFactory { List sortItemList = sort.getOrderKeys() .stream() .map(orderKey -> { - Expression item = bindSlot(orderKey.getExpr(), plan, ctx); - item = bindSlot(item, plan.children(), ctx); + Expression item = bindSlot(orderKey.getExpr(), plan, ctx, true, false); + item = bindSlot(item, plan.children(), ctx, true, false); item = bindFunction(item, ctx); return new OrderKey(item, orderKey.isAsc(), orderKey.isNullFirst()); }).collect(Collectors.toList()); @@ -561,21 +565,33 @@ public class BindExpression implements AnalysisRuleFactory { @SuppressWarnings("unchecked") private E bindSlot(E expr, Plan input, CascadesContext cascadesContext) { - return bindSlot(expr, input, cascadesContext, true); + return bindSlot(expr, input, cascadesContext, true, true); } private E bindSlot(E expr, Plan input, CascadesContext cascadesContext, boolean enableExactMatch) { - return (E) new SlotBinder(toScope(input.getOutput()), cascadesContext, enableExactMatch).bind(expr); + return bindSlot(expr, input, cascadesContext, enableExactMatch, true); + } + + private E bindSlot(E expr, Plan input, CascadesContext cascadesContext, + boolean enableExactMatch, boolean bindSlotInOuterScope) { + return (E) new SlotBinder(toScope(cascadesContext, input.getOutput()), cascadesContext, + enableExactMatch, bindSlotInOuterScope).bind(expr); } @SuppressWarnings("unchecked") private E bindSlot(E expr, List inputs, CascadesContext cascadesContext, boolean enableExactMatch) { + return bindSlot(expr, inputs, cascadesContext, enableExactMatch, true); + } + + private E bindSlot(E expr, List inputs, CascadesContext cascadesContext, + boolean enableExactMatch, boolean bindSlotInOuterScope) { List boundedSlots = inputs.stream() .flatMap(input -> input.getOutput().stream()) .collect(Collectors.toList()); - return (E) new SlotBinder(toScope(boundedSlots), cascadesContext, enableExactMatch).bind(expr); + return (E) new SlotBinder(toScope(cascadesContext, boundedSlots), cascadesContext, + enableExactMatch, bindSlotInOuterScope).bind(expr); } @SuppressWarnings("unchecked") @@ -648,4 +664,8 @@ public class BindExpression implements AnalysisRuleFactory { function = (BoundFunction) TypeCoercion.INSTANCE.rewrite(function, null); return function; } + + public boolean canBind(Plan plan) { + return !plan.hasUnboundExpression() || plan.canBind(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 8ba0997572..b30cd4ad5c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -30,10 +30,13 @@ import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.analyzer.CTEContext; +import org.apache.doris.nereids.analyzer.Unbound; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.pattern.MatchingContext; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.EqualTo; @@ -70,26 +73,37 @@ public class BindRelation extends OneAnalysisRuleFactory { @Override public Rule build() { return unboundRelation().thenApply(ctx -> { - List nameParts = ctx.root.getNameParts(); - switch (nameParts.size()) { - case 1: { // table - // Use current database name from catalog. - return bindWithCurrentDb(ctx.cascadesContext, ctx.root); - } - case 2: { // db.table - // Use database name from table name parts. - return bindWithDbNameFromNamePart(ctx.cascadesContext, ctx.root); - } - case 3: { // catalog.db.table - // Use catalog and database name from name parts. - return bindWithCatalogNameFromNamePart(ctx.cascadesContext, ctx.root); - } - default: - throw new IllegalStateException("Table name [" + ctx.root.getTableName() + "] is invalid."); + Plan plan = doBindRelation(ctx); + if (!(plan instanceof Unbound)) { + // init output and allocate slot id immediately, so that the slot id increase + // in the order in which the table appears. + LogicalProperties logicalProperties = plan.getLogicalProperties(); + logicalProperties.getOutput(); } + return plan; }).toRule(RuleType.BINDING_RELATION); } + private Plan doBindRelation(MatchingContext ctx) { + List nameParts = ctx.root.getNameParts(); + switch (nameParts.size()) { + case 1: { // table + // Use current database name from catalog. + return bindWithCurrentDb(ctx.cascadesContext, ctx.root); + } + case 2: { // db.table + // Use database name from table name parts. + return bindWithDbNameFromNamePart(ctx.cascadesContext, ctx.root); + } + case 3: { // catalog.db.table + // Use catalog and database name from name parts. + return bindWithCatalogNameFromNamePart(ctx.cascadesContext, ctx.root); + } + default: + throw new IllegalStateException("Table name [" + ctx.root.getTableName() + "] is invalid."); + } + } + private TableIf getTable(String catalogName, String dbName, String tableName, Env env) { CatalogIf catalog = env.getCatalogMgr().getCatalog(catalogName); if (catalog == null) { @@ -209,12 +223,12 @@ public class BindRelation extends OneAnalysisRuleFactory { private Plan parseAndAnalyzeView(String viewSql, CascadesContext parentContext) { LogicalPlan parsedViewPlan = new NereidsParser().parseSingle(viewSql); - CascadesContext viewContext = new Memo(parsedViewPlan) - .newCascadesContext(parentContext.getStatementContext()); + CascadesContext viewContext = CascadesContext.newRewriteContext( + parentContext.getStatementContext(), parsedViewPlan, PhysicalProperties.ANY); viewContext.newAnalyzer().analyze(); // we should remove all group expression of the plan which in other memo, so the groupId would not conflict - return viewContext.getMemo().copyOut(false); + return viewContext.getRewritePlan(); } private List getPartitionIds(TableIf t, UnboundRelation unboundRelation) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckPolicy.java index 65df581bbf..e3c2ca5c68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckPolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckPolicy.java @@ -17,9 +17,11 @@ package org.apache.doris.nereids.rules.analysis; +import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalCheckPolicy; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; @@ -39,15 +41,13 @@ public class CheckPolicy implements AnalysisRuleFactory { public List buildRules() { return ImmutableList.of( RuleType.CHECK_ROW_POLICY.build( - logicalCheckPolicy(logicalSubQueryAlias()).then(checkPolicy -> checkPolicy.child()) - ), - RuleType.CHECK_ROW_POLICY.build( - logicalCheckPolicy(logicalFilter()).then(checkPolicy -> checkPolicy.child()) - ), - RuleType.CHECK_ROW_POLICY.build( - logicalCheckPolicy(logicalRelation()).thenApply(ctx -> { - LogicalCheckPolicy checkPolicy = ctx.root; - LogicalRelation relation = checkPolicy.child(); + logicalCheckPolicy(any().when(child -> !(child instanceof UnboundRelation))).thenApply(ctx -> { + LogicalCheckPolicy checkPolicy = ctx.root; + Plan child = checkPolicy.child(); + if (!(child instanceof LogicalRelation)) { + return child; + } + LogicalRelation relation = (LogicalRelation) child; Optional filter = checkPolicy.getFilter(relation, ctx.connectContext); if (!filter.isPresent()) { return relation; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/RegisterCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/RegisterCTE.java index a8b25c2577..1fd829f5f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/RegisterCTE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/RegisterCTE.java @@ -20,11 +20,9 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.analyzer.CTEContext; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; @@ -48,7 +46,7 @@ public class RegisterCTE extends OneAnalysisRuleFactory { @Override public Rule build() { return logicalCTE().thenApply(ctx -> { - LogicalCTE logicalCTE = ctx.root; + LogicalCTE logicalCTE = ctx.root; register(logicalCTE.getAliasQueries(), ctx.cascadesContext); return logicalCTE.child(); }).toRule(RuleType.REGISTER_CTE); @@ -69,10 +67,10 @@ public class RegisterCTE extends OneAnalysisRuleFactory { CTEContext localCteContext = cteCtx; Function analyzeCte = parsePlan -> { - CascadesContext localCascadesContext = new Memo(parsePlan) - .newCascadesContext(cascadesContext.getStatementContext(), localCteContext); + CascadesContext localCascadesContext = CascadesContext.newRewriteContext( + cascadesContext.getStatementContext(), parsePlan, localCteContext); localCascadesContext.newAnalyzer().analyze(); - return (LogicalPlan) localCascadesContext.getMemo().copyOut(false); + return (LogicalPlan) localCascadesContext.getRewritePlan(); }; LogicalPlan analyzedCteBody = analyzeCte.apply(aliasQuery.child()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ReplaceExpressionByChildOutput.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ReplaceExpressionByChildOutput.java index ba953307eb..b52e2f0218 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ReplaceExpressionByChildOutput.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ReplaceExpressionByChildOutput.java @@ -23,7 +23,7 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -46,7 +46,7 @@ public class ReplaceExpressionByChildOutput implements AnalysisRuleFactory { return ImmutableList.builder() .add(RuleType.REPLACE_SORT_EXPRESSION_BY_CHILD_OUTPUT.build( logicalSort(logicalProject()).then(sort -> { - LogicalProject project = sort.child(); + LogicalProject project = sort.child(); Map sMap = Maps.newHashMap(); project.getProjects().stream() .filter(Alias.class::isInstance) @@ -57,7 +57,7 @@ public class ReplaceExpressionByChildOutput implements AnalysisRuleFactory { )) .add(RuleType.REPLACE_SORT_EXPRESSION_BY_CHILD_OUTPUT.build( logicalSort(logicalAggregate()).then(sort -> { - LogicalAggregate aggregate = sort.child(); + LogicalAggregate aggregate = sort.child(); Map sMap = Maps.newHashMap(); aggregate.getOutputExpressions().stream() .filter(Alias.class::isInstance) @@ -67,7 +67,7 @@ public class ReplaceExpressionByChildOutput implements AnalysisRuleFactory { }) )).add(RuleType.REPLACE_SORT_EXPRESSION_BY_CHILD_OUTPUT.build( logicalSort(logicalHaving(logicalAggregate())).then(sort -> { - LogicalAggregate aggregate = sort.child().child(); + LogicalAggregate aggregate = sort.child().child(); Map sMap = Maps.newHashMap(); aggregate.getOutputExpressions().stream() .filter(Alias.class::isInstance) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ResolveOrdinalInOrderByAndGroupBy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ResolveOrdinalInOrderByAndGroupBy.java index c826cc98b0..9a3a2d675e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ResolveOrdinalInOrderByAndGroupBy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ResolveOrdinalInOrderByAndGroupBy.java @@ -20,13 +20,16 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewriteContext; import org.apache.doris.nereids.rules.expression.rewrite.rules.FoldConstantRule; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import com.google.common.collect.ImmutableList; @@ -43,13 +46,15 @@ public class ResolveOrdinalInOrderByAndGroupBy implements AnalysisRuleFactory { public List buildRules() { return ImmutableList.builder() .add(RuleType.RESOLVE_ORDINAL_IN_ORDER_BY.build( - logicalSort().then(sort -> { + logicalSort().thenApply(ctx -> { + LogicalSort sort = ctx.root; List childOutput = sort.child().getOutput(); List orderKeys = sort.getOrderKeys(); List orderKeysWithoutOrd = new ArrayList<>(); + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); for (OrderKey k : orderKeys) { Expression expression = k.getExpr(); - expression = FoldConstantRule.INSTANCE.rewrite(expression); + expression = FoldConstantRule.INSTANCE.rewrite(expression, context); if (expression instanceof IntegerLikeLiteral) { IntegerLikeLiteral i = (IntegerLikeLiteral) expression; int ord = i.getIntValue(); @@ -64,12 +69,14 @@ public class ResolveOrdinalInOrderByAndGroupBy implements AnalysisRuleFactory { }) )) .add(RuleType.RESOLVE_ORDINAL_IN_GROUP_BY.build( - logicalAggregate().whenNot(agg -> agg.isOrdinalIsResolved()).then(agg -> { + logicalAggregate().whenNot(agg -> agg.isOrdinalIsResolved()).thenApply(ctx -> { + LogicalAggregate agg = ctx.root; List aggOutput = agg.getOutputExpressions(); List groupByWithoutOrd = new ArrayList<>(); + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); boolean ordExists = false; for (Expression groupByExpr : agg.getGroupByExpressions()) { - groupByExpr = FoldConstantRule.INSTANCE.rewrite(groupByExpr); + groupByExpr = FoldConstantRule.INSTANCE.rewrite(groupByExpr, context); if (groupByExpr instanceof IntegerLikeLiteral) { IntegerLikeLiteral i = (IntegerLikeLiteral) groupByExpr; int ord = i.getIntValue(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java index b2fcd34918..0d1785eafb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java @@ -47,15 +47,18 @@ class SlotBinder extends SubExprAnalyzer { but enabled for order by clause TODO after remove original planner, always enable exact match mode. */ - private boolean enableExactMatch = true; + private boolean enableExactMatch; + private final boolean bindSlotInOuterScope; public SlotBinder(Scope scope, CascadesContext cascadesContext) { - this(scope, cascadesContext, true); + this(scope, cascadesContext, true, true); } - public SlotBinder(Scope scope, CascadesContext cascadesContext, boolean enableExactMatch) { + public SlotBinder(Scope scope, CascadesContext cascadesContext, + boolean enableExactMatch, boolean bindSlotInOuterScope) { super(scope, cascadesContext); this.enableExactMatch = enableExactMatch; + this.bindSlotInOuterScope = bindSlotInOuterScope; } public Expression bind(Expression expression) { @@ -81,7 +84,7 @@ class SlotBinder extends SubExprAnalyzer { Optional> boundedOpt = Optional.of(bindSlot(unboundSlot, getScope().getSlots())); boolean foundInThisScope = !boundedOpt.get().isEmpty(); // Currently only looking for symbols on the previous level. - if (!foundInThisScope && getScope().getOuterScope().isPresent()) { + if (bindSlotInOuterScope && !foundInThisScope && getScope().getOuterScope().isPresent()) { boundedOpt = Optional.of(bindSlot(unboundSlot, getScope() .getOuterScope() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java index 26d281fe44..a51e049999 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java @@ -20,7 +20,6 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.analyzer.Scope; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.trees.expressions.Exists; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.InSubquery; @@ -38,6 +37,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -140,13 +140,12 @@ class SubExprAnalyzer extends DefaultExpressionRewriter { } private AnalyzedResult analyzeSubquery(SubqueryExpr expr) { - CascadesContext subqueryContext = new Memo(expr.getQueryPlan()) - .newCascadesContext((cascadesContext.getStatementContext()), cascadesContext.getCteContext()); + CascadesContext subqueryContext = CascadesContext.newRewriteContext( + cascadesContext.getStatementContext(), expr.getQueryPlan(), cascadesContext.getCteContext()); Scope subqueryScope = genScopeWithSubquery(expr); - subqueryContext - .newAnalyzer(Optional.of(subqueryScope)) - .analyze(); - return new AnalyzedResult((LogicalPlan) subqueryContext.getMemo().copyOut(false), + subqueryContext.setOuterScope(subqueryScope); + subqueryContext.newAnalyzer().analyze(); + return new AnalyzedResult((LogicalPlan) subqueryContext.getRewritePlan(), subqueryScope.getCorrelatedSlots()); } @@ -168,7 +167,7 @@ class SubExprAnalyzer extends DefaultExpressionRewriter { private final LogicalPlan logicalPlan; private final List correlatedSlots; - public AnalyzedResult(LogicalPlan logicalPlan, List correlatedSlots) { + public AnalyzedResult(LogicalPlan logicalPlan, Collection correlatedSlots) { this.logicalPlan = Objects.requireNonNull(logicalPlan, "logicalPlan can not be null"); this.correlatedSlots = correlatedSlots == null ? new ArrayList<>() : ImmutableList.copyOf(correlatedSlots); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubquery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java similarity index 88% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubquery.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java index 1e85e87797..995a095a34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubquery.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java @@ -30,7 +30,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; @@ -46,18 +46,18 @@ import java.util.Optional; import java.util.Set; /** - * AnalyzeSubquery. translate from subquery to LogicalApply. + * SubqueryToApply. translate from subquery to LogicalApply. * In two steps * The first step is to replace the predicate corresponding to the filter where the subquery is located. * The second step converts the subquery into an apply node. */ -public class AnalyzeSubquery implements AnalysisRuleFactory { +public class SubqueryToApply implements AnalysisRuleFactory { @Override public List buildRules() { return ImmutableList.of( - RuleType.ANALYZE_FILTER_SUBQUERY.build( + RuleType.FILTER_SUBQUERY_TO_APPLY.build( logicalFilter().thenApply(ctx -> { - LogicalFilter filter = ctx.root; + LogicalFilter filter = ctx.root; Set subqueryExprs = filter.getPredicate().collect(SubqueryExpr.class::isInstance); if (subqueryExprs.isEmpty()) { return filter; @@ -66,14 +66,14 @@ public class AnalyzeSubquery implements AnalysisRuleFactory { // first step: Replace the subquery of predicate in LogicalFilter // second step: Replace subquery with LogicalApply return new LogicalFilter<>(new ReplaceSubquery().replace(filter.getConjuncts()), - analyzedSubquery( + subqueryToApply( subqueryExprs, filter.child(), ctx.cascadesContext )); }) ), - RuleType.ANALYZE_PROJECT_SUBQUERY.build( + RuleType.PROJECT_SUBQUERY_TO_APPLY.build( logicalProject().thenApply(ctx -> { - LogicalProject project = ctx.root; + LogicalProject project = ctx.root; Set subqueryExprs = new HashSet<>(); project.getProjects().stream() .filter(Alias.class::isInstance) @@ -89,17 +89,17 @@ public class AnalyzeSubquery implements AnalysisRuleFactory { return new LogicalProject(project.getProjects().stream() .map(p -> p.withChildren(new ReplaceSubquery().replace(p))) .collect(ImmutableList.toImmutableList()), - analyzedSubquery( + subqueryToApply( subqueryExprs, project.child(), ctx.cascadesContext )); }) - ) + ) ); } - private LogicalPlan analyzedSubquery(Set subqueryExprs, - LogicalPlan childPlan, CascadesContext ctx) { - LogicalPlan tmpPlan = childPlan; + private Plan subqueryToApply(Set subqueryExprs, + Plan childPlan, CascadesContext ctx) { + Plan tmpPlan = childPlan; for (SubqueryExpr subqueryExpr : subqueryExprs) { if (!ctx.subqueryIsAnalyzed(subqueryExpr)) { tmpPlan = addApply(subqueryExpr, tmpPlan, ctx); @@ -108,8 +108,7 @@ public class AnalyzeSubquery implements AnalysisRuleFactory { return tmpPlan; } - private LogicalPlan addApply(SubqueryExpr subquery, - LogicalPlan childPlan, CascadesContext ctx) { + private LogicalPlan addApply(SubqueryExpr subquery, Plan childPlan, CascadesContext ctx) { ctx.setSubqueryExprIsAnalyzed(subquery, true); LogicalApply newApply = new LogicalApply( subquery.getCorrelateSlots(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/UserAuthentication.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/UserAuthentication.java index ef1158ae2f..e51f203932 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/UserAuthentication.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/UserAuthentication.java @@ -33,7 +33,8 @@ public class UserAuthentication extends OneAnalysisRuleFactory { @Override public Rule build() { - return logicalRelation().thenApply(ctx -> checkPermission(ctx.root, ctx.connectContext)) + return logicalRelation() + .thenApply(ctx -> checkPermission(ctx.root, ctx.connectContext)) .toRule(RuleType.RELATION_AUTHENTICATION); } @@ -46,7 +47,6 @@ public class UserAuthentication extends OneAnalysisRuleFactory { ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(), dbName + ": " + tableName); throw new AnalysisException(message); - } return relation; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/ExplorationRuleFactory.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/ExplorationRuleFactory.java index 4e449a5146..a8b9f9ce53 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/ExplorationRuleFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/ExplorationRuleFactory.java @@ -17,13 +17,14 @@ package org.apache.doris.nereids.rules.exploration; +import org.apache.doris.nereids.pattern.GeneratedMemoPatterns; import org.apache.doris.nereids.rules.PlanRuleFactory; import org.apache.doris.nereids.rules.RulePromise; /** * interface for all exploration rule factories. */ -public interface ExplorationRuleFactory extends PlanRuleFactory { +public interface ExplorationRuleFactory extends PlanRuleFactory, GeneratedMemoPatterns { @Override default RulePromise defaultPromise() { return RulePromise.EXPLORE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionNormalization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionNormalization.java index 5d869f662e..360cdbf87d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionNormalization.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionNormalization.java @@ -29,7 +29,7 @@ import org.apache.doris.nereids.rules.expression.rewrite.rules.SimplifyCastRule; import org.apache.doris.nereids.rules.expression.rewrite.rules.SimplifyNotExprRule; import org.apache.doris.nereids.rules.expression.rewrite.rules.SupportJavaDateFormatter; import org.apache.doris.nereids.rules.expression.rewrite.rules.TypeCoercion; -import org.apache.doris.qe.ConnectContext; +import org.apache.doris.nereids.trees.expressions.Expression; import com.google.common.collect.ImmutableList; @@ -56,8 +56,13 @@ public class ExpressionNormalization extends ExpressionRewrite { SupportJavaDateFormatter.INSTANCE ); - public ExpressionNormalization(ConnectContext context) { - super(new ExpressionRuleExecutor(NORMALIZE_REWRITE_RULES, context)); + public ExpressionNormalization() { + super(new ExpressionRuleExecutor(NORMALIZE_REWRITE_RULES)); + } + + @Override + public Expression rewrite(Expression expression, ExpressionRewriteContext context) { + return super.rewrite(expression, context); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewrite.java index d4e033cb35..50b83de095 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewrite.java @@ -25,11 +25,16 @@ import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.functions.Function; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; +import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; @@ -56,8 +61,8 @@ public class ExpressionRewrite implements RewriteRuleFactory { this.rewriter = Objects.requireNonNull(rewriter, "rewriter is null"); } - public Expression rewrite(Expression expression) { - return rewriter.rewrite(expression); + public Expression rewrite(Expression expression, ExpressionRewriteContext expressionRewriteContext) { + return rewriter.rewrite(expression, expressionRewriteContext); } @Override @@ -77,10 +82,12 @@ public class ExpressionRewrite implements RewriteRuleFactory { private class GenerateExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalGenerate().then(generate -> { + return logicalGenerate().thenApply(ctx -> { + LogicalGenerate generate = ctx.root; + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); List generators = generate.getGenerators(); List newGenerators = generators.stream() - .map(func -> (Function) rewriter.rewrite(func)) + .map(func -> (Function) rewriter.rewrite(func, context)) .collect(ImmutableList.toImmutableList()); if (generators.equals(newGenerators)) { return generate; @@ -93,11 +100,14 @@ public class ExpressionRewrite implements RewriteRuleFactory { private class OneRowRelationExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalOneRowRelation().then(oneRowRelation -> { + return logicalOneRowRelation().thenApply(ctx -> { + LogicalOneRowRelation oneRowRelation = ctx.root; List projects = oneRowRelation.getProjects(); + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); + List newProjects = projects .stream() - .map(expr -> (NamedExpression) rewriter.rewrite(expr)) + .map(expr -> (NamedExpression) rewriter.rewrite(expr, context)) .collect(ImmutableList.toImmutableList()); if (projects.equals(newProjects)) { return oneRowRelation; @@ -110,10 +120,13 @@ public class ExpressionRewrite implements RewriteRuleFactory { private class ProjectExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalProject().then(project -> { + return logicalProject().thenApply(ctx -> { + LogicalProject project = ctx.root; + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); List projects = project.getProjects(); List newProjects = projects.stream() - .map(expr -> (NamedExpression) rewriter.rewrite(expr)).collect(ImmutableList.toImmutableList()); + .map(expr -> (NamedExpression) rewriter.rewrite(expr, context)) + .collect(ImmutableList.toImmutableList()); if (projects.equals(newProjects)) { return project; } @@ -125,9 +138,11 @@ public class ExpressionRewrite implements RewriteRuleFactory { private class FilterExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalFilter().then(filter -> { + return logicalFilter().thenApply(ctx -> { + LogicalFilter filter = ctx.root; + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); Set newConjuncts = ImmutableSet.copyOf(ExpressionUtils.extractConjunction( - rewriter.rewrite(filter.getPredicate()))); + rewriter.rewrite(filter.getPredicate(), context))); if (newConjuncts.equals(filter.getConjuncts())) { return filter; } @@ -139,13 +154,16 @@ public class ExpressionRewrite implements RewriteRuleFactory { private class AggExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalAggregate().then(agg -> { + return logicalAggregate().thenApply(ctx -> { + LogicalAggregate agg = ctx.root; List groupByExprs = agg.getGroupByExpressions(); - List newGroupByExprs = rewriter.rewrite(groupByExprs); + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); + List newGroupByExprs = rewriter.rewrite(groupByExprs, context); List outputExpressions = agg.getOutputExpressions(); List newOutputExpressions = outputExpressions.stream() - .map(expr -> (NamedExpression) rewriter.rewrite(expr)).collect(ImmutableList.toImmutableList()); + .map(expr -> (NamedExpression) rewriter.rewrite(expr, context)) + .collect(ImmutableList.toImmutableList()); if (outputExpressions.equals(newOutputExpressions)) { return agg; } @@ -158,16 +176,18 @@ public class ExpressionRewrite implements RewriteRuleFactory { private class JoinExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalJoin().then(join -> { + return logicalJoin().thenApply(ctx -> { + LogicalJoin join = ctx.root; List hashJoinConjuncts = join.getHashJoinConjuncts(); List otherJoinConjuncts = join.getOtherJoinConjuncts(); if (otherJoinConjuncts.isEmpty() && hashJoinConjuncts.isEmpty()) { return join; } + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); List rewriteHashJoinConjuncts = Lists.newArrayList(); boolean hashJoinConjunctsChanged = false; for (Expression expr : hashJoinConjuncts) { - Expression newExpr = rewriter.rewrite(expr); + Expression newExpr = rewriter.rewrite(expr, context); hashJoinConjunctsChanged = hashJoinConjunctsChanged || !newExpr.equals(expr); rewriteHashJoinConjuncts.add(newExpr); } @@ -175,7 +195,7 @@ public class ExpressionRewrite implements RewriteRuleFactory { List rewriteOtherJoinConjuncts = Lists.newArrayList(); boolean otherJoinConjunctsChanged = false; for (Expression expr : otherJoinConjuncts) { - Expression newExpr = rewriter.rewrite(expr); + Expression newExpr = rewriter.rewrite(expr, context); otherJoinConjunctsChanged = otherJoinConjunctsChanged || !newExpr.equals(expr); rewriteOtherJoinConjuncts.add(newExpr); } @@ -193,11 +213,13 @@ public class ExpressionRewrite implements RewriteRuleFactory { @Override public Rule build() { - return logicalSort().then(sort -> { + return logicalSort().thenApply(ctx -> { + LogicalSort sort = ctx.root; List orderKeys = sort.getOrderKeys(); List rewrittenOrderKeys = new ArrayList<>(); + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); for (OrderKey k : orderKeys) { - Expression expression = rewriter.rewrite(k.getExpr()); + Expression expression = rewriter.rewrite(k.getExpr(), context); rewrittenOrderKeys.add(new OrderKey(expression, k.isAsc(), k.isNullFirst())); } return sort.withOrderKeys(rewrittenOrderKeys); @@ -208,10 +230,12 @@ public class ExpressionRewrite implements RewriteRuleFactory { private class HavingExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalHaving().then(having -> { + return logicalHaving().thenApply(ctx -> { + LogicalHaving having = ctx.root; Set rewrittenExpr = new HashSet<>(); + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); for (Expression e : having.getExpressions()) { - rewrittenExpr.add(rewriter.rewrite(e)); + rewrittenExpr.add(rewriter.rewrite(e, context)); } return having.withExpressions(rewrittenExpr); }).toRule(RuleType.REWRITE_HAVING_EXPRESSION); @@ -221,15 +245,19 @@ public class ExpressionRewrite implements RewriteRuleFactory { private class LogicalRepeatRewrite extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalRepeat().then(r -> { + return logicalRepeat().thenApply(ctx -> { + LogicalRepeat repeat = ctx.root; ImmutableList.Builder> groupingExprs = ImmutableList.builder(); - for (List expressions : r.getGroupingSets()) { - groupingExprs.add(expressions.stream().map(rewriter::rewrite) - .collect(ImmutableList.toImmutableList())); + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); + for (List expressions : repeat.getGroupingSets()) { + groupingExprs.add(expressions.stream() + .map(expr -> rewriter.rewrite(expr, context)) + .collect(ImmutableList.toImmutableList()) + ); } - return r.withGroupSetsAndOutput(groupingExprs.build(), - r.getOutputExpressions().stream() - .map(rewriter::rewrite) + return repeat.withGroupSetsAndOutput(groupingExprs.build(), + repeat.getOutputExpressions().stream() + .map(output -> rewriter.rewrite(output, context)) .map(e -> (NamedExpression) e) .collect(ImmutableList.toImmutableList())); }).toRule(RuleType.REWRITE_REPEAT_EXPRESSION); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteContext.java index dd32382810..37e571a0e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteContext.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.rules.expression.rewrite; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.qe.ConnectContext; /** @@ -25,7 +26,7 @@ import org.apache.doris.qe.ConnectContext; public class ExpressionRewriteContext { public final ConnectContext connectContext; - public ExpressionRewriteContext(ConnectContext connectContext) { - this.connectContext = connectContext; + public ExpressionRewriteContext(CascadesContext cascadesContext) { + this.connectContext = cascadesContext.getConnectContext(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRuleExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRuleExecutor.java index fdcb0ba497..4058a566d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRuleExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRuleExecutor.java @@ -19,7 +19,6 @@ package org.apache.doris.nereids.rules.expression.rewrite; import org.apache.doris.nereids.rules.expression.rewrite.rules.NormalizeBinaryPredicatesRule; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; @@ -30,30 +29,23 @@ import java.util.Optional; * Expression rewrite entry, which contains all rewrite rules. */ public class ExpressionRuleExecutor { - - private final ExpressionRewriteContext ctx; private final List rules; - public ExpressionRuleExecutor(List rules, ConnectContext context) { - this.rules = rules; - this.ctx = new ExpressionRewriteContext(context); - } - public ExpressionRuleExecutor(List rules) { - this(rules, null); + this.rules = rules; } - public List rewrite(List exprs) { - return exprs.stream().map(this::rewrite).collect(ImmutableList.toImmutableList()); + public List rewrite(List exprs, ExpressionRewriteContext ctx) { + return exprs.stream().map(expr -> rewrite(expr, ctx)).collect(ImmutableList.toImmutableList()); } /** * Given an expression, returns a rewritten expression. */ - public Expression rewrite(Expression root) { + public Expression rewrite(Expression root, ExpressionRewriteContext ctx) { Expression result = root; for (ExpressionRewriteRule rule : rules) { - result = applyRule(result, rule); + result = applyRule(result, rule, ctx); } return result; } @@ -61,11 +53,11 @@ public class ExpressionRuleExecutor { /** * Given an expression, returns a rewritten expression. */ - public Optional rewrite(Optional root) { - return root.map(this::rewrite); + public Optional rewrite(Optional root, ExpressionRewriteContext ctx) { + return root.map(r -> this.rewrite(r, ctx)); } - private Expression applyRule(Expression expr, ExpressionRewriteRule rule) { + private Expression applyRule(Expression expr, ExpressionRewriteRule rule, ExpressionRewriteContext ctx) { return rule.rewrite(expr, ctx); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/FoldConstantRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/FoldConstantRule.java index fcd8a7cff2..175e40b0aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/FoldConstantRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/FoldConstantRule.java @@ -20,7 +20,6 @@ package org.apache.doris.nereids.rules.expression.rewrite.rules; import org.apache.doris.nereids.rules.expression.rewrite.AbstractExpressionRewriteRule; import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewriteContext; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.qe.ConnectContext; /** * Constant evaluation of an expression. @@ -36,11 +35,5 @@ public class FoldConstantRule extends AbstractExpressionRewriteRule { } return FoldConstantRuleOnFE.INSTANCE.rewrite(expr, ctx); } - - public Expression rewrite(Expression expr) { - ExpressionRewriteContext ctx = new ExpressionRewriteContext(ConnectContext.get()); - return rewrite(expr, ctx); - } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java index 7187a9cf52..5cf69e5f63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java @@ -87,8 +87,7 @@ public class AggregateStrategies implements ImplementationRuleFactory { @Override public List buildRules() { - PatternDescriptor> basePattern = logicalAggregate() - .when(LogicalAggregate::isNormalized); + PatternDescriptor> basePattern = logicalAggregate(); return ImmutableList.of( RuleType.STORAGE_LAYER_AGGREGATE_WITHOUT_PROJECT.build( @@ -125,12 +124,12 @@ public class AggregateStrategies implements ImplementationRuleFactory { RuleType.TWO_PHASE_AGGREGATE_WITH_COUNT_DISTINCT_MULTI.build( basePattern .when(this::containsCountDistinctMultiExpr) - .thenApplyMulti(ctx -> twoPhaseAggregateWithCountDistinctMulti(ctx.root, ctx.connectContext)) + .thenApplyMulti(ctx -> twoPhaseAggregateWithCountDistinctMulti(ctx.root, ctx.cascadesContext)) ), RuleType.THREE_PHASE_AGGREGATE_WITH_COUNT_DISTINCT_MULTI.build( basePattern .when(this::containsCountDistinctMultiExpr) - .thenApplyMulti(ctx -> threePhaseAggregateWithCountDistinctMulti(ctx.root, ctx.connectContext)) + .thenApplyMulti(ctx -> threePhaseAggregateWithCountDistinctMulti(ctx.root, ctx.cascadesContext)) ), RuleType.TWO_PHASE_AGGREGATE_WITH_DISTINCT.build( basePattern @@ -393,7 +392,7 @@ public class AggregateStrategies implements ImplementationRuleFactory { * */ private List> twoPhaseAggregateWithCountDistinctMulti( - LogicalAggregate logicalAgg, ConnectContext connectContext) { + LogicalAggregate logicalAgg, CascadesContext cascadesContext) { AggregateParam inputToBufferParam = new AggregateParam(AggPhase.LOCAL, AggMode.INPUT_TO_BUFFER); Set countDistinctArguments = logicalAgg.getDistinctArguments(); @@ -423,13 +422,14 @@ public class AggregateStrategies implements ImplementationRuleFactory { PhysicalHashAggregate gatherLocalAgg = new PhysicalHashAggregate<>( localAggGroupBy, localOutput, Optional.of(partitionExpressions), new AggregateParam(AggPhase.LOCAL, AggMode.INPUT_TO_BUFFER), - maybeUsingStreamAgg(connectContext, logicalAgg), + maybeUsingStreamAgg(cascadesContext.getConnectContext(), logicalAgg), logicalAgg.getLogicalProperties(), requireGather, logicalAgg.child() ); List distinctGroupBy = logicalAgg.getGroupByExpressions(); - LogicalAggregate countIfAgg = countDistinctMultiExprToCountIf(logicalAgg, connectContext).first; + LogicalAggregate countIfAgg = countDistinctMultiExprToCountIf( + logicalAgg, cascadesContext).first; AggregateParam distinctInputToResultParam = new AggregateParam(AggPhase.DISTINCT_LOCAL, AggMode.INPUT_TO_RESULT); @@ -509,7 +509,7 @@ public class AggregateStrategies implements ImplementationRuleFactory { * */ private List> threePhaseAggregateWithCountDistinctMulti( - LogicalAggregate logicalAgg, ConnectContext connectContext) { + LogicalAggregate logicalAgg, CascadesContext cascadesContext) { AggregateParam inputToBufferParam = new AggregateParam(AggPhase.LOCAL, AggMode.INPUT_TO_BUFFER); Set countDistinctArguments = logicalAgg.getDistinctArguments(); @@ -539,7 +539,7 @@ public class AggregateStrategies implements ImplementationRuleFactory { PhysicalHashAggregate anyLocalAgg = new PhysicalHashAggregate<>( localAggGroupBy, localOutput, Optional.of(partitionExpressions), new AggregateParam(AggPhase.LOCAL, AggMode.INPUT_TO_BUFFER), - maybeUsingStreamAgg(connectContext, logicalAgg), + maybeUsingStreamAgg(cascadesContext.getConnectContext(), logicalAgg), logicalAgg.getLogicalProperties(), requireAny, logicalAgg.child() ); @@ -571,7 +571,8 @@ public class AggregateStrategies implements ImplementationRuleFactory { bufferToBufferParam, false, logicalAgg.getLogicalProperties(), requireGather, anyLocalAgg); - LogicalAggregate countIfAgg = countDistinctMultiExprToCountIf(logicalAgg, connectContext).first; + LogicalAggregate countIfAgg = countDistinctMultiExprToCountIf( + logicalAgg, cascadesContext).first; AggregateParam distinctInputToResultParam = new AggregateParam(AggPhase.DISTINCT_LOCAL, AggMode.INPUT_TO_RESULT); @@ -1194,7 +1195,7 @@ public class AggregateStrategies implements ImplementationRuleFactory { * phase of aggregate, please normalize to slot and create a bottom project like NormalizeAggregate. */ private Pair, List> countDistinctMultiExprToCountIf( - LogicalAggregate aggregate, ConnectContext connectContext) { + LogicalAggregate aggregate, CascadesContext cascadesContext) { ImmutableList.Builder countIfList = ImmutableList.builder(); List newOutput = ExpressionUtils.rewriteDownShortCircuit( aggregate.getOutputExpressions(), outputChild -> { @@ -1206,7 +1207,7 @@ public class AggregateStrategies implements ImplementationRuleFactory { for (int i = arguments.size() - 2; i >= 0; --i) { Expression argument = count.getArgument(i); If ifNull = new If(new IsNull(argument), NullLiteral.INSTANCE, countExpr); - countExpr = assignNullType(ifNull, connectContext); + countExpr = assignNullType(ifNull, cascadesContext); } Count countIf = new Count(countExpr); countIfList.add(countIf); @@ -1224,8 +1225,9 @@ public class AggregateStrategies implements ImplementationRuleFactory { } // don't invoke the ExpressionNormalization, because the expression maybe simplified and get rid of some slots - private If assignNullType(If ifExpr, ConnectContext context) { - If ifWithCoercion = (If) TypeCoercion.INSTANCE.rewrite(ifExpr, new ExpressionRewriteContext(context)); + private If assignNullType(If ifExpr, CascadesContext cascadesContext) { + ExpressionRewriteContext context = new ExpressionRewriteContext(cascadesContext); + If ifWithCoercion = (If) TypeCoercion.INSTANCE.rewrite(ifExpr, context); Expression trueValue = ifWithCoercion.getArgument(1); if (trueValue instanceof Cast && trueValue.child(0) instanceof NullLiteral) { List newArgs = Lists.newArrayList(ifWithCoercion.getArguments()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/ImplementationRuleFactory.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/ImplementationRuleFactory.java index ae15eed50e..5e71010c74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/ImplementationRuleFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/ImplementationRuleFactory.java @@ -17,13 +17,14 @@ package org.apache.doris.nereids.rules.implementation; +import org.apache.doris.nereids.pattern.GeneratedMemoPatterns; import org.apache.doris.nereids.rules.PlanRuleFactory; import org.apache.doris.nereids.rules.RulePromise; /** * interface for all implementation rule factories. */ -public interface ImplementationRuleFactory extends PlanRuleFactory { +public interface ImplementationRuleFactory extends PlanRuleFactory, GeneratedMemoPatterns { @Override default RulePromise defaultPromise() { return RulePromise.IMPLEMENT; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java index 78dc66aa7c..1a486c7161 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.OlapTable; import org.apache.doris.common.Pair; import org.apache.doris.nereids.annotation.Developing; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; @@ -56,7 +57,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.util.ExpressionUtils; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -484,9 +484,14 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial Set nonVirtualRequiredScanOutput = requiredScanOutput.stream() .filter(slot -> !(slot instanceof VirtualSlotReference)) .collect(ImmutableSet.toImmutableSet()); - Preconditions.checkArgument(scan.getOutputSet().containsAll(nonVirtualRequiredScanOutput), - String.format("Scan's output (%s) should contains all the input required scan output (%s).", - scan.getOutput(), nonVirtualRequiredScanOutput)); + + // use if condition to skip String.format() and speed up + if (!scan.getOutputSet().containsAll(nonVirtualRequiredScanOutput)) { + throw new AnalysisException( + String.format("Scan's output (%s) should contains all the input required scan output (%s).", + scan.getOutput(), nonVirtualRequiredScanOutput)); + } + OlapTable table = scan.getTable(); switch (scan.getTable().getKeysType()) { case AGG_KEYS: diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java index 7134bd264a..9d004ca5f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.mv; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.OlapTable; import org.apache.doris.nereids.rules.Rule; @@ -114,13 +115,20 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater LogicalOlapScan scan, Supplier> requiredScanOutputSupplier, Supplier> predicatesSupplier) { - switch (scan.getTable().getKeysType()) { + OlapTable table = scan.getTable(); + long baseIndexId = table.getBaseIndexId(); + KeysType keysType = scan.getTable().getKeysType(); + switch (keysType) { case AGG_KEYS: case UNIQUE_KEYS: + break; case DUP_KEYS: + if (table.getIndexIdToMeta().size() == 1) { + return scan.withMaterializedIndexSelected(PreAggStatus.on(), baseIndexId); + } break; default: - throw new RuntimeException("Not supported keys type: " + scan.getTable().getKeysType()); + throw new RuntimeException("Not supported keys type: " + keysType); } if (scan.getTable().isDupKeysOrMergeOnWrite()) { // Set pre-aggregation to `on` to keep consistency with legacy logic. @@ -132,8 +140,16 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater return scan.withMaterializedIndexSelected(PreAggStatus.on(), selectBestIndex(candidate, scan, predicatesSupplier.get())); } else { - OlapTable table = scan.getTable(); - long baseIndexId = table.getBaseIndexId(); + final PreAggStatus preAggStatus; + if (preAggEnabledByHint(scan)) { + // PreAggStatus could be enabled by pre-aggregation hint for agg-keys and unique-keys. + preAggStatus = PreAggStatus.on(); + } else { + preAggStatus = PreAggStatus.off("No aggregate on scan."); + } + if (table.getIndexIdToMeta().size() == 1) { + return scan.withMaterializedIndexSelected(preAggStatus, baseIndexId); + } int baseIndexKeySize = table.getKeyColumnsByIndexId(table.getBaseIndexId()).size(); // No aggregate on scan. // So only base index and indexes that have all the keys could be used. @@ -143,13 +159,6 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater .filter(index -> containAllRequiredColumns(index, scan, requiredScanOutputSupplier.get())) .collect(Collectors.toList()); - final PreAggStatus preAggStatus; - if (preAggEnabledByHint(scan)) { - // PreAggStatus could be enabled by pre-aggregation hint for agg-keys and unique-keys. - preAggStatus = PreAggStatus.on(); - } else { - preAggStatus = PreAggStatus.off("No aggregate on scan."); - } if (candidates.size() == 1) { // `candidates` only have base index. return scan.withMaterializedIndexSelected(preAggStatus, baseIndexId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/BatchRewriteRuleFactory.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/BatchRewriteRuleFactory.java new file mode 100644 index 0000000000..4dcd1998b0 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/BatchRewriteRuleFactory.java @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.rules.PlanRuleFactory; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleFactory; +import org.apache.doris.nereids.rules.RulePromise; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; + +import java.util.List; + +/** BatchRewriteRuleFactory */ +public interface BatchRewriteRuleFactory extends PlanRuleFactory { + @Override + default RulePromise defaultPromise() { + return RulePromise.REWRITE; + } + + @Override + default List buildRules() { + Builder rules = ImmutableList.builder(); + for (RuleFactory ruleFactory : getRuleFactories()) { + rules.addAll(ruleFactory.buildRules()); + } + return rules.build(); + } + + List getRuleFactories(); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteRuleFactory.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteRuleFactory.java index 03c220f03b..c07d5ceb5f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteRuleFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteRuleFactory.java @@ -17,13 +17,14 @@ package org.apache.doris.nereids.rules.rewrite; +import org.apache.doris.nereids.pattern.GeneratedPlanPatterns; import org.apache.doris.nereids.rules.PlanRuleFactory; import org.apache.doris.nereids.rules.RulePromise; /** * interface for all rewrite rule factories. */ -public interface RewriteRuleFactory extends PlanRuleFactory { +public interface RewriteRuleFactory extends PlanRuleFactory, GeneratedPlanPatterns { @Override default RulePromise defaultPromise() { return RulePromise.REWRITE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/CheckAndStandardizeWindowFunctionAndFrame.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/CheckAndStandardizeWindowFunctionAndFrame.java index 2855f6376f..42195eb0dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/CheckAndStandardizeWindowFunctionAndFrame.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/CheckAndStandardizeWindowFunctionAndFrame.java @@ -24,7 +24,7 @@ import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.WindowExpression; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; import com.google.common.base.Preconditions; @@ -45,7 +45,7 @@ public class CheckAndStandardizeWindowFunctionAndFrame extends OneRewriteRuleFac ); } - private LogicalWindow checkAndStandardize(LogicalWindow logicalWindow) { + private LogicalWindow checkAndStandardize(LogicalWindow logicalWindow) { List newOutputExpressions = logicalWindow.getWindowExpressions().stream() .map(expr -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateAggregate.java index 8e3d4977af..3d89320eaa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateAggregate.java @@ -23,7 +23,7 @@ import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Project; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; @@ -34,7 +34,7 @@ public class EliminateAggregate extends OneRewriteRuleFactory { @Override public Rule build() { return logicalAggregate(logicalAggregate()).then(outerAgg -> { - LogicalAggregate innerAgg = outerAgg.child(); + LogicalAggregate innerAgg = outerAgg.child(); if (!isSame(outerAgg.getGroupByExpressions(), innerAgg.getGroupByExpressions())) { return outerAgg; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateGroupByConstant.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateGroupByConstant.java index 1d52871a77..7598352e1e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateGroupByConstant.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateGroupByConstant.java @@ -19,11 +19,14 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewriteContext; import org.apache.doris.nereids.rules.expression.rewrite.rules.FoldConstantRule; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; @@ -46,13 +49,15 @@ import java.util.Set; public class EliminateGroupByConstant extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalAggregate().then(aggregate -> { + return logicalAggregate().thenApply(ctx -> { + LogicalAggregate aggregate = ctx.root; + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); List groupByExprs = aggregate.getGroupByExpressions(); List outputExprs = aggregate.getOutputExpressions(); Set slotGroupByExprs = Sets.newLinkedHashSet(); Expression lit = null; for (Expression expression : groupByExprs) { - expression = FoldConstantRule.INSTANCE.rewrite(expression); + expression = FoldConstantRule.INSTANCE.rewrite(expression, context); if (!(expression instanceof Literal)) { slotGroupByExprs.add(expression); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateLimitUnderApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateLimitUnderApply.java index 2b66e52cd9..1f56660d9c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateLimitUnderApply.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateLimitUnderApply.java @@ -32,7 +32,7 @@ import java.util.List; public class EliminateLimitUnderApply extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalApply(group(), logicalLimit()).then(apply -> { + return logicalApply(any(), logicalLimit()).then(apply -> { List children = new ImmutableList.Builder() .add(apply.left()) .add(apply.right().child()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateNotNull.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateNotNull.java index f7a03f5dfe..039e32ea5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateNotNull.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateNotNull.java @@ -25,6 +25,8 @@ import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.PlanUtils; import org.apache.doris.nereids.util.TypeUtils; @@ -48,7 +50,8 @@ public class EliminateNotNull extends OneRewriteRuleFactory { public Rule build() { return logicalFilter() .when(filter -> filter.getConjuncts().stream().anyMatch(expr -> expr.isGeneratedIsNotNull)) - .then(filter -> { + .thenApply(ctx -> { + LogicalFilter filter = ctx.root; // Progress Example: `id > 0 and id is not null and name is not null(generated)` // predicatesNotContainIsNotNull: `id > 0` // predicatesNotContainIsNotNull infer nonNullable slots: `id` @@ -66,7 +69,8 @@ public class EliminateNotNull extends OneRewriteRuleFactory { predicatesNotContainIsNotNull.add(expr); } }); - Set inferNonNotSlots = ExpressionUtils.inferNotNullSlots(predicatesNotContainIsNotNull); + Set inferNonNotSlots = ExpressionUtils.inferNotNullSlots( + predicatesNotContainIsNotNull, ctx.cascadesContext); Set keepIsNotNull = slotsFromIsNotNull.stream() .filter(ExpressionTrait::nullable) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoin.java index 33fe48fe61..0b6980e276 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoin.java @@ -22,8 +22,8 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.util.TypeUtils; import org.apache.doris.nereids.util.Utils; @@ -45,7 +45,7 @@ public class EliminateOuterJoin extends OneRewriteRuleFactory { return logicalFilter( logicalJoin().when(join -> join.getJoinType().isOuterJoin()) ).then(filter -> { - LogicalJoin join = filter.child(); + LogicalJoin join = filter.child(); Builder conjunctsBuilder = ImmutableSet.builder(); Set notNullSlots = new HashSet<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateSortUnderApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateSortUnderApply.java index 62f092e7e6..64e61a0344 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateSortUnderApply.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateSortUnderApply.java @@ -34,7 +34,7 @@ public class EliminateSortUnderApply implements RewriteRuleFactory { public List buildRules() { return ImmutableList.of( RuleType.ELIMINATE_SORT_UNDER_APPLY.build( - logicalApply(group(), logicalSort()).then(apply -> { + logicalApply(any(), logicalSort()).then(apply -> { List children = new ImmutableList.Builder() .add(apply.left()) .add(apply.right().child()) @@ -42,8 +42,8 @@ public class EliminateSortUnderApply implements RewriteRuleFactory { return apply.withChildren(children); }) ), - RuleType.ELIMINATE_SORT_UNDER_APPLY.build( - logicalApply(group(), logicalProject(logicalSort())).then(apply -> { + RuleType.ELIMINATE_SORT_UNDER_APPLY_PROJECT.build( + logicalApply(any(), logicalProject(logicalSort())).then(apply -> { List children = new ImmutableList.Builder() .add(apply.left()) .add(apply.right().withChildren(apply.right().child().child())) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProject.java index 0754e6b6ee..f62ae4e96e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProject.java @@ -17,63 +17,92 @@ package org.apache.doris.nereids.rules.rewrite.logical; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; +import org.apache.doris.nereids.annotation.DependsRules; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; +import org.apache.doris.nereids.trees.plans.logical.OutputSavePoint; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; -import com.google.common.collect.ImmutableList; - +import java.util.ArrayList; import java.util.List; /** * remove the project that output same with its child to avoid we get two consecutive projects in best plan. * for more information, please see this PR */ -public class EliminateUnnecessaryProject implements RewriteRuleFactory { +@DependsRules(ColumnPruning.class) +public class EliminateUnnecessaryProject implements CustomRewriter { + @Override - public List buildRules() { - return ImmutableList.of( - RuleType.MARK_NECESSARY_PROJECT.build( - logicalSetOperation(logicalProject(), group()) - .thenApply(ctx -> { - LogicalProject project = (LogicalProject) ctx.root.child(0); - return ctx.root.withChildren(project.withEliminate(false), ctx.root.child(1)); - }) - ), - RuleType.MARK_NECESSARY_PROJECT.build( - logicalSetOperation(group(), logicalProject()) - .thenApply(ctx -> { - LogicalProject project = (LogicalProject) ctx.root.child(1); - return ctx.root.withChildren(ctx.root.child(0), project.withEliminate(false)); - }) - ), - RuleType.ELIMINATE_UNNECESSARY_PROJECT.build( - logicalProject(any()) - .when(LogicalProject::canEliminate) - .when(project -> project.getOutputSet().equals(project.child().getOutputSet())) - .thenApply(ctx -> { - int rootGroupId = ctx.cascadesContext.getMemo().getRoot().getGroupId().asInt(); - LogicalProject project = ctx.root; - // if project is root, we need to ensure the output order is same. - if (project.getGroupExpression().get().getOwnerGroup().getGroupId().asInt() - == rootGroupId) { - if (project.getOutput().equals(project.child().getOutput())) { - return project.child(); - } else { - return null; - } - } else { - return project.child(); - } - }) - ), - RuleType.ELIMINATE_UNNECESSARY_PROJECT.build( - logicalProject(logicalEmptyRelation()) - .then(project -> new LogicalEmptyRelation(project.getProjects())) - ) - ); + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + return rewrite(plan, false); + } + + private Plan rewrite(Plan plan, boolean outputSavePoint) { + if (plan instanceof LogicalSetOperation) { + return rewriteLogicalSetOperation((LogicalSetOperation) plan, outputSavePoint); + } else if (plan instanceof LogicalProject) { + return rewriteProject((LogicalProject) plan, outputSavePoint); + } else if (plan instanceof OutputSavePoint) { + return rewriteChildren(plan, true); + } else { + return rewriteChildren(plan, outputSavePoint); + } + } + + private Plan rewriteProject(LogicalProject project, boolean outputSavePoint) { + if (project.child() instanceof LogicalEmptyRelation) { + // eliminate unnecessary project + return new LogicalEmptyRelation(project.getProjects()); + } else if (project.canEliminate() && outputSavePoint + && project.getOutputSet().equals(project.child().getOutputSet())) { + // eliminate unnecessary project + return rewrite(project.child(), outputSavePoint); + } else if (project.canEliminate() && project.getOutput().equals(project.child().getOutput())) { + // eliminate unnecessary project + return rewrite(project.child(), outputSavePoint); + } else { + return rewriteChildren(project, true); + } + } + + private Plan rewriteLogicalSetOperation(LogicalSetOperation set, boolean outputSavePoint) { + if (set.arity() == 2) { + Plan left = set.child(0); + Plan right = set.child(1); + boolean changed = false; + if (isCanEliminateProject(left)) { + changed = true; + left = ((LogicalProject) left).withEliminate(false); + } + if (isCanEliminateProject(right)) { + changed = true; + right = ((LogicalProject) right).withEliminate(false); + } + if (changed) { + set = (LogicalSetOperation) set.withChildren(left, right); + } + } + return rewriteChildren(set, outputSavePoint); + } + + private Plan rewriteChildren(Plan plan, boolean outputSavePoint) { + List newChildren = new ArrayList<>(); + boolean hasNewChildren = false; + for (Plan child : plan.children()) { + Plan newChild = rewrite(child, outputSavePoint); + if (newChild != child) { + hasNewChildren = true; + } + newChildren.add(newChild); + } + return hasNewChildren ? plan.withChildren(newChildren) : plan; + } + + private static boolean isCanEliminateProject(Plan plan) { + return plan instanceof LogicalProject && ((LogicalProject) plan).canEliminate(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoin.java index f0adb815a4..6d01541d3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoin.java @@ -21,8 +21,8 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.PlanUtils; @@ -40,7 +40,7 @@ public class ExtractFilterFromCrossJoin extends OneRewriteRuleFactory { public Rule build() { return crossLogicalJoin() .then(join -> { - LogicalJoin newJoin = new LogicalJoin<>(JoinType.CROSS_JOIN, + LogicalJoin newJoin = new LogicalJoin<>(JoinType.CROSS_JOIN, ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, join.getHint(), join.left(), join.right()); Set predicates = Stream.concat(join.getHashJoinConjuncts().stream(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractSingleTableExpressionFromDisjunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractSingleTableExpressionFromDisjunction.java index 3f88fb2533..5bb029e035 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractSingleTableExpressionFromDisjunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractSingleTableExpressionFromDisjunction.java @@ -115,7 +115,6 @@ public class ExtractSingleTableExpressionFromDisjunction extends OneRewriteRuleF redundants.add(ExpressionUtils.or(extractForAll)); } } - } if (redundants.isEmpty()) { return new LogicalFilter<>(filter.getConjuncts(), true, filter.child()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoin.java index 6ee7546cf0..9700048e5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoin.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.annotation.DependsRules; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; @@ -43,6 +44,9 @@ import java.util.List; * CAUTION: * This rule must be applied after BindSlotReference */ +@DependsRules({ + PushFilterInsideJoin.class +}) public class FindHashConditionForJoin extends OneRewriteRuleFactory { @Override public Rule build() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/HideOneRowRelationUnderUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/HideOneRowRelationUnderUnion.java index 79e7305c1d..2577a31be2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/HideOneRowRelationUnderUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/HideOneRowRelationUnderUnion.java @@ -50,7 +50,7 @@ public class HideOneRowRelationUnderUnion implements AnalysisRuleFactory { public List buildRules() { return ImmutableList.of( RuleType.HIDE_ONE_ROW_RELATION_UNDER_UNION.build( - logicalUnion(logicalOneRowRelation().when(LogicalOneRowRelation::buildUnionNode), group()) + logicalUnion(logicalOneRowRelation().when(LogicalOneRowRelation::buildUnionNode), any()) .then(union -> { List newChildren = new ImmutableList.Builder() .add(((LogicalOneRowRelation) union.child(0)).withBuildUnionNode(false)) @@ -60,7 +60,7 @@ public class HideOneRowRelationUnderUnion implements AnalysisRuleFactory { }) ), RuleType.HIDE_ONE_ROW_RELATION_UNDER_UNION.build( - logicalUnion(group(), logicalOneRowRelation().when(LogicalOneRowRelation::buildUnionNode)) + logicalUnion(any(), logicalOneRowRelation().when(LogicalOneRowRelation::buildUnionNode)) .then(union -> { List children = new ImmutableList.Builder() .add(union.child(0)) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNull.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNull.java index 02fac54ff3..f83f9369cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNull.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNull.java @@ -21,6 +21,8 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.PlanUtils; @@ -43,9 +45,10 @@ public class InferFilterNotNull extends OneRewriteRuleFactory { public Rule build() { return logicalFilter() .when(filter -> filter.getConjuncts().stream().noneMatch(expr -> expr.isGeneratedIsNotNull)) - .then(filter -> { + .thenApply(ctx -> { + LogicalFilter filter = ctx.root; Set predicates = filter.getConjuncts(); - Set isNotNull = ExpressionUtils.inferNotNull(predicates); + Set isNotNull = ExpressionUtils.inferNotNull(predicates, ctx.cascadesContext); if (isNotNull.isEmpty() || predicates.containsAll(isNotNull)) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferJoinNotNull.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferJoinNotNull.java index c4ef028426..4887898637 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferJoinNotNull.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferJoinNotNull.java @@ -44,7 +44,8 @@ public class InferJoinNotNull extends OneRewriteRuleFactory { // TODO: maybe consider ANTI? return logicalJoin().when(join -> join.getJoinType().isInnerJoin() || join.getJoinType().isSemiJoin()) .whenNot(LogicalJoin::isGenerateIsNotNull) - .then(join -> { + .thenApply(ctx -> { + LogicalJoin join = ctx.root; Set conjuncts = new HashSet<>(); conjuncts.addAll(join.getHashJoinConjuncts()); conjuncts.addAll(join.getOtherJoinConjuncts()); @@ -52,15 +53,19 @@ public class InferJoinNotNull extends OneRewriteRuleFactory { Plan left = join.left(); Plan right = join.right(); if (join.getJoinType().isInnerJoin()) { - Set leftNotNull = ExpressionUtils.inferNotNull(conjuncts, join.left().getOutputSet()); - Set rightNotNull = ExpressionUtils.inferNotNull(conjuncts, join.right().getOutputSet()); + Set leftNotNull = ExpressionUtils.inferNotNull( + conjuncts, join.left().getOutputSet(), ctx.cascadesContext); + Set rightNotNull = ExpressionUtils.inferNotNull( + conjuncts, join.right().getOutputSet(), ctx.cascadesContext); left = PlanUtils.filterOrSelf(leftNotNull, join.left()); right = PlanUtils.filterOrSelf(rightNotNull, join.right()); } else if (join.getJoinType() == JoinType.LEFT_SEMI_JOIN) { - Set leftNotNull = ExpressionUtils.inferNotNull(conjuncts, join.left().getOutputSet()); + Set leftNotNull = ExpressionUtils.inferNotNull( + conjuncts, join.left().getOutputSet(), ctx.cascadesContext); left = PlanUtils.filterOrSelf(leftNotNull, join.left()); } else { - Set rightNotNull = ExpressionUtils.inferNotNull(conjuncts, join.right().getOutputSet()); + Set rightNotNull = ExpressionUtils.inferNotNull( + conjuncts, join.right().getOutputSet(), ctx.cascadesContext); right = PlanUtils.filterOrSelf(rightNotNull, join.right()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferPredicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferPredicates.java index 12eb6f0e0a..38aa6bc6c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferPredicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferPredicates.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; import org.apache.doris.nereids.util.ExpressionUtils; @@ -49,10 +50,15 @@ import java.util.stream.Collectors; * 2. put these predicates into `otherJoinConjuncts` , these predicates are processed in the next * round of predicate push-down */ -public class InferPredicates extends DefaultPlanRewriter { +public class InferPredicates extends DefaultPlanRewriter implements CustomRewriter { private final PredicatePropagation propagation = new PredicatePropagation(); private final PullUpPredicates pollUpPredicates = new PullUpPredicates(); + @Override + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + return plan.accept(this, jobContext); + } + @Override public Plan visitLogicalJoin(LogicalJoin join, JobContext context) { join = (LogicalJoin) super.visit(join, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeFilters.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeFilters.java index a992ca68c9..6b1cedd5f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeFilters.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeFilters.java @@ -30,16 +30,12 @@ import com.google.common.collect.ImmutableSet; * this rule aims to merge consecutive filters. * For example: * logical plan tree: - * project - * | * filter(a>0) * | * filter(b>0) * | * scan * transformed to: - * project - * | * filter(a>0 and b>0) * | * scan diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeGenerates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeGenerates.java index b596e0eba9..3a42bde534 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeGenerates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeGenerates.java @@ -22,7 +22,7 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.Function; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import com.google.common.collect.Lists; @@ -37,7 +37,7 @@ public class MergeGenerates extends OneRewriteRuleFactory { @Override public Rule build() { return logicalGenerate(logicalGenerate()).then(top -> { - LogicalGenerate bottom = top.child(); + LogicalGenerate bottom = top.child(); Set topGeneratorSlots = top.getInputSlots(); if (bottom.getGeneratorOutput().stream().anyMatch(topGeneratorSlots::contains)) { // top generators use bottom's generator's output, cannot merge. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjects.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjects.java index 80dfcb7082..4afbbba883 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjects.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjects.java @@ -21,7 +21,7 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import java.util.List; @@ -47,7 +47,7 @@ public class MergeProjects extends OneRewriteRuleFactory { @Override public Rule build() { return logicalProject(logicalProject()).then(project -> { - LogicalProject childProject = project.child(); + LogicalProject childProject = project.child(); List projectExpressions = project.mergeProjections(childProject); return new LogicalProject<>(projectExpressions, childProject.child(0)); }).toRule(RuleType.MERGE_PROJECTS); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeSetOperations.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeSetOperations.java index d3a7cf1526..56f924684c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeSetOperations.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeSetOperations.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; import com.google.common.collect.ImmutableList; import java.util.List; +import java.util.stream.Stream; /** * optimization. @@ -51,48 +52,44 @@ public class MergeSetOperations implements RewriteRuleFactory { public List buildRules() { return ImmutableList.of( RuleType.MERGE_SET_OPERATION.build( - logicalSetOperation(logicalSetOperation(), group()).thenApply(ctx -> { - LogicalSetOperation parentSetOperation = ctx.root; - LogicalSetOperation childSetOperation = (LogicalSetOperation) parentSetOperation.child(0); + logicalSetOperation(any(), any()).when(MergeSetOperations::canMerge).then(parentSetOperation -> { + List newChildren = parentSetOperation.children() + .stream() + .flatMap(child -> { + if (canMerge(parentSetOperation, child)) { + return child.children().stream(); + } else { + return Stream.of(child); + } + }).collect(ImmutableList.toImmutableList()); - if (isSameClass(parentSetOperation, childSetOperation) - && isSameQualifierOrChildQualifierIsAll(parentSetOperation, childSetOperation)) { - List newChildren = new ImmutableList.Builder() - .addAll(childSetOperation.children()) - .add(parentSetOperation.child(1)) - .build(); - return parentSetOperation.withChildren(newChildren); - } - return parentSetOperation; - }) - ), - RuleType.MERGE_SET_OPERATION.build( - logicalSetOperation(group(), logicalSetOperation()).thenApply(ctx -> { - LogicalSetOperation parentSetOperation = ctx.root; - LogicalSetOperation childSetOperation = (LogicalSetOperation) parentSetOperation.child(1); - - if (isSameClass(parentSetOperation, childSetOperation) - && isSameQualifierOrChildQualifierIsAll(parentSetOperation, childSetOperation)) { - List newChildren = new ImmutableList.Builder() - .add(parentSetOperation.child(0)) - .addAll(childSetOperation.children()) - .build(); - return parentSetOperation.withNewChildren(newChildren); - } - return parentSetOperation; + return parentSetOperation.withChildren(newChildren); }) ) ); } - private boolean isSameQualifierOrChildQualifierIsAll(LogicalSetOperation parentSetOperation, + /** canMerge */ + public static boolean canMerge(LogicalSetOperation parent) { + Plan left = parent.child(0); + if (canMerge(parent, left)) { + return true; + } + Plan right = parent.child(1); + if (canMerge(parent, right)) { + return true; + } + return false; + } + + public static final boolean canMerge(LogicalSetOperation parent, Plan child) { + return child.getClass().equals(parent.getClass()) + && isSameQualifierOrChildQualifierIsAll(parent, (LogicalSetOperation) child); + } + + public static final boolean isSameQualifierOrChildQualifierIsAll(LogicalSetOperation parentSetOperation, LogicalSetOperation childSetOperation) { return parentSetOperation.getQualifier() == childSetOperation.getQualifier() || childSetOperation.getQualifier() == Qualifier.ALL; } - - private boolean isSameClass(LogicalSetOperation parentSetOperation, - LogicalSetOperation childSetOperation) { - return parentSetOperation.getClass().isAssignableFrom(childSetOperation.getClass()); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneAggChildColumns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneAggChildColumns.java index d8a31543fa..47df6ad4f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneAggChildColumns.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneAggChildColumns.java @@ -23,7 +23,7 @@ import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.util.ExpressionUtils; @@ -78,7 +78,7 @@ public class PruneAggChildColumns extends OneRewriteRuleFactory { * @return null, if there exists an aggregation function that its parameters contains non-constant expr. * else return a slot with min data type. */ - private boolean isAggregateWithConstant(LogicalAggregate agg) { + private boolean isAggregateWithConstant(LogicalAggregate agg) { for (NamedExpression output : agg.getOutputExpressions()) { if (output.anyMatch(SlotReference.class::isInstance)) { return false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneFilterChildColumns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneFilterChildColumns.java index bcd925338e..b1f28def1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneFilterChildColumns.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneFilterChildColumns.java @@ -19,7 +19,6 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -51,7 +50,7 @@ import java.util.stream.Stream; * | * scan(k1,k2,k3,v1) */ -public class PruneFilterChildColumns extends AbstractPushDownProjectRule> { +public class PruneFilterChildColumns extends AbstractPushDownProjectRule> { public PruneFilterChildColumns() { setRuleType(RuleType.COLUMN_PRUNE_FILTER_CHILD); @@ -59,7 +58,7 @@ public class PruneFilterChildColumns extends AbstractPushDownProjectRule filter, Set references) { + protected Plan pushDownProject(LogicalFilter filter, Set references) { Set filterInputSlots = filter.getInputSlots(); Set required = Stream.concat(references.stream(), filterInputSlots.stream()).collect(Collectors.toSet()); if (required.containsAll(filter.child().getOutput())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneJoinChildrenColumns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneJoinChildrenColumns.java index f05271d32d..daf4ad3a3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneJoinChildrenColumns.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneJoinChildrenColumns.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -56,7 +55,7 @@ import java.util.stream.Stream; * scan scan */ public class PruneJoinChildrenColumns - extends AbstractPushDownProjectRule> { + extends AbstractPushDownProjectRule> { public PruneJoinChildrenColumns() { setRuleType(RuleType.COLUMN_PRUNE_JOIN_CHILD); @@ -64,7 +63,7 @@ public class PruneJoinChildrenColumns } @Override - protected Plan pushDownProject(LogicalJoin joinPlan, + protected Plan pushDownProject(LogicalJoin joinPlan, Set references) { Set exprIds = Stream.of(references, joinPlan.getInputSlots()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneSortChildColumns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneSortChildColumns.java index 372d03f8d0..90adbd067a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneSortChildColumns.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneSortChildColumns.java @@ -19,7 +19,6 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; @@ -34,7 +33,7 @@ import java.util.stream.Stream; * prune join children output. * pattern: project(sort()) */ -public class PruneSortChildColumns extends AbstractPushDownProjectRule> { +public class PruneSortChildColumns extends AbstractPushDownProjectRule> { public PruneSortChildColumns() { setRuleType(RuleType.COLUMN_PRUNE_SORT_CHILD); @@ -42,7 +41,7 @@ public class PruneSortChildColumns extends AbstractPushDownProjectRule sortPlan, Set references) { + protected Plan pushDownProject(LogicalSort sortPlan, Set references) { Set sortSlots = sortPlan.getOutputSet(); Set required = Stream.concat(references.stream(), sortSlots.stream()).collect(Collectors.toSet()); if (required.containsAll(sortPlan.child().getOutput())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnProjectUnderAgg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpCorrelatedFilterUnderApplyAggregateProject.java similarity index 82% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnProjectUnderAgg.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpCorrelatedFilterUnderApplyAggregateProject.java index 232d5d3190..403008a9d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnProjectUnderAgg.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpCorrelatedFilterUnderApplyAggregateProject.java @@ -21,12 +21,13 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import java.util.List; @@ -57,15 +58,15 @@ import java.util.List; * child * */ -public class ApplyPullFilterOnProjectUnderAgg extends OneRewriteRuleFactory { +public class PullUpCorrelatedFilterUnderApplyAggregateProject extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalApply(group(), logicalAggregate(logicalProject(logicalFilter()))) + return logicalApply(any(), logicalAggregate(logicalProject(logicalFilter()))) .when(LogicalApply::isCorrelated).then(apply -> { - LogicalAggregate>> agg = apply.right(); + LogicalAggregate>> agg = apply.right(); - LogicalProject> project = agg.child(); - LogicalFilter filter = project.child(); + LogicalProject> project = agg.child(); + LogicalFilter filter = project.child(); List newProjects = Lists.newArrayList(); newProjects.addAll(project.getProjects()); filter.child().getOutput().forEach(slot -> { @@ -76,10 +77,9 @@ public class ApplyPullFilterOnProjectUnderAgg extends OneRewriteRuleFactory { LogicalProject newProject = new LogicalProject<>(newProjects, filter.child()); LogicalFilter newFilter = new LogicalFilter<>(filter.getConjuncts(), newProject); - LogicalAggregate newAgg = new LogicalAggregate<>(agg.getGroupByExpressions(), - agg.getOutputExpressions(), agg.isOrdinalIsResolved(), newFilter); + LogicalAggregate newAgg = agg.withChildren(ImmutableList.of(newFilter)); return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), apply.getCorrelationFilter(), apply.left(), newAgg); - }).toRule(RuleType.APPLY_PULL_FILTER_ON_PROJECT_UNDER_AGG); + }).toRule(RuleType.PULL_UP_CORRELATED_FILTER_UNDER_APPLY_AGGREGATE_PROJECT); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushApplyUnderProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpProjectUnderApply.java similarity index 82% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushApplyUnderProject.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpProjectUnderApply.java index 0945b2706e..7f993f1798 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushApplyUnderProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpProjectUnderApply.java @@ -20,8 +20,8 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.ScalarSubquery; -import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -47,10 +47,10 @@ import java.util.List; * / \ * Input(output:b) child */ -public class PushApplyUnderProject extends OneRewriteRuleFactory { +public class PullUpProjectUnderApply extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalApply(group(), logicalProject(any())) + return logicalApply(any(), logicalProject(any())) .when(LogicalApply::isCorrelated) .whenNot(apply -> apply.right().child() instanceof LogicalFilter && apply.isIn()) .whenNot(LogicalApply::alreadyExecutedEliminateFilter) @@ -58,12 +58,12 @@ public class PushApplyUnderProject extends OneRewriteRuleFactory { LogicalProject project = apply.right(); LogicalApply newCorrelate = new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), apply.getCorrelationFilter(), apply.left(), project.child()); - List newSlots = new ArrayList<>(); - newSlots.addAll(apply.left().getOutput()); + List newProjects = new ArrayList<>(); + newProjects.addAll(apply.left().getOutput()); if (apply.getSubqueryExpr() instanceof ScalarSubquery) { - newSlots.add(apply.right().getOutput().get(0)); + newProjects.add(project.getProjects().get(0)); } - return new LogicalProject(newSlots, newCorrelate); - }).toRule(RuleType.PUSH_APPLY_UNDER_PROJECT); + return new LogicalProject(newProjects, newCorrelate); + }).toRule(RuleType.PULL_UP_PROJECT_UNDER_APPLY); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoin.java index 0ea7f62f86..b2d192c163 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoin.java @@ -17,11 +17,12 @@ package org.apache.doris.nereids.rules.rewrite.logical; +import org.apache.doris.nereids.annotation.DependsRules; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import com.google.common.collect.Lists; @@ -31,6 +32,10 @@ import java.util.List; /** * Push the predicate in the LogicalFilter to the join children. */ +@DependsRules({ + InferPredicates.class, + EliminateOuterJoin.class +}) public class PushFilterInsideJoin extends OneRewriteRuleFactory { @Override public Rule build() { @@ -40,7 +45,7 @@ public class PushFilterInsideJoin extends OneRewriteRuleFactory { || filter.child().getJoinType().isInnerJoin()) .then(filter -> { List otherConditions = Lists.newArrayList(filter.getConjuncts()); - LogicalJoin join = filter.child(); + LogicalJoin join = filter.child(); otherConditions.addAll(join.getOtherJoinConjuncts()); return new LogicalJoin<>(join.getJoinType(), join.getHashJoinConjuncts(), otherConditions, join.getHint(), join.left(), join.right()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughAggregation.java index 575f5c419e..a891eb07ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughAggregation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughAggregation.java @@ -22,7 +22,6 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -66,7 +65,7 @@ public class PushdownFilterThroughAggregation extends OneRewriteRuleFactory { @Override public Rule build() { return logicalFilter(logicalAggregate()).then(filter -> { - LogicalAggregate aggregate = filter.child(); + LogicalAggregate aggregate = filter.child(); Set canPushDownSlots = new HashSet<>(); if (aggregate.hasRepeat()) { // When there is a repeat, the push-down condition is consistent with the repeat diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoin.java index e95f7bb229..64ff0cd514 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoin.java @@ -24,8 +24,8 @@ import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.util.PlanUtils; @@ -94,7 +94,7 @@ public class PushdownFilterThroughJoin extends OneRewriteRuleFactory { public Rule build() { return logicalFilter(logicalJoin()).then(filter -> { - LogicalJoin join = filter.child(); + LogicalJoin join = filter.child(); Set predicates = filter.getConjuncts(); @@ -113,9 +113,9 @@ public class PushdownFilterThroughJoin extends OneRewriteRuleFactory { } } - Set leftPredicates = Sets.newHashSet(); - Set rightPredicates = Sets.newHashSet(); - Set remainingPredicates = Sets.newHashSet(); + Set leftPredicates = Sets.newLinkedHashSet(); + Set rightPredicates = Sets.newLinkedHashSet(); + Set remainingPredicates = Sets.newLinkedHashSet(); for (Expression p : filterPredicates) { Set slots = p.collect(SlotReference.class::isInstance); if (slots.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java index ac56572491..8712340f7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java @@ -19,12 +19,17 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.util.ExpressionUtils; +import com.google.common.collect.ImmutableList; + +import java.util.List; + /** * Push down filter through project. * input: @@ -32,18 +37,38 @@ import org.apache.doris.nereids.util.ExpressionUtils; * output: * project(c+d as a, e as b) -> filter(c+d>2, e=0). */ -public class PushdownFilterThroughProject extends OneRewriteRuleFactory { +public class PushdownFilterThroughProject implements RewriteRuleFactory { @Override - public Rule build() { - return logicalFilter(logicalProject()).then(filter -> { - LogicalProject project = filter.child(); - return new LogicalProject<>( - project.getProjects(), - new LogicalFilter<>( - ExpressionUtils.replace(filter.getConjuncts(), project.getAliasToProducer()), - project.child() - ) - ); - }).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_PROJECT); + public List buildRules() { + return ImmutableList.of( + RuleType.PUSHDOWN_FILTER_THROUGH_PROJECT.build( + logicalFilter(logicalProject()).then(filter -> { + LogicalProject project = filter.child(); + return new LogicalProject<>( + project.getProjects(), + new LogicalFilter<>( + ExpressionUtils.replace(filter.getConjuncts(), project.getAliasToProducer()), + project.child() + ) + ); + }) + ), + // filter(project(limit)) will change to filter(limit(project)) by PushdownProjectThroughLimit, + // then we should change filter(limit(project)) to project(filter(limit)) + RuleType.PUSHDOWN_FILTER_THROUGH_PROJECT_UNDER_LIMIT.build( + logicalFilter(logicalLimit(logicalProject())).then(filter -> { + LogicalLimit> limit = filter.child(); + LogicalProject project = limit.child(); + + return new LogicalProject<>( + project.getProjects(), + new LogicalFilter<>( + ExpressionUtils.replace(filter.getConjuncts(), project.getAliasToProducer()), + limit.withChildren(project.child()) + ) + ); + }) + ) + ); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughRepeat.java index 0d5f2c3f64..e19b364042 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughRepeat.java @@ -22,7 +22,6 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; @@ -65,7 +64,7 @@ public class PushdownFilterThroughRepeat extends OneRewriteRuleFactory { @Override public Rule build() { return logicalFilter(logicalRepeat()).then(filter -> { - LogicalRepeat repeat = filter.child(); + LogicalRepeat repeat = filter.child(); Set commonGroupingSetExpressions = repeat.getCommonGroupingSetExpressions(); if (commonGroupingSetExpressions.isEmpty()) { return filter; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java index 4faed0a5cf..b9f0f70d2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java @@ -20,7 +20,7 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -48,10 +48,10 @@ public class PushdownProjectThroughLimit extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalProject(logicalLimit(group())).thenApply(ctx -> { - LogicalProject> logicalProject = ctx.root; - LogicalLimit logicalLimit = logicalProject.child(); - return new LogicalLimit>(logicalLimit.getLimit(), + return logicalProject(logicalLimit(any())).thenApply(ctx -> { + LogicalProject> logicalProject = ctx.root; + LogicalLimit logicalLimit = logicalProject.child(); + return new LogicalLimit<>(logicalLimit.getLimit(), logicalLimit.getOffset(), new LogicalProject<>(logicalProject.getProjects(), logicalLimit.child())); }).toRule(RuleType.PUSHDOWN_PROJECT_THROUGH_LIMIT); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java index caea65b552..ede6586a3a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java @@ -18,12 +18,12 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.annotation.DependsRules; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinHint; import org.apache.doris.nereids.trees.plans.JoinHint.JoinHintType; import org.apache.doris.nereids.trees.plans.JoinType; @@ -68,6 +68,9 @@ import java.util.stream.Collectors; *
  • MultiJoin to {Join cluster}
    • * */ +@DependsRules({ + MergeFilters.class +}) public class ReorderJoin extends OneRewriteRuleFactory { @Override public Rule build() { @@ -94,8 +97,8 @@ public class ReorderJoin extends OneRewriteRuleFactory { */ public Plan joinToMultiJoin(Plan plan, Map planToHintType) { // subtree can't specify the end of Pattern. so end can be GroupPlan or Filter - if (plan instanceof GroupPlan - || (plan instanceof LogicalFilter && plan.child(0) instanceof GroupPlan)) { + if (nonJoinAndNonFilter(plan) + || (plan instanceof LogicalFilter && nonJoinAndNonFilter(plan.child(0)))) { return plan; } @@ -368,4 +371,8 @@ public class ReorderJoin extends OneRewriteRuleFactory { throw new RuntimeException("findInnerJoin: can't reach here"); } + + private boolean nonJoinAndNonFilter(Plan plan) { + return !(plan instanceof LogicalJoin) && !(plan instanceof LogicalFilter); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnAgg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyAggregateFilter.java similarity index 89% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnAgg.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyAggregateFilter.java index 8998ba6afe..2b85431711 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnAgg.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyAggregateFilter.java @@ -22,7 +22,7 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -59,12 +59,12 @@ import java.util.Map; * Filter(Uncorrelated predicate) * */ -public class ApplyPullFilterOnAgg extends OneRewriteRuleFactory { +public class UnCorrelatedApplyAggregateFilter extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalApply(group(), logicalAggregate(logicalFilter())).when(LogicalApply::isCorrelated).then(apply -> { - LogicalAggregate> agg = apply.right(); - LogicalFilter filter = agg.child(); + return logicalApply(any(), logicalAggregate(logicalFilter())).when(LogicalApply::isCorrelated).then(apply -> { + LogicalAggregate> agg = apply.right(); + LogicalFilter filter = agg.child(); Map> split = Utils.splitCorrelatedConjuncts( filter.getConjuncts(), apply.getCorrelationSlot()); List correlatedPredicate = split.get(true); @@ -89,6 +89,6 @@ public class ApplyPullFilterOnAgg extends OneRewriteRuleFactory { apply.getSubqueryExpr(), ExpressionUtils.optionalAnd(correlatedPredicate), apply.left(), newAgg); - }).toRule(RuleType.APPLY_PULL_FILTER_ON_AGG); + }).toRule(RuleType.UN_CORRELATED_APPLY_AGGREGATE_FILTER); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushApplyUnderFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyFilter.java similarity index 89% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushApplyUnderFilter.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyFilter.java index b8833a7a9d..1ffbcf2214 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushApplyUnderFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyFilter.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -51,11 +50,11 @@ import java.util.Set; * / \ * Input(output:b) Filter(UnCorrelated predicate) */ -public class PushApplyUnderFilter extends OneRewriteRuleFactory { +public class UnCorrelatedApplyFilter extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalApply(group(), logicalFilter()).when(LogicalApply::isCorrelated).then(apply -> { - LogicalFilter filter = apply.right(); + return logicalApply(any(), logicalFilter()).when(LogicalApply::isCorrelated).then(apply -> { + LogicalFilter filter = apply.right(); Set conjuncts = filter.getConjuncts(); Map> split = Utils.splitCorrelatedConjuncts( conjuncts, apply.getCorrelationSlot()); @@ -71,6 +70,6 @@ public class PushApplyUnderFilter extends OneRewriteRuleFactory { return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), ExpressionUtils.optionalAnd(correlatedPredicate), apply.left(), child); - }).toRule(RuleType.PUSH_APPLY_UNDER_FILTER); + }).toRule(RuleType.UN_CORRELATED_APPLY_FILTER); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateFilterUnderApplyProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyProjectFilter.java similarity index 90% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateFilterUnderApplyProject.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyProjectFilter.java index e72302b9cb..23d9ec025a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateFilterUnderApplyProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyProjectFilter.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -60,15 +59,15 @@ import java.util.Set; * | * child */ -public class EliminateFilterUnderApplyProject extends OneRewriteRuleFactory { +public class UnCorrelatedApplyProjectFilter extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalApply(group(), logicalProject(logicalFilter())) + return logicalApply(any(), logicalProject(logicalFilter())) .when(LogicalApply::isCorrelated) .when(LogicalApply::isIn) .then(apply -> { - LogicalProject> project = apply.right(); - LogicalFilter filter = project.child(); + LogicalProject> project = apply.right(); + LogicalFilter filter = project.child(); Set conjuncts = filter.getConjuncts(); Map> split = Utils.splitCorrelatedConjuncts( conjuncts, apply.getCorrelationSlot()); @@ -92,6 +91,6 @@ public class EliminateFilterUnderApplyProject extends OneRewriteRuleFactory { return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), ExpressionUtils.optionalAnd(correlatedPredicate), apply.left(), newProject); - }).toRule(RuleType.ELIMINATE_FILTER_UNDER_APPLY_PROJECT); + }).toRule(RuleType.UN_CORRELATED_APPLY_PROJECT_FILTER); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java index 0320e3ee95..d8f10a362d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java @@ -179,4 +179,21 @@ public interface TreeNode> { return false; }); } + + /** + * equals by the full tree nodes + * @param that other tree node + * @return true if all the tree is equals + */ + default boolean deepEquals(TreeNode that) { + if (!equals(that)) { + return false; + } + for (int i = 0; i < arity(); i++) { + if (!child(i).deepEquals(that.child(i))) { + return false; + } + } + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java index d9cd7c695e..7f2628e03f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java @@ -77,15 +77,11 @@ public class Exists extends SubqueryExpr implements LeafExpression { @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { + if (!super.equals(o)) { return false; } Exists other = (Exists) o; - return Objects.equals(this.queryPlan, other.getQueryPlan()) - && Objects.equals(this.isNot, other.isNot()); + return this.isNot == other.isNot; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java index d3080b283c..06697f82da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java @@ -199,7 +199,15 @@ public abstract class Expression extends AbstractTreeNode implements * This expression has unbound symbols or not. */ public boolean hasUnbound() { - return this.anyMatch(Unbound.class::isInstance); + if (this instanceof Unbound) { + return true; + } + for (Expression child : children) { + if (child.hasUnbound()) { + return true; + } + } + return false; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java index 77423b165f..fe1dc5428f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java @@ -95,16 +95,12 @@ public class InSubquery extends SubqueryExpr { @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { + if (!super.equals(o)) { return false; } InSubquery inSubquery = (InSubquery) o; return Objects.equals(this.compareExpr, inSubquery.getCompareExpr()) - && checkEquals(this.queryPlan, inSubquery.queryPlan) - && Objects.equals(this.isNot, inSubquery.isNot); + && this.isNot == inSubquery.isNot; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java index 047c985de0..8e5e9080df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java @@ -60,7 +60,7 @@ public class IsNull extends Expression implements UnaryExpression { @Override public String toString() { - return toSql(); + return child().toString() + " IS NULL"; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index e378b580ee..b0f4e829de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -148,7 +148,8 @@ public class SlotReference extends Slot { // The contains method needs to use hashCode, so similar to equals, it only compares exprId @Override public int hashCode() { - return Objects.hash(exprId); + // direct return exprId to speed up + return exprId.asInt(); } public Optional getColumn() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java index 44146728c9..d98a72f06a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java @@ -77,6 +77,11 @@ public abstract class SubqueryExpr extends Expression { return queryPlan; } + @Override + public boolean hasUnbound() { + return super.hasUnbound() || !queryPlan.bound(); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -86,39 +91,8 @@ public abstract class SubqueryExpr extends Expression { return false; } SubqueryExpr other = (SubqueryExpr) o; - return checkEquals(queryPlan, other.queryPlan) - && Objects.equals(correlateSlots, other.correlateSlots); - } - - /** - * Compare whether all logical nodes under query are the same. - * @param i original query. - * @param o compared query. - * @return equal ? true : false; - */ - protected boolean checkEquals(Object i, Object o) { - if (!(i instanceof LogicalPlan) || !(o instanceof LogicalPlan)) { - return false; - } - LogicalPlan other = (LogicalPlan) o; - LogicalPlan input = (LogicalPlan) i; - if (other.children().size() != input.children().size()) { - return false; - } - boolean equal; - for (int j = 0; j < input.children().size(); j++) { - LogicalPlan childInput = (LogicalPlan) input.child(j); - LogicalPlan childOther = (LogicalPlan) other.child(j); - equal = Objects.equals(childInput, childOther); - if (!equal) { - return false; - } - if (childInput.children().size() != childOther.children().size()) { - return false; - } - checkEquals(childInput, childOther); - } - return true; + return Objects.equals(correlateSlots, other.correlateSlots) + && queryPlan.deepEquals(other.queryPlan); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java index f3f829eade..198a28742a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.trees.plans; import org.apache.doris.nereids.analyzer.Unbound; -import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.metrics.CounterType; @@ -32,6 +31,8 @@ import org.apache.doris.nereids.properties.UnboundLogicalProperties; import org.apache.doris.nereids.trees.AbstractTreeNode; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.util.MutableState; +import org.apache.doris.nereids.util.MutableState.EmptyMutableState; import org.apache.doris.nereids.util.TreeStringUtils; import org.apache.doris.statistics.StatsDeriveResult; @@ -52,11 +53,19 @@ public abstract class AbstractPlan extends AbstractTreeNode implements Pla EventChannel.getDefaultChannel() .addEnhancers(new AddCounterEventEnhancer()) .addConsumers(new LogConsumer(CounterEvent.class, EventChannel.LOG))); + protected final StatsDeriveResult statsDeriveResult; protected final PlanType type; protected final Optional groupExpression; protected final Supplier logicalPropertiesSupplier; + // this field is special, because other fields in tree node is immutable, but in some scenes, mutable + // state is necessary. e.g. the rewrite framework need distinguish whether the plan is created by + // rules, the framework can set this field to a state variable to quickly judge without new big plan. + // we should avoid using it as much as possible, because mutable state is easy to cause bugs and + // difficult to locate. + private MutableState mutableState = EmptyMutableState.INSTANCE; + public AbstractPlan(PlanType type, Plan... children) { this(type, Optional.empty(), Optional.empty(), null, children); } @@ -96,9 +105,7 @@ public abstract class AbstractPlan extends AbstractTreeNode implements Pla @Override public boolean canBind() { - return !bound() - && !(this instanceof Unbound) - && childrenBound(); + return !bound() && childrenBound(); } /** @@ -161,9 +168,19 @@ public abstract class AbstractPlan extends AbstractTreeNode implements Pla @Override public LogicalProperties getLogicalProperties() { - if (this instanceof UnboundRelation) { + if (this instanceof Unbound) { return UnboundLogicalProperties.INSTANCE; } return logicalPropertiesSupplier.get(); } + + @Override + public Optional getMutableState(String key) { + return mutableState.get(key); + } + + @Override + public void setMutableState(String key, Object state) { + this.mutableState = this.mutableState.set(key, state); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java index 91a392732f..1708878fa4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java @@ -22,6 +22,8 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.MutableState; +import org.apache.doris.nereids.util.MutableState.MultiMutableState; import com.google.common.collect.ImmutableList; @@ -33,6 +35,7 @@ import java.util.Optional; * Used for unit test only. */ public class FakePlan implements Plan { + private MutableState mutableState = new MultiMutableState(); @Override public List children() { @@ -108,4 +111,14 @@ public class FakePlan implements Plan { public Plan withLogicalProperties(Optional logicalProperties) { return this; } + + @Override + public Optional getMutableState(String key) { + return (Optional) Optional.ofNullable(mutableState.get(key)); + } + + @Override + public void setMutableState(String key, Object mutableState) { + this.mutableState = this.mutableState.set(key, mutableState); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java index e0702937a3..018af58ba6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java @@ -33,6 +33,7 @@ import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -129,4 +130,19 @@ public interface Plan extends TreeNode { Plan withGroupExpression(Optional groupExpression); Plan withLogicalProperties(Optional logicalProperties); + + Optional getMutableState(String key); + + /** getOrInitMutableState */ + default T getOrInitMutableState(String key, Supplier initState) { + Optional mutableState = getMutableState(key); + if (!mutableState.isPresent()) { + T state = initState.get(); + setMutableState(key, state); + return state; + } + return mutableState.get(); + } + + void setMutableState(String key, Object value); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index 5afe8b4cf6..50ce5ff669 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -83,5 +83,9 @@ public enum PlanType { PHYSICAL_ASSERT_NUM_ROWS, PHYSICAL_UNION, PHYSICAL_EXCEPT, - PHYSICAL_INTERSECT + PHYSICAL_INTERSECT, + + COMMAND, + EXPLAIN_COMMAND, + CREATE_POLICY_COMMAND } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java index 9cec62708b..9bf9fb04b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java @@ -21,9 +21,13 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.statistics.StatsDeriveResult; + +import org.jetbrains.annotations.Nullable; import java.util.List; import java.util.Optional; @@ -31,75 +35,90 @@ import java.util.Optional; /** * All DDL and DML commands' super class. */ -public interface Command extends LogicalPlan { +public abstract class Command extends AbstractPlan implements LogicalPlan { + + public Command(PlanType type, Plan... children) { + super(type, children); + } + + public Command(PlanType type, Optional optLogicalProperties, Plan... children) { + super(type, optLogicalProperties, children); + } + + public Command(PlanType type, Optional groupExpression, + Optional optLogicalProperties, + @Nullable StatsDeriveResult statsDeriveResult, + Plan... children) { + super(type, groupExpression, optLogicalProperties, statsDeriveResult, children); + } @Override - default Optional getGroupExpression() { + public Optional getGroupExpression() { throw new RuntimeException("Command do not implement getGroupExpression"); } @Override - default List children() { + public List children() { throw new RuntimeException("Command do not implement children"); } @Override - default Plan child(int index) { + public Plan child(int index) { throw new RuntimeException("Command do not implement child"); } @Override - default int arity() { + public int arity() { throw new RuntimeException("Command do not implement arity"); } @Override - default Plan withChildren(List children) { + public Plan withChildren(List children) { throw new RuntimeException("Command do not implement withChildren"); } @Override - default PlanType getType() { + public PlanType getType() { throw new RuntimeException("Command do not implement getType"); } @Override - default List getExpressions() { + public List getExpressions() { throw new RuntimeException("Command do not implement getExpressions"); } @Override - default LogicalProperties getLogicalProperties() { + public LogicalProperties getLogicalProperties() { throw new RuntimeException("Command do not implement getLogicalProperties"); } @Override - default boolean canBind() { + public boolean canBind() { throw new RuntimeException("Command do not implement canResolve"); } @Override - default List getOutput() { + public List getOutput() { throw new RuntimeException("Command do not implement getOutput"); } @Override - default List getNonUserVisibleOutput() { + public List getNonUserVisibleOutput() { throw new RuntimeException("Command do not implement getNonUserVisibleOutput"); } @Override - default String treeString() { + public String treeString() { throw new RuntimeException("Command do not implement treeString"); } @Override - default Plan withGroupExpression(Optional groupExpression) { + public Plan withGroupExpression(Optional groupExpression) { throw new RuntimeException("Command do not implement withGroupExpression"); } @Override - default Plan withLogicalProperties(Optional logicalProperties) { + public Plan withLogicalProperties(Optional logicalProperties) { throw new RuntimeException("Command do not implement withLogicalProperties"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreatePolicyCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreatePolicyCommand.java index f6ddc8217e..a405a61cb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreatePolicyCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreatePolicyCommand.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.commands; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.policy.PolicyTypeEnum; @@ -26,13 +27,14 @@ import java.util.Optional; /** * Create policy command. */ -public class CreatePolicyCommand implements Command { +public class CreatePolicyCommand extends Command { private PolicyTypeEnum type; private final Optional wherePredicate; public CreatePolicyCommand(PolicyTypeEnum type, Expression expr) { + super(PlanType.CREATE_POLICY_COMMAND); this.type = type; this.wherePredicate = Optional.of(expr); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java index adf9cd5ba4..c96a95cc23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java @@ -17,13 +17,14 @@ package org.apache.doris.nereids.trees.plans.commands; +import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; /** * explain command. */ -public class ExplainCommand implements Command { +public class ExplainCommand extends Command { /** * explain level. @@ -51,6 +52,7 @@ public class ExplainCommand implements Command { private final LogicalPlan logicalPlan; public ExplainCommand(ExplainLevel level, LogicalPlan logicalPlan) { + super(PlanType.EXPLAIN_COMMAND); this.level = level; this.logicalPlan = logicalPlan; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java index feef64e3a6..704a0dd222 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java @@ -25,12 +25,15 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import java.util.Optional; +import java.util.function.Supplier; /** * Abstract class for all concrete logical plan. */ public abstract class AbstractLogicalPlan extends AbstractPlan implements LogicalPlan { + private Supplier hasUnboundExpressions = () -> super.hasUnboundExpression(); + public AbstractLogicalPlan(PlanType type, Plan... children) { super(type, children); } @@ -44,9 +47,15 @@ public abstract class AbstractLogicalPlan extends AbstractPlan implements Logica super(type, groupExpression, logicalProperties, null, children); } + @Override + public boolean hasUnboundExpression() { + return hasUnboundExpressions.get(); + } + @Override public LogicalProperties computeLogicalProperties() { - boolean hasUnboundChild = children.stream().map(Plan::getLogicalProperties) + boolean hasUnboundChild = children.stream() + .map(Plan::getLogicalProperties) .anyMatch(UnboundLogicalProperties.class::isInstance); if (hasUnboundChild || hasUnboundExpression()) { return UnboundLogicalProperties.INSTANCE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index b844c7eafa..6f3c0778de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -52,7 +52,7 @@ import java.util.Optional; */ public class LogicalAggregate extends LogicalUnary - implements Aggregate { + implements Aggregate, OutputSavePoint { private final boolean normalized; private final List groupByExpressions; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLeaf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLeaf.java index 56101773ee..f874ae8c4f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLeaf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLeaf.java @@ -29,7 +29,7 @@ import java.util.Optional; /** * Abstract class for all logical plan that have no child. */ -public abstract class LogicalLeaf extends AbstractLogicalPlan implements LeafPlan { +public abstract class LogicalLeaf extends AbstractLogicalPlan implements LeafPlan, OutputSavePoint { public LogicalLeaf(PlanType nodeType, Optional groupExpression, Optional logicalProperties) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index db4b9e2f0e..3d6a9f7363 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -297,6 +297,9 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, @Override public List computeNonUserVisibleOutput() { + // if (getTable().getIndexIdToMeta().size() == 1) { + // return ImmutableList.of(); + // } Set baseSchemaColNames = table.getBaseSchema().stream() .map(Column::getName) .collect(Collectors.toSet()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index 12037f81b2..101031220b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -40,7 +40,8 @@ import java.util.Optional; /** * Logical project plan. */ -public class LogicalProject extends LogicalUnary implements Project { +public class LogicalProject extends LogicalUnary + implements Project, OutputSavePoint { private final List projects; private final List excepts; @@ -116,6 +117,7 @@ public class LogicalProject extends LogicalUnary extends LogicalUnary implements Repeat { +public class LogicalRepeat extends LogicalUnary + implements Repeat, OutputSavePoint { // max num of distinct sets in grouping sets clause public static final int MAX_GROUPING_SETS_NUM = 64; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java index 0be6e0efdf..b0534d04f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java @@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.catalog.Type; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.UnboundLogicalProperties; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -51,7 +52,7 @@ import java.util.Optional; * * eg: select k1, k2 from t1 union select 1, 2 union select d1, d2 from t2; */ -public abstract class LogicalSetOperation extends AbstractLogicalPlan implements SetOperation { +public abstract class LogicalSetOperation extends AbstractLogicalPlan implements SetOperation, OutputSavePoint { // eg value: qualifier:DISTINCT protected final Qualifier qualifier; @@ -82,6 +83,19 @@ public abstract class LogicalSetOperation extends AbstractLogicalPlan implements this.outputs = ImmutableList.copyOf(outputs); } + @Override + public boolean hasUnboundExpression() { + return outputs.isEmpty() || super.hasUnboundExpression(); + } + + @Override + public LogicalProperties computeLogicalProperties() { + if (outputs.isEmpty()) { + return UnboundLogicalProperties.INSTANCE; + } + return super.computeLogicalProperties(); + } + @Override public List computeOutput() { return outputs.stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/OutputSavePoint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/OutputSavePoint.java new file mode 100644 index 0000000000..fa566d14c1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/OutputSavePoint.java @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.logical; + +/** OutputSavePoint is used to point out a plan whether exist output field */ +public interface OutputSavePoint { +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CheckAnalysisJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CustomRewriter.java similarity index 60% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CheckAnalysisJob.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CustomRewriter.java index 45da20f408..fe719ee1f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CheckAnalysisJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CustomRewriter.java @@ -15,21 +15,14 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.jobs.batch; +package org.apache.doris.nereids.trees.plans.visitor; -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.analysis.CheckAnalysis; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.trees.plans.Plan; -import com.google.common.collect.ImmutableList; +/** CustomRewriter */ +public interface CustomRewriter { -/** - * Execute check analysis rules. - */ -public class CheckAnalysisJob extends BatchRulesJob { - public CheckAnalysisJob(CascadesContext cascadesContext) { - super(cascadesContext); - rulesJob.addAll(ImmutableList.of( - bottomUpBatch(ImmutableList.of(new CheckAnalysis())) - )); - } + // entrance method + Plan rewriteRoot(Plan plan, JobContext jobContext); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 6128aa39dc..f9cbd57a89 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.util; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewriteContext; import org.apache.doris.nereids.rules.expression.rewrite.rules.FoldConstantRule; import org.apache.doris.nereids.trees.TreeNode; @@ -344,7 +345,7 @@ public class ExpressionUtils { /** * infer notNulls slot from predicate */ - public static Set inferNotNullSlots(Set predicates) { + public static Set inferNotNullSlots(Set predicates, CascadesContext cascadesContext) { Literal nullLiteral = Literal.of(null); Set notNullSlots = Sets.newHashSet(); for (Expression predicate : predicates) { @@ -353,7 +354,7 @@ public class ExpressionUtils { replaceMap.put(slot, nullLiteral); Expression evalExpr = FoldConstantRule.INSTANCE.rewrite( ExpressionUtils.replace(predicate, replaceMap), - new ExpressionRewriteContext(null)); + new ExpressionRewriteContext(cascadesContext)); if (nullLiteral.equals(evalExpr) || BooleanLiteral.FALSE.equals(evalExpr)) { notNullSlots.add(slot); } @@ -365,8 +366,8 @@ public class ExpressionUtils { /** * infer notNulls slot from predicate */ - public static Set inferNotNull(Set predicates) { - return inferNotNullSlots(predicates).stream() + public static Set inferNotNull(Set predicates, CascadesContext cascadesContext) { + return inferNotNullSlots(predicates, cascadesContext).stream() .map(slot -> { Not isNotNull = new Not(new IsNull(slot)); isNotNull.isGeneratedIsNotNull = true; @@ -377,8 +378,9 @@ public class ExpressionUtils { /** * infer notNulls slot from predicate but these slots must be in the given slots. */ - public static Set inferNotNull(Set predicates, Set slots) { - return inferNotNullSlots(predicates).stream() + public static Set inferNotNull(Set predicates, Set slots, + CascadesContext cascadesContext) { + return inferNotNullSlots(predicates, cascadesContext).stream() .filter(slots::contains) .map(slot -> { Not isNotNull = new Not(new IsNull(slot)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/MutableState.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/MutableState.java new file mode 100644 index 0000000000..95ba886e3f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/MutableState.java @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.util; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +/** MutableState */ +public interface MutableState { + Optional get(String key); + + MutableState set(String key, Object value); + + /** EmptyMutableState */ + class EmptyMutableState implements MutableState { + public static final EmptyMutableState INSTANCE = new EmptyMutableState(); + + private EmptyMutableState() {} + + @Override + public Optional get(String key) { + return Optional.empty(); + } + + @Override + public MutableState set(String key, Object value) { + return new SingleMutableState(key, value); + } + } + + /** SingleMutableState */ + class SingleMutableState implements MutableState { + public final String key; + public final Object value; + + public SingleMutableState(String key, Object value) { + this.key = key; + this.value = value; + } + + @Override + public Optional get(String key) { + if (this.key.equals(key)) { + return (Optional) Optional.ofNullable(value); + } + return Optional.empty(); + } + + @Override + public MutableState set(String key, Object value) { + if (this.key.equals(key)) { + return new SingleMutableState(key, value); + } + MultiMutableState multiMutableState = new MultiMutableState(); + multiMutableState.set(this.key, this.value); + multiMutableState.set(key, value); + return multiMutableState; + } + } + + /** MultiMutableState */ + class MultiMutableState implements MutableState { + private final Map states = new LinkedHashMap<>(); + + @Override + public Optional get(String key) { + return (Optional) Optional.ofNullable(states.get(key)); + } + + @Override + public MutableState set(String key, Object value) { + states.put(key, value); + return this; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java index a5989d595f..e963d6131a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java @@ -244,6 +244,7 @@ public class Utils { for (int i = 0; i < list.size(); i++) { if (list.get(i) == item) { list.remove(i); + i--; return; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/JoinHintTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/JoinHintTest.java index 76b2cf6dd0..1b0129d917 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/JoinHintTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/JoinHintTest.java @@ -23,7 +23,7 @@ import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; import org.apache.doris.nereids.trees.plans.JoinHint; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.MatchingUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.planner.HashJoinNode; import org.apache.doris.planner.HashJoinNode.DistributionMode; @@ -38,7 +38,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -class JoinHintTest extends TestWithFeService implements PatternMatchSupported { +class JoinHintTest extends TestWithFeService implements MemoPatternMatchSupported { @Override protected void runBeforeAll() throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/datasets/ssb/SSBJoinReorderTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/datasets/ssb/SSBJoinReorderTest.java index 5f4fb55b23..583250afd1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/datasets/ssb/SSBJoinReorderTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/datasets/ssb/SSBJoinReorderTest.java @@ -17,7 +17,7 @@ package org.apache.doris.nereids.datasets.ssb; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import com.google.common.collect.ImmutableList; @@ -25,7 +25,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -public class SSBJoinReorderTest extends SSBTestBase implements PatternMatchSupported { +public class SSBJoinReorderTest extends SSBTestBase implements MemoPatternMatchSupported { @Test public void q4_1() { test( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java index 7ddf8e9b1e..e268f5a091 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java @@ -40,8 +40,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.apache.doris.qe.ConnectContext; @@ -56,7 +56,7 @@ import java.util.ArrayList; import java.util.Map; import java.util.Objects; -class MemoTest implements PatternMatchSupported { +class MemoTest implements MemoPatternMatchSupported { private final ConnectContext connectContext = MemoTestUtils.createConnectContext(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/metrics/EventTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/metrics/EventTest.java index 94efff68d0..631022d175 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/metrics/EventTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/metrics/EventTest.java @@ -78,6 +78,11 @@ public class EventTest extends TestWithFeService { channel.start(); } + @Override + protected void runBeforeEach() throws Exception { + CounterEvent.clearCounter(); + } + @Override public void runAfterAll() { channel.stop(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParserTestBase.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParserTestBase.java index cd35c5aef8..cb179ef531 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParserTestBase.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParserTestBase.java @@ -18,13 +18,13 @@ package org.apache.doris.nereids.parser; import org.apache.doris.nereids.util.ExpressionParseChecker; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanParseChecker; /** * Base class to check SQL parsing result. */ -public abstract class ParserTestBase implements PatternMatchSupported { +public abstract class ParserTestBase implements MemoPatternMatchSupported { public PlanParseChecker parsePlan(String sql) { return new PlanParseChecker(sql); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java index efec2a2c67..4947570ace 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java @@ -319,7 +319,7 @@ public class GroupExpressionMatchingTest { } } - private org.apache.doris.nereids.pattern.GeneratedPatterns patterns() { + private org.apache.doris.nereids.pattern.GeneratedMemoPatterns patterns() { return () -> RulePromise.REWRITE; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubQueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubQueryTest.java index d55083b718..5a568a8586 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubQueryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubQueryTest.java @@ -30,8 +30,8 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.util.FieldChecker; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; @@ -40,7 +40,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -public class AnalyzeSubQueryTest extends TestWithFeService implements PatternMatchSupported { +public class AnalyzeSubQueryTest extends TestWithFeService implements MemoPatternMatchSupported { private final NereidsParser parser = new NereidsParser(); @@ -142,20 +142,20 @@ public class AnalyzeSubQueryTest extends TestWithFeService implements PatternMat ) ) ).when(FieldChecker.check("projects", ImmutableList.of( - new SlotReference(new ExprId(0), "id", BigIntType.INSTANCE, true, ImmutableList.of("TT2")), - new SlotReference(new ExprId(1), "score", BigIntType.INSTANCE, true, ImmutableList.of("TT2")))) + new SlotReference(new ExprId(2), "id", BigIntType.INSTANCE, true, ImmutableList.of("TT2")), + new SlotReference(new ExprId(3), "score", BigIntType.INSTANCE, true, ImmutableList.of("TT2")))) ) ) .when(FieldChecker.check("otherJoinConjuncts", ImmutableList.of(new EqualTo( - new SlotReference(new ExprId(2), "id", BigIntType.INSTANCE, true, ImmutableList.of("TT1")), - new SlotReference(new ExprId(0), "id", BigIntType.INSTANCE, true, ImmutableList.of("T"))))) + new SlotReference(new ExprId(0), "id", BigIntType.INSTANCE, true, ImmutableList.of("TT1")), + new SlotReference(new ExprId(2), "id", BigIntType.INSTANCE, true, ImmutableList.of("T"))))) ) ).when(FieldChecker.check("projects", ImmutableList.of( - new SlotReference(new ExprId(2), "id", BigIntType.INSTANCE, true, ImmutableList.of("TT1")), - new SlotReference(new ExprId(3), "score", BigIntType.INSTANCE, true, ImmutableList.of("TT1")), - new SlotReference(new ExprId(0), "id", BigIntType.INSTANCE, true, ImmutableList.of("T")), - new SlotReference(new ExprId(1), "score", BigIntType.INSTANCE, true, ImmutableList.of("T")))) + new SlotReference(new ExprId(0), "id", BigIntType.INSTANCE, true, ImmutableList.of("TT1")), + new SlotReference(new ExprId(1), "score", BigIntType.INSTANCE, true, ImmutableList.of("TT1")), + new SlotReference(new ExprId(2), "id", BigIntType.INSTANCE, true, ImmutableList.of("T")), + new SlotReference(new ExprId(3), "score", BigIntType.INSTANCE, true, ImmutableList.of("T")))) ) ); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java index 0ab04d7456..b23b28b1c8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java @@ -26,15 +26,15 @@ import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.implementation.AggregateStrategies; -import org.apache.doris.nereids.rules.rewrite.logical.ApplyPullFilterOnAgg; -import org.apache.doris.nereids.rules.rewrite.logical.ApplyPullFilterOnProjectUnderAgg; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateFilterUnderApplyProject; import org.apache.doris.nereids.rules.rewrite.logical.ExistsApplyToJoin; import org.apache.doris.nereids.rules.rewrite.logical.InApplyToJoin; import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects; -import org.apache.doris.nereids.rules.rewrite.logical.PushApplyUnderFilter; -import org.apache.doris.nereids.rules.rewrite.logical.PushApplyUnderProject; +import org.apache.doris.nereids.rules.rewrite.logical.PullUpCorrelatedFilterUnderApplyAggregateProject; +import org.apache.doris.nereids.rules.rewrite.logical.PullUpProjectUnderApply; import org.apache.doris.nereids.rules.rewrite.logical.ScalarApplyToJoin; +import org.apache.doris.nereids.rules.rewrite.logical.UnCorrelatedApplyAggregateFilter; +import org.apache.doris.nereids.rules.rewrite.logical.UnCorrelatedApplyFilter; +import org.apache.doris.nereids.rules.rewrite.logical.UnCorrelatedApplyProjectFilter; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -47,8 +47,8 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.util.FieldChecker; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; @@ -61,7 +61,7 @@ import org.junit.jupiter.api.Test; import java.util.List; import java.util.Optional; -public class AnalyzeWhereSubqueryTest extends TestWithFeService implements PatternMatchSupported { +public class AnalyzeWhereSubqueryTest extends TestWithFeService implements MemoPatternMatchSupported { private final NereidsParser parser = new NereidsParser(); // scalar @@ -132,14 +132,18 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte }; for (String sql : testSql) { - NamedExpressionUtil.clear(); - StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); - PhysicalPlan plan = new NereidsPlanner(statementContext).plan( - parser.parseSingle(sql), - PhysicalProperties.ANY - ); - // Just to check whether translate will throw exception - new PhysicalPlanTranslator().translatePlan(plan, new PlanTranslatorContext()); + try { + NamedExpressionUtil.clear(); + StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); + PhysicalPlan plan = new NereidsPlanner(statementContext).plan( + parser.parseSingle(sql), + PhysicalProperties.ANY + ); + // Just to check whether translate will throw exception + new PhysicalPlanTranslator().translatePlan(plan, new PlanTranslatorContext()); + } catch (Throwable t) { + throw new IllegalStateException("Test sql failed: " + t.getMessage() + ", sql:\n" + sql, t); + } } } @@ -175,7 +179,7 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte // after aggFilter rule PlanChecker.from(connectContext) .analyze(sql2) - .applyBottomUp(new ApplyPullFilterOnAgg()) + .applyBottomUp(new UnCorrelatedApplyAggregateFilter()) .matches( logicalApply( any(), @@ -209,7 +213,7 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte // after Scalar CorrelatedJoin to join PlanChecker.from(connectContext) .analyze(sql2) - .applyBottomUp(new ApplyPullFilterOnAgg()) + .applyBottomUp(new UnCorrelatedApplyAggregateFilter()) .applyBottomUp(new ScalarApplyToJoin()) .matches( logicalJoin( @@ -247,7 +251,7 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte public void testInSql4AfterEliminateFilterUnderApplyProjectRule() { PlanChecker.from(connectContext) .analyze(sql4) - .applyBottomUp(new EliminateFilterUnderApplyProject()) + .applyBottomUp(new UnCorrelatedApplyProjectFilter()) .matches( logicalApply( any(), @@ -271,7 +275,7 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte public void testInSql4AfterInToJoin() { PlanChecker.from(connectContext) .analyze(sql4) - .applyBottomUp(new EliminateFilterUnderApplyProject()) + .applyBottomUp(new UnCorrelatedApplyProjectFilter()) .applyBottomUp(new InApplyToJoin()) .matches( logicalJoin().when(FieldChecker.check("joinType", JoinType.LEFT_SEMI_JOIN)) @@ -310,7 +314,7 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte public void testExistSql6AfterPushProjectRule() { PlanChecker.from(connectContext) .analyze(sql6) - .applyBottomUp(new PushApplyUnderProject()) + .applyBottomUp(new PullUpProjectUnderApply()) .matches( logicalProject( logicalApply().when(FieldChecker.check("correlationFilter", Optional.empty())) @@ -329,8 +333,8 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte public void testExistSql6AfterPushFilterRule() { PlanChecker.from(connectContext) .analyze(sql6) - .applyBottomUp(new PushApplyUnderProject()) - .applyBottomUp(new PushApplyUnderFilter()) + .applyBottomUp(new PullUpProjectUnderApply()) + .applyBottomUp(new UnCorrelatedApplyFilter()) .matches( logicalApply().when(FieldChecker.check("correlationFilter", Optional.of( new EqualTo(new SlotReference(new ExprId(1), "k2", BigIntType.INSTANCE, true, @@ -344,8 +348,8 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte public void testExistSql6AfterInToJoin() { PlanChecker.from(connectContext) .analyze(sql6) - .applyBottomUp(new PushApplyUnderProject()) - .applyBottomUp(new PushApplyUnderFilter()) + .applyBottomUp(new PullUpProjectUnderApply()) + .applyBottomUp(new UnCorrelatedApplyFilter()) .applyBottomUp(new ExistsApplyToJoin()) .matches( logicalJoin().when(FieldChecker.check("joinType", JoinType.LEFT_SEMI_JOIN)) @@ -407,7 +411,7 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte .analyze(sql10) .applyBottomUp(new LogicalSubQueryAliasToLogicalProject()) .applyTopDown(new MergeProjects()) - .applyBottomUp(new ApplyPullFilterOnProjectUnderAgg()) + .applyBottomUp(new PullUpCorrelatedFilterUnderApplyAggregateProject()) .matches( logicalApply( any(), @@ -439,8 +443,8 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte .analyze(sql10) .applyBottomUp(new LogicalSubQueryAliasToLogicalProject()) .applyTopDown(new MergeProjects()) - .applyBottomUp(new ApplyPullFilterOnProjectUnderAgg()) - .applyBottomUp(new ApplyPullFilterOnAgg()) + .applyBottomUp(new PullUpCorrelatedFilterUnderApplyAggregateProject()) + .applyBottomUp(new UnCorrelatedApplyAggregateFilter()) .matches( logicalApply( any(), @@ -465,8 +469,8 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte .analyze(sql10) .applyBottomUp(new LogicalSubQueryAliasToLogicalProject()) .applyTopDown(new MergeProjects()) - .applyBottomUp(new ApplyPullFilterOnProjectUnderAgg()) - .applyBottomUp(new ApplyPullFilterOnAgg()) + .applyBottomUp(new PullUpCorrelatedFilterUnderApplyAggregateProject()) + .applyBottomUp(new UnCorrelatedApplyAggregateFilter()) .applyBottomUp(new ScalarApplyToJoin()) .matches( logicalJoin( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindFunctionTest.java index b5aebdc124..27d99fff42 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindFunctionTest.java @@ -21,13 +21,13 @@ import org.apache.doris.common.Config; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.expressions.LessThan; import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; import org.junit.jupiter.api.Test; -public class BindFunctionTest extends TestWithFeService implements PatternMatchSupported { +public class BindFunctionTest extends TestWithFeService implements MemoPatternMatchSupported { private final NereidsParser parser = new NereidsParser(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckExpressionLegalityTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckExpressionLegalityTest.java index 553ee35e44..4c003cf283 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckExpressionLegalityTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckExpressionLegalityTest.java @@ -23,14 +23,14 @@ import org.apache.doris.nereids.jobs.batch.CheckLegalityAfterRewrite; import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewrite; import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.qe.ConnectContext; import org.junit.jupiter.api.Test; -public class CheckExpressionLegalityTest implements PatternMatchSupported { +public class CheckExpressionLegalityTest implements MemoPatternMatchSupported { @Test public void testAvg() { ConnectContext connectContext = MemoTestUtils.createConnectContext(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlotsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlotsTest.java index 3ecbd52a46..154a30955f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlotsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlotsTest.java @@ -39,7 +39,7 @@ import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.util.FieldChecker; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import com.google.common.collect.ImmutableList; @@ -49,7 +49,7 @@ import org.junit.jupiter.api.Test; import java.util.stream.Collectors; -public class FillUpMissingSlotsTest extends AnalyzeCheckTestBase implements PatternMatchSupported { +public class FillUpMissingSlotsTest extends AnalyzeCheckTestBase implements MemoPatternMatchSupported { @Override public void runBeforeAll() throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java index 487fae2c58..a421aa0ac2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java @@ -31,8 +31,8 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Year; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.qe.ConnectContext; @@ -44,7 +44,7 @@ import java.util.List; import java.util.Map; // this ut will add more test case later -public class FunctionRegistryTest implements PatternMatchSupported { +public class FunctionRegistryTest implements MemoPatternMatchSupported { private ConnectContext connectContext = MemoTestUtils.createConnectContext(); @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeatTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeatTest.java index f32f5959d9..37508d5387 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeatTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeatTest.java @@ -26,15 +26,15 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; -public class NormalizeRepeatTest implements PatternMatchSupported { +public class NormalizeRepeatTest implements MemoPatternMatchSupported { @Test public void testKeepNullableAfterNormalizeRepeat() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/RegisterCTETest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/RegisterCTETest.java index 95b39c48a9..7142eecab5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/RegisterCTETest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/RegisterCTETest.java @@ -31,8 +31,8 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.implementation.AggregateStrategies; import org.apache.doris.nereids.rules.rewrite.logical.InApplyToJoin; -import org.apache.doris.nereids.rules.rewrite.logical.PushApplyUnderFilter; -import org.apache.doris.nereids.rules.rewrite.logical.PushApplyUnderProject; +import org.apache.doris.nereids.rules.rewrite.logical.PullUpProjectUnderApply; +import org.apache.doris.nereids.rules.rewrite.logical.UnCorrelatedApplyFilter; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -46,8 +46,8 @@ import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.util.FieldChecker; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; @@ -60,7 +60,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -public class RegisterCTETest extends TestWithFeService implements PatternMatchSupported { +public class RegisterCTETest extends TestWithFeService implements MemoPatternMatchSupported { private final NereidsParser parser = new NereidsParser(); @@ -205,8 +205,8 @@ public class RegisterCTETest extends TestWithFeService implements PatternMatchSu PlanChecker.from(connectContext) .analyze(sql3) - .applyBottomUp(new PushApplyUnderProject()) - .applyBottomUp(new PushApplyUnderFilter()) + .applyBottomUp(new PullUpProjectUnderApply()) + .applyBottomUp(new UnCorrelatedApplyFilter()) .applyBottomUp(new InApplyToJoin()) .matches( logicalProject( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ReplaceExpressionByChildOutputTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ReplaceExpressionByChildOutputTest.java index 67d043b88c..3238791145 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ReplaceExpressionByChildOutputTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ReplaceExpressionByChildOutputTest.java @@ -29,8 +29,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -40,7 +40,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -public class ReplaceExpressionByChildOutputTest implements PatternMatchSupported { +public class ReplaceExpressionByChildOutputTest implements MemoPatternMatchSupported { @Test void testSortProject() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProjectTest.java index c272ce97cf..2111253168 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProjectTest.java @@ -32,8 +32,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -43,7 +43,7 @@ import org.junit.jupiter.api.Test; import java.util.List; import java.util.Objects; -class InnerJoinLAsscomProjectTest implements PatternMatchSupported { +class InnerJoinLAsscomProjectTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomTest.java index 02b340fbd2..5d89c29f8a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomTest.java @@ -32,8 +32,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -42,7 +42,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -public class InnerJoinLAsscomTest implements PatternMatchSupported { +public class InnerJoinLAsscomTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociateTest.java index fede3fe0df..7b4b7187b8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociateTest.java @@ -22,14 +22,14 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.junit.jupiter.api.Test; -class InnerJoinLeftAssociateTest implements PatternMatchSupported { +class InnerJoinLeftAssociateTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); private final LogicalOlapScan scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociateTest.java index 82d284dade..c0c07513b6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociateTest.java @@ -22,14 +22,14 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.junit.jupiter.api.Test; -class InnerJoinRightAssociateTest implements PatternMatchSupported { +class InnerJoinRightAssociateTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); private final LogicalOlapScan scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinCommuteTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinCommuteTest.java index 0e70e82083..20323d108e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinCommuteTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinCommuteTest.java @@ -22,14 +22,14 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.junit.jupiter.api.Test; -public class JoinCommuteTest implements PatternMatchSupported { +public class JoinCommuteTest implements MemoPatternMatchSupported { @Test public void testInnerJoinCommute() { LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinExchangeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinExchangeTest.java index 2909b960c2..8faa5d0a85 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinExchangeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinExchangeTest.java @@ -22,15 +22,15 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; -class JoinExchangeTest implements PatternMatchSupported { +class JoinExchangeTest implements MemoPatternMatchSupported { @Test public void testSimple() { LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocTest.java index 49e7a13795..90eeefee02 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocTest.java @@ -18,12 +18,13 @@ package org.apache.doris.nereids.rules.exploration.join; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.trees.expressions.NamedExpressionUtil; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -31,10 +32,18 @@ import org.junit.jupiter.api.Test; import java.util.Objects; -class OuterJoinAssocTest implements PatternMatchSupported { - LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); - LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); - LogicalOlapScan scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); +class OuterJoinAssocTest implements MemoPatternMatchSupported { + LogicalOlapScan scan1; + LogicalOlapScan scan2; + LogicalOlapScan scan3; + + public OuterJoinAssocTest() throws Exception { + // clear id so that slot id keep consistent every running + NamedExpressionUtil.clear(); + scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); + scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); + } @Test public void testInnerLeft() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java index 79d705f231..90b8266f25 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java @@ -25,8 +25,8 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -36,7 +36,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -class OuterJoinLAsscomProjectTest implements PatternMatchSupported { +class OuterJoinLAsscomProjectTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomTest.java index 93143e22e1..7b75231c04 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomTest.java @@ -26,8 +26,8 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -37,7 +37,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -public class OuterJoinLAsscomTest implements PatternMatchSupported { +public class OuterJoinLAsscomTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProjectTest.java index fb4586d34e..ba82d58f2a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProjectTest.java @@ -22,15 +22,15 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; -public class SemiJoinSemiJoinTransposeProjectTest implements PatternMatchSupported { +public class SemiJoinSemiJoinTransposeProjectTest implements MemoPatternMatchSupported { public static final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); public static final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); public static final LogicalOlapScan scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteTestHelper.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteTestHelper.java index 75824b2441..f75f474257 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteTestHelper.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteTestHelper.java @@ -17,12 +17,15 @@ package org.apache.doris.nereids.rules.expression.rewrite; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.rules.expression.rewrite.rules.TypeCoercion; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.DataType; @@ -31,7 +34,9 @@ import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; +import org.apache.doris.nereids.util.MemoTestUtils; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.junit.jupiter.api.Assertions; @@ -43,21 +48,29 @@ public abstract class ExpressionRewriteTestHelper { protected static final NereidsParser PARSER = new NereidsParser(); protected ExpressionRuleExecutor executor; + protected ExpressionRewriteContext context; + + public ExpressionRewriteTestHelper() { + CascadesContext cascadesContext = MemoTestUtils.createCascadesContext( + new UnboundRelation(new RelationId(1), ImmutableList.of("tbl"))); + context = new ExpressionRewriteContext(cascadesContext); + } + protected final void assertRewrite(String expression, String expected) { Expression needRewriteExpression = PARSER.parseExpression(expression); Expression expectedExpression = PARSER.parseExpression(expected); - Expression rewrittenExpression = executor.rewrite(needRewriteExpression); + Expression rewrittenExpression = executor.rewrite(needRewriteExpression, context); Assertions.assertEquals(expectedExpression, rewrittenExpression); } protected void assertRewrite(String expression, Expression expectedExpression) { Expression needRewriteExpression = PARSER.parseExpression(expression); - Expression rewrittenExpression = executor.rewrite(needRewriteExpression); + Expression rewrittenExpression = executor.rewrite(needRewriteExpression, context); Assertions.assertEquals(expectedExpression, rewrittenExpression); } protected void assertRewrite(Expression expression, Expression expectedExpression) { - Expression rewrittenExpression = executor.rewrite(expression); + Expression rewrittenExpression = executor.rewrite(expression, context); Assertions.assertEquals(expectedExpression, rewrittenExpression); } @@ -66,7 +79,7 @@ public abstract class ExpressionRewriteTestHelper { Expression needRewriteExpression = PARSER.parseExpression(expression); needRewriteExpression = typeCoercion(replaceUnboundSlot(needRewriteExpression, mem)); Expression expectedExpression = PARSER.parseExpression(expected); - Expression rewrittenExpression = executor.rewrite(needRewriteExpression); + Expression rewrittenExpression = executor.rewrite(needRewriteExpression, context); Assertions.assertEquals(expectedExpression.toSql(), rewrittenExpression.toSql()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/FoldConstantTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/FoldConstantTest.java index d600d0a2da..7e8d132d06 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/FoldConstantTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/FoldConstantTest.java @@ -134,7 +134,7 @@ public class FoldConstantTest extends ExpressionRewriteTestHelper { // cast '1' as tinyint Cast c = new Cast(Literal.of("1"), TinyIntType.INSTANCE); - Expression rewritten = executor.rewrite(c); + Expression rewritten = executor.rewrite(c, context); Literal expected = Literal.of((byte) 1); Assertions.assertEquals(rewritten, expected); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/SimplifyRangeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/SimplifyRangeTest.java index 617ad971f7..5022828713 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/SimplifyRangeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/SimplifyRangeTest.java @@ -17,12 +17,15 @@ package org.apache.doris.nereids.rules.expression.rewrite; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.rules.expression.rewrite.rules.SimplifyRange; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.DataType; @@ -30,6 +33,7 @@ import org.apache.doris.nereids.types.DoubleType; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.TinyIntType; +import org.apache.doris.nereids.util.MemoTestUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -44,6 +48,13 @@ public class SimplifyRangeTest { private static final NereidsParser PARSER = new NereidsParser(); private ExpressionRuleExecutor executor; + private ExpressionRewriteContext context; + + public SimplifyRangeTest() { + CascadesContext cascadesContext = MemoTestUtils.createCascadesContext( + new UnboundRelation(new RelationId(1), ImmutableList.of("tbl"))); + context = new ExpressionRewriteContext(cascadesContext); + } @Test public void testSimplify() { @@ -99,7 +110,7 @@ public class SimplifyRangeTest { Map mem = Maps.newHashMap(); Expression needRewriteExpression = replaceUnboundSlot(PARSER.parseExpression(expression), mem); Expression expectedExpression = replaceUnboundSlot(PARSER.parseExpression(expected), mem); - Expression rewrittenExpression = executor.rewrite(needRewriteExpression); + Expression rewrittenExpression = executor.rewrite(needRewriteExpression, context); Assertions.assertEquals(expectedExpression, rewrittenExpression); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java index 512e3f1e74..403d13b2cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java @@ -31,7 +31,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.HllUnionAgg; import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.ScanNode; @@ -53,7 +53,7 @@ import java.util.stream.Collectors; * Tests ported from {@link org.apache.doris.planner.MaterializedViewFunctionTest} */ @Disabled("Disabled until nereids support advanced mv") -public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements PatternMatchSupported { +public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoPatternMatchSupported { private static final String EMPS_TABLE_NAME = "emps"; private static final String EMPS_MV_NAME = "emps_mv"; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java index 8f596b0d18..947a30e449 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java @@ -21,14 +21,14 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.nereids.rules.analysis.LogicalSubQueryAliasToLogicalProject; import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects; import org.apache.doris.nereids.trees.plans.PreAggStatus; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -class SelectRollupIndexTest extends BaseMaterializedIndexSelectTest implements PatternMatchSupported { +class SelectRollupIndexTest extends BaseMaterializedIndexSelectTest implements MemoPatternMatchSupported { @Override protected void beforeCreatingConnectContext() throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/AggregateStrategiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/AggregateStrategiesTest.java index 29b729fe1c..33c5c1f909 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/AggregateStrategiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/AggregateStrategiesTest.java @@ -37,8 +37,8 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -55,7 +55,7 @@ import java.util.List; import java.util.Optional; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class AggregateStrategiesTest implements PatternMatchSupported { +public class AggregateStrategiesTest implements MemoPatternMatchSupported { private Plan rStudent; @BeforeAll diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/CheckAndStandardizeWindowFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/CheckAndStandardizeWindowFunctionTest.java index 9faa34487c..5bfa3635ef 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/CheckAndStandardizeWindowFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/CheckAndStandardizeWindowFunctionTest.java @@ -40,8 +40,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -55,7 +55,7 @@ import org.junit.jupiter.api.TestInstance; import java.util.List; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class CheckAndStandardizeWindowFunctionTest implements PatternMatchSupported { +public class CheckAndStandardizeWindowFunctionTest implements MemoPatternMatchSupported { private LogicalPlan rStudent; private NamedExpression gender; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java index bf2daa712e..7379df6bea 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java @@ -22,7 +22,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.types.DoubleType; import org.apache.doris.nereids.types.IntegerType; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; @@ -35,7 +35,7 @@ import java.util.stream.Collectors; /** * column prune ut. */ -public class ColumnPruningTest extends TestWithFeService implements PatternMatchSupported { +public class ColumnPruningTest extends TestWithFeService implements MemoPatternMatchSupported { @Override protected void runBeforeAll() throws Exception { createDatabase("test"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateDedupJoinConditionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateDedupJoinConditionTest.java index 24510cd97d..4011e79960 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateDedupJoinConditionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateDedupJoinConditionTest.java @@ -21,15 +21,15 @@ import org.apache.doris.common.Pair; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; -class EliminateDedupJoinConditionTest implements PatternMatchSupported { +class EliminateDedupJoinConditionTest implements MemoPatternMatchSupported { @Test void testEliminate() { LogicalPlan plan = new LogicalPlanBuilder(PlanConstructor.scan1) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java index 4777251c56..623b29c159 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java @@ -20,13 +20,14 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.common.Pair; import org.apache.doris.nereids.trees.expressions.And; import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.NamedExpressionUtil; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -34,9 +35,16 @@ import org.junit.jupiter.api.Test; import java.util.Objects; -class EliminateOuterJoinTest implements PatternMatchSupported { - private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); - private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); +class EliminateOuterJoinTest implements MemoPatternMatchSupported { + private final LogicalOlapScan scan1; + private final LogicalOlapScan scan2; + + public EliminateOuterJoinTest() throws Exception { + // clear id so that slot id keep consistent every running + NamedExpressionUtil.clear(); + scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); + } @Test void testEliminateLeft() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProjectTest.java index 4c440e68f8..ecd6de1cd5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProjectTest.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.plans.Plan; @@ -33,12 +32,9 @@ import org.apache.doris.nereids.util.PlanConstructor; import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.List; - /** * test ELIMINATE_UNNECESSARY_PROJECT rule. */ @@ -66,11 +62,10 @@ public class EliminateUnnecessaryProjectTest extends TestWithFeService { .build(); CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(unnecessaryProject); - List rules = Lists.newArrayList(new EliminateUnnecessaryProject().buildRules()); - cascadesContext.topDownRewrite(rules); + cascadesContext.topDownRewrite(new EliminateUnnecessaryProject()); Plan actual = cascadesContext.getMemo().copyOut(); - Assertions.assertTrue(actual.child(0) instanceof LogicalOlapScan); + Assertions.assertTrue(actual.child(0) instanceof LogicalProject); } @Test @@ -80,8 +75,7 @@ public class EliminateUnnecessaryProjectTest extends TestWithFeService { .build(); CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(unnecessaryProject); - List rules = Lists.newArrayList(new EliminateUnnecessaryProject().buildRules()); - cascadesContext.topDownRewrite(rules); + cascadesContext.topDownRewrite(new EliminateUnnecessaryProject()); Plan actual = cascadesContext.getMemo().copyOut(); Assertions.assertTrue(actual instanceof LogicalOlapScan); @@ -89,13 +83,12 @@ public class EliminateUnnecessaryProjectTest extends TestWithFeService { @Test public void testNotEliminateTopProjectWhenOutputNotEquals() { - LogicalPlan unnecessaryProject = new LogicalPlanBuilder(PlanConstructor.newLogicalOlapScan(0, "t1", 0)) + LogicalPlan necessaryProject = new LogicalPlanBuilder(PlanConstructor.newLogicalOlapScan(0, "t1", 0)) .project(ImmutableList.of(1, 0)) .build(); - CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(unnecessaryProject); - List rules = Lists.newArrayList(new EliminateUnnecessaryProject().buildRules()); - cascadesContext.topDownRewrite(rules); + CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(necessaryProject); + cascadesContext.topDownRewrite(new EliminateUnnecessaryProject()); Plan actual = cascadesContext.getMemo().copyOut(); Assertions.assertTrue(actual instanceof LogicalProject); @@ -109,8 +102,7 @@ public class EliminateUnnecessaryProjectTest extends TestWithFeService { .project(ImmutableList.of(1, 0)) .build(); CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(unnecessaryProject); - List rules = Lists.newArrayList(new EliminateUnnecessaryProject().buildRules()); - cascadesContext.topDownRewrite(rules); + cascadesContext.topDownRewrite(new EliminateUnnecessaryProject()); Plan actual = cascadesContext.getMemo().copyOut(); Assertions.assertTrue(actual instanceof LogicalEmptyRelation); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractAndNormalizeWindowExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractAndNormalizeWindowExpressionTest.java index 39d62974a9..96c833a1de 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractAndNormalizeWindowExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractAndNormalizeWindowExpressionTest.java @@ -34,8 +34,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -48,7 +48,7 @@ import org.junit.jupiter.api.TestInstance; import java.util.List; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class ExtractAndNormalizeWindowExpressionTest implements PatternMatchSupported { +public class ExtractAndNormalizeWindowExpressionTest implements MemoPatternMatchSupported { private LogicalPlan rStudent; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoinTest.java index dc3aa0aa8f..48db2844f4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoinTest.java @@ -22,14 +22,14 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.junit.jupiter.api.Test; -class ExtractFilterFromCrossJoinTest implements PatternMatchSupported { +class ExtractFilterFromCrossJoinTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractSingleTableExpressionFromDisjunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractSingleTableExpressionFromDisjunctionTest.java index eb1c619a9b..bfc7e1aaa2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractSingleTableExpressionFromDisjunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractSingleTableExpressionFromDisjunctionTest.java @@ -29,8 +29,8 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -44,7 +44,7 @@ import org.junit.jupiter.api.TestInstance; import java.util.Set; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class ExtractSingleTableExpressionFromDisjunctionTest implements PatternMatchSupported { +public class ExtractSingleTableExpressionFromDisjunctionTest implements MemoPatternMatchSupported { Plan student; Plan course; SlotReference courseCid; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNullTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNullTest.java index 7c0d2623cc..3c689a2be1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNullTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNullTest.java @@ -25,14 +25,14 @@ import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.junit.jupiter.api.Test; -class InferFilterNotNullTest implements PatternMatchSupported { +class InferFilterNotNullTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferJoinNotNullTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferJoinNotNullTest.java index 81cd74874a..6ba1ed4bed 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferJoinNotNullTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferJoinNotNullTest.java @@ -22,14 +22,14 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.junit.jupiter.api.Test; -class InferJoinNotNullTest implements PatternMatchSupported { +class InferJoinNotNullTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); private final LogicalOlapScan scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); @@ -43,8 +43,8 @@ class InferJoinNotNullTest implements PatternMatchSupported { .applyTopDown(new InferJoinNotNull()) .matches( innerLogicalJoin( - logicalFilter().when(f -> f.getPredicate().toString().equals("( not id IS NULL)")), - logicalFilter().when(f -> f.getPredicate().toString().equals("( not id IS NULL)")) + logicalFilter().when(f -> f.getPredicate().toString().equals("( not id#0 IS NULL)")), + logicalFilter().when(f -> f.getPredicate().toString().equals("( not id#2 IS NULL)")) ) ); @@ -55,7 +55,7 @@ class InferJoinNotNullTest implements PatternMatchSupported { .applyTopDown(new InferJoinNotNull()) .matches( leftSemiLogicalJoin( - logicalFilter().when(f -> f.getPredicate().toString().equals("( not id IS NULL)")), + logicalFilter().when(f -> f.getPredicate().toString().equals("( not id#0 IS NULL)")), logicalOlapScan() ) ); @@ -68,7 +68,7 @@ class InferJoinNotNullTest implements PatternMatchSupported { .matches( rightSemiLogicalJoin( logicalOlapScan(), - logicalFilter().when(f -> f.getPredicate().toString().equals("( not id IS NULL)")) + logicalFilter().when(f -> f.getPredicate().toString().equals("( not id#2 IS NULL)")) ) ); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferPredicatesTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferPredicatesTest.java index a2addbbeb7..e7af340c32 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferPredicatesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferPredicatesTest.java @@ -17,13 +17,13 @@ package org.apache.doris.nereids.rules.rewrite.logical; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; import org.junit.jupiter.api.Test; -public class InferPredicatesTest extends TestWithFeService implements PatternMatchSupported { +public class InferPredicatesTest extends TestWithFeService implements MemoPatternMatchSupported { @Override protected void runBeforeAll() throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/LimitPushDownTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/LimitPushDownTest.java index ef251c911f..89a2b4f0b5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/LimitPushDownTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/LimitPushDownTest.java @@ -29,8 +29,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.apache.doris.planner.OlapScanNode; @@ -48,7 +48,7 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; -class LimitPushDownTest extends TestWithFeService implements PatternMatchSupported { +class LimitPushDownTest extends TestWithFeService implements MemoPatternMatchSupported { private Plan scanScore = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.score); private Plan scanStudent = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/LogicalWindowToPhysicalWindowTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/LogicalWindowToPhysicalWindowTest.java index 1abe5f56ca..0ba1f76ec3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/LogicalWindowToPhysicalWindowTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/LogicalWindowToPhysicalWindowTest.java @@ -30,8 +30,8 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -45,7 +45,7 @@ import org.junit.jupiter.api.TestInstance; import java.util.List; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class LogicalWindowToPhysicalWindowTest implements PatternMatchSupported { +public class LogicalWindowToPhysicalWindowTest implements MemoPatternMatchSupported { private Plan rStudent; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjectsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjectsTest.java index 024890753f..4c04874a06 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjectsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjectsTest.java @@ -25,8 +25,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -39,7 +39,7 @@ import java.util.Objects; /** * MergeConsecutiveProjects ut */ -public class MergeProjectsTest implements PatternMatchSupported { +public class MergeProjectsTest implements MemoPatternMatchSupported { LogicalOlapScan score = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.score); @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeAggregateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeAggregateTest.java index 53848e4c50..254684eedb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeAggregateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeAggregateTest.java @@ -33,8 +33,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; import org.apache.doris.nereids.util.FieldChecker; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -47,7 +47,7 @@ import org.junit.jupiter.api.TestInstance; import java.util.List; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class NormalizeAggregateTest implements PatternMatchSupported { +public class NormalizeAggregateTest implements MemoPatternMatchSupported { private LogicalPlan rStudent; @BeforeAll diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PhysicalStorageLayerAggregateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PhysicalStorageLayerAggregateTest.java index 22e888ee7a..1556438652 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PhysicalStorageLayerAggregateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PhysicalStorageLayerAggregateTest.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.pattern.GeneratedPatterns; +import org.apache.doris.nereids.pattern.GeneratedMemoPatterns; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RulePromise; import org.apache.doris.nereids.rules.RuleType; @@ -42,7 +42,7 @@ import org.junit.jupiter.api.Test; import java.util.Collections; import java.util.Optional; -public class PhysicalStorageLayerAggregateTest implements GeneratedPatterns { +public class PhysicalStorageLayerAggregateTest implements GeneratedMemoPatterns { @Test public void testWithoutProject() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoinTest.java index 5111fea12c..1fd6cb9ac2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoinTest.java @@ -23,14 +23,14 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.junit.jupiter.api.Test; -class PushFilterInsideJoinTest implements PatternMatchSupported { +class PushFilterInsideJoinTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownExpressionsInHashConditionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownExpressionsInHashConditionTest.java index c002db8b9b..4b4278be9e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownExpressionsInHashConditionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownExpressionsInHashConditionTest.java @@ -22,7 +22,7 @@ import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.NamedExpressionUtil; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; @@ -32,7 +32,7 @@ import org.junit.jupiter.api.Test; import java.util.List; import java.util.Set; -public class PushdownExpressionsInHashConditionTest extends TestWithFeService implements PatternMatchSupported { +public class PushdownExpressionsInHashConditionTest extends TestWithFeService implements MemoPatternMatchSupported { @Override protected void runBeforeAll() throws Exception { createDatabase("test"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughAggregationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughAggregationTest.java index bc6a08d994..474f1acd49 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughAggregationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughAggregationTest.java @@ -29,8 +29,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -38,7 +38,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import org.junit.jupiter.api.Test; -public class PushdownFilterThroughAggregationTest implements PatternMatchSupported { +public class PushdownFilterThroughAggregationTest implements MemoPatternMatchSupported { /*- * origin plan: @@ -135,8 +135,8 @@ public class PushdownFilterThroughAggregationTest implements PatternMatchSupport logicalAggregate( logicalFilter( logicalOlapScan() - ).when(filter -> ImmutableList.copyOf(filter.getConjuncts()).get(0) instanceof LessThanEqual - && ImmutableList.copyOf(filter.getConjuncts()).get(1) instanceof GreaterThan) + ).when(filter -> ImmutableList.copyOf(filter.getConjuncts()).get(0) instanceof GreaterThan + && ImmutableList.copyOf(filter.getConjuncts()).get(1) instanceof LessThanEqual) ) ).when(filter -> ImmutableList.copyOf(filter.getConjuncts()).get(0) instanceof EqualTo) ) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoinTest.java index ef357db025..620c1bf44b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoinTest.java @@ -27,8 +27,8 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -44,7 +44,7 @@ import java.util.Set; * PushdownFilterThroughJoinTest UT. */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class PushdownFilterThroughJoinTest implements PatternMatchSupported { +public class PushdownFilterThroughJoinTest implements MemoPatternMatchSupported { private LogicalPlan rStudent; private LogicalPlan rScore; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimitTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimitTest.java index f82f3c0787..77bf76af4c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimitTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimitTest.java @@ -19,15 +19,15 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; -public class PushdownProjectThroughLimitTest implements PatternMatchSupported { +public class PushdownProjectThroughLimitTest implements MemoPatternMatchSupported { @Test public void testPushdownProjectThroughLimit() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoinTest.java index 23f69e9b7c..bf37e5666f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoinTest.java @@ -23,8 +23,8 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -33,7 +33,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -class ReorderJoinTest implements PatternMatchSupported { +class ReorderJoinTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java index 7e81c648a2..48b5026d1e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java @@ -69,7 +69,7 @@ public class InferTest extends SqlTestBase { logicalOlapScan() ) ).when(f -> f.getPredicate().toString() - .equals("((id#0 = 4) OR ((id#0 > 4) AND score IS NULL))")) + .equals("((id#0 = 4) OR ((id#0 > 4) AND score#3 IS NULL))")) ); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java index f3b8899b5a..3151c2d7bd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java @@ -18,10 +18,10 @@ package org.apache.doris.nereids.sqltest; import org.apache.doris.nereids.trees.expressions.NamedExpressionUtil; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.utframe.TestWithFeService; -public abstract class SqlTestBase extends TestWithFeService implements PatternMatchSupported { +public abstract class SqlTestBase extends TestWithFeService implements MemoPatternMatchSupported { @Override protected void runBeforeAll() throws Exception { createDatabase("test"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectExceptTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectExceptTest.java index 23606f95f8..781c7840e4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectExceptTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectExceptTest.java @@ -22,8 +22,8 @@ import org.apache.doris.nereids.analyzer.UnboundStar; import org.apache.doris.nereids.exceptions.ParseException; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -31,7 +31,7 @@ import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -public class SelectExceptTest implements PatternMatchSupported { +public class SelectExceptTest implements MemoPatternMatchSupported { @Test public void testExcept() { LogicalOlapScan olapScan = PlanConstructor.newLogicalOlapScan(0, "t1", 1); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java index 4a72965010..71ab84f5a8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java @@ -26,8 +26,8 @@ import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.analysis.LogicalSubQueryAliasToLogicalProject; import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; @@ -36,7 +36,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -public class ViewTest extends TestWithFeService implements PatternMatchSupported { +public class ViewTest extends TestWithFeService implements MemoPatternMatchSupported { @Override protected void runBeforeAll() throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java index 8da90832fd..c09fa8623b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java @@ -91,7 +91,7 @@ public class PlanToStringTest { LogicalProject plan = new LogicalProject<>(ImmutableList.of( new SlotReference(new ExprId(0), "a", BigIntType.INSTANCE, true, Lists.newArrayList())), child); - Assertions.assertTrue(plan.toString().matches("LogicalProject \\( projects=\\[a#\\d+], excepts=\\[], canEliminate=true \\)")); + Assertions.assertTrue(plan.toString().matches("LogicalProject \\( distinct=false, projects=\\[a#\\d+], excepts=\\[], canEliminate=true \\)")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MatchingUtils.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MatchingUtils.java index bc5bb1d78d..c698cdc0f4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MatchingUtils.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MatchingUtils.java @@ -36,20 +36,20 @@ public class MatchingUtils { Memo memo = new Memo(plan); if (plan instanceof PhysicalPlan) { assertMatches(memo, () -> new GroupExpressionMatching(patternDesc.pattern, - memo.getRoot().getPhysicalExpressions().get(0)).iterator().hasNext()); + memo.getRoot().getPhysicalExpressions().get(0)).iterator().hasNext(), + () -> plan.treeString()); } else if (plan instanceof LogicalPlan) { assertMatches(memo, () -> new GroupExpressionMatching(patternDesc.pattern, - memo.getRoot().getLogicalExpression()).iterator().hasNext()); + memo.getRoot().getLogicalExpression()).iterator().hasNext(), + () -> plan.treeString()); } else { throw new IllegalStateException("Input plan should be LogicalPlan or PhysicalPlan, but meet " + plan); } } - private static void assertMatches(Memo memo, Supplier asserter) { + private static void assertMatches(Memo memo, Supplier asserter, Supplier planString) { Assertions.assertTrue(asserter.get(), - () -> "pattern not match, plan :\n" - + memo.getRoot().getLogicalExpression().getPlan().treeString() - + "\n" + () -> "pattern not match, plan:\n" + planString.get() + "\n" ); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PatternMatchSupported.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoPatternMatchSupported.java similarity index 87% rename from fe/fe-core/src/test/java/org/apache/doris/nereids/util/PatternMatchSupported.java rename to fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoPatternMatchSupported.java index 3c32a581c2..5916b6b1a0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PatternMatchSupported.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoPatternMatchSupported.java @@ -17,10 +17,10 @@ package org.apache.doris.nereids.util; -import org.apache.doris.nereids.pattern.GeneratedPatterns; +import org.apache.doris.nereids.pattern.GeneratedMemoPatterns; import org.apache.doris.nereids.rules.RulePromise; -public interface PatternMatchSupported extends GeneratedPatterns { +public interface MemoPatternMatchSupported extends GeneratedMemoPatterns { @Override default RulePromise defaultPromise() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoTestUtils.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoTestUtils.java index f63b0842fc..f59b7a1342 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoTestUtils.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoTestUtils.java @@ -81,8 +81,9 @@ public class MemoTestUtils { public static CascadesContext createCascadesContext(StatementContext statementContext, Plan initPlan) { PhysicalProperties requestProperties = NereidsPlanner.buildInitRequireProperties(initPlan); - CascadesContext cascadesContext = CascadesContext.newContext( + CascadesContext cascadesContext = CascadesContext.newRewriteContext( statementContext, initPlan, requestProperties); + cascadesContext.toMemo(); MemoValidator.validateInitState(cascadesContext.getMemo(), initPlan); return cascadesContext; } @@ -90,7 +91,7 @@ public class MemoTestUtils { public static LogicalPlan analyze(String sql) { CascadesContext cascadesContext = createCascadesContext(sql); cascadesContext.newAnalyzer().analyze(); - return (LogicalPlan) cascadesContext.getMemo().copyOut(); + return (LogicalPlan) cascadesContext.getRewritePlan(); } /** diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index 31fb819fa4..f49711aa89 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -26,8 +26,8 @@ import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator; import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.nereids.jobs.JobContext; -import org.apache.doris.nereids.jobs.batch.NereidsRewriteJobExecutor; -import org.apache.doris.nereids.jobs.batch.OptimizeRulesJob; +import org.apache.doris.nereids.jobs.batch.CascadesOptimizer; +import org.apache.doris.nereids.jobs.batch.NereidsRewriter; import org.apache.doris.nereids.jobs.cascades.DeriveStatsJob; import org.apache.doris.nereids.jobs.joinorder.JoinOrderJob; import org.apache.doris.nereids.memo.CopyInResult; @@ -107,20 +107,17 @@ public class PlanChecker { return this; } - public PlanChecker analyze() { - MemoTestUtils.createCascadesContext(connectContext, parsedPlan); - return this; - } - public PlanChecker analyze(String sql) { this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, sql); this.cascadesContext.newAnalyzer().analyze(); + this.cascadesContext.toMemo(); return this; } public PlanChecker analyze(Plan plan) { this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan); this.cascadesContext.newAnalyzer().analyze(); + this.cascadesContext.toMemo(); MemoValidator.validate(cascadesContext.getMemo()); return this; } @@ -176,13 +173,14 @@ public class PlanChecker { } public PlanChecker rewrite() { - new NereidsRewriteJobExecutor(cascadesContext).execute(); + new NereidsRewriter(cascadesContext).execute(); + cascadesContext.toMemo(); return this; } public PlanChecker optimize() { double now = System.currentTimeMillis(); - new OptimizeRulesJob(cascadesContext).execute(); + new CascadesOptimizer(cascadesContext).execute(); System.out.println("cascades:" + (System.currentTimeMillis() - now)); return this; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/OptimizeRulesJob.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanPatternMatchSupported.java similarity index 66% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/OptimizeRulesJob.java rename to fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanPatternMatchSupported.java index 61c6887233..e664f219b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/OptimizeRulesJob.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanPatternMatchSupported.java @@ -15,20 +15,14 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.jobs.batch; +package org.apache.doris.nereids.util; -import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.pattern.GeneratedPlanPatterns; +import org.apache.doris.nereids.rules.RulePromise; -import com.google.common.collect.ImmutableList; - -/** - * cascade optimizer added. - */ -public class OptimizeRulesJob extends BatchRulesJob { - public OptimizeRulesJob(CascadesContext cascadesContext) { - super(cascadesContext); - rulesJob.addAll(ImmutableList.of( - optimize() - )); +public interface PlanPatternMatchSupported extends GeneratedPlanPatterns { + @Override + default RulePromise defaultPromise() { + return RulePromise.PLAN_CHECK; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index 35f730726a..3068340c0b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -53,12 +53,10 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.parser.NereidsParser; -import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.NamedExpressionUtil; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.planner.Planner; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.OriginStatement; @@ -177,15 +175,14 @@ public abstract class TestWithFeService { protected CascadesContext createCascadesContext(String sql) { StatementContext statementCtx = createStatementCtx(sql); - LogicalPlan initPlan = new NereidsParser().parseSingle(sql); - PhysicalProperties requestProperties = NereidsPlanner.buildInitRequireProperties(initPlan); - return CascadesContext.newContext(statementCtx, initPlan, requestProperties); + return MemoTestUtils.createCascadesContext(statementCtx, sql); } public LogicalPlan analyze(String sql) { CascadesContext cascadesContext = createCascadesContext(sql); cascadesContext.newAnalyzer().analyze(); - return (LogicalPlan) cascadesContext.getMemo().copyOut(); + cascadesContext.toMemo(); + return (LogicalPlan) cascadesContext.getRewritePlan(); } protected ConnectContext createCtx(UserIdentity user, String host) throws IOException { diff --git a/regression-test/suites/nereids_benchmark_p2/explain_clickbench_benchmark.groovy b/regression-test/suites/nereids_benchmark_p2/explain_clickbench_benchmark.groovy index a8c491d855..8c52b52e9f 100644 --- a/regression-test/suites/nereids_benchmark_p2/explain_clickbench_benchmark.groovy +++ b/regression-test/suites/nereids_benchmark_p2/explain_clickbench_benchmark.groovy @@ -182,10 +182,11 @@ suite("explain_clickbench_benchmark") { sql "SET enable_nereids_planner=true" sql "SET enable_vectorized_engine=true" sql "SET enable_fallback_to_original_planner=false" + sql "SET enable_dphyp_optimizer=true" benchmark { warmUp true - executeTimes 3 + executeTimes 300 skipFailure false printResult true