diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/RewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/RewriteJob.java index 956ba0c961..2aea7994fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/RewriteJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/RewriteJob.java @@ -62,17 +62,17 @@ public class RewriteJob extends BatchRulesJob { .addAll(new ConvertApplyToJoinJob(cascadesContext).rulesJob) .add(topDownBatch(ImmutableList.of(new ExpressionNormalization()))) .add(topDownBatch(ImmutableList.of(new NormalizeAggregate()))) - .add(topDownBatch(ImmutableList.of(new ReorderJoin()))) + .add(topDownBatch(ImmutableList.of(new ReorderJoin()))) .add(topDownBatch(ImmutableList.of(new FindHashConditionForJoin()))) - .add(topDownBatch(ImmutableList.of(new PushPredicateThroughJoin()))) .add(topDownBatch(ImmutableList.of(new NormalizeAggregate()))) .add(topDownBatch(ImmutableList.of(new ColumnPruning()))) + .add(topDownBatch(ImmutableList.of(new PushPredicateThroughJoin(), + new PushdownProjectThroughLimit(), + new PushdownFilterThroughProject(), + new MergeConsecutiveProjects(), + new MergeConsecutiveFilters(), + new MergeConsecutiveLimits()))) .add(topDownBatch(ImmutableList.of(new AggregateDisassemble()))) - .add(topDownBatch(ImmutableList.of(new PushdownProjectThroughLimit()))) - .add(topDownBatch(ImmutableList.of(new PushdownFilterThroughProject()))) - .add(bottomUpBatch(ImmutableList.of(new MergeConsecutiveProjects()))) - .add(topDownBatch(ImmutableList.of(new MergeConsecutiveFilters()))) - .add(bottomUpBatch(ImmutableList.of(new MergeConsecutiveLimits()))) .add(topDownBatch(ImmutableList.of(new EliminateLimit()))) .add(topDownBatch(ImmutableList.of(new EliminateFilter()))) .add(topDownBatch(ImmutableList.of(new PruneOlapScanPartition()))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 0be0d61646..7920ae6dae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.InsertStmt; import org.apache.doris.analysis.KillStmt; import org.apache.doris.analysis.Queriable; import org.apache.doris.analysis.QueryStmt; +import org.apache.doris.analysis.SelectStmt; import org.apache.doris.analysis.SqlParser; import org.apache.doris.analysis.SqlScanner; import org.apache.doris.analysis.StatementBase; @@ -234,11 +235,14 @@ public class ConnectProcessor { boolean alreadyAddedToAuditInfoList = false; try { List stmts = null; - if (ctx.getSessionVariable().isEnableNereidsPlanner()) { + Exception nereidsParseException = null; + // ctx could be null in some unit tests + if (ctx != null && ctx.getSessionVariable().isEnableNereidsPlanner()) { NereidsParser nereidsParser = new NereidsParser(); try { stmts = nereidsParser.parseSQL(originStmt); } catch (Exception e) { + nereidsParseException = e; // TODO: We should catch all exception here until we support all query syntax. LOG.warn(" Fallback to stale planner." + " Nereids cannot process this statement: \"{}\".", originStmt, e); @@ -255,6 +259,11 @@ public class ConnectProcessor { ctx.resetReturnRows(); } parsedStmt = stmts.get(i); + if (parsedStmt instanceof SelectStmt) { + if (!ctx.getSessionVariable().enableFallbackToOriginalPlanner) { + throw new Exception(String.format("SQL: {}", parsedStmt.toSql()), nereidsParseException); + } + } parsedStmt.setOrigStmt(new OriginStatement(originStmt, i)); parsedStmt.setUserInfo(ctx.getCurrentUserIdentity()); executor = new StmtExecutor(ctx, parsedStmt); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index a12ee9059c..e27f5ff2f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -195,6 +195,8 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_NEREIDS_PLANNER = "enable_nereids_planner"; + public static final String ENABLE_FALLBACK_TO_ORIGINAL_PLANNER = "enable_fallback_to_original_planner"; + public static final String ENABLE_NEREIDS_REORDER_TO_ELIMINATE_CROSS_JOIN = "enable_nereids_reorder_to_eliminate_cross_join"; @@ -534,6 +536,7 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = ENABLE_LOCAL_EXCHANGE) public boolean enableLocalExchange = false; + /** * For debugg purpose, dont' merge unique key and agg key when reading data. */ @@ -546,6 +549,13 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = SKIP_DELETE_PREDICATE) public boolean skipDeletePredicate = false; + // This variable is used to avoid FE fallback to the original parser. When we execute SQL in regression tests + // for nereids, fallback will cause the Doris return the correct result although the syntax is unsupported + // in nereids for some mistaken modification. You should set it on the + @VariableMgr.VarAttr(name = ENABLE_FALLBACK_TO_ORIGINAL_PLANNER) + public boolean enableFallbackToOriginalPlanner = true; + + public String getBlockEncryptionMode() { return blockEncryptionMode; } diff --git a/regression-test/suites/nereids_syntax_p0/agg_with_const.groovy b/regression-test/suites/nereids_syntax_p0/agg_with_const.groovy index ed2e5b60f1..fb4fd81fe7 100644 --- a/regression-test/suites/nereids_syntax_p0/agg_with_const.groovy +++ b/regression-test/suites/nereids_syntax_p0/agg_with_const.groovy @@ -17,6 +17,9 @@ suite("agg_with_const") { + sql "SET enable_vectorized_engine=true" + sql "SET enable_nereids_planner=true" + sql """ DROP TABLE IF EXISTS t1 """ @@ -33,6 +36,8 @@ suite("agg_with_const") { insert into t1 values(1994, 1994, 1995) """ + sql "SET enable_fallback_to_original_planner=false" + qt_select """ select count(2) + 1, sum(2) + sum(col1) from t1 """ diff --git a/regression-test/suites/nereids_syntax_p0/empty_relation.groovy b/regression-test/suites/nereids_syntax_p0/empty_relation.groovy index 05e1af14ee..4e1babd166 100644 --- a/regression-test/suites/nereids_syntax_p0/empty_relation.groovy +++ b/regression-test/suites/nereids_syntax_p0/empty_relation.groovy @@ -19,6 +19,7 @@ suite("empty_relation") { // enable nereids and vectorized engine sql "SET enable_vectorized_engine=true" sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" test { sql "select *, substring(s_name, 1, 2) from supplier limit 0" diff --git a/regression-test/suites/nereids_syntax_p0/explain.groovy b/regression-test/suites/nereids_syntax_p0/explain.groovy index 865e08d553..52e13de757 100644 --- a/regression-test/suites/nereids_syntax_p0/explain.groovy +++ b/regression-test/suites/nereids_syntax_p0/explain.groovy @@ -24,6 +24,8 @@ suite("explain") { SET enable_nereids_planner=true """ + sql "SET enable_fallback_to_original_planner=false" + explain { sql("select count(2) + 1, sum(2) + sum(lo_suppkey) from lineorder") contains "projections: lo_suppkey" diff --git a/regression-test/suites/nereids_syntax_p0/function.groovy b/regression-test/suites/nereids_syntax_p0/function.groovy index f0b44ae2e6..c4099a0798 100644 --- a/regression-test/suites/nereids_syntax_p0/function.groovy +++ b/regression-test/suites/nereids_syntax_p0/function.groovy @@ -23,6 +23,7 @@ suite("function") { sql """ SET enable_nereids_planner=true """ + sql "SET enable_fallback_to_original_planner=false" order_qt_max """ SELECT max(lo_discount), max(lo_extendedprice) AS max_extendedprice FROM lineorder; diff --git a/regression-test/suites/nereids_syntax_p0/having.groovy b/regression-test/suites/nereids_syntax_p0/having.groovy index 4f6abf2e61..d7d61f5687 100644 --- a/regression-test/suites/nereids_syntax_p0/having.groovy +++ b/regression-test/suites/nereids_syntax_p0/having.groovy @@ -43,6 +43,7 @@ suite("test_nereids_having") { (3, 3, 9) """ + sql "SET enable_fallback_to_original_planner=false" order_qt_select "SELECT a1 as value FROM t1 GROUP BY a1 HAVING a1 > 0"; order_qt_select "SELECT a1 as value FROM t1 GROUP BY a1 HAVING value > 0"; diff --git a/regression-test/suites/nereids_syntax_p0/inpredicate.groovy b/regression-test/suites/nereids_syntax_p0/inpredicate.groovy index be6f405396..de73bb106b 100644 --- a/regression-test/suites/nereids_syntax_p0/inpredicate.groovy +++ b/regression-test/suites/nereids_syntax_p0/inpredicate.groovy @@ -24,6 +24,8 @@ suite("inpredicate") { SET enable_nereids_planner=true """ + sql "SET enable_fallback_to_original_planner=false" + order_qt_in_predicate_1 """ SELECT * FROM supplier WHERE s_suppkey in (1, 2, 3); """ diff --git a/regression-test/suites/nereids_syntax_p0/join.groovy b/regression-test/suites/nereids_syntax_p0/join.groovy index 5faab7d912..700fe1a49a 100644 --- a/regression-test/suites/nereids_syntax_p0/join.groovy +++ b/regression-test/suites/nereids_syntax_p0/join.groovy @@ -24,6 +24,8 @@ suite("join") { SET enable_nereids_planner=true """ + sql "SET enable_fallback_to_original_planner=false" + order_qt_inner_join_1 """ SELECT * FROM lineorder JOIN supplier ON lineorder.lo_suppkey = supplier.s_suppkey """ diff --git a/regression-test/suites/nereids_syntax_p0/one_row_relation.groovy b/regression-test/suites/nereids_syntax_p0/one_row_relation.groovy index 9a73edf1bb..ace60c735d 100644 --- a/regression-test/suites/nereids_syntax_p0/one_row_relation.groovy +++ b/regression-test/suites/nereids_syntax_p0/one_row_relation.groovy @@ -19,6 +19,7 @@ suite("one_row_relation") { // enable nereids and vectorized engine sql "SET enable_vectorized_engine=true" sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" test { sql "select 100, 'abc', substring('abc', 1, 2), substring(substring('abcdefg', 4, 3), 1, 2)" diff --git a/regression-test/suites/nereids_syntax_p0/rollup.groovy b/regression-test/suites/nereids_syntax_p0/rollup.groovy index aabb980138..b7830a4484 100644 --- a/regression-test/suites/nereids_syntax_p0/rollup.groovy +++ b/regression-test/suites/nereids_syntax_p0/rollup.groovy @@ -67,6 +67,8 @@ suite("rollup") { sql "set enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + explain { sql("select k2, sum(v1) from rollup_t1 group by k2") contains("rollup_t1(r1)") diff --git a/regression-test/suites/nereids_syntax_p0/sub_query_alias.groovy b/regression-test/suites/nereids_syntax_p0/sub_query_alias.groovy index 8a50987089..e4cdcec5b5 100644 --- a/regression-test/suites/nereids_syntax_p0/sub_query_alias.groovy +++ b/regression-test/suites/nereids_syntax_p0/sub_query_alias.groovy @@ -24,6 +24,8 @@ suite("sub_query_alias") { SET enable_nereids_planner=true """ + sql "SET enable_fallback_to_original_planner=false" + qt_select_1 """ select t.c_custkey, t.lo_custkey from ( diff --git a/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy b/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy index abc28073a2..41a3bd7237 100644 --- a/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy +++ b/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy @@ -89,6 +89,8 @@ suite ("sub_query_correlated") { insert into subquery4 values (5,4), (5,2), (8,3), (5,4), (6,7), (8,9) """ + sql "SET enable_fallback_to_original_planner=false" + //------------------Correlated----------------- qt_scalar_less_than_corr """ select * from subquery1 where subquery1.k1 < (select sum(subquery3.k3) from subquery3 where subquery3.v2 = subquery1.k2) order by k1 diff --git a/regression-test/suites/nereids_syntax_p0/view.groovy b/regression-test/suites/nereids_syntax_p0/view.groovy index 29ca5a34f3..15956a24cc 100644 --- a/regression-test/suites/nereids_syntax_p0/view.groovy +++ b/regression-test/suites/nereids_syntax_p0/view.groovy @@ -46,6 +46,8 @@ suite("view") { on v1.c_custkey = t.lo_custkey; """ + sql "SET enable_fallback_to_original_planner=false" + qt_select_1 """ select * from v1