database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](nereids) merge push down and remove redundant operator rules into one batch (#12569)

Browse Source 
1. For some related rules, we need to execute them together to get the expected plan.
2. Add session variables to avoid fallback to stale planner when running regression tests of nereids for piggyback.
This commit is contained in:
Kikyou1997
2022-09-14 14:37:36 +08:00
committed by GitHub
parent 08ee84ef67
commit 3543f85ae5
15 changed files with 50 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/RewriteJob.java
View File

@ -62,17 +62,17 @@ public class RewriteJob extends BatchRulesJob {
.addAll(new ConvertApplyToJoinJob(cascadesContext).rulesJob)
.add(topDownBatch(ImmutableList.of(new ExpressionNormalization())))
.add(topDownBatch(ImmutableList.of(new NormalizeAggregate())))
.add(topDownBatch(ImmutableList.of(new ReorderJoin())))
.add(topDownBatch(ImmutableList.of(new ReorderJoin())))
.add(topDownBatch(ImmutableList.of(new FindHashConditionForJoin())))
.add(topDownBatch(ImmutableList.of(new PushPredicateThroughJoin())))
.add(topDownBatch(ImmutableList.of(new NormalizeAggregate())))
.add(topDownBatch(ImmutableList.of(new ColumnPruning())))
.add(topDownBatch(ImmutableList.of(new PushPredicateThroughJoin(),
new PushdownProjectThroughLimit(),
new PushdownFilterThroughProject(),
new MergeConsecutiveProjects(),
new MergeConsecutiveFilters(),
new MergeConsecutiveLimits())))
.add(topDownBatch(ImmutableList.of(new AggregateDisassemble())))
.add(topDownBatch(ImmutableList.of(new PushdownProjectThroughLimit())))
.add(topDownBatch(ImmutableList.of(new PushdownFilterThroughProject())))
.add(bottomUpBatch(ImmutableList.of(new MergeConsecutiveProjects())))
.add(topDownBatch(ImmutableList.of(new MergeConsecutiveFilters())))
.add(bottomUpBatch(ImmutableList.of(new MergeConsecutiveLimits())))
.add(topDownBatch(ImmutableList.of(new EliminateLimit())))
.add(topDownBatch(ImmutableList.of(new EliminateFilter())))
.add(topDownBatch(ImmutableList.of(new PruneOlapScanPartition())))

11
fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.analysis.InsertStmt;
import org.apache.doris.analysis.KillStmt;
import org.apache.doris.analysis.Queriable;
import org.apache.doris.analysis.QueryStmt;
import org.apache.doris.analysis.SelectStmt;
import org.apache.doris.analysis.SqlParser;
import org.apache.doris.analysis.SqlScanner;
import org.apache.doris.analysis.StatementBase;
@ -234,11 +235,14 @@ public class ConnectProcessor {
boolean alreadyAddedToAuditInfoList = false;
try {
List<StatementBase> stmts = null;
if (ctx.getSessionVariable().isEnableNereidsPlanner()) {
Exception nereidsParseException = null;
// ctx could be null in some unit tests
if (ctx != null && ctx.getSessionVariable().isEnableNereidsPlanner()) {
NereidsParser nereidsParser = new NereidsParser();
try {
stmts = nereidsParser.parseSQL(originStmt);
} catch (Exception e) {
nereidsParseException = e;
// TODO: We should catch all exception here until we support all query syntax.
LOG.warn(" Fallback to stale planner."
+ " Nereids cannot process this statement: \"{}\".", originStmt, e);
@ -255,6 +259,11 @@ public class ConnectProcessor {
ctx.resetReturnRows();
}
parsedStmt = stmts.get(i);
if (parsedStmt instanceof SelectStmt) {
if (!ctx.getSessionVariable().enableFallbackToOriginalPlanner) {
throw new Exception(String.format("SQL: {}", parsedStmt.toSql()), nereidsParseException);
}
}
parsedStmt.setOrigStmt(new OriginStatement(originStmt, i));
parsedStmt.setUserInfo(ctx.getCurrentUserIdentity());
executor = new StmtExecutor(ctx, parsedStmt);

10
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -195,6 +195,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String ENABLE_NEREIDS_PLANNER = "enable_nereids_planner";
public static final String ENABLE_FALLBACK_TO_ORIGINAL_PLANNER = "enable_fallback_to_original_planner";
public static final String ENABLE_NEREIDS_REORDER_TO_ELIMINATE_CROSS_JOIN =
"enable_nereids_reorder_to_eliminate_cross_join";
@ -534,6 +536,7 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = ENABLE_LOCAL_EXCHANGE)
public boolean enableLocalExchange = false;
/**
* For debugg purpose, dont' merge unique key and agg key when reading data.
*/
@ -546,6 +549,13 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = SKIP_DELETE_PREDICATE)
public boolean skipDeletePredicate = false;
// This variable is used to avoid FE fallback to the original parser. When we execute SQL in regression tests
// for nereids, fallback will cause the Doris return the correct result although the syntax is unsupported
// in nereids for some mistaken modification. You should set it on the
@VariableMgr.VarAttr(name = ENABLE_FALLBACK_TO_ORIGINAL_PLANNER)
public boolean enableFallbackToOriginalPlanner = true;
public String getBlockEncryptionMode() {
return blockEncryptionMode;
}

5
regression-test/suites/nereids_syntax_p0/agg_with_const.groovy
View File

@ -17,6 +17,9 @@
suite("agg_with_const") {
sql "SET enable_vectorized_engine=true"
sql "SET enable_nereids_planner=true"
sql """
DROP TABLE IF EXISTS t1
"""
@ -33,6 +36,8 @@ suite("agg_with_const") {
insert into t1 values(1994, 1994, 1995)
"""
sql "SET enable_fallback_to_original_planner=false"
qt_select """
select count(2) + 1, sum(2) + sum(col1) from t1
"""

1
regression-test/suites/nereids_syntax_p0/empty_relation.groovy
View File

@ -19,6 +19,7 @@ suite("empty_relation") {
// enable nereids and vectorized engine
sql "SET enable_vectorized_engine=true"
sql "SET enable_nereids_planner=true"
sql "SET enable_fallback_to_original_planner=false"
test {
sql "select *, substring(s_name, 1, 2) from supplier limit 0"

2
regression-test/suites/nereids_syntax_p0/explain.groovy
View File

@ -24,6 +24,8 @@ suite("explain") {
SET enable_nereids_planner=true
"""
sql "SET enable_fallback_to_original_planner=false"
explain {
sql("select count(2) + 1, sum(2) + sum(lo_suppkey) from lineorder")
contains "projections: lo_suppkey"

1
regression-test/suites/nereids_syntax_p0/function.groovy
View File

@ -23,6 +23,7 @@ suite("function") {
sql """
SET enable_nereids_planner=true
"""
sql "SET enable_fallback_to_original_planner=false"
order_qt_max """
SELECT max(lo_discount), max(lo_extendedprice) AS max_extendedprice FROM lineorder;

1
regression-test/suites/nereids_syntax_p0/having.groovy
View File

@ -43,6 +43,7 @@ suite("test_nereids_having") {
(3, 3, 9)
"""
sql "SET enable_fallback_to_original_planner=false"
order_qt_select "SELECT a1 as value FROM t1 GROUP BY a1 HAVING a1 > 0";
order_qt_select "SELECT a1 as value FROM t1 GROUP BY a1 HAVING value > 0";

2
regression-test/suites/nereids_syntax_p0/inpredicate.groovy
View File

@ -24,6 +24,8 @@ suite("inpredicate") {
SET enable_nereids_planner=true
"""
sql "SET enable_fallback_to_original_planner=false"
order_qt_in_predicate_1 """
SELECT * FROM supplier WHERE s_suppkey in (1, 2, 3);
"""

2
regression-test/suites/nereids_syntax_p0/join.groovy
View File

@ -24,6 +24,8 @@ suite("join") {
SET enable_nereids_planner=true
"""
sql "SET enable_fallback_to_original_planner=false"
order_qt_inner_join_1 """
SELECT * FROM lineorder JOIN supplier ON lineorder.lo_suppkey = supplier.s_suppkey
"""

1
regression-test/suites/nereids_syntax_p0/one_row_relation.groovy
View File

@ -19,6 +19,7 @@ suite("one_row_relation") {
// enable nereids and vectorized engine
sql "SET enable_vectorized_engine=true"
sql "SET enable_nereids_planner=true"
sql "SET enable_fallback_to_original_planner=false"
test {
sql "select 100, 'abc', substring('abc', 1, 2), substring(substring('abcdefg', 4, 3), 1, 2)"

2
regression-test/suites/nereids_syntax_p0/rollup.groovy
View File

@ -67,6 +67,8 @@ suite("rollup") {
sql "set enable_nereids_planner=true"
sql "SET enable_fallback_to_original_planner=false"
explain {
sql("select k2, sum(v1) from rollup_t1 group by k2")
contains("rollup_t1(r1)")

2
regression-test/suites/nereids_syntax_p0/sub_query_alias.groovy
View File

@ -24,6 +24,8 @@ suite("sub_query_alias") {
SET enable_nereids_planner=true
"""
sql "SET enable_fallback_to_original_planner=false"
qt_select_1 """
select t.c_custkey, t.lo_custkey
from (

2
regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy
View File

@ -89,6 +89,8 @@ suite ("sub_query_correlated") {
insert into subquery4 values (5,4), (5,2), (8,3), (5,4), (6,7), (8,9)
"""
sql "SET enable_fallback_to_original_planner=false"
//------------------Correlated-----------------
qt_scalar_less_than_corr """
select * from subquery1 where subquery1.k1 < (select sum(subquery3.k3) from subquery3 where subquery3.v2 = subquery1.k2) order by k1

2
regression-test/suites/nereids_syntax_p0/view.groovy
View File

@ -46,6 +46,8 @@ suite("view") {
on v1.c_custkey = t.lo_custkey;
"""
sql "SET enable_fallback_to_original_planner=false"
qt_select_1 """
select *
from v1