database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](nereids) merge push down and remove redundant operator rules into one batch (#12569)

Browse Source 
1. For some related rules, we need to execute them together to get the expected plan.
2. Add session variables to avoid fallback to stale planner when running regression tests of nereids for piggyback.
This commit is contained in:
Kikyou1997
2022-09-14 14:37:36 +08:00
committed by GitHub
parent 08ee84ef67
commit 3543f85ae5
15 changed files with 50 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/RewriteJob.java
View File

@ -62,17 +62,17 @@ public class RewriteJob extends BatchRulesJob {
.addAll(new ConvertApplyToJoinJob(cascadesContext).rulesJob)
.add(topDownBatch(ImmutableList.of(new ExpressionNormalization())))
.add(topDownBatch(ImmutableList.of(new NormalizeAggregate())))
.add(topDownBatch(ImmutableList.of(new ReorderJoin())))
.add(topDownBatch(ImmutableList.of(new ReorderJoin())))
.add(topDownBatch(ImmutableList.of(new FindHashConditionForJoin())))
.add(topDownBatch(ImmutableList.of(new PushPredicateThroughJoin())))
.add(topDownBatch(ImmutableList.of(new NormalizeAggregate())))
.add(topDownBatch(ImmutableList.of(new ColumnPruning())))
.add(topDownBatch(ImmutableList.of(new PushPredicateThroughJoin(),
new PushdownProjectThroughLimit(),
new PushdownFilterThroughProject(),
new MergeConsecutiveProjects(),
new MergeConsecutiveFilters(),
new MergeConsecutiveLimits())))
.add(topDownBatch(ImmutableList.of(new AggregateDisassemble())))
.add(topDownBatch(ImmutableList.of(new PushdownProjectThroughLimit())))
.add(topDownBatch(ImmutableList.of(new PushdownFilterThroughProject())))
.add(bottomUpBatch(ImmutableList.of(new MergeConsecutiveProjects())))
.add(topDownBatch(ImmutableList.of(new MergeConsecutiveFilters())))
.add(bottomUpBatch(ImmutableList.of(new MergeConsecutiveLimits())))
.add(topDownBatch(ImmutableList.of(new EliminateLimit())))
.add(topDownBatch(ImmutableList.of(new EliminateFilter())))
.add(topDownBatch(ImmutableList.of(new PruneOlapScanPartition())))

11
fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.analysis.InsertStmt;
import org.apache.doris.analysis.KillStmt;
import org.apache.doris.analysis.Queriable;
import org.apache.doris.analysis.QueryStmt;
import org.apache.doris.analysis.SelectStmt;
import org.apache.doris.analysis.SqlParser;
import org.apache.doris.analysis.SqlScanner;
import org.apache.doris.analysis.StatementBase;
@ -234,11 +235,14 @@ public class ConnectProcessor {
boolean alreadyAddedToAuditInfoList = false;
try {
List<StatementBase> stmts = null;
if (ctx.getSessionVariable().isEnableNereidsPlanner()) {
Exception nereidsParseException = null;
// ctx could be null in some unit tests
if (ctx != null && ctx.getSessionVariable().isEnableNereidsPlanner()) {
NereidsParser nereidsParser = new NereidsParser();
try {
stmts = nereidsParser.parseSQL(originStmt);
} catch (Exception e) {
nereidsParseException = e;
// TODO: We should catch all exception here until we support all query syntax.
LOG.warn(" Fallback to stale planner."
+ " Nereids cannot process this statement: \"{}\".", originStmt, e);
@ -255,6 +259,11 @@ public class ConnectProcessor {
ctx.resetReturnRows();
}
parsedStmt = stmts.get(i);
if (parsedStmt instanceof SelectStmt) {
if (!ctx.getSessionVariable().enableFallbackToOriginalPlanner) {
throw new Exception(String.format("SQL: {}", parsedStmt.toSql()), nereidsParseException);
}
}
parsedStmt.setOrigStmt(new OriginStatement(originStmt, i));
parsedStmt.setUserInfo(ctx.getCurrentUserIdentity());
executor = new StmtExecutor(ctx, parsedStmt);

10
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -195,6 +195,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String ENABLE_NEREIDS_PLANNER = "enable_nereids_planner";
public static final String ENABLE_FALLBACK_TO_ORIGINAL_PLANNER = "enable_fallback_to_original_planner";
public static final String ENABLE_NEREIDS_REORDER_TO_ELIMINATE_CROSS_JOIN =
"enable_nereids_reorder_to_eliminate_cross_join";
@ -534,6 +536,7 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = ENABLE_LOCAL_EXCHANGE)
public boolean enableLocalExchange = false;
/**
* For debugg purpose, dont' merge unique key and agg key when reading data.
*/
@ -546,6 +549,13 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = SKIP_DELETE_PREDICATE)
public boolean skipDeletePredicate = false;
// This variable is used to avoid FE fallback to the original parser. When we execute SQL in regression tests
// for nereids, fallback will cause the Doris return the correct result although the syntax is unsupported
// in nereids for some mistaken modification. You should set it on the
@VariableMgr.VarAttr(name = ENABLE_FALLBACK_TO_ORIGINAL_PLANNER)
public boolean enableFallbackToOriginalPlanner = true;
public String getBlockEncryptionMode() {
return blockEncryptionMode;
}