database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Feat](Nereids) join hint support stage one (#27378)

Browse Source 
support view as a independent unit of leading hint
add random test check of leading hint query
add more test with data of leading hint query
add random test check of distribute hint
This commit is contained in:
LiBinfeng
2023-11-29 21:08:08 +08:00
committed by GitHub
parent 6cdaf8ea32
commit 83ed8d3cba
52 changed files with 9019 additions and 1711 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
View File

@ -27,6 +27,7 @@ import org.apache.doris.nereids.analyzer.UnboundOlapTableSink;
import org.apache.doris.nereids.analyzer.UnboundOneRowRelation;
import org.apache.doris.nereids.analyzer.UnboundRelation;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.hint.Hint;
import org.apache.doris.nereids.jobs.Job;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.executor.Analyzer;
@ -114,6 +115,9 @@ public class CascadesContext implements ScheduleContext {
private final Optional<CascadesContext> parent;
private final List<MaterializationContext> materializationContexts;
private boolean isLeadingJoin = false;
private final Map<String, Hint> hintMap = Maps.newLinkedHashMap();
/**
* Constructor of OptimizerContext.
@ -615,4 +619,16 @@ public class CascadesContext implements ScheduleContext {
p.value().setStatistics(updatedConsumerStats);
}
}
public boolean isLeadingJoin() {
return isLeadingJoin;
}
public void setLeadingJoin(boolean leadingJoin) {
isLeadingJoin = leadingJoin;
}
public Map<String, Hint> getHintMap() {
return hintMap;
}
}

23
fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
View File

@ -68,7 +68,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
@ -353,30 +352,30 @@ public class NereidsPlanner extends Planner {
/**
* getting hints explain string, which specified by enumerate and show in lists
* @param hintMap hint map recorded in statement context
* @param hints hint map recorded in statement context
* @return explain string shows using of hint
*/
public String getHintExplainString(Map<String, Hint> hintMap) {
public String getHintExplainString(List<Hint> hints) {
String used = "";
String unUsed = "";
String syntaxError = "";
for (Map.Entry<String, Hint> entry : hintMap.entrySet()) {
switch (entry.getValue().getStatus()) {
for (Hint hint : hints) {
switch (hint.getStatus()) {
case UNUSED:
unUsed = unUsed + " " + entry.getValue().getExplainString();
unUsed = unUsed + " " + hint.getExplainString();
break;
case SYNTAX_ERROR:
syntaxError = syntaxError + " " + entry.getValue().getExplainString()
+ " Msg:" + entry.getValue().getErrorMessage();
syntaxError = syntaxError + " " + hint.getExplainString()
+ " Msg:" + hint.getErrorMessage();
break;
case SUCCESS:
used = used + " " + entry.getValue().getExplainString();
used = used + " " + hint.getExplainString();
break;
default:
break;
}
}
return "\nUsed:" + used + "\nUnUsed:" + unUsed + "\nSyntaxError:" + syntaxError;
return "\nHint log:" + "\nUsed:" + used + "\nUnUsed:" + unUsed + "\nSyntaxError:" + syntaxError;
}
@Override
@ -417,8 +416,8 @@ public class NereidsPlanner extends Planner {
default:
plan = super.getExplainString(explainOptions);
}
if (!statementContext.getHintMap().isEmpty()) {
String hint = getHintExplainString(cascadesContext.getStatementContext().getHintMap());
if (statementContext != null && !statementContext.getHints().isEmpty()) {
String hint = getHintExplainString(statementContext.getHints());
return plan + hint;
}
return plan;

26
fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
View File

@ -42,6 +42,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -72,8 +73,6 @@ public class StatementContext {
private boolean isDpHyp = false;
private boolean isOtherJoinReorder = false;
private boolean isLeadingJoin = false;
private final IdGenerator<ExprId> exprIdGenerator = ExprId.createGenerator();
private final IdGenerator<ObjectId> objectIdGenerator = ObjectId.createGenerator();
private final IdGenerator<RelationId> relationIdGenerator = RelationId.createGenerator();
@ -87,9 +86,10 @@ public class StatementContext {
private final Map<CTEId, List<Pair<Map<Slot, Slot>, Group>>> cteIdToConsumerGroup = new HashMap<>();
private final Map<CTEId, LogicalPlan> rewrittenCteProducer = new HashMap<>();
private final Map<CTEId, LogicalPlan> rewrittenCteConsumer = new HashMap<>();
private final Map<String, Hint> hintMap = Maps.newLinkedHashMap();
private final Set<String> viewDdlSqlSet = Sets.newHashSet();
private final List<Hint> hints = new ArrayList<>();
public StatementContext() {
this.connectContext = ConnectContext.get();
}
@ -147,14 +147,6 @@ public class StatementContext {
isDpHyp = dpHyp;
}
public boolean isLeadingJoin() {
return isLeadingJoin;
}
public void setLeadingJoin(boolean leadingJoin) {
isLeadingJoin = leadingJoin;
}
public boolean isOtherJoinReorder() {
return isOtherJoinReorder;
}
@ -193,10 +185,6 @@ public class StatementContext {
return supplier.get();
}
public Map<String, Hint> getHintMap() {
return hintMap;
}
public ColumnAliasGenerator getColumnAliasGenerator() {
return columnAliasGenerator == null
? columnAliasGenerator = new ColumnAliasGenerator()
@ -246,4 +234,12 @@ public class StatementContext {
public List<String> getViewDdlSqls() {
return ImmutableList.copyOf(viewDdlSqlSet);
}
public void addHint(Hint hint) {
this.hints.add(hint);
}
public List<Hint> getHints() {
return ImmutableList.copyOf(hints);
}
}

106
fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java
View File

@ -31,6 +31,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
import org.apache.doris.nereids.util.JoinUtils;
import com.google.common.collect.Maps;
@ -61,10 +62,14 @@ public class LeadingHint extends Hint {
private final List<Pair<Long, Expression>> filters = new ArrayList<>();
private final Map<Expression, JoinType> conditionJoinType = Maps.newLinkedHashMap();
private final List<JoinConstraint> joinConstraintList = new ArrayList<>();
private Long innerJoinBitmap = 0L;
private Long totalBitmap = 0L;
public LeadingHint(String hintName) {
super(hintName);
}
@ -191,6 +196,30 @@ public class LeadingHint extends Hint {
return filters;
}
public void putConditionJoinType(Expression filter, JoinType joinType) {
conditionJoinType.put(filter, joinType);
}
/**
* find out whether conditions can match original joinType
* @param conditions conditions needs to put on this join
* @param joinType join type computed by join constraint
* @return can conditions matched
*/
public boolean isConditionJoinTypeMatched(List<Expression> conditions, JoinType joinType) {
for (Expression condition : conditions) {
JoinType originalJoinType = conditionJoinType.get(condition);
if (originalJoinType.equals(joinType)
|| originalJoinType.isOneSideOuterJoin() && joinType.isOneSideOuterJoin()
|| originalJoinType.isSemiJoin() && joinType.isSemiJoin()
|| originalJoinType.isAntiJoin() && joinType.isAntiJoin()) {
continue;
}
return false;
}
return true;
}
public List<JoinConstraint> getJoinConstraintList() {
return joinConstraintList;
}
@ -203,6 +232,31 @@ public class LeadingHint extends Hint {
this.innerJoinBitmap = innerJoinBitmap;
}
public Long getTotalBitmap() {
return totalBitmap;
}
/**
* set total bitmap used in leading before we get into leading join
*/
public void setTotalBitmap() {
Long totalBitmap = 0L;
if (hasSameName()) {
this.setStatus(HintStatus.SYNTAX_ERROR);
this.setErrorMessage("duplicated table");
}
for (int index = 0; index < getTablelist().size(); index++) {
RelationId id = findRelationIdAndTableName(getTablelist().get(index));
if (id == null) {
this.setStatus(HintStatus.SYNTAX_ERROR);
this.setErrorMessage("can not find table: " + getTablelist().get(index));
return;
}
totalBitmap = LongBitmap.set(totalBitmap, id.asInt());
}
this.totalBitmap = totalBitmap;
}
/**
* try to get join constraint, if can not get, it means join is inner join,
* @param joinTableBitmap table bitmap below this join
@ -218,6 +272,22 @@ public class LeadingHint extends Hint {
JoinConstraint matchedJoinConstraint = null;
for (JoinConstraint joinConstraint : joinConstraintList) {
if (joinConstraint.getJoinType().isFullOuterJoin()) {
if (leftTableBitmap.equals(joinConstraint.getLeftHand())
&& rightTableBitmap.equals(joinConstraint.getRightHand())
|| rightTableBitmap.equals(joinConstraint.getLeftHand())
&& leftTableBitmap.equals(joinConstraint.getRightHand())) {
if (matchedJoinConstraint != null) {
return Pair.of(null, false);
}
matchedJoinConstraint = joinConstraint;
reversed = false;
break;
} else {
continue;
}
}
if (!LongBitmap.isOverlap(joinConstraint.getMinRightHand(), joinTableBitmap)) {
continue;
}
@ -337,7 +407,6 @@ public class LeadingHint extends Hint {
* @return plan
*/
public Plan generateLeadingJoinPlan() {
this.setStatus(HintStatus.SUCCESS);
Stack<Pair<Integer, LogicalPlan>> stack = new Stack<>();
int index = 0;
LogicalPlan logicalPlan = getLogicalPlanByName(getTablelist().get(index));
@ -365,8 +434,16 @@ public class LeadingHint extends Hint {
getFilters(), newStackTop.second, logicalPlan);
Pair<List<Expression>, List<Expression>> pair = JoinUtils.extractExpressionForHashTable(
newStackTop.second.getOutput(), logicalPlan.getOutput(), conditions);
// leading hint would set status inside if not success
JoinType joinType = computeJoinType(getBitmap(newStackTop.second),
getBitmap(logicalPlan), conditions);
if (joinType == null) {
this.setStatus(HintStatus.SYNTAX_ERROR);
this.setErrorMessage("JoinType can not be null");
} else if (!isConditionJoinTypeMatched(conditions, joinType)) {
this.setStatus(HintStatus.UNUSED);
this.setErrorMessage("condition does not matched joinType");
}
if (!this.isSuccess()) {
return null;
}
@ -379,7 +456,13 @@ public class LeadingHint extends Hint {
logicalPlan);
logicalJoin.setBitmap(LongBitmap.or(getBitmap(newStackTop.second), getBitmap(logicalPlan)));
if (stackTopLevel > 0) {
stackTopLevel--;
if (index < getTablelist().size()) {
if (stackTopLevel > getLevellist().get(index)) {
stackTopLevel--;
}
} else {
stackTopLevel--;
}
}
if (!stack.isEmpty()) {
newStackTop = stack.peek();
@ -401,17 +484,7 @@ public class LeadingHint extends Hint {
LogicalJoin finalJoin = (LogicalJoin) stack.pop().second;
// we want all filters been remove
if (!getFilters().isEmpty()) {
List<Expression> conditions = getLastConditions(getFilters());
Pair<List<Expression>, List<Expression>> pair = JoinUtils.extractExpressionForHashTable(
finalJoin.left().getOutput(), finalJoin.right().getOutput(), conditions);
finalJoin = new LogicalJoin<>(finalJoin.getJoinType(), pair.first,
pair.second,
JoinHint.NONE,
Optional.empty(),
finalJoin.left(),
finalJoin.right());
}
assert (filters.isEmpty());
if (finalJoin != null) {
this.setStatus(HintStatus.SUCCESS);
}
@ -468,6 +541,8 @@ public class LeadingHint extends Hint {
return getBitmap((LogicalPlan) root.child(0));
} else if (root instanceof LogicalProject) {
return getBitmap((LogicalPlan) root.child(0));
} else if (root instanceof LogicalSubQueryAlias) {
return LongBitmap.set(0L, (((LogicalSubQueryAlias) root).getRelationId().asInt()));
} else {
return null;
}
@ -484,11 +559,6 @@ public class LeadingHint extends Hint {
this.setErrorMessage("duplicated table");
return totalBitmap;
}
if (tables != null && getTablelist().size() != tables.size()) {
this.setStatus(HintStatus.SYNTAX_ERROR);
this.setErrorMessage("tables should be same as join tables");
return totalBitmap;
}
for (int index = 0; index < getTablelist().size(); index++) {
RelationId id = findRelationIdAndTableName(getTablelist().get(index));
if (id == null) {

35
fe/fe-core/src/main/java/org/apache/doris/nereids/hint/OrderedHint.java Normal file
View File

@ -0,0 +1,35 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.hint;
/**
* ordered hint.
*/
public class OrderedHint extends Hint {
public OrderedHint(String hintName) {
super(hintName);
}
@Override
public String getExplainString() {
StringBuilder out = new StringBuilder();
out.append("ORDERED");
return out.toString();
}
}

17
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
View File

@ -19,8 +19,6 @@ package org.apache.doris.nereids.jobs.executor;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.jobs.rewrite.RewriteJob;
import org.apache.doris.nereids.processor.pre.EliminateLogicalSelectHint;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.analysis.AdjustAggregateNullableForEmptySet;
import org.apache.doris.nereids.rules.analysis.AnalyzeCTE;
import org.apache.doris.nereids.rules.analysis.BindExpression;
@ -30,8 +28,12 @@ import org.apache.doris.nereids.rules.analysis.BindSink;
import org.apache.doris.nereids.rules.analysis.CheckAfterBind;
import org.apache.doris.nereids.rules.analysis.CheckAnalysis;
import org.apache.doris.nereids.rules.analysis.CheckPolicy;
import org.apache.doris.nereids.rules.analysis.CollectJoinConstraint;
import org.apache.doris.nereids.rules.analysis.CollectSubQueryAlias;
import org.apache.doris.nereids.rules.analysis.EliminateGroupByConstant;
import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint;
import org.apache.doris.nereids.rules.analysis.FillUpMissingSlots;
import org.apache.doris.nereids.rules.analysis.LeadingJoin;
import org.apache.doris.nereids.rules.analysis.NormalizeAggregate;
import org.apache.doris.nereids.rules.analysis.NormalizeRepeat;
import org.apache.doris.nereids.rules.analysis.OneRowRelationExtractAggregate;
@ -41,6 +43,7 @@ import org.apache.doris.nereids.rules.analysis.ReplaceExpressionByChildOutput;
import org.apache.doris.nereids.rules.analysis.ResolveOrdinalInOrderByAndGroupBy;
import org.apache.doris.nereids.rules.analysis.SubqueryToApply;
import org.apache.doris.nereids.rules.analysis.UserAuthentication;
import org.apache.doris.nereids.rules.rewrite.JoinCommute;
import java.util.List;
import java.util.Objects;
@ -84,9 +87,9 @@ public class Analyzer extends AbstractBatchJobExecutor {
private static List<RewriteJob> buildAnalyzeJobs(Optional<CustomTableResolver> customTableResolver) {
return jobs(
// we should eliminate hint after "Subquery unnesting" because some hint maybe exist in the CTE or subquery.
custom(RuleType.ELIMINATE_HINT, EliminateLogicalSelectHint::new),
// we should eliminate hint before "Subquery unnesting".
topDown(new AnalyzeCTE()),
topDown(new EliminateLogicalSelectHint()),
bottomUp(
new BindRelation(customTableResolver),
new CheckPolicy(),
@ -121,6 +124,12 @@ public class Analyzer extends AbstractBatchJobExecutor {
bottomUp(new CheckAnalysis()),
topDown(new EliminateGroupByConstant()),
topDown(new NormalizeAggregate()),
bottomUp(new JoinCommute()),
bottomUp(
new CollectSubQueryAlias(),
new CollectJoinConstraint()
),
topDown(new LeadingJoin()),
bottomUp(new SubqueryToApply())
);
}

15
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
View File

@ -43,7 +43,6 @@ import org.apache.doris.nereids.rules.rewrite.CheckDataTypes;
import org.apache.doris.nereids.rules.rewrite.CheckMatchExpression;
import org.apache.doris.nereids.rules.rewrite.CheckMultiDistinct;
import org.apache.doris.nereids.rules.rewrite.CollectFilterAboveConsumer;
import org.apache.doris.nereids.rules.rewrite.CollectJoinConstraint;
import org.apache.doris.nereids.rules.rewrite.CollectProjectAboveConsumer;
import org.apache.doris.nereids.rules.rewrite.ColumnPruning;
import org.apache.doris.nereids.rules.rewrite.ConvertInnerOrCrossJoin;
@ -74,7 +73,7 @@ import org.apache.doris.nereids.rules.rewrite.InferFilterNotNull;
import org.apache.doris.nereids.rules.rewrite.InferJoinNotNull;
import org.apache.doris.nereids.rules.rewrite.InferPredicates;
import org.apache.doris.nereids.rules.rewrite.InferSetOperatorDistinct;
import org.apache.doris.nereids.rules.rewrite.LeadingJoin;
import org.apache.doris.nereids.rules.rewrite.JoinCommute;
import org.apache.doris.nereids.rules.rewrite.LimitSortToTopN;
import org.apache.doris.nereids.rules.rewrite.MergeFilters;
import org.apache.doris.nereids.rules.rewrite.MergeOneRowRelationIntoUnion;
@ -106,7 +105,6 @@ import org.apache.doris.nereids.rules.rewrite.PushProjectIntoOneRowRelation;
import org.apache.doris.nereids.rules.rewrite.PushProjectThroughUnion;
import org.apache.doris.nereids.rules.rewrite.ReorderJoin;
import org.apache.doris.nereids.rules.rewrite.RewriteCteChildren;
import org.apache.doris.nereids.rules.rewrite.SemiJoinCommute;
import org.apache.doris.nereids.rules.rewrite.SimplifyAggGroupBy;
import org.apache.doris.nereids.rules.rewrite.SplitLimit;
import org.apache.doris.nereids.rules.rewrite.TransposeSemiJoinAgg;
@ -226,7 +224,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
),
// push down SEMI Join
bottomUp(
new SemiJoinCommute(),
new JoinCommute(),
new TransposeSemiJoinLogicalJoin(),
new TransposeSemiJoinLogicalJoinProject(),
new TransposeSemiJoinAgg(),
@ -240,15 +238,6 @@ public class Rewriter extends AbstractBatchJobExecutor {
bottomUp(new EliminateNotNull()),
topDown(new ConvertInnerOrCrossJoin())
),
topic("LEADING JOIN",
bottomUp(
new CollectJoinConstraint()
),
custom(RuleType.LEADING_JOIN, LeadingJoin::new),
bottomUp(
new ExpressionRewrite(CheckLegalityAfterRewrite.INSTANCE)
)
),
topic("Column pruning and infer predicate",
custom(RuleType.COLUMN_PRUNING, ColumnPruning::new),
custom(RuleType.INFER_PREDICATES, InferPredicates::new),

8
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/bitmap/LongBitmap.java
View File

@ -150,9 +150,17 @@ public class LongBitmap {
return Long.numberOfTrailingZeros(bitmap);
}
/**
* use to calculate table bitmap
* @param relationIdSet relationIds
* @return bitmap
*/
public static Long computeTableBitmap(Set<RelationId> relationIdSet) {
Long totalBitmap = 0L;
for (RelationId id : relationIdSet) {
if (id == null) {
continue;
}
totalBitmap = LongBitmap.set(totalBitmap, (id.asInt()));
}
return totalBitmap;

4
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View File

@ -193,6 +193,7 @@ import org.apache.doris.nereids.exceptions.ParseException;
import org.apache.doris.nereids.properties.OrderKey;
import org.apache.doris.nereids.properties.SelectHint;
import org.apache.doris.nereids.properties.SelectHintLeading;
import org.apache.doris.nereids.properties.SelectHintOrdered;
import org.apache.doris.nereids.properties.SelectHintSetVar;
import org.apache.doris.nereids.trees.TableSample;
import org.apache.doris.nereids.trees.expressions.Add;
@ -2556,6 +2557,9 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
}
hints.put(hintName, new SelectHintLeading(hintName, leadingParameters));
break;
case "ordered":
hints.put(hintName, new SelectHintOrdered(hintName));
break;
default:
break;
}

1
fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PlanPreprocessors.java
View File

@ -46,7 +46,6 @@ public class PlanPreprocessors {
public List<PlanPreprocessor> getProcessors() {
// add processor if we need
return ImmutableList.of(
new EliminateLogicalSelectHint(),
new TurnOffPipelineForDml()
);
}

32
fe/fe-core/src/main/java/org/apache/doris/nereids/properties/SelectHintOrdered.java Normal file
View File

@ -0,0 +1,32 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.properties;
/**
* select hint.
*/
public class SelectHintOrdered extends SelectHint {
public SelectHintOrdered(String hintName) {
super(hintName);
}
@Override
public String toString() {
return super.getHintName();
}
}

3
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View File

@ -108,7 +108,10 @@ public enum RuleType {
SIMPLIFY_AGG_GROUP_BY(RuleTypeClass.REWRITE),
DISTINCT_AGGREGATE_DISASSEMBLE(RuleTypeClass.REWRITE),
LOGICAL_SUB_QUERY_ALIAS_TO_LOGICAL_PROJECT(RuleTypeClass.REWRITE),
COLLECT_SUB_QUERY_ALIAS(RuleTypeClass.REWRITE),
ELIMINATE_GROUP_BY_CONSTANT(RuleTypeClass.REWRITE),
ELIMINATE_LOGICAL_SELECT_HINT(RuleTypeClass.REWRITE),
ELIMINATE_ORDER_BY_CONSTANT(RuleTypeClass.REWRITE),
ELIMINATE_HINT(RuleTypeClass.REWRITE),
ELIMINATE_JOIN_ON_EMPTYRELATION(RuleTypeClass.REWRITE),

9
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
View File

@ -128,9 +128,8 @@ public class BindRelation extends OneAnalysisRuleFactory {
if (analyzedCte.isPresent()) {
LogicalCTEConsumer consumer = new LogicalCTEConsumer(unboundRelation.getRelationId(),
cteContext.getCteId(), tableName, analyzedCte.get());
if (cascadesContext.getStatementContext().isLeadingJoin()) {
LeadingHint leading = (LeadingHint) cascadesContext.getStatementContext()
.getHintMap().get("Leading");
if (cascadesContext.isLeadingJoin()) {
LeadingHint leading = (LeadingHint) cascadesContext.getHintMap().get("Leading");
leading.putRelationIdAndTableName(Pair.of(consumer.getRelationId(), tableName));
leading.getRelationIdToScanMap().put(consumer.getRelationId(), consumer);
}
@ -156,8 +155,8 @@ public class BindRelation extends OneAnalysisRuleFactory {
// TODO: should generate different Scan sub class according to table's type
LogicalPlan scan = getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
if (cascadesContext.getStatementContext().isLeadingJoin()) {
LeadingHint leading = (LeadingHint) cascadesContext.getStatementContext().getHintMap().get("Leading");
if (cascadesContext.isLeadingJoin()) {
LeadingHint leading = (LeadingHint) cascadesContext.getHintMap().get("Leading");
leading.putRelationIdAndTableName(Pair.of(unboundRelation.getRelationId(), tableName));
leading.getRelationIdToScanMap().put(unboundRelation.getRelationId(), scan);
}

56
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectJoinConstraint.java → fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite;
package org.apache.doris.nereids.rules.analysis;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.hint.Hint;
@ -24,11 +24,11 @@ import org.apache.doris.nereids.hint.LeadingHint;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
@ -47,24 +47,17 @@ public class CollectJoinConstraint implements RewriteRuleFactory {
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
logicalRelation().thenApply(ctx -> {
LeadingHint leading = (LeadingHint) ctx.cascadesContext
.getStatementContext().getHintMap().get("Leading");
if (leading == null) {
return ctx.root;
} else if (leading.isSyntaxError()) {
return ctx.root;
}
return ctx.root;
}).toRule(RuleType.COLLECT_JOIN_CONSTRAINT),
logicalJoin().thenApply(ctx -> {
LeadingHint leading = (LeadingHint) ctx.cascadesContext
.getStatementContext().getHintMap().get("Leading");
if (leading == null) {
if (!ctx.cascadesContext.isLeadingJoin()) {
return ctx.root;
}
LeadingHint leading = (LeadingHint) ctx.cascadesContext
.getHintMap().get("Leading");
LogicalJoin join = ctx.root;
if (join.getJoinType().isNullAwareLeftAntiJoin()) {
leading.setStatus(Hint.HintStatus.UNUSED);
leading.setErrorMessage("condition does not matched joinType");
}
List<Expression> expressions = join.getHashJoinConjuncts();
Long totalFilterBitMap = 0L;
Long nonNullableSlotBitMap = 0L;
@ -74,6 +67,7 @@ public class CollectJoinConstraint implements RewriteRuleFactory {
Long filterBitMap = calSlotsTableBitMap(leading, expression.getInputSlots(), false);
totalFilterBitMap = LongBitmap.or(totalFilterBitMap, filterBitMap);
leading.getFilters().add(Pair.of(filterBitMap, expression));
leading.putConditionJoinType(expression, join.getJoinType());
}
expressions = join.getOtherJoinConjuncts();
for (Expression expression : expressions) {
@ -82,6 +76,7 @@ public class CollectJoinConstraint implements RewriteRuleFactory {
Long filterBitMap = calSlotsTableBitMap(leading, expression.getInputSlots(), false);
totalFilterBitMap = LongBitmap.or(totalFilterBitMap, filterBitMap);
leading.getFilters().add(Pair.of(filterBitMap, expression));
leading.putConditionJoinType(expression, join.getJoinType());
}
Long leftHand = LongBitmap.computeTableBitmap(join.left().getInputRelations());
Long rightHand = LongBitmap.computeTableBitmap(join.right().getInputRelations());
@ -91,28 +86,13 @@ public class CollectJoinConstraint implements RewriteRuleFactory {
return ctx.root;
}).toRule(RuleType.COLLECT_JOIN_CONSTRAINT),
logicalFilter().thenApply(ctx -> {
LeadingHint leading = (LeadingHint) ctx.cascadesContext
.getStatementContext().getHintMap().get("Leading");
if (leading == null) {
return ctx.root;
}
LogicalFilter filter = ctx.root;
Set<Expression> expressions = filter.getConjuncts();
for (Expression expression : expressions) {
Long filterBitMap = calSlotsTableBitMap(leading, expression.getInputSlots(), false);
leading.getFilters().add(Pair.of(filterBitMap, expression));
}
return ctx.root;
}).toRule(RuleType.COLLECT_JOIN_CONSTRAINT),
logicalProject(logicalOlapScan()).thenApply(
ctx -> {
LeadingHint leading = (LeadingHint) ctx.cascadesContext
.getStatementContext().getHintMap().get("Leading");
if (leading == null) {
if (!ctx.cascadesContext.isLeadingJoin()) {
return ctx.root;
}
LeadingHint leading = (LeadingHint) ctx.cascadesContext
.getHintMap().get("Leading");
LogicalProject<LogicalOlapScan> project = ctx.root;
LogicalOlapScan scan = project.child();
leading.getRelationIdToScanMap().put(scan.getRelationId(), project);
@ -195,15 +175,11 @@ public class CollectJoinConstraint implements RewriteRuleFactory {
if (getNotNullable && slot.nullable()) {
continue;
}
if (!slot.isColumnFromTable()) {
if (!slot.isColumnFromTable() && (slot.getQualifier() == null || slot.getQualifier().isEmpty())) {
// we can not get info from column not from table
continue;
}
String tableName = leading.getExprIdToTableNameMap().get(slot.getExprId());
if (tableName == null) {
tableName = slot.getQualifier().get(slot.getQualifier().size() - 1);
leading.getExprIdToTableNameMap().put(slot.getExprId(), tableName);
}
String tableName = slot.getQualifier().get(slot.getQualifier().size() - 1);
RelationId id = leading.findRelationIdAndTableName(tableName);
if (id == null) {
leading.setStatus(Hint.HintStatus.SYNTAX_ERROR);

69
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectSubQueryAlias.java Normal file
View File

@ -0,0 +1,69 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.analysis;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.hint.LeadingHint;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import com.google.common.collect.ImmutableList;
import java.util.List;
/**
* Eliminate the logical sub query and alias node after analyze and before rewrite
* If we match the alias node and return its child node, in the execute() of the job
* <p>
* TODO: refactor group merge strategy to support the feature above
*/
public class CollectSubQueryAlias implements RewriteRuleFactory {
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
logicalSubQueryAlias().thenApply(ctx -> {
if (ctx.cascadesContext.isLeadingJoin()) {
String aliasName = ctx.root.getAlias();
LeadingHint leading = (LeadingHint) ctx.cascadesContext.getHintMap().get("Leading");
RelationId newId = ctx.statementContext.getNextRelationId();
leading.putRelationIdAndTableName(Pair.of(newId, aliasName));
leading.getRelationIdToScanMap().put(newId, ctx.root);
ctx.root.setRelationId(newId);
}
return ctx.root;
}).toRule(RuleType.COLLECT_JOIN_CONSTRAINT),
logicalRelation().thenApply(ctx -> {
if (ctx.cascadesContext.isLeadingJoin()) {
LeadingHint leading = (LeadingHint) ctx.cascadesContext.getHintMap().get("Leading");
LogicalRelation relation = (LogicalRelation) ctx.root;
RelationId relationId = relation.getRelationId();
if (ctx.root instanceof LogicalCatalogRelation) {
String relationName = ((LogicalCatalogRelation) ctx.root).getTable().getName();
leading.putRelationIdAndTableName(Pair.of(relationId, relationName));
}
leading.getRelationIdToScanMap().put(relationId, ctx.root);
}
return ctx.root;
}).toRule(RuleType.COLLECT_JOIN_CONSTRAINT)
);
}
}

84
fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/EliminateLogicalSelectHint.java → fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java
View File

@ -15,22 +15,25 @@
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.processor.pre;
package org.apache.doris.nereids.rules.analysis;
import org.apache.doris.analysis.SetVar;
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.common.DdlException;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.hint.Hint;
import org.apache.doris.nereids.hint.LeadingHint;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.hint.OrderedHint;
import org.apache.doris.nereids.properties.SelectHint;
import org.apache.doris.nereids.properties.SelectHintLeading;
import org.apache.doris.nereids.properties.SelectHintSetVar;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSelectHint;
import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.VariableMgr;
@ -38,38 +41,44 @@ import org.apache.doris.qe.VariableMgr;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
/**
* eliminate set var hint, and set var to session variables.
* eliminate logical select hint and set them to cascade context
*/
public class EliminateLogicalSelectHint extends PlanPreprocessor implements CustomRewriter {
public class EliminateLogicalSelectHint extends OneRewriteRuleFactory {
private Logger logger = LoggerFactory.getLogger(getClass());
@Override
public Plan rewriteRoot(Plan plan, JobContext jobContext) {
return plan.accept(this, jobContext.getCascadesContext().getStatementContext());
}
@Override
public LogicalPlan visitLogicalSelectHint(
LogicalSelectHint<? extends Plan> selectHintPlan,
StatementContext context) {
for (Entry<String, SelectHint> hint : selectHintPlan.getHints().entrySet()) {
String hintName = hint.getKey();
if (hintName.equalsIgnoreCase("SET_VAR")) {
setVar((SelectHintSetVar) hint.getValue(), context);
} else if (hintName.equalsIgnoreCase("ORDERED")) {
ConnectContext.get().getSessionVariable().setDisableJoinReorder(true);
} else if (hintName.equalsIgnoreCase("LEADING")) {
extractLeading((SelectHintLeading) hint.getValue(), context);
} else {
logger.warn("Can not process select hint '{}' and skip it", hint.getKey());
public Rule build() {
return logicalSelectHint().thenApply(ctx -> {
LogicalSelectHint<Plan> selectHintPlan = ctx.root;
for (Entry<String, SelectHint> hint : selectHintPlan.getHints().entrySet()) {
String hintName = hint.getKey();
if (hintName.equalsIgnoreCase("SET_VAR")) {
setVar((SelectHintSetVar) hint.getValue(), ctx.statementContext);
} else if (hintName.equalsIgnoreCase("ORDERED")) {
try {
ctx.cascadesContext.getConnectContext().getSessionVariable()
.disableNereidsJoinReorderOnce();
} catch (DdlException e) {
throw new RuntimeException(e);
}
OrderedHint ordered = new OrderedHint("Ordered");
ordered.setStatus(Hint.HintStatus.SUCCESS);
ctx.cascadesContext.getHintMap().put("Ordered", ordered);
ctx.statementContext.addHint(ordered);
} else if (hintName.equalsIgnoreCase("LEADING")) {
extractLeading((SelectHintLeading) hint.getValue(), ctx.cascadesContext,
ctx.statementContext, selectHintPlan.getHints());
} else {
logger.warn("Can not process select hint '{}' and skip it", hint.getKey());
}
}
}
return (LogicalPlan) selectHintPlan.child();
return selectHintPlan.child();
}).toRule(RuleType.ELIMINATE_LOGICAL_SELECT_HINT);
}
private void setVar(SelectHintSetVar selectHint, StatementContext context) {
@ -84,7 +93,7 @@ public class EliminateLogicalSelectHint extends PlanPreprocessor implements Cust
VariableMgr.setVar(sessionVariable, new SetVar(key, new StringLiteral(value.get())));
} catch (Throwable t) {
throw new AnalysisException("Can not set session variable '"
+ key + "' = '" + value.get() + "'", t);
+ key + "' = '" + value.get() + "'", t);
}
}
}
@ -101,15 +110,28 @@ public class EliminateLogicalSelectHint extends PlanPreprocessor implements Cust
}
}
private void extractLeading(SelectHintLeading selectHint, StatementContext context) {
private void extractLeading(SelectHintLeading selectHint, CascadesContext context,
StatementContext statementContext, Map<String, SelectHint> hints) {
LeadingHint hint = new LeadingHint("Leading", selectHint.getParameters(), selectHint.toString());
if (context.getHintMap().get("Leading") != null) {
hint.setStatus(Hint.HintStatus.SYNTAX_ERROR);
hint.setErrorMessage("can only have one leading clause");
context.getHintMap().get("Leading").setStatus(Hint.HintStatus.UNUSED);
hint.setErrorMessage("one query block can only have one leading clause");
statementContext.addHint(hint);
context.setLeadingJoin(false);
return;
}
hint.setStatus(Hint.HintStatus.SUCCESS);
statementContext.addHint(hint);
context.getHintMap().put("Leading", hint);
context.setLeadingJoin(true);
if (hints.get("ordered") != null || ConnectContext.get().getSessionVariable().isDisableJoinReorder()) {
context.setLeadingJoin(false);
hint.setStatus(Hint.HintStatus.UNUSED);
} else {
context.setLeadingJoin(true);
}
assert (selectHint != null);
assert (context != null);
}
}

69
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/LeadingJoin.java Normal file
View File

@ -0,0 +1,69 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.analysis;
import org.apache.doris.common.DdlException;
import org.apache.doris.nereids.hint.Hint;
import org.apache.doris.nereids.hint.LeadingHint;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
import java.util.List;
/**
* Leading join is used to generate leading join and replace original logical join
*/
public class LeadingJoin implements RewriteRuleFactory {
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
logicalJoin()
.whenNot(join -> ConnectContext.get().getSessionVariable().isDisableJoinReorder())
.thenApply(ctx -> {
if (!ctx.cascadesContext.isLeadingJoin()) {
return ctx.root;
}
Hint leadingHint = ctx.cascadesContext.getHintMap().get("Leading");
((LeadingHint) leadingHint).setTotalBitmap();
Long currentBitMap = LongBitmap.computeTableBitmap(ctx.root.getInputRelations());
if (((LeadingHint) leadingHint).getTotalBitmap().equals(currentBitMap)
&& leadingHint.isSuccess()) {
Plan leadingJoin = ((LeadingHint) leadingHint).generateLeadingJoinPlan();
if (leadingHint.isSuccess() && leadingJoin != null) {
try {
ctx.cascadesContext.getConnectContext().getSessionVariable()
.disableNereidsJoinReorderOnce();
ctx.cascadesContext.setLeadingJoin(false);
} catch (DdlException e) {
throw new RuntimeException(e);
}
return leadingJoin;
}
}
return ctx.root;
}).toRule(RuleType.LEADING_JOIN)
);
}
}

23
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/LogicalSubQueryAliasToLogicalProject.java
View File

@ -17,15 +17,10 @@
package org.apache.doris.nereids.rules.analysis;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.hint.Hint;
import org.apache.doris.nereids.hint.LeadingHint;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import com.google.common.collect.ImmutableList;
@ -42,24 +37,6 @@ public class LogicalSubQueryAliasToLogicalProject extends OneRewriteRuleFactory
logicalSubQueryAlias().thenApply(ctx -> {
LogicalProject project = new LogicalProject<>(
ImmutableList.copyOf(ctx.root.getOutput()), ctx.root.child());
if (ctx.cascadesContext.getStatementContext().isLeadingJoin()) {
String aliasName = ctx.root.getAlias();
LeadingHint leading = (LeadingHint) ctx.cascadesContext.getStatementContext()
.getHintMap().get("Leading");
if (!(project.child() instanceof LogicalRelation)) {
if (leading.getTablelist().contains(aliasName)) {
leading.setStatus(Hint.HintStatus.SYNTAX_ERROR);
leading.setErrorMessage("Leading alias can only be table name alias");
}
} else {
RelationId id = leading.findRelationIdAndTableName(aliasName);
if (id == null) {
id = ((LogicalRelation) project.child()).getRelationId();
}
leading.putRelationIdAndTableName(Pair.of(id, aliasName));
leading.getRelationIdToScanMap().put(id, project);
}
}
return project;
})
);

4
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SemiJoinCommute.java → fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/JoinCommute.java
View File

@ -24,8 +24,10 @@ import org.apache.doris.qe.ConnectContext;
/**
* RightSemiJoin -> LeftSemiJoin
* RightAntiJoin -> LeftAntiJoin
* RightOuterJoin -> LeftOuterJoin
*/
public class SemiJoinCommute extends OneRewriteRuleFactory {
public class JoinCommute extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalJoin()

84
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/LeadingJoin.java
View File

@ -1,84 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.common.DdlException;
import org.apache.doris.nereids.hint.Hint;
import org.apache.doris.nereids.hint.LeadingHint;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
import org.apache.doris.nereids.rules.rewrite.LeadingJoin.LeadingContext;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
/**
* Leading join is used to generate leading join and replace original logical join
*/
public class LeadingJoin extends DefaultPlanRewriter<LeadingContext> implements CustomRewriter {
@Override
public Plan rewriteRoot(Plan plan, JobContext jobContext) {
if (jobContext.getCascadesContext().getStatementContext().isLeadingJoin()) {
Hint leadingHint = jobContext.getCascadesContext().getStatementContext().getHintMap().get("Leading");
Plan leadingPlan = plan.accept(this, new LeadingContext(
(LeadingHint) leadingHint, ((LeadingHint) leadingHint)
.getLeadingTableBitmap(jobContext.getCascadesContext().getTables())));
if (leadingHint.isSuccess()) {
try {
jobContext.getCascadesContext().getConnectContext().getSessionVariable()
.disableNereidsJoinReorderOnce();
} catch (DdlException e) {
throw new RuntimeException(e);
}
} else {
return plan;
}
return leadingPlan;
}
return plan;
}
@Override
public Plan visit(Plan plan, LeadingContext context) {
Long currentBitMap = LongBitmap.computeTableBitmap(plan.getInputRelations());
if (LongBitmap.isSubset(currentBitMap, context.totalBitmap)
&& plan instanceof LogicalJoin && !context.leading.isSyntaxError()) {
Plan leadingJoin = context.leading.generateLeadingJoinPlan();
if (context.leading.isSuccess() && leadingJoin != null) {
return leadingJoin;
}
} else {
return (LogicalPlan) super.visit(plan, context);
}
return plan;
}
/** LeadingContext */
public static class LeadingContext {
public LeadingHint leading;
public Long totalBitmap;
public LeadingContext(LeadingHint leading, Long totalBitmap) {
this.leading = leading;
this.totalBitmap = totalBitmap;
}
}
}

19
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
View File

@ -17,10 +17,7 @@
package org.apache.doris.nereids.trees.copier;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.hint.Hint;
import org.apache.doris.nereids.hint.LeadingHint;
import org.apache.doris.nereids.properties.OrderKey;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
@ -33,7 +30,6 @@ import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.nereids.trees.expressions.SubqueryExpr;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalApply;
import org.apache.doris.nereids.trees.plans.logical.LogicalAssertNumRows;
@ -67,7 +63,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalTopN;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
@ -173,7 +168,6 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter<DeepCopierContext
olapScan.getManuallySpecifiedPartitions(), olapScan.getSelectedTabletIds(),
olapScan.getHints(), olapScan.getTableSample());
}
updateLeadingRelationIdMap(newOlapScan.getRelationId(), newOlapScan.getTable().getName(), newOlapScan);
newOlapScan.getOutput();
context.putRelation(olapScan.getRelationId(), newOlapScan);
updateReplaceMapWithOutput(olapScan, newOlapScan, context.exprIdReplaceMap);
@ -200,7 +194,6 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter<DeepCopierContext
}
LogicalSchemaScan newSchemaScan = new LogicalSchemaScan(StatementScopeIdGenerator.newRelationId(),
schemaScan.getTable(), schemaScan.getQualifier());
updateLeadingRelationIdMap(newSchemaScan.getRelationId(), newSchemaScan.getTable().getName(), newSchemaScan);
updateReplaceMapWithOutput(schemaScan, newSchemaScan, context.exprIdReplaceMap);
context.putRelation(schemaScan.getRelationId(), newSchemaScan);
return newSchemaScan;
@ -213,7 +206,6 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter<DeepCopierContext
}
LogicalFileScan newFileScan = new LogicalFileScan(StatementScopeIdGenerator.newRelationId(),
fileScan.getTable(), fileScan.getQualifier(), fileScan.getTableSample());
updateLeadingRelationIdMap(newFileScan.getRelationId(), fileScan.getTable().getName(), newFileScan);
updateReplaceMapWithOutput(fileScan, newFileScan, context.exprIdReplaceMap);
context.putRelation(fileScan.getRelationId(), newFileScan);
Set<Expression> conjuncts = fileScan.getConjuncts().stream()
@ -241,7 +233,6 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter<DeepCopierContext
}
LogicalJdbcScan newJdbcScan = new LogicalJdbcScan(StatementScopeIdGenerator.newRelationId(),
jdbcScan.getTable(), jdbcScan.getQualifier());
updateLeadingRelationIdMap(newJdbcScan.getRelationId(), jdbcScan.getTable().getName(), newJdbcScan);
updateReplaceMapWithOutput(jdbcScan, newJdbcScan, context.exprIdReplaceMap);
context.putRelation(jdbcScan.getRelationId(), newJdbcScan);
return newJdbcScan;
@ -255,7 +246,6 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter<DeepCopierContext
LogicalEsScan newEsScan = new LogicalEsScan(StatementScopeIdGenerator.newRelationId(),
esScan.getTable(), esScan.getQualifier());
updateReplaceMapWithOutput(esScan, newEsScan, context.exprIdReplaceMap);
updateLeadingRelationIdMap(newEsScan.getRelationId(), esScan.getTable().getName(), newEsScan);
context.putRelation(esScan.getRelationId(), newEsScan);
return newEsScan;
}
@ -455,7 +445,6 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter<DeepCopierContext
StatementScopeIdGenerator.newRelationId(),
cteConsumer.getCteId(), cteConsumer.getName(),
consumerToProducerOutputMap, producerToConsumerOutputMap);
updateLeadingRelationIdMap(newCTEConsumer.getRelationId(), cteConsumer.getName(), newCTEConsumer);
context.putRelation(cteConsumer.getRelationId(), newCTEConsumer);
return newCTEConsumer;
}
@ -474,12 +463,4 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter<DeepCopierContext
}
}
private void updateLeadingRelationIdMap(RelationId id, String tableName, LogicalPlan plan) {
if (!ConnectContext.get().getStatementContext().isLeadingJoin()) {
return;
}
Hint leading = ConnectContext.get().getStatementContext().getHintMap().get("Leading");
((LeadingHint) leading).updateRelationIdByTableName(Pair.of(id, tableName));
((LeadingHint) leading).getRelationIdToScanMap().put(id, plan);
}
}

8
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java
View File

@ -185,10 +185,18 @@ public enum JoinType {
return this == LEFT_SEMI_JOIN || this == RIGHT_SEMI_JOIN;
}
public final boolean isAntiJoin() {
return this == LEFT_ANTI_JOIN || this == RIGHT_ANTI_JOIN;
}
public final boolean isOuterJoin() {
return this == LEFT_OUTER_JOIN || this == RIGHT_OUTER_JOIN || this == FULL_OUTER_JOIN;
}
public final boolean isOneSideOuterJoin() {
return this == LEFT_OUTER_JOIN || this == RIGHT_OUTER_JOIN;
}
public final boolean isRemainLeftJoin() {
return this != RIGHT_SEMI_JOIN && this != RIGHT_ANTI_JOIN;
}

7
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
View File

@ -64,7 +64,6 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
private final JoinType joinType;
private final List<Expression> otherJoinConjuncts;
private final List<Expression> hashJoinConjuncts;
private final JoinHint hint;
// When the predicate condition contains subqueries and disjunctions, the join will be marked as MarkJoin.
private final Optional<MarkJoinSlotReference> markJoinSlotReference;
@ -74,6 +73,8 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
// Table bitmap for tables below this join
private long bitmap = LongBitmap.newBitmap();
private JoinHint hint;
public LogicalJoin(JoinType joinType, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) {
this(joinType, ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE,
Optional.empty(), Optional.empty(), Optional.empty(), leftChild, rightChild);
@ -216,6 +217,10 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
return hint;
}
public void setHint(JoinHint hint) {
this.hint = hint;
}
public boolean isMarkJoin() {
return markJoinSlotReference.isPresent();
}

19
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java
View File

@ -24,11 +24,13 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
import java.util.HashMap;
@ -36,6 +38,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
/**
@ -45,6 +48,7 @@ import java.util.function.Supplier;
*/
public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYPE> {
protected RelationId relationId;
private final List<String> qualifier;
private final Optional<List<String>> columnAliases;
@ -170,4 +174,19 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<
builder.replace(replaceMap);
return builder.build();
}
public void setRelationId(RelationId relationId) {
this.relationId = relationId;
}
public RelationId getRelationId() {
return relationId;
}
@Override
public Set<RelationId> getInputRelations() {
Set<RelationId> relationIdSet = Sets.newHashSet();
relationIdSet.add(relationId);
return relationIdSet;
}
}