[fix](compile) fe compile failed when generate doc and FE UT failed (#26164)
1. FE could not compile because below error. Intro by PR #25933 ``` [INFO] --- exec:3.1.0:java (doc) @ fe-core --- ... Failed to generate doc for ignoreRuntimeFilterIds ``` 2. fix UT bugs intro by below PRs > - #25951 > - #26031 3. because fe could not compile, FE UT CI do not work well. So, some UT failed be introduced by the PRs merged after PR #25933 merged. So this PR revert them to fix FE UT > - Revert "[Bug](materialized-view) SelectMaterializedIndexWithAggregate do not change plan > when match ba… (#26145)" > This reverts commit 8d7abf60f94d2d1208b71e96b9290ea02122b8d8. > - Revert "[enhancement](Nereids): optimize GroupExpressionMatching (#26130)" > This reverts commit 19122b55cd95af097b4ef7b6eb809f37db29765f. > - Revert "[Performance](Nereids): optimize GroupExpressionMatching (#26084)" > This reverts commit 0d956e90cf920039b8baa79c170a298be56a128d.
This commit is contained in:
@ -298,9 +298,6 @@ public class CreateMaterializedViewStmt extends DdlStmt {
|
||||
if (tableRefList.size() != 1) {
|
||||
throw new AnalysisException("The materialized view only support one table in from clause.");
|
||||
}
|
||||
if (!isReplay && tableRefList.get(0).hasExplicitAlias()) {
|
||||
throw new AnalysisException("The materialized view not support table with alias.");
|
||||
}
|
||||
TableName tableName = tableRefList.get(0).getName();
|
||||
if (tableName == null) {
|
||||
throw new AnalysisException("table in from clause is invalid, please check if it's single table "
|
||||
|
||||
@ -723,12 +723,12 @@ public class Column implements Writable, GsonPostProcessable {
|
||||
// show change datetimeV2/dateV2 to datetime/date
|
||||
if (isCompatible) {
|
||||
if (type.isDatetimeV2()) {
|
||||
sb.append("datetime");
|
||||
sb.append("DATETIME");
|
||||
if (((ScalarType) type).getScalarScale() > 0) {
|
||||
sb.append("(").append(((ScalarType) type).getScalarScale()).append(")");
|
||||
}
|
||||
} else if (type.isDateV2()) {
|
||||
sb.append("date");
|
||||
sb.append("DATE");
|
||||
} else if (type.isDecimalV3()) {
|
||||
sb.append("DECIMAL");
|
||||
ScalarType sType = (ScalarType) type;
|
||||
|
||||
@ -61,7 +61,7 @@ public class ApplyRuleJob extends Job {
|
||||
}
|
||||
|
||||
@Override
|
||||
public final void execute() throws AnalysisException {
|
||||
public void execute() throws AnalysisException {
|
||||
if (groupExpression.hasApplied(rule)
|
||||
|| groupExpression.isUnused()) {
|
||||
return;
|
||||
|
||||
@ -20,6 +20,7 @@ package org.apache.doris.nereids.pattern;
|
||||
import org.apache.doris.nereids.memo.Group;
|
||||
import org.apache.doris.nereids.memo.GroupExpression;
|
||||
import org.apache.doris.nereids.properties.LogicalProperties;
|
||||
import org.apache.doris.nereids.trees.plans.GroupPlan;
|
||||
import org.apache.doris.nereids.trees.plans.Plan;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
@ -54,7 +55,6 @@ public class GroupExpressionMatching implements Iterable<Plan> {
|
||||
public static class GroupExpressionIterator implements Iterator<Plan> {
|
||||
private final List<Plan> results = Lists.newArrayList();
|
||||
private int resultIndex = 0;
|
||||
private int resultsSize;
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
@ -69,19 +69,21 @@ public class GroupExpressionMatching implements Iterable<Plan> {
|
||||
|
||||
int childrenGroupArity = groupExpression.arity();
|
||||
int patternArity = pattern.arity();
|
||||
// (logicalFilter(), multi()) match (logicalFilter()),
|
||||
// but (logicalFilter(), logicalFilter(), multi()) not match (logicalFilter())
|
||||
boolean extraMulti = patternArity == childrenGroupArity + 1
|
||||
&& (pattern.hasMultiChild() || pattern.hasMultiGroupChild());
|
||||
if (patternArity > childrenGroupArity && !extraMulti) {
|
||||
return;
|
||||
}
|
||||
if (!(pattern instanceof SubTreePattern)) {
|
||||
// (logicalFilter(), multi()) match (logicalFilter()),
|
||||
// but (logicalFilter(), logicalFilter(), multi()) not match (logicalFilter())
|
||||
boolean extraMulti = patternArity == childrenGroupArity + 1
|
||||
&& (pattern.hasMultiChild() || pattern.hasMultiGroupChild());
|
||||
if (patternArity > childrenGroupArity && !extraMulti) {
|
||||
return;
|
||||
}
|
||||
|
||||
// (multi()) match (logicalFilter(), logicalFilter()),
|
||||
// but (logicalFilter()) not match (logicalFilter(), logicalFilter())
|
||||
if (!pattern.isAny() && patternArity < childrenGroupArity
|
||||
&& !pattern.hasMultiChild() && !pattern.hasMultiGroupChild()) {
|
||||
return;
|
||||
// (multi()) match (logicalFilter(), logicalFilter()),
|
||||
// but (logicalFilter()) not match (logicalFilter(), logicalFilter())
|
||||
if (!pattern.isAny() && patternArity < childrenGroupArity
|
||||
&& !pattern.hasMultiChild() && !pattern.hasMultiGroupChild()) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Pattern.GROUP / Pattern.MULTI / Pattern.MULTI_GROUP can not match GroupExpression
|
||||
@ -91,7 +93,7 @@ public class GroupExpressionMatching implements Iterable<Plan> {
|
||||
|
||||
// getPlan return the plan with GroupPlan as children
|
||||
Plan root = groupExpression.getPlan();
|
||||
if (patternArity == 0) {
|
||||
if (patternArity == 0 && !(pattern instanceof SubTreePattern)) {
|
||||
if (pattern.matchPredicates(root)) {
|
||||
// if no children pattern, we treat all children as GROUP. e.g. Pattern.ANY.
|
||||
// leaf plan will enter this branch too, e.g. logicalRelation().
|
||||
@ -101,16 +103,20 @@ public class GroupExpressionMatching implements Iterable<Plan> {
|
||||
// matching children group, one List<Plan> per child
|
||||
// first dimension is every child group's plan
|
||||
// second dimension is all matched plan in one group
|
||||
List<Plan>[] childrenPlans = new List[childrenGroupArity];
|
||||
List<List<Plan>> childrenPlans = Lists.newArrayListWithCapacity(childrenGroupArity);
|
||||
for (int i = 0; i < childrenGroupArity; ++i) {
|
||||
Group childGroup = groupExpression.child(i);
|
||||
List<Plan> childrenPlan = matchingChildGroup(pattern, childGroup, i);
|
||||
|
||||
if (childrenPlan.isEmpty()) {
|
||||
// current pattern is match but children patterns not match
|
||||
return;
|
||||
if (pattern instanceof SubTreePattern) {
|
||||
childrenPlan = ImmutableList.of(new GroupPlan(childGroup));
|
||||
} else {
|
||||
// current pattern is match but children patterns not match
|
||||
return;
|
||||
}
|
||||
}
|
||||
childrenPlans[i] = childrenPlan;
|
||||
childrenPlans.add(childrenPlan);
|
||||
}
|
||||
assembleAllCombinationPlanTree(root, pattern, groupExpression, childrenPlans);
|
||||
} else if (patternArity == 1 && (pattern.hasMultiChild() || pattern.hasMultiGroupChild())) {
|
||||
@ -121,22 +127,25 @@ public class GroupExpressionMatching implements Iterable<Plan> {
|
||||
results.add(root);
|
||||
}
|
||||
}
|
||||
this.resultsSize = results.size();
|
||||
}
|
||||
|
||||
private List<Plan> matchingChildGroup(Pattern<? extends Plan> parentPattern,
|
||||
Group childGroup, int childIndex) {
|
||||
Pattern<? extends Plan> childPattern;
|
||||
boolean isLastPattern = childIndex + 1 >= parentPattern.arity();
|
||||
int patternChildIndex = isLastPattern ? parentPattern.arity() - 1 : childIndex;
|
||||
if (parentPattern instanceof SubTreePattern) {
|
||||
childPattern = parentPattern;
|
||||
} else {
|
||||
boolean isLastPattern = childIndex + 1 >= parentPattern.arity();
|
||||
int patternChildIndex = isLastPattern ? parentPattern.arity() - 1 : childIndex;
|
||||
|
||||
childPattern = parentPattern.child(patternChildIndex);
|
||||
// translate MULTI and MULTI_GROUP to ANY and GROUP
|
||||
if (isLastPattern) {
|
||||
if (childPattern.isMulti()) {
|
||||
childPattern = Pattern.ANY;
|
||||
} else if (childPattern.isMultiGroup()) {
|
||||
childPattern = Pattern.GROUP;
|
||||
childPattern = parentPattern.child(patternChildIndex);
|
||||
// translate MULTI and MULTI_GROUP to ANY and GROUP
|
||||
if (isLastPattern) {
|
||||
if (childPattern.isMulti()) {
|
||||
childPattern = Pattern.ANY;
|
||||
} else if (childPattern.isMultiGroup()) {
|
||||
childPattern = Pattern.GROUP;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -145,37 +154,40 @@ public class GroupExpressionMatching implements Iterable<Plan> {
|
||||
}
|
||||
|
||||
private void assembleAllCombinationPlanTree(Plan root, Pattern<Plan> rootPattern,
|
||||
GroupExpression groupExpression, List<Plan>[] childrenPlans) {
|
||||
int childrenPlansSize = childrenPlans.length;
|
||||
int[] childrenPlanIndex = new int[childrenPlansSize];
|
||||
GroupExpression groupExpression,
|
||||
List<List<Plan>> childrenPlans) {
|
||||
int[] childrenPlanIndex = new int[childrenPlans.size()];
|
||||
int offset = 0;
|
||||
LogicalProperties logicalProperties = groupExpression.getOwnerGroup().getLogicalProperties();
|
||||
|
||||
// assemble all combination of plan tree by current root plan and children plan
|
||||
Optional<GroupExpression> groupExprOption = Optional.of(groupExpression);
|
||||
Optional<LogicalProperties> logicalPropOption = Optional.of(logicalProperties);
|
||||
while (offset < childrenPlansSize) {
|
||||
ImmutableList.Builder<Plan> childrenBuilder = ImmutableList.builderWithExpectedSize(childrenPlansSize);
|
||||
for (int i = 0; i < childrenPlansSize; i++) {
|
||||
childrenBuilder.add(childrenPlans[i].get(childrenPlanIndex[i]));
|
||||
while (offset < childrenPlans.size()) {
|
||||
ImmutableList.Builder<Plan> childrenBuilder =
|
||||
ImmutableList.builderWithExpectedSize(childrenPlans.size());
|
||||
for (int i = 0; i < childrenPlans.size(); i++) {
|
||||
childrenBuilder.add(childrenPlans.get(i).get(childrenPlanIndex[i]));
|
||||
}
|
||||
List<Plan> children = childrenBuilder.build();
|
||||
|
||||
// assemble children: replace GroupPlan to real plan,
|
||||
// withChildren will erase groupExpression, so we must
|
||||
// withGroupExpression too.
|
||||
Plan rootWithChildren = root.withGroupExprLogicalPropChildren(groupExprOption,
|
||||
logicalPropOption, children);
|
||||
Plan rootWithChildren = root.withGroupExprLogicalPropChildren(Optional.of(groupExpression),
|
||||
Optional.of(logicalProperties), children);
|
||||
if (rootPattern.matchPredicates(rootWithChildren)) {
|
||||
results.add(rootWithChildren);
|
||||
}
|
||||
for (offset = 0; offset < childrenPlansSize; offset++) {
|
||||
offset = 0;
|
||||
while (true) {
|
||||
childrenPlanIndex[offset]++;
|
||||
if (childrenPlanIndex[offset] == childrenPlans[offset].size()) {
|
||||
// Reset the index when it reaches the size of the current child plan list
|
||||
if (childrenPlanIndex[offset] == childrenPlans.get(offset).size()) {
|
||||
childrenPlanIndex[offset] = 0;
|
||||
offset++;
|
||||
if (offset == childrenPlans.size()) {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
break; // Break the loop when the index is within the size of the current child plan list
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -183,7 +195,7 @@ public class GroupExpressionMatching implements Iterable<Plan> {
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return resultIndex < resultsSize;
|
||||
return resultIndex < results.size();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -45,6 +45,11 @@ public class GroupMatching {
|
||||
matchingPlans.add(plan);
|
||||
}
|
||||
}
|
||||
for (GroupExpression groupExpression : group.getPhysicalExpressions()) {
|
||||
for (Plan plan : new GroupExpressionMatching(pattern, groupExpression)) {
|
||||
matchingPlans.add(plan);
|
||||
}
|
||||
}
|
||||
}
|
||||
return matchingPlans;
|
||||
}
|
||||
|
||||
@ -38,11 +38,13 @@ import com.google.common.collect.Lists;
|
||||
public class EnforceMissingPropertiesHelper {
|
||||
private static final EventProducer ENFORCER_TRACER = new EventProducer(EnforcerEvent.class,
|
||||
EventChannel.getDefaultChannel().addConsumers(new LogConsumer(EnforcerEvent.class, EventChannel.LOG)));
|
||||
private final JobContext context;
|
||||
private final GroupExpression groupExpression;
|
||||
private Cost curTotalCost;
|
||||
|
||||
public EnforceMissingPropertiesHelper(JobContext context, GroupExpression groupExpression,
|
||||
Cost curTotalCost) {
|
||||
this.context = context;
|
||||
this.groupExpression = groupExpression;
|
||||
this.curTotalCost = curTotalCost;
|
||||
}
|
||||
|
||||
@ -116,10 +116,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
|
||||
agg.getGroupByExpressions(),
|
||||
new HashSet<>(agg.getExpressions()));
|
||||
|
||||
if (result.indexId == scan.getTable().getBaseIndexId()) {
|
||||
return ctx.root;
|
||||
}
|
||||
|
||||
LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
|
||||
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
|
||||
|
||||
@ -166,10 +162,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
|
||||
requiredExpr
|
||||
);
|
||||
|
||||
if (result.indexId == scan.getTable().getBaseIndexId()) {
|
||||
return ctx.root;
|
||||
}
|
||||
|
||||
LogicalOlapScan mvPlan =
|
||||
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
|
||||
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
|
||||
@ -215,10 +207,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
|
||||
collectRequireExprWithAggAndProject(agg.getExpressions(), project.getProjects())
|
||||
);
|
||||
|
||||
if (result.indexId == scan.getTable().getBaseIndexId()) {
|
||||
return ctx.root;
|
||||
}
|
||||
|
||||
LogicalOlapScan mvPlan =
|
||||
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
|
||||
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
|
||||
@ -277,10 +265,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
|
||||
requiredExpr
|
||||
);
|
||||
|
||||
if (result.indexId == scan.getTable().getBaseIndexId()) {
|
||||
return ctx.root;
|
||||
}
|
||||
|
||||
LogicalOlapScan mvPlan =
|
||||
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
|
||||
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
|
||||
@ -338,10 +322,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
|
||||
requiredExpr
|
||||
);
|
||||
|
||||
if (result.indexId == scan.getTable().getBaseIndexId()) {
|
||||
return ctx.root;
|
||||
}
|
||||
|
||||
LogicalOlapScan mvPlan =
|
||||
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
|
||||
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
|
||||
@ -389,10 +369,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
|
||||
nonVirtualGroupByExprs(agg),
|
||||
new HashSet<>(agg.getExpressions()));
|
||||
|
||||
if (result.indexId == scan.getTable().getBaseIndexId()) {
|
||||
return ctx.root;
|
||||
}
|
||||
|
||||
LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
|
||||
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
|
||||
|
||||
@ -446,10 +422,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
|
||||
requiredExpr
|
||||
);
|
||||
|
||||
if (result.indexId == scan.getTable().getBaseIndexId()) {
|
||||
return ctx.root;
|
||||
}
|
||||
|
||||
LogicalOlapScan mvPlan =
|
||||
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
|
||||
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
|
||||
@ -502,10 +474,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
|
||||
collectRequireExprWithAggAndProject(agg.getExpressions(), project.getProjects())
|
||||
);
|
||||
|
||||
if (result.indexId == scan.getTable().getBaseIndexId()) {
|
||||
return ctx.root;
|
||||
}
|
||||
|
||||
LogicalOlapScan mvPlan =
|
||||
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
|
||||
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
|
||||
@ -571,10 +539,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
|
||||
requiredExpr
|
||||
);
|
||||
|
||||
if (result.indexId == scan.getTable().getBaseIndexId()) {
|
||||
return ctx.root;
|
||||
}
|
||||
|
||||
LogicalOlapScan mvPlan =
|
||||
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
|
||||
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
|
||||
@ -641,10 +605,6 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
|
||||
requiredExpr
|
||||
);
|
||||
|
||||
if (result.indexId == scan.getTable().getBaseIndexId()) {
|
||||
return ctx.root;
|
||||
}
|
||||
|
||||
LogicalOlapScan mvPlan =
|
||||
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
|
||||
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
|
||||
|
||||
@ -17,6 +17,10 @@
|
||||
|
||||
package org.apache.doris.nereids.trees;
|
||||
|
||||
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
|
||||
import org.apache.doris.nereids.trees.plans.ObjectId;
|
||||
import org.apache.doris.planner.PlanNodeId;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
|
||||
import java.util.List;
|
||||
@ -29,6 +33,7 @@ import java.util.List;
|
||||
*/
|
||||
public abstract class AbstractTreeNode<NODE_TYPE extends TreeNode<NODE_TYPE>>
|
||||
implements TreeNode<NODE_TYPE> {
|
||||
protected final ObjectId id = StatementScopeIdGenerator.newObjectId();
|
||||
protected final List<NODE_TYPE> children;
|
||||
// TODO: Maybe we should use a GroupPlan to avoid TreeNode hold the GroupExpression.
|
||||
// https://github.com/apache/doris/pull/9807#discussion_r884829067
|
||||
@ -54,4 +59,12 @@ public abstract class AbstractTreeNode<NODE_TYPE extends TreeNode<NODE_TYPE>>
|
||||
public int arity() {
|
||||
return children.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* used for PhysicalPlanTranslator only
|
||||
* @return PlanNodeId
|
||||
*/
|
||||
public PlanNodeId translatePlanNodeId() {
|
||||
return id.toPlanNodeId();
|
||||
}
|
||||
}
|
||||
|
||||
@ -37,7 +37,6 @@ import org.apache.doris.nereids.types.coercion.AnyDataType;
|
||||
import org.apache.doris.nereids.util.Utils;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Lists;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
@ -45,6 +44,7 @@ import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* Abstract class for all Expression in Nereids.
|
||||
@ -247,19 +247,8 @@ public abstract class Expression extends AbstractTreeNode<Expression> implements
|
||||
return collect(Slot.class::isInstance);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all the input slot ids of the expression.
|
||||
* <p>
|
||||
* Note that the input slots of subquery's inner plan is not included.
|
||||
*/
|
||||
public final Set<ExprId> getInputSlotExprIds() {
|
||||
ImmutableSet.Builder<ExprId> result = ImmutableSet.builder();
|
||||
foreach(node -> {
|
||||
if (node instanceof Slot) {
|
||||
result.add(((Slot) node).getExprId());
|
||||
}
|
||||
});
|
||||
return result.build();
|
||||
return getInputSlots().stream().map(NamedExpression::getExprId).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
public boolean isLiteral() {
|
||||
|
||||
@ -24,11 +24,9 @@ import org.apache.doris.nereids.properties.UnboundLogicalProperties;
|
||||
import org.apache.doris.nereids.trees.AbstractTreeNode;
|
||||
import org.apache.doris.nereids.trees.expressions.ExprId;
|
||||
import org.apache.doris.nereids.trees.expressions.Slot;
|
||||
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
|
||||
import org.apache.doris.nereids.util.MutableState;
|
||||
import org.apache.doris.nereids.util.MutableState.EmptyMutableState;
|
||||
import org.apache.doris.nereids.util.TreeStringUtils;
|
||||
import org.apache.doris.planner.PlanNodeId;
|
||||
import org.apache.doris.statistics.Statistics;
|
||||
|
||||
import com.google.common.base.Supplier;
|
||||
@ -47,7 +45,6 @@ import javax.annotation.Nullable;
|
||||
*/
|
||||
public abstract class AbstractPlan extends AbstractTreeNode<Plan> implements Plan {
|
||||
public static final String FRAGMENT_ID = "fragment";
|
||||
protected final ObjectId id = StatementScopeIdGenerator.newObjectId();
|
||||
|
||||
protected final Statistics statistics;
|
||||
protected final PlanType type;
|
||||
@ -171,12 +168,4 @@ public abstract class AbstractPlan extends AbstractTreeNode<Plan> implements Pla
|
||||
public void setMutableState(String key, Object state) {
|
||||
this.mutableState = this.mutableState.set(key, state);
|
||||
}
|
||||
|
||||
/**
|
||||
* used for PhysicalPlanTranslator only
|
||||
* @return PlanNodeId
|
||||
*/
|
||||
public PlanNodeId translatePlanNodeId() {
|
||||
return id.toPlanNodeId();
|
||||
}
|
||||
}
|
||||
|
||||
@ -43,9 +43,9 @@ import org.apache.doris.statistics.Statistics;
|
||||
import org.apache.doris.thrift.TRuntimeFilterType;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Sets;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
@ -113,25 +113,22 @@ public class PhysicalHashJoin<
|
||||
* Return pair of left used slots and right used slots.
|
||||
*/
|
||||
public Pair<List<ExprId>, List<ExprId>> getHashConjunctsExprIds() {
|
||||
int size = hashJoinConjuncts.size();
|
||||
|
||||
List<ExprId> exprIds1 = new ArrayList<>(size);
|
||||
List<ExprId> exprIds2 = new ArrayList<>(size);
|
||||
List<ExprId> exprIds1 = Lists.newArrayListWithCapacity(hashJoinConjuncts.size());
|
||||
List<ExprId> exprIds2 = Lists.newArrayListWithCapacity(hashJoinConjuncts.size());
|
||||
|
||||
Set<ExprId> leftExprIds = left().getOutputExprIdSet();
|
||||
Set<ExprId> rightExprIds = right().getOutputExprIdSet();
|
||||
|
||||
for (Expression expr : hashJoinConjuncts) {
|
||||
for (ExprId exprId : expr.getInputSlotExprIds()) {
|
||||
expr.getInputSlotExprIds().forEach(exprId -> {
|
||||
if (leftExprIds.contains(exprId)) {
|
||||
exprIds1.add(exprId);
|
||||
} else if (rightExprIds.contains(exprId)) {
|
||||
exprIds2.add(exprId);
|
||||
} else {
|
||||
throw new RuntimeException("Invalid ExprId found: " + exprId
|
||||
+ ". Cannot generate valid equal on clause slot pairs for join.");
|
||||
throw new RuntimeException("Could not generate valid equal on clause slot pairs for join");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
return Pair.of(exprIds1, exprIds2);
|
||||
}
|
||||
|
||||
@ -38,7 +38,6 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute;
|
||||
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
|
||||
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.qe.SessionVariable;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Lists;
|
||||
@ -67,10 +66,9 @@ public class JoinUtils {
|
||||
* check if the row count of the left child in the broadcast join is less than a threshold value.
|
||||
*/
|
||||
public static boolean checkBroadcastJoinStats(PhysicalHashJoin<? extends Plan, ? extends Plan> join) {
|
||||
SessionVariable sessionVariable = ConnectContext.get().getSessionVariable();
|
||||
double memLimit = sessionVariable.getMaxExecMemByte();
|
||||
double rowsLimit = sessionVariable.getBroadcastRowCountLimit();
|
||||
double brMemlimit = sessionVariable.getBroadcastHashtableMemLimitPercentage();
|
||||
double memLimit = ConnectContext.get().getSessionVariable().getMaxExecMemByte();
|
||||
double rowsLimit = ConnectContext.get().getSessionVariable().getBroadcastRowCountLimit();
|
||||
double brMemlimit = ConnectContext.get().getSessionVariable().getBroadcastHashtableMemLimitPercentage();
|
||||
double datasize = join.getGroupExpression().get().child(1).getStatistics().computeSize();
|
||||
double rowCount = join.getGroupExpression().get().child(1).getStatistics().getRowCount();
|
||||
return rowCount <= rowsLimit && datasize <= memLimit * brMemlimit;
|
||||
@ -116,12 +114,12 @@ public class JoinUtils {
|
||||
* @return true if the equal can be used as hash join condition
|
||||
*/
|
||||
public boolean isHashJoinCondition(EqualTo equalTo) {
|
||||
Set<Slot> equalLeft = equalTo.left().getInputSlots();
|
||||
Set<Slot> equalLeft = equalTo.left().collect(Slot.class::isInstance);
|
||||
if (equalLeft.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Set<Slot> equalRight = equalTo.right().getInputSlots();
|
||||
Set<Slot> equalRight = equalTo.right().collect(Slot.class::isInstance);
|
||||
if (equalRight.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -55,7 +55,7 @@ public class PlanUtils {
|
||||
* normalize comparison predicate on a binary plan to its two sides are corresponding to the child's output.
|
||||
*/
|
||||
public static ComparisonPredicate maybeCommuteComparisonPredicate(ComparisonPredicate expression, Plan left) {
|
||||
Set<Slot> slots = expression.left().getInputSlots();
|
||||
Set<Slot> slots = expression.left().collect(Slot.class::isInstance);
|
||||
Set<Slot> leftSlots = left.getOutputSet();
|
||||
Set<Slot> buffer = Sets.newHashSet(slots);
|
||||
buffer.removeAll(leftSlots);
|
||||
|
||||
@ -1253,7 +1253,8 @@ public class SessionVariable implements Serializable, Writable {
|
||||
public boolean fasterFloatConvert = false;
|
||||
|
||||
@VariableMgr.VarAttr(name = IGNORE_RUNTIME_FILTER_IDS,
|
||||
description = {"the runtime filter id in IGNORE_RUNTIME_FILTER_IDS list will not be generated"})
|
||||
description = {"在IGNORE_RUNTIME_FILTER_IDS列表中的runtime filter将不会被生成",
|
||||
"the runtime filter id in IGNORE_RUNTIME_FILTER_IDS list will not be generated"})
|
||||
|
||||
public String ignoreRuntimeFilterIds = "";
|
||||
public static final String IGNORE_RUNTIME_FILTER_IDS = "ignore_runtime_filter_ids";
|
||||
|
||||
Reference in New Issue
Block a user