database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](Nereids): add equal set in functional dependencies (#33642)

Browse Source
This commit is contained in:
谢健
2024-04-25 16:31:38 +08:00
committed by yiguolei
parent 2b4f4ca796
commit a237f7ec6e
28 changed files with 575 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java
View File

@ -18,11 +18,13 @@
package org.apache.doris.nereids.properties;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.util.ImmutableEqualSet;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@ -36,15 +38,20 @@ public class FunctionalDependencies {
public static final FunctionalDependencies EMPTY_FUNC_DEPS
= new FunctionalDependencies(new NestedSet().toImmutable(),
new NestedSet().toImmutable(), new ImmutableSet.Builder<FdItem>().build());
new NestedSet().toImmutable(), new ImmutableSet.Builder<FdItem>().build(),
ImmutableEqualSet.empty());
private final NestedSet uniqueSet;
private final NestedSet uniformSet;
private final ImmutableSet<FdItem> fdItems;
private FunctionalDependencies(NestedSet uniqueSet, NestedSet uniformSet, ImmutableSet<FdItem> fdItems) {
private final ImmutableEqualSet<Slot> equalSet;
private FunctionalDependencies(
NestedSet uniqueSet, NestedSet uniformSet, ImmutableSet<FdItem> fdItems, ImmutableEqualSet<Slot> equalSet) {
this.uniqueSet = uniqueSet;
this.uniformSet = uniformSet;
this.fdItems = fdItems;
this.equalSet = equalSet;
}
public boolean isEmpty() {
@ -87,13 +94,26 @@ public class FunctionalDependencies {
return slotSet.stream().noneMatch(Slot::nullable) && isUniform(slotSet);
}
public boolean isNullSafeEqual(Slot l, Slot r) {
return equalSet.isEqual(l, r);
}
public boolean isEqualAndNotNotNull(Slot l, Slot r) {
return equalSet.isEqual(l, r) && !l.nullable() && !r.nullable();
}
public List<Set<Slot>> calAllEqualSet() {
return equalSet.calEqualSetList();
}
public ImmutableSet<FdItem> getFdItems() {
return fdItems;
}
@Override
public String toString() {
return String.format("FuncDeps[uniform:%s, unique:%s, fdItems:%s]", uniformSet, uniqueSet, fdItems);
return String.format("FuncDeps[uniform:%s, unique:%s, fdItems:%s, equalSet:%s]",
uniformSet, uniqueSet, fdItems, equalSet);
}
/**
@ -103,17 +123,21 @@ public class FunctionalDependencies {
private final NestedSet uniqueSet;
private final NestedSet uniformSet;
private ImmutableSet<FdItem> fdItems;
private final ImmutableEqualSet.Builder<Slot> equalSetBuilder;
public Builder() {
uniqueSet = new NestedSet();
uniformSet = new NestedSet();
fdItems = new ImmutableSet.Builder<FdItem>().build();
equalSetBuilder = new ImmutableEqualSet.Builder<>();
}
public Builder(FunctionalDependencies other) {
this.uniformSet = new NestedSet(other.uniformSet);
this.uniqueSet = new NestedSet(other.uniqueSet);
this.fdItems = ImmutableSet.copyOf(other.fdItems);
equalSetBuilder = new ImmutableEqualSet.Builder<>(other.equalSet);
}
public void addUniformSlot(Slot slot) {
@ -147,11 +171,20 @@ public class FunctionalDependencies {
public void addFunctionalDependencies(FunctionalDependencies fd) {
uniformSet.add(fd.uniformSet);
uniqueSet.add(fd.uniqueSet);
equalSetBuilder.addEqualSet(fd.equalSet);
}
public void addEqualPair(Slot l, Slot r) {
equalSetBuilder.addEqualPair(l, r);
}
public void addEqualSet(FunctionalDependencies functionalDependencies) {
equalSetBuilder.addEqualSet(functionalDependencies.equalSet);
}
public FunctionalDependencies build() {
return new FunctionalDependencies(uniqueSet.toImmutable(), uniformSet.toImmutable(),
ImmutableSet.copyOf(fdItems));
ImmutableSet.copyOf(fdItems), equalSetBuilder.build());
}
public void pruneSlots(Set<Slot> outputSlots) {
@ -162,6 +195,7 @@ public class FunctionalDependencies {
public void replace(Map<Slot, Slot> replaceMap) {
uniformSet.replace(replaceMap);
uniqueSet.replace(replaceMap);
equalSetBuilder.replace(replaceMap);
}
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java
View File

@ -46,4 +46,9 @@ public interface BlockFuncDepsPropagation extends LogicalPlan {
default void computeUniform(FunctionalDependencies.Builder fdBuilder) {
// don't generate
}
@Override
default void computeEqualSet(FunctionalDependencies.Builder fdBuilder) {
// don't generate
}
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java
View File

@ -66,4 +66,9 @@ public interface PropagateFuncDeps extends LogicalPlan {
default void computeUniform(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
@Override
default void computeEqualSet(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addEqualSet(child(0).getLogicalProperties().getFunctionalDependencies());
}
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java
View File

@ -397,4 +397,9 @@ public class LogicalAggregate<CHILD_TYPE extends Plan>
return builder.build();
}
@Override
public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies());
}
}

6
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.AssertNumRowsElement;
@ -145,4 +146,9 @@ public class LogicalAssertNumRows<CHILD_TYPE extends Plan> extends LogicalUnary<
getOutput().forEach(fdBuilder::addUniformSlot);
}
}
@Override
public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies());
}
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java
View File

@ -215,4 +215,9 @@ public abstract class LogicalCatalogRelation extends LogicalRelation implements
}
return slotSet.build();
}
@Override
public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) {
// don't generate any equal pair
}
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java
View File

@ -209,4 +209,9 @@ public class LogicalDeferMaterializeTopN<CHILD_TYPE extends Plan> extends Logica
"columnIdSlot", columnIdSlot
);
}
@Override
public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies());
}
}

15
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.ExprFdItem;
import org.apache.doris.nereids.properties.FdFactory;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@ -148,6 +149,20 @@ public class LogicalExcept extends LogicalSetOperation {
fdBuilder.replace(replaceMap);
}
@Override
public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addEqualSet(child(0).getLogicalProperties().getFunctionalDependencies());
Map<Slot, Slot> replaceMap = new HashMap<>();
List<Slot> output = getOutput();
List<? extends Slot> originalOutputs = regularChildrenOutputs.isEmpty()
? child(0).getOutput()
: regularChildrenOutputs.get(0);
for (int i = 0; i < output.size(); i++) {
replaceMap.put(originalOutputs.get(i), output.get(i));
}
fdBuilder.replace(replaceMap);
}
@Override
public void computeUniform(Builder fdBuilder) {
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());

10
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
@ -166,4 +167,13 @@ public class LogicalFilter<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_T
getConjuncts().forEach(e -> fdBuilder.addUniformSlot(ExpressionUtils.extractUniformSlot(e)));
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
@Override
public void computeEqualSet(Builder fdBuilder) {
fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies());
for (Expression expression : getConjuncts()) {
Optional<Pair<Slot, Slot>> equalSlot = ExpressionUtils.extractEqualSlot(expression);
equalSlot.ifPresent(slotSlotPair -> fdBuilder.addEqualPair(slotSlotPair.first, slotSlotPair.second));
}
}
}

6
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
@ -169,4 +170,9 @@ public class LogicalGenerate<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD
public void computeUniform(Builder fdBuilder) {
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
@Override
public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies());
}
}

10
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
@ -138,6 +139,15 @@ public class LogicalHaving<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_T
return builder.build();
}
@Override
public void computeEqualSet(Builder fdBuilder) {
fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies());
for (Expression expression : getConjuncts()) {
Optional<Pair<Slot, Slot>> equalSlot = ExpressionUtils.extractEqualSlot(expression);
equalSlot.ifPresent(slotSlotPair -> fdBuilder.addEqualPair(slotSlotPair.first, slotSlotPair.second));
}
}
@Override
public String toString() {
return Utils.toSqlString("LogicalHaving", "predicates", getPredicate());

9
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java
View File

@ -139,6 +139,15 @@ public class LogicalIntersect extends LogicalSetOperation {
}
}
@Override
public void computeEqualSet(Builder fdBuilder) {
for (Plan child : children) {
fdBuilder.addEqualSet(
child.getLogicalProperties().getFunctionalDependencies());
replaceSlotInFuncDeps(fdBuilder, child.getOutput(), getOutput());
}
}
@Override
public ImmutableSet<FdItem> computeFdItems() {
Set<NamedExpression> output = ImmutableSet.copyOf(getOutput());

16
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
View File

@ -696,4 +696,20 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
fdBuilder.addUniformSlot(left().getLogicalProperties().getFunctionalDependencies());
}
}
@Override
public void computeEqualSet(Builder fdBuilder) {
if (!joinType.isLeftSemiOrAntiJoin()) {
fdBuilder.addEqualSet(right().getLogicalProperties().getFunctionalDependencies());
}
if (!joinType.isRightSemiOrAntiJoin()) {
fdBuilder.addEqualSet(left().getLogicalProperties().getFunctionalDependencies());
}
if (joinType.isInnerJoin()) {
for (Expression expression : getHashJoinConjuncts()) {
Optional<Pair<Slot, Slot>> equalSlot = ExpressionUtils.extractEqualSlot(expression);
equalSlot.ifPresent(slotSlotPair -> fdBuilder.addEqualPair(slotSlotPair.first, slotSlotPair.second));
}
}
}
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java
View File

@ -181,4 +181,9 @@ public class LogicalLimit<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TY
}
return fdItems;
}
@Override
public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies());
}
}

16
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java
View File

@ -23,6 +23,7 @@ import org.apache.doris.nereids.properties.FdFactory;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
@ -37,7 +38,9 @@ import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
@ -157,4 +160,17 @@ public class LogicalOneRowRelation extends LogicalRelation implements OneRowRela
return builder.build();
}
@Override
public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) {
Map<Expression, NamedExpression> aliasMap = new HashMap<>();
for (NamedExpression namedExpr : getOutputs()) {
if (namedExpr instanceof Alias) {
if (aliasMap.containsKey(namedExpr.child(0))) {
fdBuilder.addEqualPair(namedExpr.toSlot(), aliasMap.get(namedExpr.child(0)).toSlot());
}
aliasMap.put(namedExpr.child(0), namedExpr);
}
}
}
}

3
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java
View File

@ -63,6 +63,7 @@ public interface LogicalPlan extends Plan {
FunctionalDependencies.Builder fdBuilder = new FunctionalDependencies.Builder();
computeUniform(fdBuilder);
computeUnique(fdBuilder);
computeEqualSet(fdBuilder);
ImmutableSet<FdItem> fdItems = computeFdItems();
fdBuilder.addFdItems(fdItems);
return fdBuilder.build();
@ -73,4 +74,6 @@ public interface LogicalPlan extends Plan {
void computeUnique(FunctionalDependencies.Builder fdBuilder);
void computeUniform(FunctionalDependencies.Builder fdBuilder);
void computeEqualSet(FunctionalDependencies.Builder fdBuilder);
}

21
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
View File

@ -23,6 +23,7 @@ import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.BoundStar;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@ -41,7 +42,9 @@ import com.google.common.collect.ImmutableList.Builder;
import com.google.common.collect.ImmutableSet;
import org.json.JSONObject;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
@ -282,4 +285,22 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
}
fdBuilder.pruneSlots(getOutputSet());
}
@Override
public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) {
Map<Expression, NamedExpression> aliasMap = new HashMap<>();
fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies());
for (NamedExpression expr : getProjects()) {
if (expr instanceof Alias) {
if (aliasMap.containsKey(expr.child(0))) {
fdBuilder.addEqualPair(expr.toSlot(), aliasMap.get(expr.child(0)).toSlot());
}
aliasMap.put(expr.child(0), expr);
if (expr.child(0).isSlot()) {
fdBuilder.addEqualPair(expr.toSlot(), (Slot) expr.child(0));
}
}
}
fdBuilder.pruneSlots(getOutputSet());
}
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java
View File

@ -204,4 +204,9 @@ public class LogicalRepeat<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_T
return builder.build();
}
@Override
public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies());
}
}

21
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSqlCache.java
View File

@ -20,11 +20,10 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.analysis.Expr;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.TreeStringPlan;
@ -36,14 +35,13 @@ import org.apache.doris.qe.ResultSet;
import org.apache.doris.thrift.TUniqueId;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
/** LogicalSqlCache */
public class LogicalSqlCache extends LogicalLeaf implements SqlCache, TreeStringPlan {
public class LogicalSqlCache extends LogicalLeaf implements SqlCache, TreeStringPlan, BlockFuncDepsPropagation {
private final TUniqueId queryId;
private final List<String> columnLabels;
private final List<Expr> resultExprs;
@ -137,19 +135,4 @@ public class LogicalSqlCache extends LogicalLeaf implements SqlCache, TreeString
public String getChildrenTreeString() {
return planBody;
}
@Override
public ImmutableSet<FdItem> computeFdItems() {
return ImmutableSet.of();
}
@Override
public void computeUnique(Builder fdBuilder) {
}
@Override
public void computeUniform(Builder fdBuilder) {
}
}

11
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java
View File

@ -191,6 +191,17 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<
return ImmutableSet.of();
}
@Override
public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addEqualSet(child(0).getLogicalProperties().getFunctionalDependencies());
Map<Slot, Slot> replaceMap = new HashMap<>();
List<Slot> outputs = getOutput();
for (int i = 0; i < outputs.size(); i++) {
replaceMap.put(child(0).getOutput().get(i), outputs.get(i));
}
fdBuilder.replace(replaceMap);
}
public void setRelationId(RelationId relationId) {
this.relationId = relationId;
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java
View File

@ -187,6 +187,11 @@ public class LogicalTopN<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYP
}
}
@Override
public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addEqualSet(child(0).getLogicalProperties().getFunctionalDependencies());
}
@Override
public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet<FdItem> fdItems = child(0).getLogicalProperties().getFunctionalDependencies().getFdItems();

61
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java
View File

@ -25,6 +25,7 @@ import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
@ -36,7 +37,12 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
@ -196,6 +202,61 @@ public class LogicalUnion extends LogicalSetOperation implements Union, OutputPr
// don't propagate uniform slots
}
private List<Set<Integer>> mapSlotToIndex(Plan plan, List<Set<Slot>> equalSlotsList) {
Map<Slot, Integer> slotToIndex = new HashMap<>();
for (int i = 0; i < plan.getOutput().size(); i++) {
slotToIndex.put(plan.getOutput().get(i), i);
}
List<Set<Integer>> equalSlotIndicesList = new ArrayList<>();
for (Set<Slot> equalSlots : equalSlotsList) {
Set<Integer> equalSlotIndices = new HashSet<>();
for (Slot slot : equalSlots) {
if (slotToIndex.containsKey(slot)) {
equalSlotIndices.add(slotToIndex.get(slot));
}
}
if (equalSlotIndices.size() > 1) {
equalSlotIndicesList.add(equalSlotIndices);
}
}
return equalSlotIndicesList;
}
@Override
public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) {
List<Set<Integer>> unionEqualSlotIndicesList = new ArrayList<>();
for (Plan child : children) {
List<Set<Slot>> childEqualSlotsList =
child.getLogicalProperties().getFunctionalDependencies().calAllEqualSet();
List<Set<Integer>> childEqualSlotsIndicesList = mapSlotToIndex(child, childEqualSlotsList);
if (unionEqualSlotIndicesList.isEmpty()) {
unionEqualSlotIndicesList = childEqualSlotsIndicesList;
} else {
// Only all child of union has the equal pair, we keep the equal pair.
// It means we should calculate the intersection of all child
for (Set<Integer> childEqualSlotIndices : childEqualSlotsIndicesList) {
for (Set<Integer> unionEqualSlotIndices : unionEqualSlotIndicesList) {
if (Collections.disjoint(childEqualSlotIndices, unionEqualSlotIndices)) {
unionEqualSlotIndices.retainAll(childEqualSlotIndices);
}
}
}
}
List<Slot> ouputList = getOutput();
for (Set<Integer> equalSlotIndices : unionEqualSlotIndicesList) {
if (equalSlotIndices.size() <= 1) {
continue;
}
int first = equalSlotIndices.iterator().next();
for (int idx : equalSlotIndices) {
fdBuilder.addEqualPair(ouputList.get(first), ouputList.get(idx));
}
}
}
}
@Override
public ImmutableSet<FdItem> computeFdItems() {
Set<NamedExpression> output = ImmutableSet.copyOf(getOutput());

6
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.catalog.View;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
@ -146,4 +147,9 @@ public class LogicalView<BODY extends Plan> extends LogicalUnary<BODY> {
public void computeUniform(Builder fdBuilder) {
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
@Override
public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addEqualSet(child(0).getLogicalProperties().getFunctionalDependencies());
}
}

6
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
@ -324,4 +325,9 @@ public class LogicalWindow<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_T
}
}
}
@Override
public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addEqualSet(child(0).getLogicalProperties().getFunctionalDependencies());
}
}

8
fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.nereids.util;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.MaterializedViewException;
import org.apache.doris.common.NereidsException;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule;
@ -130,6 +131,13 @@ public class ExpressionUtils {
}
}
public static Optional<Pair<Slot, Slot>> extractEqualSlot(Expression expr) {
if (expr instanceof EqualTo && expr.child(0).isSlot() && expr.child(1).isSlot()) {
return Optional.of(Pair.of((Slot) expr.child(0), (Slot) expr.child(1)));
}
return Optional.empty();
}
public static Optional<Expression> optionalAnd(List<Expression> expressions) {
if (expressions.isEmpty()) {
return Optional.empty();

69
fe/fe-core/src/main/java/org/apache/doris/nereids/util/ImmutableEqualSet.java
View File

@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
/**
@ -44,34 +45,66 @@ public class ImmutableEqualSet<T> {
* Builder for ImmutableEqualSet.
*/
public static class Builder<T> {
private final Map<T, T> parent = new LinkedHashMap<>();
private final Map<T, Integer> size = new LinkedHashMap<>();
private Map<T, T> parent;
Builder(Map<T, T> parent) {
this.parent = parent;
}
public Builder() {
this(new LinkedHashMap<>());
}
public Builder(ImmutableEqualSet<T> equalSet) {
this(new LinkedHashMap<>(equalSet.root));
}
/**
* replace all key value according replace map
*/
public void replace(Map<T, T> replaceMap) {
Map<T, T> newMap = new LinkedHashMap<>();
for (Entry<T, T> entry : parent.entrySet()) {
newMap.put(replaceMap.getOrDefault(entry.getKey(), entry.getKey()),
replaceMap.getOrDefault(entry.getValue(), entry.getValue()));
}
parent = newMap;
}
/**
* Add a equal pair
*/
public void addEqualPair(T a, T b) {
if (!parent.containsKey(a)) {
parent.put(a, a);
}
if (!parent.containsKey(b)) {
parent.put(b, b);
}
T root1 = findRoot(a);
T root2 = findRoot(b);
if (root1 != root2) {
parent.put(b, root1);
findRoot(b);
parent.put(root1, root2);
}
}
public void addEqualSet(ImmutableEqualSet<T> equalSet) {
this.parent.putAll(equalSet.root);
}
private T findRoot(T a) {
parent.putIfAbsent(a, a); // Ensure that the element is added
size.putIfAbsent(a, 1); // Initialize size to 1
if (!parent.get(a).equals(a)) {
parent.put(a, findRoot(parent.get(a))); // Path compression
if (a.equals(parent.get(a))) {
return parent.get(a);
}
return parent.get(a);
return findRoot(parent.get(a));
}
public ImmutableEqualSet<T> build() {
parent.keySet().forEach(this::findRoot);
return new ImmutableEqualSet<>(parent);
ImmutableMap.Builder<T, T> foldMapBuilder = new ImmutableMap.Builder<>();
for (T k : parent.keySet()) {
foldMapBuilder.put(k, findRoot(k));
}
return new ImmutableEqualSet<>(foldMapBuilder.build());
}
}
@ -103,4 +136,16 @@ public class ImmutableEqualSet<T> {
public Set<T> getAllItemSet() {
return ImmutableSet.copyOf(root.keySet());
}
public boolean isEqual(T l, T r) {
if (!root.containsKey(l) || !root.containsKey(r)) {
return false;
}
return root.get(l) == root.get(r);
}
@Override
public String toString() {
return root.toString();
}
}