diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOlapTableSink.java index 1fd371d856..d9a039a8c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOlapTableSink.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.UnboundLogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Sink; @@ -40,7 +41,8 @@ import java.util.Optional; /** * Represent an olap table sink plan node that has not been bound. */ -public class UnboundOlapTableSink extends LogicalSink implements Unbound, Sink { +public class UnboundOlapTableSink extends LogicalSink + implements Unbound, Sink, BlockFuncDepsPropagation { private final List nameParts; private final List colNames; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java index d2574c6ff2..160b7210f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.properties.UnboundLogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -41,7 +42,8 @@ import java.util.Optional; * A relation that contains only one row consist of some constant expressions. * e.g. select 100, 'value' */ -public class UnboundOneRowRelation extends LogicalRelation implements Unbound, OneRowRelation { +public class UnboundOneRowRelation extends LogicalRelation implements Unbound, OneRowRelation, + BlockFuncDepsPropagation { private final List projects; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index dd0033e9e5..080e749c05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.properties.UnboundLogicalProperties; import org.apache.doris.nereids.trees.TableSample; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -43,7 +44,7 @@ import java.util.Optional; /** * Represent a relation plan node that has not been bound. */ -public class UnboundRelation extends LogicalRelation implements Unbound { +public class UnboundRelation extends LogicalRelation implements Unbound, BlockFuncDepsPropagation { private final List nameParts; private final List partNames; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java index 99a3a591f2..6f378e9911 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Sink; @@ -38,7 +39,8 @@ import java.util.Optional; /** * unbound result sink */ -public class UnboundResultSink extends LogicalSink implements Unbound, Sink { +public class UnboundResultSink extends LogicalSink + implements Unbound, Sink, BlockFuncDepsPropagation { public UnboundResultSink(CHILD_TYPE child) { super(PlanType.LOGICAL_UNBOUND_RESULT_SINK, ImmutableList.of(), child); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java index 51ffb722b7..13b8336933 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; +import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -38,7 +39,8 @@ import java.util.Objects; import java.util.Optional; /** UnboundTVFRelation */ -public class UnboundTVFRelation extends LogicalRelation implements TVFRelation, Unbound { +public class UnboundTVFRelation extends LogicalRelation implements TVFRelation, Unbound, + BlockFuncDepsPropagation { private final String functionName; private final Properties properties; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java index 31a6786ac3..2eb386ce21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.memo.CopyInResult; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.memo.Memo; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; @@ -214,7 +215,7 @@ public class PlanReceiver implements AbstractReceiver { List otherConjuncts) { // Check whether only NSL can be performed LogicalProperties joinProperties = new LogicalProperties( - () -> JoinUtils.getJoinOutput(joinType, left, right)); + () -> JoinUtils.getJoinOutput(joinType, left, right), () -> FunctionalDependencies.EMPTY_FUNC_DEPS); List plans = Lists.newArrayList(); if (JoinUtils.shouldNestedLoopJoin(joinType, hashConjuncts)) { plans.add(new PhysicalNestedLoopJoin<>(joinType, hashConjuncts, otherConjuncts, @@ -378,7 +379,9 @@ public class PlanReceiver implements AbstractReceiver { .collect(Collectors.toList()); if (!outputSet.equals(new HashSet<>(projects))) { LogicalProperties projectProperties = new LogicalProperties( - () -> projects.stream().map(p -> p.toSlot()).collect(Collectors.toList())); + () -> projects.stream() + .map(p -> p.toSlot()) + .collect(Collectors.toList()), () -> FunctionalDependencies.EMPTY_FUNC_DEPS); allChild = allChild.stream() .map(c -> new PhysicalProject<>(projects, projectProperties, c)) .collect(Collectors.toList()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java new file mode 100644 index 0000000000..73a78b0da2 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java @@ -0,0 +1,232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.properties; + +import org.apache.doris.nereids.trees.expressions.Slot; + +import com.google.common.collect.ImmutableSet; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Record functional dependencies, aka func deps, including + * 1. unique slot: it means the column that has ndv = row count, can be null + * 2. uniform slotL it means the column that has ndv = 1, can be null + */ +public class FunctionalDependencies { + public static final FunctionalDependencies EMPTY_FUNC_DEPS + = new FunctionalDependencies(new NestedSet().toImmutable(), new NestedSet().toImmutable()); + private final NestedSet uniqueSet; + private final NestedSet uniformSet; + + private FunctionalDependencies(NestedSet uniqueSet, NestedSet uniformSet) { + this.uniqueSet = uniqueSet; + this.uniformSet = uniformSet; + } + + public boolean isEmpty() { + return uniformSet.isEmpty() && uniqueSet.isEmpty(); + } + + public boolean isUnique(Slot slot) { + return uniqueSet.contains(slot); + } + + public boolean isUnique(Set slotSet) { + return !slotSet.isEmpty() && uniqueSet.containsAnySub(slotSet); + } + + public boolean isUniform(Slot slot) { + return uniformSet.contains(slot); + } + + public boolean isUniform(ImmutableSet slotSet) { + return !slotSet.isEmpty() + && (uniformSet.contains(slotSet) || slotSet.stream().allMatch(uniformSet::contains)); + } + + public boolean isUniqueAndNotNull(Slot slot) { + return !slot.nullable() && isUnique(slot); + } + + public boolean isUniqueAndNotNull(Set slotSet) { + return slotSet.stream().noneMatch(Slot::nullable) && isUnique(slotSet); + } + + public boolean isUniformAndNotNull(Slot slot) { + return !slot.nullable() && isUniform(slot); + } + + public boolean isUniformAndNotNull(ImmutableSet slotSet) { + return slotSet.stream().noneMatch(Slot::nullable) && isUniform(slotSet); + } + + @Override + public String toString() { + return "uniform:" + uniformSet + + "\t unique:" + uniqueSet; + } + + /** + * Builder of Func Deps + */ + public static class Builder { + private final NestedSet uniqueSet; + private final NestedSet uniformSet; + + public Builder() { + uniqueSet = new NestedSet(); + uniformSet = new NestedSet(); + } + + public Builder(FunctionalDependencies other) { + this.uniformSet = new NestedSet(other.uniformSet); + this.uniqueSet = new NestedSet(other.uniqueSet); + } + + public void addUniformSlot(Slot slot) { + uniformSet.add(slot); + } + + public void addUniformSlot(FunctionalDependencies functionalDependencies) { + uniformSet.add(functionalDependencies.uniformSet); + } + + public void addUniformSlot(ImmutableSet slotSet) { + uniformSet.add(slotSet); + } + + public void addUniqueSlot(Slot slot) { + uniqueSet.add(slot); + } + + public void addUniqueSlot(ImmutableSet slotSet) { + uniqueSet.add(slotSet); + } + + public void addFunctionalDependencies(FunctionalDependencies fd) { + uniformSet.add(fd.uniformSet); + uniqueSet.add(fd.uniqueSet); + } + + public FunctionalDependencies build() { + return new FunctionalDependencies(uniqueSet.toImmutable(), uniformSet.toImmutable()); + } + + public void pruneSlots(Set outputSlots) { + uniformSet.removeNotContain(outputSlots); + uniqueSet.removeNotContain(outputSlots); + } + + public void replace(Map replaceMap) { + uniformSet.replace(replaceMap); + uniqueSet.replace(replaceMap); + } + } + + static class NestedSet { + Set slots; + Set> slotSets; + + NestedSet() { + slots = new HashSet<>(); + slotSets = new HashSet<>(); + } + + NestedSet(NestedSet o) { + this.slots = new HashSet<>(o.slots); + this.slotSets = new HashSet<>(o.slotSets); + } + + NestedSet(Set slots, Set> slotSets) { + this.slots = slots; + this.slotSets = slotSets; + } + + public boolean contains(Slot slot) { + return slots.contains(slot); + } + + public boolean contains(ImmutableSet slotSet) { + if (slotSet.size() == 1) { + return slots.contains(slotSet.iterator().next()); + } + return slotSets.contains(slotSet); + } + + public boolean containsAnySub(Set slotSet) { + return slotSet.stream().anyMatch(s -> slots.contains(s)) + || slotSets.stream().anyMatch(slotSet::containsAll); + } + + public void removeNotContain(Set slotSet) { + slots = slots.stream() + .filter(slotSet::contains) + .collect(Collectors.toSet()); + slotSets = slotSets.stream() + .filter(slotSet::containsAll) + .collect(Collectors.toSet()); + } + + public void add(Slot slot) { + slots.add(slot); + } + + public void add(ImmutableSet slotSet) { + if (slotSet.isEmpty()) { + return; + } + if (slotSet.size() == 1) { + slots.add(slotSet.iterator().next()); + return; + } + slotSets.add(slotSet); + } + + public void add(NestedSet nestedSet) { + slots.addAll(nestedSet.slots); + slotSets.addAll(nestedSet.slotSets); + } + + public boolean isEmpty() { + return slots.isEmpty() && slotSets.isEmpty(); + } + + @Override + public String toString() { + return "{" + slots + slotSets + "}"; + } + + public void replace(Map replaceMap) { + slots = slots.stream() + .map(s -> replaceMap.getOrDefault(s, s)) + .collect(Collectors.toSet()); + slotSets = slotSets.stream() + .map(set -> set.stream().map(replaceMap::get).collect(ImmutableSet.toImmutableSet())) + .collect(Collectors.toSet()); + } + + public NestedSet toImmutable() { + return new NestedSet(ImmutableSet.copyOf(slots), ImmutableSet.copyOf(slotSets)); + } + } +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java index e72d07e309..f82b43eaea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java @@ -42,10 +42,11 @@ public class LogicalProperties { protected final Supplier> outputSetSupplier; protected final Supplier> outputMapSupplier; protected final Supplier> outputExprIdSetSupplier; + protected final Supplier fdSupplier; private Integer hashCode = null; - public LogicalProperties(Supplier> outputSupplier) { - this(outputSupplier, ImmutableList::of); + public LogicalProperties(Supplier> outputSupplier, Supplier fdSupplier) { + this(outputSupplier, fdSupplier, ImmutableList::of); } /** @@ -54,7 +55,8 @@ public class LogicalProperties { * @param outputSupplier provide the output. Supplier can lazy compute output without * throw exception for which children have UnboundRelation */ - public LogicalProperties(Supplier> outputSupplier, Supplier> nonUserVisibleOutputSupplier) { + public LogicalProperties(Supplier> outputSupplier, + Supplier fdSupplier, Supplier> nonUserVisibleOutputSupplier) { this.outputSupplier = Suppliers.memoize( Objects.requireNonNull(outputSupplier, "outputSupplier can not be null") ); @@ -73,6 +75,9 @@ public class LogicalProperties { .map(NamedExpression::getExprId) .collect(ImmutableSet.toImmutableSet()) ); + this.fdSupplier = Suppliers.memoize( + Objects.requireNonNull(fdSupplier, "FunctionalDependencies can not be null") + ); } public List getOutput() { @@ -91,6 +96,10 @@ public class LogicalProperties { return outputExprIdSetSupplier.get(); } + public FunctionalDependencies getFunctionalDependencies() { + return fdSupplier.get(); + } + public List getOutputExprIds() { return outputExprIdsSupplier.get(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/UnboundLogicalProperties.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/UnboundLogicalProperties.java index a086e54e7a..886b7aceb1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/UnboundLogicalProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/UnboundLogicalProperties.java @@ -32,7 +32,7 @@ public class UnboundLogicalProperties extends LogicalProperties { public static final UnboundLogicalProperties INSTANCE = new UnboundLogicalProperties(); private UnboundLogicalProperties() { - super(ImmutableList::of); + super(ImmutableList::of, () -> FunctionalDependencies.EMPTY_FUNC_DEPS); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MultiJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MultiJoin.java index 6f25e90320..83e849086a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MultiJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MultiJoin.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -52,7 +53,7 @@ import java.util.Optional; * firstChild [RIGHT (OUTER/SEMI/ANTI) JOIN] children[1, last]. * eg: MJ([ROJ] A, B, C, D) is {A} [ROJ] {B C D}. */ -public class MultiJoin extends AbstractLogicalPlan { +public class MultiJoin extends AbstractLogicalPlan implements BlockFuncDepsPropagation { /* * topJoin * / \ MultiJoin diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java index 1e4fbdcf41..764091e351 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java @@ -19,12 +19,14 @@ package org.apache.doris.nereids.trees.plans; import org.apache.doris.nereids.analyzer.Unbound; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.UnboundLogicalProperties; import org.apache.doris.nereids.trees.AbstractTreeNode; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.MutableState; import org.apache.doris.nereids.util.MutableState.EmptyMutableState; import org.apache.doris.nereids.util.TreeStringUtils; @@ -169,7 +171,11 @@ public abstract class AbstractPlan extends AbstractTreeNode implements Pla if (hasUnboundChild || hasUnboundExpression()) { return UnboundLogicalProperties.INSTANCE; } else { - return new LogicalProperties(this::computeOutput); + Supplier> outputSupplier = Suppliers.memoize(this::computeOutput); + Supplier fdSupplier = () -> this instanceof LogicalPlan + ? ((LogicalPlan) this).computeFuncDeps(outputSupplier) + : FunctionalDependencies.EMPTY_FUNC_DEPS; + return new LogicalProperties(outputSupplier, fdSupplier); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java new file mode 100644 index 0000000000..3f60727576 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans; + +import org.apache.doris.nereids.properties.FunctionalDependencies; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; + +import java.util.List; +import java.util.function.Supplier; + +/** + * Block fd propagation, it always returns an empty fd + */ +public interface BlockFuncDepsPropagation extends LogicalPlan { + @Override + default FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + return FunctionalDependencies.EMPTY_FUNC_DEPS; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java index b964bec262..ad186c680c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -77,7 +78,7 @@ public class FakePlan implements Plan { @Override public LogicalProperties getLogicalProperties() { - return new LogicalProperties(ArrayList::new); + return new LogicalProperties(ArrayList::new, () -> FunctionalDependencies.EMPTY_FUNC_DEPS); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/GroupPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/GroupPlan.java index 0c3f519038..49056add39 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/GroupPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/GroupPlan.java @@ -37,7 +37,7 @@ import java.util.Optional; * Used in {@link org.apache.doris.nereids.pattern.GroupExpressionMatching.GroupExpressionIterator}, * as a place-holder when do match root. */ -public class GroupPlan extends LogicalLeaf { +public class GroupPlan extends LogicalLeaf implements BlockFuncDepsPropagation { private final Group group; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java index 6470b184c8..6b4c063986 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java @@ -160,6 +160,10 @@ public enum JoinType { return this == LEFT_ANTI_JOIN; } + public final boolean isLefSemiJoin() { + return this == LEFT_SEMI_JOIN; + } + public final boolean isRightSemiOrAntiJoin() { return this == RIGHT_SEMI_JOIN || this == RIGHT_ANTI_JOIN; } @@ -168,6 +172,10 @@ public enum JoinType { return this == RIGHT_ANTI_JOIN; } + public final boolean isRightSemiJoin() { + return this == RIGHT_SEMI_JOIN; + } + public final boolean isSemiOrAntiJoin() { return this == LEFT_SEMI_JOIN || this == RIGHT_SEMI_JOIN || this == LEFT_ANTI_JOIN || this == NULL_AWARE_LEFT_ANTI_JOIN || this == RIGHT_ANTI_JOIN; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java new file mode 100644 index 0000000000..332a80c99e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans; + +import org.apache.doris.nereids.properties.FunctionalDependencies; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; + +import java.util.List; +import java.util.function.Supplier; + +/** + * Propagate fd, keep children's fd + */ +public interface PropagateFuncDeps extends LogicalPlan { + @Override + default FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + if (children().size() == 1) { + // Note when changing function dependencies, we always clone it. + // So it's safe to return a reference + return child(0).getLogicalProperties().getFunctionalDependencies(); + } + FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder(); + children().stream() + .map(p -> p.getLogicalProperties().getFunctionalDependencies()) + .forEach(builder::addFunctionalDependencies); + return builder.build(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java index 2adc9d9042..b32d3bed44 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.AbstractPlan; +import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; @@ -36,7 +37,7 @@ import java.util.Optional; /** * All DDL and DML commands' super class. */ -public abstract class Command extends AbstractPlan implements LogicalPlan { +public abstract class Command extends AbstractPlan implements LogicalPlan, BlockFuncDepsPropagation { protected Command(PlanType type) { super(type, Optional.empty(), Optional.empty(), null, ImmutableList.of()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index 072993f5f9..53877ff091 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -18,23 +18,36 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; +import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.VirtualSlotReference; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.expressions.functions.agg.Ndv; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -282,4 +295,72 @@ public class LogicalAggregate return new LogicalAggregate<>(normalizedGroupBy, normalizedOutput, true, ordinalIsResolved, generated, hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), normalizedChild); } + + private void updateFuncDepsGroupByUnique(NamedExpression namedExpression, Builder fdBuilder) { + if (ExpressionUtils.isInjective(namedExpression)) { + fdBuilder.addUniqueSlot(ImmutableSet.copyOf(namedExpression.getInputSlots())); + return; + } + + if (!(namedExpression instanceof Alias && namedExpression.child(0) instanceof AggregateFunction)) { + return; + } + + AggregateFunction agg = (AggregateFunction) namedExpression.child(0); + if (agg instanceof Count || agg instanceof Ndv) { + fdBuilder.addUniformSlot(namedExpression.toSlot()); + return; + } + + if (ExpressionUtils.isInjectiveAgg(agg) + && child().getLogicalProperties().getFunctionalDependencies().isUniqueAndNotNull(agg.getInputSlots())) { + fdBuilder.addUniqueSlot(namedExpression.toSlot()); + } + } + + @Override + public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + FunctionalDependencies childFd = child(0).getLogicalProperties().getFunctionalDependencies(); + Set outputSet = new HashSet<>(outputSupplier.get()); + Builder fdBuilder = new Builder(); + // when group by all tuples, the result only have one row + if (groupByExpressions.isEmpty()) { + outputSet.forEach(s -> { + fdBuilder.addUniformSlot(s); + fdBuilder.addUniqueSlot(s); + }); + return fdBuilder.build(); + } + + // when group by complicate expression or virtual slot, just propagate uniform slots + if (groupByExpressions.stream() + .anyMatch(s -> !(s instanceof SlotReference) || s instanceof VirtualSlotReference)) { + fdBuilder.addUniformSlot(childFd); + fdBuilder.pruneSlots(outputSet); + return fdBuilder.build(); + } + + // when group by uniform slot, the result only have one row + ImmutableSet groupByKeys = groupByExpressions.stream() + .map(s -> (Slot) s) + .collect(ImmutableSet.toImmutableSet()); + if (childFd.isUniformAndNotNull(groupByKeys)) { + outputSupplier.get().forEach(s -> { + fdBuilder.addUniformSlot(s); + fdBuilder.addUniqueSlot(s); + }); + } + + // when group by unique slot, the result depends on agg func + if (childFd.isUniqueAndNotNull(groupByKeys)) { + for (NamedExpression namedExpression : getOutputExpressions()) { + updateFuncDepsGroupByUnique(namedExpression, fdBuilder); + } + } + + // group by keys is unique + fdBuilder.addUniqueSlot(groupByKeys); + fdBuilder.pruneSlots(outputSet); + return fdBuilder.build(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java index b4f25e9771..f6cf13dfc7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -44,7 +45,7 @@ import java.util.Optional; * @param subquery. */ public class LogicalApply - extends LogicalBinary { + extends LogicalBinary implements PropagateFuncDeps { // correlation column private final List correlationSlot; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java index 69b5942c72..3ba0a29842 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -40,7 +41,8 @@ import java.util.Optional; * If the number of rows is more than the desired num of rows, the query will be cancelled. * The cancelled reason will be reported by Backend and displayed back to the user. */ -public class LogicalAssertNumRows extends LogicalUnary { +public class LogicalAssertNumRows extends LogicalUnary implements + PropagateFuncDeps { private final AssertNumRowsElement assertNumRowsElement; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java index e21726ad7c..7a75dd5c1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -36,7 +37,7 @@ import java.util.Optional; /** * Logical Node for CTE */ -public class LogicalCTE extends LogicalUnary { +public class LogicalCTE extends LogicalUnary implements PropagateFuncDeps { private final List> aliasQueries; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEAnchor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEAnchor.java index c7b6bb5158..67aef52f97 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEAnchor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEAnchor.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -36,7 +37,8 @@ import java.util.Optional; * LogicalCTEAnchor */ public class LogicalCTEAnchor extends LogicalBinary { + RIGHT_CHILD_TYPE extends Plan> extends LogicalBinary implements + PropagateFuncDeps { private final CTEId cteId; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java index c96d71e5da..97ea18600c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -40,7 +41,8 @@ import java.util.Optional; /** * LogicalCTEConsumer */ -public class LogicalCTEConsumer extends LogicalRelation { +//TODO: find cte producer and propagate its functional dependencies +public class LogicalCTEConsumer extends LogicalRelation implements BlockFuncDepsPropagation { private final String name; private final CTEId cteId; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEProducer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEProducer.java index 64f2831d07..aeae13302d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEProducer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEProducer.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -37,7 +38,8 @@ import java.util.Optional; /** * LogicalCTEProducer */ -public class LogicalCTEProducer extends LogicalUnary { +public class LogicalCTEProducer extends LogicalUnary implements + PropagateFuncDeps { private final CTEId cteId; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java index 4dd5121ef8..a2c2c68ddc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java @@ -19,10 +19,13 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.TableIf; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; +import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -33,11 +36,13 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.function.Supplier; /** * abstract class catalog relation for logical relation @@ -113,4 +118,18 @@ public abstract class LogicalCatalogRelation extends LogicalRelation implements } return Utils.qualifiedName(qualifier, table.getName()); } + + @Override + public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + Builder fdBuilder = new Builder(); + if (table instanceof OlapTable && ((OlapTable) table).getKeysType().isAggregationFamily()) { + ImmutableSet slotSet = computeOutput().stream() + .filter(SlotReference.class::isInstance) + .filter(s -> ((SlotReference) s).getColumn().isPresent() + && ((SlotReference) s).getColumn().get().isKey()) + .collect(ImmutableSet.toImmutableSet()); + fdBuilder.addUniqueSlot(slotSet); + } + return fdBuilder.build(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java index b9fc7ecfe6..12250bb7d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; @@ -46,7 +47,8 @@ import java.util.Optional; /** * Logical Check Policy */ -public class LogicalCheckPolicy extends LogicalUnary { +public class LogicalCheckPolicy extends LogicalUnary implements + PropagateFuncDeps { public LogicalCheckPolicy(CHILD_TYPE child) { super(PlanType.LOGICAL_CHECK_POLICY, child); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeResultSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeResultSink.java index 277d0dc903..ab6efb8977 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeResultSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeResultSink.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; import org.apache.doris.nereids.trees.plans.algebra.Sink; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -37,7 +38,7 @@ import java.util.Optional; * use for defer materialize top n */ public class LogicalDeferMaterializeResultSink - extends LogicalSink implements Sink { + extends LogicalSink implements Sink, PropagateFuncDeps { private final LogicalResultSink logicalResultSink; private final OlapTable olapTable; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java index b775cb1db2..6b6f6d432e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java @@ -18,6 +18,8 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; +import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -37,11 +39,13 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; /** * use for defer materialize top n */ -public class LogicalDeferMaterializeTopN extends LogicalUnary implements TopN { +public class LogicalDeferMaterializeTopN extends LogicalUnary + implements TopN { private final LogicalTopN logicalTopN; @@ -111,6 +115,19 @@ public class LogicalDeferMaterializeTopN extends Logica .collect(ImmutableList.toImmutableList()); } + @Override + public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + FunctionalDependencies fd = child(0).getLogicalProperties().getFunctionalDependencies(); + if (getLimit() == 1) { + Builder builder = new Builder(); + List output = outputSupplier.get(); + output.forEach(builder::addUniformSlot); + output.forEach(builder::addUniqueSlot); + fd = builder.build(); + } + return fd; + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitLogicalDeferMaterializeTopN(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java index 32e7e1144a..2f6018fae9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -39,7 +40,8 @@ import java.util.Optional; * e.g. * select * from tbl limit 0 */ -public class LogicalEmptyRelation extends LogicalRelation implements EmptyRelation, OutputPrunable { +public class LogicalEmptyRelation extends LogicalRelation + implements EmptyRelation, OutputPrunable, BlockFuncDepsPropagation { private final List projects; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java index f7e2c20fbf..c1753f147e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java @@ -18,8 +18,10 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -27,9 +29,13 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.function.Supplier; /** * Logical Except. @@ -97,4 +103,23 @@ public class LogicalExcept extends LogicalSetOperation { return new LogicalExcept(qualifier, newOutputs, regularChildrenOutputs, Optional.empty(), Optional.empty(), children); } + + @Override + public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + FunctionalDependencies.Builder builder = new FunctionalDependencies + .Builder(child(0).getLogicalProperties().getFunctionalDependencies()); + Map replaceMap = new HashMap<>(); + List output = outputSupplier.get(); + List originalOutputs = regularChildrenOutputs.isEmpty() + ? child(0).getOutput() + : regularChildrenOutputs.get(0); + for (int i = 0; i < output.size(); i++) { + replaceMap.put(originalOutputs.get(i), output.get(i)); + } + builder.replace(replaceMap); + if (qualifier == Qualifier.DISTINCT) { + builder.addUniqueSlot(ImmutableSet.copyOf(outputSupplier.get())); + } + return builder.build(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java index 0dda497d28..3ca559c519 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; import org.apache.doris.nereids.trees.plans.algebra.Sink; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; @@ -36,7 +37,8 @@ import java.util.Optional; /** * logicalFileSink for select into outfile */ -public class LogicalFileSink extends LogicalSink implements Sink { +public class LogicalFileSink extends LogicalSink + implements Sink, PropagateFuncDeps { private final String filePath; private final String format; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java index cb5502d1b6..0c4699fcbe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java @@ -18,6 +18,8 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; +import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -26,6 +28,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Filter; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; @@ -37,6 +40,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -136,4 +140,12 @@ public class LogicalFilter extends LogicalUnary withConjunctsAndChild(Set conjuncts, Plan child) { return new LogicalFilter<>(conjuncts, child); } + + @Override + public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + Builder fdBuilder = new Builder( + child().getLogicalProperties().getFunctionalDependencies()); + getConjuncts().forEach(e -> fdBuilder.addUniformSlot(ExpressionUtils.extractUniformSlot(e))); + return fdBuilder.build(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java index 8036afe256..0de6e46ce2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -35,6 +36,7 @@ import com.google.common.collect.Lists; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.function.Supplier; /** * plan for table generator, the statement like: SELECT * FROM tbl LATERAL VIEW EXPLODE(c1) g as (gc1); @@ -139,4 +141,11 @@ public class LogicalGenerate extends LogicalUnary> outputSupplier) { + FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder(); + builder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); + return builder.build(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java index 8524b35d82..25600c56d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java @@ -18,6 +18,8 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; +import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -25,6 +27,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Filter; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; @@ -35,6 +38,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; /** * Logical Having plan @@ -114,6 +118,14 @@ public class LogicalHaving extends LogicalUnary> outputSupplier) { + Builder fdBuilder = new Builder( + child().getLogicalProperties().getFunctionalDependencies()); + getConjuncts().forEach(e -> fdBuilder.addUniformSlot(ExpressionUtils.extractUniformSlot(e))); + return fdBuilder.build(); + } + @Override public String toString() { return Utils.toSqlString("LogicalHaving", "predicates", getPredicate()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java index 3f08f4e10e..2d3006bda5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java @@ -18,8 +18,10 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -27,9 +29,13 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.function.Supplier; /** * Logical Intersect. @@ -98,4 +104,27 @@ public class LogicalIntersect extends LogicalSetOperation { return new LogicalIntersect(qualifier, newOutputs, regularChildrenOutputs, Optional.empty(), Optional.empty(), children); } + + void replaceSlotInFuncDeps(FunctionalDependencies.Builder builder, + List originalOutputs, List newOutputs) { + Map replaceMap = new HashMap<>(); + for (int i = 0; i < newOutputs.size(); i++) { + replaceMap.put(originalOutputs.get(i), newOutputs.get(i)); + } + builder.replace(replaceMap); + } + + @Override + public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder(); + for (Plan child : children) { + builder.addFunctionalDependencies( + child.getLogicalProperties().getFunctionalDependencies()); + replaceSlotInFuncDeps(builder, child.getOutput(), outputSupplier.get()); + } + if (qualifier == Qualifier.DISTINCT) { + builder.addUniqueSlot(ImmutableSet.copyOf(outputSupplier.get())); + } + return builder.build(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java index b5459acf35..c898e12e2f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java @@ -17,10 +17,14 @@ package org.apache.doris.nereids.trees.plans.logical; +import org.apache.doris.common.Pair; import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; +import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.rules.exploration.join.JoinReorderContext; +import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; @@ -37,17 +41,19 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import org.json.JSONObject; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.annotation.Nullable; /** * Logical join plan. @@ -269,7 +275,7 @@ public class LogicalJoin getExpressions() { - return new Builder() + return new ImmutableList.Builder() .addAll(hashJoinConjuncts) .addAll(otherJoinConjuncts) .build(); @@ -357,6 +363,92 @@ public class LogicalJoin, Set> extractHashKeys() { + Set leftKeys = new HashSet<>(); + Set rightKeys = new HashSet<>(); + for (Expression expression : hashJoinConjuncts) { + // Note we don't support null-safe predicate right now, because we just check uniqueness for join keys + if (!(expression instanceof EqualTo + && ((EqualTo) expression).left() instanceof Slot + && ((EqualTo) expression).right() instanceof Slot)) { + return null; + } + Slot leftKey = (Slot) ((EqualTo) expression).left(); + Slot rightKey = (Slot) ((EqualTo) expression).right(); + if (left().getOutputSet().contains(leftKey)) { + leftKeys.add(leftKey); + rightKeys.add(rightKey); + } else { + leftKeys.add(rightKey); + rightKeys.add(leftKey); + } + } + return Pair.of(leftKeys, rightKeys); + } + + @Override + public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + //1. NALAJ and FOJ block functional dependencies + if (joinType.isNullAwareLeftAntiJoin() || joinType.isFullOuterJoin()) { + return FunctionalDependencies.EMPTY_FUNC_DEPS; + } + + // left/right semi/anti join propagate left/right functional dependencies + if (joinType.isLeftAntiJoin() || joinType.isLefSemiJoin()) { + return left().getLogicalProperties().getFunctionalDependencies(); + } + if (joinType.isRightSemiJoin() || joinType.isRightAntiJoin()) { + return right().getLogicalProperties().getFunctionalDependencies(); + } + + // if there is non-equal join conditions, block functional dependencies + if (!otherJoinConjuncts.isEmpty()) { + return FunctionalDependencies.EMPTY_FUNC_DEPS; + } + + Pair, Set> keys = extractHashKeys(); + if (keys == null) { + return FunctionalDependencies.EMPTY_FUNC_DEPS; + } + + // Note here we only check whether the left is unique. + // So the hash condition can't be null-safe + // TODO: consider Null-safe hash condition when left and rigth is not nullable + boolean isLeftUnique = left().getLogicalProperties() + .getFunctionalDependencies().isUnique(keys.first); + boolean isRightUnique = left().getLogicalProperties() + .getFunctionalDependencies().isUnique(keys.first); + Builder fdBuilder = new Builder(); + if (joinType.isInnerJoin()) { + // inner join propagate uniforms slots + // And if the hash keys is unique, inner join can propagate all functional dependencies + if (isLeftUnique && isRightUnique) { + fdBuilder.addFunctionalDependencies(left().getLogicalProperties().getFunctionalDependencies()); + fdBuilder.addFunctionalDependencies(right().getLogicalProperties().getFunctionalDependencies()); + } else { + fdBuilder.addUniformSlot(left().getLogicalProperties().getFunctionalDependencies()); + fdBuilder.addUniformSlot(right().getLogicalProperties().getFunctionalDependencies()); + } + } + + // left/right outer join propagate left/right uniforms slots + // And if the right/left hash keys is unique, + // join can propagate left/right functional dependencies + if (joinType.isLeftOuterJoin()) { + if (isRightUnique) { + return left().getLogicalProperties().getFunctionalDependencies(); + } + fdBuilder.addUniformSlot(left().getLogicalProperties().getFunctionalDependencies()); + } + if (joinType.isRightOuterJoin()) { + if (isLeftUnique) { + return left().getLogicalProperties().getFunctionalDependencies(); + } + fdBuilder.addUniformSlot(left().getLogicalProperties().getFunctionalDependencies()); + } + return fdBuilder.build(); + } + @Override public JSONObject toJson() { JSONObject logicalJoin = super.toJson(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java index e896cf2240..df18b134f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java @@ -18,6 +18,8 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; +import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -34,6 +36,7 @@ import com.google.common.collect.ImmutableList; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.function.Supplier; /** * Logical limit plan @@ -140,4 +143,16 @@ public class LogicalLimit extends LogicalUnary(limit, offset, phase, children.get(0)); } + + @Override + public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + FunctionalDependencies fd = child(0).getLogicalProperties().getFunctionalDependencies(); + if (getLimit() == 1 && !phase.isLocal()) { + Builder builder = new Builder(); + outputSupplier.get().forEach(builder::addUniformSlot); + outputSupplier.get().forEach(builder::addUniqueSlot); + fd = builder.build(); + } + return fd; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapTableSink.java index a3641057a7..664ac8f993 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapTableSink.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; import org.apache.doris.nereids.trees.plans.algebra.Sink; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -39,7 +40,8 @@ import java.util.Optional; /** * logical olap table sink for insert command */ -public class LogicalOlapTableSink extends LogicalSink implements Sink { +public class LogicalOlapTableSink extends LogicalSink + implements Sink, PropagateFuncDeps { // bound data sink private final Database database; private final OlapTable targetTable; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java index ffc7c0d625..c0c25f7969 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -34,6 +35,7 @@ import com.google.common.collect.ImmutableList; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.function.Supplier; /** * A relation that contains only one row consist of some constant expressions. @@ -126,4 +128,14 @@ public class LogicalOneRowRelation extends LogicalRelation implements OneRowRela public Plan pruneOutputs(List prunedOutputs) { return withProjects(prunedOutputs); } + + @Override + public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder(); + outputSupplier.get().forEach(s -> { + builder.addUniformSlot(s); + builder.addUniqueSlot(s); + }); + return builder.build(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPartitionTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPartitionTopN.java index 003b273727..85ee1e25de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPartitionTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPartitionTopN.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.expressions.functions.window.Rank; import org.apache.doris.nereids.trees.expressions.functions.window.RowNumber; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; import org.apache.doris.nereids.trees.plans.WindowFuncType; import org.apache.doris.nereids.trees.plans.algebra.PartitionTopN; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; @@ -43,7 +44,8 @@ import java.util.Optional; /** * Logical partition-top-N plan. */ -public class LogicalPartitionTopN extends LogicalUnary implements PartitionTopN { +public class LogicalPartitionTopN extends LogicalUnary + implements PartitionTopN, PropagateFuncDeps { private final WindowFuncType function; private final List partitionKeys; private final List orderKeys; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java index 15b289415d..3c058c6e4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java @@ -17,10 +17,13 @@ package org.apache.doris.nereids.trees.plans.logical; +import org.apache.doris.nereids.properties.FunctionalDependencies; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import com.google.common.collect.ImmutableList; +import java.util.List; import java.util.Optional; import java.util.function.BiFunction; import java.util.function.Supplier; @@ -50,4 +53,11 @@ public interface LogicalPlan extends Plan { return (LogicalPlan) withChildren(ImmutableList.copyOf(children())); } + /** + * Compute FunctionalDependencies for different plan + * Note: Unless you really know what you're doing, please use the following interface. + * - BlockFDPropagation: clean the fd + * - PropagateFD: propagate the fd + */ + FunctionalDependencies computeFuncDeps(Supplier> outputSupplier); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index 46b3f3d84a..af2861ad19 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -20,11 +20,14 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.analyzer.Unbound; import org.apache.doris.nereids.analyzer.UnboundStar; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.BoundStar; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Uuid; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Project; @@ -34,11 +37,14 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.json.JSONObject; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.function.Supplier; /** * Logical project plan. @@ -200,4 +206,26 @@ public class LogicalProject extends LogicalUnary> outputSupplier) { + FunctionalDependencies childFuncDeps = child().getLogicalProperties().getFunctionalDependencies(); + FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder(childFuncDeps); + builder.pruneSlots(new HashSet<>(outputSupplier.get())); + projects.stream().filter(Alias.class::isInstance).forEach(proj -> { + if (proj.child(0).isConstant()) { + builder.addUniformSlot(proj.toSlot()); + } else if (proj.child(0) instanceof Uuid) { + builder.addUniqueSlot(proj.toSlot()); + } else if (ExpressionUtils.isInjective(proj.child(0))) { + ImmutableSet inputs = ImmutableSet.copyOf(proj.getInputSlots()); + if (childFuncDeps.isUnique(inputs)) { + builder.addUniqueSlot(proj.toSlot()); + } else if (childFuncDeps.isUniform(inputs)) { + builder.addUniformSlot(proj.toSlot()); + } + } + }); + return builder.build(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java index 61fc8584d4..8ee9e465a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -36,6 +37,7 @@ import com.google.common.collect.ImmutableList; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.function.Supplier; /** * LogicalRepeat. @@ -177,4 +179,14 @@ public class LogicalRepeat extends LogicalUnary output.containsType(VirtualSlotReference.class)); } + + @Override + public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder(); + // Note uniform does not reject nullable slots + outputSupplier.get().stream() + .filter(child(0).getLogicalProperties().getFunctionalDependencies()::isUniform) + .forEach(builder::addUniformSlot); + return builder.build(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalResultSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalResultSink.java index 6e754b491a..564938dd8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalResultSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalResultSink.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; import org.apache.doris.nereids.trees.plans.algebra.Sink; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -34,7 +35,8 @@ import java.util.Optional; /** * result sink */ -public class LogicalResultSink extends LogicalSink implements Sink { +public class LogicalResultSink extends LogicalSink + implements Sink, PropagateFuncDeps { public LogicalResultSink(List outputExprs, CHILD_TYPE child) { super(PlanType.LOGICAL_RESULT_SINK, outputExprs, child); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java index 260d29a1ab..1595666285 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.SelectHint; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; @@ -40,7 +41,8 @@ import java.util.stream.Collectors; * select hint plan. * e.g. LogicalSelectHint (set_var(query_timeout='1800', exec_mem_limit='2147483648')) */ -public class LogicalSelectHint extends LogicalUnary { +public class LogicalSelectHint extends LogicalUnary + implements BlockFuncDepsPropagation { private final Map hints; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java index f139656f01..9d9d321e65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; import org.apache.doris.nereids.trees.plans.algebra.Sort; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -42,7 +43,8 @@ import java.util.Optional; * orderKeys: list of column information after order by. eg:[a, asc],[b, desc]. * OrderKey: Contains order expression information and sorting method. Default is ascending. */ -public class LogicalSort extends LogicalUnary implements Sort { +public class LogicalSort extends LogicalUnary + implements Sort, PropagateFuncDeps { private final List orderKeys; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java index 5bef42f4f2..5dff119af8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -30,9 +31,12 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import org.apache.commons.lang3.StringUtils; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.function.Supplier; /** * The node of logical plan for sub query and alias @@ -153,4 +157,17 @@ public class LogicalSubQueryAlias extends LogicalUnary< return new LogicalSubQueryAlias<>(qualifier, columnAliases, groupExpression, logicalProperties, children.get(0)); } + + @Override + public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + FunctionalDependencies.Builder builder = new FunctionalDependencies + .Builder(child(0).getLogicalProperties().getFunctionalDependencies()); + Map replaceMap = new HashMap<>(); + List outputs = outputSupplier.get(); + for (int i = 0; i < outputs.size(); i++) { + replaceMap.put(child(0).getOutput().get(i), outputs.get(i)); + } + builder.replace(replaceMap); + return builder.build(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java index 4527ffa316..f5842c30dc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; +import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -37,7 +38,7 @@ import java.util.Objects; import java.util.Optional; /** LogicalTableValuedFunctionRelation */ -public class LogicalTVFRelation extends LogicalRelation implements TVFRelation { +public class LogicalTVFRelation extends LogicalRelation implements TVFRelation, BlockFuncDepsPropagation { private final TableValuedFunction function; private final ImmutableList qualifier; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java index 02b239f173..a38778d8d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java @@ -18,6 +18,8 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; +import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.trees.expressions.Expression; @@ -34,6 +36,7 @@ import com.google.common.collect.ImmutableList; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.function.Supplier; /** * Logical top-N plan. @@ -148,4 +151,17 @@ public class LogicalTopN extends LogicalUnary(orderKeys, limit, offset, groupExpression, logicalProperties, children.get(0)); } + + @Override + public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + FunctionalDependencies fd = child(0).getLogicalProperties().getFunctionalDependencies(); + if (getLimit() == 1) { + Builder builder = new Builder(); + List output = outputSupplier.get(); + output.forEach(builder::addUniformSlot); + output.forEach(builder::addUniqueSlot); + fd = builder.build(); + } + return fd; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java index a5ab24e839..17c4ef4a51 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java @@ -18,9 +18,11 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -30,10 +32,12 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.function.Supplier; /** * Logical Union. @@ -170,4 +174,14 @@ public class LogicalUnion extends LogicalSetOperation implements Union, OutputPr public LogicalUnion pruneOutputs(List prunedOutputs) { return withNewOutputs(prunedOutputs); } + + @Override + public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + if (qualifier != Qualifier.DISTINCT) { + return FunctionalDependencies.EMPTY_FUNC_DEPS; + } + FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder(); + builder.addUniqueSlot(ImmutableSet.copyOf(outputSupplier.get())); + return builder.build(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java index fcfe9906d6..e8c16b4902 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -36,10 +37,12 @@ import org.apache.doris.qe.ConnectContext; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.function.Supplier; /** * logical node to deal with window functions; @@ -224,4 +227,46 @@ public class LogicalWindow extends LogicalUnary partitionKeys = windowExpr.getPartitionKeys(); + + // Now we only support slot type keys + if (!partitionKeys.stream().allMatch(Slot.class::isInstance)) { + return; + } + ImmutableSet slotSet = partitionKeys.stream() + .map(s -> (Slot) s) + .collect(ImmutableSet.toImmutableSet()); + + // if partition by keys are unique, output is uniform + if (child(0).getLogicalProperties().getFunctionalDependencies().isUniqueAndNotNull(slotSet)) { + if (windowExpr.getFunction() instanceof RowNumber + || windowExpr.getFunction() instanceof Rank + || windowExpr.getFunction() instanceof DenseRank) { + builder.addUniformSlot(namedExpression.toSlot()); + } + } + + // if partition by keys are uniform, output is unique + if (child(0).getLogicalProperties().getFunctionalDependencies().isUniformAndNotNull(slotSet)) { + if (windowExpr.getFunction() instanceof RowNumber) { + builder.addUniqueSlot(namedExpression.toSlot()); + } + } + } + + @Override + public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder( + child(0).getLogicalProperties().getFunctionalDependencies()); + for (NamedExpression namedExpression : windowExpressions) { + updateFuncDepsByWindowExpr(namedExpression, builder); + } + return builder.build(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/UsingJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/UsingJoin.java index dd4ffbe796..4796b33e1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/UsingJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/UsingJoin.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.JoinHint; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; @@ -41,7 +42,7 @@ import java.util.Optional; * select col1 from t1 join t2 using(col1); */ public class UsingJoin - extends LogicalBinary implements Join { + extends LogicalBinary implements Join, BlockFuncDepsPropagation { private final JoinType joinType; private final ImmutableList otherJoinConjuncts; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 4793395cfe..c27254a7ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -37,6 +37,11 @@ import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.Avg; +import org.apache.doris.nereids.trees.expressions.functions.agg.Max; +import org.apache.doris.nereids.trees.expressions.functions.agg.Min; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; @@ -444,6 +449,32 @@ public class ExpressionUtils { .collect(ImmutableSet.toImmutableSet()); } + /** + * extract uniform slot for the given predicate, such as a = 1 and b = 2 + */ + public static ImmutableSet extractUniformSlot(Expression expression) { + ImmutableSet.Builder builder = new ImmutableSet.Builder<>(); + if (expression instanceof And) { + builder.addAll(extractUniformSlot(expression.child(0))); + builder.addAll(extractUniformSlot(expression.child(1))); + } + if (expression instanceof EqualTo) { + if (isInjective(expression.child(0)) && expression.child(1).isConstant()) { + builder.add((Slot) expression.child(0)); + } + } + return builder.build(); + } + + // TODO: Add more injective functions + public static boolean isInjective(Expression expression) { + return expression instanceof Slot; + } + + public static boolean isInjectiveAgg(AggregateFunction agg) { + return agg instanceof Sum || agg instanceof Avg || agg instanceof Max || agg instanceof Min; + } + public static Set mutableCollect(List expressions, Predicate> predicate) { return expressions.stream() diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java index ad8ed9b6bf..fbe87e079c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids.glue.translator; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.GreaterThan; @@ -63,7 +64,7 @@ public class PhysicalPlanTranslatorTest { t1Output.add(col1); t1Output.add(col2); t1Output.add(col3); - LogicalProperties t1Properties = new LogicalProperties(() -> t1Output); + LogicalProperties t1Properties = new LogicalProperties(() -> t1Output, () -> FunctionalDependencies.EMPTY_FUNC_DEPS); PhysicalOlapScan scan = new PhysicalOlapScan(StatementScopeIdGenerator.newRelationId(), t1, qualifier, t1.getBaseIndexId(), Collections.emptyList(), Collections.emptyList(), null, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), t1Properties, Optional.empty()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java index fd60eb2e00..0b63dacf78 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -87,7 +88,7 @@ public class DeriveStatsJobTest { OlapTable table1 = PlanConstructor.newOlapTable(tableId1, "t1", 0); return (LogicalOlapScan) new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), table1, Collections.emptyList()).withGroupExprLogicalPropChildren(Optional.empty(), - Optional.of(new LogicalProperties(() -> ImmutableList.of(slot1))), ImmutableList.of()); + Optional.of(new LogicalProperties(() -> ImmutableList.of(slot1), () -> FunctionalDependencies.EMPTY_FUNC_DEPS)), ImmutableList.of()); } private LogicalAggregate constructAgg(Plan child) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java index 62d62ebf43..8943787a30 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java @@ -22,6 +22,7 @@ import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.cost.Cost; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.properties.UnboundLogicalProperties; @@ -89,16 +90,18 @@ class MemoTest implements MemoPatternMatchSupported { @Test void testMergeGroup() { Group srcGroup = new Group(new GroupId(2), new GroupExpression(new FakePlan()), - new LogicalProperties(ArrayList::new)); + new LogicalProperties(ArrayList::new, () -> FunctionalDependencies.EMPTY_FUNC_DEPS)); Group dstGroup = new Group(new GroupId(3), new GroupExpression(new FakePlan()), - new LogicalProperties(ArrayList::new)); + new LogicalProperties(ArrayList::new, () -> FunctionalDependencies.EMPTY_FUNC_DEPS)); FakePlan fakePlan = new FakePlan(); GroupExpression srcParentExpression = new GroupExpression(fakePlan, Lists.newArrayList(srcGroup)); - Group srcParentGroup = new Group(new GroupId(0), srcParentExpression, new LogicalProperties(ArrayList::new)); + Group srcParentGroup = new Group(new GroupId(0), srcParentExpression, + new LogicalProperties(ArrayList::new, () -> FunctionalDependencies.EMPTY_FUNC_DEPS)); srcParentGroup.setBestPlan(srcParentExpression, Cost.zeroV1(), PhysicalProperties.ANY); GroupExpression dstParentExpression = new GroupExpression(fakePlan, Lists.newArrayList(dstGroup)); - Group dstParentGroup = new Group(new GroupId(1), dstParentExpression, new LogicalProperties(ArrayList::new)); + Group dstParentGroup = new Group(new GroupId(1), dstParentExpression, + new LogicalProperties(ArrayList::new, () -> FunctionalDependencies.EMPTY_FUNC_DEPS)); Memo memo = new Memo(); Map groups = Deencapsulation.getField(memo, "groups"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java index 729d988d93..4b273b5508 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.processor.post.MergeProjectPostProcessor; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -74,7 +75,7 @@ public class MergeProjectPostProcessTest { t1Output.add(a); t1Output.add(b); t1Output.add(c); - LogicalProperties t1Properties = new LogicalProperties(() -> t1Output); + LogicalProperties t1Properties = new LogicalProperties(() -> t1Output, () -> FunctionalDependencies.EMPTY_FUNC_DEPS); PhysicalOlapScan scan = new PhysicalOlapScan(RelationId.createGenerator().getNextId(), t1, qualifier, 0L, Collections.emptyList(), Collections.emptyList(), null, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), t1Properties, Optional.empty()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushDownFilterThroughProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushDownFilterThroughProjectTest.java index b817f80000..08ccc56f9b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushDownFilterThroughProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushDownFilterThroughProjectTest.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.processor.post.PushDownFilterThroughProject; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.EqualTo; @@ -85,7 +86,7 @@ public class PushDownFilterThroughProjectTest { t1Output.add(a); t1Output.add(b); t1Output.add(c); - LogicalProperties t1Properties = new LogicalProperties(() -> t1Output); + LogicalProperties t1Properties = new LogicalProperties(() -> t1Output, () -> FunctionalDependencies.EMPTY_FUNC_DEPS); PhysicalOlapScan scan = new PhysicalOlapScan(RelationId.createGenerator().getNextId(), t1, qualifier, 0L, Collections.emptyList(), Collections.emptyList(), null, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), t1Properties, diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FunctionalDependenciesTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FunctionalDependenciesTest.java new file mode 100644 index 0000000000..ff72b8cb4b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FunctionalDependenciesTest.java @@ -0,0 +1,387 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.properties; + +import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +class FunctionalDependenciesTest extends TestWithFeService { + Slot slot1 = new SlotReference("1", IntegerType.INSTANCE, false); + Slot slot2 = new SlotReference("2", IntegerType.INSTANCE, false); + Slot slot3 = new SlotReference("1", IntegerType.INSTANCE, false); + Slot slot4 = new SlotReference("1", IntegerType.INSTANCE, false); + + @Override + protected void runBeforeAll() throws Exception { + createDatabase("test"); + createTable("create table test.agg (\n" + + "id int not null,\n" + + "name varchar(128) replace not null )\n" + + "AGGREGATE KEY(id)\n" + + "distributed by hash(id) buckets 10\n" + + "properties('replication_num' = '1');"); + createTable("create table test.uni (\n" + + "id int not null,\n" + + "name varchar(128) not null)\n" + + "UNIQUE KEY(id)\n" + + "distributed by hash(id) buckets 10\n" + + "properties('replication_num' = '1');"); + connectContext.setDatabase("default_cluster:test"); + } + + @Test + void testUniform() { + Builder fdBuilder = new Builder(); + fdBuilder.addUniformSlot(slot1); + FunctionalDependencies fd = fdBuilder.build(); + Assertions.assertTrue(fd.isUniformAndNotNull(slot1)); + Assertions.assertFalse(fd.isUniformAndNotNull(slot2)); + fdBuilder.addUniformSlot(ImmutableSet.of(slot2)); + fd = fdBuilder.build(); + Assertions.assertTrue(fd.isUniformAndNotNull(slot2)); + ImmutableSet slotSet = ImmutableSet.of(slot1, slot2, slot3); + fdBuilder.addUniformSlot(slotSet); + fd = fdBuilder.build(); + Assertions.assertTrue(fd.isUniformAndNotNull(slotSet)); + Assertions.assertFalse(fd.isUniformAndNotNull(ImmutableSet.of(slot1, slot2, slot3, slot4))); + Assertions.assertTrue(fd.isUniformAndNotNull(ImmutableSet.of(slot1, slot2))); + Assertions.assertFalse(fd.isUniformAndNotNull(ImmutableSet.of(slot3, slot2))); + } + + @Test + void testUnique() { + Builder fdBuilder = new Builder(); + fdBuilder.addUniqueSlot(slot1); + FunctionalDependencies fd = fdBuilder.build(); + Assertions.assertTrue(fd.isUniqueAndNotNull(slot1)); + Assertions.assertFalse(fd.isUniqueAndNotNull(slot2)); + fdBuilder.addUniqueSlot(slot2); + fd = fdBuilder.build(); + Assertions.assertTrue(fd.isUniqueAndNotNull(slot2)); + ImmutableSet slotSet = ImmutableSet.of(slot1, slot2, slot3); + fdBuilder.addUniqueSlot(slotSet); + fd = fdBuilder.build(); + Assertions.assertTrue(fd.isUniqueAndNotNull(slotSet)); + Assertions.assertTrue(fd.isUniqueAndNotNull(ImmutableSet.of(slot1, slot2, slot3, slot4))); + Assertions.assertFalse(fd.isUniqueAndNotNull(ImmutableSet.of(slot3, slot4))); + } + + @Test + void testMergeFD() { + Builder fdBuilder1 = new Builder(); + fdBuilder1.addUniformSlot(slot1); + Builder fdBuilder2 = new Builder(); + fdBuilder2.addUniformSlot(slot2); + + fdBuilder1.addFunctionalDependencies(fdBuilder2.build()); + FunctionalDependencies fd = fdBuilder1.build(); + Assertions.assertTrue(fd.isUniformAndNotNull(slot1)); + Assertions.assertTrue(fd.isUniformAndNotNull(slot2)); + } + + @Test + void testScan() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select id from agg") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isUniqueAndNotNull(plan.getOutput().get(0))); + plan = PlanChecker.from(connectContext) + .analyze("select id from uni") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isUniqueAndNotNull(plan.getOutput().get(0))); + } + + @Test + void testFilter() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select name from agg where name = \"1\"") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isUniformAndNotNull(plan.getOutput().get(0))); + } + + @Test + void testJoin() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select * from agg full outer join uni " + + "on agg.id = uni.id") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isEmpty()); + + plan = PlanChecker.from(connectContext) + .analyze("select agg.id, uni.id from agg full outer join uni " + + "on agg.id = uni.id") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isEmpty()); + + plan = PlanChecker.from(connectContext) + .analyze("select agg.id from agg left outer join uni " + + "on agg.id = uni.id") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0))); + + plan = PlanChecker.from(connectContext) + .analyze("select agg.id from agg left semi join uni " + + "on agg.id = uni.id") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0))); + + plan = PlanChecker.from(connectContext) + .analyze("select agg.id from agg left anti join uni " + + "on agg.id = uni.id") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0))); + + plan = PlanChecker.from(connectContext) + .analyze("select uni.id from agg right outer join uni " + + "on agg.id = uni.id") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0))); + + plan = PlanChecker.from(connectContext) + .analyze("select uni.id, agg.id from agg inner join uni " + + "on agg.id = uni.id") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0))); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(1))); + } + + @Test + void testAgg() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select count(1), name from agg group by name") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(1))); + + plan = PlanChecker.from(connectContext) + .analyze("select count(1), max(id), id from agg group by id") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(1))); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(2))); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0))); + + plan = PlanChecker.from(connectContext) + .analyze("select count(1), id from agg where id = 1 group by id") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(1))); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0))); + + plan = PlanChecker.from(connectContext) + .analyze("select count(name) from agg") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0))); + + plan = PlanChecker.from(connectContext) + .analyze("select id from agg where id = 1 group by cube(id, name)") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniform(plan.getOutput().get(0))); + } + + @Test + void testGenerate() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select k1 from (select 1 k1) as t lateral view explode([1,2,3]) tmp1 as e1") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniform(plan.getOutput().get(0))); + } + + @Test + void testWindow() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select row_number() over(partition by id) from agg") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0))); + + plan = PlanChecker.from(connectContext) + .analyze("select row_number() over(partition by name) from agg where name = '1'") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0))); + + plan = PlanChecker.from(connectContext) + .analyze("select row_number() over(partition by name) from agg where name = '1' limit 1") + .rewrite() + .getPlan(); + LogicalPartitionTopN ptopn = (LogicalPartitionTopN) plan.child(0).child(0).child(0).child(0).child(0); + System.out.println(ptopn.getLogicalProperties().getFunctionalDependencies()); + System.out.println(ptopn.getOutput()); + Assertions.assertTrue(ptopn.getLogicalProperties() + .getFunctionalDependencies().isUniformAndNotNull(ImmutableSet.copyOf(ptopn.getOutputSet()))); + + plan = PlanChecker.from(connectContext) + .analyze("select row_number() over(partition by name) from agg limit 1") + .rewrite() + .getPlan(); + ptopn = (LogicalPartitionTopN) plan.child(0).child(0).child(0).child(0).child(0); + Assertions.assertTrue(ptopn.getLogicalProperties() + .getFunctionalDependencies().isEmpty()); + } + + @Test + void testProject() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select name from agg where id = 1") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isEmpty()); + } + + @Test + void testLimitAndTopN() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select name from agg limit 1") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0))); + + plan = PlanChecker.from(connectContext) + .analyze("select name from agg order by id limit 1") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0))); + } + + @Test + void testSubQuery() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select id from (select * from agg) t") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0))); + } + + @Disabled + @Test + void testCTE() { + Plan plan = PlanChecker.from(connectContext) + .analyze("with t as (select * from agg) select id from t where id = 1") + .getPlan(); + System.out.println(plan.treeString()); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0))); + } + + @Test + void testSetOperation() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select * from agg union select * from uni") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(ImmutableSet.copyOf(plan.getOutput()))); + + plan = PlanChecker.from(connectContext) + .analyze("select * from agg union all select * from uni") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertFalse(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(ImmutableSet.copyOf(plan.getOutput()))); + + plan = PlanChecker.from(connectContext) + .analyze("select * from agg intersect select * from uni where name = \"1\"") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(ImmutableSet.copyOf(plan.getOutput()))); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0))); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(1))); + + plan = PlanChecker.from(connectContext) + .analyze("select * from agg except select * from uni") + .rewrite() + .getPlan(); + System.out.println(plan.getLogicalProperties().getFunctionalDependencies()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(ImmutableSet.copyOf(plan.getOutput()))); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0))); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/JoinEstimateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/JoinEstimateTest.java index 2735e26da4..282f88fc77 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/JoinEstimateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/JoinEstimateTest.java @@ -20,6 +20,7 @@ package org.apache.doris.nereids.stats; import org.apache.doris.common.IdGenerator; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupId; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Slot; @@ -73,14 +74,14 @@ public class JoinEstimateTest { public List get() { return Lists.newArrayList(a); } - }))); + }, () -> FunctionalDependencies.EMPTY_FUNC_DEPS))); GroupPlan right = new GroupPlan(new Group(idGenerator.getNextId(), new LogicalProperties( new Supplier>() { @Override public List get() { return Lists.newArrayList(b); } - }))); + }, () -> FunctionalDependencies.EMPTY_FUNC_DEPS))); LogicalJoin join = new LogicalJoin(JoinType.INNER_JOIN, Lists.newArrayList(eq), left, right); Statistics outputStats = JoinEstimation.estimate(leftStats, rightStats, join); @@ -124,14 +125,14 @@ public class JoinEstimateTest { public List get() { return Lists.newArrayList(a); } - }))); + }, () -> FunctionalDependencies.EMPTY_FUNC_DEPS))); GroupPlan right = new GroupPlan(new Group(idGenerator.getNextId(), new LogicalProperties( new Supplier>() { @Override public List get() { return Lists.newArrayList(b, c); } - }))); + }, () -> FunctionalDependencies.EMPTY_FUNC_DEPS))); LogicalJoin join = new LogicalJoin(JoinType.LEFT_OUTER_JOIN, Lists.newArrayList(eq), left, right); Statistics outputStats = JoinEstimation.estimate(leftStats, rightStats, join); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java index 3dfe2d0bdf..28591ab8cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.And; import org.apache.doris.nereids.trees.expressions.EqualTo; @@ -61,7 +62,8 @@ public class StatsCalculatorTest { private Group newFakeGroup() { GroupExpression groupExpression = new GroupExpression(scan); - Group group = new Group(null, groupExpression, new LogicalProperties(Collections::emptyList)); + Group group = new Group(null, groupExpression, + new LogicalProperties(Collections::emptyList, () -> FunctionalDependencies.EMPTY_FUNC_DEPS)); group.getLogicalExpressions().remove(0); return group; } @@ -250,7 +252,7 @@ public class StatsCalculatorTest { LogicalOlapScan logicalOlapScan1 = (LogicalOlapScan) new LogicalOlapScan( StatementScopeIdGenerator.newRelationId(), table1, Collections.emptyList()).withGroupExprLogicalPropChildren(Optional.empty(), - Optional.of(new LogicalProperties(() -> ImmutableList.of(slot1))), ImmutableList.of()); + Optional.of(new LogicalProperties(() -> ImmutableList.of(slot1), () -> FunctionalDependencies.EMPTY_FUNC_DEPS)), ImmutableList.of()); GroupExpression groupExpression = new GroupExpression(logicalOlapScan1, ImmutableList.of()); Group ownerGroup = new Group(null, groupExpression, null);