From 92d28e497b1ab96ce99303a4338abc805149ba87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=A2=E5=81=A5?= Date: Thu, 11 Apr 2024 15:57:38 +0800 Subject: [PATCH] [refactor](Nereids): compute unique and uniform property respectively (#32908) --- .../properties/FunctionalDependencies.java | 4 + .../nereids/trees/plans/AbstractPlan.java | 2 +- .../trees/plans/BlockFuncDepsPropagation.java | 18 +- .../trees/plans/PropagateFuncDeps.java | 20 +- .../trees/plans/logical/LogicalAggregate.java | 122 +++--- .../plans/logical/LogicalAssertNumRows.java | 33 +- .../plans/logical/LogicalCatalogRelation.java | 23 +- .../logical/LogicalDeferMaterializeTopN.java | 29 +- .../trees/plans/logical/LogicalExcept.java | 59 +-- .../trees/plans/logical/LogicalFilter.java | 25 +- .../trees/plans/logical/LogicalGenerate.java | 20 +- .../trees/plans/logical/LogicalHaving.java | 19 +- .../trees/plans/logical/LogicalIntersect.java | 27 +- .../trees/plans/logical/LogicalJoin.java | 130 +++--- .../trees/plans/logical/LogicalLimit.java | 30 +- .../plans/logical/LogicalOneRowRelation.java | 21 +- .../trees/plans/logical/LogicalPlan.java | 17 +- .../trees/plans/logical/LogicalProject.java | 67 +-- .../trees/plans/logical/LogicalRepeat.java | 19 +- .../plans/logical/LogicalSubQueryAlias.java | 26 +- .../trees/plans/logical/LogicalTopN.java | 30 +- .../trees/plans/logical/LogicalUnion.java | 22 +- .../trees/plans/logical/LogicalView.java | 22 +- .../trees/plans/logical/LogicalWindow.java | 75 ++-- .../doris/nereids/util/ExpressionUtils.java | 4 +- .../FunctionalDependenciesTest.java | 4 +- .../doris/nereids/properties/UniformTest.java | 207 ++++++++++ .../doris/nereids/properties/UniqueTest.java | 385 ++++++++++++++++++ 28 files changed, 1075 insertions(+), 385 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/properties/UniformTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/properties/UniqueTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java index c7e6030e13..2b0c1f5c91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java @@ -136,6 +136,10 @@ public class FunctionalDependencies { uniqueSet.add(slotSet); } + public void addUniqueSlot(FunctionalDependencies functionalDependencies) { + uniqueSet.add(functionalDependencies.uniqueSet); + } + public void addFdItems(ImmutableSet items) { fdItems = ImmutableSet.copyOf(items); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java index 286a92aab7..4747aa8489 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java @@ -179,7 +179,7 @@ public abstract class AbstractPlan extends AbstractTreeNode implements Pla } else { Supplier> outputSupplier = Suppliers.memoize(this::computeOutput); Supplier fdSupplier = () -> this instanceof LogicalPlan - ? ((LogicalPlan) this).computeFuncDeps(outputSupplier) + ? ((LogicalPlan) this).computeFuncDeps() : FunctionalDependencies.EMPTY_FUNC_DEPS; return new LogicalProperties(outputSupplier, fdSupplier); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java index 07804587e3..a679ad0f7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java @@ -19,25 +19,31 @@ package org.apache.doris.nereids.trees.plans; import org.apache.doris.nereids.properties.FdItem; import org.apache.doris.nereids.properties.FunctionalDependencies; -import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import com.google.common.collect.ImmutableSet; -import java.util.List; -import java.util.function.Supplier; - /** * Block fd propagation, it always returns an empty fd */ public interface BlockFuncDepsPropagation extends LogicalPlan { @Override - default FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + default FunctionalDependencies computeFuncDeps() { return FunctionalDependencies.EMPTY_FUNC_DEPS; } @Override - default ImmutableSet computeFdItems(Supplier> outputSupplier) { + default ImmutableSet computeFdItems() { return ImmutableSet.of(); } + + @Override + default void computeUnique(FunctionalDependencies.Builder fdBuilder) { + // don't generate + } + + @Override + default void computeUniform(FunctionalDependencies.Builder fdBuilder) { + // don't generate + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java index c344f22ca2..f37059049a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java @@ -19,20 +19,16 @@ package org.apache.doris.nereids.trees.plans; import org.apache.doris.nereids.properties.FdItem; import org.apache.doris.nereids.properties.FunctionalDependencies; -import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import com.google.common.collect.ImmutableSet; -import java.util.List; -import java.util.function.Supplier; - /** * Propagate fd, keep children's fd */ public interface PropagateFuncDeps extends LogicalPlan { @Override - default FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + default FunctionalDependencies computeFuncDeps() { if (children().size() == 1) { // Note when changing function dependencies, we always clone it. // So it's safe to return a reference @@ -42,13 +38,13 @@ public interface PropagateFuncDeps extends LogicalPlan { children().stream() .map(p -> p.getLogicalProperties().getFunctionalDependencies()) .forEach(builder::addFunctionalDependencies); - ImmutableSet fdItems = computeFdItems(outputSupplier); + ImmutableSet fdItems = computeFdItems(); builder.addFdItems(fdItems); return builder.build(); } @Override - default ImmutableSet computeFdItems(Supplier> outputSupplier) { + default ImmutableSet computeFdItems() { if (children().size() == 1) { // Note when changing function dependencies, we always clone it. // So it's safe to return a reference @@ -60,4 +56,14 @@ public interface PropagateFuncDeps extends LogicalPlan { .forEach(builder::addAll); return builder.build(); } + + @Override + default void computeUnique(FunctionalDependencies.Builder fdBuilder) { + fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies()); + } + + @Override + default void computeUniform(FunctionalDependencies.Builder fdBuilder) { + fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index fa4f891e7a..1e5e5a45ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -21,17 +21,13 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.FdFactory; import org.apache.doris.nereids.properties.FdItem; import org.apache.doris.nereids.properties.FunctionalDependencies; -import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.TableFdItem; -import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.VirtualSlotReference; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; -import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.agg.Ndv; import org.apache.doris.nereids.trees.plans.Plan; @@ -45,12 +41,9 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.Set; -import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -300,80 +293,85 @@ public class LogicalAggregate hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), normalizedChild); } - private void updateFuncDepsGroupByUnique(NamedExpression namedExpression, Builder fdBuilder) { - if (ExpressionUtils.isInjective(namedExpression)) { - fdBuilder.addUniqueSlot(ImmutableSet.copyOf(namedExpression.getInputSlots())); - return; + private boolean isUniqueGroupByUnique(NamedExpression namedExpression) { + if (namedExpression.children().size() != 1) { + return false; } + Expression agg = namedExpression.child(0); + return ExpressionUtils.isInjectiveAgg(agg) + && child().getLogicalProperties().getFunctionalDependencies().isUniqueAndNotNull(agg.getInputSlots()); + } - if (!(namedExpression instanceof Alias && namedExpression.child(0) instanceof AggregateFunction)) { - return; - } - - AggregateFunction agg = (AggregateFunction) namedExpression.child(0); - if (agg instanceof Count || agg instanceof Ndv) { - fdBuilder.addUniformSlot(namedExpression.toSlot()); - return; - } - - if (ExpressionUtils.isInjectiveAgg(agg) - && child().getLogicalProperties().getFunctionalDependencies().isUniqueAndNotNull(agg.getInputSlots())) { - fdBuilder.addUniqueSlot(namedExpression.toSlot()); + private boolean isUniformGroupByUnique(NamedExpression namedExpression) { + if (namedExpression.children().size() != 1) { + return false; } + Expression agg = namedExpression.child(0); + return agg instanceof Count || agg instanceof Ndv; } @Override - public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + public void computeUnique(FunctionalDependencies.Builder fdBuilder) { + if (this.sourceRepeat.isPresent()) { + // roll up may generate new data + return; + } FunctionalDependencies childFd = child(0).getLogicalProperties().getFunctionalDependencies(); - Set outputSet = new HashSet<>(outputSupplier.get()); - Builder fdBuilder = new Builder(); - // when group by all tuples, the result only have one row - if (groupByExpressions.isEmpty()) { - outputSet.forEach(s -> { - fdBuilder.addUniformSlot(s); - fdBuilder.addUniqueSlot(s); - }); - return fdBuilder.build(); - } - - // when group by complicate expression or virtual slot, just propagate uniform slots - if (groupByExpressions.stream() - .anyMatch(s -> !(s instanceof SlotReference) || s instanceof VirtualSlotReference)) { - fdBuilder.addUniformSlot(childFd); - fdBuilder.pruneSlots(outputSet); - return fdBuilder.build(); - } - - // when group by uniform slot, the result only have one row ImmutableSet groupByKeys = groupByExpressions.stream() .map(s -> (Slot) s) .collect(ImmutableSet.toImmutableSet()); - if (childFd.isUniformAndNotNull(groupByKeys)) { - outputSupplier.get().forEach(s -> { - fdBuilder.addUniformSlot(s); - fdBuilder.addUniqueSlot(s); - }); + // when group by all tuples, the result only have one row + if (groupByExpressions.isEmpty() || childFd.isUniformAndNotNull(groupByKeys)) { + getOutput().forEach(fdBuilder::addUniqueSlot); + return; } - // when group by unique slot, the result depends on agg func - if (childFd.isUniqueAndNotNull(groupByKeys)) { - for (NamedExpression namedExpression : getOutputExpressions()) { - updateFuncDepsGroupByUnique(namedExpression, fdBuilder); - } - } + // propagate all unique slots + fdBuilder.addUniqueSlot(childFd); // group by keys is unique fdBuilder.addUniqueSlot(groupByKeys); - fdBuilder.pruneSlots(outputSet); - ImmutableSet fdItems = computeFdItems(outputSupplier); - fdBuilder.addFdItems(fdItems); - - return fdBuilder.build(); + // group by unique may has unique aggregate result + if (childFd.isUniqueAndNotNull(groupByKeys)) { + for (NamedExpression namedExpression : getOutputExpressions()) { + if (isUniqueGroupByUnique(namedExpression)) { + fdBuilder.addUniqueSlot(namedExpression.toSlot()); + } + } + } } @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { + public void computeUniform(FunctionalDependencies.Builder fdBuilder) { + // always propagate uniform + FunctionalDependencies childFd = child(0).getLogicalProperties().getFunctionalDependencies(); + fdBuilder.addUniformSlot(childFd); + + if (this.sourceRepeat.isPresent()) { + // roll up may generate new data + return; + } + ImmutableSet groupByKeys = groupByExpressions.stream() + .map(s -> (Slot) s) + .collect(ImmutableSet.toImmutableSet()); + // when group by all tuples, the result only have one row + if (groupByExpressions.isEmpty() || childFd.isUniformAndNotNull(groupByKeys)) { + getOutput().forEach(fdBuilder::addUniformSlot); + return; + } + + if (childFd.isUniqueAndNotNull(groupByKeys)) { + for (NamedExpression namedExpression : getOutputExpressions()) { + if (isUniformGroupByUnique(namedExpression)) { + fdBuilder.addUniformSlot(namedExpression.toSlot()); + } + } + } + } + + @Override + public ImmutableSet computeFdItems() { ImmutableSet.Builder builder = ImmutableSet.builder(); ImmutableSet groupByExprs = getGroupByExpressions().stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java index 0279c9701c..cad8d6e14d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java @@ -18,18 +18,21 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.FdItem; +import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.AssertNumRowsElement; +import org.apache.doris.nereids.trees.expressions.AssertNumRowsElement.Assertion; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; -import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Objects; @@ -42,8 +45,7 @@ import java.util.stream.Collectors; * If the number of rows is more than the desired num of rows, the query will be cancelled. * The cancelled reason will be reported by Backend and displayed back to the user. */ -public class LogicalAssertNumRows extends LogicalUnary implements - PropagateFuncDeps { +public class LogicalAssertNumRows extends LogicalUnary { private final AssertNumRowsElement assertNumRowsElement; @@ -118,4 +120,29 @@ public class LogicalAssertNumRows extends LogicalUnary< public List computeOutput() { return child().getOutput().stream().map(o -> o.withNullable(true)).collect(Collectors.toList()); } + + @Override + public ImmutableSet computeFdItems() { + return ImmutableSet.of(); + } + + @Override + public void computeUnique(Builder fdBuilder) { + if (assertNumRowsElement.getDesiredNumOfRows() == 1 + && (assertNumRowsElement.getAssertion() == Assertion.EQ + || assertNumRowsElement.getAssertion() == Assertion.LT + || assertNumRowsElement.getAssertion() == Assertion.LE)) { + getOutput().forEach(fdBuilder::addUniqueSlot); + } + } + + @Override + public void computeUniform(Builder fdBuilder) { + if (assertNumRowsElement.getDesiredNumOfRows() == 1 + && (assertNumRowsElement.getAssertion() == Assertion.EQ + || assertNumRowsElement.getAssertion() == Assertion.LT + || assertNumRowsElement.getAssertion() == Assertion.LE)) { + getOutput().forEach(fdBuilder::addUniformSlot); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java index 5f7982aae4..433feb741b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java @@ -49,7 +49,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.function.Supplier; /** * abstract class catalog relation for logical relation @@ -128,9 +127,16 @@ public abstract class LogicalCatalogRelation extends LogicalRelation implements } @Override - public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { + public FunctionalDependencies computeFuncDeps() { Builder fdBuilder = new Builder(); - Set outputSet = Utils.fastToImmutableSet(outputSupplier.get()); + computeUnique(fdBuilder); + fdBuilder.addFdItems(computeFdItems(Utils.fastToImmutableSet(getOutputSet()))); + return fdBuilder.build(); + } + + @Override + public void computeUnique(FunctionalDependencies.Builder fdBuilder) { + Set outputSet = Utils.fastToImmutableSet(getOutputSet()); if (table instanceof OlapTable && ((OlapTable) table).getKeysType().isAggregationFamily()) { ImmutableSet.Builder uniqSlots = ImmutableSet.builderWithExpectedSize(outputSet.size()); for (Slot slot : outputSet) { @@ -154,13 +160,16 @@ public abstract class LogicalCatalogRelation extends LogicalRelation implements Set columns = c.getUniqueKeys(table); fdBuilder.addUniqueSlot((ImmutableSet) findSlotsByColumn(outputSet, columns)); } - fdBuilder.addFdItems(computeFdItems(outputSet)); - return fdBuilder.build(); } @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { - return computeFdItems(Utils.fastToImmutableSet(outputSupplier.get())); + public void computeUniform(FunctionalDependencies.Builder fdBuilder) { + // No uniform slot for catalog relation + } + + @Override + public ImmutableSet computeFdItems() { + return computeFdItems(Utils.fastToImmutableSet(getOutputSet())); } private ImmutableSet computeFdItems(Set outputSet) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java index 5bf0afe1fa..06c03f17f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java @@ -22,7 +22,6 @@ import org.apache.doris.nereids.properties.ExprFdItem; import org.apache.doris.nereids.properties.FdFactory; import org.apache.doris.nereids.properties.FdItem; import org.apache.doris.nereids.properties.FunctionalDependencies; -import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -43,7 +42,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.function.Supplier; /** * use for defer materialize top n @@ -120,26 +118,29 @@ public class LogicalDeferMaterializeTopN extends Logica } @Override - public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { - FunctionalDependencies fd = child(0).getLogicalProperties().getFunctionalDependencies(); + public void computeUnique(FunctionalDependencies.Builder fdBuilder) { if (getLimit() == 1) { - Builder builder = new Builder(); - List output = outputSupplier.get(); - output.forEach(builder::addUniformSlot); - output.forEach(builder::addUniqueSlot); - ImmutableSet fdItems = computeFdItems(outputSupplier); - builder.addFdItems(fdItems); - fd = builder.build(); + getOutput().forEach(fdBuilder::addUniqueSlot); + } else { + fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies()); } - return fd; } @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { + public void computeUniform(FunctionalDependencies.Builder fdBuilder) { + if (getLimit() == 1) { + getOutput().forEach(fdBuilder::addUniformSlot); + } else { + fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); + } + } + + @Override + public ImmutableSet computeFdItems() { ImmutableSet fdItems = child(0).getLogicalProperties().getFunctionalDependencies().getFdItems(); if (getLimit() == 1) { ImmutableSet.Builder builder = ImmutableSet.builder(); - List output = outputSupplier.get(); + List output = getOutput(); ImmutableSet slotSet = output.stream() .filter(SlotReference.class::isInstance) .map(SlotReference.class::cast) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java index 707da68fe8..d022395441 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java @@ -21,7 +21,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.ExprFdItem; import org.apache.doris.nereids.properties.FdFactory; import org.apache.doris.nereids.properties.FdItem; -import org.apache.doris.nereids.properties.FunctionalDependencies; +import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -39,7 +39,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.function.Supplier; /** * Logical Except. @@ -109,29 +108,8 @@ public class LogicalExcept extends LogicalSetOperation { } @Override - public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { - FunctionalDependencies.Builder builder = new FunctionalDependencies - .Builder(child(0).getLogicalProperties().getFunctionalDependencies()); - Map replaceMap = new HashMap<>(); - List output = outputSupplier.get(); - List originalOutputs = regularChildrenOutputs.isEmpty() - ? child(0).getOutput() - : regularChildrenOutputs.get(0); - for (int i = 0; i < output.size(); i++) { - replaceMap.put(originalOutputs.get(i), output.get(i)); - } - builder.replace(replaceMap); - if (qualifier == Qualifier.DISTINCT) { - builder.addUniqueSlot(ImmutableSet.copyOf(outputSupplier.get())); - } - ImmutableSet fdItems = computeFdItems(outputSupplier); - builder.addFdItems(fdItems); - return builder.build(); - } - - @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { - Set output = ImmutableSet.copyOf(outputSupplier.get()); + public ImmutableSet computeFdItems() { + Set output = ImmutableSet.copyOf(getOutput()); ImmutableSet.Builder builder = ImmutableSet.builder(); ImmutableSet exprs = output.stream() @@ -152,4 +130,35 @@ public class LogicalExcept extends LogicalSetOperation { return builder.build(); } + + @Override + public void computeUnique(Builder fdBuilder) { + fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies()); + if (qualifier == Qualifier.DISTINCT) { + fdBuilder.addUniqueSlot(ImmutableSet.copyOf(getOutput())); + } + Map replaceMap = new HashMap<>(); + List output = getOutput(); + List originalOutputs = regularChildrenOutputs.isEmpty() + ? child(0).getOutput() + : regularChildrenOutputs.get(0); + for (int i = 0; i < output.size(); i++) { + replaceMap.put(originalOutputs.get(i), output.get(i)); + } + fdBuilder.replace(replaceMap); + } + + @Override + public void computeUniform(Builder fdBuilder) { + fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); + Map replaceMap = new HashMap<>(); + List output = getOutput(); + List originalOutputs = regularChildrenOutputs.isEmpty() + ? child(0).getOutput() + : regularChildrenOutputs.get(0); + for (int i = 0; i < output.size(); i++) { + replaceMap.put(originalOutputs.get(i), output.get(i)); + } + fdBuilder.replace(replaceMap); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java index 4375124a22..69d378fd6a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java @@ -19,7 +19,6 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.FdItem; -import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; @@ -41,7 +40,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -149,17 +147,7 @@ public class LogicalFilter extends LogicalUnary> outputSupplier) { - Builder fdBuilder = new Builder( - child().getLogicalProperties().getFunctionalDependencies()); - getConjuncts().forEach(e -> fdBuilder.addUniformSlot(ExpressionUtils.extractUniformSlot(e))); - ImmutableSet fdItems = computeFdItems(outputSupplier); - fdBuilder.addFdItems(fdItems); - return fdBuilder.build(); - } - - @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { + public ImmutableSet computeFdItems() { ImmutableSet.Builder builder = ImmutableSet.builder(); ImmutableSet childItems = child().getLogicalProperties().getFunctionalDependencies().getFdItems(); @@ -167,4 +155,15 @@ public class LogicalFilter extends LogicalUnary fdBuilder.addUniformSlot(ExpressionUtils.extractUniformSlot(e))); + fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java index 0fb3964e51..a54a7514db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java @@ -19,7 +19,7 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.FdItem; -import org.apache.doris.nereids.properties.FunctionalDependencies; +import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -38,7 +38,6 @@ import com.google.common.collect.Lists; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.function.Supplier; /** * plan for table generator, the statement like: SELECT * FROM tbl LATERAL VIEW EXPLODE(c1) g as (gc1); @@ -157,16 +156,17 @@ public class LogicalGenerate extends LogicalUnary> outputSupplier) { - FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder(); - builder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); - ImmutableSet fdItems = computeFdItems(outputSupplier); - builder.addFdItems(fdItems); - return builder.build(); + public ImmutableSet computeFdItems() { + return ImmutableSet.of(); } @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { - return ImmutableSet.of(); + public void computeUnique(Builder fdBuilder) { + // don't generate and propagate unique + } + + @Override + public void computeUniform(Builder fdBuilder) { + fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java index e4d01f1274..da526c33af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java @@ -19,7 +19,6 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.FdItem; -import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; @@ -39,7 +38,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.function.Supplier; /** * Logical Having plan @@ -120,17 +118,18 @@ public class LogicalHaving extends LogicalUnary> outputSupplier) { - Builder fdBuilder = new Builder( - child().getLogicalProperties().getFunctionalDependencies()); - getConjuncts().forEach(e -> fdBuilder.addUniformSlot(ExpressionUtils.extractUniformSlot(e))); - ImmutableSet fdItems = computeFdItems(outputSupplier); - fdBuilder.addFdItems(fdItems); - return fdBuilder.build(); + public void computeUnique(Builder fdBuilder) { + fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies()); } @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { + public void computeUniform(Builder fdBuilder) { + getConjuncts().forEach(e -> fdBuilder.addUniformSlot(ExpressionUtils.extractUniformSlot(e))); + fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); + } + + @Override + public ImmutableSet computeFdItems() { ImmutableSet.Builder builder = ImmutableSet.builder(); ImmutableSet childItems = child().getLogicalProperties().getFunctionalDependencies().getFdItems(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java index 2fbdbd2456..e9e4889a8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.properties.ExprFdItem; import org.apache.doris.nereids.properties.FdFactory; import org.apache.doris.nereids.properties.FdItem; import org.apache.doris.nereids.properties.FunctionalDependencies; +import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -39,7 +40,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.function.Supplier; /** * Logical Intersect. @@ -119,24 +119,29 @@ public class LogicalIntersect extends LogicalSetOperation { } @Override - public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { - FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder(); + public void computeUnique(Builder fdBuilder) { for (Plan child : children) { - builder.addFunctionalDependencies( + fdBuilder.addUniqueSlot( child.getLogicalProperties().getFunctionalDependencies()); - replaceSlotInFuncDeps(builder, child.getOutput(), outputSupplier.get()); + replaceSlotInFuncDeps(fdBuilder, child.getOutput(), getOutput()); } if (qualifier == Qualifier.DISTINCT) { - builder.addUniqueSlot(ImmutableSet.copyOf(outputSupplier.get())); + fdBuilder.addUniqueSlot(ImmutableSet.copyOf(getOutput())); } - ImmutableSet fdItems = computeFdItems(outputSupplier); - builder.addFdItems(fdItems); - return builder.build(); } @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { - Set output = ImmutableSet.copyOf(outputSupplier.get()); + public void computeUniform(Builder fdBuilder) { + for (Plan child : children) { + fdBuilder.addUniformSlot( + child.getLogicalProperties().getFunctionalDependencies()); + replaceSlotInFuncDeps(fdBuilder, child.getOutput(), getOutput()); + } + } + + @Override + public ImmutableSet computeFdItems() { + Set output = ImmutableSet.copyOf(getOutput()); ImmutableSet.Builder builder = ImmutableSet.builder(); ImmutableSet exprs = output.stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java index d18a454fe7..ea92aeca22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.FdFactory; import org.apache.doris.nereids.properties.FdItem; -import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.TableFdItem; @@ -58,7 +57,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nullable; @@ -460,76 +458,7 @@ public class LogicalJoin> outputSupplier) { - if (isMarkJoin()) { - // TODO disable function dependence calculation for mark join, but need re-think this in future. - return FunctionalDependencies.EMPTY_FUNC_DEPS; - } - //1. NALAJ and FOJ block functional dependencies - if (joinType.isNullAwareLeftAntiJoin() || joinType.isFullOuterJoin()) { - return FunctionalDependencies.EMPTY_FUNC_DEPS; - } - - // left/right semi/anti join propagate left/right functional dependencies - if (joinType.isLeftAntiJoin() || joinType.isLefSemiJoin()) { - return left().getLogicalProperties().getFunctionalDependencies(); - } - if (joinType.isRightSemiJoin() || joinType.isRightAntiJoin()) { - return right().getLogicalProperties().getFunctionalDependencies(); - } - - // if there is non-equal join conditions, block functional dependencies - if (!otherJoinConjuncts.isEmpty()) { - return FunctionalDependencies.EMPTY_FUNC_DEPS; - } - - Pair, Set> keys = extractNullRejectHashKeys(); - if (keys == null) { - return FunctionalDependencies.EMPTY_FUNC_DEPS; - } - - // Note here we only check whether the left is unique. - // So the hash condition can't be null-safe - // TODO: consider Null-safe hash condition when left and rigth is not nullable - boolean isLeftUnique = left().getLogicalProperties() - .getFunctionalDependencies().isUnique(keys.first); - boolean isRightUnique = right().getLogicalProperties() - .getFunctionalDependencies().isUnique(keys.second); - Builder fdBuilder = new Builder(); - if (joinType.isInnerJoin()) { - // inner join propagate uniforms slots - // And if the hash keys is unique, inner join can propagate all functional dependencies - if (isLeftUnique && isRightUnique) { - fdBuilder.addFunctionalDependencies(left().getLogicalProperties().getFunctionalDependencies()); - fdBuilder.addFunctionalDependencies(right().getLogicalProperties().getFunctionalDependencies()); - } else { - fdBuilder.addUniformSlot(left().getLogicalProperties().getFunctionalDependencies()); - fdBuilder.addUniformSlot(right().getLogicalProperties().getFunctionalDependencies()); - } - } - - // left/right outer join propagate left/right uniforms slots - // And if the right/left hash keys is unique, - // join can propagate left/right functional dependencies - if (joinType.isLeftOuterJoin()) { - if (isRightUnique) { - return left().getLogicalProperties().getFunctionalDependencies(); - } - fdBuilder.addUniformSlot(left().getLogicalProperties().getFunctionalDependencies()); - } - if (joinType.isRightOuterJoin()) { - if (isLeftUnique) { - return left().getLogicalProperties().getFunctionalDependencies(); - } - fdBuilder.addUniformSlot(left().getLogicalProperties().getFunctionalDependencies()); - } - ImmutableSet fdItems = computeFdItems(outputSupplier); - fdBuilder.addFdItems(fdItems); - return fdBuilder.build(); - } - - @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { + public ImmutableSet computeFdItems() { ImmutableSet.Builder builder = ImmutableSet.builder(); if (isMarkJoin() || joinType.isNullAwareLeftAntiJoin() || joinType.isFullOuterJoin() @@ -710,4 +639,61 @@ public class LogicalJoin, Set> keys = extractNullRejectHashKeys(); + if (keys == null) { + return; + } + + // Note here we only check whether the left is unique. + // So the hash condition can't be null-safe + // TODO: consider Null-safe hash condition when left and rigth is not nullable + boolean isLeftUnique = left().getLogicalProperties() + .getFunctionalDependencies().isUnique(keys.first); + boolean isRightUnique = right().getLogicalProperties() + .getFunctionalDependencies().isUnique(keys.second); + + // left/right outer join propagate left/right uniforms slots + // And if the right/left hash keys is unique, + // join can propagate left/right functional dependencies + if (joinType.isLeftOuterJoin() && isRightUnique) { + fdBuilder.addUniqueSlot(left().getLogicalProperties().getFunctionalDependencies()); + } else if (joinType.isRightOuterJoin() && isLeftUnique) { + fdBuilder.addUniqueSlot(right().getLogicalProperties().getFunctionalDependencies()); + } else if (joinType.isInnerJoin() && isLeftUnique && isRightUnique) { + // inner join propagate uniforms slots + // And if the hash keys is unique, inner join can propagate all functional dependencies + fdBuilder.addFunctionalDependencies(left().getLogicalProperties().getFunctionalDependencies()); + fdBuilder.addFunctionalDependencies(right().getLogicalProperties().getFunctionalDependencies()); + } + } + + @Override + public void computeUniform(Builder fdBuilder) { + if (isMarkJoin()) { + // TODO disable function dependence calculation for mark join, but need re-think this in future. + return; + } + if (!joinType.isLeftSemiOrAntiJoin()) { + fdBuilder.addUniformSlot(right().getLogicalProperties().getFunctionalDependencies()); + } + if (!joinType.isRightSemiOrAntiJoin()) { + fdBuilder.addUniformSlot(left().getLogicalProperties().getFunctionalDependencies()); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java index 482c7e2605..02558fe2ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java @@ -22,7 +22,6 @@ import org.apache.doris.nereids.properties.ExprFdItem; import org.apache.doris.nereids.properties.FdFactory; import org.apache.doris.nereids.properties.FdItem; import org.apache.doris.nereids.properties.FunctionalDependencies; -import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -41,7 +40,6 @@ import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.function.Supplier; /** * Logical limit plan @@ -150,25 +148,29 @@ public class LogicalLimit extends LogicalUnary> outputSupplier) { - FunctionalDependencies fd = child(0).getLogicalProperties().getFunctionalDependencies(); - if (getLimit() == 1 && !phase.isLocal()) { - Builder builder = new Builder(); - outputSupplier.get().forEach(builder::addUniformSlot); - outputSupplier.get().forEach(builder::addUniqueSlot); - ImmutableSet fdItems = computeFdItems(outputSupplier); - builder.addFdItems(fdItems); - fd = builder.build(); + public void computeUnique(FunctionalDependencies.Builder fdBuilder) { + if (getLimit() == 1) { + getOutput().forEach(fdBuilder::addUniqueSlot); + } else { + fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies()); } - return fd; } @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { + public void computeUniform(FunctionalDependencies.Builder fdBuilder) { + if (getLimit() == 1) { + getOutput().forEach(fdBuilder::addUniformSlot); + } else { + fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); + } + } + + @Override + public ImmutableSet computeFdItems() { ImmutableSet fdItems = child(0).getLogicalProperties().getFunctionalDependencies().getFdItems(); if (getLimit() == 1 && !phase.isLocal()) { ImmutableSet.Builder builder = ImmutableSet.builder(); - List output = outputSupplier.get(); + List output = getOutput(); ImmutableSet slotSet = output.stream() .filter(SlotReference.class::isInstance) .map(SlotReference.class::cast) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java index d338be5cc8..78a1e9a3a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java @@ -41,7 +41,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.function.Supplier; /** * A relation that contains only one row consist of some constant expressions. @@ -136,20 +135,18 @@ public class LogicalOneRowRelation extends LogicalRelation implements OneRowRela } @Override - public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { - FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder(); - outputSupplier.get().forEach(s -> { - builder.addUniformSlot(s); - builder.addUniqueSlot(s); - }); - ImmutableSet fdItems = computeFdItems(outputSupplier); - builder.addFdItems(fdItems); - return builder.build(); + public void computeUnique(FunctionalDependencies.Builder fdBuilder) { + getOutput().forEach(fdBuilder::addUniqueSlot); } @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { - Set output = ImmutableSet.copyOf(outputSupplier.get()); + public void computeUniform(FunctionalDependencies.Builder fdBuilder) { + getOutput().forEach(fdBuilder::addUniformSlot); + } + + @Override + public ImmutableSet computeFdItems() { + Set output = ImmutableSet.copyOf(getOutput()); ImmutableSet.Builder builder = ImmutableSet.builder(); ImmutableSet slotSet = output.stream() .filter(SlotReference.class::isInstance) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java index 860bd263ac..c0fb6ae736 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java @@ -19,13 +19,11 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.properties.FdItem; import org.apache.doris.nereids.properties.FunctionalDependencies; -import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import java.util.List; import java.util.Optional; import java.util.function.BiFunction; import java.util.function.Supplier; @@ -61,7 +59,18 @@ public interface LogicalPlan extends Plan { * - BlockFDPropagation: clean the fd * - PropagateFD: propagate the fd */ - FunctionalDependencies computeFuncDeps(Supplier> outputSupplier); + default FunctionalDependencies computeFuncDeps() { + FunctionalDependencies.Builder fdBuilder = new FunctionalDependencies.Builder(); + computeUniform(fdBuilder); + computeUnique(fdBuilder); + ImmutableSet fdItems = computeFdItems(); + fdBuilder.addFdItems(fdItems); + return fdBuilder.build(); + } - ImmutableSet computeFdItems(Supplier> outputSupplier); + ImmutableSet computeFdItems(); + + void computeUnique(FunctionalDependencies.Builder fdBuilder); + + void computeUniform(FunctionalDependencies.Builder fdBuilder); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index 89dd7d4967..ebecee0d3a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.FdItem; import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.BoundStar; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -42,11 +41,9 @@ import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableSet; import org.json.JSONObject; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.function.Supplier; /** * Logical project plan. @@ -239,31 +236,7 @@ public class LogicalProject extends LogicalUnary> outputSupplier) { - FunctionalDependencies childFuncDeps = child().getLogicalProperties().getFunctionalDependencies(); - FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder(childFuncDeps); - builder.pruneSlots(new HashSet<>(outputSupplier.get())); - projects.stream().filter(Alias.class::isInstance).forEach(proj -> { - if (proj.child(0).isConstant()) { - builder.addUniformSlot(proj.toSlot()); - } else if (proj.child(0) instanceof Uuid) { - builder.addUniqueSlot(proj.toSlot()); - } else if (ExpressionUtils.isInjective(proj.child(0))) { - ImmutableSet inputs = ImmutableSet.copyOf(proj.getInputSlots()); - if (childFuncDeps.isUnique(inputs)) { - builder.addUniqueSlot(proj.toSlot()); - } else if (childFuncDeps.isUniform(inputs)) { - builder.addUniformSlot(proj.toSlot()); - } - } - }); - ImmutableSet fdItems = computeFdItems(outputSupplier); - builder.addFdItems(fdItems); - return builder.build(); - } - - @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { + public ImmutableSet computeFdItems() { ImmutableSet.Builder builder = ImmutableSet.builder(); ImmutableSet childItems = child().getLogicalProperties().getFunctionalDependencies().getFdItems(); @@ -271,4 +244,42 @@ public class LogicalProject extends LogicalUnary inputs = ImmutableSet.copyOf(proj.getInputSlots()); + if (child(0).getLogicalProperties().getFunctionalDependencies().isUnique(inputs)) { + fdBuilder.addUniqueSlot(proj.toSlot()); + } + } + } + fdBuilder.pruneSlots(getOutputSet()); + } + + @Override + public void computeUniform(FunctionalDependencies.Builder fdBuilder) { + fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); + for (NamedExpression proj : getProjects()) { + if (proj.children().isEmpty()) { + continue; + } + if (proj.child(0).isConstant()) { + fdBuilder.addUniformSlot(proj.toSlot()); + } else if (ExpressionUtils.isInjective(proj.child(0))) { + ImmutableSet inputs = ImmutableSet.copyOf(proj.getInputSlots()); + if (child(0).getLogicalProperties().getFunctionalDependencies().isUniform(inputs)) { + fdBuilder.addUniformSlot(proj.toSlot()); + } + } + } + fdBuilder.pruneSlots(getOutputSet()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java index 5e0311afe1..95ae92b686 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java @@ -39,7 +39,6 @@ import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.function.Supplier; /** * LogicalRepeat. @@ -187,19 +186,17 @@ public class LogicalRepeat extends LogicalUnary> outputSupplier) { - FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder(); - // Note uniform does not reject nullable slots - outputSupplier.get().stream() - .filter(child(0).getLogicalProperties().getFunctionalDependencies()::isUniform) - .forEach(builder::addUniformSlot); - ImmutableSet fdItems = computeFdItems(outputSupplier); - builder.addFdItems(fdItems); - return builder.build(); + public void computeUnique(FunctionalDependencies.Builder fdBuilder) { + // don't generate unique slot } @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { + public void computeUniform(FunctionalDependencies.Builder fdBuilder) { + fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); + } + + @Override + public ImmutableSet computeFdItems() { ImmutableSet.Builder builder = ImmutableSet.builder(); ImmutableSet childItems = child().getLogicalProperties().getFunctionalDependencies().getFdItems(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java index 68a451f1df..ea9fd143c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java @@ -41,7 +41,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.function.Supplier; /** * The node of logical plan for sub query and alias @@ -165,22 +164,29 @@ public class LogicalSubQueryAlias extends LogicalUnary< } @Override - public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { - FunctionalDependencies.Builder builder = new FunctionalDependencies - .Builder(child(0).getLogicalProperties().getFunctionalDependencies()); + public void computeUnique(FunctionalDependencies.Builder fdBuilder) { + fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies()); Map replaceMap = new HashMap<>(); - List outputs = outputSupplier.get(); + List outputs = getOutput(); for (int i = 0; i < outputs.size(); i++) { replaceMap.put(child(0).getOutput().get(i), outputs.get(i)); } - builder.replace(replaceMap); - ImmutableSet fdItems = computeFdItems(outputSupplier); - builder.addFdItems(fdItems); - return builder.build(); + fdBuilder.replace(replaceMap); } @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { + public void computeUniform(FunctionalDependencies.Builder fdBuilder) { + fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); + Map replaceMap = new HashMap<>(); + List outputs = getOutput(); + for (int i = 0; i < outputs.size(); i++) { + replaceMap.put(child(0).getOutput().get(i), outputs.get(i)); + } + fdBuilder.replace(replaceMap); + } + + @Override + public ImmutableSet computeFdItems() { // TODO: inherit from child with replaceMap return ImmutableSet.of(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java index 1fb5dbaab7..791ded58cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java @@ -22,7 +22,6 @@ import org.apache.doris.nereids.properties.ExprFdItem; import org.apache.doris.nereids.properties.FdFactory; import org.apache.doris.nereids.properties.FdItem; import org.apache.doris.nereids.properties.FunctionalDependencies; -import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.trees.expressions.Expression; @@ -35,6 +34,7 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -42,7 +42,6 @@ import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.function.Supplier; /** * Logical top-N plan. @@ -171,26 +170,29 @@ public class LogicalTopN extends LogicalUnary> outputSupplier) { - FunctionalDependencies fd = child(0).getLogicalProperties().getFunctionalDependencies(); + public void computeUnique(FunctionalDependencies.Builder fdBuilder) { if (getLimit() == 1) { - Builder builder = new Builder(); - List output = outputSupplier.get(); - output.forEach(builder::addUniformSlot); - output.forEach(builder::addUniqueSlot); - ImmutableSet fdItems = computeFdItems(outputSupplier); - builder.addFdItems(fdItems); - fd = builder.build(); + getOutput().forEach(fdBuilder::addUniqueSlot); + } else { + fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies()); } - return fd; } @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { + public void computeUniform(FunctionalDependencies.Builder fdBuilder) { + if (getLimit() == 1) { + getOutput().forEach(fdBuilder::addUniformSlot); + } else { + fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); + } + } + + @Override + public ImmutableSet computeFdItems() { ImmutableSet fdItems = child(0).getLogicalProperties().getFunctionalDependencies().getFdItems(); if (getLimit() == 1) { ImmutableSet.Builder builder = ImmutableSet.builder(); - List output = outputSupplier.get(); + List output = getOutput(); ImmutableSet slotSet = output.stream() .filter(SlotReference.class::isInstance) .map(SlotReference.class::cast) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java index dac6996c0c..57d944bc2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java @@ -25,7 +25,6 @@ import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -41,7 +40,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.function.Supplier; /** * Logical Union. @@ -187,20 +185,20 @@ public class LogicalUnion extends LogicalSetOperation implements Union, OutputPr } @Override - public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { - if (qualifier != Qualifier.DISTINCT) { - return FunctionalDependencies.EMPTY_FUNC_DEPS; + public void computeUnique(FunctionalDependencies.Builder fdBuilder) { + if (qualifier == Qualifier.DISTINCT) { + fdBuilder.addUniqueSlot(ImmutableSet.copyOf(getOutput())); } - FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder(); - builder.addUniqueSlot(ImmutableSet.copyOf(outputSupplier.get())); - ImmutableSet fdItems = computeFdItems(outputSupplier); - builder.addFdItems(fdItems); - return builder.build(); } @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { - Set output = ImmutableSet.copyOf(outputSupplier.get()); + public void computeUniform(FunctionalDependencies.Builder fdBuilder) { + // don't propagate uniform slots + } + + @Override + public ImmutableSet computeFdItems() { + Set output = ImmutableSet.copyOf(getOutput()); ImmutableSet.Builder builder = ImmutableSet.builder(); ImmutableSet exprs = output.stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java index e8621b1754..200ccc2ffc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java @@ -21,7 +21,7 @@ import org.apache.doris.catalog.View; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.FdItem; -import org.apache.doris.nereids.properties.FunctionalDependencies; +import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -36,7 +36,6 @@ import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.function.Supplier; /** LogicalView */ public class LogicalView extends LogicalUnary { @@ -129,17 +128,22 @@ public class LogicalView extends LogicalUnary { } @Override - public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { - return ((LogicalPlan) child()).computeFuncDeps(outputSupplier); - } - - @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { - return ((LogicalPlan) child()).computeFdItems(outputSupplier); + public ImmutableSet computeFdItems() { + return ((LogicalPlan) child()).computeFdItems(); } @Override public Plan withChildren(List children) { return new LogicalView<>(view, (LogicalPlan) children.get(0)); } + + @Override + public void computeUnique(Builder fdBuilder) { + fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies()); + } + + @Override + public void computeUniform(Builder fdBuilder) { + fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java index 457678787b..b03a336540 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java @@ -19,7 +19,7 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.FdItem; -import org.apache.doris.nereids.properties.FunctionalDependencies; +import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -43,7 +43,6 @@ import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.function.Supplier; /** * logical node to deal with window functions; @@ -228,35 +227,51 @@ public class LogicalWindow extends LogicalUnary partitionKeys = windowExpr.getPartitionKeys(); // Now we only support slot type keys if (!partitionKeys.stream().allMatch(Slot.class::isInstance)) { - return; + return false; } ImmutableSet slotSet = partitionKeys.stream() .map(s -> (Slot) s) .collect(ImmutableSet.toImmutableSet()); + // if partition by keys are uniform or empty, output is unique + if (child(0).getLogicalProperties().getFunctionalDependencies().isUniformAndNotNull(slotSet) + || slotSet.isEmpty()) { + if (windowExpr.getFunction() instanceof RowNumber) { + return true; + } + } + return false; + } + private boolean isUniform(NamedExpression namedExpression) { + if (namedExpression.children().size() != 1 || !(namedExpression.child(0) instanceof WindowExpression)) { + return false; + } + WindowExpression windowExpr = (WindowExpression) namedExpression.child(0); + List partitionKeys = windowExpr.getPartitionKeys(); + // Now we only support slot type keys + if (!partitionKeys.stream().allMatch(Slot.class::isInstance)) { + return false; + } + ImmutableSet slotSet = partitionKeys.stream() + .map(s -> (Slot) s) + .collect(ImmutableSet.toImmutableSet()); // if partition by keys are unique, output is uniform if (child(0).getLogicalProperties().getFunctionalDependencies().isUniqueAndNotNull(slotSet)) { if (windowExpr.getFunction() instanceof RowNumber || windowExpr.getFunction() instanceof Rank || windowExpr.getFunction() instanceof DenseRank) { - builder.addUniformSlot(namedExpression.toSlot()); - } - } - - // if partition by keys are uniform, output is unique - if (child(0).getLogicalProperties().getFunctionalDependencies().isUniformAndNotNull(slotSet)) { - if (windowExpr.getFunction() instanceof RowNumber) { - builder.addUniqueSlot(namedExpression.toSlot()); + return true; } } + return false; } private void updateFuncDepsByWindowExpr(NamedExpression namedExpression, ImmutableSet.Builder builder) { @@ -278,19 +293,7 @@ public class LogicalWindow extends LogicalUnary> outputSupplier) { - FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder( - child(0).getLogicalProperties().getFunctionalDependencies()); - for (NamedExpression namedExpression : windowExpressions) { - updateFuncDepsByWindowExpr(namedExpression, builder); - } - ImmutableSet fdItems = computeFdItems(outputSupplier); - builder.addFdItems(fdItems); - return builder.build(); - } - - @Override - public ImmutableSet computeFdItems(Supplier> outputSupplier) { + public ImmutableSet computeFdItems() { ImmutableSet.Builder builder = ImmutableSet.builder(); ImmutableSet childItems = child().getLogicalProperties().getFunctionalDependencies().getFdItems(); builder.addAll(childItems); @@ -301,4 +304,24 @@ public class LogicalWindow extends LogicalUnary uni.name") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isEmpty()); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isEmpty()); + plan = PlanChecker.from(connectContext) + .analyze("select uni.id, agg.id from agg inner join uni " + + "on agg.id = uni.id and agg.name > uni.name") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0))); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(1))); + } + + @Test + void testOneRowRelation() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select 1") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0))); + } + + @Test + void testProject() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select id as id2 from uni") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0))); + } + + @Test + void testRepeat() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select id from agg group by GROUPING SETS ((id, name), (id))") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isEmpty()); + plan = PlanChecker.from(connectContext) + .analyze("select id from agg group by rollup (id, name)") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isEmpty()); + } + + @Test + void testSubQuery() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select id from (select id from agg) t") + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0))); + } + + @Test + void testAssertRows() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select id from agg where id = (select id from uni)") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0))); + } + + @Test + void testWindow() { + // partition by uniform + Plan plan = PlanChecker.from(connectContext) + .analyze("select id, row_number() over(partition by id) from agg where id =1") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0))); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(1))); + + // partition by None + plan = PlanChecker.from(connectContext) + .analyze("select id, row_number() over() from agg where id =1") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0))); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(1))); + } + +}