database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[refactor](Nereids): compute unique and uniform property respectively (#32908)

Browse Source
This commit is contained in:
谢健
2024-04-11 15:57:38 +08:00
committed by yiguolei
parent 272269f9c1
commit 92d28e497b
28 changed files with 1075 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java
View File

@ -136,6 +136,10 @@ public class FunctionalDependencies {
uniqueSet.add(slotSet);
}
public void addUniqueSlot(FunctionalDependencies functionalDependencies) {
uniqueSet.add(functionalDependencies.uniqueSet);
}
public void addFdItems(ImmutableSet<FdItem> items) {
fdItems = ImmutableSet.copyOf(items);
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java
View File

@ -179,7 +179,7 @@ public abstract class AbstractPlan extends AbstractTreeNode<Plan> implements Pla
} else {
Supplier<List<Slot>> outputSupplier = Suppliers.memoize(this::computeOutput);
Supplier<FunctionalDependencies> fdSupplier = () -> this instanceof LogicalPlan
? ((LogicalPlan) this).computeFuncDeps(outputSupplier)
? ((LogicalPlan) this).computeFuncDeps()
: FunctionalDependencies.EMPTY_FUNC_DEPS;
return new LogicalProperties(outputSupplier, fdSupplier);
}

18
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java
View File

@ -19,25 +19,31 @@ package org.apache.doris.nereids.trees.plans;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.function.Supplier;
/**
* Block fd propagation, it always returns an empty fd
*/
public interface BlockFuncDepsPropagation extends LogicalPlan {
@Override
default FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
default FunctionalDependencies computeFuncDeps() {
return FunctionalDependencies.EMPTY_FUNC_DEPS;
}
@Override
default ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
default ImmutableSet<FdItem> computeFdItems() {
return ImmutableSet.of();
}
@Override
default void computeUnique(FunctionalDependencies.Builder fdBuilder) {
// don't generate
}
@Override
default void computeUniform(FunctionalDependencies.Builder fdBuilder) {
// don't generate
}
}

20
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java
View File

@ -19,20 +19,16 @@ package org.apache.doris.nereids.trees.plans;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.function.Supplier;
/**
* Propagate fd, keep children's fd
*/
public interface PropagateFuncDeps extends LogicalPlan {
@Override
default FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
default FunctionalDependencies computeFuncDeps() {
if (children().size() == 1) {
// Note when changing function dependencies, we always clone it.
// So it's safe to return a reference
@ -42,13 +38,13 @@ public interface PropagateFuncDeps extends LogicalPlan {
children().stream()
.map(p -> p.getLogicalProperties().getFunctionalDependencies())
.forEach(builder::addFunctionalDependencies);
ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
ImmutableSet<FdItem> fdItems = computeFdItems();
builder.addFdItems(fdItems);
return builder.build();
}
@Override
default ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
default ImmutableSet<FdItem> computeFdItems() {
if (children().size() == 1) {
// Note when changing function dependencies, we always clone it.
// So it's safe to return a reference
@ -60,4 +56,14 @@ public interface PropagateFuncDeps extends LogicalPlan {
.forEach(builder::addAll);
return builder.build();
}
@Override
default void computeUnique(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
@Override
default void computeUniform(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
}

122
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java
View File

@ -21,17 +21,13 @@ import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdFactory;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.TableFdItem;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.VirtualSlotReference;
import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
import org.apache.doris.nereids.trees.expressions.functions.agg.Ndv;
import org.apache.doris.nereids.trees.plans.Plan;
@ -45,12 +41,9 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
import java.util.stream.Collectors;
/**
@ -300,80 +293,85 @@ public class LogicalAggregate<CHILD_TYPE extends Plan>
hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), normalizedChild);
}
private void updateFuncDepsGroupByUnique(NamedExpression namedExpression, Builder fdBuilder) {
if (ExpressionUtils.isInjective(namedExpression)) {
fdBuilder.addUniqueSlot(ImmutableSet.copyOf(namedExpression.getInputSlots()));
return;
private boolean isUniqueGroupByUnique(NamedExpression namedExpression) {
if (namedExpression.children().size() != 1) {
return false;
}
Expression agg = namedExpression.child(0);
return ExpressionUtils.isInjectiveAgg(agg)
&& child().getLogicalProperties().getFunctionalDependencies().isUniqueAndNotNull(agg.getInputSlots());
}
if (!(namedExpression instanceof Alias && namedExpression.child(0) instanceof AggregateFunction)) {
return;
}
AggregateFunction agg = (AggregateFunction) namedExpression.child(0);
if (agg instanceof Count || agg instanceof Ndv) {
fdBuilder.addUniformSlot(namedExpression.toSlot());
return;
}
if (ExpressionUtils.isInjectiveAgg(agg)
&& child().getLogicalProperties().getFunctionalDependencies().isUniqueAndNotNull(agg.getInputSlots())) {
fdBuilder.addUniqueSlot(namedExpression.toSlot());
private boolean isUniformGroupByUnique(NamedExpression namedExpression) {
if (namedExpression.children().size() != 1) {
return false;
}
Expression agg = namedExpression.child(0);
return agg instanceof Count || agg instanceof Ndv;
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
if (this.sourceRepeat.isPresent()) {
// roll up may generate new data
return;
}
FunctionalDependencies childFd = child(0).getLogicalProperties().getFunctionalDependencies();
Set<Slot> outputSet = new HashSet<>(outputSupplier.get());
Builder fdBuilder = new Builder();
// when group by all tuples, the result only have one row
if (groupByExpressions.isEmpty()) {
outputSet.forEach(s -> {
fdBuilder.addUniformSlot(s);
fdBuilder.addUniqueSlot(s);
});
return fdBuilder.build();
}
// when group by complicate expression or virtual slot, just propagate uniform slots
if (groupByExpressions.stream()
.anyMatch(s -> !(s instanceof SlotReference) || s instanceof VirtualSlotReference)) {
fdBuilder.addUniformSlot(childFd);
fdBuilder.pruneSlots(outputSet);
return fdBuilder.build();
}
// when group by uniform slot, the result only have one row
ImmutableSet<Slot> groupByKeys = groupByExpressions.stream()
.map(s -> (Slot) s)
.collect(ImmutableSet.toImmutableSet());
if (childFd.isUniformAndNotNull(groupByKeys)) {
outputSupplier.get().forEach(s -> {
fdBuilder.addUniformSlot(s);
fdBuilder.addUniqueSlot(s);
});
// when group by all tuples, the result only have one row
if (groupByExpressions.isEmpty() || childFd.isUniformAndNotNull(groupByKeys)) {
getOutput().forEach(fdBuilder::addUniqueSlot);
return;
}
// when group by unique slot, the result depends on agg func
if (childFd.isUniqueAndNotNull(groupByKeys)) {
for (NamedExpression namedExpression : getOutputExpressions()) {
updateFuncDepsGroupByUnique(namedExpression, fdBuilder);
}
}
// propagate all unique slots
fdBuilder.addUniqueSlot(childFd);
// group by keys is unique
fdBuilder.addUniqueSlot(groupByKeys);
fdBuilder.pruneSlots(outputSet);
ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
fdBuilder.addFdItems(fdItems);
return fdBuilder.build();
// group by unique may has unique aggregate result
if (childFd.isUniqueAndNotNull(groupByKeys)) {
for (NamedExpression namedExpression : getOutputExpressions()) {
if (isUniqueGroupByUnique(namedExpression)) {
fdBuilder.addUniqueSlot(namedExpression.toSlot());
}
}
}
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
// always propagate uniform
FunctionalDependencies childFd = child(0).getLogicalProperties().getFunctionalDependencies();
fdBuilder.addUniformSlot(childFd);
if (this.sourceRepeat.isPresent()) {
// roll up may generate new data
return;
}
ImmutableSet<Slot> groupByKeys = groupByExpressions.stream()
.map(s -> (Slot) s)
.collect(ImmutableSet.toImmutableSet());
// when group by all tuples, the result only have one row
if (groupByExpressions.isEmpty() || childFd.isUniformAndNotNull(groupByKeys)) {
getOutput().forEach(fdBuilder::addUniformSlot);
return;
}
if (childFd.isUniqueAndNotNull(groupByKeys)) {
for (NamedExpression namedExpression : getOutputExpressions()) {
if (isUniformGroupByUnique(namedExpression)) {
fdBuilder.addUniformSlot(namedExpression.toSlot());
}
}
}
}
@Override
public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<SlotReference> groupByExprs = getGroupByExpressions().stream()

33
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java
View File

@ -18,18 +18,21 @@
package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.AssertNumRowsElement;
import org.apache.doris.nereids.trees.expressions.AssertNumRowsElement.Assertion;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.PropagateFuncDeps;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
@ -42,8 +45,7 @@ import java.util.stream.Collectors;
* If the number of rows is more than the desired num of rows, the query will be cancelled.
* The cancelled reason will be reported by Backend and displayed back to the user.
*/
public class LogicalAssertNumRows<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYPE> implements
PropagateFuncDeps {
public class LogicalAssertNumRows<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYPE> {
private final AssertNumRowsElement assertNumRowsElement;
@ -118,4 +120,29 @@ public class LogicalAssertNumRows<CHILD_TYPE extends Plan> extends LogicalUnary<
public List<Slot> computeOutput() {
return child().getOutput().stream().map(o -> o.withNullable(true)).collect(Collectors.toList());
}
@Override
public ImmutableSet<FdItem> computeFdItems() {
return ImmutableSet.of();
}
@Override
public void computeUnique(Builder fdBuilder) {
if (assertNumRowsElement.getDesiredNumOfRows() == 1
&& (assertNumRowsElement.getAssertion() == Assertion.EQ
|| assertNumRowsElement.getAssertion() == Assertion.LT
|| assertNumRowsElement.getAssertion() == Assertion.LE)) {
getOutput().forEach(fdBuilder::addUniqueSlot);
}
}
@Override
public void computeUniform(Builder fdBuilder) {
if (assertNumRowsElement.getDesiredNumOfRows() == 1
&& (assertNumRowsElement.getAssertion() == Assertion.EQ
|| assertNumRowsElement.getAssertion() == Assertion.LT
|| assertNumRowsElement.getAssertion() == Assertion.LE)) {
getOutput().forEach(fdBuilder::addUniformSlot);
}
}
}

23
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java
View File

@ -49,7 +49,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
/**
* abstract class catalog relation for logical relation
@ -128,9 +127,16 @@ public abstract class LogicalCatalogRelation extends LogicalRelation implements
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
public FunctionalDependencies computeFuncDeps() {
Builder fdBuilder = new Builder();
Set<Slot> outputSet = Utils.fastToImmutableSet(outputSupplier.get());
computeUnique(fdBuilder);
fdBuilder.addFdItems(computeFdItems(Utils.fastToImmutableSet(getOutputSet())));
return fdBuilder.build();
}
@Override
public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
Set<Slot> outputSet = Utils.fastToImmutableSet(getOutputSet());
if (table instanceof OlapTable && ((OlapTable) table).getKeysType().isAggregationFamily()) {
ImmutableSet.Builder<Slot> uniqSlots = ImmutableSet.builderWithExpectedSize(outputSet.size());
for (Slot slot : outputSet) {
@ -154,13 +160,16 @@ public abstract class LogicalCatalogRelation extends LogicalRelation implements
Set<Column> columns = c.getUniqueKeys(table);
fdBuilder.addUniqueSlot((ImmutableSet) findSlotsByColumn(outputSet, columns));
}
fdBuilder.addFdItems(computeFdItems(outputSet));
return fdBuilder.build();
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
return computeFdItems(Utils.fastToImmutableSet(outputSupplier.get()));
public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
// No uniform slot for catalog relation
}
@Override
public ImmutableSet<FdItem> computeFdItems() {
return computeFdItems(Utils.fastToImmutableSet(getOutputSet()));
}
private ImmutableSet<FdItem> computeFdItems(Set<Slot> outputSet) {

29
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java
View File

@ -22,7 +22,6 @@ import org.apache.doris.nereids.properties.ExprFdItem;
import org.apache.doris.nereids.properties.FdFactory;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.OrderKey;
import org.apache.doris.nereids.trees.expressions.ExprId;
@ -43,7 +42,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
/**
* use for defer materialize top n
@ -120,26 +118,29 @@ public class LogicalDeferMaterializeTopN<CHILD_TYPE extends Plan> extends Logica
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
FunctionalDependencies fd = child(0).getLogicalProperties().getFunctionalDependencies();
public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
if (getLimit() == 1) {
Builder builder = new Builder();
List<Slot> output = outputSupplier.get();
output.forEach(builder::addUniformSlot);
output.forEach(builder::addUniqueSlot);
ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
builder.addFdItems(fdItems);
fd = builder.build();
getOutput().forEach(fdBuilder::addUniqueSlot);
} else {
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
return fd;
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
if (getLimit() == 1) {
getOutput().forEach(fdBuilder::addUniformSlot);
} else {
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
}
@Override
public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet<FdItem> fdItems = child(0).getLogicalProperties().getFunctionalDependencies().getFdItems();
if (getLimit() == 1) {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
List<Slot> output = outputSupplier.get();
List<Slot> output = getOutput();
ImmutableSet<SlotReference> slotSet = output.stream()
.filter(SlotReference.class::isInstance)
.map(SlotReference.class::cast)

59
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java
View File

@ -21,7 +21,7 @@ import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.ExprFdItem;
import org.apache.doris.nereids.properties.FdFactory;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
@ -39,7 +39,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
/**
* Logical Except.
@ -109,29 +108,8 @@ public class LogicalExcept extends LogicalSetOperation {
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
FunctionalDependencies.Builder builder = new FunctionalDependencies
.Builder(child(0).getLogicalProperties().getFunctionalDependencies());
Map<Slot, Slot> replaceMap = new HashMap<>();
List<Slot> output = outputSupplier.get();
List<? extends Slot> originalOutputs = regularChildrenOutputs.isEmpty()
? child(0).getOutput()
: regularChildrenOutputs.get(0);
for (int i = 0; i < output.size(); i++) {
replaceMap.put(originalOutputs.get(i), output.get(i));
}
builder.replace(replaceMap);
if (qualifier == Qualifier.DISTINCT) {
builder.addUniqueSlot(ImmutableSet.copyOf(outputSupplier.get()));
}
ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
builder.addFdItems(fdItems);
return builder.build();
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
Set<NamedExpression> output = ImmutableSet.copyOf(outputSupplier.get());
public ImmutableSet<FdItem> computeFdItems() {
Set<NamedExpression> output = ImmutableSet.copyOf(getOutput());
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<SlotReference> exprs = output.stream()
@ -152,4 +130,35 @@ public class LogicalExcept extends LogicalSetOperation {
return builder.build();
}
@Override
public void computeUnique(Builder fdBuilder) {
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
if (qualifier == Qualifier.DISTINCT) {
fdBuilder.addUniqueSlot(ImmutableSet.copyOf(getOutput()));
}
Map<Slot, Slot> replaceMap = new HashMap<>();
List<Slot> output = getOutput();
List<? extends Slot> originalOutputs = regularChildrenOutputs.isEmpty()
? child(0).getOutput()
: regularChildrenOutputs.get(0);
for (int i = 0; i < output.size(); i++) {
replaceMap.put(originalOutputs.get(i), output.get(i));
}
fdBuilder.replace(replaceMap);
}
@Override
public void computeUniform(Builder fdBuilder) {
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
Map<Slot, Slot> replaceMap = new HashMap<>();
List<Slot> output = getOutput();
List<? extends Slot> originalOutputs = regularChildrenOutputs.isEmpty()
? child(0).getOutput()
: regularChildrenOutputs.get(0);
for (int i = 0; i < output.size(); i++) {
replaceMap.put(originalOutputs.get(i), output.get(i));
}
fdBuilder.replace(replaceMap);
}
}

25
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java
View File

@ -19,7 +19,6 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
@ -41,7 +40,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -149,17 +147,7 @@ public class LogicalFilter<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_T
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
Builder fdBuilder = new Builder(
child().getLogicalProperties().getFunctionalDependencies());
getConjuncts().forEach(e -> fdBuilder.addUniformSlot(ExpressionUtils.extractUniformSlot(e)));
ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
fdBuilder.addFdItems(fdItems);
return fdBuilder.build();
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<FdItem> childItems = child().getLogicalProperties().getFunctionalDependencies().getFdItems();
@ -167,4 +155,15 @@ public class LogicalFilter<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_T
return builder.build();
}
@Override
public void computeUnique(Builder fdBuilder) {
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
@Override
public void computeUniform(Builder fdBuilder) {
getConjuncts().forEach(e -> fdBuilder.addUniformSlot(ExpressionUtils.extractUniformSlot(e)));
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
}

20
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java
View File

@ -19,7 +19,7 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
@ -38,7 +38,6 @@ import com.google.common.collect.Lists;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Supplier;
/**
* plan for table generator, the statement like: SELECT * FROM tbl LATERAL VIEW EXPLODE(c1) g as (gc1);
@ -157,16 +156,17 @@ public class LogicalGenerate<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder();
builder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
builder.addFdItems(fdItems);
return builder.build();
public ImmutableSet<FdItem> computeFdItems() {
return ImmutableSet.of();
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
return ImmutableSet.of();
public void computeUnique(Builder fdBuilder) {
// don't generate and propagate unique
}
@Override
public void computeUniform(Builder fdBuilder) {
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
}

19
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java
View File

@ -19,7 +19,6 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
@ -39,7 +38,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
/**
* Logical Having plan
@ -120,17 +118,18 @@ public class LogicalHaving<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_T
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
Builder fdBuilder = new Builder(
child().getLogicalProperties().getFunctionalDependencies());
getConjuncts().forEach(e -> fdBuilder.addUniformSlot(ExpressionUtils.extractUniformSlot(e)));
ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
fdBuilder.addFdItems(fdItems);
return fdBuilder.build();
public void computeUnique(Builder fdBuilder) {
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
public void computeUniform(Builder fdBuilder) {
getConjuncts().forEach(e -> fdBuilder.addUniformSlot(ExpressionUtils.extractUniformSlot(e)));
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
@Override
public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<FdItem> childItems = child().getLogicalProperties().getFunctionalDependencies().getFdItems();

27
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.nereids.properties.ExprFdItem;
import org.apache.doris.nereids.properties.FdFactory;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
@ -39,7 +40,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
/**
* Logical Intersect.
@ -119,24 +119,29 @@ public class LogicalIntersect extends LogicalSetOperation {
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder();
public void computeUnique(Builder fdBuilder) {
for (Plan child : children) {
builder.addFunctionalDependencies(
fdBuilder.addUniqueSlot(
child.getLogicalProperties().getFunctionalDependencies());
replaceSlotInFuncDeps(builder, child.getOutput(), outputSupplier.get());
replaceSlotInFuncDeps(fdBuilder, child.getOutput(), getOutput());
}
if (qualifier == Qualifier.DISTINCT) {
builder.addUniqueSlot(ImmutableSet.copyOf(outputSupplier.get()));
fdBuilder.addUniqueSlot(ImmutableSet.copyOf(getOutput()));
}
ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
builder.addFdItems(fdItems);
return builder.build();
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
Set<NamedExpression> output = ImmutableSet.copyOf(outputSupplier.get());
public void computeUniform(Builder fdBuilder) {
for (Plan child : children) {
fdBuilder.addUniformSlot(
child.getLogicalProperties().getFunctionalDependencies());
replaceSlotInFuncDeps(fdBuilder, child.getOutput(), getOutput());
}
}
@Override
public ImmutableSet<FdItem> computeFdItems() {
Set<NamedExpression> output = ImmutableSet.copyOf(getOutput());
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<SlotReference> exprs = output.stream()

130
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
View File

@ -24,7 +24,6 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdFactory;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.TableFdItem;
@ -58,7 +57,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nullable;
@ -460,76 +458,7 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
if (isMarkJoin()) {
// TODO disable function dependence calculation for mark join, but need re-think this in future.
return FunctionalDependencies.EMPTY_FUNC_DEPS;
}
//1. NALAJ and FOJ block functional dependencies
if (joinType.isNullAwareLeftAntiJoin() || joinType.isFullOuterJoin()) {
return FunctionalDependencies.EMPTY_FUNC_DEPS;
}
// left/right semi/anti join propagate left/right functional dependencies
if (joinType.isLeftAntiJoin() || joinType.isLefSemiJoin()) {
return left().getLogicalProperties().getFunctionalDependencies();
}
if (joinType.isRightSemiJoin() || joinType.isRightAntiJoin()) {
return right().getLogicalProperties().getFunctionalDependencies();
}
// if there is non-equal join conditions, block functional dependencies
if (!otherJoinConjuncts.isEmpty()) {
return FunctionalDependencies.EMPTY_FUNC_DEPS;
}
Pair<Set<Slot>, Set<Slot>> keys = extractNullRejectHashKeys();
if (keys == null) {
return FunctionalDependencies.EMPTY_FUNC_DEPS;
}
// Note here we only check whether the left is unique.
// So the hash condition can't be null-safe
// TODO: consider Null-safe hash condition when left and rigth is not nullable
boolean isLeftUnique = left().getLogicalProperties()
.getFunctionalDependencies().isUnique(keys.first);
boolean isRightUnique = right().getLogicalProperties()
.getFunctionalDependencies().isUnique(keys.second);
Builder fdBuilder = new Builder();
if (joinType.isInnerJoin()) {
// inner join propagate uniforms slots
// And if the hash keys is unique, inner join can propagate all functional dependencies
if (isLeftUnique && isRightUnique) {
fdBuilder.addFunctionalDependencies(left().getLogicalProperties().getFunctionalDependencies());
fdBuilder.addFunctionalDependencies(right().getLogicalProperties().getFunctionalDependencies());
} else {
fdBuilder.addUniformSlot(left().getLogicalProperties().getFunctionalDependencies());
fdBuilder.addUniformSlot(right().getLogicalProperties().getFunctionalDependencies());
}
}
// left/right outer join propagate left/right uniforms slots
// And if the right/left hash keys is unique,
// join can propagate left/right functional dependencies
if (joinType.isLeftOuterJoin()) {
if (isRightUnique) {
return left().getLogicalProperties().getFunctionalDependencies();
}
fdBuilder.addUniformSlot(left().getLogicalProperties().getFunctionalDependencies());
}
if (joinType.isRightOuterJoin()) {
if (isLeftUnique) {
return left().getLogicalProperties().getFunctionalDependencies();
}
fdBuilder.addUniformSlot(left().getLogicalProperties().getFunctionalDependencies());
}
ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
fdBuilder.addFdItems(fdItems);
return fdBuilder.build();
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
if (isMarkJoin() || joinType.isNullAwareLeftAntiJoin()
|| joinType.isFullOuterJoin()
@ -710,4 +639,61 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
logicalJoin.put("Properties", properties);
return logicalJoin;
}
@Override
public void computeUnique(Builder fdBuilder) {
if (isMarkJoin()) {
// TODO disable function dependence calculation for mark join, but need re-think this in future.
return;
}
if (joinType.isLeftSemiOrAntiJoin()) {
fdBuilder.addUniqueSlot(left().getLogicalProperties().getFunctionalDependencies());
} else if (joinType.isRightSemiOrAntiJoin()) {
fdBuilder.addUniqueSlot(right().getLogicalProperties().getFunctionalDependencies());
}
// if there is non-equal join conditions, don't propagate unique
if (hashJoinConjuncts.isEmpty()) {
return;
}
Pair<Set<Slot>, Set<Slot>> keys = extractNullRejectHashKeys();
if (keys == null) {
return;
}
// Note here we only check whether the left is unique.
// So the hash condition can't be null-safe
// TODO: consider Null-safe hash condition when left and rigth is not nullable
boolean isLeftUnique = left().getLogicalProperties()
.getFunctionalDependencies().isUnique(keys.first);
boolean isRightUnique = right().getLogicalProperties()
.getFunctionalDependencies().isUnique(keys.second);
// left/right outer join propagate left/right uniforms slots
// And if the right/left hash keys is unique,
// join can propagate left/right functional dependencies
if (joinType.isLeftOuterJoin() && isRightUnique) {
fdBuilder.addUniqueSlot(left().getLogicalProperties().getFunctionalDependencies());
} else if (joinType.isRightOuterJoin() && isLeftUnique) {
fdBuilder.addUniqueSlot(right().getLogicalProperties().getFunctionalDependencies());
} else if (joinType.isInnerJoin() && isLeftUnique && isRightUnique) {
// inner join propagate uniforms slots
// And if the hash keys is unique, inner join can propagate all functional dependencies
fdBuilder.addFunctionalDependencies(left().getLogicalProperties().getFunctionalDependencies());
fdBuilder.addFunctionalDependencies(right().getLogicalProperties().getFunctionalDependencies());
}
}
@Override
public void computeUniform(Builder fdBuilder) {
if (isMarkJoin()) {
// TODO disable function dependence calculation for mark join, but need re-think this in future.
return;
}
if (!joinType.isLeftSemiOrAntiJoin()) {
fdBuilder.addUniformSlot(right().getLogicalProperties().getFunctionalDependencies());
}
if (!joinType.isRightSemiOrAntiJoin()) {
fdBuilder.addUniformSlot(left().getLogicalProperties().getFunctionalDependencies());
}
}
}

30
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java
View File

@ -22,7 +22,6 @@ import org.apache.doris.nereids.properties.ExprFdItem;
import org.apache.doris.nereids.properties.FdFactory;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
@ -41,7 +40,6 @@ import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Supplier;
/**
* Logical limit plan
@ -150,25 +148,29 @@ public class LogicalLimit<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TY
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
FunctionalDependencies fd = child(0).getLogicalProperties().getFunctionalDependencies();
if (getLimit() == 1 && !phase.isLocal()) {
Builder builder = new Builder();
outputSupplier.get().forEach(builder::addUniformSlot);
outputSupplier.get().forEach(builder::addUniqueSlot);
ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
builder.addFdItems(fdItems);
fd = builder.build();
public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
if (getLimit() == 1) {
getOutput().forEach(fdBuilder::addUniqueSlot);
} else {
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
return fd;
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
if (getLimit() == 1) {
getOutput().forEach(fdBuilder::addUniformSlot);
} else {
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
}
@Override
public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet<FdItem> fdItems = child(0).getLogicalProperties().getFunctionalDependencies().getFdItems();
if (getLimit() == 1 && !phase.isLocal()) {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
List<Slot> output = outputSupplier.get();
List<Slot> output = getOutput();
ImmutableSet<SlotReference> slotSet = output.stream()
.filter(SlotReference.class::isInstance)
.map(SlotReference.class::cast)

21
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java
View File

@ -41,7 +41,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
/**
* A relation that contains only one row consist of some constant expressions.
@ -136,20 +135,18 @@ public class LogicalOneRowRelation extends LogicalRelation implements OneRowRela
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder();
outputSupplier.get().forEach(s -> {
builder.addUniformSlot(s);
builder.addUniqueSlot(s);
});
ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
builder.addFdItems(fdItems);
return builder.build();
public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
getOutput().forEach(fdBuilder::addUniqueSlot);
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
Set<NamedExpression> output = ImmutableSet.copyOf(outputSupplier.get());
public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
getOutput().forEach(fdBuilder::addUniformSlot);
}
@Override
public ImmutableSet<FdItem> computeFdItems() {
Set<NamedExpression> output = ImmutableSet.copyOf(getOutput());
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<SlotReference> slotSet = output.stream()
.filter(SlotReference.class::isInstance)

17
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java
View File

@ -19,13 +19,11 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Optional;
import java.util.function.BiFunction;
import java.util.function.Supplier;
@ -61,7 +59,18 @@ public interface LogicalPlan extends Plan {
* - BlockFDPropagation: clean the fd
* - PropagateFD: propagate the fd
*/
FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier);
default FunctionalDependencies computeFuncDeps() {
FunctionalDependencies.Builder fdBuilder = new FunctionalDependencies.Builder();
computeUniform(fdBuilder);
computeUnique(fdBuilder);
ImmutableSet<FdItem> fdItems = computeFdItems();
fdBuilder.addFdItems(fdItems);
return fdBuilder.build();
}
ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier);
ImmutableSet<FdItem> computeFdItems();
void computeUnique(FunctionalDependencies.Builder fdBuilder);
void computeUniform(FunctionalDependencies.Builder fdBuilder);
}

67
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
View File

@ -23,7 +23,6 @@ import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.BoundStar;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@ -42,11 +41,9 @@ import com.google.common.collect.ImmutableList.Builder;
import com.google.common.collect.ImmutableSet;
import org.json.JSONObject;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Supplier;
/**
* Logical project plan.
@ -239,31 +236,7 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
FunctionalDependencies childFuncDeps = child().getLogicalProperties().getFunctionalDependencies();
FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder(childFuncDeps);
builder.pruneSlots(new HashSet<>(outputSupplier.get()));
projects.stream().filter(Alias.class::isInstance).forEach(proj -> {
if (proj.child(0).isConstant()) {
builder.addUniformSlot(proj.toSlot());
} else if (proj.child(0) instanceof Uuid) {
builder.addUniqueSlot(proj.toSlot());
} else if (ExpressionUtils.isInjective(proj.child(0))) {
ImmutableSet<Slot> inputs = ImmutableSet.copyOf(proj.getInputSlots());
if (childFuncDeps.isUnique(inputs)) {
builder.addUniqueSlot(proj.toSlot());
} else if (childFuncDeps.isUniform(inputs)) {
builder.addUniformSlot(proj.toSlot());
}
}
});
ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
builder.addFdItems(fdItems);
return builder.build();
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<FdItem> childItems = child().getLogicalProperties().getFunctionalDependencies().getFdItems();
@ -271,4 +244,42 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
return builder.build();
}
@Override
public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
for (NamedExpression proj : getProjects()) {
if (proj.children().isEmpty()) {
continue;
}
if (proj.child(0) instanceof Uuid) {
fdBuilder.addUniqueSlot(proj.toSlot());
} else if (ExpressionUtils.isInjective(proj.child(0))) {
ImmutableSet<Slot> inputs = ImmutableSet.copyOf(proj.getInputSlots());
if (child(0).getLogicalProperties().getFunctionalDependencies().isUnique(inputs)) {
fdBuilder.addUniqueSlot(proj.toSlot());
}
}
}
fdBuilder.pruneSlots(getOutputSet());
}
@Override
public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
for (NamedExpression proj : getProjects()) {
if (proj.children().isEmpty()) {
continue;
}
if (proj.child(0).isConstant()) {
fdBuilder.addUniformSlot(proj.toSlot());
} else if (ExpressionUtils.isInjective(proj.child(0))) {
ImmutableSet<Slot> inputs = ImmutableSet.copyOf(proj.getInputSlots());
if (child(0).getLogicalProperties().getFunctionalDependencies().isUniform(inputs)) {
fdBuilder.addUniformSlot(proj.toSlot());
}
}
}
fdBuilder.pruneSlots(getOutputSet());
}
}

19
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java
View File

@ -39,7 +39,6 @@ import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Supplier;
/**
* LogicalRepeat.
@ -187,19 +186,17 @@ public class LogicalRepeat<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_T
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder();
// Note uniform does not reject nullable slots
outputSupplier.get().stream()
.filter(child(0).getLogicalProperties().getFunctionalDependencies()::isUniform)
.forEach(builder::addUniformSlot);
ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
builder.addFdItems(fdItems);
return builder.build();
public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
// don't generate unique slot
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
@Override
public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<FdItem> childItems = child().getLogicalProperties().getFunctionalDependencies().getFdItems();

26
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java
View File

@ -41,7 +41,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
/**
* The node of logical plan for sub query and alias
@ -165,22 +164,29 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends Plan> extends LogicalUnary<
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
FunctionalDependencies.Builder builder = new FunctionalDependencies
.Builder(child(0).getLogicalProperties().getFunctionalDependencies());
public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
Map<Slot, Slot> replaceMap = new HashMap<>();
List<Slot> outputs = outputSupplier.get();
List<Slot> outputs = getOutput();
for (int i = 0; i < outputs.size(); i++) {
replaceMap.put(child(0).getOutput().get(i), outputs.get(i));
}
builder.replace(replaceMap);
ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
builder.addFdItems(fdItems);
return builder.build();
fdBuilder.replace(replaceMap);
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
Map<Slot, Slot> replaceMap = new HashMap<>();
List<Slot> outputs = getOutput();
for (int i = 0; i < outputs.size(); i++) {
replaceMap.put(child(0).getOutput().get(i), outputs.get(i));
}
fdBuilder.replace(replaceMap);
}
@Override
public ImmutableSet<FdItem> computeFdItems() {
// TODO: inherit from child with replaceMap
return ImmutableSet.of();
}

30
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java
View File

@ -22,7 +22,6 @@ import org.apache.doris.nereids.properties.ExprFdItem;
import org.apache.doris.nereids.properties.FdFactory;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.OrderKey;
import org.apache.doris.nereids.trees.expressions.Expression;
@ -35,6 +34,7 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
@ -42,7 +42,6 @@ import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Supplier;
/**
* Logical top-N plan.
@ -171,26 +170,29 @@ public class LogicalTopN<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYP
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
FunctionalDependencies fd = child(0).getLogicalProperties().getFunctionalDependencies();
public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
if (getLimit() == 1) {
Builder builder = new Builder();
List<Slot> output = outputSupplier.get();
output.forEach(builder::addUniformSlot);
output.forEach(builder::addUniqueSlot);
ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
builder.addFdItems(fdItems);
fd = builder.build();
getOutput().forEach(fdBuilder::addUniqueSlot);
} else {
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
return fd;
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
if (getLimit() == 1) {
getOutput().forEach(fdBuilder::addUniformSlot);
} else {
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
}
@Override
public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet<FdItem> fdItems = child(0).getLogicalProperties().getFunctionalDependencies().getFdItems();
if (getLimit() == 1) {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
List<Slot> output = outputSupplier.get();
List<Slot> output = getOutput();
ImmutableSet<SlotReference> slotSet = output.stream()
.filter(SlotReference.class::isInstance)
.map(SlotReference.class::cast)

22
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java
View File

@ -25,7 +25,6 @@ import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
@ -41,7 +40,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
/**
* Logical Union.
@ -187,20 +185,20 @@ public class LogicalUnion extends LogicalSetOperation implements Union, OutputPr
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
if (qualifier != Qualifier.DISTINCT) {
return FunctionalDependencies.EMPTY_FUNC_DEPS;
public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
if (qualifier == Qualifier.DISTINCT) {
fdBuilder.addUniqueSlot(ImmutableSet.copyOf(getOutput()));
}
FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder();
builder.addUniqueSlot(ImmutableSet.copyOf(outputSupplier.get()));
ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
builder.addFdItems(fdItems);
return builder.build();
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
Set<NamedExpression> output = ImmutableSet.copyOf(outputSupplier.get());
public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
// don't propagate uniform slots
}
@Override
public ImmutableSet<FdItem> computeFdItems() {
Set<NamedExpression> output = ImmutableSet.copyOf(getOutput());
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<SlotReference> exprs = output.stream()

22
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java
View File

@ -21,7 +21,7 @@ import org.apache.doris.catalog.View;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
@ -36,7 +36,6 @@ import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Supplier;
/** LogicalView */
public class LogicalView<BODY extends Plan> extends LogicalUnary<BODY> {
@ -129,17 +128,22 @@ public class LogicalView<BODY extends Plan> extends LogicalUnary<BODY> {
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
return ((LogicalPlan) child()).computeFuncDeps(outputSupplier);
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
return ((LogicalPlan) child()).computeFdItems(outputSupplier);
public ImmutableSet<FdItem> computeFdItems() {
return ((LogicalPlan) child()).computeFdItems();
}
@Override
public Plan withChildren(List<Plan> children) {
return new LogicalView<>(view, (LogicalPlan) children.get(0));
}
@Override
public void computeUnique(Builder fdBuilder) {
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
@Override
public void computeUniform(Builder fdBuilder) {
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
}

75
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java
View File

@ -19,7 +19,7 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@ -43,7 +43,6 @@ import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Supplier;
/**
* logical node to deal with window functions;
@ -228,35 +227,51 @@ public class LogicalWindow<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_T
return Optional.ofNullable(window);
}
private void updateFuncDepsByWindowExpr(NamedExpression namedExpression, FunctionalDependencies.Builder builder) {
private boolean isUnique(NamedExpression namedExpression) {
if (namedExpression.children().size() != 1 || !(namedExpression.child(0) instanceof WindowExpression)) {
return;
return false;
}
WindowExpression windowExpr = (WindowExpression) namedExpression.child(0);
List<Expression> partitionKeys = windowExpr.getPartitionKeys();
// Now we only support slot type keys
if (!partitionKeys.stream().allMatch(Slot.class::isInstance)) {
return;
return false;
}
ImmutableSet<Slot> slotSet = partitionKeys.stream()
.map(s -> (Slot) s)
.collect(ImmutableSet.toImmutableSet());
// if partition by keys are uniform or empty, output is unique
if (child(0).getLogicalProperties().getFunctionalDependencies().isUniformAndNotNull(slotSet)
|| slotSet.isEmpty()) {
if (windowExpr.getFunction() instanceof RowNumber) {
return true;
}
}
return false;
}
private boolean isUniform(NamedExpression namedExpression) {
if (namedExpression.children().size() != 1 || !(namedExpression.child(0) instanceof WindowExpression)) {
return false;
}
WindowExpression windowExpr = (WindowExpression) namedExpression.child(0);
List<Expression> partitionKeys = windowExpr.getPartitionKeys();
// Now we only support slot type keys
if (!partitionKeys.stream().allMatch(Slot.class::isInstance)) {
return false;
}
ImmutableSet<Slot> slotSet = partitionKeys.stream()
.map(s -> (Slot) s)
.collect(ImmutableSet.toImmutableSet());
// if partition by keys are unique, output is uniform
if (child(0).getLogicalProperties().getFunctionalDependencies().isUniqueAndNotNull(slotSet)) {
if (windowExpr.getFunction() instanceof RowNumber
|| windowExpr.getFunction() instanceof Rank
|| windowExpr.getFunction() instanceof DenseRank) {
builder.addUniformSlot(namedExpression.toSlot());
}
}
// if partition by keys are uniform, output is unique
if (child(0).getLogicalProperties().getFunctionalDependencies().isUniformAndNotNull(slotSet)) {
if (windowExpr.getFunction() instanceof RowNumber) {
builder.addUniqueSlot(namedExpression.toSlot());
return true;
}
}
return false;
}
private void updateFuncDepsByWindowExpr(NamedExpression namedExpression, ImmutableSet.Builder<FdItem> builder) {
@ -278,19 +293,7 @@ public class LogicalWindow<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_T
}
@Override
public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>> outputSupplier) {
FunctionalDependencies.Builder builder = new FunctionalDependencies.Builder(
child(0).getLogicalProperties().getFunctionalDependencies());
for (NamedExpression namedExpression : windowExpressions) {
updateFuncDepsByWindowExpr(namedExpression, builder);
}
ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
builder.addFdItems(fdItems);
return builder.build();
}
@Override
public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier) {
public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<FdItem> childItems = child().getLogicalProperties().getFunctionalDependencies().getFdItems();
builder.addAll(childItems);
@ -301,4 +304,24 @@ public class LogicalWindow<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_T
return builder.build();
}
@Override
public void computeUnique(Builder fdBuilder) {
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
for (NamedExpression namedExpression : windowExpressions) {
if (isUnique(namedExpression)) {
fdBuilder.addUniqueSlot(namedExpression.toSlot());
}
}
}
@Override
public void computeUniform(Builder fdBuilder) {
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
for (NamedExpression namedExpression : windowExpressions) {
if (isUniform(namedExpression)) {
fdBuilder.addUniformSlot(namedExpression.toSlot());
}
}
}
}

4
fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
View File

@ -41,7 +41,6 @@ import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.WindowExpression;
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.Avg;
import org.apache.doris.nereids.trees.expressions.functions.agg.Max;
import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
@ -738,7 +737,8 @@ public class ExpressionUtils {
return expression instanceof Slot;
}
public static boolean isInjectiveAgg(AggregateFunction agg) {
// if the input is unique, the output of agg is unique, too
public static boolean isInjectiveAgg(Expression agg) {
return agg instanceof Sum || agg instanceof Avg || agg instanceof Max || agg instanceof Min;
}

4
fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FunctionalDependenciesTest.java
View File

@ -258,12 +258,12 @@ class FunctionalDependenciesTest extends TestWithFeService {
@Test
void testWindow() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select row_number() over(partition by id) from agg")
.analyze("select id, row_number() over(partition by id) from agg")
.rewrite()
.getPlan();
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0)));
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(1)));
plan = PlanChecker.from(connectContext)
.analyze("select row_number() over(partition by name) from agg where name = '1'")

207
fe/fe-core/src/test/java/org/apache/doris/nereids/properties/UniformTest.java Normal file
View File

@ -0,0 +1,207 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.properties;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.utframe.TestWithFeService;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
class UniformTest extends TestWithFeService {
Slot slot1 = new SlotReference("1", IntegerType.INSTANCE, false);
Slot slot2 = new SlotReference("2", IntegerType.INSTANCE, false);
Slot slot3 = new SlotReference("1", IntegerType.INSTANCE, false);
Slot slot4 = new SlotReference("1", IntegerType.INSTANCE, false);
@Override
protected void runBeforeAll() throws Exception {
createDatabase("test");
createTable("create table test.agg (\n"
+ "id int not null,\n"
+ "name varchar(128) replace not null )\n"
+ "AGGREGATE KEY(id)\n"
+ "distributed by hash(id) buckets 10\n"
+ "properties('replication_num' = '1');");
createTable("create table test.uni (\n"
+ "id int not null,\n"
+ "name varchar(128) not null)\n"
+ "UNIQUE KEY(id)\n"
+ "distributed by hash(id) buckets 10\n"
+ "properties('replication_num' = '1');");
connectContext.setDatabase("test");
}
@Test
void testAgg() {
// group by unique
Plan plan = PlanChecker.from(connectContext)
.analyze("select count(id) from agg group by id")
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniform(plan.getOutput().get(0)));
// propagate uniform
plan = PlanChecker.from(connectContext)
.analyze("select id from agg where id = 1 group by id ")
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniform(plan.getOutput().get(0)));
// group by all/uniform
plan = PlanChecker.from(connectContext)
.analyze("select sum(id) from agg ")
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniform(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
.analyze("select sum(id) from agg where id = 1 group by id")
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniform(plan.getOutput().get(0)));
}
@Test
void testTopNLimit() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select name from agg limit 1")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniformAndNotNull(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
.analyze("select name from agg order by name limit 1 ")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniformAndNotNull(plan.getOutput().get(0)));
}
@Test
void testSetOp() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select name from agg limit 1 except select name from agg")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniform(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
.analyze("select id from agg intersect select name from agg limit 1")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniform(plan.getOutput().get(0)));
}
@Test
void testFilterHaving() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select id from agg where id = 1")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniformAndNotNull(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
.analyze("select name from uni group by name having name = \"\"")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniformAndNotNull(plan.getOutput().get(0)));
}
@Test
void testGenerate() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select id from agg lateral view explode([1,2,3]) tmp1 as e1")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isEmpty());
}
@Test
void testJoin() {
// foj doesn't propagate any
Plan plan = PlanChecker.from(connectContext)
.analyze("select uni.id from agg join uni "
+ "on agg.id = uni.id where uni.id = 1")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isUniform(plan.getOutput().get(0)));
}
@Test
void testOneRowRelation() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select 1")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniform(plan.getOutput().get(0)));
}
@Test
void testProject() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select id as id2, 1 as id3 from uni where id = 1")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0)));
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(1)));
}
@Test
void testSubQuery() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select id from (select id from agg where id = 1) t")
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0)));
}
@Test
void testWindow() {
// partition by uniform
Plan plan = PlanChecker.from(connectContext)
.analyze("select row_number() over(partition by id) from agg")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniform(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
.analyze("select rank() over(partition by id) from agg")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniform(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
.analyze("select dense_rank() over(partition by id) from agg")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniform(plan.getOutput().get(0)));
}
}

385
fe/fe-core/src/test/java/org/apache/doris/nereids/properties/UniqueTest.java Normal file
View File

@ -0,0 +1,385 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.properties;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.utframe.TestWithFeService;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
class UniqueTest extends TestWithFeService {
Slot slot1 = new SlotReference("1", IntegerType.INSTANCE, false);
Slot slot2 = new SlotReference("2", IntegerType.INSTANCE, false);
Slot slot3 = new SlotReference("1", IntegerType.INSTANCE, false);
Slot slot4 = new SlotReference("1", IntegerType.INSTANCE, false);
@Override
protected void runBeforeAll() throws Exception {
createDatabase("test");
createTable("create table test.agg (\n"
+ "id int not null,\n"
+ "name varchar(128) replace not null )\n"
+ "AGGREGATE KEY(id)\n"
+ "distributed by hash(id) buckets 10\n"
+ "properties('replication_num' = '1');");
createTable("create table test.uni (\n"
+ "id int not null,\n"
+ "name varchar(128) not null)\n"
+ "UNIQUE KEY(id)\n"
+ "distributed by hash(id) buckets 10\n"
+ "properties('replication_num' = '1');");
connectContext.setDatabase("test");
}
@Test
void testAgg() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select name from agg group by name")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniqueAndNotNull(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
.analyze("select sum(id) from agg")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUnique(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
.analyze("select id, sum(id), avg(id), max(id), min(id) from agg group by id")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUnique(plan.getOutput().get(0)));
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUnique(plan.getOutput().get(1)));
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUnique(plan.getOutput().get(2)));
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUnique(plan.getOutput().get(3)));
}
@Test
void testScan() throws Exception {
// test agg key
Plan plan = PlanChecker.from(connectContext)
.analyze("select id from agg")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniqueAndNotNull(plan.getOutput().get(0)));
// test unique key
plan = PlanChecker.from(connectContext)
.analyze("select id from uni")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniqueAndNotNull(plan.getOutput().get(0)));
// test unique constraint
addConstraint("alter table agg add constraint uq unique(name)");
plan = PlanChecker.from(connectContext)
.analyze("select name from agg")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniqueAndNotNull(plan.getOutput().get(0)));
dropConstraint("alter table agg drop constraint uq");
// test primary constraint
addConstraint("alter table agg add constraint pk primary key(name)");
plan = PlanChecker.from(connectContext)
.analyze("select name from agg")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniqueAndNotNull(plan.getOutput().get(0)));
dropConstraint("alter table agg drop constraint pk");
}
@Test
void testTopNLimit() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select name from agg limit 1")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniqueAndNotNull(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
.analyze("select name from agg order by name limit 1 ")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniqueAndNotNull(plan.getOutput().get(0)));
}
@Test
void testSetOp() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select name from agg limit 1 except select name from agg")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniqueAndNotNull(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
.analyze("select id from agg intersect select name from agg")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniqueAndNotNull(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
.analyze("select id from agg union all select name from agg")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isEmpty());
plan = PlanChecker.from(connectContext)
.analyze("select name, id from agg union select name, id from agg")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUnique(plan.getOutputSet()));
}
@Test
void testFilterHaving() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select id from agg where id = 1")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniqueAndNotNull(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
.analyze("select name from uni group by name having name = \"\"")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniqueAndNotNull(plan.getOutput().get(0)));
}
@Test
void testGenerate() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select id from agg lateral view explode([1,2,3]) tmp1 as e1")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isEmpty());
}
@Test
void testJoin() {
// foj doesn't propagate any
Plan plan = PlanChecker.from(connectContext)
.analyze("select * from agg full outer join uni "
+ "on agg.id = uni.id")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isEmpty());
// loj propagate unique when right is unique
plan = PlanChecker.from(connectContext)
.analyze("select agg.id, uni.id from agg left outer join uni "
+ "on agg.id = uni.name")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isEmpty());
plan = PlanChecker.from(connectContext)
.analyze("select agg.id, uni.id from agg left outer join uni "
+ "on agg.id = uni.id")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isUnique(plan.getOutput().get(0)));
Assertions.assertFalse(plan.getLogicalProperties().getFunctionalDependencies().isUnique(plan.getOutput().get(1)));
// roj propagate unique when left is unique
plan = PlanChecker.from(connectContext)
.analyze("select agg.id, uni.id from agg right outer join uni "
+ "on agg.id = uni.name")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isEmpty());
plan = PlanChecker.from(connectContext)
.analyze("select agg.id, uni.id from agg right outer join uni "
+ "on agg.id = uni.id")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isUnique(plan.getOutput().get(1)));
Assertions.assertFalse(plan.getLogicalProperties().getFunctionalDependencies().isUnique(plan.getOutput().get(0)));
// semi/anti join propagate all
plan = PlanChecker.from(connectContext)
.analyze("select agg.id from agg left semi join uni "
+ "on agg.id = uni.name")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isUnique(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
.analyze("select agg.id from agg left anti join uni "
+ "on agg.id = uni.name")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isUnique(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
.analyze("select uni.id from agg right semi join uni "
+ "on agg.id = uni.name")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isUnique(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
.analyze("select uni.id from agg right anti join uni "
+ "on agg.id = uni.name")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isUnique(plan.getOutput().get(0)));
// inner join propagate unique only when join key is unique
plan = PlanChecker.from(connectContext)
.analyze("select uni.id, agg.id from agg inner join uni "
+ "on agg.id = uni.id")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(1)));
plan = PlanChecker.from(connectContext)
.analyze("select uni.id, agg.id from agg inner join uni "
+ "on agg.id = uni.name")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isEmpty());
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isEmpty());
plan = PlanChecker.from(connectContext)
.analyze("select uni.id, agg.id from agg inner join uni "
+ "on agg.id < uni.id")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isEmpty());
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isEmpty());
plan = PlanChecker.from(connectContext)
.analyze("select uni.id, agg.id from agg inner join uni "
+ "on agg.id = uni.id or agg.name > uni.name")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isEmpty());
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isEmpty());
plan = PlanChecker.from(connectContext)
.analyze("select uni.id, agg.id from agg inner join uni "
+ "on agg.id = uni.id and agg.name > uni.name")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(1)));
}
@Test
void testOneRowRelation() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select 1")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
}
@Test
void testProject() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select id as id2 from uni")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
}
@Test
void testRepeat() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select id from agg group by GROUPING SETS ((id, name), (id))")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isEmpty());
plan = PlanChecker.from(connectContext)
.analyze("select id from agg group by rollup (id, name)")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isEmpty());
}
@Test
void testSubQuery() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select id from (select id from agg) t")
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
}
@Test
void testAssertRows() {
Plan plan = PlanChecker.from(connectContext)
.analyze("select id from agg where id = (select id from uni)")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
}
@Test
void testWindow() {
// partition by uniform
Plan plan = PlanChecker.from(connectContext)
.analyze("select id, row_number() over(partition by id) from agg where id =1")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(1)));
// partition by None
plan = PlanChecker.from(connectContext)
.analyze("select id, row_number() over() from agg where id =1")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(1)));
}
}