database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](Nereids) avoid generate some array in BindExpression (#23854)

Browse Source
This commit is contained in:
morrySnow
2023-09-06 11:12:32 +08:00
committed by GitHub
parent 1bdb0c14fb
commit aee773502b
7 changed files with 89 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View File

@ -1826,10 +1826,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
Optional<AggClauseContext> aggClause,
Optional<HavingClauseContext> havingClause) {
return ParserUtils.withOrigin(ctx, () -> {
// TODO: add lateral views
// from -> where -> group by -> having -> select
LogicalPlan filter = withFilter(inputRelation, whereClause);
SelectColumnClauseContext selectColumnCtx = selectClause.selectColumnClause();
LogicalPlan aggregate = withAggregate(filter, selectColumnCtx, aggClause);
@ -1838,7 +1835,6 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
throw new ParseException("cannot combine SELECT DISTINCT with aggregate functions or GROUP BY",
selectClause);
}
// TODO: replace and process having at this position
if (!(aggregate instanceof Aggregate) && havingClause.isPresent()) {
// create a project node for pattern match of ProjectToGlobalAggregate rule
// then ProjectToGlobalAggregate rule can insert agg node as LogicalHaving node's child
@ -1849,10 +1845,10 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
throw new ParseException("only column name is supported in except clause", selectColumnCtx);
}
project = new LogicalProject<>(ImmutableList.of(new UnboundStar(ImmutableList.of())),
expressions, aggregate, isDistinct);
expressions, isDistinct, aggregate);
} else {
List<NamedExpression> projects = getNamedExpressions(selectColumnCtx.namedExpressionSeq());
project = new LogicalProject<>(projects, ImmutableList.of(), aggregate, isDistinct);
project = new LogicalProject<>(projects, ImmutableList.of(), isDistinct, aggregate);
}
return new LogicalHaving<>(ExpressionUtils.extractConjunctionToSet(
getExpression((havingClause.get().booleanExpression()))), project);
@ -2011,7 +2007,6 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
private LogicalPlan withProjection(LogicalPlan input, SelectColumnClauseContext selectCtx,
Optional<AggClauseContext> aggCtx, boolean isDistinct) {
return ParserUtils.withOrigin(selectCtx, () -> {
// TODO: skip if havingClause exists
if (aggCtx.isPresent()) {
return input;
} else {
@ -2021,10 +2016,10 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
throw new ParseException("only column name is supported in except clause", selectCtx);
}
return new LogicalProject<>(ImmutableList.of(new UnboundStar(ImmutableList.of())),
expressions, input, isDistinct);
expressions, isDistinct, input);
} else {
List<NamedExpression> projects = getNamedExpressions(selectCtx.namedExpressionSeq());
return new LogicalProject<>(projects, Collections.emptyList(), input, isDistinct);
return new LogicalProject<>(projects, Collections.emptyList(), isDistinct, input);
}
}
});

28
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
View File

@ -133,21 +133,23 @@ public class BindExpression implements AnalysisRuleFactory {
logicalProject().thenApply(ctx -> {
LogicalProject<Plan> project = ctx.root;
List<NamedExpression> boundProjections =
bindSlot(project.getProjects(), project.children(), ctx.cascadesContext);
List<NamedExpression> boundExceptions = bindSlot(project.getExcepts(), project.children(),
ctx.cascadesContext);
boundProjections = flatBoundStar(boundProjections, boundExceptions);
bindSlot(project.getProjects(), project.child(), ctx.cascadesContext);
if (boundProjections.stream().anyMatch(BoundStar.class::isInstance)) {
List<NamedExpression> boundExceptions = project.getExcepts().isEmpty() ? ImmutableList.of()
: bindSlot(project.getExcepts(), project.child(), ctx.cascadesContext);
boundProjections = flatBoundStar(boundProjections, boundExceptions);
}
boundProjections = boundProjections.stream()
.map(expr -> bindFunction(expr, ctx.cascadesContext))
.collect(ImmutableList.toImmutableList());
return new LogicalProject<>(boundProjections, project.child(), project.isDistinct());
return new LogicalProject<>(boundProjections, project.isDistinct(), project.child());
})
),
RuleType.BINDING_FILTER_SLOT.build(
logicalFilter().thenApply(ctx -> {
LogicalFilter<Plan> filter = ctx.root;
Set<Expression> boundConjuncts = filter.getConjuncts().stream()
.map(expr -> bindSlot(expr, filter.children(), ctx.cascadesContext))
.map(expr -> bindSlot(expr, filter.child(), ctx.cascadesContext))
.map(expr -> bindFunction(expr, ctx.cascadesContext))
.collect(ImmutableSet.toImmutableSet());
return new LogicalFilter<>(boundConjuncts, filter.child());
@ -216,7 +218,7 @@ public class BindExpression implements AnalysisRuleFactory {
logicalAggregate().thenApply(ctx -> {
LogicalAggregate<Plan> agg = ctx.root;
List<NamedExpression> output = agg.getOutputExpressions().stream()
.map(expr -> bindSlot(expr, agg.children(), ctx.cascadesContext))
.map(expr -> bindSlot(expr, agg.child(), ctx.cascadesContext))
.map(expr -> bindFunction(expr, ctx.cascadesContext))
.collect(ImmutableList.toImmutableList());
@ -348,7 +350,7 @@ public class BindExpression implements AnalysisRuleFactory {
logicalRepeat().thenApply(ctx -> {
LogicalRepeat<Plan> repeat = ctx.root;
List<NamedExpression> output = repeat.getOutputExpressions().stream()
.map(expr -> bindSlot(expr, repeat.children(), ctx.cascadesContext))
.map(expr -> bindSlot(expr, repeat.child(), ctx.cascadesContext))
.map(expr -> bindFunction(expr, ctx.cascadesContext))
.collect(ImmutableList.toImmutableList());
@ -381,7 +383,7 @@ public class BindExpression implements AnalysisRuleFactory {
List<List<Expression>> groupingSets = replacedGroupingSets
.stream()
.map(groupingSet -> groupingSet.stream()
.map(expr -> bindSlot(expr, repeat.children(), ctx.cascadesContext))
.map(expr -> bindSlot(expr, repeat.child(), ctx.cascadesContext))
.map(expr -> bindFunction(expr, ctx.cascadesContext))
.collect(ImmutableList.toImmutableList()))
.collect(ImmutableList.toImmutableList());
@ -449,7 +451,7 @@ public class BindExpression implements AnalysisRuleFactory {
Aggregate<Plan> childPlan = having.child();
Set<Expression> boundConjuncts = having.getConjuncts().stream()
.map(expr -> {
expr = bindSlot(expr, childPlan.children(), ctx.cascadesContext, false);
expr = bindSlot(expr, childPlan.child(), ctx.cascadesContext, false);
return bindSlot(expr, childPlan, ctx.cascadesContext, false);
})
.map(expr -> bindFunction(expr, ctx.cascadesContext))
@ -527,7 +529,7 @@ public class BindExpression implements AnalysisRuleFactory {
logicalGenerate().thenApply(ctx -> {
LogicalGenerate<Plan> generate = ctx.root;
List<Function> boundSlotGenerators
= bindSlot(generate.getGenerators(), generate.children(), ctx.cascadesContext);
= bindSlot(generate.getGenerators(), generate.child(), ctx.cascadesContext);
List<Function> boundFunctionGenerators = boundSlotGenerators.stream()
.map(f -> bindTableGeneratingFunction((UnboundFunction) f, ctx.cascadesContext))
.collect(Collectors.toList());
@ -612,10 +614,10 @@ public class BindExpression implements AnalysisRuleFactory {
}
private <E extends Expression> List<E> bindSlot(
List<E> exprList, List<Plan> inputs, CascadesContext cascadesContext) {
List<E> exprList, Plan input, CascadesContext cascadesContext) {
List<E> slots = new ArrayList<>(exprList.size());
for (E expr : exprList) {
E result = bindSlot(expr, inputs, cascadesContext);
E result = bindSlot(expr, input, cascadesContext);
slots.add(result);
}
return slots;

43
fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
View File

@ -155,26 +155,33 @@ public class ExpressionEstimation extends ExpressionVisitor<ColumnStatistic, Sta
}
private ColumnStatistic castMinMax(ColumnStatistic colStats, DataType targetType) {
if (colStats.minExpr instanceof StringLiteral && targetType.isDateLikeType()) {
ColumnStatisticBuilder builder = new ColumnStatisticBuilder(colStats);
if (colStats.minExpr != null && colStats.maxExpr != null) {
String strMin = colStats.minExpr.getStringValue();
try {
DateLiteral dateMinLiteral = new DateLiteral(strMin);
long min = dateMinLiteral.getValue();
builder.setMinValue(min);
builder.setMinExpr(dateMinLiteral.toLegacyLiteral());
String strMax = colStats.maxExpr.getStringValue();
DateLiteral dateMaxLiteral = new DateLiteral(strMax);
long max = dateMaxLiteral.getValue();
builder.setMaxValue(max);
builder.setMaxExpr(dateMaxLiteral.toLegacyLiteral());
} catch (AnalysisException e) {
// ignore exception. do not convert min max
if (colStats.minExpr instanceof StringLiteral || colStats.maxExpr instanceof StringLiteral) {
if (targetType.isDateLikeType()) {
ColumnStatisticBuilder builder = new ColumnStatisticBuilder(colStats);
if (colStats.minExpr != null) {
try {
String strMin = colStats.minExpr.getStringValue();
DateLiteral dateMinLiteral = new DateLiteral(strMin);
long min = dateMinLiteral.getValue();
builder.setMinValue(min);
builder.setMinExpr(dateMinLiteral.toLegacyLiteral());
} catch (AnalysisException e) {
// ignore exception. do not convert min
}
}
if (colStats.maxExpr != null) {
try {
String strMax = colStats.maxExpr.getStringValue();
DateLiteral dateMaxLiteral = new DateLiteral(strMax);
long max = dateMaxLiteral.getValue();
builder.setMaxValue(max);
builder.setMaxExpr(dateMaxLiteral.toLegacyLiteral());
} catch (AnalysisException e) {
// ignore exception. do not convert max
}
}
return builder.build();
}
return builder.build();
}
return colStats;
}

8
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/IdenticalSignature.java
View File

@ -18,9 +18,9 @@
package org.apache.doris.nereids.trees.expressions.functions;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.coercion.AnyDataType;
import org.apache.doris.nereids.types.coercion.FollowToAnyDataType;
import java.util.List;
@ -36,10 +36,8 @@ public interface IdenticalSignature extends ComputeSignature {
try {
// TODO: copy matchesType to DataType
// TODO: resolve AnyDataType invoke toCatalogDataType
if (signatureType instanceof ArrayType) {
if (((ArrayType) signatureType).getItemType() instanceof AnyDataType) {
return false;
}
if (signatureType instanceof AnyDataType || signatureType instanceof FollowToAnyDataType) {
return false;
}
return realType.toCatalogDataType().matchesType(signatureType.toCatalogDataType());
} catch (Throwable t) {

13
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/NullOrIdenticalSignature.java
View File

@ -18,7 +18,6 @@
package org.apache.doris.nereids.trees.expressions.functions;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.NullType;
import org.apache.doris.nereids.types.coercion.AnyDataType;
@ -37,13 +36,13 @@ public interface NullOrIdenticalSignature extends ComputeSignature {
try {
// TODO: copy matchesType to DataType
// TODO: resolve AnyDataType invoke toCatalogDataType
if (signatureType instanceof ArrayType) {
if (((ArrayType) signatureType).getItemType() instanceof AnyDataType) {
return false;
}
if (realType instanceof NullType) {
return true;
}
return realType instanceof NullType
|| realType.toCatalogDataType().matchesType(signatureType.toCatalogDataType());
if (signatureType instanceof AnyDataType) {
return false;
}
return realType.toCatalogDataType().matchesType(signatureType.toCatalogDataType());
} catch (Throwable t) {
// the signatureType maybe DataType and can not cast to catalog data type.
return false;

71
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
View File

@ -48,42 +48,38 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
private final List<NamedExpression> projects;
private final List<NamedExpression> excepts;
private final boolean isDistinct;
// For project nodes under union, erasure cannot be configured, so add this flag.
private final boolean canEliminate;
private final boolean isDistinct;
public LogicalProject(List<NamedExpression> projects, CHILD_TYPE child) {
this(projects, ImmutableList.of(), true, child, false);
this(projects, ImmutableList.of(), false, true, child);
}
/**
* only for test.
*/
public LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts, CHILD_TYPE child) {
this(projects, excepts, true, child, false);
this(projects, excepts, false, true, child);
}
public LogicalProject(List<NamedExpression> projects, boolean isDistinct, CHILD_TYPE child) {
this(projects, ImmutableList.of(), isDistinct, true, child);
}
public LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts,
boolean canEliminate, CHILD_TYPE child) {
this(projects, excepts, canEliminate, Optional.empty(), Optional.empty(), child, false);
}
public LogicalProject(List<NamedExpression> projects, CHILD_TYPE child, boolean isDistinct) {
this(projects, ImmutableList.of(), true, child, isDistinct);
}
public LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts, CHILD_TYPE child,
boolean isDistinct) {
this(projects, excepts, true, child, isDistinct);
boolean isDistinct, CHILD_TYPE child) {
this(projects, excepts, isDistinct, true, child);
}
private LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts,
boolean canEliminate, CHILD_TYPE child, boolean isDistinct) {
this(projects, excepts, canEliminate, Optional.empty(), Optional.empty(), child, isDistinct);
boolean isDistinct, boolean canEliminate, CHILD_TYPE child) {
this(projects, excepts, isDistinct, canEliminate, Optional.empty(), Optional.empty(), child);
}
private LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts, boolean canEliminate,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
CHILD_TYPE child, boolean isDistinct) {
private LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts, boolean isDistinct,
boolean canEliminate, Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, CHILD_TYPE child) {
super(PlanType.LOGICAL_PROJECT, groupExpression, logicalProperties, child);
Preconditions.checkArgument(projects != null, "projects can not be null");
// only ColumnPrune rule may produce empty projects, this happens in rewrite phase
@ -94,8 +90,8 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
? ImmutableList.of(ExpressionUtils.selectMinimumColumn(child.getOutput()))
: projects;
this.excepts = ImmutableList.copyOf(excepts);
this.canEliminate = canEliminate;
this.isDistinct = isDistinct;
this.canEliminate = canEliminate;
}
/**
@ -156,7 +152,7 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
&& excepts.equals(that.excepts)
&& canEliminate == that.canEliminate
&& isDistinct == that.isDistinct;
// TODO: should add exprId for UnBoundStar and BoundStar for equality comparasion
// TODO: should add exprId for UnBoundStar and BoundStar for equality comparison
if (!projects.isEmpty() && (projects.get(0) instanceof UnboundStar || projects.get(0) instanceof BoundStar)) {
equal = child().getLogicalProperties().equals(that.child().getLogicalProperties());
}
@ -171,43 +167,43 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
@Override
public LogicalProject<Plan> withChildren(List<Plan> children) {
Preconditions.checkArgument(children.size() == 1);
return new LogicalProject<>(projects, excepts, canEliminate, children.get(0), isDistinct);
return new LogicalProject<>(projects, excepts, isDistinct, canEliminate, children.get(0));
}
@Override
public LogicalProject<Plan> withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalProject<>(projects, excepts, canEliminate,
groupExpression, Optional.of(getLogicalProperties()), child(), isDistinct);
return new LogicalProject<>(projects, excepts, isDistinct, canEliminate,
groupExpression, Optional.of(getLogicalProperties()), child());
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
Preconditions.checkArgument(children.size() == 1);
return new LogicalProject<>(projects, excepts, canEliminate,
groupExpression, logicalProperties, children.get(0), isDistinct);
return new LogicalProject<>(projects, excepts, isDistinct, canEliminate,
groupExpression, logicalProperties, children.get(0));
}
public LogicalProject<Plan> withEliminate(boolean isEliminate) {
return new LogicalProject<>(projects, excepts, isEliminate, child(), isDistinct);
return new LogicalProject<>(projects, excepts, isDistinct, isEliminate, child());
}
public LogicalProject<Plan> withProjects(List<NamedExpression> projects) {
return new LogicalProject<>(projects, excepts, canEliminate, child(), isDistinct);
return new LogicalProject<>(projects, excepts, isDistinct, canEliminate, child());
}
public LogicalProject<Plan> withProjectsAndChild(List<NamedExpression> projects, Plan child) {
return new LogicalProject<>(projects, excepts, canEliminate, child, isDistinct);
}
public boolean canEliminate() {
return canEliminate;
return new LogicalProject<>(projects, excepts, isDistinct, canEliminate, child);
}
public boolean isDistinct() {
return isDistinct;
}
public boolean canEliminate() {
return canEliminate;
}
@Override
public List<NamedExpression> getOutputs() {
return projects;
@ -229,9 +225,4 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
logicalProject.put("Properties", properties);
return logicalProject;
}
public LogicalProject<Plan> readFromJson(JSONObject logicalProject) {
return new LogicalProject<>(ImmutableList.of(new UnboundStar(ImmutableList.of())),
null, null, isDistinct);
}
}

6
fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
View File

@ -91,7 +91,9 @@ import org.apache.doris.nereids.types.TimeType;
import org.apache.doris.nereids.types.TimeV2Type;
import org.apache.doris.nereids.types.TinyIntType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.nereids.types.coercion.AnyDataType;
import org.apache.doris.nereids.types.coercion.CharacterType;
import org.apache.doris.nereids.types.coercion.FollowToAnyDataType;
import org.apache.doris.nereids.types.coercion.FractionalType;
import org.apache.doris.nereids.types.coercion.IntegralType;
import org.apache.doris.nereids.types.coercion.NumericType;
@ -183,7 +185,9 @@ public class TypeCoercionUtils {
// TODO: complete the cast logic like FunctionCallExpr.analyzeImpl
boolean legacyCastCompatible = false;
try {
legacyCastCompatible = !input.toCatalogDataType().matchesType(expected.toCatalogDataType());
if (!(expected instanceof AnyDataType) && !(expected instanceof FollowToAnyDataType)) {
legacyCastCompatible = !input.toCatalogDataType().matchesType(expected.toCatalogDataType());
}
} catch (Throwable t) {
// ignore.
}