database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](Nerieds) fix bucket shuffle plan and cost model bugs and add new function add_months (#18836)

Browse Source 
fix
1. fix varchar(1) compare to varchar(2) bug
2. fix bucket shuffle join's cost model bug

feature:
1. support add_months function
This commit is contained in:
924060929
2023-04-26 13:52:44 +08:00
committed by GitHub
parent ca19b972cc
commit 7a786c3b09
10 changed files with 64 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
View File

@ -541,7 +541,7 @@ public class BuiltinScalarFunctions implements FunctionHelper {
scalar(MonthCeil.class, "month_ceil"),
scalar(MonthFloor.class, "month_floor"),
scalar(MonthName.class, "monthname"),
scalar(MonthsAdd.class, "months_add"),
scalar(MonthsAdd.class, "months_add", "add_months"),
scalar(MonthsDiff.class, "months_diff"),
scalar(MonthsSub.class, "months_sub"),
scalar(MultiMatchAny.class, "multi_match_any"),

13
fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java
View File

@ -32,7 +32,9 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.statistics.Statistics;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@ -42,6 +44,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
/**
* Representation for group expression in cascades optimizer.
@ -202,6 +205,11 @@ public class GroupExpression {
return lowestCostTable.get(require).second;
}
public List<PhysicalProperties> getInputPropertiesListOrEmpty(PhysicalProperties require) {
Pair<Cost, List<PhysicalProperties>> costAndChildRequire = lowestCostTable.get(require);
return costAndChildRequire == null ? ImmutableList.of() : costAndChildRequire.second;
}
/**
* Add a (outputProperties) -> (cost, childrenInputProperties) in lowestCostTable.
* if the outputProperties exists, will be covered.
@ -318,9 +326,8 @@ public class GroupExpression {
builder.append(" cost=").append(format.format((long) cost));
builder.append(" estRows=").append(format.format(estOutputRowCount));
builder.append(" (plan=").append(plan.toString()).append(") children=[");
for (Group group : children) {
builder.append(group.getGroupId()).append(" ");
}
builder.append(Joiner.on(", ").join(
children.stream().map(Group::getGroupId).collect(Collectors.toList())));
builder.append("]");
return builder.toString();
}

9
fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
View File

@ -717,13 +717,16 @@ public class Memo {
}
}
builder.append(" lowest Plan(cost, properties, plan)");
builder.append(" lowest Plan(cost, properties, plan, childrenRequires)");
group.getAllProperties().forEach(
prop -> {
Optional<Pair<Cost, GroupExpression>> costAndGroupExpression = group.getLowestCostPlan(prop);
if (costAndGroupExpression.isPresent()) {
builder.append("\n " + costAndGroupExpression.get().first.getValue() + " " + prop)
.append("\n ").append(costAndGroupExpression.get().second);
Cost cost = costAndGroupExpression.get().first;
GroupExpression child = costAndGroupExpression.get().second;
builder.append("\n " + cost.getValue() + " " + prop)
.append("\n ").append(child)
.append("\n " + child.getInputPropertiesListOrEmpty(prop));
}
}
);

24
fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
View File

@ -21,8 +21,11 @@ import org.apache.doris.nereids.PlanContext;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction;
import org.apache.doris.nereids.trees.plans.Plan;
@ -43,6 +46,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.util.JoinUtils;
import org.apache.doris.qe.ConnectContext;
@ -137,11 +141,16 @@ public class ChildOutputPropertyDeriver extends PlanVisitor<PhysicalProperties,
for (NamedExpression namedExpression : project.getProjects()) {
if (namedExpression instanceof Alias) {
Alias alias = (Alias) namedExpression;
if (alias.child() instanceof SlotReference) {
projections.put(((SlotReference) alias.child()).getExprId(), alias.getExprId());
Expression child = alias.child();
if (child instanceof SlotReference) {
projections.put(((SlotReference) child).getExprId(), alias.getExprId());
} else if (child instanceof Cast && child.child(0) instanceof Slot
&& isSameHashValue(child.child(0).getDataType(), child.getDataType())) {
// cast(slot as varchar(10)) can do projection if slot is varchar(3)
projections.put(((Slot) child.child(0)).getExprId(), alias.getExprId());
} else {
obstructions.addAll(
alias.child().getInputSlots().stream()
child.getInputSlots().stream()
.map(NamedExpression::getExprId)
.collect(Collectors.toSet()));
}
@ -268,4 +277,13 @@ public class ChildOutputPropertyDeriver extends PlanVisitor<PhysicalProperties,
Preconditions.checkState(childrenOutputProperties.size() == 1);
return childrenOutputProperties.get(0);
}
private boolean isSameHashValue(DataType originType, DataType castType) {
if (originType.isStringLikeType() && (castType.isVarcharType() || castType.isStringType())
&& (castType.width() >= originType.width() || castType.width() < 0)) {
return true;
} else {
return false;
}
}
}

6
fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
View File

@ -126,8 +126,8 @@ public class ChildrenPropertiesRegulator extends PlanVisitor<Double, Void> {
if (ConnectContext.get().getSessionVariable().isEnableBucketShuffleJoin()) {
// We need to recalculate the required property of right child,
// to make right child compatible with left child.
PhysicalProperties rightRequireProperties = calRightRequiredOfBucketShuffleJoin(leftHashSpec,
rightHashSpec);
PhysicalProperties rightRequireProperties = calRightRequiredOfBucketShuffleJoin(
leftHashSpec, rightHashSpec);
if (!rightOutput.equals(rightRequireProperties)) {
updateChildEnforceAndCost(rightChild, rightOutput,
(DistributionSpecHash) rightRequireProperties.getDistributionSpec(), rightLowest.first);
@ -193,8 +193,8 @@ public class ChildrenPropertiesRegulator extends PlanVisitor<Double, Void> {
GroupExpression enforcer = outputDistributionSpec.addEnforcer(child.getOwnerGroup());
jobContext.getCascadesContext().getMemo().addEnforcerPlan(enforcer, child.getOwnerGroup());
Cost totalCost = CostCalculator.addChildCost(enforcer.getPlan(),
currentCost,
CostCalculator.calculateCost(enforcer, Lists.newArrayList(childOutput)),
currentCost,
0);
if (enforcer.updateLowestCostTable(newOutputProperty,

46
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java
View File

@ -33,8 +33,6 @@ import org.apache.doris.nereids.types.LargeIntType;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.commons.lang3.StringUtils;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.Locale;
@ -137,45 +135,7 @@ public abstract class Literal extends Expression implements LeafExpression, Comp
*/
@Override
public int compareTo(Literal other) {
if (isNullLiteral() && other.isNullLiteral()) {
return 0;
} else if (isNullLiteral() || other.isNullLiteral()) {
return isNullLiteral() ? -1 : 1;
}
DataType oType = other.getDataType();
DataType type = getDataType();
if (type.isVarcharType() && oType.isVarcharType()) {
// VarChar type can be different, e.g., VarChar(1) = VarChar(2)
return StringUtils.compare((String) getValue(), (String) other.getValue());
} else if (!type.equals(oType)) {
throw new RuntimeException("data type not equal!");
} else if (type.isBooleanType()) {
return Boolean.compare((boolean) getValue(), (boolean) other.getValue());
} else if (type.isTinyIntType()) {
return Byte.compare((byte) getValue(), (byte) other.getValue());
} else if (type.isSmallIntType()) {
return Short.compare((short) getValue(), (short) other.getValue());
} else if (type.isIntegerType()) {
return Integer.compare((int) getValue(), (int) other.getValue());
} else if (type.isBigIntType()) {
return Long.compare((long) getValue(), (long) other.getValue());
} else if (type.isLargeIntType()) {
return ((BigInteger) getValue()).compareTo((BigInteger) other.getValue());
} else if (type.isFloatType()) {
return Float.compare((float) getValue(), (float) other.getValue());
} else if (type.isDoubleType()) {
return Double.compare((double) getValue(), (double) other.getValue());
} else if (type.isDateLikeType()) {
return Long.compare((Long) getValue(), (Long) other.getValue());
} else if (type.isDecimalV2Type()) {
return ((BigDecimal) getValue()).compareTo((BigDecimal) other.getValue());
} else if (type.isStringLikeType()) {
return StringUtils.compare((String) getValue(), (String) other.getValue());
} else {
throw new RuntimeException(String.format("Literal {} is not supported!", type.toString()));
}
return toLegacyLiteral().compareLiteral(other.toLegacyLiteral());
}
/**
@ -268,10 +228,6 @@ public abstract class Literal extends Expression implements LeafExpression, Comp
throw new AnalysisException("cannot cast " + desc + " from type " + this.dataType + " to type " + targetType);
}
public boolean isCharacterLiteral() {
return this instanceof StringLiteral || this instanceof CharLiteral || this instanceof VarcharLiteral;
}
/** fromLegacyLiteral */
public static Literal fromLegacyLiteral(LiteralExpr literalExpr, Type type) {
DataType dataType = DataType.fromCatalogType(type);

4
fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
View File

@ -367,6 +367,10 @@ public abstract class DataType implements AbstractDataType {
return this instanceof IntegralType && !(this instanceof LargeIntType);
}
public boolean isFloatLikeType() {
return this.isFloatType() || isDoubleType() || isDecimalLikeType();
}
public boolean isTinyIntType() {
return this instanceof TinyIntType;
}

6
fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
View File

@ -296,11 +296,13 @@ public class TypeCoercionUtils {
// process by constant folding
return (T) op.withChildren(left, right);
}
if (left instanceof Literal && ((Literal) left).isCharacterLiteral()) {
if (left instanceof Literal && ((Literal) left).isStringLikeLiteral()
&& !right.getDataType().isStringLikeType()) {
left = TypeCoercionUtils.characterLiteralTypeCoercion(
((Literal) left).getStringValue(), right.getDataType()).orElse(left);
}
if (right instanceof Literal && ((Literal) right).isCharacterLiteral()) {
if (right instanceof Literal && ((Literal) right).isStringLikeLiteral()
&& !left.getDataType().isStringLikeType()) {
right = TypeCoercionUtils.characterLiteralTypeCoercion(
((Literal) right).getStringValue(), left.getDataType()).orElse(right);

9
fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java
View File

@ -154,6 +154,15 @@ public class Statistics {
@Override
public String toString() {
if (Double.isNaN(rowCount)) {
return "NaN";
}
if (Double.POSITIVE_INFINITY == rowCount) {
return "Infinite";
}
if (Double.NEGATIVE_INFINITY == rowCount) {
return "-Infinite";
}
DecimalFormat format = new DecimalFormat("#,###.##");
return format.format(rowCount);
}