database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](decimal) use new way for decimal arithmetic precision promotion (#27787)

Browse Source 
* [DNM](decimal) use new way for decimal arithmetic precision promotion

* [improvement](decimal) [DNM](decimal) use new way for decimal arithmetic precision promotion
1. [DNM](decimal) use new way for decimal arithmetic precision promotion
2. throw exception if it overflows for decimal arithmetics
3. throw exception if it overflows when casting among number types

* fix compile error of gcc

* improvement

---------

Co-authored-by: morrySnow <morrysnow@126.com>
This commit is contained in:
TengJianPing
2023-12-05 12:54:40 +08:00
committed by GitHub
parent ca6949ee3e
commit 17016b9797
121 changed files with 17662 additions and 12466 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
View File

@ -39,7 +39,6 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.TreeNode;
import org.apache.doris.common.io.Writable;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.rewrite.mvrewrite.MVExprEquivalent;
import org.apache.doris.statistics.ExprStats;
@ -2364,22 +2363,12 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
}
if (fn.functionName().equalsIgnoreCase(Operator.MULTIPLY.getName())
&& fn.getReturnType().isDecimalV3()) {
if (ConnectContext.get() != null
&& ConnectContext.get().getSessionVariable().checkOverflowForDecimal()) {
return true;
} else {
return hasNullableChild(children);
}
return hasNullableChild(children);
}
if ((fn.functionName().equalsIgnoreCase(Operator.ADD.getName())
|| fn.functionName().equalsIgnoreCase(Operator.SUBTRACT.getName()))
&& fn.getReturnType().isDecimalV3()) {
if (ConnectContext.get() != null
&& ConnectContext.get().getSessionVariable().checkOverflowForDecimal()) {
return true;
} else {
return hasNullableChild(children);
}
return hasNullableChild(children);
}
if (fn.functionName().equalsIgnoreCase("group_concat")) {
int size = Math.min(fn.getNumArgs(), children.size());

15
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Add.java
View File

@ -18,7 +18,7 @@
package org.apache.doris.nereids.trees.expressions;
import org.apache.doris.analysis.ArithmeticExpr.Operator;
import org.apache.doris.nereids.trees.expressions.functions.CheckOverflowNullable;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.DecimalV3Type;
@ -31,7 +31,7 @@ import java.util.List;
/**
* Add Expression.
*/
public class Add extends BinaryArithmetic implements CheckOverflowNullable {
public class Add extends BinaryArithmetic implements PropagateNullable {
public Add(Expression left, Expression right) {
super(ImmutableList.of(left, right), Operator.ADD);
@ -49,9 +49,9 @@ public class Add extends BinaryArithmetic implements CheckOverflowNullable {
@Override
public DecimalV3Type getDataTypeForDecimalV3(DecimalV3Type t1, DecimalV3Type t2) {
DecimalV3Type decimalV3Type = (DecimalV3Type) DecimalV3Type.widerDecimalV3Type(t1, t2, false);
return DecimalV3Type.createDecimalV3Type(decimalV3Type.getPrecision() + 1,
decimalV3Type.getScale());
int targetScale = Math.max(t1.getScale(), t2.getScale());
int integralPart = Math.max(t1.getRange(), t2.getRange());
return processDecimalV3OverFlow(integralPart + 1, targetScale, integralPart);
}
@Override
@ -59,11 +59,6 @@ public class Add extends BinaryArithmetic implements CheckOverflowNullable {
return super.getDataTypeForOthers(t1, t2).promotion();
}
@Override
public boolean nullable() {
return CheckOverflowNullable.super.nullable();
}
@Override
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitAdd(this, context);

23
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryArithmetic.java
View File

@ -27,6 +27,7 @@ import org.apache.doris.nereids.types.DecimalV2Type;
import org.apache.doris.nereids.types.DecimalV3Type;
import org.apache.doris.nereids.types.coercion.NumericType;
import org.apache.doris.nereids.util.TypeCoercionUtils;
import org.apache.doris.qe.ConnectContext;
import java.util.List;
@ -93,4 +94,26 @@ public abstract class BinaryArithmetic extends BinaryOperator implements Propaga
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitBinaryArithmetic(this, context);
}
protected DecimalV3Type processDecimalV3OverFlow(int integralPart, int targetScale, int maxIntegralPart) {
int precision = integralPart + targetScale;
int scale = targetScale;
boolean enableDecimal256 = false;
ConnectContext connectContext = ConnectContext.get();
if (connectContext != null) {
enableDecimal256 = connectContext.getSessionVariable().isEnableDecimal256();
}
if (enableDecimal256) {
if (precision > DecimalV3Type.MAX_DECIMAL256_PRECISION) {
precision = DecimalV3Type.MAX_DECIMAL256_PRECISION;
scale = precision - maxIntegralPart;
}
} else {
if (precision > DecimalV3Type.MAX_DECIMAL128_PRECISION) {
precision = DecimalV3Type.MAX_DECIMAL128_PRECISION;
scale = precision - maxIntegralPart;
}
}
return DecimalV3Type.createDecimalV3Type(precision, scale);
}
}

50
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Divide.java
View File

@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.DecimalV3Type;
import org.apache.doris.nereids.types.DoubleType;
import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@ -67,21 +68,40 @@ public class Divide extends BinaryArithmetic implements AlwaysNullable {
@Override
public DecimalV3Type getDataTypeForDecimalV3(DecimalV3Type t1, DecimalV3Type t2) {
int retPercision = t1.getPrecision() + t2.getScale() + Config.div_precision_increment;
Preconditions.checkState(retPercision <= DecimalV3Type.MAX_DECIMAL256_PRECISION,
"target precision " + retPercision + " larger than precision "
+ DecimalV3Type.MAX_DECIMAL256_PRECISION + " in Divide return type");
int retScale = t1.getScale() + t2.getScale()
+ Config.div_precision_increment;
int targetPercision = retPercision;
int targetScale = t1.getScale() + t2.getScale();
Preconditions.checkState(targetPercision >= targetScale,
"target scale " + targetScale + " larger than precision " + retPercision
+ " in Divide return type");
Preconditions.checkState(retPercision >= retScale,
"scale " + retScale + " larger than precision " + retPercision
+ " in Divide return type");
return DecimalV3Type.createDecimalV3Type(retPercision, retScale);
int precision = t1.getPrecision() + t2.getScale() + Config.div_precision_increment;
int scale = t1.getScale();
boolean enableDecimal256 = false;
int defaultScale = 6;
ConnectContext connectContext = ConnectContext.get();
if (connectContext != null) {
enableDecimal256 = connectContext.getSessionVariable().isEnableDecimal256();
defaultScale = connectContext.getSessionVariable().decimalOverflowScale;
}
if (enableDecimal256 && precision > DecimalV3Type.MAX_DECIMAL256_PRECISION) {
int integralPartBoundary = DecimalV3Type.MAX_DECIMAL256_PRECISION - defaultScale;
if (precision - scale < integralPartBoundary) {
// retains more int part
scale = DecimalV3Type.MAX_DECIMAL256_PRECISION - (precision - scale);
} else if (precision - scale > integralPartBoundary && scale < defaultScale) {
// scale not change, retains more scale part
} else {
scale = defaultScale;
}
precision = DecimalV3Type.MAX_DECIMAL256_PRECISION;
} else if (!enableDecimal256 && precision > DecimalV3Type.MAX_DECIMAL128_PRECISION) {
int integralPartBoundary = DecimalV3Type.MAX_DECIMAL128_PRECISION - defaultScale;
if (precision - scale < integralPartBoundary) {
// retains more int part
scale = DecimalV3Type.MAX_DECIMAL128_PRECISION - (precision - scale);
} else if (precision - scale > integralPartBoundary && scale < defaultScale) {
// scale not change, retains more scale part
} else {
scale = defaultScale;
}
precision = DecimalV3Type.MAX_DECIMAL128_PRECISION;
}
scale = Math.min(precision, scale + t2.getScale() + Config.div_precision_increment);
return DecimalV3Type.createDecimalV3Type(precision, scale);
}
@Override

8
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Mod.java
View File

@ -49,11 +49,9 @@ public class Mod extends BinaryArithmetic implements AlwaysNullable {
@Override
public DecimalV3Type getDataTypeForDecimalV3(DecimalV3Type t1, DecimalV3Type t2) {
// TODO use max int part + max scale of two operands as result type
// because BE require the result and operands types are the exact the same decimalv3 type
int scale = Math.max(t1.getScale(), t2.getScale());
int precision = Math.max(t1.getRange(), t2.getRange()) + scale;
return DecimalV3Type.createDecimalV3Type(precision, scale);
int targetScale = Math.max(t1.getScale(), t2.getScale());
int integralPart = Math.max(t1.getRange(), t2.getRange());
return processDecimalV3OverFlow(integralPart, targetScale, integralPart);
}
@Override

59
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Multiply.java
View File

@ -18,7 +18,7 @@
package org.apache.doris.nereids.trees.expressions;
import org.apache.doris.analysis.ArithmeticExpr.Operator;
import org.apache.doris.nereids.trees.expressions.functions.CheckOverflowNullable;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.DecimalV3Type;
@ -32,7 +32,7 @@ import java.util.List;
/**
* Multiply Expression.
*/
public class Multiply extends BinaryArithmetic implements CheckOverflowNullable {
public class Multiply extends BinaryArithmetic implements PropagateNullable {
public Multiply(Expression left, Expression right) {
super(ImmutableList.of(left, right), Operator.MULTIPLY);
@ -50,26 +50,42 @@ public class Multiply extends BinaryArithmetic implements CheckOverflowNullable
@Override
public DecimalV3Type getDataTypeForDecimalV3(DecimalV3Type t1, DecimalV3Type t2) {
int retPercision = t1.getPrecision() + t2.getPrecision();
int retPrecision = t1.getPrecision() + t2.getPrecision();
int retScale = t1.getScale() + t2.getScale();
if (retPercision > DecimalV3Type.MAX_DECIMAL128_PRECISION) {
boolean enableDecimal256 = false;
ConnectContext connectContext = ConnectContext.get();
if (connectContext != null) {
enableDecimal256 = connectContext.getSessionVariable().isEnableDecimal256();
}
if (enableDecimal256) {
if (retPercision > DecimalV3Type.MAX_DECIMAL256_PRECISION) {
retPercision = DecimalV3Type.MAX_DECIMAL256_PRECISION;
}
} else {
retPercision = DecimalV3Type.MAX_DECIMAL128_PRECISION;
}
boolean enableDecimal256 = false;
int defaultScale = 6;
ConnectContext connectContext = ConnectContext.get();
if (connectContext != null) {
enableDecimal256 = connectContext.getSessionVariable().isEnableDecimal256();
defaultScale = connectContext.getSessionVariable().decimalOverflowScale;
}
Preconditions.checkState(retPercision >= retScale,
"scale " + retScale + " larger than precision " + retPercision
if (!enableDecimal256 && retPrecision > DecimalV3Type.MAX_DECIMAL128_PRECISION) {
int integralPartBoundary = DecimalV3Type.MAX_DECIMAL128_PRECISION - defaultScale;
if (retPrecision - retScale < integralPartBoundary) {
// retains more int part
retScale = DecimalV3Type.MAX_DECIMAL128_PRECISION - (retPrecision - retScale);
} else if (retPrecision - retScale > integralPartBoundary && retScale < defaultScale) {
// retScale not change, retains more scale part
} else {
retScale = defaultScale;
}
retPrecision = DecimalV3Type.MAX_DECIMAL128_PRECISION;
} else if (enableDecimal256 && retPrecision > DecimalV3Type.MAX_DECIMAL256_PRECISION) {
int integralPartBoundary = DecimalV3Type.MAX_DECIMAL256_PRECISION - defaultScale;
if (retPrecision - retScale < integralPartBoundary) {
// retains more int part
retScale = DecimalV3Type.MAX_DECIMAL256_PRECISION - (retPrecision - retScale);
} else if (retPrecision - retScale > integralPartBoundary && retScale < defaultScale) {
// retScale not change, retains more scale part
} else {
retScale = defaultScale;
}
retPrecision = DecimalV3Type.MAX_DECIMAL256_PRECISION;
}
Preconditions.checkState(retPrecision >= retScale,
"scale " + retScale + " larger than precision " + retPrecision
+ " in Multiply return type");
return DecimalV3Type.createDecimalV3Type(retPercision, retScale);
return DecimalV3Type.createDecimalV3Type(retPrecision, retScale);
}
@Override
@ -77,11 +93,6 @@ public class Multiply extends BinaryArithmetic implements CheckOverflowNullable
return super.getDataTypeForOthers(t1, t2).promotion();
}
@Override
public boolean nullable() {
return CheckOverflowNullable.super.nullable();
}
@Override
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitMultiply(this, context);

15
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Subtract.java
View File

@ -18,7 +18,7 @@
package org.apache.doris.nereids.trees.expressions;
import org.apache.doris.analysis.ArithmeticExpr.Operator;
import org.apache.doris.nereids.trees.expressions.functions.CheckOverflowNullable;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.DecimalV3Type;
@ -31,7 +31,7 @@ import java.util.List;
/**
* Subtract Expression. BinaryExpression.
*/
public class Subtract extends BinaryArithmetic implements CheckOverflowNullable {
public class Subtract extends BinaryArithmetic implements PropagateNullable {
public Subtract(Expression left, Expression right) {
super(ImmutableList.of(left, right), Operator.SUBTRACT);
@ -49,9 +49,9 @@ public class Subtract extends BinaryArithmetic implements CheckOverflowNullable
@Override
public DecimalV3Type getDataTypeForDecimalV3(DecimalV3Type t1, DecimalV3Type t2) {
DecimalV3Type decimalV3Type = (DecimalV3Type) DecimalV3Type.widerDecimalV3Type(t1, t2, false);
return DecimalV3Type.createDecimalV3Type(decimalV3Type.getPrecision() + 1,
decimalV3Type.getScale());
int targetScale = Math.max(t1.getScale(), t2.getScale());
int integralPart = Math.max(t1.getRange(), t2.getRange());
return processDecimalV3OverFlow(integralPart + 1, targetScale, integralPart);
}
@Override
@ -59,11 +59,6 @@ public class Subtract extends BinaryArithmetic implements CheckOverflowNullable
return super.getDataTypeForOthers(t1, t2).promotion();
}
@Override
public boolean nullable() {
return CheckOverflowNullable.super.nullable();
}
@Override
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitSubtract(this, context);

34
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/CheckOverflowNullable.java
View File

@ -1,34 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.expressions.functions;
import org.apache.doris.qe.ConnectContext;
/**
* if session variable check_overflow_for_decimal set to true, the expression's return always nullable
*/
public interface CheckOverflowNullable extends PropagateNullable {
@Override
default boolean nullable() {
if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().checkOverflowForDecimal()) {
return true;
} else {
return PropagateNullable.super.nullable();
}
}
}

40
fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
View File

@ -104,7 +104,6 @@ import org.apache.doris.nereids.types.coercion.FractionalType;
import org.apache.doris.nereids.types.coercion.IntegralType;
import org.apache.doris.nereids.types.coercion.NumericType;
import org.apache.doris.nereids.types.coercion.PrimitiveType;
import org.apache.doris.qe.ConnectContext;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
@ -686,20 +685,15 @@ public class TypeCoercionUtils {
DataType commonType = DoubleType.INSTANCE;
if (t1.isFloatLikeType() || t2.isFloatLikeType()) {
// double type
} else if (t1.isDecimalV3Type() || t2.isDecimalV3Type()) {
} else if (t1.isDecimalV3Type() || t2.isDecimalV3Type()
// decimalv2 vs bigint, largeint treat as decimalv3
|| ((t1.isBigIntType() || t1.isLargeIntType()) && t2.isDecimalV2Type())
|| (t1.isDecimalV2Type() && (t2.isBigIntType() || t2.isLargeIntType()))) {
// divide should cast to precision and target scale
DecimalV3Type retType;
DecimalV3Type dt1 = DecimalV3Type.forType(t1);
DecimalV3Type dt2 = DecimalV3Type.forType(t2);
try {
retType = divide.getDataTypeForDecimalV3(dt1, dt2);
} catch (Exception e) {
// exception means overflow.
return castChildren(divide, left, right, DoubleType.INSTANCE);
}
return divide.withChildren(castIfNotSameType(left,
DecimalV3Type.createDecimalV3Type(retType.getPrecision(), retType.getScale())),
castIfNotSameType(right, dt2));
DecimalV3Type retType = divide.getDataTypeForDecimalV3(dt1, dt2);
return divide.withChildren(castIfNotSameType(left, retType), castIfNotSameType(right, dt2));
} else if (t1.isDecimalV2Type() || t2.isDecimalV2Type()) {
commonType = DecimalV2Type.SYSTEM_DEFAULT;
}
@ -792,18 +786,16 @@ public class TypeCoercionUtils {
commonType = DoubleType.INSTANCE;
}
if (t1.isDecimalV3Type() && t2.isDecimalV2Type()
|| t1.isDecimalV2Type() && t2.isDecimalV3Type()) {
// we treat decimalv2 vs dicimalv3, largeint or bigint as decimalv3 way.
if ((t1.isDecimalV3Type() || t1.isBigIntType() || t1.isLargeIntType()) && t2.isDecimalV2Type()
|| t1.isDecimalV2Type() && (t2.isDecimalV3Type() || t2.isBigIntType() || t2.isLargeIntType())) {
return processDecimalV3BinaryArithmetic(binaryArithmetic, left, right);
}
if (t1.isDecimalV2Type() || t2.isDecimalV2Type()) {
// to be consitent with old planner
// to be consistent with old planner
// see findCommonType() method in ArithmeticExpr.java
commonType = t1.isDecimalV2Type() && t2.isDecimalV2Type()
|| (ConnectContext.get() != null
&& ConnectContext.get().getSessionVariable().roundPreciseDecimalV2Value)
? DecimalV2Type.SYSTEM_DEFAULT : DoubleType.INSTANCE;
commonType = DecimalV2Type.SYSTEM_DEFAULT;
}
boolean isBitArithmetic = binaryArithmetic instanceof BitAnd
@ -833,7 +825,7 @@ public class TypeCoercionUtils {
return castChildren(binaryArithmetic, left, right, commonType);
}
// double and float already process, we only process decimalv2 and fixed point number.
// double and float already process, we only process decimalv3 and fixed point number.
if (t1 instanceof DecimalV3Type || t2 instanceof DecimalV3Type) {
return processDecimalV3BinaryArithmetic(binaryArithmetic, left, right);
}
@ -1591,13 +1583,7 @@ public class TypeCoercionUtils {
DecimalV3Type.forType(TypeCoercionUtils.getNumResultType(right.getDataType()));
// check return type whether overflow, if true, turn to double
DecimalV3Type retType;
try {
retType = binaryArithmetic.getDataTypeForDecimalV3(dt1, dt2);
} catch (Exception e) {
// exception means overflow.
return castChildren(binaryArithmetic, left, right, DoubleType.INSTANCE);
}
DecimalV3Type retType = binaryArithmetic.getDataTypeForDecimalV3(dt1, dt2);
// add, subtract and mod should cast children to exactly same type as return type
if (binaryArithmetic instanceof Add || binaryArithmetic instanceof Subtract

13
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -242,6 +242,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String CHECK_OVERFLOW_FOR_DECIMAL = "check_overflow_for_decimal";
public static final String DECIMAL_OVERFLOW_SCALE = "decimal_overflow_scale";
public static final String TRIM_TAILING_SPACES_FOR_EXTERNAL_TABLE_QUERY
= "trim_tailing_spaces_for_external_table_query";
@ -923,8 +925,15 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = ENABLE_PROJECTION)
private boolean enableProjection = true;
@VariableMgr.VarAttr(name = CHECK_OVERFLOW_FOR_DECIMAL)
private boolean checkOverflowForDecimal = false;
@VariableMgr.VarAttr(name = CHECK_OVERFLOW_FOR_DECIMAL, varType = VariableAnnotation.DEPRECATED)
private boolean checkOverflowForDecimal = true;
@VariableMgr.VarAttr(name = DECIMAL_OVERFLOW_SCALE, needForward = true, description = {
"当decimal数值计算结果精度溢出时，计算结果最多可保留的小数位数", "When the precision of the result of"
+ " a decimal numerical calculation overflows,"
+ "the maximum number of decimal scale that the result can be retained"
})
public int decimalOverflowScale = 6;
@VariableMgr.VarAttr(name = ENABLE_DPHYP_OPTIMIZER)
public boolean enableDPHypOptimizer = false;