database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[bug](fold) fix fold date/datetime error as null (#33845)

Browse Source 
the LocalDateTime/LocalDate value maybe null, so need check it firstly.
if it's null, could return NullLiteral directly.
This commit is contained in:
zhangstar333
2024-04-23 20:51:40 +08:00
committed by yiguolei
parent f6ec64c6ad
commit d5275c55b4
3 changed files with 51 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
View File

@ -33,11 +33,12 @@ import org.apache.doris.nereids.rules.expression.ExpressionMatchingContext;
import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher;
import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Match;
import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Sleep;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Tokenize;
import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral;
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
@ -197,15 +198,14 @@ public class FoldConstantRuleOnBE implements ExpressionPatternRuleFactory {
|| expr.getDataType().isIPv6Type()) {
return;
}
if (skipSleepFunction(expr) || (expr instanceof TableGeneratingFunction)) {
// first need pass PlanTranslatorContext value,
// and ArrayItemReference translate, can't findColumnRef
// Match need give more info rather then as left child a NULL, in
// match_phrase_prefix/MATCH_PHRASE/MATCH_PHRASE/MATCH_ANY
if (skipSleepFunction(expr) || (expr instanceof TableGeneratingFunction)
|| (expr instanceof ArrayItemReference) || (expr instanceof Match)) {
return;
}
// Tokenize function want check the second child literal must be string type
// and properties format, it's a little special,
// maybe check in checkLegalityBeforeTypeCoercion function?
if (expr instanceof Tokenize) {
expr.checkLegalityAfterRewrite();
}
String id = idGenerator.getNextId().toString();
constMap.put(id, expr);
Expr staleExpr;
@ -286,7 +286,7 @@ public class FoldConstantRuleOnBE implements ExpressionPatternRuleFactory {
List<Literal> resultExpression = getResultExpression(type, resultContent);
if (resultExpression.isEmpty()) {
ret = constMap.get(e1.getKey());
LOG.debug("Be constant folding convert {} to {} failed query_id: {}", e1.getKey(), ret,
LOG.warn("Be constant folding convert {} to {} failed query_id: {}", e1.getKey(), ret,
DebugUtil.printId(context.queryId()));
} else {
ret = resultExpression.get(0);
@ -320,8 +320,7 @@ public class FoldConstantRuleOnBE implements ExpressionPatternRuleFactory {
if (type.isNullType()) {
int num = resultContent.getNullMapCount();
for (int i = 0; i < num; ++i) {
Literal literal = new NullLiteral(type);
res.add(literal);
res.add(new NullLiteral(type));
}
} else if (type.isBooleanType()) {
int num = resultContent.getUint32ValueCount();
@ -412,19 +411,27 @@ public class FoldConstantRuleOnBE implements ExpressionPatternRuleFactory {
for (int i = 0; i < num; ++i) {
long uint64Value = resultContent.getUint64Value(i);
LocalDateTime dateTimeV2 = convertToJavaDateTimeV2(uint64Value);
Literal literal = new DateTimeV2Literal((DateTimeV2Type) type, dateTimeV2.getYear(),
dateTimeV2.getMonthValue(), dateTimeV2.getDayOfMonth(), dateTimeV2.getHour(),
dateTimeV2.getMinute(), dateTimeV2.getSecond(), dateTimeV2.getNano() / 1000);
res.add(literal);
if (dateTimeV2 == null && resultContent.hasHasNull()) {
res.add(new NullLiteral(type));
} else {
Literal literal = new DateTimeV2Literal((DateTimeV2Type) type, dateTimeV2.getYear(),
dateTimeV2.getMonthValue(), dateTimeV2.getDayOfMonth(), dateTimeV2.getHour(),
dateTimeV2.getMinute(), dateTimeV2.getSecond(), dateTimeV2.getNano() / 1000);
res.add(literal);
}
}
} else if (type.isDateV2Type()) {
int num = resultContent.getUint32ValueCount();
for (int i = 0; i < num; ++i) {
int uint32Value = resultContent.getUint32Value(i);
LocalDate localDate = convertToJavaDateV2(uint32Value);
DateV2Literal dateV2Literal = new DateV2Literal(localDate.getYear(), localDate.getMonthValue(),
localDate.getDayOfMonth());
res.add(dateV2Literal);
if (localDate == null && resultContent.hasHasNull()) {
res.add(new NullLiteral(type));
} else {
DateV2Literal dateV2Literal = new DateV2Literal(localDate.getYear(), localDate.getMonthValue(),
localDate.getDayOfMonth());
res.add(dateV2Literal);
}
}
} else if (type.isIPv4Type()) {
int num = resultContent.getUint32ValueCount();
@ -438,8 +445,11 @@ public class FoldConstantRuleOnBE implements ExpressionPatternRuleFactory {
for (int i = 0; i < num; ++i) {
String stringValue = resultContent.getStringValue(i);
// maybe need handle NULL_IN_CSV_FOR_ORDINARY_TYPE = "\\N";
JsonLiteral jsonLiteral = new JsonLiteral(stringValue);
res.add(jsonLiteral);
if ("\\N".equalsIgnoreCase(stringValue) && resultContent.hasHasNull()) {
res.add(new NullLiteral(type));
} else {
res.add(new JsonLiteral(stringValue));
}
}
} else if (type.isStringLikeType()) {
int num = resultContent.getStringValueCount();

10
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Tokenize.java
View File

@ -23,6 +23,7 @@ import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@ -54,8 +55,13 @@ public class Tokenize extends ScalarFunction
}
@Override
public void checkLegalityAfterRewrite() {
if (!(child(1) instanceof StringLikeLiteral)) {
public void checkLegalityBeforeTypeCoercion() {
Expression rightChild = child(1);
// tokenize(k7, null) could return NULL
if (rightChild instanceof NullLiteral) {
return;
}
if (!(rightChild instanceof StringLikeLiteral)) {
throw new AnalysisException("tokenize second argument must be string literal");
}
String properties = ((StringLikeLiteral) child(1)).value;

13
regression-test/suites/query_p0/explain/test_pushdown_explain.groovy
View File

@ -23,4 +23,17 @@ suite("test_pushdown_explain") {
contains "PREDICATES:"
}
qt_select "select k1 from baseall where k1 = 1"
explain {
sql("select cast(0 as datetime)")
contains "NULL"
}
explain {
sql("select cast(0 as date)")
contains "NULL"
}
explain {
sql("select cast(\"6.a8\" as json)")
contains "NULL"
}
}