database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](function) fix str_to_date default return type scale for nereids (#24932)

Browse Source 
fix str_to_date default return type scale for nereids
This commit is contained in:
zclllyybb
2023-10-20 12:55:49 +08:00
committed by GitHub
parent 0510d548c6
commit dc47087560
8 changed files with 40 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
be/src/vec/functions/function_timestamp.cpp
View File

@ -72,6 +72,7 @@ struct StrToDate {
static DataTypes get_variadic_argument_types() { return {}; }
static DataTypePtr get_return_type_impl(const DataTypes& arguments) {
//TODO: it doesn't matter now. maybe sometime we should find the function signature with return_type together
return make_nullable(std::make_shared<DataTypeDateTime>());
}
@ -115,6 +116,8 @@ struct StrToDate {
auto& rdata = specific_char_column->get_chars();
auto& roffsets = specific_char_column->get_offsets();
// Because of we cant distinguish by return_type when we find function. so the return_type may NOT be same with real return type
// which decided by FE. that's found by which.
ColumnPtr res = nullptr;
WhichDataType which(remove_nullable(block.get_by_position(result).type));
if (which.is_date_time_v2()) {

24
be/src/vec/runtime/vdatetime_value.cpp
View File

@ -2300,14 +2300,28 @@ bool DateV2Value<T>::from_date_format_str(const char* format, int format_len, co
break;
// Micro second
case 'f':
tmp = val + min(6, val_end - val);
if (!str_to_int64(val, &tmp, &int_value)) {
return false;
tmp = val;
// when there's still something to the end, fix the scale of ms.
while (tmp < val_end && isdigit(*tmp)) {
tmp++;
}
if (tmp - val > 6) {
const char* tmp2 = val + 6;
if (!str_to_int64(val, &tmp2, &int_value)) {
return false;
}
} else {
if (!str_to_int64(val, &tmp, &int_value)) {
return false;
}
}
if constexpr (is_datetime) {
microsecond = int_value * int_exp10(6 - min(6, tmp - val));
frac_part_used = true;
}
microsecond = int_value * int_exp10(6 - min(6, val_end - val));
val = tmp;
time_part_used = true;
frac_part_used = true;
break;
// AM/PM
case 'p':

10
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java
View File

@ -28,7 +28,7 @@ import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.DateTimeType;
import org.apache.doris.nereids.types.DateTimeV2Type;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.VarcharType;
@ -44,8 +44,9 @@ public class StrToDate extends ScalarFunction
implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(DateTimeType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT),
FunctionSignature.ret(DateTimeType.INSTANCE).args(StringType.INSTANCE, StringType.INSTANCE)
FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT,
VarcharType.SYSTEM_DEFAULT),
FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT).args(StringType.INSTANCE, StringType.INSTANCE)
);
/**
@ -86,6 +87,9 @@ public class StrToDate extends ScalarFunction
if (getArgument(1) instanceof StringLikeLiteral) {
if (DateLiteral.hasTimePart(((StringLikeLiteral) getArgument(1)).getStringValue())) {
returnType = DataType.fromCatalogType(ScalarType.getDefaultDateType(Type.DATETIME));
if (returnType.isDateTimeV2Type()) {
returnType = DataType.fromCatalogType(Type.DATETIMEV2_WITH_MAX_SCALAR);
}
} else {
returnType = DataType.fromCatalogType(ScalarType.getDefaultDateType(Type.DATE));
}

4
gensrc/script/doris_builtins_functions.py
View File

@ -939,8 +939,6 @@ visible_functions = {
[['datediff'], 'INT', ['DATETIME', 'DATETIME'], 'ALWAYS_NULLABLE'],
[['timediff'], 'TIME', ['DATETIME', 'DATETIME'], 'ALWAYS_NULLABLE'],
[['str_to_date'], 'DATETIME', ['VARCHAR', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['str_to_date'], 'DATETIME', ['STRING', 'STRING'], 'ALWAYS_NULLABLE'],
[['date_format'], 'VARCHAR', ['DATETIME', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['date_format'], 'VARCHAR', ['DATE', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['date', 'to_date'], 'DATE', ['DATETIME'], 'ALWAYS_NULLABLE'],
@ -1104,6 +1102,8 @@ visible_functions = {
[['datediff'], 'INT', ['DATEV2', 'DATEV2'], ''],
[['timediff'], 'TIMEV2', ['DATEV2', 'DATEV2'], ''],
[['str_to_date'], 'DATETIMEV2', ['VARCHAR', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['str_to_date'], 'DATETIMEV2', ['STRING', 'STRING'], 'ALWAYS_NULLABLE'],
[['date_format'], 'VARCHAR', ['DATETIMEV2', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['date_format'], 'VARCHAR', ['DATEV2', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['date', 'to_date', 'datev2', 'to_datev2'], 'DATEV2', ['DATETIMEV2'], 'ALWAYS_NULLABLE'],

5
regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out
View File

@ -235,7 +235,10 @@ February
2014-12-21T12:34:56
-- !sql --
2014-12-21T12:34:56
2014-12-21T12:34:56.789
-- !sql --
2023-07-05T02:09:55.880
-- !sql --
2004-10-18

3
regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out
View File

@ -275,6 +275,9 @@ February
-- !sql --
2014-12-21T12:34:56
-- !sql --
2023-07-05T02:09:55
-- !sql --
2004-10-18

1
regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy
View File

@ -304,6 +304,7 @@ suite("test_date_function") {
qt_sql """ select str_to_date(test_datetime, '%Y-%m-%d %H:%i:%s') from ${tableName}; """
qt_sql """ select str_to_date("2014-12-21 12:34%3A56", '%Y-%m-%d %H:%i%%3A%s'); """
qt_sql """ select str_to_date("2014-12-21 12:34:56.789 PM", '%Y-%m-%d %h:%i:%s.%f %p'); """
qt_sql """ select str_to_date('2023-07-05T02:09:55.880Z','%Y-%m-%dT%H:%i:%s.%fZ') """
qt_sql """ select str_to_date('200442 Monday', '%X%V %W') """
sql """ truncate table ${tableName} """
sql """ insert into ${tableName} values ("2020-09-01") """

1
regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
View File

@ -333,6 +333,7 @@ suite("test_date_function") {
qt_sql """ select str_to_date(test_datetime, '%Y-%m-%d %H:%i:%s') from ${tableName}; """
qt_sql """ select str_to_date("2014-12-21 12:34%3A56", '%Y-%m-%d %H:%i%%3A%s'); """
qt_sql """ select str_to_date("2014-12-21 12:34:56.789 PM", '%Y-%m-%d %h:%i:%s.%f %p'); """
qt_sql """ select str_to_date('2023-07-05T02:09:55.880Z','%Y-%m-%dT%H:%i:%s.%fZ') """
qt_sql """ select str_to_date('200442 Monday', '%X%V %W') """
sql """ truncate table ${tableName} """
sql """ insert into ${tableName} values ("2020-09-01") """