From d67029c83009d363f4e8af235e8ff4b2198cdd78 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 26 Jul 2022 22:36:13 +0800 Subject: [PATCH] [feature-wip] (datetimev2) support `cast` between datetimev2 with different scales (#11198) * [feature-wip] (datetimev2) support `cast` between datetimev2 with different scale --- be/src/vec/data_types/data_type_factory.cpp | 6 +- be/src/vec/data_types/data_type_time_v2.h | 19 +++++ be/src/vec/functions/function_cast.h | 21 ++++-- be/src/vec/io/io_helper.h | 8 +-- be/src/vec/runtime/vdatetime_value.cpp | 23 ++++--- be/src/vec/runtime/vdatetime_value.h | 2 +- be/test/vec/runtime/vdatetime_value_test.cpp | 64 +++++++++++++++++ .../apache/doris/analysis/DateLiteral.java | 69 +++++++++++++------ .../org/apache/doris/catalog/ScalarType.java | 1 + .../doris/common/proc/PartitionsProcDir.java | 4 +- .../doris/common/proc/RollupProcDir.java | 4 +- .../common/proc/SchemaChangeProcDir.java | 4 +- .../doris/common/util/PropertyAnalyzer.java | 2 +- .../org/apache/doris/load/DeleteHandler.java | 10 ++- .../apache/doris/qe/cache/PartitionRange.java | 2 +- .../doris/analysis/DateLiteralTest.java | 6 +- .../apache/doris/planner/QueryPlanTest.java | 8 +-- .../rewrite/RewriteDateLiteralRuleTest.java | 28 ++++---- 18 files changed, 210 insertions(+), 71 deletions(-) diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp index 0025f86ec0..249d008205 100644 --- a/be/src/vec/data_types/data_type_factory.cpp +++ b/be/src/vec/data_types/data_type_factory.cpp @@ -87,7 +87,7 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo nested = std::make_shared(); break; case TYPE_DATETIMEV2: - nested = std::make_shared(); + nested = std::make_shared(col_desc.scale); break; case TYPE_DATETIME: nested = std::make_shared(); @@ -168,7 +168,7 @@ DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type, result = std::make_shared(); break; case OLAP_FIELD_TYPE_DATETIMEV2: - result = std::make_shared(); + result = std::make_shared(scale); break; case OLAP_FIELD_TYPE_DATETIME: result = std::make_shared(); @@ -252,7 +252,7 @@ DataTypePtr DataTypeFactory::create_data_type(const PColumnMeta& pcolumn) { nested = std::make_shared(); break; case PGenericType::DATETIMEV2: - nested = std::make_shared(); + nested = std::make_shared(pcolumn.decimal_param().scale()); break; case PGenericType::DATETIME: nested = std::make_shared(); diff --git a/be/src/vec/data_types/data_type_time_v2.h b/be/src/vec/data_types/data_type_time_v2.h index 13d54d1490..0efff481e6 100644 --- a/be/src/vec/data_types/data_type_time_v2.h +++ b/be/src/vec/data_types/data_type_time_v2.h @@ -47,6 +47,15 @@ public: class DataTypeDateTimeV2 final : public DataTypeNumberBase { public: + static constexpr bool is_parametric = true; + + DataTypeDateTimeV2(UInt32 scale = 0) : scale_(scale) { + if (UNLIKELY(scale > 6)) { + LOG(FATAL) << fmt::format("Scale {} is out of bounds", scale); + } + } + + DataTypeDateTimeV2(const DataTypeDateTimeV2& rhs) : scale_(rhs.scale_) {} TypeIndex get_type_id() const override { return TypeIndex::DateTimeV2; } const char* get_family_name() const override { return "DateTimeV2"; } std::string do_get_name() const override { return "DateTimeV2"; } @@ -60,9 +69,19 @@ public: MutableColumnPtr create_column() const override; + const UInt32 get_scale() const { return scale_; } + static void cast_to_date(const UInt64 from, Int64& to); static void cast_to_date_time(const UInt64 from, Int64& to); static void cast_to_date_v2(const UInt64 from, UInt32& to); + +private: + UInt32 scale_; }; +template +constexpr bool IsDataTypeDateTimeV2 = false; +template <> +inline constexpr bool IsDataTypeDateTimeV2 = true; + } // namespace doris::vectorized diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h index be83d31b44..5494056fd8 100644 --- a/be/src/vec/functions/function_cast.h +++ b/be/src/vec/functions/function_cast.h @@ -163,8 +163,8 @@ struct ConvertImpl { IsDateV2Type) { DataTypeDateTimeV2::cast_to_date_v2(vec_from[i], vec_to[i]); } else { - // TODO: conversion between datetimev2 with different scales - vec_to[i] = vec_from[i]; + UInt32 scale = additions; + vec_to[i] = vec_from[i] / std::pow(10, 6 - scale); } } else if constexpr (IsTimeType) { if constexpr (IsDateTimeType && IsDateV2Type) { @@ -420,7 +420,8 @@ bool try_parse_impl(typename DataType::FieldType& x, ReadBuffer& rb, const DateL } if constexpr (IsDateTimeV2Type) { - return try_read_datetime_v2_text(x, rb); + UInt32 scale = additions; + return try_read_datetime_v2_text(x, rb, scale); } if constexpr (std::is_floating_point_v) { @@ -669,9 +670,16 @@ private: ret_status = ConvertImpl::execute( block, arguments, result, input_rows_count, scale); - } else + } else if constexpr (IsDataTypeDateTimeV2) { + const ColumnWithTypeAndName& scale_column = block.get_by_position(result); + auto type = + check_and_get_data_type(scale_column.type.get()); + ret_status = ConvertImpl::execute( + block, arguments, result, input_rows_count, type->get_scale()); + } else { ret_status = ConvertImpl::execute( block, arguments, result, input_rows_count); + } return true; }; @@ -894,6 +902,11 @@ struct ConvertThroughParsing { if constexpr (IsDataTypeDecimal) { parsed = try_parse_impl(vec_to[i], read_buffer, local_time_zone, vec_to.get_scale()); + } else if constexpr (IsDataTypeDateTimeV2) { + auto type = check_and_get_data_type( + block.get_by_position(result).type.get()); + parsed = try_parse_impl(vec_to[i], read_buffer, local_time_zone, + type->get_scale()); } else { parsed = try_parse_impl(vec_to[i], read_buffer, local_time_zone); } diff --git a/be/src/vec/io/io_helper.h b/be/src/vec/io/io_helper.h index 00cddc22e9..64cca34005 100644 --- a/be/src/vec/io/io_helper.h +++ b/be/src/vec/io/io_helper.h @@ -293,10 +293,10 @@ bool read_date_v2_text_impl(T& x, ReadBuffer& buf) { } template -bool read_datetime_v2_text_impl(T& x, ReadBuffer& buf) { +bool read_datetime_v2_text_impl(T& x, ReadBuffer& buf, UInt32 scale = -1) { static_assert(std::is_same_v); auto dv = binary_cast>(x); - auto ans = dv.from_date_str(buf.position(), buf.count()); + auto ans = dv.from_date_str(buf.position(), buf.count(), scale); // only to match the is_all_read() check to prevent return null buf.position() = buf.end(); @@ -374,7 +374,7 @@ bool try_read_date_v2_text(T& x, ReadBuffer& in) { } template -bool try_read_datetime_v2_text(T& x, ReadBuffer& in) { - return read_datetime_v2_text_impl(x, in); +bool try_read_datetime_v2_text(T& x, ReadBuffer& in, UInt32 scale) { + return read_datetime_v2_text_impl(x, in, scale); } } // namespace doris::vectorized diff --git a/be/src/vec/runtime/vdatetime_value.cpp b/be/src/vec/runtime/vdatetime_value.cpp index a48661b6d5..9056e44c46 100644 --- a/be/src/vec/runtime/vdatetime_value.cpp +++ b/be/src/vec/runtime/vdatetime_value.cpp @@ -1684,7 +1684,7 @@ bool DateV2Value::is_invalid(uint32_t year, uint32_t month, uint32_t day, uin // YYYY-MM-DD HH-MM-DD.FFFFFF AM in default format // 0 1 2 3 4 5 6 7 template -bool DateV2Value::from_date_str(const char* date_str, int len) { +bool DateV2Value::from_date_str(const char* date_str, int len, int scale) { const char* ptr = date_str; const char* end = date_str + len; // ONLY 2, 6 can follow by a sapce @@ -1732,6 +1732,11 @@ bool DateV2Value::from_date_str(const char* date_str, int len) { if (field_idx == 6) { // Microsecond temp_val *= std::pow(10, 6 - (end - start)); + if constexpr (is_datetime) { + if (scale >= 0) { + temp_val /= std::pow(10, 6 - scale); + } + } } // Imposible if (temp_val > 999999L) { @@ -2258,15 +2263,15 @@ int32_t DateV2Value::to_buffer(char* buffer, int scale) const { /* Second */ *buffer++ = (char)('0' + (date_v2_value_.second_ / 10)); *buffer++ = (char)('0' + (date_v2_value_.second_ % 10)); - if (scale != 0) { + if (scale != 0 && date_v2_value_.microsecond_ > 0) { *buffer++ = '.'; - } - /* Microsecond */ - uint32_t ms = date_v2_value_.microsecond_; - int ms_width = scale == -1 ? 6 : std::min(6, scale); - for (int i = 0; i < ms_width; i++) { - *buffer++ = (char)('0' + (ms / std::pow(10, 5 - i))); - ms %= (uint32_t)std::pow(10, 5 - i); + /* Microsecond */ + uint32_t ms = date_v2_value_.microsecond_; + int ms_width = scale == -1 ? 6 : std::min(6, scale); + for (int i = 0; i < ms_width; i++) { + *buffer++ = (char)('0' + (ms / std::pow(10, 5 - i))); + ms %= (uint32_t)std::pow(10, 5 - i); + } } } return buffer - start; diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h index 7cf696b8c7..5b6663c796 100644 --- a/be/src/vec/runtime/vdatetime_value.h +++ b/be/src/vec/runtime/vdatetime_value.h @@ -801,7 +801,7 @@ public: // 'YYMMDD', 'YYYYMMDD', 'YYMMDDHHMMSS', 'YYYYMMDDHHMMSS' // 'YY-MM-DD', 'YYYY-MM-DD', 'YY-MM-DD HH.MM.SS' // 'YYYYMMDDTHHMMSS' - bool from_date_str(const char* str, int len); + bool from_date_str(const char* str, int len, int scale = -1); // Convert this value to string // this will check type to decide which format to convert diff --git a/be/test/vec/runtime/vdatetime_value_test.cpp b/be/test/vec/runtime/vdatetime_value_test.cpp index 809705dead..312b0ed423 100644 --- a/be/test/vec/runtime/vdatetime_value_test.cpp +++ b/be/test/vec/runtime/vdatetime_value_test.cpp @@ -483,4 +483,68 @@ TEST(VDateTimeValueTest, date_diff_test) { } } +TEST(VDateTimeValueTest, date_v2_to_string_test) { + uint16_t year = 2022; + uint8_t month = 5; + uint8_t day = 24; + uint8_t hour = 23; + uint8_t minute = 50; + uint8_t second = 50; + uint32_t ms = 555000; + + { + DateV2Value date_v2; + date_v2.set_time(year, month, day, 0, 0, 0, 0); + + char buf[30]; + int len = date_v2.to_buffer(buf); + EXPECT_TRUE(std::string(buf, len) == std::string("2022-05-24")); + } + + { + DateV2Value date_v2; + date_v2.set_time(year, month, day, hour, minute, second, ms); + + char buf[30]; + int len = date_v2.to_buffer(buf); + EXPECT_TRUE(std::string(buf, len) == std::string("2022-05-24 23:50:50.555000")); + } + + { + DateV2Value date_v2; + date_v2.set_time(year, month, day, hour, minute, second, ms); + + char buf[30]; + int len = date_v2.to_buffer(buf, 3); + EXPECT_TRUE(std::string(buf, len) == std::string("2022-05-24 23:50:50.555")); + } + + { + DateV2Value date_v2; + date_v2.set_time(year, month, day, hour, minute, second, ms); + + char buf[30]; + int len = date_v2.to_buffer(buf, 2); + EXPECT_TRUE(std::string(buf, len) == std::string("2022-05-24 23:50:50.55")); + } + + { + DateV2Value date_v2; + date_v2.set_time(year, month, day, hour, minute, second, ms); + + char buf[30]; + int len = date_v2.to_buffer(buf, 6); + EXPECT_TRUE(std::string(buf, len) == std::string("2022-05-24 23:50:50.555000")); + } + + { + DateV2Value date_v2; + date_v2.set_time(year, month, day, hour, minute, second, 0); + + char buf[30]; + int len = date_v2.to_buffer(buf); + EXPECT_TRUE(std::string(buf, len) == std::string("2022-05-24 23:50:50")); + } +} + } // namespace doris::vectorized diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java index 52b2029e04..2a08cb6e75 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -31,6 +31,7 @@ import org.apache.doris.thrift.TExprNode; import org.apache.doris.thrift.TExprNodeType; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -46,10 +47,12 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; +import java.time.format.DateTimeParseException; import java.time.format.TextStyle; import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.TimeZone; @@ -74,6 +77,7 @@ public class DateLiteral extends LiteralExpr { private static DateTimeFormatter DATE_TIME_FORMATTER = null; private static DateTimeFormatter DATE_TIME_FORMATTER_TO_MICRO_SECOND = null; private static DateTimeFormatter DATE_FORMATTER = null; + private static List formatterList = null; /* * The datekey type is widely used in data warehouses * For example, 20121229 means '2012-12-29' @@ -99,8 +103,18 @@ public class DateLiteral extends LiteralExpr { DATETIMEKEY_FORMATTER = formatBuilder("%Y%m%d%H%i%s").toFormatter(); DATE_TIME_FORMATTER_TO_MICRO_SECOND = new DateTimeFormatterBuilder() .appendPattern("uuuu-MM-dd HH:mm:ss") - .appendFraction(ChronoField.NANO_OF_SECOND, 0, 6, true) + .appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true) .toFormatter(); + formatterList = Lists.newArrayList( + formatBuilder("%Y%m%d").appendLiteral('T').appendPattern("HHmmss") + .appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true).toFormatter(), + formatBuilder("%Y%m%d").appendLiteral('T').appendPattern("HHmmss") + .appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, false).toFormatter(), + formatBuilder("%Y%m%d%H%i%s") + .appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true).toFormatter(), + formatBuilder("%Y%m%d%H%i%s") + .appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, false).toFormatter(), + DATETIMEKEY_FORMATTER, DATEKEY_FORMATTER); } catch (AnalysisException e) { LOG.error("invalid date format", e); System.exit(-1); @@ -321,13 +335,22 @@ public class DateLiteral extends LiteralExpr { private void init(String s, Type type) throws AnalysisException { try { Preconditions.checkArgument(type.isDateType()); - TemporalAccessor dateTime; - if (s.length() == DATEKEY_LENGTH && !s.contains("-")) { + TemporalAccessor dateTime = null; + boolean parsed = false; + if (!s.contains("-")) { // handle format like 20210106, but should not handle 2021-1-6 - dateTime = DATEKEY_FORMATTER.parse(s); - } else if (s.length() == DATETIMEKEY_LENGTH && !s.contains("-")) { - // handle format like 20210106, but should not handle 2021-1-6 - dateTime = DATETIMEKEY_FORMATTER.parse(s); + for (DateTimeFormatter formatter : formatterList) { + try { + dateTime = formatter.parse(s); + parsed = true; + break; + } catch (DateTimeParseException ex) { + // ignore + } + } + if (!parsed) { + throw new AnalysisException("Invalid date value: " + s); + } } else { String[] datePart = s.contains(" ") ? s.split(" ")[0].split("-") : s.split("-"); DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); @@ -380,7 +403,7 @@ public class DateLiteral extends LiteralExpr { builder.appendPattern(String.join("", Collections.nCopies(timePart[i].contains(".") ? timePart[i].split("\\.")[0].length() : timePart[i].length(), "s"))); if (timePart[i].contains(".")) { - builder.appendFraction(ChronoField.NANO_OF_SECOND, 0, 6, true); + builder.appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true); } break; default: @@ -392,8 +415,10 @@ public class DateLiteral extends LiteralExpr { } DateTimeFormatter formatter = builder.toFormatter(); dateTime = formatter.parse(s); + parsed = true; } + Preconditions.checkArgument(parsed); year = getOrDefault(dateTime, ChronoField.YEAR, 0); month = getOrDefault(dateTime, ChronoField.MONTH_OF_YEAR, 0); day = getOrDefault(dateTime, ChronoField.DAY_OF_MONTH, 0); @@ -402,10 +427,9 @@ public class DateLiteral extends LiteralExpr { second = getOrDefault(dateTime, ChronoField.SECOND_OF_MINUTE, 0); microsecond = getOrDefault(dateTime, ChronoField.MICRO_OF_SECOND, 0); if (type.isDatetimeV2()) { - this.type = ScalarType.createDatetimeV2Type(6); - } else { - this.type = type; + this.roundFloor(((ScalarType) type).getScalarScale()); } + this.type = type; } catch (Exception ex) { throw new AnalysisException("date literal [" + s + "] is invalid: " + ex.getMessage()); } @@ -513,14 +537,14 @@ public class DateLiteral extends LiteralExpr { long remain = Double.valueOf(microsecond % (Math.pow(10, 6 - newScale))).longValue(); if (remain != 0) { microsecond = Double.valueOf((microsecond + (Math.pow(10, 6 - newScale))) - / (Math.pow(10, 6 - newScale))).longValue(); + / (Math.pow(10, 6 - newScale)) * (Math.pow(10, 6 - newScale))).longValue(); } type = ScalarType.createDatetimeV2Type(newScale); } public void roundFloor(int newScale) { - Preconditions.checkArgument(type.isDatetimeV2()); - microsecond = Double.valueOf(microsecond / (Math.pow(10, 6 - newScale))).longValue(); + microsecond = Double.valueOf(microsecond / (Math.pow(10, 6 - newScale)) + * (Math.pow(10, 6 - newScale))).longValue(); type = ScalarType.createDatetimeV2Type(newScale); } @@ -532,8 +556,12 @@ public class DateLiteral extends LiteralExpr { if (type == PrimitiveType.DATE || type == PrimitiveType.DATEV2) { return String.format("%04d-%02d-%02d", year, month, day); } else if (type == PrimitiveType.DATETIMEV2) { - return String.format("%04d-%02d-%02d %02d:%02d:%02d.%06d", - year, month, day, hour, minute, second, microsecond); + String tmp = String.format("%04d-%02d-%02d %02d:%02d:%02d", + year, month, day, hour, minute, second); + if (microsecond == 0) { + return tmp; + } + return tmp + String.format(".%06d", microsecond); } else { return String.format("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second); } @@ -673,8 +701,7 @@ public class DateLiteral extends LiteralExpr { this.type = Type.DATE; } else if (dateLiteralType == DateLiteralType.DATETIMEV2.value()) { fromPackedDatetime(in.readLong()); - int scale = in.readInt(); - this.type = ScalarType.createDatetimeV2Type(scale); + this.type = ScalarType.createDatetimeV2Type(in.readInt()); } else if (dateLiteralType == DateLiteralType.DATEV2.value()) { fromPackedDatetime(in.readLong()); this.type = Type.DATEV2; @@ -782,6 +809,7 @@ public class DateLiteral extends LiteralExpr { builder.appendValue(ChronoField.ALIGNED_WEEK_OF_YEAR, 2); break; case 'x': + case 'Y': // %Y Year, numeric, four digits // %x Year for the week, where Monday is the first day of the week, // numeric, four digits; used with %v builder.appendValue(ChronoField.YEAR, 4); @@ -789,9 +817,6 @@ public class DateLiteral extends LiteralExpr { case 'W': // %W Weekday name (Sunday..Saturday) builder.appendText(ChronoField.DAY_OF_WEEK, TextStyle.FULL); break; - case 'Y': // %Y Year, numeric, four digits - builder.appendPattern("uuuu"); - break; case 'y': // %y Year, numeric (two digits) builder.appendValueReduced(ChronoField.YEAR, 2, 2, 1970); break; @@ -843,7 +868,7 @@ public class DateLiteral extends LiteralExpr { final int hour = getOrDefault(accessor, ChronoField.HOUR_OF_DAY, 0); final int minute = getOrDefault(accessor, ChronoField.MINUTE_OF_HOUR, 0); final int second = getOrDefault(accessor, ChronoField.SECOND_OF_MINUTE, 0); - final int microSeconds = getOrDefault(accessor, ChronoField.NANO_OF_SECOND, 0); + final int microSeconds = getOrDefault(accessor, ChronoField.MICRO_OF_SECOND, 0); return LocalDateTime.of(year, month, dayOfMonth, hour, minute, second, microSeconds); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarType.java index 6fb59c20c6..f3f3784b6f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarType.java @@ -78,6 +78,7 @@ public class ScalarType extends Type { public static final int MAX_DECIMAL32_PRECISION = 9; public static final int MAX_DECIMAL64_PRECISION = 18; public static final int MAX_DECIMAL128_PRECISION = 38; + public static final int MAX_DATETIMEV2_SCALE = 6; private static final Logger LOG = LogManager.getLogger(ScalarType.class); @SerializedName(value = "type") diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java index 72665d62a5..14bab261bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java @@ -104,9 +104,11 @@ public class PartitionsProcDir implements ProcDirInterface { type = Type.DATETIME; break; case DATEV2: - case DATETIMEV2: type = Type.DATETIMEV2; break; + case DATETIMEV2: + type = subExpr.getChild(1).getType(); + break; default: throw new AnalysisException("Invalid date type: " + subExpr.getChild(1).getType()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/RollupProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/RollupProcDir.java index c3d52787c4..fb65f5231c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/RollupProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/RollupProcDir.java @@ -161,9 +161,11 @@ public class RollupProcDir implements ProcDirInterface { type = Type.DATETIME; break; case DATEV2: - case DATETIMEV2: type = Type.DATETIMEV2; break; + case DATETIMEV2: + type = subExpr.getChild(1).getType(); + break; default: throw new AnalysisException("Invalid date type: " + subExpr.getChild(1).getType()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/SchemaChangeProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/SchemaChangeProcDir.java index a7e171bb21..a2ff07f966 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/SchemaChangeProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/SchemaChangeProcDir.java @@ -80,9 +80,11 @@ public class SchemaChangeProcDir implements ProcDirInterface { type = Type.DATETIME; break; case DATEV2: - case DATETIMEV2: type = Type.DATETIMEV2; break; + case DATETIMEV2: + type = subExpr.getChild(1).getType(); + break; default: throw new AnalysisException("Invalid date type: " + subExpr.getChild(1).getType()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index b7c7369f90..317beb78d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -163,7 +163,7 @@ public class PropertyAnalyzer { } else if (key.equalsIgnoreCase(PROPERTIES_REMOTE_STORAGE_POLICY)) { remoteStoragePolicy = value; } else if (key.equalsIgnoreCase(PROPERTIES_DATA_BASE_TIME)) { - DateLiteral dateLiteral = new DateLiteral(value, Type.DATETIME); + DateLiteral dateLiteral = new DateLiteral(value, DateLiteral.getDefaultDateType(Type.DATETIME)); dataBaseTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); } else if (!hasStoragePolicy && key.equalsIgnoreCase(PROPERTIES_STORAGE_POLICY)) { if (!Strings.isNullOrEmpty(value)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java index 0344bffe89..e2fdabfdab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java @@ -37,6 +37,7 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Replica; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.TabletInvertedIndex; @@ -583,12 +584,17 @@ public class DeleteHandler implements Writable { } } else if (column.getDataType() == PrimitiveType.DATE || column.getDataType() == PrimitiveType.DATETIME - || column.getDataType() == PrimitiveType.DATEV2 - || column.getDataType() == PrimitiveType.DATETIMEV2) { + || column.getDataType() == PrimitiveType.DATEV2) { DateLiteral dateLiteral = new DateLiteral(value, Type.fromPrimitiveType(column.getDataType())); value = dateLiteral.getStringValue(); binaryPredicate.setChild(1, LiteralExpr.create(value, Type.fromPrimitiveType(column.getDataType()))); + } else if (column.getDataType() == PrimitiveType.DATETIMEV2) { + DateLiteral dateLiteral = new DateLiteral(value, + ScalarType.createDecimalType(ScalarType.MAX_DATETIMEV2_SCALE)); + value = dateLiteral.getStringValue(); + binaryPredicate.setChild(1, LiteralExpr.create(value, + ScalarType.createDecimalType(ScalarType.MAX_DATETIMEV2_SCALE))); } LiteralExpr.create(value, Type.fromPrimitiveType(column.getDataType())); } catch (AnalysisException e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java index 1c5b5bb4a0..3a4e7dd45b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java @@ -459,7 +459,7 @@ public class PartitionRange { LiteralExpr newLiteral; if (key.keyType == KeyType.DATE) { try { - newLiteral = new DateLiteral(key.toString(), Type.DATE); + newLiteral = new DateLiteral(key.toString(), DateLiteral.getDefaultDateType(Type.DATE)); } catch (Exception e) { LOG.warn("Date's format is error {},{}", key.toString(), e); continue; diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java index 60cee42cfe..dacce11b67 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java @@ -254,16 +254,16 @@ public class DateLiteralTest { public void testDateFormatForDatetimeV2() { boolean hasException = false; try { - DateLiteral literal = new DateLiteral("1997-10-7 00:00:00.123456", Type.DATETIMEV2); + DateLiteral literal = new DateLiteral("1997-10-7 00:00:00.123456", ScalarType.createDatetimeV2Type(6)); Assert.assertEquals(1997, literal.getYear()); Assert.assertEquals(123456, literal.getMicrosecond()); - literal = new DateLiteral("2021-06-1 00:00:00.123456", Type.DATETIMEV2); + literal = new DateLiteral("2021-06-1 00:00:00.123456", ScalarType.createDatetimeV2Type(6)); Assert.assertEquals(2021, literal.getYear()); Assert.assertEquals(6, literal.getMonth()); Assert.assertEquals(1, literal.getDay()); - literal = new DateLiteral("2022-6-01 00:00:00.123456", Type.DATETIMEV2); + literal = new DateLiteral("2022-6-01 00:00:00.123456", ScalarType.createDatetimeV2Type(6)); Assert.assertEquals(2022, literal.getYear()); Assert.assertEquals(6, literal.getMonth()); Assert.assertEquals(1, literal.getDay()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java index 14d2478317..925a720591 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java @@ -2012,10 +2012,10 @@ public class QueryPlanTest extends TestWithFeService { rewriteDateLiteralRuleTest.testWithIntFormatDate(); rewriteDateLiteralRuleTest.testWithInvalidFormatDate(); rewriteDateLiteralRuleTest.testWithStringFormatDate(); - // rewriteDateLiteralRuleTest.testWithDoubleFormatDateV2(); - // rewriteDateLiteralRuleTest.testWithIntFormatDateV2(); - // rewriteDateLiteralRuleTest.testWithInvalidFormatDateV2(); - // rewriteDateLiteralRuleTest.testWithStringFormatDateV2(); + rewriteDateLiteralRuleTest.testWithDoubleFormatDateV2(); + rewriteDateLiteralRuleTest.testWithIntFormatDateV2(); + rewriteDateLiteralRuleTest.testWithInvalidFormatDateV2(); + rewriteDateLiteralRuleTest.testWithStringFormatDateV2(); rewriteDateLiteralRuleTest.after(); } // --end-- diff --git a/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java b/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java index 9f2d106bd9..40bc9bb2c3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java @@ -64,13 +64,13 @@ public class RewriteDateLiteralRuleTest { public void testWithIntFormatDateV2() throws Exception { String query = "select * from " + DB_NAME + ".tb2 where k1 > 20210301"; String planString = dorisAssert.query(query).explainQuery(); - Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00.000000'")); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00'")); query = "select k1 > 20210301 from " + DB_NAME + ".tb2"; planString = dorisAssert.query(query).explainQuery(); - Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00.000000'")); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00'")); query = "select k1 > 20210301223344 from " + DB_NAME + ".tb2"; planString = dorisAssert.query(query).explainQuery(); - Assert.assertTrue(planString.contains("`k1` > '2021-03-01 22:33:44.000000'")); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 22:33:44'")); } public void testWithStringFormatDate() throws Exception { @@ -122,47 +122,47 @@ public class RewriteDateLiteralRuleTest { public void testWithStringFormatDateV2() throws Exception { String query = "select * from " + DB_NAME + ".tb2 where k1 > '2021030112334455'"; String planString = dorisAssert.query(query).explainQuery(); - Assert.assertTrue(planString.contains("`k1` > '2021-03-01 12:33:44.000000'")); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 12:33:44.550000'")); query = "select k1 > '20210301' from " + DB_NAME + ".tb2"; planString = dorisAssert.query(query).explainQuery(); - Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00.000000'")); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00'")); query = "select k1 > '20210301233234.34' from " + DB_NAME + ".tb2"; planString = dorisAssert.query(query).explainQuery(); - Assert.assertTrue(planString.contains("`k1` > '2021-03-01 23:32:34.000000'")); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 23:32:34.340000'")); query = "select * from " + DB_NAME + ".tb2 where k1 > '2021-03-01'"; planString = dorisAssert.query(query).explainQuery(); - Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00.000000'")); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00'")); query = "select k1 > '2021-03-01 11:22:33' from " + DB_NAME + ".tb2"; planString = dorisAssert.query(query).explainQuery(); Assert.assertTrue(planString.contains("`k1` > '2021-03-01 11:22:33'")); - query = "select k1 > '2021-03-01 16:22:33' from " + DB_NAME + ".tb2"; + query = "select k1 > '2021-03-01 16:22:33' from " + DB_NAME + ".tb2"; planString = dorisAssert.query(query).explainQuery(); Assert.assertTrue(planString.contains("`k1` > '2021-03-01 16:22:33'")); query = "select k1 > '2021-03-01 11:22' from " + DB_NAME + ".tb2"; planString = dorisAssert.query(query).explainQuery(); - Assert.assertTrue(planString.contains("`k1` > '2021-03-01 11:22:00.000000'")); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 11:22:00'")); query = "select k1 > '20210301T221133' from " + DB_NAME + ".tb2"; planString = dorisAssert.query(query).explainQuery(); - Assert.assertTrue(planString.contains("`k1` > '2021-03-01 22:11:33.000000'")); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 22:11:33'")); - query = "select k1 > '2021-03-01dd 11:22' from " + DB_NAME + ".tb2"; + query = "select k1 > '2021-03-01 11:22' from " + DB_NAME + ".tb2"; planString = dorisAssert.query(query).explainQuery(); - Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00.000000'")); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 11:22:00'")); query = "select k1 > '80-03-01 11:22' from " + DB_NAME + ".tb2"; planString = dorisAssert.query(query).explainQuery(); - Assert.assertTrue(planString.contains("`k1` > '1980-03-01 11:22:00.000000'")); + Assert.assertTrue(planString.contains("`k1` > '1980-03-01 11:22:00'")); query = "select k1 > '12-03-01 11:22' from " + DB_NAME + ".tb2"; planString = dorisAssert.query(query).explainQuery(); - Assert.assertTrue(planString.contains("`k1` > '2012-03-01 11:22:00.000000'")); + Assert.assertTrue(planString.contains("`k1` > '2012-03-01 11:22:00'")); } public void testWithDoubleFormatDate() throws Exception {