diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp index c8271daa0d..29512a504a 100644 --- a/be/src/exec/es/es_scroll_parser.cpp +++ b/be/src/exec/es/es_scroll_parser.cpp @@ -455,10 +455,6 @@ Status ScrollParser::fill_tuple(const TupleDescriptor* tuple_desc, RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type); } - if (ts_slot->year() < 1900) { - RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type); - } - if (type == TYPE_DATE) { ts_slot->cast_to_date(); } else { diff --git a/be/src/exec/tablet_sink.cpp b/be/src/exec/tablet_sink.cpp index 54ec061287..0cdbe641b7 100644 --- a/be/src/exec/tablet_sink.cpp +++ b/be/src/exec/tablet_sink.cpp @@ -822,27 +822,6 @@ int OlapTableSink::_validate_data(RuntimeState* state, RowBatch* batch, Bitmap* LOG(INFO) << ss.str(); #else state->append_error_msg_to_file("", ss.str()); -#endif - filtered_rows++; - row_valid = false; - filter_bitmap->Set(row_no, true); - continue; - } - break; - } - case TYPE_DATE: - case TYPE_DATETIME: { - static DateTimeValue s_min_value = DateTimeValue(19000101000000UL); - // static DateTimeValue s_max_value = DateTimeValue(99991231235959UL); - DateTimeValue* date_val = (DateTimeValue*)slot; - if (*date_val < s_min_value) { - std::stringstream ss; - ss << "datetime value is not valid, column=" << desc->col_name() - << ", value=" << date_val->debug_string(); -#if BE_TEST - LOG(INFO) << ss.str(); -#else - state->append_error_msg_to_file("", ss.str()); #endif filtered_rows++; row_valid = false; diff --git a/be/src/exec/text_converter.hpp b/be/src/exec/text_converter.hpp index 6d3001aa2e..1f2b1dce8c 100644 --- a/be/src/exec/text_converter.hpp +++ b/be/src/exec/text_converter.hpp @@ -129,11 +129,6 @@ inline bool TextConverter::write_slot(const SlotDescriptor* slot_desc, parse_result = StringParser::PARSE_FAILURE; break; } - // For compatibility with DPP, which only support years after 1900 - if (ts_slot->year() < 1900) { - parse_result = StringParser::PARSE_FAILURE; - break; - } ts_slot->cast_to_date(); break; @@ -144,11 +139,6 @@ inline bool TextConverter::write_slot(const SlotDescriptor* slot_desc, if (!ts_slot->from_date_str(data, len)) { parse_result = StringParser::PARSE_FAILURE; } - // For compatibility with DPP, which only support years after 1900 - if (ts_slot->year() < 1900) { - parse_result = StringParser::PARSE_FAILURE; - break; - } ts_slot->to_datetime(); break; diff --git a/be/test/olap/column_reader_test.cpp b/be/test/olap/column_reader_test.cpp index 60e53e483f..0008672805 100644 --- a/be/test/olap/column_reader_test.cpp +++ b/be/test/olap/column_reader_test.cpp @@ -1780,6 +1780,69 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithPresent) { ASSERT_NE(_column_reader->next_vector( _col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); } +TEST_F(TestColumn, VectorizedDatetimeColumnZero) { + // write data + TabletSchema tablet_schema; + + SetTabletSchemaWithOneColumn( + "DatetimeColumnWithoutPresent", + "DATETIME", + "REPLACE", + 8, + true, + true, &tablet_schema); + CreateColumnWriter(tablet_schema); + + RowCursor write_row; + write_row.init(tablet_schema); + + RowBlock block(&tablet_schema); + RowBlockInfo block_info; + block_info.row_num = 10000; + block.init(block_info); + + write_row.set_null(0); + block.set_row(0, write_row); + block.finalize(1); + ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); + + std::vector val_string_array; + val_string_array.push_back("1000-01-01 00:00:00"); + OlapTuple tuple(val_string_array); + write_row.from_tuple(tuple); + write_row.set_not_null(0); + block.set_row(0, write_row); + block.finalize(1); + ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); + + ColumnDataHeaderMessage header; + ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); + + // read data + CreateColumnReader(tablet_schema); + + RowCursor read_row; + read_row.init(tablet_schema); + + _col_vector.reset(new ColumnVector()); + ASSERT_EQ(_column_reader->next_vector( + _col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); + bool *is_null = _col_vector->is_null(); + ASSERT_EQ(is_null[0], true); + + char *data = reinterpret_cast(_col_vector->col_data()); + ASSERT_EQ(is_null[1], false); + + data += sizeof(uint64_t); + read_row.set_field_content(0, data, _mem_pool.get()); + std::cout << read_row.to_string() << std::endl; + ASSERT_TRUE(strncmp(read_row.to_string().c_str(), + "0&1000-01-01 00:00:00", strlen("0&1000-01-01 00:00:00")) == 0); + + ASSERT_NE(_column_reader->next_vector( + _col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS); +} + TEST_F(TestColumn, VectorizedDateColumnWithoutPresent) { // write data diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE TABLE.md index 8c0b035e21..cf0663a8e7 100644 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE TABLE.md +++ b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE TABLE.md @@ -68,9 +68,9 @@ under the License. 其中整数部分为 1 ~ 18 不支持科学计数法 DATE(3字节) - 范围:1900-01-01 ~ 9999-12-31 + 范围:0000-01-01 ~ 9999-12-31 DATETIME(8字节) - 范围:1900-01-01 00:00:00 ~ 9999-12-31 23:59:59 + 范围:0000-01-01 00:00:00 ~ 9999-12-31 23:59:59 CHAR[(length)] 定长字符串。长度范围:1 ~ 255。默认为1 VARCHAR[(length)] diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATE.md index b06e5ea968..bcaf2d9350 100644 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATE.md +++ b/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATE.md @@ -24,7 +24,7 @@ under the License. DATE(expr) 将输入的类型转化为DATE类型 DATE类型 - 日期类型,目前的取值范围是['1900-01-01', '9999-12-31'], 默认的打印形式是'YYYY-MM-DD' + 日期类型,目前的取值范围是['0000-01-01', '9999-12-31'], 默认的打印形式是'YYYY-MM-DD' ## example mysql> SELECT DATE('2003-12-31 01:02:03'); diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATETIME.md b/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATETIME.md index 875149d371..992f00c894 100644 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATETIME.md +++ b/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATETIME.md @@ -20,7 +20,7 @@ under the License. # DATETIME ## description DATETIME - 日期时间类型,取值范围是['1000-01-01 00:00:00', '9999-12-31 23:59:59']. + 日期时间类型,取值范围是['0000-01-01 00:00:00', '9999-12-31 23:59:59']. 打印的形式是'YYYY-MM-DD HH:MM:SS' ## keyword diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE TABLE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE TABLE_EN.md index 319d1f3307..84bf32f49c 100644 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE TABLE_EN.md +++ b/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE TABLE_EN.md @@ -67,9 +67,9 @@ Syntax: fractional part: 0 ~ 9 Not support scientific notation DATE(3 Bytes) - Range: 1900-01-01 ~ 9999-12-31 + Range: 0000-01-01 ~ 9999-12-31 DATETIME(8 Bytes) - Range: 1900-01-01 00:00:00 ~ 9999-12-31 23:59:59 + Range: 0000-01-01 00:00:00 ~ 9999-12-31 23:59:59 CHAR[(length)] Fixed length string. Range: 1 ~ 255. Default: 1 VARCHAR[(length)] diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/DATETIME_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Types/DATETIME_EN.md index 930994e46a..cfdc388dbf 100644 --- a/docs/documentation/en/sql-reference/sql-statements/Data Types/DATETIME_EN.md +++ b/docs/documentation/en/sql-reference/sql-statements/Data Types/DATETIME_EN.md @@ -20,7 +20,7 @@ under the License. # DATETIME ## Description DATETIME -Date and time type, value range is ['1000-01-01 00:00:00','9999-12-31 23:59:59']. +Date and time type, value range is ['0000-01-01 00:00:00','9999-12-31 23:59:59']. The form of printing is'YYYY-MM-DD HH:MM:SS' ##keyword diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/DATE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Types/DATE_EN.md index a5ec0fbf70..ff42017cf7 100644 --- a/docs/documentation/en/sql-reference/sql-statements/Data Types/DATE_EN.md +++ b/docs/documentation/en/sql-reference/sql-statements/Data Types/DATE_EN.md @@ -24,7 +24,7 @@ Syntax: Date Convert input type to DATE type date -Date type, the current range of values is ['1900-01-01','9999-12-31'], and the default print form is'YYYYY-MM-DD'. +Date type, the current range of values is ['0000-01-01','9999-12-31'], and the default print form is'YYYYY-MM-DD'. ## example mysql> SELECT DATE('2003-12-31 01:02:03'); diff --git a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java index 4812b68876..cce2ad876b 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -48,9 +48,9 @@ import com.google.common.base.Preconditions; public class DateLiteral extends LiteralExpr { private static final Logger LOG = LogManager.getLogger(DateLiteral.class); - private static final DateLiteral MIN_DATE = new DateLiteral(1900, 1, 1); + private static final DateLiteral MIN_DATE = new DateLiteral(0000, 1, 1); private static final DateLiteral MAX_DATE = new DateLiteral(9999, 12, 31); - private static final DateLiteral MIN_DATETIME = new DateLiteral(1900, 1, 1, 0, 0, 0); + private static final DateLiteral MIN_DATETIME = new DateLiteral(0000, 1, 1, 0, 0, 0); private static final DateLiteral MAX_DATETIME = new DateLiteral(9999, 12, 31, 23, 59, 59); public static final DateLiteral UNIX_EPOCH_TIME = new DateLiteral(1970, 01, 01, 00, 00, 00); diff --git a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java index 37e1c4fa73..789895779e 100644 --- a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java +++ b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java @@ -94,10 +94,10 @@ public class TimeUtils { TIME_FORMAT.setTimeZone(TIME_ZONE); try { - MIN_DATE = DATE_FORMAT.parse("1900-01-01"); + MIN_DATE = DATE_FORMAT.parse("0000-01-01"); MAX_DATE = DATE_FORMAT.parse("9999-12-31"); - MIN_DATETIME = DATETIME_FORMAT.parse("1900-01-01 00:00:00"); + MIN_DATETIME = DATETIME_FORMAT.parse("0000-01-01 00:00:00"); MAX_DATETIME = DATETIME_FORMAT.parse("9999-12-31 23:59:59"); } catch (ParseException e) { diff --git a/fe/src/test/java/org/apache/doris/analysis/LiteralExprCompareTest.java b/fe/src/test/java/org/apache/doris/analysis/LiteralExprCompareTest.java index d3c197d95e..b8ed2c8087 100644 --- a/fe/src/test/java/org/apache/doris/analysis/LiteralExprCompareTest.java +++ b/fe/src/test/java/org/apache/doris/analysis/LiteralExprCompareTest.java @@ -87,8 +87,8 @@ public class LiteralExprCompareTest { LiteralExpr minDatetime1Same = new DateLiteral(ScalarType.DATETIME, false); LiteralExpr date8 = new DateLiteral("9999-12-31", ScalarType.DATE); LiteralExpr date9 = new DateLiteral("9999-12-31 23:59:59", ScalarType.DATETIME); - LiteralExpr date10 = new DateLiteral("1900-01-01", ScalarType.DATE); - LiteralExpr date11 = new DateLiteral("1900-01-01 00:00:00", ScalarType.DATETIME); + LiteralExpr date10 = new DateLiteral("0000-01-01", ScalarType.DATE); + LiteralExpr date11 = new DateLiteral("0000-01-01 00:00:00", ScalarType.DATETIME); Assert.assertTrue(date1.equals(date1Same) && date1.compareLiteral(date1Same) == 0); Assert.assertTrue(date1.equals(date1Same) && date1.compareLiteral(date1Same) == 0); diff --git a/fe/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java b/fe/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java index c71f8cf33a..5c0142ed2b 100644 --- a/fe/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java +++ b/fe/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java @@ -134,7 +134,7 @@ public class PartitionKeyTest { pk1 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("-128"), new PartitionValue("-32768"), new PartitionValue("-2147483648"), new PartitionValue("-9223372036854775808"), new PartitionValue("-170141183460469231731687303715884105728"), - new PartitionValue("1900-01-01"), new PartitionValue("1900-01-01 00:00:00")), + new PartitionValue("0000-01-01"), new PartitionValue("0000-01-01 00:00:00")), allColumns); pk2 = PartitionKey.createInfinityPartitionKey(allColumns, false); Assert.assertTrue(pk1.equals(pk2) && pk1.compareTo(pk2) == 0); diff --git a/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java b/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java index 4e3c110ac2..218c0ba9ec 100644 --- a/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java +++ b/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java @@ -57,9 +57,9 @@ public class TimeUtilsTest { Assert.assertNotNull(TimeUtils.getStartTime()); Assert.assertTrue(TimeUtils.getEstimatedTime(0L) > 0); - Assert.assertEquals(-2209017600000L, TimeUtils.MIN_DATE.getTime()); + Assert.assertEquals(-62167420800000L, TimeUtils.MIN_DATE.getTime()); Assert.assertEquals(253402185600000L, TimeUtils.MAX_DATE.getTime()); - Assert.assertEquals(-2209017600000L, TimeUtils.MIN_DATETIME.getTime()); + Assert.assertEquals(-62167420800000L, TimeUtils.MIN_DATETIME.getTime()); Assert.assertEquals(253402271999000L, TimeUtils.MAX_DATETIME.getTime()); } @@ -74,6 +74,7 @@ public class TimeUtilsTest { validDateList.add("9999-12-31"); validDateList.add("1900-01-01"); validDateList.add("2013-2-28"); + validDateList.add("0000-01-01"); for (String validDate : validDateList) { try { TimeUtils.parseDate(validDate, PrimitiveType.DATE); @@ -112,6 +113,7 @@ public class TimeUtilsTest { validDateTimeList.add("2013-2-28 23:59:59"); validDateTimeList.add("2013-2-28 2:3:4"); validDateTimeList.add("2014-05-07 19:8:50"); + validDateTimeList.add("0000-01-01 00:00:00"); for (String validDateTime : validDateTimeList) { try { TimeUtils.parseDate(validDateTime, PrimitiveType.DATETIME);