database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Open datetime min value limit (#3158)

Browse Source 
the min_value in olap/type.h of datetime is 0000-01-01 00:00:00, so we don't need restrict datetime min in tablet_sink
This commit is contained in:
HangyuanLiu
2020-03-24 10:52:57 +08:00
committed by GitHub
parent dff3c0d57e
commit d4c1938b5c
15 changed files with 82 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
be/src/exec/es/es_scroll_parser.cpp
View File

@ -455,10 +455,6 @@ Status ScrollParser::fill_tuple(const TupleDescriptor* tuple_desc,
RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type);
}
if (ts_slot->year() < 1900) {
RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type);
}
if (type == TYPE_DATE) {
ts_slot->cast_to_date();
} else {

21
be/src/exec/tablet_sink.cpp
View File

@ -822,27 +822,6 @@ int OlapTableSink::_validate_data(RuntimeState* state, RowBatch* batch, Bitmap*
LOG(INFO) << ss.str();
#else
state->append_error_msg_to_file("", ss.str());
#endif
filtered_rows++;
row_valid = false;
filter_bitmap->Set(row_no, true);
continue;
}
break;
}
case TYPE_DATE:
case TYPE_DATETIME: {
static DateTimeValue s_min_value = DateTimeValue(19000101000000UL);
// static DateTimeValue s_max_value = DateTimeValue(99991231235959UL);
DateTimeValue* date_val = (DateTimeValue*)slot;
if (*date_val < s_min_value) {
std::stringstream ss;
ss << "datetime value is not valid, column=" << desc->col_name()
<< ", value=" << date_val->debug_string();
#if BE_TEST
LOG(INFO) << ss.str();
#else
state->append_error_msg_to_file("", ss.str());
#endif
filtered_rows++;
row_valid = false;

10
be/src/exec/text_converter.hpp
View File

@ -129,11 +129,6 @@ inline bool TextConverter::write_slot(const SlotDescriptor* slot_desc,
parse_result = StringParser::PARSE_FAILURE;
break;
}
// For compatibility with DPP, which only support years after 1900
if (ts_slot->year() < 1900) {
parse_result = StringParser::PARSE_FAILURE;
break;
}
ts_slot->cast_to_date();
break;
@ -144,11 +139,6 @@ inline bool TextConverter::write_slot(const SlotDescriptor* slot_desc,
if (!ts_slot->from_date_str(data, len)) {
parse_result = StringParser::PARSE_FAILURE;
}
// For compatibility with DPP, which only support years after 1900
if (ts_slot->year() < 1900) {
parse_result = StringParser::PARSE_FAILURE;
break;
}
ts_slot->to_datetime();
break;

63
be/test/olap/column_reader_test.cpp
View File

@ -1780,6 +1780,69 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithPresent) {
ASSERT_NE(_column_reader->next_vector(
_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS);
}
TEST_F(TestColumn, VectorizedDatetimeColumnZero) {
// write data
TabletSchema tablet_schema;
SetTabletSchemaWithOneColumn(
"DatetimeColumnWithoutPresent",
"DATETIME",
"REPLACE",
8,
true,
true, &tablet_schema);
CreateColumnWriter(tablet_schema);
RowCursor write_row;
write_row.init(tablet_schema);
RowBlock block(&tablet_schema);
RowBlockInfo block_info;
block_info.row_num = 10000;
block.init(block_info);
write_row.set_null(0);
block.set_row(0, write_row);
block.finalize(1);
ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS);
std::vector<string> val_string_array;
val_string_array.push_back("1000-01-01 00:00:00");
OlapTuple tuple(val_string_array);
write_row.from_tuple(tuple);
write_row.set_not_null(0);
block.set_row(0, write_row);
block.finalize(1);
ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS);
ColumnDataHeaderMessage header;
ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS);
// read data
CreateColumnReader(tablet_schema);
RowCursor read_row;
read_row.init(tablet_schema);
_col_vector.reset(new ColumnVector());
ASSERT_EQ(_column_reader->next_vector(
_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS);
bool *is_null = _col_vector->is_null();
ASSERT_EQ(is_null[0], true);
char *data = reinterpret_cast<char *>(_col_vector->col_data());
ASSERT_EQ(is_null[1], false);
data += sizeof(uint64_t);
read_row.set_field_content(0, data, _mem_pool.get());
std::cout << read_row.to_string() << std::endl;
ASSERT_TRUE(strncmp(read_row.to_string().c_str(),
"0&1000-01-01 00:00:00", strlen("0&1000-01-01 00:00:00")) == 0);
ASSERT_NE(_column_reader->next_vector(
_col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS);
}
TEST_F(TestColumn, VectorizedDateColumnWithoutPresent) {
// write data

4
docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE TABLE.md
View File

@ -68,9 +68,9 @@ under the License.
其中整数部分为 1 ~ 18
不支持科学计数法
DATE（3字节）
范围：1900-01-01 ~ 9999-12-31
范围：0000-01-01 ~ 9999-12-31
DATETIME（8字节）
范围：1900-01-01 00:00:00 ~ 9999-12-31 23:59:59
范围：0000-01-01 00:00:00 ~ 9999-12-31 23:59:59
CHAR[(length)]
定长字符串。长度范围：1 ~ 255。默认为1
VARCHAR[(length)]

2
docs/documentation/cn/sql-reference/sql-statements/Data Types/DATE.md
View File

@ -24,7 +24,7 @@ under the License.
DATE(expr)
将输入的类型转化为DATE类型
DATE类型
日期类型，目前的取值范围是['1900-01-01', '9999-12-31'], 默认的打印形式是'YYYY-MM-DD'
日期类型，目前的取值范围是['0000-01-01', '9999-12-31'], 默认的打印形式是'YYYY-MM-DD'
## example
mysql> SELECT DATE('2003-12-31 01:02:03');

2
docs/documentation/cn/sql-reference/sql-statements/Data Types/DATETIME.md
View File

@ -20,7 +20,7 @@ under the License.
# DATETIME
## description
DATETIME
日期时间类型，取值范围是['1000-01-01 00:00:00', '9999-12-31 23:59:59'].
日期时间类型，取值范围是['0000-01-01 00:00:00', '9999-12-31 23:59:59'].
打印的形式是'YYYY-MM-DD HH:MM:SS'
## keyword

4
docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE TABLE_EN.md
View File

@ -67,9 +67,9 @@ Syntax:
fractional part: 0 ~ 9
Not support scientific notation
DATE(3 Bytes)
Range: 1900-01-01 ~ 9999-12-31
Range: 0000-01-01 ~ 9999-12-31
DATETIME(8 Bytes)
Range: 1900-01-01 00:00:00 ~ 9999-12-31 23:59:59
Range: 0000-01-01 00:00:00 ~ 9999-12-31 23:59:59
CHAR[(length)]
Fixed length string. Range: 1 ~ 255. Default: 1
VARCHAR[(length)]

2
docs/documentation/en/sql-reference/sql-statements/Data Types/DATETIME_EN.md
View File

@ -20,7 +20,7 @@ under the License.
# DATETIME
## Description
DATETIME
Date and time type, value range is ['1000-01-01 00:00:00','9999-12-31 23:59:59'].
Date and time type, value range is ['0000-01-01 00:00:00','9999-12-31 23:59:59'].
The form of printing is'YYYY-MM-DD HH:MM:SS'
##keyword

2
docs/documentation/en/sql-reference/sql-statements/Data Types/DATE_EN.md
View File

@ -24,7 +24,7 @@ Syntax:
Date
Convert input type to DATE type
date
Date type, the current range of values is ['1900-01-01','9999-12-31'], and the default print form is'YYYYY-MM-DD'.
Date type, the current range of values is ['0000-01-01','9999-12-31'], and the default print form is'YYYYY-MM-DD'.
## example
mysql> SELECT DATE('2003-12-31 01:02:03');

4
fe/src/main/java/org/apache/doris/analysis/DateLiteral.java
View File

@ -48,9 +48,9 @@ import com.google.common.base.Preconditions;
public class DateLiteral extends LiteralExpr {
private static final Logger LOG = LogManager.getLogger(DateLiteral.class);
private static final DateLiteral MIN_DATE = new DateLiteral(1900, 1, 1);
private static final DateLiteral MIN_DATE = new DateLiteral(0000, 1, 1);
private static final DateLiteral MAX_DATE = new DateLiteral(9999, 12, 31);
private static final DateLiteral MIN_DATETIME = new DateLiteral(1900, 1, 1, 0, 0, 0);
private static final DateLiteral MIN_DATETIME = new DateLiteral(0000, 1, 1, 0, 0, 0);
private static final DateLiteral MAX_DATETIME = new DateLiteral(9999, 12, 31, 23, 59, 59);
public static final DateLiteral UNIX_EPOCH_TIME = new DateLiteral(1970, 01, 01, 00, 00, 00);

4
fe/src/main/java/org/apache/doris/common/util/TimeUtils.java
View File

@ -94,10 +94,10 @@ public class TimeUtils {
TIME_FORMAT.setTimeZone(TIME_ZONE);
try {
MIN_DATE = DATE_FORMAT.parse("1900-01-01");
MIN_DATE = DATE_FORMAT.parse("0000-01-01");
MAX_DATE = DATE_FORMAT.parse("9999-12-31");
MIN_DATETIME = DATETIME_FORMAT.parse("1900-01-01 00:00:00");
MIN_DATETIME = DATETIME_FORMAT.parse("0000-01-01 00:00:00");
MAX_DATETIME = DATETIME_FORMAT.parse("9999-12-31 23:59:59");
} catch (ParseException e) {

4
fe/src/test/java/org/apache/doris/analysis/LiteralExprCompareTest.java
View File

@ -87,8 +87,8 @@ public class LiteralExprCompareTest {
LiteralExpr minDatetime1Same = new DateLiteral(ScalarType.DATETIME, false);
LiteralExpr date8 = new DateLiteral("9999-12-31", ScalarType.DATE);
LiteralExpr date9 = new DateLiteral("9999-12-31 23:59:59", ScalarType.DATETIME);
LiteralExpr date10 = new DateLiteral("1900-01-01", ScalarType.DATE);
LiteralExpr date11 = new DateLiteral("1900-01-01 00:00:00", ScalarType.DATETIME);
LiteralExpr date10 = new DateLiteral("0000-01-01", ScalarType.DATE);
LiteralExpr date11 = new DateLiteral("0000-01-01 00:00:00", ScalarType.DATETIME);
Assert.assertTrue(date1.equals(date1Same) && date1.compareLiteral(date1Same) == 0);
Assert.assertTrue(date1.equals(date1Same) && date1.compareLiteral(date1Same) == 0);

2
fe/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java
View File

@ -134,7 +134,7 @@ public class PartitionKeyTest {
pk1 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("-128"), new PartitionValue("-32768"),
new PartitionValue("-2147483648"), new PartitionValue("-9223372036854775808"),
new PartitionValue("-170141183460469231731687303715884105728"),
new PartitionValue("1900-01-01"), new PartitionValue("1900-01-01 00:00:00")),
new PartitionValue("0000-01-01"), new PartitionValue("0000-01-01 00:00:00")),
allColumns);
pk2 = PartitionKey.createInfinityPartitionKey(allColumns, false);
Assert.assertTrue(pk1.equals(pk2) && pk1.compareTo(pk2) == 0);

6
fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java
View File

@ -57,9 +57,9 @@ public class TimeUtilsTest {
Assert.assertNotNull(TimeUtils.getStartTime());
Assert.assertTrue(TimeUtils.getEstimatedTime(0L) > 0);
Assert.assertEquals(-2209017600000L, TimeUtils.MIN_DATE.getTime());
Assert.assertEquals(-62167420800000L, TimeUtils.MIN_DATE.getTime());
Assert.assertEquals(253402185600000L, TimeUtils.MAX_DATE.getTime());
Assert.assertEquals(-2209017600000L, TimeUtils.MIN_DATETIME.getTime());
Assert.assertEquals(-62167420800000L, TimeUtils.MIN_DATETIME.getTime());
Assert.assertEquals(253402271999000L, TimeUtils.MAX_DATETIME.getTime());
}
@ -74,6 +74,7 @@ public class TimeUtilsTest {
validDateList.add("9999-12-31");
validDateList.add("1900-01-01");
validDateList.add("2013-2-28");
validDateList.add("0000-01-01");
for (String validDate : validDateList) {
try {
TimeUtils.parseDate(validDate, PrimitiveType.DATE);
@ -112,6 +113,7 @@ public class TimeUtilsTest {
validDateTimeList.add("2013-2-28 23:59:59");
validDateTimeList.add("2013-2-28 2:3:4");
validDateTimeList.add("2014-05-07 19:8:50");
validDateTimeList.add("0000-01-01 00:00:00");
for (String validDateTime : validDateTimeList) {
try {
TimeUtils.parseDate(validDateTime, PrimitiveType.DATETIME);