diff --git a/be/src/exec/olap_common.h b/be/src/exec/olap_common.h index 41d8f7aa5c..952e092cab 100644 --- a/be/src/exec/olap_common.h +++ b/be/src/exec/olap_common.h @@ -314,7 +314,8 @@ private: typedef std::variant, ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange<__int128>, ColumnValueRange, ColumnValueRange, - ColumnValueRange, ColumnValueRange> + ColumnValueRange, ColumnValueRange, + ColumnValueRange> ColumnValueRangeType; template diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp index 542363b6e1..b346081b5d 100644 --- a/be/src/exec/tablet_info.cpp +++ b/be/src/exec/tablet_info.cpp @@ -578,14 +578,25 @@ Status VOlapTablePartitionParam::_create_partition_key(const TExprNode& t_expr, auto column = std::move(*part_key->first->get_by_position(pos).column).mutate(); switch (t_expr.node_type) { case TExprNodeType::DATE_LITERAL: { - vectorized::VecDateTimeValue dt; - if (!dt.from_date_str(t_expr.date_literal.value.c_str(), - t_expr.date_literal.value.size())) { - std::stringstream ss; - ss << "invalid date literal in partition column, date=" << t_expr.date_literal; - return Status::InternalError(ss.str()); + if (TypeDescriptor::from_thrift(t_expr.type).is_date_v2_type()) { + vectorized::DateV2Value dt; + if (!dt.from_date_str(t_expr.date_literal.value.c_str(), + t_expr.date_literal.value.size())) { + std::stringstream ss; + ss << "invalid date literal in partition column, date=" << t_expr.date_literal; + return Status::InternalError(ss.str()); + } + column->insert_data(reinterpret_cast(&dt), 0); + } else { + vectorized::VecDateTimeValue dt; + if (!dt.from_date_str(t_expr.date_literal.value.c_str(), + t_expr.date_literal.value.size())) { + std::stringstream ss; + ss << "invalid date literal in partition column, date=" << t_expr.date_literal; + return Status::InternalError(ss.str()); + } + column->insert_data(reinterpret_cast(&dt), 0); } - column->insert_data(reinterpret_cast(&dt), 0); break; } case TExprNodeType::INT_LITERAL: { diff --git a/be/src/exec/tablet_sink.cpp b/be/src/exec/tablet_sink.cpp index a718fa0475..fc3e976d7d 100644 --- a/be/src/exec/tablet_sink.cpp +++ b/be/src/exec/tablet_sink.cpp @@ -777,6 +777,7 @@ Status OlapTableSink::prepare(RuntimeState* state) { case TYPE_VARCHAR: case TYPE_DATE: case TYPE_DATETIME: + case TYPE_DATEV2: case TYPE_HLL: case TYPE_OBJECT: case TYPE_STRING: diff --git a/be/src/exprs/anyval_util.cpp b/be/src/exprs/anyval_util.cpp index f22d857360..374c5a3130 100644 --- a/be/src/exprs/anyval_util.cpp +++ b/be/src/exprs/anyval_util.cpp @@ -142,6 +142,9 @@ FunctionContext::TypeDesc AnyValUtil::column_type_to_type_desc(const TypeDescrip case TYPE_DATETIME: out.type = FunctionContext::TYPE_DATETIME; break; + case TYPE_DATEV2: + out.type = FunctionContext::TYPE_DATEV2; + break; case TYPE_VARCHAR: out.type = FunctionContext::TYPE_VARCHAR; out.len = type.len; diff --git a/be/src/exprs/bloomfilter_predicate.h b/be/src/exprs/bloomfilter_predicate.h index d9e8c8b34e..cb0736dcca 100644 --- a/be/src/exprs/bloomfilter_predicate.h +++ b/be/src/exprs/bloomfilter_predicate.h @@ -248,6 +248,15 @@ struct DateFindOp : public CommonFindOp { } }; +template +struct DateV2FindOp : public CommonFindOp { + bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) const { + doris::vectorized::DateV2Value value; + value.from_date(*reinterpret_cast(data)); + return bloom_filter.test(Slice((char*)&value, sizeof(doris::vectorized::DateV2Value))); + } +}; + template struct DecimalV2FindOp : public CommonFindOp { bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) const { @@ -275,6 +284,11 @@ struct BloomFilterTypeTraits { using FindOp = DateFindOp; }; +template +struct BloomFilterTypeTraits { + using FindOp = DateV2FindOp; +}; + template struct BloomFilterTypeTraits { using FindOp = DateTimeFindOp; diff --git a/be/src/exprs/create_predicate_function.h b/be/src/exprs/create_predicate_function.h index b8acae5964..364634057f 100644 --- a/be/src/exprs/create_predicate_function.h +++ b/be/src/exprs/create_predicate_function.h @@ -92,6 +92,8 @@ typename Traits::BasePtr create_predicate_function(PrimitiveType type) { return Creator::template create(); case TYPE_DATETIME: return Creator::template create(); + case TYPE_DATEV2: + return Creator::template create(); case TYPE_CHAR: return Creator::template create(); diff --git a/be/src/exprs/expr.h b/be/src/exprs/expr.h index e4cb02ea1b..56ff05e8a3 100644 --- a/be/src/exprs/expr.h +++ b/be/src/exprs/expr.h @@ -516,6 +516,15 @@ Status create_texpr_literal_node(const void* data, TExprNode* node) { } else if (origin_value->type() == TimeType::TIME_TIME) { (*node).__set_type(create_type_desc(PrimitiveType::TYPE_TIME)); } + } else if constexpr (std::is_same_v) { + auto origin_value = reinterpret_cast(data); + TDateLiteral date_literal; + char convert_buffer[30]; + origin_value->to_string(convert_buffer); + date_literal.__set_value(convert_buffer); + (*node).__set_date_literal(date_literal); + (*node).__set_node_type(TExprNodeType::DATE_LITERAL); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DATEV2)); } else if constexpr (std::is_same_v) { auto origin_value = reinterpret_cast(data); (*node).__set_node_type(TExprNodeType::DECIMAL_LITERAL); diff --git a/be/src/exprs/expr_context.cpp b/be/src/exprs/expr_context.cpp index 84a51ba619..199076cdcb 100644 --- a/be/src/exprs/expr_context.cpp +++ b/be/src/exprs/expr_context.cpp @@ -256,7 +256,9 @@ void* ExprContext::get_value(Expr* e, TupleRow* row) { return &_result.string_val; } case TYPE_DATE: - case TYPE_DATETIME: { + case TYPE_DATETIME: + case TYPE_DATEV2: + case TYPE_DATETIMEV2: { doris_udf::DateTimeVal v = e->get_datetime_val(this, row); if (v.is_null) { return nullptr; diff --git a/be/src/exprs/literal.cpp b/be/src/exprs/literal.cpp index 94b9a97f0c..462ec259be 100644 --- a/be/src/exprs/literal.cpp +++ b/be/src/exprs/literal.cpp @@ -81,6 +81,8 @@ Literal::Literal(const TExprNode& node) : Expr(node) { break; case TYPE_DATE: case TYPE_DATETIME: + case TYPE_DATEV2: + case TYPE_DATETIMEV2: _value.datetime_val.from_date_str(node.date_literal.value.c_str(), node.date_literal.value.size()); break; diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp index 2533c8c2d1..bab230d73b 100644 --- a/be/src/exprs/runtime_filter.cpp +++ b/be/src/exprs/runtime_filter.cpp @@ -74,6 +74,9 @@ TExprNodeType::type get_expr_node_type(PrimitiveType type) { case TYPE_DATETIME: return TExprNodeType::DATE_LITERAL; + case TYPE_DATEV2: + return TExprNodeType::DATE_LITERAL; + case TYPE_CHAR: case TYPE_VARCHAR: case TYPE_HLL: @@ -109,6 +112,8 @@ PColumnType to_proto(PrimitiveType type) { return PColumnType::COLUMN_TYPE_DOUBLE; case TYPE_DATE: return PColumnType::COLUMN_TYPE_DATE; + case TYPE_DATEV2: + return PColumnType::COLUMN_TYPE_DATEV2; case TYPE_DATETIME: return PColumnType::COLUMN_TYPE_DATETIME; case TYPE_DECIMALV2: @@ -148,6 +153,8 @@ PrimitiveType to_primitive_type(PColumnType type) { return TYPE_DOUBLE; case PColumnType::COLUMN_TYPE_DATE: return TYPE_DATE; + case PColumnType::COLUMN_TYPE_DATEV2: + return TYPE_DATEV2; case PColumnType::COLUMN_TYPE_DATETIME: return TYPE_DATETIME; case PColumnType::COLUMN_TYPE_DECIMALV2: @@ -233,6 +240,10 @@ Status create_literal(ObjectPool* pool, PrimitiveType type, const void* data, vo create_texpr_literal_node(data, &node); break; } + case TYPE_DATEV2: { + create_texpr_literal_node(data, &node); + break; + } case TYPE_DATE: case TYPE_DATETIME: { create_texpr_literal_node(data, &node); @@ -669,6 +680,14 @@ public: }); break; } + case TYPE_DATEV2: { + batch_assign(in_filter, [](std::unique_ptr& set, PColumnValue& column, + ObjectPool* pool) { + auto date_v2_val = column.intval(); + set->insert(&date_v2_val); + }); + break; + } case TYPE_DATETIME: case TYPE_DATE: { batch_assign(in_filter, [](std::unique_ptr& set, PColumnValue& column, @@ -773,6 +792,11 @@ public: double max_val = static_cast(minmax_filter->max_val().doubleval()); return _minmax_func->assign(&min_val, &max_val); } + case TYPE_DATEV2: { + int32_t min_val = minmax_filter->min_val().intval(); + int32_t max_val = minmax_filter->max_val().intval(); + return _minmax_func->assign(&min_val, &max_val); + } case TYPE_DATETIME: case TYPE_DATE: { auto& min_val_ref = minmax_filter->min_val().stringval(); @@ -1246,6 +1270,13 @@ void IRuntimeFilter::to_protobuf(PInFilter* filter) { }); return; } + case TYPE_DATEV2: { + batch_copy( + filter, it, [](PColumnValue* column, const doris::vectorized::DateV2Value* value) { + column->set_intval(*reinterpret_cast(value)); + }); + return; + } case TYPE_DATE: case TYPE_DATETIME: { batch_copy(filter, it, [](PColumnValue* column, const DateTimeValue* value) { @@ -1328,6 +1359,11 @@ void IRuntimeFilter::to_protobuf(PMinMaxFilter* filter) { filter->mutable_max_val()->set_doubleval(*reinterpret_cast(max_data)); return; } + case TYPE_DATEV2: { + filter->mutable_min_val()->set_intval(*reinterpret_cast(min_data)); + filter->mutable_max_val()->set_intval(*reinterpret_cast(max_data)); + return; + } case TYPE_DATE: case TYPE_DATETIME: { char convert_buffer[30]; diff --git a/be/src/olap/aggregate_func.cpp b/be/src/olap/aggregate_func.cpp index 781ab39082..dda4d0f780 100644 --- a/be/src/olap/aggregate_func.cpp +++ b/be/src/olap/aggregate_func.cpp @@ -100,6 +100,7 @@ AggregateFuncResolver::AggregateFuncResolver() { add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); + add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); @@ -147,6 +148,7 @@ AggregateFuncResolver::AggregateFuncResolver() { add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); + add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); @@ -162,6 +164,7 @@ AggregateFuncResolver::AggregateFuncResolver() { add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); + add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); @@ -188,6 +191,7 @@ AggregateFuncResolver::AggregateFuncResolver() { add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); + add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); @@ -204,6 +208,7 @@ AggregateFuncResolver::AggregateFuncResolver() { add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); + add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); diff --git a/be/src/olap/column_vector.cpp b/be/src/olap/column_vector.cpp index c4962926f2..14817c85cc 100644 --- a/be/src/olap/column_vector.cpp +++ b/be/src/olap/column_vector.cpp @@ -91,6 +91,10 @@ Status ColumnVectorBatch::create(size_t init_capacity, bool is_nullable, const T local.reset(new ScalarColumnVectorBatch::CppType>( type_info, is_nullable)); break; + case OLAP_FIELD_TYPE_DATEV2: + local.reset(new ScalarColumnVectorBatch::CppType>( + type_info, is_nullable)); + break; case OLAP_FIELD_TYPE_DATETIME: local.reset( new ScalarColumnVectorBatch::CppType>( diff --git a/be/src/olap/comparison_predicate.cpp b/be/src/olap/comparison_predicate.cpp index 3424bbb6c9..870be42185 100644 --- a/be/src/olap/comparison_predicate.cpp +++ b/be/src/olap/comparison_predicate.cpp @@ -585,6 +585,7 @@ COMPARISON_PRED_CONSTRUCTOR_DECLARATION(GreaterEqualPredicate) template void CLASS::evaluate(VectorizedRowBatch* batch) const; \ template void CLASS::evaluate(VectorizedRowBatch* batch) const; \ template void CLASS::evaluate(VectorizedRowBatch* batch) const; \ + template void CLASS::evaluate(VectorizedRowBatch* batch) const; \ template void CLASS::evaluate(VectorizedRowBatch* batch) const; \ template void CLASS::evaluate(VectorizedRowBatch* batch) const; @@ -615,6 +616,8 @@ COMPARISON_PRED_EVALUATE_DECLARATION(GreaterEqualPredicate) const; \ template void CLASS::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) \ const; \ + template void CLASS::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) \ + const; \ template void CLASS::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) \ const; \ template void CLASS::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const; @@ -657,6 +660,9 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_DECLARATION(GreaterEqualPredicate) template Status CLASS::evaluate(const Schema& schema, \ const std::vector& iterators, \ uint32_t num_rows, roaring::Roaring* bitmap) const; \ + template Status CLASS::evaluate(const Schema& schema, \ + const std::vector& iterators, \ + uint32_t num_rows, roaring::Roaring* bitmap) const; \ template Status CLASS::evaluate(const Schema& schema, \ const std::vector& iterators, \ uint32_t num_rows, roaring::Roaring* bitmap) const; \ @@ -692,6 +698,8 @@ COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(GreaterEqualPredicate) uint16_t size) const; \ template uint16_t CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, \ uint16_t size) const; \ + template uint16_t CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, \ + uint16_t size) const; \ template uint16_t CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, \ uint16_t size) const; \ template uint16_t CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, \ diff --git a/be/src/olap/delete_handler.cpp b/be/src/olap/delete_handler.cpp index 4e39bc4c73..69e5f0b7af 100644 --- a/be/src/olap/delete_handler.cpp +++ b/be/src/olap/delete_handler.cpp @@ -146,6 +146,7 @@ bool DeleteConditionHandler::is_condition_value_valid(const TabletColumn& column return value_str.size() <= config::string_type_length_soft_limit_bytes; case OLAP_FIELD_TYPE_DATE: case OLAP_FIELD_TYPE_DATETIME: + case OLAP_FIELD_TYPE_DATEV2: return valid_datetime(value_str); case OLAP_FIELD_TYPE_BOOL: return valid_bool(value_str); diff --git a/be/src/olap/key_coder.cpp b/be/src/olap/key_coder.cpp index 3e2ab5ba55..49a06a13d8 100644 --- a/be/src/olap/key_coder.cpp +++ b/be/src/olap/key_coder.cpp @@ -73,6 +73,7 @@ private: add_mapping(); add_mapping(); add_mapping(); + add_mapping(); } template diff --git a/be/src/olap/key_coder.h b/be/src/olap/key_coder.h index b6afcf6c9e..7f452350d4 100644 --- a/be/src/olap/key_coder.h +++ b/be/src/olap/key_coder.h @@ -170,6 +170,40 @@ public: } }; +template <> +class KeyCoderTraits { +public: + using CppType = typename CppTypeTraits::CppType; + using UnsignedCppType = typename CppTypeTraits::UnsignedCppType; + +public: + static void full_encode_ascending(const void* value, std::string* buf) { + UnsignedCppType unsigned_val; + memcpy(&unsigned_val, value, sizeof(unsigned_val)); + // make it bigendian + unsigned_val = BigEndian::FromHost32(unsigned_val); + buf->append((char*)&unsigned_val, sizeof(unsigned_val)); + } + + static void encode_ascending(const void* value, size_t index_size, std::string* buf) { + full_encode_ascending(value, buf); + } + + static Status decode_ascending(Slice* encoded_key, size_t index_size, uint8_t* cell_ptr, + MemPool* pool) { + if (encoded_key->size < sizeof(UnsignedCppType)) { + return Status::InvalidArgument(Substitute("Key too short, need=$0 vs real=$1", + sizeof(UnsignedCppType), encoded_key->size)); + } + UnsignedCppType unsigned_val; + memcpy(&unsigned_val, encoded_key->data, sizeof(UnsignedCppType)); + unsigned_val = BigEndian::FromHost32(unsigned_val); + memcpy(cell_ptr, &unsigned_val, sizeof(UnsignedCppType)); + encoded_key->remove_prefix(sizeof(UnsignedCppType)); + return Status::OK(); + } +}; + template <> class KeyCoderTraits { public: diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index 7396a3cbf1..9bec6cfebe 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -143,7 +143,9 @@ enum FieldType { OLAP_FIELD_TYPE_BOOL = 24, OLAP_FIELD_TYPE_OBJECT = 25, OLAP_FIELD_TYPE_STRING = 26, - OLAP_FIELD_TYPE_QUANTILE_STATE = 27 + OLAP_FIELD_TYPE_QUANTILE_STATE = 27, + OLAP_FIELD_TYPE_DATEV2 = 28, + OLAP_FIELD_TYPE_DATETIMEV2 = 29 }; // Define all aggregation methods supported by Field diff --git a/be/src/olap/reader.cpp b/be/src/olap/reader.cpp index 497c014e46..3d8a3f398f 100644 --- a/be/src/olap/reader.cpp +++ b/be/src/olap/reader.cpp @@ -533,6 +533,11 @@ void TabletReader::_init_conditions_param(const ReaderParams& read_params) { predicate = new PREDICATE(index, value, opposite); \ break; \ } \ + case OLAP_FIELD_TYPE_DATEV2: { \ + uint32_t value = timestamp_from_date_v2(cond); \ + predicate = new PREDICATE(index, value, opposite); \ + break; \ + } \ case OLAP_FIELD_TYPE_DATETIME: { \ uint64_t value = timestamp_from_datetime(cond); \ predicate = new PREDICATE(index, value, opposite); \ @@ -745,6 +750,19 @@ ColumnPredicate* TabletReader::_parse_to_predicate(const TCondition& condition, } break; } + case OLAP_FIELD_TYPE_DATEV2: { + phmap::flat_hash_set values; + for (auto& cond_val : condition.condition_values) { + uint32_t value = timestamp_from_date_v2(cond_val); + values.insert(value); + } + if (condition.condition_op == "*=") { + predicate = new InListPredicate(index, std::move(values), opposite); + } else { + predicate = new NotInListPredicate(index, std::move(values), opposite); + } + break; + } case OLAP_FIELD_TYPE_DATETIME: { phmap::flat_hash_set values; for (auto& cond_val : condition.condition_values) { diff --git a/be/src/olap/row_block2.cpp b/be/src/olap/row_block2.cpp index 947bdd4055..393fe40adf 100644 --- a/be/src/olap/row_block2.cpp +++ b/be/src/olap/row_block2.cpp @@ -245,6 +245,11 @@ Status RowBlockV2::_copy_data_to_column(int cid, } break; } + case OLAP_FIELD_TYPE_DATEV2: { + auto column_int = assert_cast*>(column); + insert_data_directly(cid, column_int); + break; + } case OLAP_FIELD_TYPE_DATETIME: { auto column_int = assert_cast*>(column); @@ -507,6 +512,11 @@ Status RowBlockV2::_append_data_to_column(const ColumnVectorBatch* batch, size_t } break; } + case OLAP_FIELD_TYPE_DATEV2: { + auto column_int = assert_cast*>(column); + insert_data_directly(batch, column_int, start, len); + break; + } case OLAP_FIELD_TYPE_DATETIME: { auto column_int = assert_cast*>(column); diff --git a/be/src/olap/rowset/column_reader.cpp b/be/src/olap/rowset/column_reader.cpp index ee6c7fb1ad..8c6cc1676c 100644 --- a/be/src/olap/rowset/column_reader.cpp +++ b/be/src/olap/rowset/column_reader.cpp @@ -610,6 +610,12 @@ ColumnReader* ColumnReader::create(uint32_t column_id, const std::vector(mem_pool->allocate(size * sizeof(uint32_t))); + uint32_t value = 0; + std::stringstream ss(_default_value); + ss >> value; + for (int i = 0; i < size; ++i) { + ((uint32_t*)_values)[i] = value; + } + break; + } case OLAP_FIELD_TYPE_DATETIME: { _values = reinterpret_cast(mem_pool->allocate(size * sizeof(uint64_t))); uint64_t value = timestamp_from_datetime(_default_value); @@ -824,6 +834,8 @@ typedef IntegerColumnReaderWrapper DateColumnReader; // Internal use LONG implementation typedef IntegerColumnReaderWrapper DateTimeColumnReader; +typedef IntegerColumnReaderWrapper DateV2ColumnReader; + } // namespace doris #endif // DORIS_BE_SRC_OLAP_ROWSET_COLUMN_READER_H diff --git a/be/src/olap/rowset/column_writer.cpp b/be/src/olap/rowset/column_writer.cpp index 8053d92621..af212a6927 100644 --- a/be/src/olap/rowset/column_writer.cpp +++ b/be/src/olap/rowset/column_writer.cpp @@ -96,6 +96,11 @@ ColumnWriter* ColumnWriter::create(uint32_t column_id, const TabletSchema& schem DateColumnWriter(column_id, stream_factory, column, num_rows_per_row_block, bf_fpp); break; } + case OLAP_FIELD_TYPE_DATEV2: { + column_writer = new (std::nothrow) DateV2ColumnWriter(column_id, stream_factory, column, + num_rows_per_row_block, bf_fpp); + break; + } case OLAP_FIELD_TYPE_DECIMAL: { column_writer = new (std::nothrow) DecimalColumnWriter(column_id, stream_factory, column, num_rows_per_row_block, bf_fpp); diff --git a/be/src/olap/rowset/column_writer.h b/be/src/olap/rowset/column_writer.h index e4904de363..7771189e74 100644 --- a/be/src/olap/rowset/column_writer.h +++ b/be/src/olap/rowset/column_writer.h @@ -489,6 +489,9 @@ typedef IntegerColumnWriterWrapper DateColumnWriter; // DateTime is implemented with int64 typedef IntegerColumnWriterWrapper DateTimeColumnWriter; +// DateTime is implemented with int64 +typedef IntegerColumnWriterWrapper DateV2ColumnWriter; + class DecimalColumnWriter : public ColumnWriter { public: DecimalColumnWriter(uint32_t column_id, OutStreamFactory* stream_factory, diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp index 93cac8b5d2..9308c432b4 100644 --- a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp @@ -220,6 +220,9 @@ Status BitmapIndexWriter::create(const TypeInfo* type_info, case OLAP_FIELD_TYPE_DATETIME: res->reset(new BitmapIndexWriterImpl(type_info)); break; + case OLAP_FIELD_TYPE_DATEV2: + res->reset(new BitmapIndexWriterImpl(type_info)); + break; case OLAP_FIELD_TYPE_LARGEINT: res->reset(new BitmapIndexWriterImpl(type_info)); break; diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 8b0c8bc649..d790c8aee3 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -888,6 +888,13 @@ void DefaultValueColumnIterator::insert_default_data(vectorized::MutableColumnPt insert_column_data(); break; } + case OLAP_FIELD_TYPE_DATEV2: { + assert(_type_size == sizeof(FieldTypeTraits::CppType)); //uint32_t + + int128 = *((FieldTypeTraits::CppType*)_mem_value); + insert_column_data(); + break; + } case OLAP_FIELD_TYPE_DECIMAL: { assert(_type_size == sizeof(FieldTypeTraits::CppType)); //decimal12_t diff --git a/be/src/olap/rowset/segment_v2/encoding_info.cpp b/be/src/olap/rowset/segment_v2/encoding_info.cpp index 2c9a83e8a3..ac3a1a624f 100644 --- a/be/src/olap/rowset/segment_v2/encoding_info.cpp +++ b/be/src/olap/rowset/segment_v2/encoding_info.cpp @@ -135,6 +135,20 @@ struct TypeEncodingTraits +struct TypeEncodingTraits::CppType> { + static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) { + *builder = new FrameOfReferencePageBuilder(opts); + return Status::OK(); + } + static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts, + PageDecoder** decoder) { + *decoder = new FrameOfReferencePageDecoder(data, opts); + return Status::OK(); + } +}; + template struct TypeEncodingTraits::value>::type> { @@ -268,6 +282,10 @@ EncodingInfoResolver::EncodingInfoResolver() { _add_map(); _add_map(); + _add_map(); + _add_map(); + _add_map(); + _add_map(); _add_map(); _add_map(); diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 0769db56eb..61adede9f2 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -831,7 +831,10 @@ void SegmentIterator::_init_current_block( if (column_desc->type() == OLAP_FIELD_TYPE_DATE) { current_columns[cid]->set_date_type(); } else if (column_desc->type() == OLAP_FIELD_TYPE_DATETIME) { + // TODO(Gabriel): support datetime v2 current_columns[cid]->set_datetime_type(); + } else if (column_desc->type() == OLAP_FIELD_TYPE_DATEV2) { + current_columns[cid]->set_date_v2_type(); } current_columns[cid]->reserve(_opts.block_row_max); } diff --git a/be/src/olap/schema.cpp b/be/src/olap/schema.cpp index a5e7896147..dce43a1bdb 100644 --- a/be/src/olap/schema.cpp +++ b/be/src/olap/schema.cpp @@ -153,6 +153,9 @@ vectorized::IColumn::MutablePtr Schema::get_predicate_column_ptr(FieldType type) case OLAP_FIELD_TYPE_DATE: return doris::vectorized::PredicateColumnType::create(); + case OLAP_FIELD_TYPE_DATEV2: + return doris::vectorized::PredicateColumnType::create(); + case OLAP_FIELD_TYPE_DATETIME: return doris::vectorized::PredicateColumnType::create(); diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index 683dff9d62..cda1370b7f 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -58,6 +58,8 @@ FieldType TabletColumn::get_field_type_by_string(const std::string& type_str) { type = OLAP_FIELD_TYPE_CHAR; } else if (0 == upper_type_str.compare("DATE")) { type = OLAP_FIELD_TYPE_DATE; + } else if (0 == upper_type_str.compare("DATEV2")) { + type = OLAP_FIELD_TYPE_DATEV2; } else if (0 == upper_type_str.compare("DATETIME")) { type = OLAP_FIELD_TYPE_DATETIME; } else if (0 == upper_type_str.compare(0, 7, "DECIMAL")) { @@ -165,6 +167,9 @@ std::string TabletColumn::get_string_by_field_type(FieldType type) { case OLAP_FIELD_TYPE_DATE: return "DATE"; + case OLAP_FIELD_TYPE_DATEV2: + return "DATEV2"; + case OLAP_FIELD_TYPE_DATETIME: return "DATETIME"; @@ -251,6 +256,8 @@ uint32_t TabletColumn::get_field_length_by_type(TPrimitiveType::type type, uint3 return 16; case TPrimitiveType::DATE: return 3; + case TPrimitiveType::DATEV2: + return 4; case TPrimitiveType::DATETIME: return 8; case TPrimitiveType::FLOAT: diff --git a/be/src/olap/types.cpp b/be/src/olap/types.cpp index bced5024c1..d1b2f345bf 100644 --- a/be/src/olap/types.cpp +++ b/be/src/olap/types.cpp @@ -83,6 +83,7 @@ const TypeInfo* get_scalar_type_info(FieldType field_type) { get_scalar_type_info(), get_scalar_type_info(), get_scalar_type_info(), + get_scalar_type_info(), }; return field_type_array[field_type]; } @@ -147,6 +148,7 @@ const TypeInfo* get_array_type_info(FieldType leaf_type, int32_t iterations) { INIT_ARRAY_TYPE_INFO_LIST(OLAP_FIELD_TYPE_OBJECT), INIT_ARRAY_TYPE_INFO_LIST(OLAP_FIELD_TYPE_STRING), INIT_ARRAY_TYPE_INFO_LIST(OLAP_FIELD_TYPE_QUANTILE_STATE), + INIT_ARRAY_TYPE_INFO_LIST(OLAP_FIELD_TYPE_DATEV2), }; return array_type_Info_arr[leaf_type][iterations]; } diff --git a/be/src/olap/types.h b/be/src/olap/types.h index 915239b0ee..6f5bb48ef3 100644 --- a/be/src/olap/types.h +++ b/be/src/olap/types.h @@ -500,6 +500,11 @@ struct CppTypeTraits { using UnsignedCppType = uint24_t; }; template <> +struct CppTypeTraits { + using CppType = uint32_t; + using UnsignedCppType = uint32_t; +}; +template <> struct CppTypeTraits { using CppType = int64_t; using UnsignedCppType = uint64_t; @@ -926,6 +931,15 @@ struct FieldTypeTraits : public BaseFieldtypeTraits(part1 % 100); *reinterpret_cast(dest) = (year << 9) + (mon << 5) + mday; return Status::OK(); + } else if (src_type->type() == FieldType::OLAP_FIELD_TYPE_DATEV2) { + using SrcType = typename CppTypeTraits::CppType; + SrcType src_value = *reinterpret_cast(src); + //only need part one + CppType year = static_cast((src_value & 0xFFFF0000) >> 16); + CppType mon = static_cast((src_value & 0xFF00) >> 8); + CppType mday = static_cast(src_value & 0xFF); + *reinterpret_cast(dest) = (year << 9) + (mon << 5) + mday; + return Status::OK(); } if (src_type->type() == FieldType::OLAP_FIELD_TYPE_INT) { @@ -974,6 +988,104 @@ struct FieldTypeTraits : public BaseFieldtypeTraits +struct FieldTypeTraits + : public BaseFieldtypeTraits { + static Status from_string(void* buf, const std::string& scan_key) { + tm time_tm; + char* res = strptime(scan_key.c_str(), "%Y-%m-%d", &time_tm); + + if (nullptr != res) { + uint32_t value = ((time_tm.tm_year + 1900) << 16) | ((time_tm.tm_mon + 1) << 8) | + time_tm.tm_mday; + *reinterpret_cast(buf) = value; + } else { + *reinterpret_cast(buf) = doris::vectorized::MIN_DATE_V2; + } + + return Status::OK(); + } + static std::string to_string(const void* src) { + CppType tmp = *reinterpret_cast(src); + doris::vectorized::DateV2Value value = + binary_cast(tmp); + string format = "%Y-%m-%d"; + string res; + res.resize(12); + res.reserve(12); + value.to_format_string(format.c_str(), format.size(), res.data()); + return res; + } + static Status convert_from(void* dest, const void* src, const TypeInfo* src_type, + MemPool* mem_pool, size_t variable_len = 0) { + if (src_type->type() == FieldType::OLAP_FIELD_TYPE_DATETIME) { + using SrcType = typename CppTypeTraits::CppType; + SrcType src_value = *reinterpret_cast(src); + //only need part one + SrcType part1 = (src_value / 1000000L); + CppType year = static_cast((part1 / 10000L) % 10000); + CppType mon = static_cast((part1 / 100) % 100); + CppType mday = static_cast(part1 % 100); + *reinterpret_cast(dest) = (year << 16) | (mon << 8) | mday; + return Status::OK(); + } + + if (src_type->type() == FieldType::OLAP_FIELD_TYPE_DATE) { + using SrcType = typename CppTypeTraits::CppType; + SrcType value = *reinterpret_cast(src); + int day = static_cast(value & 31); + int mon = static_cast(value >> 5 & 15); + int year = static_cast(value >> 9); + *reinterpret_cast(dest) = (year << 16) | (mon << 8) | day; + return Status::OK(); + } + + if (src_type->type() == FieldType::OLAP_FIELD_TYPE_INT) { + using SrcType = typename CppTypeTraits::CppType; + SrcType src_value = *reinterpret_cast(src); + doris::vectorized::DateV2Value dt; + if (!dt.from_date_int64(src_value)) { + return Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA); + } + CppType year = static_cast(src_value / 10000); + CppType month = static_cast((src_value % 10000) / 100); + CppType day = static_cast(src_value % 100); + *reinterpret_cast(dest) = (year << 16) | (month << 8) | day; + return Status::OK(); + } + + if (src_type->type() == OLAP_FIELD_TYPE_VARCHAR || + src_type->type() == OLAP_FIELD_TYPE_CHAR || + src_type->type() == OLAP_FIELD_TYPE_STRING) { + if (src_type->type() == OLAP_FIELD_TYPE_CHAR) { + prepare_char_before_convert(src); + } + using SrcType = typename CppTypeTraits::CppType; + auto src_value = *reinterpret_cast(src); + doris::vectorized::DateV2Value dt; + for (const auto& format : DATE_FORMATS) { + if (dt.from_date_format_str(format.c_str(), format.length(), src_value.get_data(), + src_value.get_size())) { + *reinterpret_cast(dest) = + (dt.year() << 16) | (dt.month() << 8) | dt.day(); + return Status::OK(); + } + } + return Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA); + } + + return Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA); + } + static void set_to_max(void* buf) { + // max is 9999 * 16 * 32 + 12 * 32 + 31; + *reinterpret_cast(buf) = doris::vectorized::MAX_DATE_V2; + } + static void set_to_min(void* buf) { + // min is 0 * 16 * 32 + 1 * 32 + 1; + *reinterpret_cast(buf) = doris::vectorized::MIN_DATE_V2; + } +}; + template <> struct FieldTypeTraits : public BaseFieldtypeTraits { @@ -1023,6 +1135,14 @@ struct FieldTypeTraits int year = static_cast(value >> 9); *reinterpret_cast(dest) = (year * 10000L + mon * 100L + day) * 1000000; return Status::OK(); + } else if (src_type->type() == FieldType::OLAP_FIELD_TYPE_DATEV2) { + using SrcType = typename CppTypeTraits::CppType; + auto value = *reinterpret_cast(src); + int day = static_cast(value & 0xFF); + int mon = static_cast((value & 0xFF00) >> 8); + int year = static_cast((value & 0xFFFF0000) >> 16); + *reinterpret_cast(dest) = (year * 10000L + mon * 100L + day) * 1000000; + return Status::OK(); } return Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA); } diff --git a/be/src/olap/utils.h b/be/src/olap/utils.h index b84fe024ce..8cd6f2b2c4 100644 --- a/be/src/olap/utils.h +++ b/be/src/olap/utils.h @@ -266,7 +266,8 @@ bool valid_unsigned_number(const std::string& value_str) { bool valid_decimal(const std::string& value_str, const uint32_t precision, const uint32_t frac); -// 粗略检查date或者datetime类型是否正确 +// Validate for date/datetime roughly. The format is 'yyyy-MM-dd HH:mm:ss' +// TODO: support 'yyyy-MM-dd HH:mm:ss.SSS' bool valid_datetime(const std::string& value_str); bool valid_bool(const std::string& value_str); diff --git a/be/src/runtime/datetime_value.h b/be/src/runtime/datetime_value.h index 21ed59b2d1..20caa2a428 100644 --- a/be/src/runtime/datetime_value.h +++ b/be/src/runtime/datetime_value.h @@ -570,6 +570,8 @@ private: friend class UnusedClass; friend void doris::vectorized::VecDateTimeValue::convert_vec_dt_to_dt(DateTimeValue* dt); friend void doris::vectorized::VecDateTimeValue::convert_dt_to_vec_dt(DateTimeValue* dt); + friend void doris::vectorized::DateV2Value::convert_date_v2_to_dt(DateTimeValue* dt); + friend void doris::vectorized::DateV2Value::convert_dt_to_date_v2(DateTimeValue* dt); void from_packed_time(int64_t packed_time) { _microsecond = packed_time % (1LL << 24); diff --git a/be/src/runtime/primitive_type.cpp b/be/src/runtime/primitive_type.cpp index 2521559ab6..1a69e3368b 100644 --- a/be/src/runtime/primitive_type.cpp +++ b/be/src/runtime/primitive_type.cpp @@ -65,6 +65,12 @@ PrimitiveType convert_type_to_primitive(FunctionContext::Type type) { return PrimitiveType::TYPE_LARGEINT; case FunctionContext::Type::TYPE_DATE: return PrimitiveType::TYPE_DATE; + case FunctionContext::Type::TYPE_DATEV2: + return PrimitiveType::TYPE_DATEV2; + case FunctionContext::Type::TYPE_DATETIMEV2: + return PrimitiveType::TYPE_DATETIMEV2; + case FunctionContext::Type::TYPE_TIMEV2: + return PrimitiveType::TYPE_TIMEV2; default: DCHECK(false); } @@ -81,6 +87,8 @@ bool is_enumeration_type(PrimitiveType type) { case TYPE_VARCHAR: case TYPE_STRING: case TYPE_DATETIME: + case TYPE_DATETIMEV2: + case TYPE_TIMEV2: case TYPE_DECIMALV2: case TYPE_BOOLEAN: case TYPE_ARRAY: @@ -92,6 +100,7 @@ bool is_enumeration_type(PrimitiveType type) { case TYPE_BIGINT: case TYPE_LARGEINT: case TYPE_DATE: + case TYPE_DATEV2: return true; case INVALID_TYPE: @@ -103,7 +112,8 @@ bool is_enumeration_type(PrimitiveType type) { } bool is_date_type(PrimitiveType type) { - return type == TYPE_DATETIME || type == TYPE_DATE; + return type == TYPE_DATETIME || type == TYPE_DATE || type == TYPE_DATETIMEV2 || + type == TYPE_DATEV2; } bool is_string_type(PrimitiveType type) { @@ -151,6 +161,10 @@ int get_byte_size(PrimitiveType type) { return 16; case INVALID_TYPE: + // datev2/datetimev2/timev2 is not supported on row-based engine + case TYPE_DATEV2: + case TYPE_DATETIMEV2: + case TYPE_TIMEV2: default: DCHECK(false); } @@ -233,6 +247,15 @@ PrimitiveType thrift_to_type(TPrimitiveType::type ttype) { case TPrimitiveType::DATETIME: return TYPE_DATETIME; + case TPrimitiveType::DATEV2: + return TYPE_DATEV2; + + case TPrimitiveType::DATETIMEV2: + return TYPE_DATETIMEV2; + + case TPrimitiveType::TIMEV2: + return TYPE_TIMEV2; + case TPrimitiveType::TIME: return TYPE_TIME; @@ -309,6 +332,15 @@ TPrimitiveType::type to_thrift(PrimitiveType ptype) { case TYPE_TIME: return TPrimitiveType::TIME; + case TYPE_DATEV2: + return TPrimitiveType::DATEV2; + + case TYPE_DATETIMEV2: + return TPrimitiveType::DATETIMEV2; + + case TYPE_TIMEV2: + return TPrimitiveType::TIMEV2; + case TYPE_VARCHAR: return TPrimitiveType::VARCHAR; @@ -382,6 +414,15 @@ std::string type_to_string(PrimitiveType t) { case TYPE_TIME: return "TIME"; + case TYPE_DATEV2: + return "DATEV2"; + + case TYPE_DATETIMEV2: + return "DATETIMEV2"; + + case TYPE_TIMEV2: + return "TIMEV2"; + case TYPE_VARCHAR: return "VARCHAR"; @@ -456,6 +497,15 @@ std::string type_to_odbc_string(PrimitiveType t) { case TYPE_DATETIME: return "datetime"; + case TYPE_DATEV2: + return "datev2"; + + case TYPE_DATETIMEV2: + return "datetimev2"; + + case TYPE_TIMEV2: + return "timev2"; + case TYPE_VARCHAR: return "string"; @@ -535,6 +585,7 @@ int get_slot_size(PrimitiveType type) { return 2; case TYPE_INT: + case TYPE_DATEV2: case TYPE_FLOAT: return 4; @@ -548,6 +599,8 @@ int get_slot_size(PrimitiveType type) { case TYPE_DATE: case TYPE_DATETIME: + case TYPE_DATETIMEV2: + case TYPE_TIMEV2: // This is the size of the slot, the actual size of the data is 12. return sizeof(DateTimeValue); diff --git a/be/src/runtime/primitive_type.h b/be/src/runtime/primitive_type.h index 7ba53588b4..3c0254013a 100644 --- a/be/src/runtime/primitive_type.h +++ b/be/src/runtime/primitive_type.h @@ -58,10 +58,13 @@ enum PrimitiveType { TYPE_HLL, /* 19 */ TYPE_DECIMALV2, /* 20 */ - TYPE_TIME, /* 21 */ - TYPE_OBJECT, /* 22 */ - TYPE_STRING, /* 23 */ - TYPE_QUANTILE_STATE /* 24 */ + TYPE_TIME, /* 21 */ + TYPE_OBJECT, /* 22 */ + TYPE_STRING, /* 23 */ + TYPE_QUANTILE_STATE, /* 24 */ + TYPE_DATEV2, /* 25 */ + TYPE_DATETIMEV2, /* 26 */ + TYPE_TIMEV2, /* 27 */ }; PrimitiveType convert_type_to_primitive(FunctionContext::Type type); @@ -141,6 +144,11 @@ struct PrimitiveTypeTraits { using ColumnType = vectorized::ColumnVector; }; template <> +struct PrimitiveTypeTraits { + using CppType = doris::vectorized::DateV2Value; + using ColumnType = vectorized::ColumnVector; +}; +template <> struct PrimitiveTypeTraits { using CppType = DecimalV2Value; using ColumnType = vectorized::ColumnDecimal; @@ -188,4 +196,14 @@ struct PredicatePrimitiveTypeTraits { using PredicateFieldType = uint64_t; }; +template <> +struct PredicatePrimitiveTypeTraits { + using PredicateFieldType = uint32_t; +}; + +template <> +struct PredicatePrimitiveTypeTraits { + using PredicateFieldType = uint64_t; +}; + } // namespace doris diff --git a/be/src/runtime/raw_value.cpp b/be/src/runtime/raw_value.cpp index ac0a95c0c4..740ca3e47a 100644 --- a/be/src/runtime/raw_value.cpp +++ b/be/src/runtime/raw_value.cpp @@ -84,6 +84,10 @@ void RawValue::print_value_as_bytes(const void* value, const TypeDescriptor& typ stream->write(chars, sizeof(DateTimeValue)); break; + case TYPE_DATEV2: + stream->write(chars, sizeof(doris::vectorized::DateV2Value)); + break; + case TYPE_DECIMALV2: stream->write(chars, sizeof(DecimalV2Value)); break; @@ -162,6 +166,10 @@ void RawValue::print_value(const void* value, const TypeDescriptor& type, int sc *stream << *reinterpret_cast(value); break; + case TYPE_DATEV2: + *stream << *reinterpret_cast(value); + break; + case TYPE_DECIMALV2: *stream << DecimalV2Value(reinterpret_cast(value)->value).to_string(); break; @@ -293,6 +301,11 @@ void RawValue::write(const void* value, void* dst, const TypeDescriptor& type, M *reinterpret_cast(dst) = *reinterpret_cast(value); break; + case TYPE_DATEV2: + *reinterpret_cast(dst) = + *reinterpret_cast(value); + break; + case TYPE_DECIMALV2: *reinterpret_cast(dst) = *reinterpret_cast(value); break; @@ -378,6 +391,10 @@ void RawValue::write(const void* value, const TypeDescriptor& type, void* dst, u case TYPE_DATETIME: *reinterpret_cast(dst) = *reinterpret_cast(value); break; + case TYPE_DATEV2: + *reinterpret_cast(dst) = + *reinterpret_cast(value); + break; case TYPE_VARCHAR: case TYPE_CHAR: case TYPE_STRING: { @@ -481,6 +498,12 @@ int RawValue::compare(const void* v1, const void* v2, const TypeDescriptor& type ts_value2 = reinterpret_cast(v2); return *ts_value1 > *ts_value2 ? 1 : (*ts_value1 < *ts_value2 ? -1 : 0); + case TYPE_DATEV2: { + auto date_v2_value1 = reinterpret_cast(v1); + auto date_v2_value2 = reinterpret_cast(v2); + return *date_v2_value1 > *date_v2_value2 ? 1 : (*date_v2_value1 < *date_v2_value2 ? -1 : 0); + } + case TYPE_DECIMALV2: { DecimalV2Value decimal_value1(reinterpret_cast(v1)->value); DecimalV2Value decimal_value2(reinterpret_cast(v2)->value); diff --git a/be/src/runtime/raw_value.h b/be/src/runtime/raw_value.h index 6804eecaad..424ede11e8 100644 --- a/be/src/runtime/raw_value.h +++ b/be/src/runtime/raw_value.h @@ -160,6 +160,10 @@ inline bool RawValue::lt(const void* v1, const void* v2, const TypeDescriptor& t return *reinterpret_cast(v1) < *reinterpret_cast(v2); + case TYPE_DATEV2: + return *reinterpret_cast(v1) < + *reinterpret_cast(v2); + case TYPE_DECIMALV2: return reinterpret_cast(v1)->value < reinterpret_cast(v2)->value; @@ -212,6 +216,10 @@ inline bool RawValue::eq(const void* v1, const void* v2, const TypeDescriptor& t return *reinterpret_cast(v1) == *reinterpret_cast(v2); + case TYPE_DATEV2: + return *reinterpret_cast(v1) == + *reinterpret_cast(v2); + case TYPE_DECIMALV2: return reinterpret_cast(v1)->value == reinterpret_cast(v2)->value; @@ -273,6 +281,9 @@ inline uint32_t RawValue::get_hash_value(const void* v, const PrimitiveType& typ case TYPE_DATETIME: return HashUtil::hash(v, 16, seed); + case TYPE_DATEV2: + return HashUtil::hash(v, 4, seed); + case TYPE_DECIMALV2: return HashUtil::hash(v, 16, seed); @@ -330,6 +341,9 @@ inline uint32_t RawValue::get_hash_value_fvn(const void* v, const PrimitiveType& case TYPE_DATETIME: return HashUtil::fnv_hash(v, 16, seed); + case TYPE_DATEV2: + return HashUtil::fnv_hash(v, 4, seed); + case TYPE_DECIMALV2: return HashUtil::fnv_hash(v, 16, seed); @@ -392,6 +406,12 @@ inline uint32_t RawValue::zlib_crc32(const void* v, const TypeDescriptor& type, int len = date_val->to_buffer(buf); return HashUtil::zlib_crc_hash(buf, len, seed); } + case TYPE_DATEV2: { + const vectorized::DateV2Value* date_v2_val = (const vectorized::DateV2Value*)v; + char buf[64]; + int len = date_v2_val->to_buffer(buf); + return HashUtil::zlib_crc_hash(buf, len, seed); + } case TYPE_DECIMALV2: { const DecimalV2Value* dec_val = (const DecimalV2Value*)v; @@ -446,6 +466,12 @@ inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const TypeDescri int len = date_val->to_buffer(buf); return HashUtil::zlib_crc_hash(buf, len, seed); } + case TYPE_DATEV2: { + auto* date_v2_val = (const vectorized::DateV2Value*)v; + char buf[64]; + int date_v2_len = date_v2_val->to_buffer(buf); + return HashUtil::zlib_crc_hash(buf, date_v2_len, seed); + } case TYPE_DECIMALV2: { const DecimalV2Value* dec_val = (const DecimalV2Value*)v; diff --git a/be/src/runtime/type_limit.h b/be/src/runtime/type_limit.h index 7cc567549d..b868e25447 100644 --- a/be/src/runtime/type_limit.h +++ b/be/src/runtime/type_limit.h @@ -47,4 +47,16 @@ struct type_limit { static DateTimeValue max() { return DateTimeValue::datetime_max_value(); } }; +template <> +struct type_limit { + static doris::vectorized::DateV2Value min() { + uint32_t min = doris::vectorized::MIN_DATE_V2; + return binary_cast(min); + } + static doris::vectorized::DateV2Value max() { + uint32_t max = doris::vectorized::MAX_DATE_V2; + return binary_cast(max); + } +}; + } // namespace doris diff --git a/be/src/runtime/types.h b/be/src/runtime/types.h index 5852c2fb91..20cdc4663b 100644 --- a/be/src/runtime/types.h +++ b/be/src/runtime/types.h @@ -167,6 +167,8 @@ struct TypeDescriptor { bool is_date_type() const { return type == TYPE_DATE || type == TYPE_DATETIME; } + bool is_date_v2_type() const { return type == TYPE_DATEV2 || type == TYPE_DATETIMEV2; } + bool is_decimal_type() const { return (type == TYPE_DECIMALV2); } bool is_datetime_type() const { return type == TYPE_DATETIME; } diff --git a/be/src/udf/udf.h b/be/src/udf/udf.h index f46b7c6129..2a5296f477 100644 --- a/be/src/udf/udf.h +++ b/be/src/udf/udf.h @@ -87,7 +87,10 @@ public: TYPE_DECIMALV2, TYPE_OBJECT, TYPE_ARRAY, - TYPE_QUANTILE_STATE + TYPE_QUANTILE_STATE, + TYPE_DATEV2, + TYPE_DATETIMEV2, + TYPE_TIMEV2 }; struct TypeDesc { diff --git a/be/src/util/binary_cast.hpp b/be/src/util/binary_cast.hpp index 077885482b..013c9e598e 100644 --- a/be/src/util/binary_cast.hpp +++ b/be/src/util/binary_cast.hpp @@ -62,6 +62,13 @@ union VecDateTimeInt64Union { __int64_t i64; ~VecDateTimeInt64Union() {} }; + +union DateV2UInt32Union { + doris::vectorized::DateV2Value dt; + uint32_t ui32; + ~DateV2UInt32Union() {} +}; + // similar to reinterpret_cast but won't break strict-aliasing rules template To binary_cast(From from) { @@ -79,10 +86,16 @@ To binary_cast(From from) { constexpr bool from_i128_to_decv2 = match_v; constexpr bool from_decv2_to_i128 = match_v; + constexpr bool from_ui32_to_date_v2 = + match_v; + + constexpr bool from_date_v2_to_ui32 = + match_v; + static_assert(from_u64_to_db || from_i64_to_db || from_db_to_i64 || from_db_to_u64 || from_decv2_to_packed128 || from_i128_to_dt || from_dt_to_i128 || from_i64_to_vec_dt || from_vec_dt_to_i64 || from_i128_to_decv2 || - from_decv2_to_i128); + from_decv2_to_i128 || from_ui32_to_date_v2 || from_date_v2_to_ui32); if constexpr (from_u64_to_db) { TypeConverter conv; @@ -113,6 +126,12 @@ To binary_cast(From from) { } else if constexpr (from_i64_to_vec_dt) { VecDateTimeInt64Union conv = {.i64 = from}; return conv.dt; + } else if constexpr (from_ui32_to_date_v2) { + DateV2UInt32Union conv = {.ui32 = from}; + return conv.dt; + } else if constexpr (from_date_v2_to_ui32) { + DateV2UInt32Union conv = {.dt = from}; + return conv.ui32; } else if constexpr (from_vec_dt_to_i64) { VecDateTimeInt64Union conv = {.dt = from}; return conv.i64; diff --git a/be/src/util/date_func.cpp b/be/src/util/date_func.cpp index 93559b684f..4f7add3ad9 100644 --- a/be/src/util/date_func.cpp +++ b/be/src/util/date_func.cpp @@ -19,6 +19,8 @@ #include +#include "vec/runtime/vdatetime_value.h" + namespace doris { uint64_t timestamp_from_datetime(const std::string& datetime_str) { @@ -53,6 +55,20 @@ uint24_t timestamp_from_date(const std::string& date_str) { return uint24_t(value); } + +uint32_t timestamp_from_date_v2(const std::string& date_str) { + tm time_tm; + char* res = strptime(date_str.c_str(), "%Y-%m-%d", &time_tm); + + uint32_t value = 0; + if (nullptr != res) { + value = ((time_tm.tm_year + 1900) << 16) | ((time_tm.tm_mon + 1) << 8) | time_tm.tm_mday; + } else { + value = doris::vectorized::MIN_DATE_V2; + } + + return value; +} // refer to https://dev.mysql.com/doc/refman/5.7/en/time.html // the time value between '-838:59:59' and '838:59:59' int32_t time_to_buffer_from_double(double time, char* buffer) { diff --git a/be/src/util/date_func.h b/be/src/util/date_func.h index f3cb926a62..507ee9652e 100644 --- a/be/src/util/date_func.h +++ b/be/src/util/date_func.h @@ -29,5 +29,6 @@ namespace doris { uint64_t timestamp_from_datetime(const std::string& datetime_str); uint24_t timestamp_from_date(const std::string& date_str); int32_t time_to_buffer_from_double(double time, char* buffer); +uint32_t timestamp_from_date_v2(const std::string& date_str); } // namespace doris diff --git a/be/src/vec/CMakeLists.txt b/be/src/vec/CMakeLists.txt index 69aaf816ba..d6549701b0 100644 --- a/be/src/vec/CMakeLists.txt +++ b/be/src/vec/CMakeLists.txt @@ -78,6 +78,7 @@ set(VEC_FILES data_types/nested_utils.cpp data_types/data_type_date.cpp data_types/data_type_date_time.cpp + data_types/data_type_time_v2.cpp exec/vaggregation_node.cpp exec/varrow_scanner.cpp exec/ves_http_scan_node.cpp diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp b/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp index 813674a312..be4ce7ef0d 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp @@ -50,6 +50,10 @@ static IAggregateFunction* create_aggregate_function_single_value(const String& return new AggregateFunctionTemplate>, false>( argument_type); } + if (which.idx == TypeIndex::DateV2) { + return new AggregateFunctionTemplate>, false>( + argument_type); + } if (which.idx == TypeIndex::Decimal128) { return new AggregateFunctionTemplate>, false>( argument_type); diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_max_by.cpp b/be/src/vec/aggregate_functions/aggregate_function_min_max_by.cpp index 0b88f5106e..8b834f4202 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_min_max_by.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_min_max_by.cpp @@ -47,6 +47,10 @@ static IAggregateFunction* create_aggregate_function_min_max_by_impl( return new AggregateFunctionTemplate>, false>( value_arg_type, key_arg_type); } + if (which.idx == TypeIndex::DateV2) { + return new AggregateFunctionTemplate>, false>( + value_arg_type, key_arg_type); + } if (which.idx == TypeIndex::Decimal128) { return new AggregateFunctionTemplate>, false>( value_arg_type, key_arg_type); @@ -82,6 +86,11 @@ static IAggregateFunction* create_aggregate_function_min_max_by(const String& na SingleValueDataFixed>( argument_types); } + if (which.idx == TypeIndex::DateV2) { + return create_aggregate_function_min_max_by_impl>( + argument_types); + } if (which.idx == TypeIndex::Decimal128) { return create_aggregate_function_min_max_by_impl>( diff --git a/be/src/vec/aggregate_functions/aggregate_function_window.h b/be/src/vec/aggregate_functions/aggregate_function_window.h index 37b73bd7c1..ab519156ab 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_window.h +++ b/be/src/vec/aggregate_functions/aggregate_function_window.h @@ -579,6 +579,10 @@ static IAggregateFunction* create_function_single_value(const String& name, return new AggregateFunctionTemplate< Data>>(argument_types); } + if (which.is_date_v2()) { + return new AggregateFunctionTemplate< + Data>>(argument_types); + } if (which.is_string_or_fixed_string()) { return new AggregateFunctionTemplate< Data>>( diff --git a/be/src/vec/aggregate_functions/aggregate_function_window_funnel.cpp b/be/src/vec/aggregate_functions/aggregate_function_window_funnel.cpp index a889299f33..dda947bb10 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_window_funnel.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_window_funnel.cpp @@ -28,7 +28,7 @@ AggregateFunctionPtr create_aggregate_function_window_funnel(const std::string& const DataTypes& argument_types, const Array& parameters, const bool result_is_nullable) { - return std::make_shared(argument_types); + return std::make_shared>(argument_types); } void register_aggregate_function_window_funnel(AggregateFunctionSimpleFactory& factory) { diff --git a/be/src/vec/aggregate_functions/aggregate_function_window_funnel.h b/be/src/vec/aggregate_functions/aggregate_function_window_funnel.h index 7b62ea2ad6..92eb8d99f0 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_window_funnel.h +++ b/be/src/vec/aggregate_functions/aggregate_function_window_funnel.h @@ -29,8 +29,9 @@ namespace doris::vectorized { +template struct WindowFunnelState { - std::vector> events; + std::vector> events; int max_event_level; bool sorted; int64_t window; @@ -48,7 +49,7 @@ struct WindowFunnelState { events.shrink_to_fit(); } - void add(const VecDateTimeValue& timestamp, int event_idx, int event_num, int64_t win) { + void add(const DateValueType& timestamp, int event_idx, int event_num, int64_t win) { window = win; max_event_level = event_num; if (sorted && events.size() > 0) { @@ -69,17 +70,17 @@ struct WindowFunnelState { } int get() const { - std::vector> events_timestamp(max_event_level); + std::vector> events_timestamp(max_event_level); for (int64_t i = 0; i < events.size(); i++) { const int& event_idx = events[i].second; - const VecDateTimeValue& timestamp = events[i].first; + const DateValueType& timestamp = events[i].first; if (event_idx == 0) { events_timestamp[0] = timestamp; continue; } if (events_timestamp[event_idx - 1].has_value()) { - const VecDateTimeValue& first_timestamp = events_timestamp[event_idx - 1].value(); - VecDateTimeValue last_timestamp = first_timestamp; + const DateValueType& first_timestamp = events_timestamp[event_idx - 1].value(); + DateValueType last_timestamp = first_timestamp; TimeInterval interval(SECOND, window, false); last_timestamp.date_add_interval(interval, SECOND); @@ -132,8 +133,7 @@ struct WindowFunnelState { write_var_int(events.size(), out); for (int64_t i = 0; i < events.size(); i++) { - int64_t timestamp = - binary_cast(events[i].first); + int64_t timestamp = binary_cast(events[i].first); int event_idx = events[i].second; write_var_int(timestamp, out); write_var_int(event_idx, out); @@ -153,19 +153,23 @@ struct WindowFunnelState { read_var_int(timestamp, in); read_var_int(event_idx, in); - VecDateTimeValue time_value = - binary_cast(timestamp); + DateValueType time_value = binary_cast(timestamp); add(time_value, (int)event_idx, max_event_level, window); } } }; +template class AggregateFunctionWindowFunnel - : public IAggregateFunctionDataHelper { + : public IAggregateFunctionDataHelper< + WindowFunnelState, + AggregateFunctionWindowFunnel> { public: AggregateFunctionWindowFunnel(const DataTypes& argument_types_) - : IAggregateFunctionDataHelper( - argument_types_, {}) {} + : IAggregateFunctionDataHelper< + WindowFunnelState, + AggregateFunctionWindowFunnel>(argument_types_, + {}) {} String get_name() const override { return "window_funnel"; } @@ -180,14 +184,15 @@ public: // TODO: handle mode in the future. // be/src/olap/row_block2.cpp copy_data_to_column const auto& timestamp = - static_cast&>(*columns[2]).get_data()[row_num]; + static_cast&>(*columns[2]).get_data()[row_num]; const int NON_EVENT_NUM = 3; - for (int i = NON_EVENT_NUM; i < get_argument_types().size(); i++) { + for (int i = NON_EVENT_NUM; i < IAggregateFunction::get_argument_types().size(); i++) { const auto& is_set = static_cast&>(*columns[i]).get_data()[row_num]; if (is_set) { - this->data(place).add(timestamp, i - NON_EVENT_NUM, - get_argument_types().size() - NON_EVENT_NUM, window); + this->data(place).add( + timestamp, i - NON_EVENT_NUM, + IAggregateFunction::get_argument_types().size() - NON_EVENT_NUM, window); } } } @@ -208,7 +213,11 @@ public: void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override { this->data(const_cast(place)).sort(); - assert_cast(to).get_data().push_back(data(place).get()); + assert_cast(to).get_data().push_back( + IAggregateFunctionDataHelper< + WindowFunnelState, + AggregateFunctionWindowFunnel>::data(place) + .get()); } }; diff --git a/be/src/vec/columns/column.h b/be/src/vec/columns/column.h index af40c96e07..35035d419f 100644 --- a/be/src/vec/columns/column.h +++ b/be/src/vec/columns/column.h @@ -475,14 +475,17 @@ public: virtual void replace_column_data_default(size_t self_row = 0) = 0; virtual bool is_date_type() const { return is_date; } + virtual bool is_date_v2_type() const { return is_date_v2; } virtual bool is_datetime_type() const { return is_date_time; } virtual void set_date_type() { is_date = true; } + virtual void set_date_v2_type() { is_date_v2 = true; } virtual void set_datetime_type() { is_date_time = true; } // todo(wb): a temporary implemention, need re-abstract here bool is_date = false; bool is_date_time = false; + bool is_date_v2 = false; protected: /// Template is to devirtualize calls to insert_from method. diff --git a/be/src/vec/columns/column_nullable.h b/be/src/vec/columns/column_nullable.h index e6f4495c30..31e1f2b80e 100644 --- a/be/src/vec/columns/column_nullable.h +++ b/be/src/vec/columns/column_nullable.h @@ -168,8 +168,10 @@ public: } bool is_date_type() const override { return get_nested_column().is_date_type(); } + bool is_date_v2_type() const override { return get_nested_column().is_date_v2_type(); } bool is_datetime_type() const override { return get_nested_column().is_datetime_type(); } void set_date_type() override { get_nested_column().set_date_type(); } + void set_date_v2_type() override { get_nested_column().set_date_v2_type(); } void set_datetime_type() override { get_nested_column().set_datetime_type(); } bool is_nullable() const override { return true; } diff --git a/be/src/vec/columns/columns_number.h b/be/src/vec/columns/columns_number.h index 500bab40aa..af878136d3 100644 --- a/be/src/vec/columns/columns_number.h +++ b/be/src/vec/columns/columns_number.h @@ -40,6 +40,7 @@ using ColumnInt64 = ColumnVector; using ColumnInt128 = ColumnVector; using ColumnDate = ColumnVector; +using ColumnDateV2 = ColumnVector; using ColumnDateTime = ColumnVector; using ColumnFloat32 = ColumnVector; diff --git a/be/src/vec/columns/predicate_column.h b/be/src/vec/columns/predicate_column.h index 27b1b7e5e0..0565ad0ff9 100644 --- a/be/src/vec/columns/predicate_column.h +++ b/be/src/vec/columns/predicate_column.h @@ -438,9 +438,19 @@ public: } else if constexpr (std::is_same_v) { insert_date_to_res_column(sel, sel_size, reinterpret_cast*>(col_ptr)); - } else if constexpr (std::is_same_v) { // a trick type judge, need refactor it. - insert_date32_to_res_column( - sel, sel_size, reinterpret_cast*>(col_ptr)); + } else if constexpr (std::is_same_v) { + if (const vectorized::ColumnVector* date_col = + check_and_get_column>( + const_cast(col_ptr))) { + // a trick type judge, need refactor it. + insert_date32_to_res_column(sel, sel_size, + const_cast*>(date_col)); + } else { + insert_default_value_res_column( + sel, sel_size, + reinterpret_cast*>( + col_ptr)); + } } else if constexpr (std::is_same_v) { insert_default_value_res_column( sel, sel_size, diff --git a/be/src/vec/core/call_on_type_index.h b/be/src/vec/core/call_on_type_index.h index 9b552ac381..d62288611d 100644 --- a/be/src/vec/core/call_on_type_index.h +++ b/be/src/vec/core/call_on_type_index.h @@ -159,6 +159,7 @@ inline bool call_on_basic_types(TypeIndex type_num1, TypeIndex type_num2, F&& f) } class DataTypeDate; +class DataTypeDateV2; class DataTypeDateTime; class DataTypeString; template @@ -205,6 +206,8 @@ bool call_on_index_and_data_type(TypeIndex number, F&& f) { case TypeIndex::Date: return f(TypePair()); + case TypeIndex::DateV2: + return f(TypePair()); case TypeIndex::DateTime: return f(TypePair()); diff --git a/be/src/vec/core/types.h b/be/src/vec/core/types.h index 650b64a627..a9c61bb31e 100644 --- a/be/src/vec/core/types.h +++ b/be/src/vec/core/types.h @@ -74,6 +74,9 @@ enum class TypeIndex { LowCardinality, BitMap, HLL, + DateV2, + DateTimeV2, + TimeV2, }; struct Consted { @@ -255,6 +258,7 @@ struct TypeId { using Date = Int64; using DateTime = Int64; +using DateV2 = UInt32; /// Own FieldType for Decimal. /// It is only a "storage" for decimal. To perform operations, you also have to provide a scale (number of digits after point). @@ -401,6 +405,12 @@ inline const char* getTypeName(TypeIndex idx) { return "Date"; case TypeIndex::DateTime: return "DateTime"; + case TypeIndex::DateV2: + return "DateV2"; + case TypeIndex::DateTimeV2: + return "DateTimeV2"; + case TypeIndex::TimeV2: + return "TimeV2"; case TypeIndex::String: return TypeName::get(); case TypeIndex::FixedString: diff --git a/be/src/vec/data_types/data_type.cpp b/be/src/vec/data_types/data_type.cpp index 414959a517..4be9a6112a 100644 --- a/be/src/vec/data_types/data_type.cpp +++ b/be/src/vec/data_types/data_type.cpp @@ -131,6 +131,8 @@ PGenericType_TypeId IDataType::get_pdata_type(const IDataType* data_type) { return PGenericType::STRING; case TypeIndex::Date: return PGenericType::DATE; + case TypeIndex::DateV2: + return PGenericType::DATEV2; case TypeIndex::DateTime: return PGenericType::DATETIME; case TypeIndex::BitMap: diff --git a/be/src/vec/data_types/data_type.h b/be/src/vec/data_types/data_type.h index 6a4dc16899..047c9fa0b6 100644 --- a/be/src/vec/data_types/data_type.h +++ b/be/src/vec/data_types/data_type.h @@ -295,7 +295,10 @@ struct WhichDataType { bool is_date() const { return idx == TypeIndex::Date; } bool is_date_time() const { return idx == TypeIndex::DateTime; } + bool is_date_v2() const { return idx == TypeIndex::DateV2; } + bool is_date_time_v2() const { return idx == TypeIndex::DateTimeV2; } bool is_date_or_datetime() const { return is_date() || is_date_time(); } + bool is_date_v2_or_datetime_v2() const { return is_date_v2() || is_date_time_v2(); } bool is_string() const { return idx == TypeIndex::String; } bool is_fixed_string() const { return idx == TypeIndex::FixedString; } @@ -318,9 +321,15 @@ struct WhichDataType { inline bool is_date(const DataTypePtr& data_type) { return WhichDataType(data_type).is_date(); } +inline bool is_date_v2(const DataTypePtr& data_type) { + return WhichDataType(data_type).is_date_v2(); +} inline bool is_date_or_datetime(const DataTypePtr& data_type) { return WhichDataType(data_type).is_date_or_datetime(); } +inline bool is_date_v2_or_datetime_v2(const DataTypePtr& data_type) { + return WhichDataType(data_type).is_date_v2_or_datetime_v2(); +} inline bool is_enum(const DataTypePtr& data_type) { return WhichDataType(data_type).is_enum(); } @@ -376,7 +385,7 @@ template inline bool is_columned_as_number(const T& data_type) { WhichDataType which(data_type); return which.is_int() || which.is_uint() || which.is_float() || which.is_date_or_datetime() || - which.is_uuid(); + which.is_uuid() || which.is_date_v2(); } template diff --git a/be/src/vec/data_types/data_type_date_time.h b/be/src/vec/data_types/data_type_date_time.h index 8085407253..ebee94ac70 100644 --- a/be/src/vec/data_types/data_type_date_time.h +++ b/be/src/vec/data_types/data_type_date_time.h @@ -22,6 +22,7 @@ #include "vec/data_types/data_type_date.h" #include "vec/data_types/data_type_number_base.h" +#include "vec/data_types/data_type_time_v2.h" class DateLUTImpl; @@ -81,6 +82,14 @@ constexpr bool IsDateType = false; template <> inline constexpr bool IsDateType = true; +template +constexpr bool IsDateV2Type = false; +template <> +inline constexpr bool IsDateV2Type = true; + +template +constexpr bool IsDateTimeV2Type = false; + template constexpr bool IsTimeType = IsDateTimeType || IsDateType; diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp index 43897e1f44..8ab9d6a933 100644 --- a/be/src/vec/data_types/data_type_factory.cpp +++ b/be/src/vec/data_types/data_type_factory.cpp @@ -81,6 +81,10 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo case TYPE_DATE: nested = std::make_shared(); break; + case TYPE_DATEV2: + nested = std::make_shared(); + break; + case TYPE_DATETIMEV2: case TYPE_DATETIME: nested = std::make_shared(); break; @@ -150,6 +154,9 @@ DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type) case OLAP_FIELD_TYPE_DATE: result = std::make_shared(); break; + case OLAP_FIELD_TYPE_DATEV2: + result = std::make_shared(); + break; case OLAP_FIELD_TYPE_DATETIME: result = std::make_shared(); break; @@ -223,6 +230,9 @@ DataTypePtr DataTypeFactory::create_data_type(const PColumnMeta& pcolumn) { case PGenericType::DATE: nested = std::make_shared(); break; + case PGenericType::DATEV2: + nested = std::make_shared(); + break; case PGenericType::DATETIME: nested = std::make_shared(); break; diff --git a/be/src/vec/data_types/data_type_factory.hpp b/be/src/vec/data_types/data_type_factory.hpp index e64bdcf3ac..08dc6a9f31 100644 --- a/be/src/vec/data_types/data_type_factory.hpp +++ b/be/src/vec/data_types/data_type_factory.hpp @@ -62,6 +62,7 @@ public: {"Float32", std::make_shared()}, {"Float64", std::make_shared()}, {"Date", std::make_shared()}, + {"DateV2", std::make_shared()}, {"DateTime", std::make_shared()}, {"String", std::make_shared()}, {"Decimal", std::make_shared>(27, 9)}, diff --git a/be/src/vec/data_types/data_type_time_v2.cpp b/be/src/vec/data_types/data_type_time_v2.cpp new file mode 100644 index 0000000000..ce8a7cd3eb --- /dev/null +++ b/be/src/vec/data_types/data_type_time_v2.cpp @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/data_types/data_type_time_v2.h" + +#include "util/binary_cast.hpp" +#include "vec/columns/columns_number.h" +#include "vec/runtime/vdatetime_value.h" + +namespace doris::vectorized { +bool DataTypeDateV2::equals(const IDataType& rhs) const { + return typeid(rhs) == typeid(*this); +} + +std::string DataTypeDateV2::to_string(const IColumn& column, size_t row_num) const { + UInt32 int_val = + assert_cast(*column.convert_to_full_column_if_const().get()) + .get_data()[row_num]; + std::stringstream ss; + // Year + ss << (int_val & 0xFFFF0000) << '-'; + // Month + ss << (int_val & 0x0000FF00) << '-'; + // Day + ss << int_val % 0xFF; + return ss.str(); +} + +void DataTypeDateV2::to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const { + UInt32 int_val = + assert_cast(*column.convert_to_full_column_if_const().get()) + .get_data()[row_num]; + DateV2Value value = binary_cast(int_val); + + char buf[64]; + char* pos = value.to_string(buf); + // DateTime to_string the end is /0 + ostr.write(buf, pos - buf - 1); +} + +MutableColumnPtr DataTypeDateV2::create_column() const { + auto col = DataTypeNumberBase::create_column(); + col->set_date_v2_type(); + return col; +} + +void DataTypeDateV2::cast_to_date_time(const UInt32& from, Int64& to) { + auto& to_value = (doris::vectorized::VecDateTimeValue&)to; + auto& from_value = (doris::vectorized::DateV2Value&)from; + to_value.create_from_date_v2(from_value, TimeType::TIME_DATETIME); +} + +void DataTypeDateV2::cast_to_date(const UInt32& from, Int64& to) { + auto& to_value = (doris::vectorized::VecDateTimeValue&)(to); + auto& from_value = (doris::vectorized::DateV2Value&)from; + to_value.create_from_date_v2(from_value, TimeType::TIME_DATE); +} + +} // namespace doris::vectorized diff --git a/be/src/vec/data_types/data_type_time_v2.h b/be/src/vec/data_types/data_type_time_v2.h new file mode 100644 index 0000000000..c126b0966b --- /dev/null +++ b/be/src/vec/data_types/data_type_time_v2.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "vec/data_types/data_type_number_base.h" + +namespace doris::vectorized { + +/** + * Use UInt32 as underlying type to represent DateV2 type. + * Specifically, a dateV2 type is represented as (YYYY (16 bits), MM (8 bits), DD (8 bits)). + */ +class DataTypeDateV2 final : public DataTypeNumberBase { +public: + TypeIndex get_type_id() const override { return TypeIndex::DateV2; } + const char* get_family_name() const override { return "DateV2"; } + std::string do_get_name() const override { return "DateV2"; } + + bool can_be_used_as_version() const override { return true; } + bool can_be_inside_nullable() const override { return true; } + + bool equals(const IDataType& rhs) const override; + std::string to_string(const IColumn& column, size_t row_num) const override; + void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override; + + MutableColumnPtr create_column() const override; + + static void cast_to_date(const UInt32& from, Int64& to); + static void cast_to_date_time(const UInt32& from, Int64& to); +}; + +} // namespace doris::vectorized diff --git a/be/src/vec/data_types/get_least_supertype.cpp b/be/src/vec/data_types/get_least_supertype.cpp index a661035af9..1aa9da5c55 100644 --- a/be/src/vec/data_types/get_least_supertype.cpp +++ b/be/src/vec/data_types/get_least_supertype.cpp @@ -146,6 +146,19 @@ DataTypePtr get_least_supertype(const DataTypes& types) { } } + { + UInt32 have_date_v2 = type_ids.count(TypeIndex::DateV2); + + if (have_date_v2) { + if (type_ids.size() != have_date_v2) { + LOG(FATAL) << get_exception_message_prefix(types) + << " because some of them are DateV2 and some of them are not"; + } + + return std::make_shared(); + } + } + /// Decimals { UInt32 have_decimal32 = type_ids.count(TypeIndex::Decimal32); diff --git a/be/src/vec/exec/join/vhash_join_node.cpp b/be/src/vec/exec/join/vhash_join_node.cpp index 4471034ea4..fa1400a626 100644 --- a/be/src/vec/exec/join/vhash_join_node.cpp +++ b/be/src/vec/exec/join/vhash_join_node.cpp @@ -1204,6 +1204,7 @@ void HashJoinNode::_hash_table_init() { break; case TYPE_INT: case TYPE_FLOAT: + case TYPE_DATEV2: _hash_table_variants.emplace(); break; case TYPE_BIGINT: diff --git a/be/src/vec/exec/vaggregation_node.cpp b/be/src/vec/exec/vaggregation_node.cpp index 8910674467..bc699b855c 100644 --- a/be/src/vec/exec/vaggregation_node.cpp +++ b/be/src/vec/exec/vaggregation_node.cpp @@ -135,6 +135,7 @@ void AggregationNode::_init_hash_method(std::vector& probe_exprs) return; case TYPE_INT: case TYPE_FLOAT: + case TYPE_DATEV2: _agg_data.init(AggregatedDataVariants::Type::int32_key, is_nullable); return; case TYPE_BIGINT: diff --git a/be/src/vec/exec/varrow_scanner.cpp b/be/src/vec/exec/varrow_scanner.cpp index 14ae373988..237c65fb62 100644 --- a/be/src/vec/exec/varrow_scanner.cpp +++ b/be/src/vec/exec/varrow_scanner.cpp @@ -254,9 +254,9 @@ Status VArrowScanner::_append_batch_to_src_block(Block* block) { } auto* array = _batch->column(column_pos++).get(); auto& column_with_type_and_name = block->get_by_name(slot_desc->col_name()); - RETURN_IF_ERROR(arrow_column_to_doris_column(array, _arrow_batch_cur_idx, - column_with_type_and_name.column, num_elements, - _state->timezone())); + RETURN_IF_ERROR(arrow_column_to_doris_column( + array, _arrow_batch_cur_idx, column_with_type_and_name.column, + column_with_type_and_name.type, num_elements, _state->timezone())); } _arrow_batch_cur_idx += num_elements; diff --git a/be/src/vec/exec/volap_scan_node.cpp b/be/src/vec/exec/volap_scan_node.cpp index aef88e4e0f..cd7e39492b 100644 --- a/be/src/vec/exec/volap_scan_node.cpp +++ b/be/src/vec/exec/volap_scan_node.cpp @@ -676,6 +676,13 @@ Status VOlapScanNode::normalize_conjuncts() { break; } + case TYPE_DATEV2: { + ColumnValueRange range(slots[slot_idx]->col_name(), + slots[slot_idx]->type().type); + normalize_predicate(range, slots[slot_idx]); + break; + } + case TYPE_DECIMALV2: { ColumnValueRange range(slots[slot_idx]->col_name(), slots[slot_idx]->type().type); @@ -829,7 +836,8 @@ Status VOlapScanNode::normalize_predicate(ColumnValueRange& range, SlotDescri } static bool ignore_cast(SlotDescriptor* slot, Expr* expr) { - if (slot->type().is_date_type() && expr->type().is_date_type()) { + if ((slot->type().is_date_type() || slot->type().is_date_v2_type()) && + (expr->type().is_date_type() || expr->type().is_date_v2_type())) { return true; } if (slot->type().is_string_type() && expr->type().is_string_type()) { @@ -952,6 +960,19 @@ Status VOlapScanNode::change_fixed_value_range(ColumnValueRange& temp_range, func(temp_range, reinterpret_cast(&v)); break; } + case TYPE_DATEV2: { + DateTimeValue date_value = *reinterpret_cast(value); + if (!date_value.check_loss_accuracy_cast_to_date()) { + doris::vectorized::DateV2Value date_v2; + date_v2.convert_dt_to_date_v2(&date_value); + if constexpr (std::is_same_v) { + func(temp_range, &date_v2); + } else { + __builtin_unreachable(); + } + } + break; + } default: { LOG(WARNING) << "Normalize filter fail, Unsupported Primitive type. [type=" << type << "]"; return Status::InternalError("Normalize filter fail, Unsupported Primitive type"); @@ -1296,6 +1317,22 @@ Status VOlapScanNode::normalize_noneq_binary_predicate(SlotDescriptor* slot, *reinterpret_cast(&date_value)); break; } + case TYPE_DATEV2: { + DateTimeValue date_value = *reinterpret_cast(value); + if (date_value.check_loss_accuracy_cast_to_date()) { + if (pred->op() == TExprOpcode::LT || pred->op() == TExprOpcode::GE) { + ++date_value; + } + } + doris::vectorized::DateV2Value date_v2; + date_v2.convert_dt_to_date_v2(&date_value); + if constexpr (std::is_same_v) { + range->add_range(to_olap_filter_type(pred->op(), child_idx), date_v2); + break; + } else { + __builtin_unreachable(); + } + } case TYPE_TINYINT: case TYPE_DECIMALV2: case TYPE_CHAR: diff --git a/be/src/vec/exec/volap_scan_node.h b/be/src/vec/exec/volap_scan_node.h index 23ce669cf1..2222390652 100644 --- a/be/src/vec/exec/volap_scan_node.h +++ b/be/src/vec/exec/volap_scan_node.h @@ -17,10 +17,13 @@ #pragma once -#include "exec/olap_scan_node.h" +#include "exec/olap_common.h" +#include "exec/scan_node.h" +#include "exprs/bloomfilter_predicate.h" #include "exprs/in_predicate.h" #include "exprs/runtime_filter.h" #include "gen_cpp/PlanNodes_types.h" +#include "olap/tablet.h" #include "util/progress_updater.h" namespace doris { diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp index bacf6b7d56..07e8d638f5 100644 --- a/be/src/vec/exec/vschema_scan_node.cpp +++ b/be/src/vec/exec/vschema_scan_node.cpp @@ -399,6 +399,12 @@ Status VSchemaScanNode::write_slot_to_vectorized_column(void* slot, SlotDescript break; } + case TYPE_DATEV2: { + uint32_t num = *reinterpret_cast(slot); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + case TYPE_DATETIME: { VecDateTimeValue value; DateTimeValue* ts_slot = reinterpret_cast(slot); diff --git a/be/src/vec/exec/vset_operation_node.cpp b/be/src/vec/exec/vset_operation_node.cpp index 8d27120ed1..e3a2659b73 100644 --- a/be/src/vec/exec/vset_operation_node.cpp +++ b/be/src/vec/exec/vset_operation_node.cpp @@ -160,6 +160,7 @@ void VSetOperationNode::hash_table_init() { break; case TYPE_INT: case TYPE_FLOAT: + case TYPE_DATEV2: _hash_table_variants.emplace(); break; case TYPE_BIGINT: diff --git a/be/src/vec/exprs/vliteral.cpp b/be/src/vec/exprs/vliteral.cpp index 73f7f231fd..673c8237b1 100644 --- a/be/src/vec/exprs/vliteral.cpp +++ b/be/src/vec/exprs/vliteral.cpp @@ -94,6 +94,12 @@ void VLiteral::init(const TExprNode& node) { field = Int64(*reinterpret_cast<__int64_t*>(&value)); break; } + case TYPE_DATEV2: { + DateV2Value value; + value.from_date_str(node.date_literal.value.c_str(), node.date_literal.value.size()); + field = value.to_date_uint32(); + break; + } case TYPE_DATETIME: { VecDateTimeValue value; value.from_date_str(node.date_literal.value.c_str(), node.date_literal.value.size()); diff --git a/be/src/vec/functions/array/function_array_element.h b/be/src/vec/functions/array/function_array_element.h index 315ce5cb2a..4f6eb12a07 100644 --- a/be/src/vec/functions/array/function_array_element.h +++ b/be/src/vec/functions/array/function_array_element.h @@ -200,6 +200,10 @@ private: if (nested_column->is_date_type()) { res = _execute_number(offsets, *nested_column, src_null_map, *arguments[1].column, nested_null_map, dst_null_map); + } else if (nested_column->is_date_v2_type()) { + res = _execute_number(offsets, *nested_column, src_null_map, + *arguments[1].column, nested_null_map, + dst_null_map); } else if (nested_column->is_datetime_type()) { res = _execute_number(offsets, *nested_column, src_null_map, *arguments[1].column, nested_null_map, diff --git a/be/src/vec/functions/array/function_array_index.h b/be/src/vec/functions/array/function_array_index.h index 046846424f..241bbda6b2 100644 --- a/be/src/vec/functions/array/function_array_index.h +++ b/be/src/vec/functions/array/function_array_index.h @@ -174,6 +174,9 @@ private: } else if (right_column.is_date_type()) { return _execute_number(offsets, nested_null_map, nested_column, right_column); + } else if (right_column.is_date_v2_type()) { + return _execute_number(offsets, nested_null_map, + nested_column, right_column); } else if (right_column.is_datetime_type()) { return _execute_number(offsets, nested_null_map, nested_column, right_column); @@ -245,10 +248,15 @@ private: return_column = _execute_number_expanded( offsets, nested_null_map, *nested_column, *right_column); } - } else if (is_date_or_datetime(right_type) && is_date_or_datetime(left_element_type)) { + } else if ((is_date_or_datetime(right_type) || is_date_v2_or_datetime_v2(right_type)) && + (is_date_or_datetime(left_element_type) || + is_date_v2_or_datetime_v2(left_element_type))) { if (nested_column->is_date_type()) { return_column = _execute_number_expanded(offsets, nested_null_map, *nested_column, *right_column); + } else if (nested_column->is_date_v2_type()) { + return_column = _execute_number_expanded( + offsets, nested_null_map, *nested_column, *right_column); } else if (nested_column->is_datetime_type()) { return_column = _execute_number_expanded( offsets, nested_null_map, *nested_column, *right_column); diff --git a/be/src/vec/functions/date_time_transforms.h b/be/src/vec/functions/date_time_transforms.h index 7381a96885..d42cd2d336 100644 --- a/be/src/vec/functions/date_time_transforms.h +++ b/be/src/vec/functions/date_time_transforms.h @@ -33,14 +33,25 @@ namespace doris::vectorized { -#define TIME_FUNCTION_IMPL(CLASS, UNIT, FUNCTION) \ - struct CLASS { \ - static constexpr auto name = #UNIT; \ - static inline auto execute(const Int64& t, bool& is_null) { \ - const auto& date_time_value = (doris::vectorized::VecDateTimeValue&)(t); \ - is_null = !date_time_value.is_valid_date(); \ - return date_time_value.FUNCTION; \ - } \ +#define TIME_FUNCTION_IMPL(CLASS, UNIT, FUNCTION) \ + template \ + struct CLASS { \ + using ARG_TYPE = ArgType; \ + static constexpr auto name = #UNIT; \ + \ + static inline auto execute(const ARG_TYPE& t, bool& is_null) { \ + const auto& date_time_value = (DateValueType&)(t); \ + is_null = !date_time_value.is_valid_date(); \ + return date_time_value.FUNCTION; \ + } \ + \ + static DataTypes get_variadic_argument_types() { \ + if constexpr (std::is_same_v) { \ + return {std::make_shared()}; \ + } else { \ + return {std::make_shared()}; \ + } \ + } \ } #define TO_TIME_FUNCTION(CLASS, UNIT) TIME_FUNCTION_IMPL(CLASS, UNIT, UNIT()) @@ -61,49 +72,76 @@ TIME_FUNCTION_IMPL(WeekDayImpl, weekday, weekday()); // TODO: the method should be always not nullable TIME_FUNCTION_IMPL(ToDaysImpl, to_days, daynr()); -#define TIME_FUNCTION_ONE_ARG_IMPL(CLASS, UNIT, FUNCTION) \ - struct CLASS { \ - static constexpr auto name = #UNIT; \ - static inline auto execute(const Int64& t, bool& is_null) { \ - const auto& date_time_value = (doris::vectorized::VecDateTimeValue&)(t); \ - is_null = !date_time_value.is_valid_date(); \ - return date_time_value.FUNCTION; \ - } \ - \ - static DataTypes get_variadic_argument_types() { \ - return {std::make_shared()}; \ - } \ +#define TIME_FUNCTION_ONE_ARG_IMPL(CLASS, UNIT, FUNCTION) \ + template \ + struct CLASS { \ + using ARG_TYPE = ArgType; \ + static constexpr auto name = #UNIT; \ + \ + static inline auto execute(const ARG_TYPE& t, bool& is_null) { \ + const auto& date_time_value = (DateValueType&)(t); \ + is_null = !date_time_value.is_valid_date(); \ + return date_time_value.FUNCTION; \ + } \ + \ + static DataTypes get_variadic_argument_types() { \ + if constexpr (std::is_same_v) { \ + return {std::make_shared()}; \ + } else { \ + return {std::make_shared()}; \ + } \ + } \ } TIME_FUNCTION_ONE_ARG_IMPL(ToWeekOneArgImpl, week, week(mysql_week_mode(0))); TIME_FUNCTION_ONE_ARG_IMPL(ToYearWeekOneArgImpl, yearweek, year_week(mysql_week_mode(0))); +template struct ToDateImpl { + using ARG_TYPE = ArgType; static constexpr auto name = "to_date"; - static inline auto execute(const Int64& t, bool& is_null) { - auto dt = binary_cast(t); + static inline auto execute(const ArgType& t, bool& is_null) { + auto dt = binary_cast(t); is_null = !dt.is_valid_date(); - dt.cast_to_date(); - return binary_cast(dt); + if constexpr (!std::is_same_v) { + dt.cast_to_date(); + } + return binary_cast(dt); + } + + static DataTypes get_variadic_argument_types() { + if constexpr (std::is_same_v) { + return {std::make_shared()}; + } else { + return {std::make_shared()}; + } } }; -struct DateImpl : public ToDateImpl { + +template +struct DateImpl : public ToDateImpl { static constexpr auto name = "date"; }; // TODO: This function look like no need do indeed copy here, we should optimize // this function +template struct TimeStampImpl { + using ARG_TYPE = ArgType; static constexpr auto name = "timestamp"; - static inline auto execute(const Int64& t, bool& is_null) { return t; } + + static inline auto execute(const ARG_TYPE& t, bool& is_null) { return t; } }; +template struct DayNameImpl { + using ARG_TYPE = ArgType; + using DATE_TYPE = DateValueType; static constexpr auto name = "dayname"; static constexpr auto max_size = MAX_DAY_NAME_LEN; - static inline auto execute(const VecDateTimeValue& dt, ColumnString::Chars& res_data, + static inline auto execute(const DateValueType& dt, ColumnString::Chars& res_data, size_t& offset, bool& is_null) { const auto* day_name = dt.day_name(); is_null = !dt.is_valid_date(); @@ -118,14 +156,25 @@ struct DayNameImpl { } return offset; } + + static DataTypes get_variadic_argument_types() { + if constexpr (std::is_same_v) { + return {std::make_shared()}; + } else { + return {std::make_shared()}; + } + } }; +template struct MonthNameImpl { + using ARG_TYPE = ArgType; + using DATE_TYPE = DateValueType; static constexpr auto name = "monthname"; static constexpr auto max_size = MAX_MONTH_NAME_LEN; - static inline auto execute(const VecDateTimeValue& dt, ColumnString::Chars& res_data, - size_t& offset, bool& is_null) { + static inline auto execute(const DATE_TYPE& dt, ColumnString::Chars& res_data, size_t& offset, + bool& is_null) { const auto* month_name = dt.month_name(); is_null = !dt.is_valid_date(); if (month_name == nullptr || is_null) { @@ -139,16 +188,25 @@ struct MonthNameImpl { } return offset; } + + static DataTypes get_variadic_argument_types() { + if constexpr (std::is_same_v) { + return {std::make_shared()}; + } else { + return {std::make_shared()}; + } + } }; +template struct DateFormatImpl { - using FromType = Int64; + using FromType = ArgType; static constexpr auto name = "date_format"; - static inline auto execute(const Int64& t, StringRef format, ColumnString::Chars& res_data, + static inline auto execute(const FromType& t, StringRef format, ColumnString::Chars& res_data, size_t& offset) { - const auto& dt = (VecDateTimeValue&)t; + const auto& dt = (DateType&)t; if (format.size > 128) { offset += 1; res_data.emplace_back(0); @@ -166,9 +224,20 @@ struct DateFormatImpl { offset += len; return std::pair {offset, false}; } + + static DataTypes get_variadic_argument_types() { + if constexpr (std::is_same_v) { + return std::vector {std::make_shared(), + std::make_shared()}; + } else { + return std::vector {std::make_shared(), + std::make_shared()}; + } + } }; // TODO: This function should be depend on argments not always nullable +template struct FromUnixTimeImpl { using FromType = Int32; @@ -179,7 +248,7 @@ struct FromUnixTimeImpl { // TODO: use default time zone, slowly and incorrect, just for test use static cctz::time_zone time_zone = cctz::fixed_time_zone(cctz::seconds(8 * 60 * 60)); - VecDateTimeValue dt; + DateType dt; if (format.size > 128 || val < 0 || val > INT_MAX || !dt.from_unixtime(val, time_zone)) { offset += 1; res_data.emplace_back(0); @@ -217,6 +286,22 @@ struct TransformerToStringOneArgument { reinterpret_cast(null_map[i])); } } + + static void vector(const PaddedPODArray& ts, ColumnString::Chars& res_data, + ColumnString::Offsets& res_offsets, NullMap& null_map) { + const auto len = ts.size(); + res_data.resize(len * Transform::max_size); + res_offsets.resize(len); + null_map.resize_fill(len, false); + + size_t offset = 0; + for (int i = 0; i < len; ++i) { + const auto& t = ts[i]; + const auto& date_time_value = reinterpret_cast(t); + res_offsets[i] = Transform::execute(date_time_value, res_data, offset, + reinterpret_cast(null_map[i])); + } + } }; template diff --git a/be/src/vec/functions/function.h b/be/src/vec/functions/function.h index c4464e4956..2272430a8e 100644 --- a/be/src/vec/functions/function.h +++ b/be/src/vec/functions/function.h @@ -292,7 +292,14 @@ public: is_date_or_datetime(get_return_type(arguments)->is_nullable() ? ((DataTypeNullable*)get_return_type(arguments).get()) ->get_nested_type() - : get_return_type(arguments)))) + : get_return_type(arguments))) || + (is_date_v2(return_type->is_nullable() + ? ((DataTypeNullable*)return_type.get())->get_nested_type() + : return_type) && + is_date_v2(get_return_type(arguments)->is_nullable() + ? ((DataTypeNullable*)get_return_type(arguments).get()) + ->get_nested_type() + : get_return_type(arguments)))) << " with " << return_type->get_name() << " and " << func_return_type->get_name(); return build_impl(arguments, return_type); diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h index 5799e60d93..490d414418 100644 --- a/be/src/vec/functions/function_cast.h +++ b/be/src/vec/functions/function_cast.h @@ -83,7 +83,8 @@ struct ConvertImpl { ColumnDecimal, ColumnVector>; if constexpr (IsDataTypeDecimal || IsDataTypeDecimal) { - if constexpr (!(IsDataTypeDecimalOrNumber || IsTimeType) || + if constexpr (!(IsDataTypeDecimalOrNumber || IsTimeType || + IsDateV2Type) || !IsDataTypeDecimalOrNumber) return Status::RuntimeError( fmt::format("Illegal column {} of first argument of function {}", @@ -121,21 +122,41 @@ struct ConvertImpl { vec_to[i] = convert_to_decimal( reinterpret_cast(vec_from[i]).to_int64(), vec_to.get_scale()); + } else if constexpr (IsDateV2Type && + IsDataTypeDecimal) { + vec_to[i] = convert_to_decimal( + reinterpret_cast(vec_from[i]).to_date_uint32(), + vec_to.get_scale()); } } else if constexpr (IsTimeType) { if constexpr (IsTimeType) { vec_to[i] = static_cast(vec_from[i]); - if constexpr (IsDateType && IsDateTimeType) { + if constexpr (IsDateTimeType) { DataTypeDateTime::cast_to_date_time(vec_to[i]); } else { DataTypeDate::cast_to_date(vec_to[i]); } + } else if constexpr (IsDateV2Type) { + auto date_v2 = binary_cast(vec_to[i]); + date_v2.from_date_int64(vec_from[i]); } else { vec_to[i] = reinterpret_cast(vec_from[i]).to_int64(); } - } else + } else if constexpr (IsDateV2Type) { + if constexpr (IsTimeType || IsDateTimeV2Type) { + if constexpr (IsDateTimeType || IsDateTimeV2Type) { + DataTypeDateV2::cast_to_date_time(vec_from[i], vec_to[i]); + } else { + DataTypeDateV2::cast_to_date(vec_from[i], vec_to[i]); + } + } else { + vec_to[i] = + reinterpret_cast(vec_from[i]).to_date_uint32(); + } + } else { vec_to[i] = static_cast(vec_from[i]); + } } // TODO: support boolean cast more reasonable @@ -180,7 +201,10 @@ struct ConvertImplToTimeType { using ColVecFrom = std::conditional_t, ColumnDecimal, ColumnVector>; - using ColVecTo = ColumnVector; + + using DateValueType = + std::conditional_t, DateV2Value, VecDateTimeValue>; + using ColVecTo = ColumnVector; if (const ColVecFrom* col_from = check_and_get_column(named_from.column.get())) { @@ -197,11 +221,13 @@ struct ConvertImplToTimeType { auto& vec_null_map_to = col_null_map_to->get_data(); for (size_t i = 0; i < size; ++i) { - auto& date_value = reinterpret_cast(vec_to[i]); + auto& date_value = reinterpret_cast(vec_to[i]); if constexpr (IsDecimalNumber) { vec_null_map_to[i] = !date_value.from_date_int64( convert_from_decimal( vec_from[i], vec_from.get_scale())); + } else if constexpr (IsDateV2Type) { + DataTypeDateV2::cast_to_date_time(vec_from[i], vec_to[i]); } else { vec_null_map_to[i] = !date_value.from_date_int64(vec_from[i]); } @@ -352,6 +378,10 @@ bool try_parse_impl(typename DataType::FieldType& x, ReadBuffer& rb, const DateL return try_read_date_text(x, rb); } + if constexpr (IsDateV2Type) { + return try_read_date_v2_text(x, rb); + } + if constexpr (std::is_floating_point_v) { return try_read_float_text(x, rb); } @@ -640,6 +670,7 @@ using FunctionToDecimal64 = using FunctionToDecimal128 = FunctionConvert, NameToDecimal128, UnknownMonotonicity>; using FunctionToDate = FunctionConvert; +using FunctionToDateV2 = FunctionConvert; using FunctionToDateTime = FunctionConvert; template @@ -705,6 +736,10 @@ struct FunctionTo { using Type = FunctionToDate; }; template <> +struct FunctionTo { + using Type = FunctionToDateV2; +}; +template <> struct FunctionTo { using Type = FunctionToDateTime; }; @@ -968,9 +1003,11 @@ private: /// In case when converting to Nullable type, we apply different parsing rule, /// that will not throw an exception but return NULL in case of malformed input. function = FunctionConvertFromString::create(); - } else if (requested_result_is_nullable && IsTimeType && - !(check_and_get_data_type(from_type.get()) || - check_and_get_data_type(from_type.get()))) { + } else if (requested_result_is_nullable && + (IsTimeType || IsDateV2Type)&&!( + check_and_get_data_type(from_type.get()) || + check_and_get_data_type(from_type.get()) || + check_and_get_data_type(from_type.get()))) { function = FunctionConvertToTimeType::create(); } else { function = FunctionTo::Type::create(); @@ -1008,7 +1045,7 @@ private: WhichDataType which(type_index); bool ok = which.is_int() || which.is_native_uint() || which.is_decimal() || - which.is_float() || which.is_date_or_datetime() || + which.is_float() || which.is_date_or_datetime() || which.is_date_v2() || which.is_string_or_fixed_string(); if (!ok) { LOG(FATAL) << fmt::format( @@ -1236,8 +1273,9 @@ private: block.get_by_position(result).column = tmp_block.get_by_position(result).column; return Status::OK(); }; - } else + } else { return wrapper; + } } /// 'from_type' and 'to_type' are nested types in case of Nullable. @@ -1267,7 +1305,8 @@ private: std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v) { + std::is_same_v || + std::is_same_v) { ret = create_wrapper(from_type, check_and_get_data_type(to_type.get()), requested_result_is_nullable); return true; @@ -1350,6 +1389,11 @@ protected: arguments[0].type->get_type_id() != TypeIndex::DateTime) && (type->get_type_id() == TypeIndex::Date || type->get_type_id() == TypeIndex::DateTime); + // 4. from_type is not DateTimeV2/DateV2, to_type is DateTimeV2/DateV2 + need_to_be_nullable |= (arguments[0].type->get_type_id() != TypeIndex::DateV2 && + arguments[0].type->get_type_id() != TypeIndex::DateTimeV2) && + (type->get_type_id() == TypeIndex::DateV2 || + type->get_type_id() == TypeIndex::DateTimeV2); if (need_to_be_nullable) { return make_nullable(type); } diff --git a/be/src/vec/functions/function_date_or_datetime_computation.cpp b/be/src/vec/functions/function_date_or_datetime_computation.cpp index 7469dba76e..6a923d7d5e 100644 --- a/be/src/vec/functions/function_date_or_datetime_computation.cpp +++ b/be/src/vec/functions/function_date_or_datetime_computation.cpp @@ -21,36 +21,117 @@ namespace doris::vectorized { -using FunctionAddSeconds = FunctionDateOrDateTimeComputation; -using FunctionAddMinutes = FunctionDateOrDateTimeComputation; -using FunctionAddHours = FunctionDateOrDateTimeComputation; -using FunctionAddDays = FunctionDateOrDateTimeComputation; -using FunctionAddWeeks = FunctionDateOrDateTimeComputation; -using FunctionAddMonths = FunctionDateOrDateTimeComputation; -using FunctionAddQuarters = FunctionDateOrDateTimeComputation; -using FunctionAddYears = FunctionDateOrDateTimeComputation; +using FunctionAddSeconds = FunctionDateOrDateTimeComputation< + AddSecondsImpl>; +using FunctionAddSecondsV2 = + FunctionDateOrDateTimeComputation>; +using FunctionAddMinutes = FunctionDateOrDateTimeComputation< + AddMinutesImpl>; +using FunctionAddHours = + FunctionDateOrDateTimeComputation>; +using FunctionAddDays = + FunctionDateOrDateTimeComputation>; +using FunctionAddWeeks = + FunctionDateOrDateTimeComputation>; +using FunctionAddMonths = + FunctionDateOrDateTimeComputation>; +using FunctionAddMinutesV2 = + FunctionDateOrDateTimeComputation>; +using FunctionAddHoursV2 = + FunctionDateOrDateTimeComputation>; +using FunctionAddDaysV2 = + FunctionDateOrDateTimeComputation>; +using FunctionAddWeeksV2 = + FunctionDateOrDateTimeComputation>; +using FunctionAddMonthsV2 = + FunctionDateOrDateTimeComputation>; +using FunctionAddQuarters = FunctionDateOrDateTimeComputation< + AddQuartersImpl>; +using FunctionAddQuartersV2 = + FunctionDateOrDateTimeComputation>; +using FunctionAddYears = + FunctionDateOrDateTimeComputation>; +using FunctionAddYearsV2 = + FunctionDateOrDateTimeComputation>; -using FunctionSubSeconds = FunctionDateOrDateTimeComputation; -using FunctionSubMinutes = FunctionDateOrDateTimeComputation; -using FunctionSubHours = FunctionDateOrDateTimeComputation; -using FunctionSubDays = FunctionDateOrDateTimeComputation; -using FunctionSubWeeks = FunctionDateOrDateTimeComputation; -using FunctionSubMonths = FunctionDateOrDateTimeComputation; -using FunctionSubQuarters = FunctionDateOrDateTimeComputation; -using FunctionSubYears = FunctionDateOrDateTimeComputation; +using FunctionSubSeconds = FunctionDateOrDateTimeComputation< + SubtractSecondsImpl>; +using FunctionSubMinutes = FunctionDateOrDateTimeComputation< + SubtractMinutesImpl>; +using FunctionSubHours = FunctionDateOrDateTimeComputation< + SubtractHoursImpl>; +using FunctionSubDays = FunctionDateOrDateTimeComputation< + SubtractDaysImpl>; +using FunctionSubWeeks = FunctionDateOrDateTimeComputation< + SubtractWeeksImpl>; +using FunctionSubMonths = FunctionDateOrDateTimeComputation< + SubtractMonthsImpl>; +using FunctionSubQuarters = FunctionDateOrDateTimeComputation< + SubtractQuartersImpl>; +using FunctionSubYears = FunctionDateOrDateTimeComputation< + SubtractYearsImpl>; +using FunctionSubSecondsV2 = FunctionDateOrDateTimeComputation< + SubtractSecondsImpl>; +using FunctionSubMinutesV2 = FunctionDateOrDateTimeComputation< + SubtractMinutesImpl>; +using FunctionSubHoursV2 = FunctionDateOrDateTimeComputation< + SubtractHoursImpl>; +using FunctionSubDaysV2 = + FunctionDateOrDateTimeComputation>; +using FunctionSubWeeksV2 = FunctionDateOrDateTimeComputation< + SubtractWeeksImpl>; +using FunctionSubMonthsV2 = FunctionDateOrDateTimeComputation< + SubtractMonthsImpl>; +using FunctionSubQuartersV2 = FunctionDateOrDateTimeComputation< + SubtractQuartersImpl>; +using FunctionSubYearsV2 = FunctionDateOrDateTimeComputation< + SubtractYearsImpl>; -using FunctionDateDiff = FunctionDateOrDateTimeComputation; -using FunctionTimeDiff = FunctionDateOrDateTimeComputation; -using FunctionYearsDiff = FunctionDateOrDateTimeComputation; -using FunctionMonthsDiff = FunctionDateOrDateTimeComputation; -using FunctionDaysDiff = FunctionDateOrDateTimeComputation; -using FunctionWeeksDiff = FunctionDateOrDateTimeComputation; -using FunctionHoursDiff = FunctionDateOrDateTimeComputation; -using FunctionMinutesDiff = FunctionDateOrDateTimeComputation; -using FunctionSecondsDiff = FunctionDateOrDateTimeComputation; +using FunctionDateDiff = FunctionDateOrDateTimeComputation>; +using FunctionTimeDiff = FunctionDateOrDateTimeComputation>; +using FunctionYearsDiff = FunctionDateOrDateTimeComputation>; +using FunctionMonthsDiff = FunctionDateOrDateTimeComputation>; +using FunctionDaysDiff = FunctionDateOrDateTimeComputation>; +using FunctionWeeksDiff = FunctionDateOrDateTimeComputation>; +using FunctionHoursDiff = FunctionDateOrDateTimeComputation>; +using FunctionMinutesDiff = FunctionDateOrDateTimeComputation>; +using FunctionSecondsDiff = FunctionDateOrDateTimeComputation>; +using FunctionDateDiffV2 = FunctionDateOrDateTimeComputation< + DateDiffImpl>; +using FunctionTimeDiffV2 = FunctionDateOrDateTimeComputation< + TimeDiffImpl>; +using FunctionYearsDiffV2 = FunctionDateOrDateTimeComputation< + YearsDiffImpl>; +using FunctionMonthsDiffV2 = FunctionDateOrDateTimeComputation< + MonthsDiffImpl>; +using FunctionDaysDiffV2 = FunctionDateOrDateTimeComputation< + DaysDiffImpl>; +using FunctionWeeksDiffV2 = FunctionDateOrDateTimeComputation< + WeeksDiffImpl>; +using FunctionHoursDiffV2 = FunctionDateOrDateTimeComputation< + HoursDiffImpl>; +using FunctionMinutesDiffV2 = FunctionDateOrDateTimeComputation< + MintueSDiffImpl>; +using FunctionSecondsDiffV2 = FunctionDateOrDateTimeComputation< + SecondsDiffImpl>; -using FunctionToYearWeekTwoArgs = FunctionDateOrDateTimeComputation; -using FunctionToWeekTwoArgs = FunctionDateOrDateTimeComputation; +using FunctionToYearWeekTwoArgs = FunctionDateOrDateTimeComputation< + ToYearWeekTwoArgsImpl>; +using FunctionToYearWeekTwoArgsV2 = FunctionDateOrDateTimeComputation< + ToYearWeekTwoArgsImpl>; +using FunctionToWeekTwoArgs = FunctionDateOrDateTimeComputation< + ToWeekTwoArgsImpl>; +using FunctionToWeekTwoArgsV2 = + FunctionDateOrDateTimeComputation>; struct NowFunctionName { static constexpr auto name = "now"; @@ -83,8 +164,12 @@ struct CurrentDateFunctionName { static constexpr auto name = "current_date"; }; -using FunctionCurDate = FunctionCurrentDateOrDateTime>; -using FunctionCurrentDate = FunctionCurrentDateOrDateTime>; +FunctionBuilderPtr createCurrentDateFunctionBuilderFunction() { + return std::make_shared>(); +} +FunctionBuilderPtr createCurDateFunctionBuilderFunction() { + return std::make_shared>(); +} struct CurTimeFunctionName { static constexpr auto name = "curtime"; @@ -102,23 +187,39 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); factory.register_alias("days_add", "date_add"); factory.register_alias("days_add", "adddate"); factory.register_function(); factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); factory.register_function(); factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); factory.register_alias("days_sub", "date_sub"); factory.register_alias("days_sub", "subdate"); factory.register_function(); factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); factory.register_function(); factory.register_function(); @@ -129,16 +230,67 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + + // Functions below make dateV2 be compatible with V1 + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); + + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); factory.register_function(); factory.register_function(); factory.register_function(); factory.register_function(); - factory.register_function(); - factory.register_function(); + factory.register_function(CurrentDateFunctionName::name, + &createCurrentDateFunctionBuilderFunction); + factory.register_function(CurDateFunctionName::name, &createCurDateFunctionBuilderFunction); factory.register_function(); factory.register_function(); factory.register_function(); diff --git a/be/src/vec/functions/function_date_or_datetime_computation.h b/be/src/vec/functions/function_date_or_datetime_computation.h index efca50e65e..f748d5ad6d 100644 --- a/be/src/vec/functions/function_date_or_datetime_computation.h +++ b/be/src/vec/functions/function_date_or_datetime_computation.h @@ -41,14 +41,28 @@ inline Int64 date_time_add(const Int64& t, Int64 delta, bool& is_null) { return binary_cast(ts_value); } -#define ADD_TIME_FUNCTION_IMPL(CLASS, NAME, UNIT) \ - struct CLASS { \ - using ReturnType = DataTypeDateTime; \ - static constexpr auto name = #NAME; \ - static constexpr auto is_nullable = true; \ - static inline Int64 execute(const Int64& t, Int64 delta, bool& is_null) { \ - return date_time_add(t, delta, is_null); \ - } \ +template +inline Int64 date_time_add(const UInt32& t, Int64 delta, bool& is_null) { + auto ts_value = binary_cast(t); + TimeInterval interval(unit, delta, false); + is_null = !ts_value.date_add_interval(interval, unit); + + return binary_cast(ts_value); +} + +#define ADD_TIME_FUNCTION_IMPL(CLASS, NAME, UNIT) \ + template \ + struct CLASS { \ + using ReturnType = ResultType; \ + static constexpr auto name = #NAME; \ + static constexpr auto is_nullable = true; \ + static inline ArgType execute(const ArgType& t, Int64 delta, bool& is_null) { \ + return date_time_add(t, delta, is_null); \ + } \ + \ + static DataTypes get_variadic_argument_types() { \ + return {std::make_shared(), std::make_shared()}; \ + } \ } ADD_TIME_FUNCTION_IMPL(AddSecondsImpl, seconds_add, SECOND); @@ -59,84 +73,132 @@ ADD_TIME_FUNCTION_IMPL(AddWeeksImpl, weeks_add, WEEK); ADD_TIME_FUNCTION_IMPL(AddMonthsImpl, months_add, MONTH); ADD_TIME_FUNCTION_IMPL(AddYearsImpl, years_add, YEAR); +template struct AddQuartersImpl { - using ReturnType = DataTypeDateTime; + using ReturnType = ResultType; static constexpr auto name = "quarters_add"; static constexpr auto is_nullable = true; - static inline Int64 execute(const Int64& t, Int64 delta, bool& is_null) { + static inline Int64 execute(const ArgType& t, Int64 delta, bool& is_null) { return date_time_add(t, delta * 3, is_null); } + + static DataTypes get_variadic_argument_types() { return {std::make_shared()}; } }; -template +template struct SubtractIntervalImpl { - using ReturnType = DataTypeDateTime; + using ReturnType = ResultType; static constexpr auto is_nullable = true; - static inline Int64 execute(const Int64& t, Int64 delta, bool& is_null) { + static inline Int64 execute(const ArgType& t, Int64 delta, bool& is_null) { return Transform::execute(t, -delta, is_null); } + + static DataTypes get_variadic_argument_types() { + return {std::make_shared(), std::make_shared()}; + } }; -struct SubtractSecondsImpl : SubtractIntervalImpl { +template +struct SubtractSecondsImpl : SubtractIntervalImpl, + DateType, ArgType, ResultType> { static constexpr auto name = "seconds_sub"; }; -struct SubtractMinutesImpl : SubtractIntervalImpl { + +template +struct SubtractMinutesImpl : SubtractIntervalImpl, + DateType, ArgType, ResultType> { static constexpr auto name = "minutes_sub"; }; -struct SubtractHoursImpl : SubtractIntervalImpl { + +template +struct SubtractHoursImpl : SubtractIntervalImpl, + DateType, ArgType, ResultType> { static constexpr auto name = "hours_sub"; }; -struct SubtractDaysImpl : SubtractIntervalImpl { + +template +struct SubtractDaysImpl : SubtractIntervalImpl, DateType, + ArgType, ResultType> { static constexpr auto name = "days_sub"; }; -struct SubtractWeeksImpl : SubtractIntervalImpl { + +template +struct SubtractWeeksImpl : SubtractIntervalImpl, + DateType, ArgType, ResultType> { static constexpr auto name = "weeks_sub"; }; -struct SubtractMonthsImpl : SubtractIntervalImpl { + +template +struct SubtractMonthsImpl : SubtractIntervalImpl, + DateType, ArgType, ResultType> { static constexpr auto name = "months_sub"; }; -struct SubtractQuartersImpl : SubtractIntervalImpl { + +template +struct SubtractQuartersImpl : SubtractIntervalImpl, + DateType, ArgType, ResultType> { static constexpr auto name = "quarters_sub"; }; -struct SubtractYearsImpl : SubtractIntervalImpl { + +template +struct SubtractYearsImpl : SubtractIntervalImpl, + DateType, ArgType, ResultType> { static constexpr auto name = "years_sub"; }; +template struct DateDiffImpl { using ReturnType = DataTypeInt32; static constexpr auto name = "datediff"; static constexpr auto is_nullable = false; - static inline Int32 execute(const Int64& t0, const Int64& t1, bool& is_null) { - const auto& ts0 = reinterpret_cast(t0); - const auto& ts1 = reinterpret_cast(t1); + static inline Int32 execute(const ArgType1& t0, const ArgType2& t1, bool& is_null) { + const auto& ts0 = reinterpret_cast(t0); + const auto& ts1 = reinterpret_cast(t1); is_null = !ts0.is_valid_date() || !ts1.is_valid_date(); return ts0.daynr() - ts1.daynr(); } + + static DataTypes get_variadic_argument_types() { + return {std::make_shared(), std::make_shared()}; + } }; +template struct TimeDiffImpl { using ReturnType = DataTypeFloat64; static constexpr auto name = "timediff"; static constexpr auto is_nullable = false; - static inline double execute(const Int64& t0, const Int64& t1, bool& is_null) { - const auto& ts0 = reinterpret_cast(t0); - const auto& ts1 = reinterpret_cast(t1); + static inline double execute(const ArgType1& t0, const ArgType2& t1, bool& is_null) { + const auto& ts0 = reinterpret_cast(t0); + const auto& ts1 = reinterpret_cast(t1); is_null = !ts0.is_valid_date() || !ts1.is_valid_date(); return ts0.second_diff(ts1); } + + static DataTypes get_variadic_argument_types() { + return {std::make_shared(), std::make_shared()}; + } }; -#define TIME_DIFF_FUNCTION_IMPL(CLASS, NAME, UNIT) \ - struct CLASS { \ - using ReturnType = DataTypeInt64; \ - static constexpr auto name = #NAME; \ - static constexpr auto is_nullable = false; \ - static inline int64_t execute(const Int64& t0, const Int64& t1, bool& is_null) { \ - const auto& ts0 = reinterpret_cast(t0); \ - const auto& ts1 = reinterpret_cast(t1); \ - is_null = !ts0.is_valid_date() || !ts1.is_valid_date(); \ - return VecDateTimeValue::datetime_diff(ts1, ts0); \ - } \ +#define TIME_DIFF_FUNCTION_IMPL(CLASS, NAME, UNIT) \ + template \ + struct CLASS { \ + using ReturnType = DataTypeInt64; \ + static constexpr auto name = #NAME; \ + static constexpr auto is_nullable = false; \ + static inline Int64 execute(const ArgType1& t0, const ArgType2& t1, bool& is_null) { \ + const auto& ts0 = reinterpret_cast(t0); \ + const auto& ts1 = reinterpret_cast(t1); \ + is_null = !ts0.is_valid_date() || !ts1.is_valid_date(); \ + return datetime_diff(ts1, ts0); \ + } \ + \ + static DataTypes get_variadic_argument_types() { \ + return {std::make_shared(), std::make_shared()}; \ + } \ } TIME_DIFF_FUNCTION_IMPL(YearsDiffImpl, years_diff, YEAR); @@ -147,29 +209,30 @@ TIME_DIFF_FUNCTION_IMPL(HoursDiffImpl, hours_diff, HOUR); TIME_DIFF_FUNCTION_IMPL(MintueSDiffImpl, minutes_diff, MINUTE); TIME_DIFF_FUNCTION_IMPL(SecondsDiffImpl, seconds_diff, SECOND); -#define TIME_FUNCTION_TWO_ARGS_IMPL(CLASS, NAME, FUNCTION) \ - struct CLASS { \ - using ReturnType = DataTypeInt32; \ - static constexpr auto name = #NAME; \ - static constexpr auto is_nullable = false; \ - static inline int64_t execute(const Int64& t0, const Int32 mode, bool& is_null) { \ - const auto& ts0 = reinterpret_cast(t0); \ - is_null = !ts0.is_valid_date(); \ - return ts0.FUNCTION; \ - } \ - static DataTypes get_variadic_argument_types() { \ - return {std::make_shared(), std::make_shared()}; \ - } \ +#define TIME_FUNCTION_TWO_ARGS_IMPL(CLASS, NAME, FUNCTION) \ + template \ + struct CLASS { \ + using ReturnType = DataTypeInt32; \ + static constexpr auto name = #NAME; \ + static constexpr auto is_nullable = false; \ + static inline int64_t execute(const ArgType& t0, const Int32 mode, bool& is_null) { \ + const auto& ts0 = reinterpret_cast(t0); \ + is_null = !ts0.is_valid_date(); \ + return ts0.FUNCTION; \ + } \ + static DataTypes get_variadic_argument_types() { \ + return {std::make_shared(), std::make_shared()}; \ + } \ } TIME_FUNCTION_TWO_ARGS_IMPL(ToYearWeekTwoArgsImpl, yearweek, year_week(mysql_week_mode(mode))); TIME_FUNCTION_TWO_ARGS_IMPL(ToWeekTwoArgsImpl, week, week(mysql_week_mode(mode))); -template +template struct DateTimeOp { // use for (DateTime, DateTime) -> other_type - static void vector_vector(const PaddedPODArray& vec_from0, - const PaddedPODArray& vec_from1, + static void vector_vector(const PaddedPODArray& vec_from0, + const PaddedPODArray& vec_from1, PaddedPODArray& vec_to, NullMap& null_map) { size_t size = vec_from0.size(); vec_to.resize(size); @@ -185,7 +248,7 @@ struct DateTimeOp { } // use for (DateTime, int32) -> other_type - static void vector_vector(const PaddedPODArray& vec_from0, + static void vector_vector(const PaddedPODArray& vec_from0, const PaddedPODArray& vec_from1, PaddedPODArray& vec_to, NullMap& null_map) { size_t size = vec_from0.size(); @@ -198,7 +261,7 @@ struct DateTimeOp { } // use for (DateTime, const DateTime) -> other_type - static void vector_constant(const PaddedPODArray& vec_from, + static void vector_constant(const PaddedPODArray& vec_from, PaddedPODArray& vec_to, NullMap& null_map, Int128& delta) { size_t size = vec_from.size(); vec_to.resize(size); @@ -211,7 +274,7 @@ struct DateTimeOp { } // use for (DateTime, const ColumnNumber) -> other_type - static void vector_constant(const PaddedPODArray& vec_from, + static void vector_constant(const PaddedPODArray& vec_from, PaddedPODArray& vec_to, NullMap& null_map, Int64 delta) { size_t size = vec_from.size(); vec_to.resize(size); @@ -224,7 +287,7 @@ struct DateTimeOp { } // use for (const DateTime, ColumnNumber) -> other_type - static void constant_vector(const FromType& from, PaddedPODArray& vec_to, + static void constant_vector(const FromType1& from, PaddedPODArray& vec_to, NullMap& null_map, const IColumn& delta) { size_t size = delta.size(); vec_to.resize(size); @@ -236,8 +299,8 @@ struct DateTimeOp { } } - static void constant_vector(const FromType& from, PaddedPODArray& vec_to, - NullMap& null_map, const PaddedPODArray& delta) { + static void constant_vector(const FromType1& from, PaddedPODArray& vec_to, + NullMap& null_map, const PaddedPODArray& delta) { size_t size = delta.size(); vec_to.resize(size); null_map.resize_fill(size, false); @@ -248,14 +311,14 @@ struct DateTimeOp { } }; -template +template struct DateTimeAddIntervalImpl { static Status execute(Block& block, const ColumnNumbers& arguments, size_t result) { using ToType = typename Transform::ReturnType::FieldType; - using Op = DateTimeOp; + using Op = DateTimeOp; const ColumnPtr source_col = block.get_by_position(arguments[0]).column; - if (const auto* sources = check_and_get_column>(source_col.get())) { + if (const auto* sources = check_and_get_column>(source_col.get())) { auto col_to = ColumnVector::create(); auto null_map = ColumnUInt8::create(); const IColumn& delta_column = *block.get_by_position(arguments[1]).column; @@ -276,7 +339,7 @@ struct DateTimeAddIntervalImpl { } } else { if (const auto* delta_vec_column0 = - check_and_get_column>(delta_column)) { + check_and_get_column>(delta_column)) { Op::vector_vector(sources->get_data(), delta_vec_column0->get_data(), col_to->get_data(), null_map->get_data()); } else { @@ -291,17 +354,17 @@ struct DateTimeAddIntervalImpl { block.get_by_position(result).column = ColumnNullable::create(std::move(col_to), std::move(null_map)); } else if (const auto* sources_const = - check_and_get_column_const>(source_col.get())) { + check_and_get_column_const>(source_col.get())) { auto col_to = ColumnVector::create(); auto null_map = ColumnUInt8::create(); - if (const auto* delta_vec_column = check_and_get_column>( + if (const auto* delta_vec_column = check_and_get_column>( *block.get_by_position(arguments[1]).column)) { - Op::constant_vector(sources_const->template get_value(), + Op::constant_vector(sources_const->template get_value(), col_to->get_data(), null_map->get_data(), delta_vec_column->get_data()); } else { - Op::constant_vector(sources_const->template get_value(), + Op::constant_vector(sources_const->template get_value(), col_to->get_data(), null_map->get_data(), *block.get_by_position(arguments[1]).column); } @@ -344,7 +407,8 @@ public: } if (arguments.size() == 2) { - if (!is_date_or_datetime(arguments[0].type)) { + if (!is_date_or_datetime(arguments[0].type) && + !is_date_v2_or_datetime_v2(arguments[0].type)) { LOG(FATAL) << fmt::format( "Illegal type {} of argument of function {}. Should be a date or a date " "with time", @@ -352,6 +416,7 @@ public: } } else { if (!WhichDataType(arguments[0].type).is_date_time() || + !WhichDataType(arguments[0].type).is_date_time_v2() || !WhichDataType(arguments[2].type).is_string()) { LOG(FATAL) << fmt::format( "Function {} supports 2 or 3 arguments. The 1st argument must be of type " @@ -368,15 +433,56 @@ public: Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) override { - const IDataType* from_type = block.get_by_position(arguments[0]).type.get(); - WhichDataType which(from_type); + const IDataType* first_arg_type = block.get_by_position(arguments[0]).type.get(); + const IDataType* second_arg_type = block.get_by_position(arguments[1]).type.get(); + WhichDataType which1(first_arg_type); + WhichDataType which2(second_arg_type); - if (which.is_date()) { + if (which1.is_date() && which2.is_date()) { + return DateTimeAddIntervalImpl::execute(block, arguments, + result); + } else if (which1.is_date_time() && which2.is_date()) { + return DateTimeAddIntervalImpl::execute(block, arguments, + result); + } else if (which1.is_date_v2() && which2.is_date()) { + return DateTimeAddIntervalImpl::execute(block, arguments, + result); + } else if (which1.is_date() && which2.is_date_time()) { + return DateTimeAddIntervalImpl::execute(block, arguments, + result); + } else if (which1.is_date() && which2.is_date_v2()) { + return DateTimeAddIntervalImpl::execute(block, arguments, + result); + } else if (which1.is_date_v2() && which2.is_date_time()) { + return DateTimeAddIntervalImpl::execute(block, arguments, + result); + } else if (which1.is_date_v2() && which2.is_date_v2()) { + return DateTimeAddIntervalImpl::execute(block, arguments, + result); + } else if (which1.is_date_time() && which2.is_date_time()) { + return DateTimeAddIntervalImpl::execute(block, arguments, + result); + } else if (which1.is_date_time() && which2.is_date_v2()) { + return DateTimeAddIntervalImpl::execute(block, arguments, + result); + } else if (which1.is_date()) { return DateTimeAddIntervalImpl::execute( block, arguments, result); - } else if (which.is_date_time()) { + } else if (which1.is_date_time()) { return DateTimeAddIntervalImpl::execute( block, arguments, result); + } else if (which1.is_date_v2()) { + return DateTimeAddIntervalImpl::execute( + block, arguments, result); } else { return Status::RuntimeError( fmt::format("Illegal type {} of argument of function {}", @@ -435,28 +541,47 @@ struct CurrentDateTimeImpl { } }; -template +template struct CurrentDateImpl { - using ReturnType = DataTypeDate; + using ReturnType = DateType; static constexpr auto name = FunctionName::name; static Status execute(FunctionContext* context, Block& block, size_t result, size_t input_rows_count) { - auto col_to = ColumnVector::create(); - VecDateTimeValue dtv; - if (dtv.from_unixtime(context->impl()->state()->timestamp_ms() / 1000, - context->impl()->state()->timezone_obj())) { - reinterpret_cast(&dtv)->set_type(TIME_DATE); - auto date_packed_int = binary_cast( - *reinterpret_cast(&dtv)); - for (int i = 0; i < input_rows_count; i++) { - col_to->insert_data( - const_cast(reinterpret_cast(&date_packed_int)), 0); + auto col_to = ColumnVector::create(); + if constexpr (std::is_same_v) { + DateV2Value dtv; + if (dtv.from_unixtime(context->impl()->state()->timestamp_ms() / 1000, + context->impl()->state()->timezone_obj())) { + auto date_packed_int = + binary_cast(*reinterpret_cast(&dtv)); + for (int i = 0; i < input_rows_count; i++) { + col_to->insert_data( + const_cast(reinterpret_cast(&date_packed_int)), 0); + } + } else { + auto invalid_val = 0; + for (int i = 0; i < input_rows_count; i++) { + col_to->insert_data( + const_cast(reinterpret_cast(&invalid_val)), 0); + } } } else { - auto invalid_val = 0; - for (int i = 0; i < input_rows_count; i++) { - col_to->insert_data(const_cast(reinterpret_cast(&invalid_val)), - 0); + VecDateTimeValue dtv; + if (dtv.from_unixtime(context->impl()->state()->timestamp_ms() / 1000, + context->impl()->state()->timezone_obj())) { + reinterpret_cast(&dtv)->set_type(TIME_DATE); + auto date_packed_int = binary_cast( + *reinterpret_cast(&dtv)); + for (int i = 0; i < input_rows_count; i++) { + col_to->insert_data( + const_cast(reinterpret_cast(&date_packed_int)), 0); + } + } else { + auto invalid_val = 0; + for (int i = 0; i < input_rows_count; i++) { + col_to->insert_data( + const_cast(reinterpret_cast(&invalid_val)), 0); + } } } block.get_by_position(result).column = std::move(col_to); @@ -517,4 +642,38 @@ struct UtcTimestampImpl { } }; +template +class CurrentDateFunctionBuilder : public FunctionBuilderImpl { +public: + explicit CurrentDateFunctionBuilder() = default; + + String get_name() const override { return FunctionName::name; } + size_t get_number_of_arguments() const override { return 0; } + +protected: + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + return make_nullable(std::make_shared()); + } + DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override { + return make_nullable(std::make_shared()); + } + + bool use_default_implementation_for_nulls() const override { return false; } + + FunctionBasePtr build_impl(const ColumnsWithTypeAndName& arguments, + const DataTypePtr& return_type) const override { + DataTypes data_types(arguments.size()); + for (size_t i = 0; i < arguments.size(); ++i) data_types[i] = arguments[i].type; + if (is_date_v2(return_type)) { + auto function = FunctionCurrentDateOrDateTime< + CurrentDateImpl>::create(); + return std::make_shared(function, data_types, return_type); + } else { + auto function = FunctionCurrentDateOrDateTime< + CurrentDateImpl>::create(); + return std::make_shared(function, data_types, return_type); + } + } +}; + } // namespace doris::vectorized diff --git a/be/src/vec/functions/function_date_or_datetime_to_something.h b/be/src/vec/functions/function_date_or_datetime_to_something.h index 1440126390..634f6064d1 100644 --- a/be/src/vec/functions/function_date_or_datetime_to_something.h +++ b/be/src/vec/functions/function_date_or_datetime_to_something.h @@ -48,14 +48,16 @@ public: DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override { if (arguments.size() == 1) { - if (!is_date_or_datetime(arguments[0].type)) { + if (!is_date_or_datetime(arguments[0].type) && + !is_date_v2_or_datetime_v2(arguments[0].type)) { LOG(FATAL) << fmt::format( "Illegal type {} of argument of function {}. Should be a date or a date " "with time", arguments[0].type->get_name(), get_name()); } } else if (arguments.size() == 2) { - if (!is_date_or_datetime(arguments[0].type)) { + if (!is_date_or_datetime(arguments[0].type) && + !is_date_v2_or_datetime_v2(arguments[0].type)) { LOG(FATAL) << fmt::format( "Illegal type {} of argument of function {}. Should be a date or a date " "with time", @@ -92,8 +94,9 @@ public: const IDataType* from_type = block.get_by_position(arguments[0]).type.get(); WhichDataType which(from_type); - return DateTimeTransformImpl::execute( - block, arguments, result, input_rows_count); + return DateTimeTransformImpl::execute(block, arguments, result, + input_rows_count); } bool has_information_about_monotonicity() const override { return true; } diff --git a/be/src/vec/functions/function_date_or_datetime_to_string.cpp b/be/src/vec/functions/function_date_or_datetime_to_string.cpp index 7815f41480..6f9d815fdd 100644 --- a/be/src/vec/functions/function_date_or_datetime_to_string.cpp +++ b/be/src/vec/functions/function_date_or_datetime_to_string.cpp @@ -24,12 +24,16 @@ namespace doris::vectorized { -using FunctionDayName = FunctionDateOrDateTimeToString; -using FunctionMonthName = FunctionDateOrDateTimeToString; +using FunctionDayName = FunctionDateOrDateTimeToString>; +using FunctionDayNameV2 = FunctionDateOrDateTimeToString>; +using FunctionMonthName = FunctionDateOrDateTimeToString>; +using FunctionMonthNameV2 = FunctionDateOrDateTimeToString>; void register_function_date_time_to_string(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); } } // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/functions/function_date_or_datetime_to_string.h b/be/src/vec/functions/function_date_or_datetime_to_string.h index 9992f189b8..8b06869489 100644 --- a/be/src/vec/functions/function_date_or_datetime_to_string.h +++ b/be/src/vec/functions/function_date_or_datetime_to_string.h @@ -32,6 +32,8 @@ template class FunctionDateOrDateTimeToString : public IFunction { public: static constexpr auto name = Transform::name; + static constexpr bool has_variadic_argument = + !std::is_void_v()))>; static FunctionPtr create() { return std::make_shared(); } String get_name() const override { return name; } @@ -42,13 +44,21 @@ public: return make_nullable(std::make_shared()); } + bool is_variadic() const override { return true; } + + DataTypes get_variadic_argument_types_impl() const override { + if constexpr (has_variadic_argument) return Transform::get_variadic_argument_types(); + return {}; + } + bool use_default_implementation_for_constants() const override { return true; } ColumnNumbers get_arguments_that_are_always_constant() const override { return {1}; } Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) override { const ColumnPtr source_col = block.get_by_position(arguments[0]).column; - const auto* sources = check_and_get_column>(source_col.get()); + const auto* sources = + check_and_get_column>(source_col.get()); auto col_res = ColumnString::create(); auto null_map = ColumnVector::create(); // Support all input of datetime is valind to make sure not null return diff --git a/be/src/vec/functions/function_datetime_string_to_string.cpp b/be/src/vec/functions/function_datetime_string_to_string.cpp index a499773669..e4685a96a7 100644 --- a/be/src/vec/functions/function_datetime_string_to_string.cpp +++ b/be/src/vec/functions/function_datetime_string_to_string.cpp @@ -21,12 +21,19 @@ namespace doris::vectorized { -using FunctionDateFormat = FunctionDateTimeStringToString; -using FunctionFromUnixTime = FunctionDateTimeStringToString; +using FunctionDateFormat = FunctionDateTimeStringToString>; +using FunctionDateFormatV2 = FunctionDateTimeStringToString>; +using FunctionFromUnixTime = FunctionDateTimeStringToString>; + +FunctionBuilderPtr createFromUnixTimeFunction() { + return std::make_shared(); +} void register_function_date_time_string_to_string(SimpleFunctionFactory& factory) { factory.register_function(); + factory.register_function(); factory.register_function(); + factory.register_function("from_unixtime", &createFromUnixTimeFunction); } } // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/functions/function_datetime_string_to_string.h b/be/src/vec/functions/function_datetime_string_to_string.h index 6502d0fd90..0cb2d9df96 100644 --- a/be/src/vec/functions/function_datetime_string_to_string.h +++ b/be/src/vec/functions/function_datetime_string_to_string.h @@ -32,12 +32,19 @@ template class FunctionDateTimeStringToString : public IFunction { public: static constexpr auto name = Transform::name; + static constexpr bool has_variadic_argument = + !std::is_void_v()))>; + static FunctionPtr create() { return std::make_shared(); } String get_name() const override { return name; } - size_t get_number_of_arguments() const override { return 0; } bool is_variadic() const override { return true; } + size_t get_number_of_arguments() const override { return 0; } + DataTypes get_variadic_argument_types_impl() const override { + if constexpr (has_variadic_argument) return Transform::get_variadic_argument_types(); + return {}; + } DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override { return make_nullable(std::make_shared()); @@ -98,4 +105,35 @@ public: } }; +class FromUnixTimeFunctionBuilder : public FunctionBuilderImpl { +public: + explicit FromUnixTimeFunctionBuilder() = default; + + String get_name() const override { return "from_unixtime"; } + bool is_variadic() const override { return true; } + size_t get_number_of_arguments() const override { return 0; } + + ColumnNumbers get_arguments_that_are_always_constant() const override { return {1}; } + +protected: + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + return make_nullable(std::make_shared()); + } + DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override { + return make_nullable(std::make_shared()); + } + + bool use_default_implementation_for_nulls() const override { return false; } + + FunctionBasePtr build_impl(const ColumnsWithTypeAndName& arguments, + const DataTypePtr& return_type) const override { + DataTypes data_types(arguments.size()); + for (size_t i = 0; i < arguments.size(); ++i) data_types[i] = arguments[i].type; + // TODO: we still use VecDateTimeValue to convert unix timestamp to string + auto function = + FunctionDateTimeStringToString>::create(); + return std::make_shared(function, data_types, return_type); + } +}; + } // namespace doris::vectorized diff --git a/be/src/vec/functions/function_timestamp.cpp b/be/src/vec/functions/function_timestamp.cpp index 22f159bd8e..691714cb33 100644 --- a/be/src/vec/functions/function_timestamp.cpp +++ b/be/src/vec/functions/function_timestamp.cpp @@ -29,16 +29,17 @@ namespace doris::vectorized { +template struct StrToDate { static constexpr auto name = "str_to_date"; - using ReturnType = DataTypeDateTime; - using ColumnType = ColumnVector; + using ReturnType = DateType; + using ColumnType = ColumnVector; static void vector_vector(FunctionContext* context, const ColumnString::Chars& ldata, const ColumnString::Offsets& loffsets, const ColumnString::Chars& rdata, - const ColumnString::Offsets& roffsets, ColumnType::Container& res, - NullMap& null_map) { + const ColumnString::Offsets& roffsets, + PaddedPODArray& res, NullMap& null_map) { size_t size = loffsets.size(); res.resize(size); for (size_t i = 0; i < size; ++i) { @@ -48,15 +49,23 @@ struct StrToDate { const char* r_raw_str = reinterpret_cast(&rdata[roffsets[i - 1]]); int r_str_size = roffsets[i] - roffsets[i - 1] - 1; - auto& ts_val = *reinterpret_cast(&res[i]); - if (!ts_val.from_date_format_str(r_raw_str, r_str_size, l_raw_str, l_str_size)) { - null_map[i] = 1; - } - if (context->impl()->get_return_type().type == - doris_udf::FunctionContext::Type::TYPE_DATETIME) { - ts_val.to_datetime(); + if constexpr (std::is_same_v || + std::is_same_v) { + auto& ts_val = *reinterpret_cast(&res[i]); + if (!ts_val.from_date_format_str(r_raw_str, r_str_size, l_raw_str, l_str_size)) { + null_map[i] = 1; + } + if (context->impl()->get_return_type().type == + doris_udf::FunctionContext::Type::TYPE_DATETIME) { + ts_val.to_datetime(); + } else { + ts_val.cast_to_date(); + } } else { - ts_val.cast_to_date(); + auto& ts_val = *reinterpret_cast(&res[i]); + if (!ts_val.from_date_format_str(r_raw_str, r_str_size, l_raw_str, l_str_size)) { + null_map[i] = 1; + } } } } @@ -66,16 +75,17 @@ struct NameMakeDate { static constexpr auto name = "makedate"; }; -template +template struct MakeDateImpl { - using ResultDataType = DataTypeDateTime; + using ResultDataType = ResultDateType; using LeftDataColumnType = ColumnVector; using RightDataColumnType = ColumnVector; - using ColumnType = ColumnVector; + using ColumnType = ColumnVector; - static void vector_vector(const typename LeftDataColumnType::Container& ldata, - const typename RightDataColumnType::Container& rdata, - ColumnType::Container& res, NullMap& null_map) { + static void vector_vector(const PaddedPODArray& ldata, + const PaddedPODArray& rdata, + PaddedPODArray& res, NullMap& null_map) { auto len = ldata.size(); res.resize(len); @@ -87,29 +97,40 @@ struct MakeDateImpl { continue; } - auto& res_val = *reinterpret_cast(&res[i]); + if constexpr (std::is_same_v || + std::is_same_v) { + auto& res_val = *reinterpret_cast(&res[i]); - VecDateTimeValue ts_value = VecDateTimeValue(); - ts_value.set_time(l, 1, 1, 0, 0, 0); + VecDateTimeValue ts_value = VecDateTimeValue(); + ts_value.set_time(l, 1, 1, 0, 0, 0); - DateTimeVal ts_val; - ts_value.to_datetime_val(&ts_val); - if (ts_val.is_null) { - null_map[i] = 1; - continue; + DateTimeVal ts_val; + ts_value.to_datetime_val(&ts_val); + if (ts_val.is_null) { + null_map[i] = 1; + continue; + } + + TimeInterval interval(DAY, r - 1, false); + res_val = VecDateTimeValue::from_datetime_val(ts_val); + if (!res_val.date_add_interval(interval, DAY)) { + null_map[i] = 1; + continue; + } + res_val.cast_to_date(); + } else { + DateV2Value* value = new (&res[i]) DateV2Value(); + value->set_time(l, 1, 1); + TimeInterval interval(DAY, r - 1, false); + if (!value->date_add_interval(interval, DAY)) { + null_map[i] = 1; + } } - - TimeInterval interval(DAY, r - 1, false); - res_val = VecDateTimeValue::from_datetime_val(ts_val); - if (!res_val.date_add_interval(interval, DAY)) { - null_map[i] = 1; - continue; - } - res_val.cast_to_date(); } } }; +template class FromDays : public IFunction { public: static constexpr auto name = "from_days"; @@ -125,7 +146,7 @@ public: bool use_default_implementation_for_nulls() const override { return true; } DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { - return make_nullable(std::make_shared()); + return make_nullable(std::make_shared()); } Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, @@ -138,15 +159,23 @@ public: auto data_col = assert_cast*>(argument_column.get()); for (int i = 0; i < input_rows_count; i++) { - const auto& cur_data = data_col->get_data()[i]; - auto& ts_value = *reinterpret_cast(&res_data[i]); - if (!ts_value.from_date_daynr(cur_data)) { - null_map->get_data()[i] = 1; - continue; + if constexpr (std::is_same_v) { + const auto& cur_data = data_col->get_data()[i]; + auto& ts_value = *reinterpret_cast(&res_data[i]); + if (!ts_value.from_date_daynr(cur_data)) { + null_map->get_data()[i] = 1; + continue; + } + DateTimeVal ts_val; + ts_value.to_datetime_val(&ts_val); + ts_value = VecDateTimeValue::from_datetime_val(ts_val); + } else { + const auto& cur_data = data_col->get_data()[i]; + auto& ts_value = *reinterpret_cast(&res_data[i]); + if (!ts_value.get_date_from_daynr(cur_data)) { + null_map->get_data()[i] = 1; + } } - DateTimeVal ts_val; - ts_value.to_datetime_val(&ts_val); - ts_value = VecDateTimeValue::from_datetime_val(ts_val); } block.replace_by_position( result, ColumnNullable::create(std::move(res_column), std::move(null_map))); @@ -154,9 +183,10 @@ public: } }; -using FunctionStrToDate = FunctionBinaryStringOperateToNullType; -using FunctionMakeDate = - FunctionBinaryToNullType; +using FunctionStrToDate = FunctionBinaryStringOperateToNullType>; + +using FunctionMakeDate = FunctionBinaryToNullType; struct UnixTimeStampImpl { static Int32 trim_timestamp(Int64 timestamp) { @@ -182,8 +212,9 @@ struct UnixTimeStampImpl { } }; +template struct UnixTimeStampDateImpl { - static DataTypes get_variadic_argument_types() { return {std::make_shared()}; } + static DataTypes get_variadic_argument_types() { return {std::make_shared()}; } static DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) { return make_nullable(std::make_shared()); @@ -210,14 +241,27 @@ struct UnixTimeStampDateImpl { } StringRef source = col_source->get_data_at(i); - const VecDateTimeValue& ts_value = - reinterpret_cast(*source.data); - int64_t timestamp; - if (!ts_value.unix_timestamp(×tamp, context->impl()->state()->timezone_obj())) { - null_map_data[i] = true; + if constexpr (std::is_same_v) { + const VecDateTimeValue& ts_value = + reinterpret_cast(*source.data); + int64_t timestamp; + if (!ts_value.unix_timestamp(×tamp, + context->impl()->state()->timezone_obj())) { + null_map_data[i] = true; + } else { + null_map_data[i] = false; + col_result_data[i] = UnixTimeStampImpl::trim_timestamp(timestamp); + } } else { - null_map_data[i] = false; - col_result_data[i] = UnixTimeStampImpl::trim_timestamp(timestamp); + const DateV2Value& ts_value = reinterpret_cast(*source.data); + int64_t timestamp; + if (!ts_value.unix_timestamp(×tamp, + context->impl()->state()->timezone_obj())) { + null_map_data[i] = true; + } else { + null_map_data[i] = false; + col_result_data[i] = UnixTimeStampImpl::trim_timestamp(timestamp); + } } } @@ -228,10 +272,9 @@ struct UnixTimeStampDateImpl { } }; -struct UnixTimeStampDatetimeImpl : public UnixTimeStampDateImpl { - static DataTypes get_variadic_argument_types() { - return {std::make_shared()}; - } +template +struct UnixTimeStampDatetimeImpl : public UnixTimeStampDateImpl { + static DataTypes get_variadic_argument_types() { return {std::make_shared()}; } }; struct UnixTimeStampStrImpl { @@ -320,11 +363,13 @@ public: void register_function_timestamp(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); - factory.register_function(); + factory.register_function>(); factory.register_function>(); - factory.register_function>(); - factory.register_function>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); + factory.register_function>>(); factory.register_function>(); } diff --git a/be/src/vec/functions/function_totype.h b/be/src/vec/functions/function_totype.h index 83ddc0cead..27a96b03ad 100644 --- a/be/src/vec/functions/function_totype.h +++ b/be/src/vec/functions/function_totype.h @@ -267,8 +267,9 @@ private: }; // func(type,type) -> nullable(type) -template typename Impl, typename Name> +template typename Impl, + typename Name> class FunctionBinaryToNullType : public IFunction { public: static constexpr auto name = Name::name; @@ -276,7 +277,8 @@ public: String get_name() const override { return name; } size_t get_number_of_arguments() const override { return 2; } DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { - using ResultDataType = typename Impl::ResultDataType; + using ResultDataType = typename Impl::ResultDataType; return make_nullable(std::make_shared()); } @@ -300,7 +302,8 @@ public: } } - using ResultDataType = typename Impl::ResultDataType; + using ResultDataType = typename Impl::ResultDataType; using T0 = typename LeftDataType::FieldType; using T1 = typename RightDataType::FieldType; @@ -323,7 +326,7 @@ public: if (auto col_left = check_and_get_column(argument_columns[0].get())) { if (auto col_right = check_and_get_column(argument_columns[1].get())) { - Impl::vector_vector( + Impl::vector_vector( col_left->get_data(), col_right->get_data(), vec_res, null_map->get_data()); block.get_by_position(result).column = ColumnNullable::create(std::move(col_res), std::move(null_map)); diff --git a/be/src/vec/functions/time_of_function.cpp b/be/src/vec/functions/time_of_function.cpp index 071704826d..86aedbae48 100644 --- a/be/src/vec/functions/time_of_function.cpp +++ b/be/src/vec/functions/time_of_function.cpp @@ -22,12 +22,31 @@ namespace doris::vectorized { -using FunctionWeekOfYear = FunctionDateOrDateTimeToSomething; -using FunctionDayOfYear = FunctionDateOrDateTimeToSomething; -using FunctionDayOfWeek = FunctionDateOrDateTimeToSomething; -using FunctionDayOfMonth = FunctionDateOrDateTimeToSomething; -using FunctionYearWeek = FunctionDateOrDateTimeToSomething; -using FunctionWeekDay = FunctionDateOrDateTimeToSomething; +using FunctionWeekOfYear = + FunctionDateOrDateTimeToSomething>; +using FunctionWeekOfYearV2 = + FunctionDateOrDateTimeToSomething>; +using FunctionDayOfYear = + FunctionDateOrDateTimeToSomething>; +using FunctionDayOfYearV2 = + FunctionDateOrDateTimeToSomething>; +using FunctionDayOfWeek = + FunctionDateOrDateTimeToSomething>; +using FunctionDayOfWeekV2 = + FunctionDateOrDateTimeToSomething>; +using FunctionDayOfMonth = + FunctionDateOrDateTimeToSomething>; +using FunctionDayOfMonthV2 = + FunctionDateOrDateTimeToSomething>; +using FunctionYearWeek = + FunctionDateOrDateTimeToSomething>; +using FunctionYearWeekV2 = + FunctionDateOrDateTimeToSomething>; +using FunctionWeekDay = + FunctionDateOrDateTimeToSomething>; +using FunctionWeekDayV2 = + FunctionDateOrDateTimeToSomething>; void register_function_time_of_function(SimpleFunctionFactory& factory) { factory.register_function(); @@ -36,5 +55,11 @@ void register_function_time_of_function(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); } } // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/functions/to_time_function.cpp b/be/src/vec/functions/to_time_function.cpp index c0e862892c..3a7c6dba78 100644 --- a/be/src/vec/functions/to_time_function.cpp +++ b/be/src/vec/functions/to_time_function.cpp @@ -23,18 +23,53 @@ namespace doris::vectorized { -using FunctionYear = FunctionDateOrDateTimeToSomething; -using FunctionQuarter = FunctionDateOrDateTimeToSomething; -using FunctionMonth = FunctionDateOrDateTimeToSomething; -using FunctionDay = FunctionDateOrDateTimeToSomething; -using FunctionWeek = FunctionDateOrDateTimeToSomething; -using FunctionHour = FunctionDateOrDateTimeToSomething; -using FunctionMinute = FunctionDateOrDateTimeToSomething; -using FunctionSecond = FunctionDateOrDateTimeToSomething; -using FunctionToDays = FunctionDateOrDateTimeToSomething; -using FunctionToDate = FunctionDateOrDateTimeToSomething; -using FunctionDate = FunctionDateOrDateTimeToSomething; -using FunctionTimeStamp = FunctionDateOrDateTimeToSomething; +using FunctionYear = + FunctionDateOrDateTimeToSomething>; +using FunctionYearV2 = + FunctionDateOrDateTimeToSomething>; +using FunctionQuarter = + FunctionDateOrDateTimeToSomething>; +using FunctionQuarterV2 = + FunctionDateOrDateTimeToSomething>; +using FunctionMonth = + FunctionDateOrDateTimeToSomething>; +using FunctionMonthV2 = + FunctionDateOrDateTimeToSomething>; +using FunctionDay = + FunctionDateOrDateTimeToSomething>; +using FunctionDayV2 = + FunctionDateOrDateTimeToSomething>; +using FunctionWeek = + FunctionDateOrDateTimeToSomething>; +using FunctionWeekV2 = + FunctionDateOrDateTimeToSomething>; +using FunctionHour = + FunctionDateOrDateTimeToSomething>; +using FunctionHourV2 = + FunctionDateOrDateTimeToSomething>; +using FunctionMinute = + FunctionDateOrDateTimeToSomething>; +using FunctionMinuteV2 = + FunctionDateOrDateTimeToSomething>; +using FunctionSecond = + FunctionDateOrDateTimeToSomething>; +using FunctionSecondV2 = + FunctionDateOrDateTimeToSomething>; +using FunctionToDays = + FunctionDateOrDateTimeToSomething>; +using FunctionToDaysV2 = + FunctionDateOrDateTimeToSomething>; +using FunctionToDate = + FunctionDateOrDateTimeToSomething>; +using FunctionToDateV2 = + FunctionDateOrDateTimeToSomething>; +using FunctionDate = + FunctionDateOrDateTimeToSomething>; +using FunctionDateV2 = + FunctionDateOrDateTimeToSomething>; +using FunctionTimeStamp = FunctionDateOrDateTimeToSomething>; +using FunctionTimeStampV2 = + FunctionDateOrDateTimeToSomething>; void register_function_to_time_function(SimpleFunctionFactory& factory) { factory.register_function(); @@ -49,6 +84,18 @@ void register_function_to_time_function(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); } } // namespace doris::vectorized diff --git a/be/src/vec/io/io_helper.h b/be/src/vec/io/io_helper.h index 1633380fcc..8425c0ec4d 100644 --- a/be/src/vec/io/io_helper.h +++ b/be/src/vec/io/io_helper.h @@ -279,6 +279,18 @@ bool read_date_text_impl(T& x, ReadBuffer& buf) { return ans; } +template +bool read_date_v2_text_impl(T& x, ReadBuffer& buf) { + static_assert(std::is_same_v); + auto dv = binary_cast(x); + auto ans = dv.from_date_str(buf.position(), buf.count()); + + // only to match the is_all_read() check to prevent return null + buf.position() = buf.end(); + x = binary_cast(dv); + return ans; +} + template bool read_decimal_text_impl(T& x, ReadBuffer& buf) { static_assert(IsDecimalNumber); @@ -335,4 +347,9 @@ template bool try_read_date_text(T& x, ReadBuffer& in) { return read_date_text_impl(x, in); } + +template +bool try_read_date_v2_text(T& x, ReadBuffer& in) { + return read_date_v2_text_impl(x, in); +} } // namespace doris::vectorized diff --git a/be/src/vec/olap/olap_data_convertor.cpp b/be/src/vec/olap/olap_data_convertor.cpp index abfaaef57f..72a55e545b 100644 --- a/be/src/vec/olap/olap_data_convertor.cpp +++ b/be/src/vec/olap/olap_data_convertor.cpp @@ -55,6 +55,10 @@ OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& co case FieldType::OLAP_FIELD_TYPE_DATE: { return std::make_unique(); } + case FieldType::OLAP_FIELD_TYPE_DATEV2: { + return std::make_unique(); + break; + } case FieldType::OLAP_FIELD_TYPE_DATETIME: { return std::make_unique(); } @@ -467,47 +471,102 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorVarChar::convert_to_olap() return Status::OK(); } +void OlapBlockDataConvertor::OlapColumnDataConvertorDate::set_source_column( + const ColumnWithTypeAndName& typed_column, size_t row_pos, size_t num_rows) { + OlapBlockDataConvertor::OlapColumnDataConvertorPaddedPODArray::set_source_column( + typed_column, row_pos, num_rows); + if (is_date_v2(typed_column.type)) { + from_date_v2_ = true; + } else { + from_date_v2_ = false; + } +} + Status OlapBlockDataConvertor::OlapColumnDataConvertorDate::convert_to_olap() { assert(_typed_column.column); - const vectorized::ColumnVector* column_datetime = nullptr; - if (_nullmap) { - auto nullable_column = - assert_cast(_typed_column.column.get()); - column_datetime = assert_cast*>( - nullable_column->get_nested_column_ptr().get()); - } else { - column_datetime = assert_cast*>( - _typed_column.column.get()); - } + if (from_date_v2_) { + const vectorized::ColumnVector* column_datetime = nullptr; + if (_nullmap) { + auto nullable_column = + assert_cast(_typed_column.column.get()); + column_datetime = assert_cast*>( + nullable_column->get_nested_column_ptr().get()); + } else { + column_datetime = assert_cast*>( + _typed_column.column.get()); + } - assert(column_datetime); + assert(column_datetime); - const VecDateTimeValue* datetime_cur = - (const VecDateTimeValue*)(column_datetime->get_data().data()) + _row_pos; - const VecDateTimeValue* datetime_end = datetime_cur + _num_rows; - uint24_t* value = _values.data(); - if (_nullmap) { - const UInt8* nullmap_cur = _nullmap + _row_pos; - while (datetime_cur != datetime_end) { - if (!*nullmap_cur) { - *value = datetime_cur->to_olap_date(); - } else { - // do nothing + const DateV2Value* datetime_cur = + (const DateV2Value*)(column_datetime->get_data().data()) + _row_pos; + const DateV2Value* datetime_end = datetime_cur + _num_rows; + uint24_t* value = _values.data(); + if (_nullmap) { + const UInt8* nullmap_cur = _nullmap + _row_pos; + while (datetime_cur != datetime_end) { + if (!*nullmap_cur) { + *value = datetime_cur->to_olap_date(); + } else { + // do nothing + } + ++value; + ++datetime_cur; + ++nullmap_cur; } - ++value; - ++datetime_cur; - ++nullmap_cur; + assert(nullmap_cur == _nullmap + _row_pos + _num_rows && + value == _values.get_end_ptr()); + } else { + while (datetime_cur != datetime_end) { + *value = datetime_cur->to_olap_date(); + ++value; + ++datetime_cur; + } + assert(value == _values.get_end_ptr()); } - assert(nullmap_cur == _nullmap + _row_pos + _num_rows && value == _values.get_end_ptr()); + return Status::OK(); } else { - while (datetime_cur != datetime_end) { - *value = datetime_cur->to_olap_date(); - ++value; - ++datetime_cur; + const vectorized::ColumnVector* column_datetime = nullptr; + if (_nullmap) { + auto nullable_column = + assert_cast(_typed_column.column.get()); + column_datetime = assert_cast*>( + nullable_column->get_nested_column_ptr().get()); + } else { + column_datetime = assert_cast*>( + _typed_column.column.get()); } - assert(value == _values.get_end_ptr()); + + assert(column_datetime); + + const VecDateTimeValue* datetime_cur = + (const VecDateTimeValue*)(column_datetime->get_data().data()) + _row_pos; + const VecDateTimeValue* datetime_end = datetime_cur + _num_rows; + uint24_t* value = _values.data(); + if (_nullmap) { + const UInt8* nullmap_cur = _nullmap + _row_pos; + while (datetime_cur != datetime_end) { + if (!*nullmap_cur) { + *value = datetime_cur->to_olap_date(); + } else { + // do nothing + } + ++value; + ++datetime_cur; + ++nullmap_cur; + } + assert(nullmap_cur == _nullmap + _row_pos + _num_rows && + value == _values.get_end_ptr()); + } else { + while (datetime_cur != datetime_end) { + *value = datetime_cur->to_olap_date(); + ++value; + ++datetime_cur; + } + assert(value == _values.get_end_ptr()); + } + return Status::OK(); } - return Status::OK(); } Status OlapBlockDataConvertor::OlapColumnDataConvertorDateTime::convert_to_olap() { diff --git a/be/src/vec/olap/olap_data_convertor.h b/be/src/vec/olap/olap_data_convertor.h index 2cf0f79dc3..97bf3a7532 100644 --- a/be/src/vec/olap/olap_data_convertor.h +++ b/be/src/vec/olap/olap_data_convertor.h @@ -192,7 +192,12 @@ private: class OlapColumnDataConvertorDate : public OlapColumnDataConvertorPaddedPODArray { public: + void set_source_column(const ColumnWithTypeAndName& typed_column, size_t row_pos, + size_t num_rows) override; Status convert_to_olap() override; + + private: + bool from_date_v2_; }; class OlapColumnDataConvertorDateTime : public OlapColumnDataConvertorPaddedPODArray { @@ -245,6 +250,96 @@ private: const T* _values = nullptr; }; + class OlapColumnDataConvertorDateV2 : public OlapColumnDataConvertorBase { + public: + OlapColumnDataConvertorDateV2() = default; + ~OlapColumnDataConvertorDateV2() override = default; + + void set_source_column(const ColumnWithTypeAndName& typed_column, size_t row_pos, + size_t num_rows) override { + OlapColumnDataConvertorBase::set_source_column(typed_column, row_pos, num_rows); + if (is_date(typed_column.type)) { + from_date_to_date_v2_ = true; + } else { + from_date_to_date_v2_ = false; + } + } + + const void* get_data() const override { return values_; } + + const void* get_data_at(size_t offset) const override { + assert(offset < _num_rows); + UInt8 null_flag = 0; + if (_nullmap) { + null_flag = _nullmap[offset]; + } + return null_flag ? nullptr : values_ + offset; + } + + Status convert_to_olap() override { + if (UNLIKELY(from_date_to_date_v2_)) { + const vectorized::ColumnVector* column_datetime = nullptr; + if (_nullmap) { + auto nullable_column = assert_cast( + _typed_column.column.get()); + column_datetime = + assert_cast*>( + nullable_column->get_nested_column_ptr().get()); + } else { + column_datetime = + assert_cast*>( + _typed_column.column.get()); + } + + assert(column_datetime); + + const VecDateTimeValue* datetime_cur = + (const VecDateTimeValue*)(column_datetime->get_data().data()) + _row_pos; + const VecDateTimeValue* datetime_end = datetime_cur + _num_rows; + uint32_t* value = const_cast(values_); + if (_nullmap) { + const UInt8* nullmap_cur = _nullmap + _row_pos; + while (datetime_cur != datetime_end) { + if (!*nullmap_cur) { + *value = datetime_cur->to_date_v2(); + } else { + // do nothing + } + ++value; + ++datetime_cur; + ++nullmap_cur; + } + } else { + while (datetime_cur != datetime_end) { + *value = datetime_cur->to_olap_date(); + ++value; + ++datetime_cur; + } + } + return Status::OK(); + } else { + const vectorized::ColumnVector* column_data = nullptr; + if (_nullmap) { + auto nullable_column = assert_cast( + _typed_column.column.get()); + column_data = assert_cast*>( + nullable_column->get_nested_column_ptr().get()); + } else { + column_data = assert_cast*>( + _typed_column.column.get()); + } + + assert(column_data); + values_ = (const uint32*)(column_data->get_data().data()) + _row_pos; + return Status::OK(); + } + } + + private: + const uint32_t* values_ = nullptr; + bool from_date_to_date_v2_; + }; + class OlapColumnDataConvertorArray : public OlapColumnDataConvertorPaddedPODArray { public: diff --git a/be/src/vec/runtime/vdatetime_value.cpp b/be/src/vec/runtime/vdatetime_value.cpp index 845ed46ac9..d7d8179ba4 100644 --- a/be/src/vec/runtime/vdatetime_value.cpp +++ b/be/src/vec/runtime/vdatetime_value.cpp @@ -44,16 +44,14 @@ uint8_t mysql_week_mode(uint32_t mode) { return mode; } -static bool is_leap(uint32_t year) { +bool is_leap(uint32_t year) { return ((year % 4) == 0) && ((year % 100 != 0) || ((year % 400) == 0 && year)); } -static uint32_t calc_days_in_year(uint32_t year) { +uint32_t calc_days_in_year(uint32_t year) { return is_leap(year) ? 366 : 365; } -RE2 VecDateTimeValue::time_zone_offset_format_reg("^[+-]{1}\\d{2}\\:\\d{2}$"); - bool VecDateTimeValue::check_range(uint32_t year, uint32_t month, uint32_t day, uint32_t hour, uint32_t minute, uint32_t second, uint16_t type) { bool time = hour > (type == TIME_TIME ? TIME_MAX_HOUR : 23) || minute > 59 || second > 59; @@ -480,30 +478,6 @@ bool VecDateTimeValue::from_date_daynr(uint64_t daynr) { return true; } -// Following code is stolen from MySQL. -uint64_t VecDateTimeValue::calc_daynr(uint32_t year, uint32_t month, uint32_t day) { - uint64_t delsum = 0; - int y = year; - - if (year == 0 && month == 0) { - return 0; - } - - /* Cast to int to be able to handle month == 0 */ - delsum = 365 * y + 31 * (month - 1) + day; - if (month <= 2) { - // No leap year - y--; - } else { - // This is great!!! - // 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 - // 0, 0, 3, 3, 4, 4, 5, 5, 5, 6, 7, 8 - delsum -= (month * 4 + 23) / 10; - } - // Every 400 year has 97 leap year, 100, 200, 300 are not leap year. - return delsum + y / 4 - y / 100 + y / 400; -} - static char* int_to_str(uint64_t val, char* to) { char buf[64]; char* ptr = buf; @@ -931,8 +905,44 @@ uint32_t VecDateTimeValue::year_week(uint8_t mode) const { return year * 100 + week; } -uint8_t VecDateTimeValue::calc_weekday(uint64_t day_nr, bool is_sunday_first_day) { - return (day_nr + 5L + (is_sunday_first_day ? 1L : 0L)) % 7; +bool VecDateTimeValue::operator>=(const DateV2Value& other) const { + int64_t ts1; + int64_t ts2; + this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone); + other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone); + return ts1 >= ts2; +} + +bool VecDateTimeValue::operator<=(const DateV2Value& other) const { + int64_t ts1; + int64_t ts2; + this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone); + other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone); + return ts1 <= ts2; +} + +bool VecDateTimeValue::operator>(const DateV2Value& other) const { + int64_t ts1; + int64_t ts2; + this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone); + other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone); + return ts1 > ts2; +} + +bool VecDateTimeValue::operator<(const DateV2Value& other) const { + int64_t ts1; + int64_t ts2; + this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone); + other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone); + return ts1 < ts2; +} + +bool VecDateTimeValue::operator==(const DateV2Value& other) const { + int64_t ts1; + int64_t ts2; + this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone); + other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone); + return ts1 == ts2; } // TODO(zhaochun): Think endptr is NULL @@ -1430,6 +1440,12 @@ bool VecDateTimeValue::from_date_format_str(const char* format, int format_len, return check_range_and_set_time(year, month, day, hour, minute, second, _type); } +int64_t VecDateTimeValue::second_diff(const DateV2Value& rhs) const { + int day_diff = daynr() - rhs.daynr(); + int time_diff = hour() * 3600 + minute() * 60 + second(); + return day_diff * 3600 * 24 + time_diff; +} + bool VecDateTimeValue::date_add_interval(const TimeInterval& interval, TimeUnit unit) { if (!is_valid_date()) return false; @@ -1585,6 +1601,12 @@ void VecDateTimeValue::set_time(uint32_t year, uint32_t month, uint32_t day, uin _second = second; } +void VecDateTimeValue::create_from_date_v2(DateV2Value& value, TimeType type) { + this->set_time(value.year(), value.month(), value.day(), 0, 0, 0); + this->set_type(type); + this->_neg = 0; +} + void VecDateTimeValue::convert_vec_dt_to_dt( doris::DateTimeValue* dt) { //use convert VecDateTimeValue to DateTimeValue dt->_neg = this->_neg; @@ -1622,8 +1644,1072 @@ std::size_t operator-(const VecDateTimeValue& v1, const VecDateTimeValue& v2) { return v1.daynr() - v2.daynr(); } +std::size_t operator-(const DateV2Value& v1, const VecDateTimeValue& v2) { + return v1.daynr() - v2.daynr(); +} + +std::size_t operator-(const VecDateTimeValue& v1, const DateV2Value& v2) { + return v1.daynr() - v2.daynr(); +} + std::size_t hash_value(VecDateTimeValue const& value) { return HashUtil::hash(&value, sizeof(VecDateTimeValue), 0); } +bool DateV2Value::is_invalid(uint32_t year, uint32_t month, uint32_t day) { + if (month == 2 && day == 29 && is_leap(year)) return false; + if (year < MIN_YEAR || year > MAX_YEAR || month == 0 || month > 12 || + day > s_days_in_month[month] || day == 0) { + return true; + } + return false; +} + +// The interval format is that with no delimiters +// YYYY-MM-DD HH-MM-DD.FFFFFF AM in default format +// 0 1 2 3 4 5 6 7 +bool DateV2Value::from_date_str(const char* date_str, int len) { + const char* ptr = date_str; + const char* end = date_str + len; + // ONLY 2, 6 can follow by a sapce + const static int allow_space_mask = 4 | 64; + const static int MAX_DATE_PARTS = 3; + uint32_t date_val[MAX_DATE_PARTS]; + int32_t date_len[MAX_DATE_PARTS] = {0, 0, 0}; + + // Skip space character + while (ptr < end && isspace(*ptr)) { + ptr++; + } + if (ptr == end || !isdigit(*ptr)) { + return false; + } + // Fix year length + const char* pos = ptr; + while (pos < end && (isdigit(*pos) || *pos == 'T')) { + pos++; + } + int year_len = 4; + int digits = pos - ptr; + bool is_interval_format = false; + + // Compatible with MySQL. Shit!!! + // For YYYYMMDD/YYYYMMDDHHMMSS is 4 digits years + if (pos == end || *pos == '.') { + if (digits == 4 || digits == 8 || digits >= 14) { + year_len = 4; + } else { + year_len = 2; + } + is_interval_format = true; + } + + int field_idx = 0; + int field_len = year_len; + while (ptr < end && isdigit(*ptr) && field_idx < MAX_DATE_PARTS) { + const char* start = ptr; + int temp_val = 0; + bool scan_to_delim = (!is_interval_format) && (field_idx != 6); + while (ptr < end && isdigit(*ptr) && (scan_to_delim || field_len--)) { + temp_val = temp_val * 10 + (*ptr++ - '0'); + } + // Imposible + if (temp_val > 999999L) { + return false; + } + date_val[field_idx] = temp_val; + date_len[field_idx] = ptr - start; + field_len = 2; + + if (ptr == end) { + field_idx++; + break; + } + if (field_idx == 2 && *ptr == 'T') { + // YYYYMMDDTHHMMDD, skip 'T' and continue + ptr++; + field_idx++; + continue; + } + + // Second part + if (field_idx == 5) { + if (*ptr == '.') { + ptr++; + field_len = 6; + } else if (isdigit(*ptr)) { + field_idx++; + break; + } + field_idx++; + continue; + } + // escape separator + while (ptr < end && (ispunct(*ptr) || isspace(*ptr))) { + if (isspace(*ptr)) { + if (((1 << field_idx) & allow_space_mask) == 0) { + return false; + } + } + ptr++; + } + field_idx++; + } + int num_field = field_idx; + if (!is_interval_format) { + year_len = date_len[0]; + } + for (; field_idx < MAX_DATE_PARTS; ++field_idx) { + date_len[field_idx] = 0; + date_val[field_idx] = 0; + } + + if (year_len == 2) { + if (date_val[0] < YY_PART_YEAR) { + date_val[0] += 2000; + } else { + date_val[0] += 1900; + } + } + + if (num_field > 3) { + return false; + } + return check_range_and_set_time(date_val[0], date_val[1], date_val[2]); +} + +void DateV2Value::set_zero(int type) { + date_v2_value_.month_ = 0; + date_v2_value_.year_ = 0; + date_v2_value_.day_ = 0; +} + +// this method is exactly same as fromDateFormatStr() in DateLiteral.java in FE +// change this method should also change that. +bool DateV2Value::from_date_format_str(const char* format, int format_len, const char* value, + int value_len, const char** sub_val_end) { + const char* ptr = format; + const char* end = format + format_len; + const char* val = value; + const char* val_end = value + value_len; + + int day_part = 0; + int weekday = -1; + int yearday = -1; + int week_num = -1; + + bool strict_week_number = false; + bool sunday_first = false; + bool strict_week_number_year_type = false; + int strict_week_number_year = -1; + bool usa_time = false; + + // TODO: should we process [hour minute second] for datev2 type here? + auto [year, month, day, hour, minute, second] = std::tuple {0, 0, 0, 0, 0, 0}; + while (ptr < end && val < val_end) { + // Skip space character + while (val < val_end && isspace(*val)) { + val++; + } + if (val >= val_end) { + break; + } + // Check switch + if (*ptr == '%' && ptr + 1 < end) { + const char* tmp = NULL; + int64_t int_value = 0; + ptr++; + switch (*ptr++) { + // Year + case 'y': + // Year, numeric (two digits) + tmp = val + min(2, val_end - val); + if (!str_to_int64(val, &tmp, &int_value)) { + return false; + } + int_value += int_value >= 70 ? 1900 : 2000; + year = int_value; + val = tmp; + break; + case 'Y': + // Year, numeric, four digits + tmp = val + min(4, val_end - val); + if (!str_to_int64(val, &tmp, &int_value)) { + return false; + } + if (tmp - val <= 2) { + int_value += int_value >= 70 ? 1900 : 2000; + } + year = int_value; + val = tmp; + break; + // Month + case 'm': + case 'c': + tmp = val + min(2, val_end - val); + if (!str_to_int64(val, &tmp, &int_value)) { + return false; + } + month = int_value; + val = tmp; + break; + case 'M': + int_value = check_word(const_cast(s_month_name), val, val_end, &val); + if (int_value < 0) { + return false; + } + month = int_value; + break; + case 'b': + int_value = check_word(s_ab_month_name, val, val_end, &val); + if (int_value < 0) { + return false; + } + month = int_value; + break; + // Day + case 'd': + case 'e': + tmp = val + min(2, val_end - val); + if (!str_to_int64(val, &tmp, &int_value)) { + return false; + } + day = int_value; + val = tmp; + break; + case 'D': + tmp = val + min(2, val_end - val); + if (!str_to_int64(val, &tmp, &int_value)) { + return false; + } + day = int_value; + val = tmp + min(2, val_end - tmp); + break; + // Hour + case 'h': + case 'I': + case 'l': + usa_time = true; + // Fall through + case 'k': + case 'H': + tmp = val + min(2, val_end - val); + if (!str_to_int64(val, &tmp, &int_value)) { + return false; + } + hour = int_value; + val = tmp; + break; + // Minute + case 'i': + tmp = val + min(2, val_end - val); + if (!str_to_int64(val, &tmp, &int_value)) { + return false; + } + minute = int_value; + val = tmp; + break; + // Second + case 's': + case 'S': + tmp = val + min(2, val_end - val); + if (!str_to_int64(val, &tmp, &int_value)) { + return false; + } + second = int_value; + val = tmp; + break; + // Micro second + case 'f': + break; + // AM/PM + case 'p': + if ((val_end - val) < 2 || toupper(*(val + 1)) != 'M' || !usa_time) { + return false; + } + if (toupper(*val) == 'P') { + // PM + day_part = 12; + } + val += 2; + break; + // Weekday + case 'W': + int_value = check_word(const_cast(s_day_name), val, val_end, &val); + if (int_value < 0) { + return false; + } + int_value++; + weekday = int_value; + break; + case 'a': + int_value = check_word(s_ab_day_name, val, val_end, &val); + if (int_value < 0) { + return false; + } + int_value++; + weekday = int_value; + break; + case 'w': + tmp = val + min(1, val_end - val); + if (!str_to_int64(val, &tmp, &int_value)) { + return false; + } + if (int_value >= 7) { + return false; + } + if (int_value == 0) { + int_value = 7; + } + weekday = int_value; + val = tmp; + break; + case 'j': + tmp = val + min(3, val_end - val); + if (!str_to_int64(val, &tmp, &int_value)) { + return false; + } + yearday = int_value; + val = tmp; + break; + case 'u': + case 'v': + case 'U': + case 'V': + sunday_first = (*(ptr - 1) == 'U' || *(ptr - 1) == 'V'); + // Used to check if there is %x or %X + strict_week_number = (*(ptr - 1) == 'V' || *(ptr - 1) == 'v'); + tmp = val + min(2, val_end - val); + if (!str_to_int64(val, &tmp, &int_value)) { + return false; + } + week_num = int_value; + if (week_num > 53 || (strict_week_number && week_num == 0)) { + return false; + } + val = tmp; + break; + // strict week number, must be used with %V or %v + case 'x': + case 'X': + strict_week_number_year_type = (*(ptr - 1) == 'X'); + tmp = val + min(4, val_end - val); + if (!str_to_int64(val, &tmp, &int_value)) { + return false; + } + strict_week_number_year = int_value; + val = tmp; + break; + case 'r': + if (!from_date_format_str("%I:%i:%S %p", 11, val, val_end - val, &tmp)) { + return false; + } + val = tmp; + break; + case 'T': + if (!from_date_format_str("%H:%i:%S", 8, val, val_end - val, &tmp)) { + return false; + } + val = tmp; + break; + case '.': + while (val < val_end && ispunct(*val)) { + val++; + } + break; + case '@': + while (val < val_end && isalpha(*val)) { + val++; + } + break; + case '#': + while (val < val_end && isdigit(*val)) { + val++; + } + break; + case '%': // %%, escape the % + if ('%' != *val) { + return false; + } + val++; + break; + default: + return false; + } + } else if (!isspace(*ptr)) { + if (*ptr != *val) { + return false; + } + ptr++; + val++; + } else { + ptr++; + } + } + + // continue to iterate pattern if has + // to find out if it has time part. + while (ptr < end) { + if (*ptr == '%' && ptr + 1 < end) { + ptr++; + switch (*ptr++) { + case 'H': + case 'h': + case 'I': + case 'i': + case 'k': + case 'l': + case 'r': + case 's': + case 'S': + case 'p': + case 'T': + break; + default: + break; + } + } else { + ptr++; + } + } + + if (usa_time) { + if (hour > 12 || hour < 1) { + return false; + } + hour = (hour % 12) + day_part; + } + if (sub_val_end) { + *sub_val_end = val; + } + + // Year day + if (yearday > 0) { + uint64_t days = calc_daynr(year, 1, 1) + yearday - 1; + if (!get_date_from_daynr(days)) { + return false; + } + } + // weekday + if (week_num >= 0 && weekday > 0) { + // Check + if ((strict_week_number && + (strict_week_number_year < 0 || strict_week_number_year_type != sunday_first)) || + (!strict_week_number && strict_week_number_year >= 0)) { + return false; + } + uint64_t days = calc_daynr(strict_week_number ? strict_week_number_year : year, 1, 1); + + uint8_t weekday_b = calc_weekday(days, sunday_first); + + if (sunday_first) { + days += ((weekday_b == 0) ? 0 : 7) - weekday_b + (week_num - 1) * 7 + weekday % 7; + } else { + days += ((weekday_b <= 3) ? 0 : 7) - weekday_b + (week_num - 1) * 7 + weekday - 1; + } + if (!get_date_from_daynr(days)) { + return false; + } + } + return check_range_and_set_time(year, month, day); +} + +int32_t DateV2Value::to_buffer(char* buffer) const { + char* start = buffer; + uint32_t temp; + // Year + temp = date_v2_value_.year_ / 100; + *buffer++ = (char)('0' + (temp / 10)); + *buffer++ = (char)('0' + (temp % 10)); + temp = date_v2_value_.year_ % 100; + *buffer++ = (char)('0' + (temp / 10)); + *buffer++ = (char)('0' + (temp % 10)); + *buffer++ = '-'; + // Month + *buffer++ = (char)('0' + (date_v2_value_.month_ / 10)); + *buffer++ = (char)('0' + (date_v2_value_.month_ % 10)); + *buffer++ = '-'; + // Day + *buffer++ = (char)('0' + (date_v2_value_.day_ / 10)); + *buffer++ = (char)('0' + (date_v2_value_.day_ % 10)); + return buffer - start; +} + +char* DateV2Value::to_string(char* to) const { + int len = to_buffer(to); + *(to + len) = '\0'; + return to + len + 1; +} + +uint32_t DateV2Value::to_date_uint32() const { + return int_val_; +} + +uint32_t DateV2Value::set_date_uint32(uint32_t int_val) { + union DateV2UInt32Union { + doris::vectorized::DateV2Value dt; + uint32_t ui32; + ~DateV2UInt32Union() {} + }; + DateV2UInt32Union conv = {.ui32 = int_val}; + if (is_invalid(conv.dt.year(), conv.dt.month(), conv.dt.day())) { + return 0; + } + this->set_time(conv.dt.year(), conv.dt.month(), conv.dt.day()); + + return int_val; +} + +uint8_t DateV2Value::week(uint8_t mode) const { + uint16_t year = 0; + return calc_week(this->daynr(), this->year(), this->month(), this->day(), mode, &year); +} + +uint32_t DateV2Value::year_week(uint8_t mode) const { + uint16_t year = 0; + // The range of the week in the year_week is 1-53, so the mode WEEK_YEAR is always true. + uint8_t week = calc_week(this->daynr(), this->year(), this->month(), this->day(), mode, &year); + // When the mode WEEK_FIRST_WEEKDAY is not set, + // the week in which the last three days of the year fall may belong to the following year. + if (week == 53 && day() >= 29 && !(mode & 4)) { + uint8_t monday_first = mode & WEEK_MONDAY_FIRST; + uint64_t daynr_of_last_day = calc_daynr(this->year(), 12, 31); + uint8_t weekday_of_last_day = calc_weekday(daynr_of_last_day, !monday_first); + + if (weekday_of_last_day - monday_first < 2) { + ++year; + week = 1; + } + } + return year * 100 + week; +} + +bool DateV2Value::get_date_from_daynr(uint64_t daynr) { + if (daynr <= 0 || daynr > DATE_MAX_DAYNR) { + return false; + } + + auto [year, month, day] = std::tuple {0, 0, 0}; + year = daynr / 365; + uint32_t days_befor_year = 0; + while (daynr < (days_befor_year = calc_daynr(year, 1, 1))) { + year--; + } + uint32_t days_of_year = daynr - days_befor_year + 1; + int leap_day = 0; + if (is_leap(year)) { + if (days_of_year > 31 + 28) { + days_of_year--; + if (days_of_year == 31 + 28) { + leap_day = 1; + } + } + } + month = 1; + while (days_of_year > s_days_in_month[month]) { + days_of_year -= s_days_in_month[month]; + month++; + } + day = days_of_year + leap_day; + + if (is_invalid(year, month, day)) { + return false; + } + set_time(year, month, day); + return true; +} + +bool DateV2Value::date_add_interval(const TimeInterval& interval, TimeUnit unit) { + if (!is_valid_date()) return false; + + switch (unit) { + case SECOND: + case MINUTE: + case HOUR: + case SECOND_MICROSECOND: + case MINUTE_MICROSECOND: + case MINUTE_SECOND: + case HOUR_MICROSECOND: + case HOUR_SECOND: + case HOUR_MINUTE: + case DAY_MICROSECOND: + case DAY_SECOND: + case DAY_MINUTE: + case DAY_HOUR: + case DAY: + case WEEK: { + // This only change day information, not change second information + uint32_t day_nr = daynr() + interval.day; + if (!get_date_from_daynr(day_nr)) { + return false; + } + break; + } + case YEAR: { + // This only change year information + date_v2_value_.year_ += interval.year; + if (date_v2_value_.year_ > 9999) { + return false; + } + if (date_v2_value_.month_ == 2 && date_v2_value_.day_ == 29 && + !is_leap(date_v2_value_.year_)) { + date_v2_value_.day_ = 28; + } + break; + } + case MONTH: + case QUARTER: + case YEAR_MONTH: { + // This will change month and year information, maybe date. + int64_t months = date_v2_value_.year_ * 12 + date_v2_value_.month_ - 1 + + 12 * interval.year + interval.month; + date_v2_value_.year_ = months / 12; + if (date_v2_value_.year_ > 9999) { + return false; + } + date_v2_value_.month_ = (months % 12) + 1; + if (date_v2_value_.day_ > s_days_in_month[date_v2_value_.month_]) { + date_v2_value_.day_ = s_days_in_month[date_v2_value_.month_]; + if (date_v2_value_.month_ == 2 && is_leap(date_v2_value_.year_)) { + date_v2_value_.day_++; + } + } + break; + } + } + return true; +} + +bool DateV2Value::unix_timestamp(int64_t* timestamp, const std::string& timezone) const { + cctz::time_zone ctz; + if (!TimezoneUtils::find_cctz_time_zone(timezone, ctz)) { + return false; + } + return unix_timestamp(timestamp, ctz); +} + +bool DateV2Value::unix_timestamp(int64_t* timestamp, const cctz::time_zone& ctz) const { + const auto tp = cctz::convert(cctz::civil_second(date_v2_value_.year_, date_v2_value_.month_, + date_v2_value_.day_, 0, 0, 0), + ctz); + *timestamp = tp.time_since_epoch().count(); + return true; +} + +bool DateV2Value::from_unixtime(int64_t timestamp, const std::string& timezone) { + cctz::time_zone ctz; + if (!TimezoneUtils::find_cctz_time_zone(timezone, ctz)) { + return false; + } + return from_unixtime(timestamp, ctz); +} + +bool DateV2Value::from_unixtime(int64_t timestamp, const cctz::time_zone& ctz) { + static const cctz::time_point epoch = + std::chrono::time_point_cast( + std::chrono::system_clock::from_time_t(0)); + cctz::time_point t = epoch + cctz::seconds(timestamp); + + const auto tp = cctz::convert(t, ctz); + + set_time(tp.year(), tp.month(), tp.day()); + return true; +} + +const char* DateV2Value::month_name() const { + if (date_v2_value_.month_ < 1 || date_v2_value_.month_ > 12) { + return nullptr; + } + return s_month_name[date_v2_value_.month_]; +} + +const char* DateV2Value::day_name() const { + int day = weekday(); + if (day < 0 || day >= 7) { + return nullptr; + } + return s_day_name[day]; +} + +void DateV2Value::set_time(uint16_t year, uint8_t month, uint8_t day) { + date_v2_value_.year_ = year; + date_v2_value_.month_ = month; + date_v2_value_.day_ = day; +} + +void DateV2Value::convert_date_v2_to_dt( + doris::DateTimeValue* dt) { //use convert VecDateTimeValue to DateTimeValue + dt->_neg = 0; + dt->_type = TIME_DATE; + dt->_hour = 0; + dt->_minute = 0; + dt->_second = 0; + dt->_year = date_v2_value_.year_; + dt->_month = date_v2_value_.month_; + dt->_day = date_v2_value_.day_; + dt->_microsecond = 0; +} + +void DateV2Value::convert_dt_to_date_v2(doris::DateTimeValue* dt) { + date_v2_value_.year_ = dt->_year; + date_v2_value_.month_ = dt->_month; + date_v2_value_.day_ = dt->_day; +} + +bool DateV2Value::to_format_string(const char* format, int len, char* to) const { + char buf[64]; + char* pos = nullptr; + const char* ptr = format; + const char* end = format + len; + char ch = '\0'; + + while (ptr < end) { + if (*ptr != '%' || (ptr + 1) == end) { + *to++ = *ptr++; + continue; + } + // Skip '%' + ptr++; + switch (ch = *ptr++) { + case 'a': + // Abbreviated weekday name + if (this->year() == 0 && this->month() == 0) { + return false; + } + to = append_string(s_ab_day_name[weekday()], to); + break; + case 'b': + // Abbreviated month name + if (this->month() == 0) { + return false; + } + to = append_string(s_ab_month_name[this->month()], to); + break; + case 'c': + // Month, numeric (0...12) + pos = int_to_str(this->month(), buf); + to = append_with_prefix(buf, pos - buf, '0', 1, to); + break; + case 'd': + // Day of month (00...31) + pos = int_to_str(this->day(), buf); + to = append_with_prefix(buf, pos - buf, '0', 2, to); + break; + case 'D': + // Day of the month with English suffix (0th, 1st, ...) + pos = int_to_str(this->day(), buf); + to = append_with_prefix(buf, pos - buf, '0', 1, to); + if (this->day() >= 10 && this->day() <= 19) { + to = append_string("th", to); + } else { + switch (this->day() % 10) { + case 1: + to = append_string("st", to); + break; + case 2: + to = append_string("nd", to); + break; + case 3: + to = append_string("rd", to); + break; + default: + to = append_string("th", to); + break; + } + } + break; + case 'e': + // Day of the month, numeric (0..31) + pos = int_to_str(this->day(), buf); + to = append_with_prefix(buf, pos - buf, '0', 1, to); + break; + case 'f': + // Microseconds (000000..999999) + pos = int_to_str(0, buf); + to = append_with_prefix(buf, pos - buf, '0', 6, to); + break; + case 'h': + case 'I': + // Hour (01..12) + pos = int_to_str(12, buf); + to = append_with_prefix(buf, pos - buf, '0', 2, to); + break; + case 'H': + // Hour (00..23) + pos = int_to_str(0, buf); + to = append_with_prefix(buf, pos - buf, '0', 2, to); + break; + case 'i': + // Minutes, numeric (00..59) + pos = int_to_str(0, buf); + to = append_with_prefix(buf, pos - buf, '0', 2, to); + break; + case 'j': + // Day of year (001..366) + pos = int_to_str(daynr() - calc_daynr(this->year(), 1, 1) + 1, buf); + to = append_with_prefix(buf, pos - buf, '0', 3, to); + break; + case 'k': + // Hour (0..23) + pos = int_to_str(0, buf); + to = append_with_prefix(buf, pos - buf, '0', 1, to); + break; + case 'l': + // Hour (1..12) + pos = int_to_str(12, buf); + to = append_with_prefix(buf, pos - buf, '0', 1, to); + break; + case 'm': + // Month, numeric (00..12) + pos = int_to_str(this->month(), buf); + to = append_with_prefix(buf, pos - buf, '0', 2, to); + break; + case 'M': + // Month name (January..December) + if (this->month() == 0) { + return false; + } + to = append_string(s_month_name[this->month()], to); + break; + case 'p': + // AM or PM + to = append_string("AM", to); + break; + case 'r': { + // Time, 12-hour (hh:mm:ss followed by AM or PM) + std::string time12_str("12:00:00 AM"); + memcpy(to, time12_str.data(), time12_str.size()); + to += time12_str.size(); + break; + } + case 's': + case 'S': + // Seconds (00..59) + pos = int_to_str(0, buf); + to = append_with_prefix(buf, pos - buf, '0', 2, to); + break; + case 'T': { + // Time, 24-hour (hh:mm:ss) + std::string time24_str("00:00:00"); + memcpy(to, time24_str.data(), time24_str.size()); + to += time24_str.size(); + break; + } + case 'u': + // Week (00..53), where Monday is the first day of the week; + // WEEK() mode 1 + pos = int_to_str(week(mysql_week_mode(1)), buf); + to = append_with_prefix(buf, pos - buf, '0', 2, to); + break; + case 'U': + // Week (00..53), where Sunday is the first day of the week; + // WEEK() mode 0 + pos = int_to_str(week(mysql_week_mode(0)), buf); + to = append_with_prefix(buf, pos - buf, '0', 2, to); + break; + case 'v': + // Week (01..53), where Monday is the first day of the week; + // WEEK() mode 3; used with %x + pos = int_to_str(week(mysql_week_mode(3)), buf); + to = append_with_prefix(buf, pos - buf, '0', 2, to); + break; + case 'V': + // Week (01..53), where Sunday is the first day of the week; + // WEEK() mode 2; used with %X + pos = int_to_str(week(mysql_week_mode(2)), buf); + to = append_with_prefix(buf, pos - buf, '0', 2, to); + break; + case 'w': + // Day of the week (0=Sunday..6=Saturday) + if (this->month() == 0 && this->year() == 0) { + return false; + } + pos = int_to_str(calc_weekday(daynr(), true), buf); + to = append_with_prefix(buf, pos - buf, '0', 1, to); + break; + case 'W': + // Weekday name (Sunday..Saturday) + to = append_string(s_day_name[weekday()], to); + break; + case 'x': { + // Year for the week, where Monday is the first day of the week, + // numeric, four digits; used with %v + uint16_t year = 0; + calc_week(this->daynr(), this->year(), this->month(), this->day(), mysql_week_mode(3), + &year); + pos = int_to_str(year, buf); + to = append_with_prefix(buf, pos - buf, '0', 4, to); + break; + } + case 'X': { + // Year for the week where Sunday is the first day of the week, + // numeric, four digits; used with %V + uint16_t year = 0; + calc_week(this->daynr(), this->year(), this->month(), this->day(), mysql_week_mode(3), + &year); + pos = int_to_str(year, buf); + to = append_with_prefix(buf, pos - buf, '0', 4, to); + break; + } + case 'y': + // Year, numeric (two digits) + pos = int_to_str(this->year() % 100, buf); + to = append_with_prefix(buf, pos - buf, '0', 2, to); + break; + case 'Y': + // Year, numeric, four digits + pos = int_to_str(this->year(), buf); + to = append_with_prefix(buf, pos - buf, '0', 4, to); + break; + default: + *to++ = ch; + break; + } + } + *to++ = '\0'; + return true; +} + +bool DateV2Value::from_date(uint32_t value) { + if (value < MIN_DATE_V2 || value > MAX_DATE_V2) { + return false; + } + + return set_date_uint32(value); +} + +int64_t DateV2Value::standardize_timevalue(int64_t value) { + if (value <= 0) { + return 0; + } + if (value >= 10000101000000L) { + // 9999-99-99 99:99:99 + if (value > 99999999999999L) { + return 0; + } + + // between 1000-01-01 00:00:00L and 9999-99-99 99:99:99 + // all digits exist. + return value; + } + // 2000-01-01 + if (value < 101) { + return 0; + } + // two digits year. 2000 ~ 2069 + if (value <= (YY_PART_YEAR - 1) * 10000L + 1231L) { + return (value + 20000000L) * 1000000L; + } + // two digits year, invalid date + if (value < YY_PART_YEAR * 10000L + 101) { + return 0; + } + // two digits year. 1970 ~ 1999 + if (value <= 991231L) { + return (value + 19000000L) * 1000000L; + } + // TODO(zhaochun): Don't allow year betwen 1000-01-01 + if (value < 10000101) { + return 0; + } + // four digits years without hour. + if (value <= 99991231L) { + return value * 1000000L; + } + // below 0000-01-01 + if (value < 101000000) { + return 0; + } + + // below is with datetime, must have hh:mm:ss + // 2000 ~ 2069 + if (value <= (YY_PART_YEAR - 1) * 10000000000L + 1231235959L) { + return value + 20000000000000L; + } + if (value < YY_PART_YEAR * 10000000000L + 101000000L) { + return 0; + } + // 1970 ~ 1999 + if (value <= 991231235959L) { + return value + 19000000000000L; + } + return value; +} + +bool DateV2Value::from_date_int64(int64_t value) { + value = standardize_timevalue(value); + if (value <= 0) { + return false; + } + uint64_t date = value / 1000000; + + auto [year, month, day] = std::tuple {0, 0, 0}; + year = date / 10000; + date %= 10000; + month = date / 100; + day = date % 100; + + return check_range_and_set_time(year, month, day); +} + +uint8_t DateV2Value::calc_week(const uint32_t& day_nr, const uint16_t& year, const uint8_t& month, + const uint8_t& day, uint8_t mode, uint16_t* to_year) { + bool monday_first = mode & WEEK_MONDAY_FIRST; + bool week_year = mode & WEEK_YEAR; + bool first_weekday = mode & WEEK_FIRST_WEEKDAY; + uint64_t daynr_first_day = calc_daynr(year, 1, 1); + uint8_t weekday_first_day = calc_weekday(daynr_first_day, !monday_first); + + int days = 0; + *to_year = year; + + // Check wether the first days of this year belongs to last year + if (month == 1 && day <= (7 - weekday_first_day)) { + if (!week_year && ((first_weekday && weekday_first_day != 0) || + (!first_weekday && weekday_first_day > 3))) { + return 0; + } + (*to_year)--; + week_year = true; + daynr_first_day -= (days = calc_days_in_year(*to_year)); + weekday_first_day = (weekday_first_day + 53 * 7 - days) % 7; + } + + // How many days since first week + if ((first_weekday && weekday_first_day != 0) || (!first_weekday && weekday_first_day > 3)) { + // days in new year belongs to last year. + days = day_nr - (daynr_first_day + (7 - weekday_first_day)); + } else { + // days in new year belongs to this year. + days = day_nr - (daynr_first_day - weekday_first_day); + } + + if (week_year && days >= 52 * 7) { + weekday_first_day = (weekday_first_day + calc_days_in_year(*to_year)) % 7; + if ((first_weekday && weekday_first_day == 0) || + (!first_weekday && weekday_first_day <= 3)) { + // Belong to next year. + (*to_year)++; + return 1; + } + } + + return days / 7 + 1; +} + +std::ostream& operator<<(std::ostream& os, const DateV2Value& value) { + char buf[11]; + value.to_string(buf); + return os << buf; +} + +// NOTE: +// only support DATE - DATE (no support DATETIME - DATETIME) +std::size_t operator-(const DateV2Value& v1, const DateV2Value& v2) { + return v1.daynr() - v2.daynr(); +} + +std::size_t hash_value(DateV2Value const& value) { + return HashUtil::hash(&value, sizeof(DateV2Value), 0); +} } // namespace doris::vectorized diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h index 4c1f4cc137..3dd325c72f 100644 --- a/be/src/vec/runtime/vdatetime_value.h +++ b/be/src/vec/runtime/vdatetime_value.h @@ -142,7 +142,49 @@ static constexpr const char* s_day_name[] = {"Monday", "Tuesday", "Wednesday", static constexpr size_t MAX_DAY_NAME_LEN = max_char_length(s_day_name, std::size(s_day_name)); static constexpr size_t MAX_MONTH_NAME_LEN = max_char_length(s_month_name, std::size(s_month_name)); +const uint32_t MAX_DATE_V2 = 31 | (12 << 8) | (9999 << 16); +const uint32_t MIN_DATE_V2 = 1 | (1 << 8) | (1000 << 16); + +const uint32_t MAX_YEAR = 9999; +const uint32_t MIN_YEAR = 1000; + +static RE2 time_zone_offset_format_reg("^[+-]{1}\\d{2}\\:\\d{2}$"); + uint8_t mysql_week_mode(uint32_t mode); +bool is_leap(uint32_t year); +uint32_t calc_days_in_year(uint32_t year); + +// Calculate how many days since 0000-01-01 +// 0000-01-01 is 1st B.C. +static uint32_t calc_daynr(uint16_t year, uint8_t month, uint8_t day) { + uint32_t delsum = 0; + int y = year; + + if (year == 0 && month == 0) { + return 0; + } + + /* Cast to int to be able to handle month == 0 */ + delsum = 365 * y + 31 * (month - 1) + day; + if (month <= 2) { + // No leap year + y--; + } else { + // This is great!!! + // 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 + // 0, 0, 3, 3, 4, 4, 5, 5, 5, 6, 7, 8 + delsum -= (month * 4 + 23) / 10; + } + // Every 400 year has 97 leap year, 100, 200, 300 are not leap year. + return delsum + y / 4 - y / 100 + y / 400; +} + +//W = (D + M*2 + 3*(M+1)/5 + Y + Y/4 -Y/100 + Y/400)%7 +static uint8_t calc_weekday(uint64_t day_nr, bool is_sunday_first_day) { + return (day_nr + 5L + (is_sunday_first_day ? 1L : 0L)) % 7; +} + +class DateV2Value; class VecDateTimeValue { // Now this type is a temp solution with little changes, maybe large refactoring follow-up. public: @@ -179,6 +221,8 @@ public: return datetime; } + void create_from_date_v2(DateV2Value& value, TimeType type); + void set_time(uint32_t year, uint32_t month, uint32_t day, uint32_t hour, uint32_t minute, uint32_t second); @@ -268,6 +312,8 @@ public: // Return true if convert success. Otherwise return false. bool from_date_int64(int64_t value); + bool from_date(int64_t value) { return from_date_int64(value); }; + // Construct time type value from int64_t value. // Return true if convert success. Otherwise return false. bool from_time_int64(int64_t value); @@ -293,71 +339,6 @@ public: static bool check_date(uint32_t year, uint32_t month, uint32_t day); - // compute the diff between two datetime value - template - static int64_t datetime_diff(const VecDateTimeValue& ts_value1, - const VecDateTimeValue& ts_value2) { - switch (unit) { - case YEAR: { - int year = (ts_value2.year() - ts_value1.year()); - if (year > 0) { - year -= (ts_value2.to_datetime_int64() % 10000000000 - - ts_value1.to_datetime_int64() % 10000000000) < 0; - } else if (year < 0) { - year += (ts_value2.to_datetime_int64() % 10000000000 - - ts_value1.to_datetime_int64() % 10000000000) > 0; - } - return year; - } - case MONTH: { - int month = (ts_value2.year() - ts_value1.year()) * 12 + - (ts_value2.month() - ts_value1.month()); - if (month > 0) { - month -= (ts_value2.to_datetime_int64() % 100000000 - - ts_value1.to_datetime_int64() % 100000000) < 0; - } else if (month < 0) { - month += (ts_value2.to_datetime_int64() % 100000000 - - ts_value1.to_datetime_int64() % 100000000) > 0; - } - return month; - } - case WEEK: { - int day = ts_value2.daynr() - ts_value1.daynr(); - if (day > 0) { - day -= ts_value2.time_part_diff(ts_value1) < 0; - } else if (day < 0) { - day += ts_value2.time_part_diff(ts_value1) > 0; - } - return day / 7; - } - case DAY: { - int day = ts_value2.daynr() - ts_value1.daynr(); - if (day > 0) { - day -= ts_value2.time_part_diff(ts_value1) < 0; - } else if (day < 0) { - day += ts_value2.time_part_diff(ts_value1) > 0; - } - return day; - } - case HOUR: { - int64_t second = ts_value2.second_diff(ts_value1); - int64_t hour = second / 60 / 60; - return hour; - } - case MINUTE: { - int64_t second = ts_value2.second_diff(ts_value1); - int64_t minute = second / 60; - return minute; - } - case SECOND: { - int64_t second = ts_value2.second_diff(ts_value1); - return second; - } - } - // Rethink the default return value - return 0; - } - // Convert this value to uint64_t // Will check its type int64_t to_int64() const; @@ -373,13 +354,6 @@ public: uint64_t daynr() const { return calc_daynr(_year, _month, _day); } - // Calculate how many days since 0000-01-01 - // 0000-01-01 is 1st B.C. - static uint64_t calc_daynr(uint32_t year, uint32_t month, uint32_t day); - - static uint8_t calc_weekday(uint64_t daynr, - bool); //W = (D + M*2 + 3*(M+1)/5 + Y + Y/4 -Y/100 + Y/400)%7 - int year() const { return _year; } int month() const { return _month; } int quarter() const { return (_month - 1) / 3 + 1; } @@ -493,6 +467,18 @@ public: return v1 > v2; } + bool operator==(const DateV2Value& other) const; + + bool operator!=(const DateV2Value& other) const { return !(*this == other); }; + + bool operator<=(const DateV2Value& other) const; + + bool operator>=(const DateV2Value& other) const; + + bool operator<(const DateV2Value& other) const; + + bool operator>(const DateV2Value& other) const; + const char* month_name() const; const char* day_name() const; @@ -523,6 +509,11 @@ public: tv->type = _type; } + uint32_t to_date_v2() const { + CHECK(_type == TIME_DATE); + return (year() << 16 | month() << 8 | day()); + }; + static VecDateTimeValue from_datetime_val(const doris_udf::DateTimeVal& tv) { VecDateTimeValue value; value.from_packed_time(tv.packed_time); @@ -562,6 +553,8 @@ public: return day_diff * 3600 * 24 + time_diff; } + int64_t second_diff(const DateV2Value& rhs) const; + int64_t time_part_diff(const VecDateTimeValue& rhs) const { int time_diff = (hour() * 3600 + minute() * 60 + second()) - (rhs.hour() * 3600 + rhs.minute() * 60 + rhs.second()); @@ -579,6 +572,7 @@ public: void convert_vec_dt_to_dt(doris::DateTimeValue* dt); void convert_dt_to_vec_dt(doris::DateTimeValue* dt); + int64_t to_datetime_int64() const; private: // Used to make sure sizeof VecDateTimeValue @@ -629,8 +623,6 @@ private: char* to_date_buffer(char* to) const; char* to_time_buffer(char* to) const; - // Used to convert to int64_t - int64_t to_datetime_int64() const; int64_t to_date_int64() const; int64_t to_time_int64() const; @@ -666,18 +658,532 @@ private: _day(day), _month(month), _year(year) {} +}; - // RE2 obj is thread safe - static RE2 time_zone_offset_format_reg; +struct DateV2ValueType { + uint32_t day_ : 8; + uint32_t month_ : 8; + uint32_t year_ : 16; + + DateV2ValueType(uint16_t year, uint8_t month, uint8_t day) + : day_(day), month_(month), year_(year) {} +}; + +class DateV2Value { // Now this type is a temp solution with little changes, maybe large refactoring follow-up. +public: + // Constructor + DateV2Value() : date_v2_value_(0, 0, 0) {} + + DateV2Value(DateV2Value& other) { int_val_ = other.to_date_uint32(); } + + DateV2Value(const DateV2Value& other) { int_val_ = other.to_date_uint32(); } + + static DateV2Value create_from_olap_date(uint64_t value) { + DateV2Value date; + date.from_olap_date(value); + return date; + } + + void set_time(uint16_t year, uint8_t month, uint8_t day); + + // TODO(gabriel): unify execution and storage + bool from_olap_date(uint64_t date) { + auto [year, month, day] = std::tuple {0, 0, 0}; + + day = date & 0x1f; + date >>= 5; + month = date & 0x0f; + date >>= 4; + year = date; + + return check_range_and_set_time(year, month, day); + } + + uint64_t to_olap_date() const { + uint64_t val; + val = date_v2_value_.year_; + val <<= 4; + val |= date_v2_value_.month_; + val <<= 5; + val |= date_v2_value_.day_; + return val; + } + + bool to_format_string(const char* format, int len, char* to) const; + + bool from_date_format_str(const char* format, int format_len, const char* value, + int value_len) { + return from_date_format_str(format, format_len, value, value_len, nullptr); + } + + // Construct Date/Datetime type value from string. + // At least the following formats are recognised (based on number of digits) + // 'YYMMDD', 'YYYYMMDD', 'YYMMDDHHMMSS', 'YYYYMMDDHHMMSS' + // 'YY-MM-DD', 'YYYY-MM-DD', 'YY-MM-DD HH.MM.SS' + // 'YYYYMMDDTHHMMSS' + bool from_date_str(const char* str, int len); + + // Convert this value to string + // this will check type to decide which format to convert + // TIME: format 'hh:mm:ss.xxxxxx' + // DATE: format 'YYYY-MM-DD' + // DATETIME: format 'YYYY-MM-DD hh:mm:ss.xxxxxx' + int32_t to_buffer(char* buffer) const; + + char* to_string(char* to) const; + + // Return true if range or date is invalid + static bool is_invalid(uint32_t year, uint32_t month, uint32_t day); + + bool check_range_and_set_time(uint16_t year, uint8_t month, uint8_t day) { + if (is_invalid(year, month, day)) { + return false; + } + set_time(year, month, day); + return true; + }; + + uint32_t daynr() const { + return calc_daynr(date_v2_value_.year_, date_v2_value_.month_, date_v2_value_.day_); + } + + int hour() const { return 0; } + int minute() const { return 0; } + int second() const { return 0; } + uint16_t year() const { return date_v2_value_.year_; } + uint8_t month() const { return date_v2_value_.month_; } + int quarter() const { return (date_v2_value_.month_ - 1) / 3 + 1; } + int week() const { return week(mysql_week_mode(0)); } //00-53 + uint8_t day() const { return date_v2_value_.day_; } + + // Weekday, from 0(Mon) to 6(Sun) + uint8_t weekday() const { return calc_weekday(daynr(), false); } + auto day_of_week() const { return (weekday() + 1) % 7 + 1; } + + // The bits in week_format has the following meaning: + // WEEK_MONDAY_FIRST (0) + // If not set: + // Sunday is first day of week + // If set: + // Monday is first day of week + // + // WEEK_YEAR (1) + // If not set: + // Week is in range 0-53 + // Week 0 is returned for the last week of the previous year (for + // a date at start of january) In this case one can get 53 for the + // first week of next year. This flag ensures that the week is + // relevant for the given year. Note that this flag is only + // relevant if WEEK_JANUARY is not set. + // If set: + // Week is in range 1-53. + // In this case one may get week 53 for a date in January (when + // the week is that last week of previous year) and week 1 for a + // date in December. + // + // WEEK_FIRST_WEEKDAY (2) + // If not set + // Weeks are numbered according to ISO 8601:1988 + // If set + // The week that contains the first 'first-day-of-week' is week 1. + // + // ISO 8601:1988 means that + // if the week containing January 1 has + // four or more days in the new year, then it is week 1; + // Otherwise it is the last week of the previous year, and the + // next week is week 1. + uint8_t week(uint8_t) const; + + uint32_t year_week(uint8_t mode) const; + + // Add interval + bool date_add_interval(const TimeInterval& interval, TimeUnit unit); + + //unix_timestamp is called with a timezone argument, + //it returns seconds of the value of date literal since '1970-01-01 00:00:00' UTC + bool unix_timestamp(int64_t* timestamp, const std::string& timezone) const; + bool unix_timestamp(int64_t* timestamp, const cctz::time_zone& ctz) const; + + //construct datetime_value from timestamp and timezone + //timestamp is an internal timestamp value representing seconds since '1970-01-01 00:00:00' UTC + bool from_unixtime(int64_t, const std::string& timezone); + bool from_unixtime(int64_t, const cctz::time_zone& ctz); + + bool operator==(const DateV2Value& other) const { + // NOTE: This is not same with MySQL. + // MySQL convert both to int with left value type and then compare + // We think all fields equals. + return this->to_date_uint32() == other.to_date_uint32(); + } + + bool operator==(const VecDateTimeValue& other) const { + int64_t ts1; + int64_t ts2; + this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone); + other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone); + return ts1 == ts2; + } + + bool operator!=(const DateV2Value& other) const { + return this->to_date_uint32() != other.to_date_uint32(); + } + + bool operator!=(const VecDateTimeValue& other) const { return !(*this == other); } + + bool operator<=(const DateV2Value& other) const { return !(*this > other); } + + bool operator<=(const VecDateTimeValue& other) const { + int64_t ts1; + int64_t ts2; + this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone); + other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone); + return ts1 <= ts2; + } + + bool operator>=(const DateV2Value& other) const { return !(*this < other); } + + bool operator>=(const VecDateTimeValue& other) const { + int64_t ts1; + int64_t ts2; + this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone); + other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone); + return ts1 >= ts2; + } + + bool operator<(const DateV2Value& other) const { + return to_date_uint32() < other.to_date_uint32(); + } + + bool operator<(const VecDateTimeValue& other) const { + int64_t ts1; + int64_t ts2; + this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone); + other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone); + return ts1 < ts2; + } + + bool operator>(const DateV2Value& other) const { + return to_date_uint32() > other.to_date_uint32(); + } + + bool operator>(const VecDateTimeValue& other) const { + int64_t ts1; + int64_t ts2; + this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone); + other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone); + return ts1 > ts2; + } + + DateV2Value& operator=(const DateV2Value& other) { + int_val_ = other.to_date_uint32(); + return *this; + } + + DateV2Value& operator=(DateV2Value& other) { + int_val_ = other.to_date_uint32(); + return *this; + } + + const char* month_name() const; + + const char* day_name() const; + + DateV2Value& operator++() { + TimeInterval interval(DAY, 1, false); + date_add_interval(interval, DAY); + return *this; + } + + uint32_t hash(int seed) const { return HashUtil::hash(this, sizeof(*this), seed); } + + int day_of_year() const { return daynr() - calc_daynr(this->year(), 1, 1) + 1; } + + std::string debug_string() const { + char buf[64]; + char* end = to_string(buf); + return std::string(buf, end - buf); + } + + bool is_valid_date() const { return !is_invalid(this->year(), this->month(), this->day()); } + + int64_t second_diff(const DateV2Value& rhs) const { + int day_diff = daynr() - rhs.daynr(); + return day_diff * 3600 * 24; + } + + int64_t second_diff(const VecDateTimeValue& rhs) const { + int day_diff = daynr() - rhs.daynr(); + int time_diff = rhs.hour() * 3600 + rhs.minute() * 60 + rhs.second(); + return day_diff * 3600 * 24 - time_diff; + }; + + void convert_date_v2_to_dt(doris::DateTimeValue* dt); + void convert_dt_to_date_v2(doris::DateTimeValue* dt); + + uint32_t to_date_uint32() const; + + bool from_date(uint32_t value); + bool from_date(int64_t value); + + bool from_date_int64(int64_t value); + uint32_t set_date_uint32(uint32_t int_val); + + bool get_date_from_daynr(uint64_t); + +private: + static uint8_t calc_week(const uint32_t& day_nr, const uint16_t& year, const uint8_t& month, + const uint8_t& day, uint8_t mode, uint16_t* to_year); + + // Used to construct from int value + int64_t standardize_timevalue(int64_t value); + + // Helper to set max, min, zero + void set_zero(int type); + + bool from_date_format_str(const char* format, int format_len, const char* value, int value_len, + const char** sub_val_end); + + union { + DateV2ValueType date_v2_value_; + uint32_t int_val_; + }; + + DateV2Value(uint16_t year, uint8_t month, uint8_t day) : date_v2_value_(year, month, day) {} }; // only support DATE - DATE (no support DATETIME - DATETIME) std::size_t operator-(const VecDateTimeValue& v1, const VecDateTimeValue& v2); +std::size_t operator-(const VecDateTimeValue& v1, const DateV2Value& v2); + +std::size_t operator-(const DateV2Value& v1, const VecDateTimeValue& v2); + std::ostream& operator<<(std::ostream& os, const VecDateTimeValue& value); std::size_t hash_value(VecDateTimeValue const& value); +std::size_t operator-(const DateV2Value& v1, const DateV2Value& v2); + +std::ostream& operator<<(std::ostream& os, const DateV2Value& value); + +std::size_t hash_value(DateV2Value const& value); + +template +int64_t datetime_diff(const VecDateTimeValue& ts_value1, const VecDateTimeValue& ts_value2) { + switch (unit) { + case YEAR: { + int year = (ts_value2.year() - ts_value1.year()); + if (year > 0) { + year -= (ts_value2.to_datetime_int64() % 10000000000 - + ts_value1.to_datetime_int64() % 10000000000) < 0; + } else if (year < 0) { + year += (ts_value2.to_datetime_int64() % 10000000000 - + ts_value1.to_datetime_int64() % 10000000000) > 0; + } + return year; + } + case MONTH: { + int month = (ts_value2.year() - ts_value1.year()) * 12 + + (ts_value2.month() - ts_value1.month()); + if (month > 0) { + month -= (ts_value2.to_datetime_int64() % 100000000 - + ts_value1.to_datetime_int64() % 100000000) < 0; + } else if (month < 0) { + month += (ts_value2.to_datetime_int64() % 100000000 - + ts_value1.to_datetime_int64() % 100000000) > 0; + } + return month; + } + case WEEK: { + int day = ts_value2.daynr() - ts_value1.daynr(); + if (day > 0) { + day -= ts_value2.time_part_diff(ts_value1) < 0; + } else if (day < 0) { + day += ts_value2.time_part_diff(ts_value1) > 0; + } + return day / 7; + } + case DAY: { + int day = ts_value2.daynr() - ts_value1.daynr(); + if (day > 0) { + day -= ts_value2.time_part_diff(ts_value1) < 0; + } else if (day < 0) { + day += ts_value2.time_part_diff(ts_value1) > 0; + } + return day; + } + case HOUR: { + int64_t second = ts_value2.second_diff(ts_value1); + int64_t hour = second / 60 / 60; + return hour; + } + case MINUTE: { + int64_t second = ts_value2.second_diff(ts_value1); + int64_t minute = second / 60; + return minute; + } + case SECOND: { + int64_t second = ts_value2.second_diff(ts_value1); + return second; + } + } + // Rethink the default return value + return 0; +} + +template +int64_t datetime_diff(const DateV2Value& ts_value1, const DateV2Value& ts_value2) { + switch (unit) { + case YEAR: { + int year = (ts_value2.year() - ts_value1.year()); + if (year > 0) { + year -= ((ts_value2.to_date_uint32() & 0x0000FFFF) - + (ts_value1.to_date_uint32() & 0x0000FFFF)) < 0; + } else if (year < 0) { + year += ((ts_value2.to_date_uint32() & 0x0000FFFF) - + (ts_value1.to_date_uint32() & 0x0000FFFF)) > 0; + } + return year; + } + case MONTH: { + int month = (ts_value2.year() - ts_value1.year()) * 12 + + (ts_value2.month() - ts_value1.month()); + if (month > 0) { + month -= (ts_value2.day() - ts_value1.day()) < 0; + } else if (month < 0) { + month += (ts_value2.day() - ts_value1.day()) > 0; + } + return month; + } + case WEEK: { + int day = ts_value2.daynr() - ts_value1.daynr(); + return day / 7; + } + case DAY: { + int day = ts_value2.daynr() - ts_value1.daynr(); + return day; + } + case HOUR: { + int64_t second = ts_value2.second_diff(ts_value1); + int64_t hour = second / 60 / 60; + return hour; + } + case MINUTE: { + int64_t second = ts_value2.second_diff(ts_value1); + int64_t minute = second / 60; + return minute; + } + case SECOND: { + int64_t second = ts_value2.second_diff(ts_value1); + return second; + } + } + // Rethink the default return value + return 0; +} + +template +int64_t datetime_diff(const DateV2Value& ts_value1, const VecDateTimeValue& ts_value2) { + switch (unit) { + case YEAR: { + int year = (ts_value2.year() - ts_value1.year()); + if (year > 0) { + year -= ((((ts_value2.to_datetime_int64() % 10000000000) / 1000000L) % 10000L) - + (ts_value1.to_date_uint32() & 0x0000FFFF)) < 0; + } else if (year < 0) { + year += ((((ts_value2.to_datetime_int64() % 10000000000) / 1000000L) % 10000L) - + (ts_value1.to_date_uint32() & 0x0000FFFF)) > 0; + } + return year; + } + case MONTH: { + int month = (ts_value2.year() - ts_value1.year()) * 12 + + (ts_value2.month() - ts_value1.month()); + if (month > 0) { + month -= (ts_value2.day() - ts_value1.day()) < 0; + } else if (month < 0) { + month += (ts_value2.day() - ts_value1.day()) > 0; + } + return month; + } + case WEEK: { + int day = ts_value2.daynr() - ts_value1.daynr(); + return day / 7; + } + case DAY: { + int day = ts_value2.daynr() - ts_value1.daynr(); + return day; + } + case HOUR: { + int64_t second = ts_value2.second_diff(ts_value1); + int64_t hour = second / 60 / 60; + return hour; + } + case MINUTE: { + int64_t second = ts_value2.second_diff(ts_value1); + int64_t minute = second / 60; + return minute; + } + case SECOND: { + int64_t second = ts_value2.second_diff(ts_value1); + return second; + } + } + // Rethink the default return value + return 0; +} + +template +int64_t datetime_diff(const VecDateTimeValue& ts_value1, const DateV2Value& ts_value2) { + switch (unit) { + case YEAR: { + int year = (ts_value2.year() - ts_value1.year()); + if (year > 0) { + year -= ((ts_value2.to_date_uint32() & 0x0000FFFF) - + (((ts_value1.to_datetime_int64() % 10000000000) / 1000000L) % 10000L)) < 0; + } else if (year < 0) { + year += ((ts_value2.to_date_uint32() & 0x0000FFFF) - + (((ts_value1.to_datetime_int64() % 10000000000) / 1000000L) % 10000L)) > 0; + } + return year; + } + case MONTH: { + int month = (ts_value2.year() - ts_value1.year()) * 12 + + (ts_value2.month() - ts_value1.month()); + if (month > 0) { + month -= (ts_value2.day() - ts_value1.day()) < 0; + } else if (month < 0) { + month += (ts_value2.day() - ts_value1.day()) > 0; + } + return month; + } + case WEEK: { + int day = ts_value2.daynr() - ts_value1.daynr(); + return day / 7; + } + case DAY: { + int day = ts_value2.daynr() - ts_value1.daynr(); + return day; + } + case HOUR: { + int64_t second = ts_value2.second_diff(ts_value1); + int64_t hour = second / 60 / 60; + return hour; + } + case MINUTE: { + int64_t second = ts_value2.second_diff(ts_value1); + int64_t minute = second / 60; + return minute; + } + case SECOND: { + int64_t second = ts_value2.second_diff(ts_value1); + return second; + } + } + // Rethink the default return value + return 0; +} + } // namespace vectorized } // namespace doris @@ -688,6 +1194,13 @@ struct hash { return doris::vectorized::hash_value(v); } }; + +template <> +struct hash { + size_t operator()(const doris::vectorized::DateV2Value& v) const { + return doris::vectorized::hash_value(v); + } +}; } // namespace std #endif diff --git a/be/src/vec/runtime/vfile_result_writer.cpp b/be/src/vec/runtime/vfile_result_writer.cpp index c1000193d8..30e32d8e7c 100644 --- a/be/src/vec/runtime/vfile_result_writer.cpp +++ b/be/src/vec/runtime/vfile_result_writer.cpp @@ -246,6 +246,14 @@ Status VFileResultWriter::_write_csv_file(const Block& block) { _plain_text_outstream << buffer; break; } + case TYPE_DATEV2: { + char buf[64]; + const DateV2Value* time_val = + (const DateV2Value*)(col.column->get_data_at(i).data); + time_val->to_string(buf); + _plain_text_outstream << buf; + break; + } case TYPE_DATE: case TYPE_DATETIME: { char buf[64]; diff --git a/be/src/vec/sink/mysql_result_writer.cpp b/be/src/vec/sink/mysql_result_writer.cpp index 1902bf9972..e9c3506dd6 100644 --- a/be/src/vec/sink/mysql_result_writer.cpp +++ b/be/src/vec/sink/mysql_result_writer.cpp @@ -200,7 +200,14 @@ Status VMysqlResultWriter::_add_one_column(const ColumnPtr& column_ptr, char* pos = time_val.to_string(buf); buf_ret = _buffer.push_string(buf, pos - buf - 1); } - + if constexpr (type == TYPE_DATEV2) { + char buf[64]; + auto time_num = data[i]; + doris::vectorized::DateV2Value date_val; + memcpy(static_cast(&date_val), &time_num, sizeof(UInt32)); + char* pos = date_val.to_string(buf); + buf_ret = _buffer.push_string(buf, pos - buf - 1); + } if constexpr (type == TYPE_DECIMALV2) { DecimalV2Value decimal_val(data[i]); auto decimal_str = decimal_val.to_string(); @@ -289,6 +296,14 @@ int VMysqlResultWriter::_add_one_cell(const ColumnPtr& column_ptr, size_t row_id DecimalV2Value decimal_val(column_data[row_idx]); auto decimal_str = decimal_val.to_string(); return buffer.push_string(decimal_str.c_str(), decimal_str.length()); + } else if (which.is_date_v2()) { + auto& column_vector = assert_cast&>(*column); + auto value = column_vector[row_idx].get(); + DateV2Value datev2; + memcpy(static_cast(&datev2), static_cast(&value), sizeof(value)); + char buf[64]; + char* pos = datev2.to_string(buf); + return buffer.push_string(buf, pos - buf - 1); } else if (which.is_array()) { auto& column_array = assert_cast(*column); auto& offsets = column_array.get_offsets(); @@ -453,6 +468,14 @@ Status VMysqlResultWriter::append_block(Block& input_block) { } break; } + case TYPE_DATEV2: { + if (type_ptr->is_nullable()) { + status = _add_one_column(column_ptr, result); + } else { + status = _add_one_column(column_ptr, result); + } + break; + } case TYPE_HLL: case TYPE_OBJECT: { if (type_ptr->is_nullable()) { diff --git a/be/src/vec/sink/vmysql_table_writer.cpp b/be/src/vec/sink/vmysql_table_writer.cpp index 0fb1cb89e4..6c7d33830e 100644 --- a/be/src/vec/sink/vmysql_table_writer.cpp +++ b/be/src/vec/sink/vmysql_table_writer.cpp @@ -186,6 +186,18 @@ Status VMysqlTableWriter::insert_row(vectorized::Block& block, size_t row) { fmt::format_to(_insert_stmt_buffer, "'{}'", str); break; } + case TYPE_DATEV2: { + uint32_t int_val = + assert_cast(*column).get_data()[row]; + vectorized::DateV2Value value = + binary_cast(int_val); + + char buf[64]; + char* pos = value.to_string(buf); + std::string str(buf, pos - buf - 1); + fmt::format_to(_insert_stmt_buffer, "'{}'", str); + break; + } default: { fmt::memory_buffer err_out; fmt::format_to(err_out, "can't convert this type to mysql type. type = {}", diff --git a/be/src/vec/utils/arrow_column_to_doris_column.cpp b/be/src/vec/utils/arrow_column_to_doris_column.cpp index 76741fdb59..5923f72f16 100644 --- a/be/src/vec/utils/arrow_column_to_doris_column.cpp +++ b/be/src/vec/utils/arrow_column_to_doris_column.cpp @@ -30,6 +30,7 @@ #include "gutil/casts.h" #include "vec/columns/column_array.h" #include "vec/columns/column_nullable.h" +#include "vec/data_types/data_type_array.h" #include "vec/data_types/data_type_decimal.h" #include "vec/runtime/vdatetime_value.h" @@ -213,6 +214,35 @@ static Status convert_column_with_timestamp_data(const arrow::Array* array, size return Status::OK(); } +template +static Status convert_column_with_date_v2_data(const arrow::Array* array, size_t array_idx, + MutableColumnPtr& data_column, size_t num_elements, + const std::string& timezone) { + auto& column_data = static_cast&>(*data_column).get_data(); + auto concrete_array = down_cast(array); + int64_t divisor = 1; + int64_t multiplier = 1; + if constexpr (std::is_same_v) { + const auto type = std::static_pointer_cast(array->type()); + divisor = time_unit_divisor(type->unit()); + if (divisor == 0L) { + return Status::InternalError(fmt::format("Invalid Time Type:{}", type->name())); + } + } else if constexpr (std::is_same_v) { + multiplier = 24 * 60 * 60; // day => secs + } else if constexpr (std::is_same_v) { + divisor = 1000; //ms => secs + } + + for (size_t value_i = array_idx; value_i < array_idx + num_elements; ++value_i) { + DateV2Value v; + v.from_unixtime(static_cast(concrete_array->Value(value_i)) / divisor * multiplier, + timezone); + column_data.emplace_back(binary_cast(v)); + } + return Status::OK(); +} + static Status convert_column_with_decimal_data(const arrow::Array* array, size_t array_idx, MutableColumnPtr& data_column, size_t num_elements) { auto& column_data = @@ -256,7 +286,8 @@ static Status convert_offset_from_list_column(const arrow::Array* array, size_t static Status convert_column_with_list_data(const arrow::Array* array, size_t array_idx, MutableColumnPtr& data_column, size_t num_elements, - const std::string& timezone) { + const std::string& timezone, + const DataTypePtr& nested_type) { size_t start_idx_of_data = 0; size_t num_of_data = 0; // get start idx and num of values from arrow offsets @@ -267,23 +298,20 @@ static Status convert_column_with_list_data(const arrow::Array* array, size_t ar std::shared_ptr arrow_data = concrete_array->values(); return arrow_column_to_doris_column(arrow_data.get(), start_idx_of_data, data_column_ptr, - num_of_data, timezone); + nested_type, num_of_data, timezone); } Status arrow_column_to_doris_column(const arrow::Array* arrow_column, size_t arrow_batch_cur_idx, - ColumnPtr& doirs_column, size_t num_elements, - const std::string& timezone) { + ColumnPtr& doris_column, const DataTypePtr& type, + size_t num_elements, const std::string& timezone) { // src column always be nullable for simpify converting - assert(doirs_column->is_nullable()); + CHECK(doris_column->is_nullable()); MutableColumnPtr data_column = nullptr; - if (doirs_column->is_nullable()) { - auto* nullable_column = reinterpret_cast( - (*std::move(doirs_column)).mutate().get()); - fill_nullable_column(arrow_column, arrow_batch_cur_idx, nullable_column, num_elements); - data_column = nullable_column->get_nested_column_ptr(); - } else { - data_column = (*std::move(doirs_column)).mutate(); - } + auto* nullable_column = reinterpret_cast( + (*std::move(doris_column)).mutate().get()); + fill_nullable_column(arrow_column, arrow_batch_cur_idx, nullable_column, num_elements); + data_column = nullable_column->get_nested_column_ptr(); + WhichDataType which_type(type); // process data switch (arrow_column->type()->id()) { case arrow::Type::STRING: @@ -303,8 +331,13 @@ Status arrow_column_to_doris_column(const arrow::Array* arrow_column, size_t arr return convert_column_with_boolean_data(arrow_column, arrow_batch_cur_idx, data_column, num_elements); case arrow::Type::DATE32: - return convert_column_with_timestamp_data( - arrow_column, arrow_batch_cur_idx, data_column, num_elements, timezone); + if (which_type.is_date_v2()) { + return convert_column_with_date_v2_data( + arrow_column, arrow_batch_cur_idx, data_column, num_elements, timezone); + } else { + return convert_column_with_timestamp_data( + arrow_column, arrow_batch_cur_idx, data_column, num_elements, timezone); + } case arrow::Type::DATE64: return convert_column_with_timestamp_data( arrow_column, arrow_batch_cur_idx, data_column, num_elements, timezone); @@ -315,8 +348,10 @@ Status arrow_column_to_doris_column(const arrow::Array* arrow_column, size_t arr return convert_column_with_decimal_data(arrow_column, arrow_batch_cur_idx, data_column, num_elements); case arrow::Type::LIST: - return convert_column_with_list_data(arrow_column, arrow_batch_cur_idx, data_column, - num_elements, timezone); + CHECK(type->have_subtypes()); + return convert_column_with_list_data( + arrow_column, arrow_batch_cur_idx, data_column, num_elements, timezone, + (reinterpret_cast(type.get()))->get_nested_type()); default: break; } diff --git a/be/src/vec/utils/arrow_column_to_doris_column.h b/be/src/vec/utils/arrow_column_to_doris_column.h index 0c75f27cda..2c9189b1be 100644 --- a/be/src/vec/utils/arrow_column_to_doris_column.h +++ b/be/src/vec/utils/arrow_column_to_doris_column.h @@ -34,7 +34,7 @@ namespace doris::vectorized { const PrimitiveType arrow_type_to_primitive_type(::arrow::Type::type type); Status arrow_column_to_doris_column(const arrow::Array* arrow_column, size_t arrow_batch_cur_idx, - ColumnPtr& doirs_column, size_t num_elements, - const std::string& timezone); + ColumnPtr& doris_column, const DataTypePtr& type, + size_t num_elements, const std::string& timezone); } // namespace doris::vectorized diff --git a/be/src/vec/utils/template_helpers.hpp b/be/src/vec/utils/template_helpers.hpp index 7fe681a8c6..afe745d1aa 100644 --- a/be/src/vec/utils/template_helpers.hpp +++ b/be/src/vec/utils/template_helpers.hpp @@ -43,7 +43,8 @@ #define TIME_TYPE_TO_COLUMN_TYPE(M) \ M(Date, ColumnInt64) \ - M(DateTime, ColumnInt64) + M(DateTime, ColumnInt64) \ + M(DateV2, ColumnUInt32) #define COMPLEX_TYPE_TO_COLUMN_TYPE(M) \ M(BitMap, ColumnBitmap) \ diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt index 6946e2f50c..b245a343d2 100644 --- a/be/test/CMakeLists.txt +++ b/be/test/CMakeLists.txt @@ -356,6 +356,7 @@ set(VEC_TEST_FILES vec/function/function_test_util.cpp vec/function/table_function_test.cpp vec/runtime/vdata_stream_test.cpp + vec/runtime/vdatetime_value_test.cpp vec/utils/arrow_column_to_doris_column_test.cpp vec/olap/char_type_padding_test.cpp ) diff --git a/be/test/olap/delta_writer_test.cpp b/be/test/olap/delta_writer_test.cpp index 632086f891..c364af49d2 100644 --- a/be/test/olap/delta_writer_test.cpp +++ b/be/test/olap/delta_writer_test.cpp @@ -151,6 +151,12 @@ static void create_tablet_request(int64_t tablet_id, int32_t schema_hash, k10.column_type.__set_scale(3); request->tablet_schema.columns.push_back(k10); + TColumn k11; + k11.column_name = "k11"; + k11.__set_is_key(true); + k11.column_type.type = TPrimitiveType::DATEV2; + request->tablet_schema.columns.push_back(k11); + TColumn v1; v1.column_name = "v1"; v1.__set_is_key(false); @@ -224,6 +230,13 @@ static void create_tablet_request(int64_t tablet_id, int32_t schema_hash, v10.column_type.__set_scale(3); v10.__set_aggregation_type(TAggregationType::SUM); request->tablet_schema.columns.push_back(v10); + + TColumn v11; + v11.column_name = "v11"; + v11.__set_is_key(false); + v11.column_type.type = TPrimitiveType::DATEV2; + v11.__set_aggregation_type(TAggregationType::REPLACE); + request->tablet_schema.columns.push_back(v11); } static void create_tablet_request_with_sequence_col(int64_t tablet_id, int32_t schema_hash, @@ -262,6 +275,13 @@ static void create_tablet_request_with_sequence_col(int64_t tablet_id, int32_t s v1.column_type.type = TPrimitiveType::DATETIME; v1.__set_aggregation_type(TAggregationType::REPLACE); request->tablet_schema.columns.push_back(v1); + + TColumn v2; + v2.column_name = "v2"; + v2.__set_is_key(false); + v2.column_type.type = TPrimitiveType::DATEV2; + v2.__set_aggregation_type(TAggregationType::REPLACE); + request->tablet_schema.columns.push_back(v2); } static TDescriptorTable create_descriptor_tablet() { @@ -288,27 +308,31 @@ static TDescriptorTable create_descriptor_tablet() { TSlotDescriptorBuilder().string_type(65).column_name("k9").column_pos(8).build()); tuple_builder.add_slot( TSlotDescriptorBuilder().decimal_type(6, 3).column_name("k10").column_pos(9).build()); + tuple_builder.add_slot( + TSlotDescriptorBuilder().type(TYPE_DATEV2).column_name("k11").column_pos(10).build()); tuple_builder.add_slot( - TSlotDescriptorBuilder().type(TYPE_TINYINT).column_name("v1").column_pos(10).build()); + TSlotDescriptorBuilder().type(TYPE_TINYINT).column_name("v1").column_pos(11).build()); tuple_builder.add_slot( - TSlotDescriptorBuilder().type(TYPE_SMALLINT).column_name("v2").column_pos(11).build()); + TSlotDescriptorBuilder().type(TYPE_SMALLINT).column_name("v2").column_pos(12).build()); tuple_builder.add_slot( - TSlotDescriptorBuilder().type(TYPE_INT).column_name("v3").column_pos(12).build()); + TSlotDescriptorBuilder().type(TYPE_INT).column_name("v3").column_pos(13).build()); tuple_builder.add_slot( - TSlotDescriptorBuilder().type(TYPE_BIGINT).column_name("v4").column_pos(13).build()); + TSlotDescriptorBuilder().type(TYPE_BIGINT).column_name("v4").column_pos(14).build()); tuple_builder.add_slot( - TSlotDescriptorBuilder().type(TYPE_LARGEINT).column_name("v5").column_pos(14).build()); + TSlotDescriptorBuilder().type(TYPE_LARGEINT).column_name("v5").column_pos(15).build()); tuple_builder.add_slot( - TSlotDescriptorBuilder().type(TYPE_DATE).column_name("v6").column_pos(15).build()); + TSlotDescriptorBuilder().type(TYPE_DATE).column_name("v6").column_pos(16).build()); tuple_builder.add_slot( - TSlotDescriptorBuilder().type(TYPE_DATETIME).column_name("v7").column_pos(16).build()); + TSlotDescriptorBuilder().type(TYPE_DATETIME).column_name("v7").column_pos(17).build()); tuple_builder.add_slot( - TSlotDescriptorBuilder().string_type(4).column_name("v8").column_pos(17).build()); + TSlotDescriptorBuilder().string_type(4).column_name("v8").column_pos(18).build()); tuple_builder.add_slot( - TSlotDescriptorBuilder().string_type(65).column_name("v9").column_pos(18).build()); + TSlotDescriptorBuilder().string_type(65).column_name("v9").column_pos(19).build()); tuple_builder.add_slot( - TSlotDescriptorBuilder().decimal_type(6, 3).column_name("v10").column_pos(19).build()); + TSlotDescriptorBuilder().decimal_type(6, 3).column_name("v10").column_pos(20).build()); + tuple_builder.add_slot( + TSlotDescriptorBuilder().type(TYPE_DATEV2).column_name("v11").column_pos(21).build()); tuple_builder.build(&dtb); return dtb.desc_tbl(); @@ -329,6 +353,8 @@ static TDescriptorTable create_descriptor_tablet_with_sequence_col() { .build()); tuple_builder.add_slot( TSlotDescriptorBuilder().type(TYPE_DATETIME).column_name("v1").column_pos(3).build()); + tuple_builder.add_slot( + TSlotDescriptorBuilder().type(TYPE_DATEV2).column_name("v2").column_pos(4).build()); tuple_builder.build(&dtb); return dtb.desc_tbl(); @@ -439,25 +465,27 @@ TEST_F(TestDeltaWriter, write) { DecimalV2Value decimal_value; decimal_value.assign_from_double(1.1); *(DecimalV2Value*)(tuple->get_slot(slots[9]->tuple_offset())) = decimal_value; - - *(int8_t*)(tuple->get_slot(slots[10]->tuple_offset())) = -127; - *(int16_t*)(tuple->get_slot(slots[11]->tuple_offset())) = -32767; - *(int32_t*)(tuple->get_slot(slots[12]->tuple_offset())) = -2147483647; - *(int64_t*)(tuple->get_slot(slots[13]->tuple_offset())) = -9223372036854775807L; - - memcpy(tuple->get_slot(slots[14]->tuple_offset()), &large_int_value, sizeof(int128_t)); - - ((DateTimeValue*)(tuple->get_slot(slots[15]->tuple_offset()))) + ((doris::vectorized::DateV2Value*)(tuple->get_slot(slots[10]->tuple_offset()))) ->from_date_str("2048-11-10", 10); + + *(int8_t*)(tuple->get_slot(slots[11]->tuple_offset())) = -127; + *(int16_t*)(tuple->get_slot(slots[12]->tuple_offset())) = -32767; + *(int32_t*)(tuple->get_slot(slots[13]->tuple_offset())) = -2147483647; + *(int64_t*)(tuple->get_slot(slots[14]->tuple_offset())) = -9223372036854775807L; + + memcpy(tuple->get_slot(slots[15]->tuple_offset()), &large_int_value, sizeof(int128_t)); + ((DateTimeValue*)(tuple->get_slot(slots[16]->tuple_offset()))) + ->from_date_str("2048-11-10", 10); + ((DateTimeValue*)(tuple->get_slot(slots[17]->tuple_offset()))) ->from_date_str("2636-08-16 19:39:43", 19); - char_ptr = (StringValue*)(tuple->get_slot(slots[17]->tuple_offset())); + char_ptr = (StringValue*)(tuple->get_slot(slots[18]->tuple_offset())); char_ptr->ptr = (char*)pool.allocate(4); memcpy(char_ptr->ptr, "abcd", 4); char_ptr->len = 4; - var_ptr = (StringValue*)(tuple->get_slot(slots[18]->tuple_offset())); + var_ptr = (StringValue*)(tuple->get_slot(slots[19]->tuple_offset())); var_ptr->ptr = (char*)pool.allocate(5); memcpy(var_ptr->ptr, "abcde", 5); var_ptr->len = 5; @@ -465,7 +493,10 @@ TEST_F(TestDeltaWriter, write) { DecimalV2Value val_decimal; val_decimal.assign_from_double(1.1); - *(DecimalV2Value*)(tuple->get_slot(slots[19]->tuple_offset())) = val_decimal; + *(DecimalV2Value*)(tuple->get_slot(slots[20]->tuple_offset())) = val_decimal; + + ((doris::vectorized::DateV2Value*)(tuple->get_slot(slots[21]->tuple_offset()))) + ->from_date_str("2048-11-10", 10); res = delta_writer->write(tuple); EXPECT_EQ(Status::OK(), res); @@ -567,36 +598,45 @@ TEST_F(TestDeltaWriter, vec_write) { decimal_value.assign_from_double(1.1); columns[9]->insert_data((const char*)&decimal_value, sizeof(decimal_value)); + doris::vectorized::DateV2Value date_v2; + date_v2.from_date_str("2048-11-10", 10); + auto date_v2_int = date_v2.to_date_uint32(); + columns[10]->insert_data((const char*)&date_v2_int, sizeof(date_v2_int)); + int8_t v1 = -127; - columns[10]->insert_data((const char*)&v1, sizeof(v1)); + columns[11]->insert_data((const char*)&v1, sizeof(v1)); int16_t v2 = -32767; - columns[11]->insert_data((const char*)&v2, sizeof(v2)); + columns[12]->insert_data((const char*)&v2, sizeof(v2)); int32_t v3 = -2147483647; - columns[12]->insert_data((const char*)&v3, sizeof(v3)); + columns[13]->insert_data((const char*)&v3, sizeof(v3)); int64_t v4 = -9223372036854775807L; - columns[13]->insert_data((const char*)&v4, sizeof(v4)); + columns[14]->insert_data((const char*)&v4, sizeof(v4)); int128_t v5 = -90000; - columns[14]->insert_data((const char*)&v5, sizeof(v5)); + columns[15]->insert_data((const char*)&v5, sizeof(v5)); DateTimeValue v6; v6.from_date_str("2048-11-10", 10); auto v6_int = v6.to_int64(); - columns[15]->insert_data((const char*)&v6_int, sizeof(v6_int)); + columns[16]->insert_data((const char*)&v6_int, sizeof(v6_int)); DateTimeValue v7; v7.from_date_str("2636-08-16 19:39:43", 19); auto v7_int = v7.to_int64(); - columns[16]->insert_data((const char*)&v7_int, sizeof(v7_int)); + columns[17]->insert_data((const char*)&v7_int, sizeof(v7_int)); - columns[17]->insert_data("abcd", 4); - columns[18]->insert_data("abcde", 5); + columns[18]->insert_data("abcd", 4); + columns[19]->insert_data("abcde", 5); decimal_value.assign_from_double(1.1); - columns[19]->insert_data((const char*)&decimal_value, sizeof(decimal_value)); + columns[20]->insert_data((const char*)&decimal_value, sizeof(decimal_value)); + + date_v2.from_date_str("2048-11-10", 10); + date_v2_int = date_v2.to_date_uint32(); + columns[21]->insert_data((const char*)&date_v2_int, sizeof(date_v2_int)); res = delta_writer->write(&block, {0}); ASSERT_TRUE(res.ok()); @@ -755,6 +795,11 @@ TEST_F(TestDeltaWriter, vec_sequence_col) { int64_t c4_int = c4.to_int64(); columns[3]->insert_data((const char*)&c4_int, sizeof(c4)); + doris::vectorized::DateV2Value c5; + c5.set_time(2022, 6, 6); + uint32_t c5_int = c5.to_date_uint32(); + columns[4]->insert_data((const char*)&c5_int, sizeof(c5)); + res = delta_writer->write(&block, {0}); ASSERT_TRUE(res.ok()); } diff --git a/be/test/olap/in_list_predicate_test.cpp b/be/test/olap/in_list_predicate_test.cpp index 285e6ea375..00a2280614 100644 --- a/be/test/olap/in_list_predicate_test.cpp +++ b/be/test/olap/in_list_predicate_test.cpp @@ -41,6 +41,15 @@ static uint24_t timestamp_from_date(const char* date_string) { return uint24_t(value); } +static uint32_t timestamp_from_date_v2(const char* date_string) { + tm time_tm; + strptime(date_string, "%Y-%m-%d", &time_tm); + + doris::vectorized::DateV2Value value; + value.set_time(time_tm.tm_year, time_tm.tm_mon, time_tm.tm_mday); + return binary_cast(value); +} + static uint64_t timestamp_from_datetime(const std::string& value_string) { tm time_tm; strptime(value_string.c_str(), "%Y-%m-%d %H:%M:%S", &time_tm); @@ -65,6 +74,18 @@ static std::string to_date_string(uint24_t& date_value) { return std::string(buf); } +static std::string to_date_v2_string(uint32_t& date_value) { + tm time_tm; + uint32_t value = date_value; + memset(&time_tm, 0, sizeof(time_tm)); + time_tm.tm_mday = static_cast(value & 0x000000FF); + time_tm.tm_mon = static_cast((value & 0x0000FF00) >> 8); + time_tm.tm_year = static_cast((value & 0xFFFF0000) >> 16); + char buf[20] = {'\0'}; + strftime(buf, sizeof(buf), "%Y-%m-%d", &time_tm); + return std::string(buf); +} + static std::string to_datetime_string(uint64_t& datetime_value) { tm time_tm; int64_t part1 = (datetime_value / 1000000L); @@ -884,6 +905,115 @@ TEST_F(TestInListPredicate, DATE_COLUMN) { delete pred; } +TEST_F(TestInListPredicate, DATE_V2_COLUMN) { + TabletSchema tablet_schema; + SetTabletSchema(std::string("DATE_V2_COLUMN"), "DATEV2", "REPLACE", 1, true, true, + &tablet_schema); + int size = 6; + std::vector return_columns; + for (int i = 0; i < tablet_schema.num_columns(); ++i) { + return_columns.push_back(i); + } + phmap::flat_hash_set values; + uint32_t value1 = datetime::timestamp_from_date_v2("2017-09-09"); + values.insert(value1); + + uint32_t value2 = datetime::timestamp_from_date_v2("2017-09-10"); + values.insert(value2); + + uint32_t value3 = datetime::timestamp_from_date_v2("2017-09-11"); + values.insert(value3); + ColumnPredicate* pred = new InListPredicate(0, std::move(values)); + + // for VectorizedBatch no nulls + InitVectorizedBatch(&tablet_schema, return_columns, size); + ColumnVector* col_vector = _vectorized_batch->column(0); + col_vector->set_no_nulls(true); + uint32_t* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(uint32_t))); + col_vector->set_col_data(col_data); + + std::vector date_array; + date_array.push_back("2017-09-07"); + date_array.push_back("2017-09-08"); + date_array.push_back("2017-09-09"); + date_array.push_back("2017-09-10"); + date_array.push_back("2017-09-11"); + date_array.push_back("2017-09-12"); + for (int i = 0; i < size; ++i) { + uint32_t timestamp = datetime::timestamp_from_date_v2(date_array[i].c_str()); + *(col_data + i) = timestamp; + } + pred->evaluate(_vectorized_batch); + EXPECT_EQ(_vectorized_batch->size(), 3); + uint16_t* sel = _vectorized_batch->selected(); + EXPECT_EQ(datetime::to_date_v2_string(*(col_data + sel[0])), "2017-09-09"); + EXPECT_EQ(datetime::to_date_v2_string(*(col_data + sel[1])), "2017-09-10"); + EXPECT_EQ(datetime::to_date_v2_string(*(col_data + sel[2])), "2017-09-11"); + + // for ColumnBlock no nulls + init_row_block(&tablet_schema, size); + ColumnBlock col_block = _row_block->column_block(0); + auto select_size = _row_block->selected_size(); + ColumnBlockView col_block_view(&col_block); + for (int i = 0; i < size; ++i, col_block_view.advance(1)) { + col_block_view.set_null_bits(1, false); + uint32_t timestamp = datetime::timestamp_from_date_v2(date_array[i].c_str()); + *reinterpret_cast(col_block_view.data()) = timestamp; + } + pred->evaluate(&col_block, _row_block->selection_vector(), &select_size); + EXPECT_EQ(select_size, 3); + EXPECT_EQ(datetime::to_date_v2_string( + *(uint32_t*)col_block.cell(_row_block->selection_vector()[0]).cell_ptr()), + "2017-09-09"); + EXPECT_EQ(datetime::to_date_v2_string( + *(uint32_t*)col_block.cell(_row_block->selection_vector()[1]).cell_ptr()), + "2017-09-10"); + EXPECT_EQ(datetime::to_date_v2_string( + *(uint32_t*)col_block.cell(_row_block->selection_vector()[2]).cell_ptr()), + "2017-09-11"); + + // for VectorizedBatch has nulls + col_vector->set_no_nulls(false); + bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); + memset(is_null, 0, size); + col_vector->set_is_null(is_null); + for (int i = 0; i < size; ++i) { + if (i % 2 == 0) { + is_null[i] = true; + } else { + uint32_t timestamp = datetime::timestamp_from_date_v2(date_array[i].c_str()); + *(col_data + i) = timestamp; + } + } + _vectorized_batch->set_size(size); + _vectorized_batch->set_selected_in_use(false); + pred->evaluate(_vectorized_batch); + EXPECT_EQ(_vectorized_batch->size(), 1); + sel = _vectorized_batch->selected(); + EXPECT_EQ(datetime::to_date_v2_string(*(col_data + sel[0])), "2017-09-10"); + + // for ColumnBlock has nulls + col_block_view = ColumnBlockView(&col_block); + for (int i = 0; i < size; ++i, col_block_view.advance(1)) { + if (i % 2 == 0) { + col_block_view.set_null_bits(1, true); + } else { + col_block_view.set_null_bits(1, false); + uint32_t timestamp = datetime::timestamp_from_date_v2(date_array[i].c_str()); + *reinterpret_cast(col_block_view.data()) = timestamp; + } + } + _row_block->clear(); + select_size = _row_block->selected_size(); + pred->evaluate(&col_block, _row_block->selection_vector(), &select_size); + EXPECT_EQ(select_size, 1); + EXPECT_EQ(datetime::to_date_v2_string( + *(uint32_t*)col_block.cell(_row_block->selection_vector()[0]).cell_ptr()), + "2017-09-10"); + + delete pred; +} + TEST_F(TestInListPredicate, DATETIME_COLUMN) { TabletSchema tablet_schema; SetTabletSchema(std::string("DATETIME_COLUMN"), "DATETIME", "REPLACE", 1, true, true, diff --git a/be/test/olap/null_predicate_test.cpp b/be/test/olap/null_predicate_test.cpp index 47bf02306f..1fe189faf6 100644 --- a/be/test/olap/null_predicate_test.cpp +++ b/be/test/olap/null_predicate_test.cpp @@ -45,6 +45,13 @@ static uint24_t to_date_timestamp(const char* date_string) { return uint24_t(value); } +static uint32_t to_date_v2_timestamp(const char* date_string) { + tm time_tm; + strptime(date_string, "%Y-%m-%d", &time_tm); + + return ((time_tm.tm_year + 1900) << 16) | ((time_tm.tm_mon + 1) << 8) | time_tm.tm_mday; +} + static uint64_t to_datetime_timestamp(const std::string& value_string) { tm time_tm; strptime(value_string.c_str(), "%Y-%m-%d %H:%M:%S", &time_tm); @@ -788,4 +795,104 @@ TEST_F(TestNullPredicate, DATETIME_COLUMN) { EXPECT_EQ(select_size, 2); } +TEST_F(TestNullPredicate, DATEV2_COLUMN) { + TabletSchema tablet_schema; + SetTabletSchema(std::string("DATEV2_COLUMN"), "DATEV2", "REPLACE", 4, true, true, + &tablet_schema); + int size = 6; + std::vector return_columns; + for (int i = 0; i < tablet_schema.num_columns(); ++i) { + return_columns.push_back(i); + } + std::unique_ptr pred(new NullPredicate(0, true)); + + // for VectorizedBatch no nulls + InitVectorizedBatch(&tablet_schema, return_columns, size); + ColumnVector* col_vector = _vectorized_batch->column(0); + col_vector->set_no_nulls(true); + uint32_t* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(uint32_t))); + col_vector->set_col_data(col_data); + + std::vector date_array; + date_array.push_back("2017-09-07"); + date_array.push_back("2017-09-08"); + date_array.push_back("2017-09-09"); + date_array.push_back("2017-09-10"); + date_array.push_back("2017-09-11"); + date_array.push_back("2017-09-12"); + for (int i = 0; i < size; ++i) { + uint32_t timestamp = datetime::to_date_v2_timestamp(date_array[i].c_str()); + *(col_data + i) = timestamp; + } + pred->evaluate(_vectorized_batch); + EXPECT_EQ(_vectorized_batch->size(), 0); + + // for ColumnBlock no nulls + init_row_block(&tablet_schema, size); + ColumnBlock col_block = _row_block->column_block(0); + auto select_size = _row_block->selected_size(); + ColumnBlockView col_block_view(&col_block); + for (int i = 0; i < size; ++i, col_block_view.advance(1)) { + col_block_view.set_null_bits(1, false); + uint32_t timestamp = datetime::to_date_v2_timestamp(date_array[i].c_str()); + *reinterpret_cast(col_block_view.data()) = timestamp; + } + pred->evaluate(&col_block, _row_block->selection_vector(), &select_size); + EXPECT_EQ(select_size, 0); + + // for vectorized::Block no null + _row_block->clear(); + select_size = _row_block->selected_size(); + vectorized::Block vec_block = tablet_schema.create_block(return_columns); + _row_block->convert_to_vec_block(&vec_block); + ColumnPtr vec_col = vec_block.get_columns()[0]; + select_size = pred->evaluate(const_cast(*vec_col), + _row_block->selection_vector(), select_size); + EXPECT_EQ(select_size, 0); + + // for VectorizedBatch has nulls + col_vector->set_no_nulls(false); + bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); + memset(is_null, 0, size); + col_vector->set_is_null(is_null); + for (int i = 0; i < size; ++i) { + if (i % 3 == 0) { + is_null[i] = true; + } else { + uint32_t timestamp = datetime::to_date_v2_timestamp(date_array[i].c_str()); + *(col_data + i) = timestamp; + } + } + _vectorized_batch->set_size(size); + _vectorized_batch->set_selected_in_use(false); + pred->evaluate(_vectorized_batch); + EXPECT_EQ(_vectorized_batch->size(), 2); + + // for ColumnBlock has nulls + col_block_view = ColumnBlockView(&col_block); + for (int i = 0; i < size; ++i, col_block_view.advance(1)) { + if (i % 3 == 0) { + col_block_view.set_null_bits(1, true); + } else { + col_block_view.set_null_bits(1, false); + uint32_t timestamp = datetime::to_date_v2_timestamp(date_array[i].c_str()); + *reinterpret_cast(col_block_view.data()) = timestamp; + } + } + _row_block->clear(); + select_size = _row_block->selected_size(); + pred->evaluate(&col_block, _row_block->selection_vector(), &select_size); + EXPECT_EQ(select_size, 2); + + // for vectorized::Block has nulls + _row_block->clear(); + select_size = _row_block->selected_size(); + vec_block = tablet_schema.create_block(return_columns); + _row_block->convert_to_vec_block(&vec_block); + vec_col = vec_block.get_columns()[0]; + select_size = pred->evaluate(const_cast(*vec_col), + _row_block->selection_vector(), select_size); + EXPECT_EQ(select_size, 2); +} + } // namespace doris diff --git a/be/test/olap/row_cursor_test.cpp b/be/test/olap/row_cursor_test.cpp index cdabe13387..96722c77d7 100644 --- a/be/test/olap/row_cursor_test.cpp +++ b/be/test/olap/row_cursor_test.cpp @@ -79,71 +79,80 @@ void set_tablet_schema_for_init(TabletSchema* tablet_schema) { column_5->set_index_length(8); ColumnPB* column_6 = tablet_schema_pb.add_column(); - column_6->set_unique_id(6); + column_6->set_unique_id(5); column_6->set_name("column_6"); - column_6->set_type("DECIMAL"); + column_6->set_type("DATEV2"); column_6->set_is_key(true); column_6->set_is_nullable(true); - column_6->set_length(12); - column_6->set_index_length(12); - column_6->set_frac(3); - column_6->set_precision(6); + column_6->set_length(4); + column_6->set_index_length(4); ColumnPB* column_7 = tablet_schema_pb.add_column(); - column_7->set_unique_id(7); + column_7->set_unique_id(6); column_7->set_name("column_7"); - column_7->set_type("CHAR"); + column_7->set_type("DECIMAL"); column_7->set_is_key(true); column_7->set_is_nullable(true); - column_7->set_length(4); - column_7->set_index_length(4); - column_7->set_default_value("char"); + column_7->set_length(12); + column_7->set_index_length(12); + column_7->set_frac(3); + column_7->set_precision(6); ColumnPB* column_8 = tablet_schema_pb.add_column(); - column_8->set_unique_id(8); + column_8->set_unique_id(7); column_8->set_name("column_8"); - column_8->set_type("BIGINT"); + column_8->set_type("CHAR"); + column_8->set_is_key(true); column_8->set_is_nullable(true); - column_8->set_length(8); - column_8->set_aggregation("SUM"); - column_8->set_is_key(false); + column_8->set_length(4); + column_8->set_index_length(4); + column_8->set_default_value("char"); ColumnPB* column_9 = tablet_schema_pb.add_column(); - column_9->set_unique_id(9); + column_9->set_unique_id(8); column_9->set_name("column_9"); - column_9->set_type("VARCHAR"); + column_9->set_type("BIGINT"); column_9->set_is_nullable(true); - column_9->set_length(16 + OLAP_VARCHAR_MAX_BYTES); - column_9->set_aggregation("REPLACE"); + column_9->set_length(8); + column_9->set_aggregation("SUM"); column_9->set_is_key(false); ColumnPB* column_10 = tablet_schema_pb.add_column(); - column_10->set_unique_id(10); + column_10->set_unique_id(9); column_10->set_name("column_10"); - column_10->set_type("LARGEINT"); + column_10->set_type("VARCHAR"); column_10->set_is_nullable(true); - column_10->set_length(16); - column_10->set_aggregation("MAX"); + column_10->set_length(16 + OLAP_VARCHAR_MAX_BYTES); + column_10->set_aggregation("REPLACE"); column_10->set_is_key(false); ColumnPB* column_11 = tablet_schema_pb.add_column(); - column_11->set_unique_id(11); + column_11->set_unique_id(10); column_11->set_name("column_11"); - column_11->set_type("DECIMAL"); + column_11->set_type("LARGEINT"); column_11->set_is_nullable(true); - column_11->set_length(12); - column_11->set_aggregation("MIN"); + column_11->set_length(16); + column_11->set_aggregation("MAX"); column_11->set_is_key(false); ColumnPB* column_12 = tablet_schema_pb.add_column(); - column_12->set_unique_id(12); + column_12->set_unique_id(11); column_12->set_name("column_12"); - column_12->set_type("HLL"); + column_12->set_type("DECIMAL"); column_12->set_is_nullable(true); - column_12->set_length(HLL_COLUMN_DEFAULT_LEN); - column_12->set_aggregation("HLL_UNION"); + column_12->set_length(12); + column_12->set_aggregation("MIN"); column_12->set_is_key(false); + ColumnPB* column_13 = tablet_schema_pb.add_column(); + column_13->set_unique_id(12); + column_13->set_name("column_13"); + column_13->set_type("HLL"); + column_13->set_is_nullable(true); + column_13->set_length(HLL_COLUMN_DEFAULT_LEN); + column_13->set_aggregation("HLL_UNION"); + column_13->set_is_key(false); + tablet_schema->init_from_pb(tablet_schema_pb); } @@ -272,7 +281,7 @@ TEST_F(TestRowCursor, InitRowCursor) { RowCursor row; Status res = row.init(tablet_schema); EXPECT_EQ(res, Status::OK()); - EXPECT_EQ(row.get_fixed_len(), 126); + EXPECT_EQ(row.get_fixed_len(), 131); EXPECT_EQ(row.get_variable_len(), 20); } @@ -300,8 +309,8 @@ TEST_F(TestRowCursor, InitRowCursorWithColIds) { RowCursor row; Status res = row.init(tablet_schema, col_ids); EXPECT_EQ(res, Status::OK()); - EXPECT_EQ(row.get_fixed_len(), 63); - EXPECT_EQ(row.get_variable_len(), 20); + EXPECT_EQ(row.get_fixed_len(), 55); + EXPECT_EQ(row.get_variable_len(), 0); } TEST_F(TestRowCursor, InitRowCursorWithScanKey) { diff --git a/be/test/olap/schema_change_test.cpp b/be/test/olap/schema_change_test.cpp index 019c7635f8..e9383f5763 100644 --- a/be/test/olap/schema_change_test.cpp +++ b/be/test/olap/schema_change_test.cpp @@ -407,6 +407,96 @@ TEST_F(TestColumn, ConvertDatetimeToDate) { EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); } +TEST_F(TestColumn, ConvertDatetimeToDateV2) { + TabletSchema tablet_schema; + set_tablet_schema("DatetimeColumn", "DATETIME", "REPLACE", 8, false, false, &tablet_schema); + create_column_writer(tablet_schema); + + RowCursor write_row; + write_row.init(tablet_schema); + RowBlock block(&tablet_schema); + RowBlockInfo block_info; + block_info.row_num = 10000; + block.init(block_info); + + std::vector val_string_array; + std::string origin_val = "2019-11-25 19:07:00"; + val_string_array.emplace_back(origin_val); + OlapTuple tuple(val_string_array); + write_row.from_tuple(tuple); + block.set_row(0, write_row); + block.finalize(1); + EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); + + ColumnDataHeaderMessage header; + EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); + + // read data + TabletSchema convert_tablet_schema; + set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 4, false, false, &convert_tablet_schema); + create_column_reader(tablet_schema); + RowCursor read_row; + read_row.init(convert_tablet_schema); + + _col_vector.reset(new ColumnVector()); + EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); + char* data = reinterpret_cast(_col_vector->col_data()); + read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); + std::string dest_string = read_row.column_schema(0)->to_string(read_row.cell_ptr(0)); + EXPECT_TRUE(strncmp(dest_string.c_str(), "2019-11-25", strlen("2019-11-25")) == 0); + + //test not support type + const auto* tp = get_scalar_type_info(); + Status st = read_row.convert_from(0, data, tp, _mem_pool.get()); + EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); +} + +TEST_F(TestColumn, ConvertDateV2ToDateTime) { + TabletSchema tablet_schema; + set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 4, false, false, &tablet_schema); + create_column_writer(tablet_schema); + + RowCursor write_row; + write_row.init(tablet_schema); + RowBlock block(&tablet_schema); + RowBlockInfo block_info; + block_info.row_num = 10000; + block.init(block_info); + + std::vector val_string_array; + std::string origin_val = "2019-11-25"; + val_string_array.emplace_back(origin_val); + OlapTuple tuple(val_string_array); + write_row.from_tuple(tuple); + block.set_row(0, write_row); + block.finalize(1); + EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); + + ColumnDataHeaderMessage header; + EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); + + // read data + TabletSchema convert_tablet_schema; + set_tablet_schema("DateTimeColumn", "DATETIME", "REPLACE", 8, false, false, + &convert_tablet_schema); + create_column_reader(tablet_schema); + RowCursor read_row; + read_row.init(convert_tablet_schema); + + _col_vector.reset(new ColumnVector()); + EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); + char* data = reinterpret_cast(_col_vector->col_data()); + read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); + std::string dest_string = read_row.column_schema(0)->to_string(read_row.cell_ptr(0)); + EXPECT_TRUE(strncmp(dest_string.c_str(), "2019-11-25 00:00:00", + strlen("2019-11-25 00:00:00")) == 0); + + //test not support type + const auto* tp = get_scalar_type_info(); + Status st = read_row.convert_from(0, data, tp, _mem_pool.get()); + EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); +} + TEST_F(TestColumn, ConvertDateToDatetime) { TabletSchema tablet_schema; set_tablet_schema("DateColumn", "DATE", "REPLACE", 3, false, false, &tablet_schema); @@ -452,6 +542,96 @@ TEST_F(TestColumn, ConvertDateToDatetime) { EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); } +TEST_F(TestColumn, ConvertDateToDateV2) { + TabletSchema tablet_schema; + set_tablet_schema("DateColumn", "DATE", "REPLACE", 3, false, false, &tablet_schema); + create_column_writer(tablet_schema); + + RowCursor write_row; + write_row.init(tablet_schema); + + RowBlock block(&tablet_schema); + RowBlockInfo block_info; + block_info.row_num = 10000; + block.init(block_info); + + std::vector val_string_array; + std::string origin_val = "2019-12-04"; + val_string_array.emplace_back(origin_val); + OlapTuple tuple(val_string_array); + write_row.from_tuple(tuple); + block.set_row(0, write_row); + block.finalize(1); + EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); + + ColumnDataHeaderMessage header_message; + EXPECT_EQ(_column_writer->finalize(&header_message), Status::OK()); + + TabletSchema convert_tablet_schema; + set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 4, false, false, &convert_tablet_schema); + create_column_reader(tablet_schema); + RowCursor read_row; + read_row.init(convert_tablet_schema); + _col_vector.reset(new ColumnVector()); + EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); + char* data = reinterpret_cast(_col_vector->col_data()); + read_row.set_field_content(0, data, _mem_pool.get()); + read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); + std::string expected = "2019-12-04"; + std::string dest_string = + read_row.column_schema(0)->to_string(read_row.cell_ptr(0)).substr(0, expected.length()); + EXPECT_TRUE(dest_string.compare("2019-12-04") == 0); + + //test not support type + const auto* tp = get_scalar_type_info(); + Status st = read_row.convert_from(0, data, tp, _mem_pool.get()); + EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); +} + +TEST_F(TestColumn, ConvertDateV2ToDate) { + TabletSchema tablet_schema; + set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 4, false, false, &tablet_schema); + create_column_writer(tablet_schema); + + RowCursor write_row; + write_row.init(tablet_schema); + + RowBlock block(&tablet_schema); + RowBlockInfo block_info; + block_info.row_num = 10000; + block.init(block_info); + + std::vector val_string_array; + std::string origin_val = "2019-12-04"; + val_string_array.emplace_back(origin_val); + OlapTuple tuple(val_string_array); + write_row.from_tuple(tuple); + block.set_row(0, write_row); + block.finalize(1); + EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); + + ColumnDataHeaderMessage header_message; + EXPECT_EQ(_column_writer->finalize(&header_message), Status::OK()); + + TabletSchema convert_tablet_schema; + set_tablet_schema("DateColumn", "DATE", "REPLACE", 3, false, false, &convert_tablet_schema); + create_column_reader(tablet_schema); + RowCursor read_row; + read_row.init(convert_tablet_schema); + _col_vector.reset(new ColumnVector()); + EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); + char* data = reinterpret_cast(_col_vector->col_data()); + read_row.set_field_content(0, data, _mem_pool.get()); + read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); + std::string dest_string = read_row.column_schema(0)->to_string(read_row.cell_ptr(0)); + EXPECT_TRUE(dest_string.compare("2019-12-04") == 0); + + //test not support type + const auto* tp = get_scalar_type_info(); + Status st = read_row.convert_from(0, data, tp, _mem_pool.get()); + EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); +} + TEST_F(TestColumn, ConvertIntToDate) { TabletSchema tablet_schema; set_tablet_schema("IntColumn", "INT", "REPLACE", 4, false, false, &tablet_schema); @@ -494,6 +674,48 @@ TEST_F(TestColumn, ConvertIntToDate) { EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); } +TEST_F(TestColumn, ConvertIntToDateV2) { + TabletSchema tablet_schema; + set_tablet_schema("IntColumn", "INT", "REPLACE", 4, false, false, &tablet_schema); + create_column_writer(tablet_schema); + + RowCursor write_row; + write_row.init(tablet_schema); + + RowBlock block(&tablet_schema); + RowBlockInfo block_info; + block_info.row_num = 10000; + block.init(block_info); + + int time_val = 20191205; + write_row.set_field_content(0, reinterpret_cast(&time_val), _mem_pool.get()); + block.set_row(0, write_row); + block.finalize(1); + EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); + + ColumnDataHeaderMessage header; + EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); + + TabletSchema convert_tablet_schema; + set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 4, false, false, &convert_tablet_schema); + create_column_reader(tablet_schema); + + RowCursor read_row; + read_row.init(convert_tablet_schema); + + _col_vector.reset(new ColumnVector()); + EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); + char* data = reinterpret_cast(_col_vector->col_data()); + read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); + std::string dest_string = read_row.column_schema(0)->to_string(read_row.cell_ptr(0)); + EXPECT_TRUE(strncmp(dest_string.c_str(), "2019-12-05", strlen("2019-12-05")) == 0); + + //test not support type + const auto* tp = get_scalar_type_info(); + Status st = read_row.convert_from(0, read_row.cell_ptr(0), tp, _mem_pool.get()); + EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); +} + TEST_F(TestColumn, ConvertVarcharToDate) { TabletSchema tablet_schema; set_tablet_schema("VarcharColumn", "VARCHAR", "REPLACE", 255, false, false, &tablet_schema); @@ -549,6 +771,64 @@ TEST_F(TestColumn, ConvertVarcharToDate) { EXPECT_EQ(st, Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); } +TEST_F(TestColumn, ConvertVarcharToDateV2) { + TabletSchema tablet_schema; + set_tablet_schema("VarcharColumn", "VARCHAR", "REPLACE", 255, false, false, &tablet_schema); + create_column_writer(tablet_schema); + + RowCursor write_row; + write_row.init(tablet_schema); + + RowBlock block(&tablet_schema); + RowBlockInfo block_info; + block_info.row_num = 10000; + block.init(block_info); + + // test valid format convert + std::vector valid_src_strs = { + "2019-12-17", "19-12-17", "20191217", "191217", "2019/12/17", "19/12/17", + }; + std::string expected_val("2019-12-17"); + for (auto src_str : valid_src_strs) { + write_row.set_field_content(0, reinterpret_cast(&src_str), _mem_pool.get()); + block.set_row(0, write_row); + block.finalize(1); + EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); + + ColumnDataHeaderMessage header; + EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); + + // because file_helper is reused in this case, we should close it. + helper.close(); + TabletSchema convert_tablet_schema; + set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 3, false, false, + &convert_tablet_schema); + create_column_reader(tablet_schema); + RowCursor read_row; + read_row.init(convert_tablet_schema); + + _col_vector.reset(new ColumnVector()); + EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); + char* data = reinterpret_cast(_col_vector->col_data()); + read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); + std::string dst_str = read_row.column_schema(0) + ->to_string(read_row.cell_ptr(0)) + .substr(0, expected_val.length()); + EXPECT_EQ(expected_val, dst_str); + } + helper.close(); + TabletSchema convert_tablet_schema; + set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 3, false, false, &convert_tablet_schema); + create_column_reader(tablet_schema); + RowCursor read_row; + read_row.init(convert_tablet_schema); + + //test not support type + const auto* tp = get_scalar_type_info(); + Status st = read_row.convert_from(0, read_row.cell_ptr(0), tp, _mem_pool.get()); + EXPECT_EQ(st, Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); +} + TEST_F(TestColumn, ConvertVarcharToTinyInt1) { test_convert_from_varchar("TINYINT", 1, "127", Status::OK()); } diff --git a/be/test/olap/storage_types_test.cpp b/be/test/olap/storage_types_test.cpp index 09e7ecc8ea..ea1bd756d1 100644 --- a/be/test/olap/storage_types_test.cpp +++ b/be/test/olap/storage_types_test.cpp @@ -140,6 +140,8 @@ TEST(TypesTest, copy_and_equal) { common_test((1988 << 9) | (2 << 5) | 1); common_test(19880201010203L); + common_test((1988 << 16) | (2 << 8) | 1); + Slice slice("12345abcde"); common_test(slice); common_test(slice); @@ -213,6 +215,10 @@ TEST(ArrayTypeTest, copy_and_equal) { (2008 << 9) | (2 << 5) | 1}; common_test_array(CollectionValue(date_array, 3, null_signs)); + uint32_t date_v2_array[3] = {(1988 << 16) | (2 << 8) | 1, (1998 << 16) | (2 << 8) | 1, + (2008 << 16) | (2 << 8) | 1}; + common_test_array(CollectionValue(date_v2_array, 3, null_signs)); + int64_t datetime_array[3] = {19880201010203L, 19980201010203L, 20080204010203L}; common_test_array(CollectionValue(datetime_array, 3, null_signs)); diff --git a/be/test/vec/core/block_test.cpp b/be/test/vec/core/block_test.cpp index fb9d2be897..d20842b017 100644 --- a/be/test/vec/core/block_test.cpp +++ b/be/test/vec/core/block_test.cpp @@ -54,7 +54,8 @@ TEST(BlockTest, RowBatchCovertToBlock) { {"k4", TYPE_VARCHAR, sizeof(StringValue), false}, {"k5", TYPE_DECIMALV2, sizeof(DecimalV2Value), false}, {"k6", TYPE_LARGEINT, sizeof(__int128), false}, - {"k7", TYPE_DATETIME, sizeof(int64_t), false}}; + {"k7", TYPE_DATETIME, sizeof(int64_t), false}, + {"k8", TYPE_DATEV2, sizeof(uint32_t), false}}; SchemaScanner schema_scanner(column_descs, sizeof(column_descs) / sizeof(SchemaScanner::ColumnDesc)); @@ -111,6 +112,13 @@ TEST(BlockTest, RowBatchCovertToBlock) { k7.date_add_interval(time_interval, vectorized::TimeUnit::DAY); memcpy(tuple->get_slot(slot_desc->tuple_offset()), &k7, column_descs[6].size); + slot_desc = tuple_desc->slots()[7]; + vectorized::DateV2Value k8; + std::string now_date("2020-12-02"); + k8.from_date_str(now_date.c_str(), now_date.size()); + k8.date_add_interval(time_interval, vectorized::TimeUnit::DAY); + memcpy(tuple->get_slot(slot_desc->tuple_offset()), &k8, column_descs[7].size); + tuple_row->set_tuple(0, tuple); row_batch.commit_last_row(); } @@ -127,6 +135,7 @@ TEST(BlockTest, RowBatchCovertToBlock) { vectorized::ColumnPtr column5 = block.get_columns()[4]; vectorized::ColumnPtr column6 = block.get_columns()[5]; vectorized::ColumnPtr column7 = block.get_columns()[6]; + vectorized::ColumnPtr column8 = block.get_columns()[7]; if (i % 5 != 0) { EXPECT_EQ((int16_t)column1->get64(i), k1); @@ -155,6 +164,16 @@ TEST(BlockTest, RowBatchCovertToBlock) { EXPECT_EQ(k7, date_time_value); + larget_int = column8->operator[](i).get(); + vectorized::DateV2Value k8; + memcpy(reinterpret_cast(&k8), &larget_int, column_descs[7].size); + vectorized::DateV2Value date_v2_value; + std::string now_date("2020-12-02"); + date_v2_value.from_date_str(now_date.c_str(), now_date.size()); + date_v2_value.date_add_interval(time_interval, vectorized::TimeUnit::DAY); + + EXPECT_EQ(k8, date_v2_value); + k1++; k3 += 0.1; } @@ -393,8 +412,19 @@ TEST(BlockTest, dump_data) { vectorized::ColumnWithTypeAndName test_datetime(column_vector_datetime->get_ptr(), datetime_type, "test_datetime"); - vectorized::Block block( - {test_int, test_string, test_decimal, test_nullable_int32, test_date, test_datetime}); + auto column_vector_date_v2 = vectorized::ColumnVector::create(); + auto& date_v2_data = column_vector_date_v2->get_data(); + for (int i = 0; i < 1024; ++i) { + vectorized::DateV2Value value; + value.from_date((uint32_t)((2022 << 16) | (6 << 8) | 6)); + date_v2_data.push_back(*reinterpret_cast(&value)); + } + vectorized::DataTypePtr date_v2_type(std::make_shared()); + vectorized::ColumnWithTypeAndName test_date_v2(column_vector_date_v2->get_ptr(), date_v2_type, + "test_datev2"); + + vectorized::Block block({test_int, test_string, test_decimal, test_nullable_int32, test_date, + test_datetime, test_date_v2}); EXPECT_GT(block.dump_data().size(), 1); } } // namespace doris diff --git a/be/test/vec/exprs/vexpr_test.cpp b/be/test/vec/exprs/vexpr_test.cpp index d8e8a6d635..4ece5fd792 100644 --- a/be/test/vec/exprs/vexpr_test.cpp +++ b/be/test/vec/exprs/vexpr_test.cpp @@ -188,6 +188,13 @@ struct literal_traits { using CXXType = std::string; }; +template <> +struct literal_traits { + const static TPrimitiveType::type ttype = TPrimitiveType::DATEV2; + const static TExprNodeType::type tnode_type = TExprNodeType::STRING_LITERAL; + using CXXType = std::string; +}; + template <> struct literal_traits { const static TPrimitiveType::type ttype = TPrimitiveType::DECIMALV2; @@ -225,6 +232,14 @@ void set_literal(TExprNode& node, const U& value) { node.__set_date_literal(date_literal); } +template ::CXXType, + std::enable_if_t = true> +void set_literal(TExprNode& node, const U& value) { + TDateLiteral date_literal; + date_literal.__set_value(value); + node.__set_date_literal(date_literal); +} + template ::CXXType, std::enable_if_t::is_iec559, bool> = true> void set_literal(TExprNode& node, const U& value) { @@ -342,6 +357,20 @@ TEST(TEST_VEXPR, LITERALTEST) { auto v = (*ctn.column)[0].get<__int64_t>(); EXPECT_EQ(v, dt); } + { + vectorized::DateV2Value data_time_value; + const char* date = "20210407"; + data_time_value.from_date_str(date, strlen(date)); + uint32_t dt; + memcpy(&dt, &data_time_value, sizeof(uint32_t)); + VLiteral literal(create_literal(std::string(date))); + Block block; + int ret = -1; + literal.execute(nullptr, &block, &ret); + auto ctn = block.safe_get_by_position(ret); + auto v = (*ctn.column)[0].get(); + EXPECT_EQ(v, dt); + } { VLiteral literal(create_literal(std::string("1234.56"))); Block block; diff --git a/be/test/vec/function/function_test_util.cpp b/be/test/vec/function/function_test_util.cpp index 0e25aa1781..32e738b349 100644 --- a/be/test/vec/function/function_test_util.cpp +++ b/be/test/vec/function/function_test_util.cpp @@ -32,6 +32,14 @@ int64_t str_to_date_time(std::string datetime_str, bool data_time) { } return binary_cast(v); } + +uint32_t str_to_date_v2(std::string datetime_str, std::string datetime_format) { + DateV2Value v; + v.from_date_format_str(datetime_format.c_str(), datetime_format.size(), datetime_str.c_str(), + datetime_str.size()); + return binary_cast(v); +} + size_t type_index_to_data_type(const std::vector& input_types, size_t index, ut_type::UTDataTypeDesc& ut_desc, DataTypePtr& type) { doris_udf::FunctionContext::TypeDesc& desc = ut_desc.type_desc; @@ -95,6 +103,14 @@ size_t type_index_to_data_type(const std::vector& input_types, size_t desc.type = doris_udf::FunctionContext::TYPE_DATE; type = std::make_shared(); return 1; + case TypeIndex::DateV2: + desc.type = doris_udf::FunctionContext::TYPE_DATEV2; + type = std::make_shared(); + return 1; + case TypeIndex::DateTimeV2: + desc.type = doris_udf::FunctionContext::TYPE_DATETIMEV2; + type = std::make_shared(); + return 1; case TypeIndex::Array: { desc.type = doris_udf::FunctionContext::TYPE_ARRAY; ut_type::UTDataTypeDesc sub_desc; @@ -200,6 +216,13 @@ bool insert_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const std::any& datetime_str.c_str(), datetime_str.size()); v.cast_to_date(); column->insert_data(reinterpret_cast(&v), 0); + } else if (type.is_date_v2()) { + static std::string date_time_format("%Y-%m-%d"); + auto datetime_str = std::any_cast(cell); + DateV2Value v; + v.from_date_format_str(date_time_format.c_str(), date_time_format.size(), + datetime_str.c_str(), datetime_str.size()); + column->insert_data(reinterpret_cast(&v), 0); } else if (type.is_array()) { auto v = std::any_cast(cell); column->insert(v); diff --git a/be/test/vec/function/function_test_util.h b/be/test/vec/function/function_test_util.h index 6ad30a9b6a..f29ab896c3 100644 --- a/be/test/vec/function/function_test_util.h +++ b/be/test/vec/function/function_test_util.h @@ -45,6 +45,7 @@ using DataSet = std::vector; using InputTypeSet = std::vector; int64_t str_to_date_time(std::string datetime_str, bool data_time = true); +uint32_t str_to_date_v2(std::string datetime_str, std::string datetime_format); namespace ut_type { using TINYINT = int8_t; @@ -220,6 +221,8 @@ void check_function(const std::string& func_name, const InputTypeSet& input_type fn_ctx_return.type = doris_udf::FunctionContext::TYPE_DOUBLE; } else if constexpr (std::is_same_v) { fn_ctx_return.type = doris_udf::FunctionContext::TYPE_DATETIME; + } else if (std::is_same_v) { + fn_ctx_return.type = doris_udf::FunctionContext::TYPE_DATEV2; } else { fn_ctx_return.type = doris_udf::FunctionContext::INVALID_TYPE; } diff --git a/be/test/vec/function/function_time_test.cpp b/be/test/vec/function/function_time_test.cpp index c046ee825a..c03a572435 100644 --- a/be/test/vec/function/function_time_test.cpp +++ b/be/test/vec/function/function_time_test.cpp @@ -455,8 +455,7 @@ TEST(VTimestampFunctionsTest, to_days_test) { DataSet data_set = {{{std::string("2021-01-01 00:00:00")}, 738156}, {{std::string("")}, Null()}, - {{std::string("2021-01-32 00:00:00")}, Null()}, - {{std::string("0000-01-01 00:00:00")}, 1}}; + {{std::string("2021-01-32 00:00:00")}, Null()}}; check_function(func_name, input_types, data_set); } @@ -469,8 +468,7 @@ TEST(VTimestampFunctionsTest, date_test) { DataSet data_set = { {{std::string("2021-01-01 06:00:00")}, str_to_date_time("2021-01-01", false)}, {{std::string("")}, Null()}, - {{Null()}, Null()}, - {{std::string("0000-01-01 00:00:00")}, str_to_date_time("0000-01-01", false)}}; + {{Null()}, Null()}}; check_function(func_name, input_types, data_set); } @@ -566,4 +564,550 @@ TEST(VTimestampFunctionsTest, weekday_test) { check_function(func_name, input_types, data_set); } + +TEST(VTimestampFunctionsTest, day_of_week_v2_test) { + std::string func_name = "dayofweek"; + + InputTypeSet input_types = {TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2001-02-03")}, 7}, + {{std::string("2020-00-01")}, Null()}, + {{std::string("2020-01-00")}, Null()}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, day_of_month_v2_test) { + std::string func_name = "dayofmonth"; + + InputTypeSet input_types = {TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2020-00-01")}, Null()}, + {{std::string("2020-01-01")}, 1}, + {{std::string("2020-02-29")}, 29}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, day_of_year_v2_test) { + std::string func_name = "dayofyear"; + + InputTypeSet input_types = {TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2020-00-01")}, Null()}, + {{std::string("2020-01-00")}, Null()}, + {{std::string("2020-02-29")}, 60}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, week_of_year_v2_test) { + std::string func_name = "weekofyear"; + + InputTypeSet input_types = {TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2020-00-01")}, Null()}, + {{std::string("2020-01-00")}, Null()}, + {{std::string("2020-02-29")}, 9}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, year_v2_test) { + std::string func_name = "year"; + + InputTypeSet input_types = {TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2021-01-01")}, 2021}, + {{std::string("2021-01-00")}, Null()}, + {{std::string("2025-05-01")}, 2025}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, quarter_v2_test) { + std::string func_name = "quarter"; + + InputTypeSet input_types = {TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2021-01-01")}, 1}, + {{std::string("")}, Null()}, + {{std::string("2021-01-32")}, Null()}, + {{std::string("2025-10-23")}, 4}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, month_v2_test) { + std::string func_name = "month"; + + InputTypeSet input_types = {TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2021-01-01")}, 1}, + {{std::string("")}, Null()}, + {{std::string("2021-01-32")}, Null()}, + {{std::string("2025-05-23")}, 5}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, day_v2_test) { + std::string func_name = "day"; + + InputTypeSet input_types = {TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2021-01-01")}, 1}, + {{std::string("")}, Null()}, + {{std::string("2021-01-32")}, Null()}, + {{std::string("2025-05-23")}, 23}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, hour_v2_test) { + std::string func_name = "hour"; + + InputTypeSet input_types = {TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2021-01-01")}, 0}, + {{std::string("2021-01-13")}, 0}, + {{std::string("")}, Null()}, + {{std::string("2025-05-23")}, 0}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, minute_v2_test) { + std::string func_name = "minute"; + + InputTypeSet input_types = {TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2021-01-01")}, 0}, + {{std::string("2021-01-13")}, 0}, + {{std::string("")}, Null()}, + {{std::string("2025-05-23")}, 0}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, second_v2_test) { + std::string func_name = "second"; + + InputTypeSet input_types = {TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2021-01-01")}, 0}, + {{std::string("2021-01-13")}, 0}, + {{std::string("")}, Null()}, + {{std::string("2025-05-23")}, 0}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, timediff_v2_test) { + std::string func_name = "timediff"; + + { + InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2019-07-18"), std::string("2019-07-18")}, 0.0}, + {{std::string("2019-07-18"), std::string("2019-07-18")}, -0.0}, + {{std::string("2019-00-18"), std::string("2019-07-18")}, Null()}, + {{std::string("2019-07-18"), std::string("2019-07-00")}, Null()}}; + + check_function(func_name, input_types, data_set); + } + + { + InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Date}; + + DataSet data_set = {{{std::string("2019-07-18"), std::string("2019-07-18")}, 0.0}, + {{std::string("2019-07-18"), std::string("2019-07-18")}, -0.0}, + {{std::string("2019-00-18"), std::string("2019-07-18")}, Null()}, + {{std::string("2019-07-18"), std::string("2019-07-00")}, Null()}}; + + check_function(func_name, input_types, data_set); + } + + { + InputTypeSet input_types = {TypeIndex::Date, TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2019-07-18"), std::string("2019-07-18")}, 0.0}, + {{std::string("2019-07-18"), std::string("2019-07-18")}, -0.0}, + {{std::string("2019-00-18"), std::string("2019-07-18")}, Null()}, + {{std::string("2019-07-18"), std::string("2019-07-00")}, Null()}}; + + check_function(func_name, input_types, data_set); + } + + { + InputTypeSet input_types = {TypeIndex::DateTime, TypeIndex::DateV2}; + + DataSet data_set = { + {{std::string("2019-07-18 00:00:00"), std::string("2019-07-18")}, 0.0}, + {{std::string("2019-07-18 00:00:00"), std::string("2019-07-18")}, -0.0}, + {{std::string("2019-00-18 00:00:00"), std::string("2019-07-18")}, Null()}, + {{std::string("2019-07-18 00:00:00"), std::string("2019-07-00")}, Null()}}; + + check_function(func_name, input_types, data_set); + } + + { + InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::DateTime}; + + DataSet data_set = { + {{std::string("2019-07-18"), std::string("2019-07-18 00:00:00")}, 0.0}, + {{std::string("2019-07-18"), std::string("2019-07-18 00:00:00")}, -0.0}, + {{std::string("2019-00-18"), std::string("2019-07-18 00:00:00")}, Null()}, + {{std::string("2019-07-18"), std::string("2019-07-00 00:00:00")}, Null()}}; + + check_function(func_name, input_types, data_set); + } +} + +TEST(VTimestampFunctionsTest, datediff_v2_test) { + std::string func_name = "datediff"; + + { + InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2019-07-18"), std::string("2019-07-19")}, -1}, + {{std::string("2019-07-18"), std::string("2019-07-17")}, 1}, + {{std::string("2019-00-18"), std::string("2019-07-18")}, Null()}, + {{std::string("2019-07-18"), std::string("2019-07-00")}, Null()}}; + + check_function(func_name, input_types, data_set); + } + + { + InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Date}; + + DataSet data_set = {{{std::string("2019-07-18"), std::string("2019-07-19")}, -1}, + {{std::string("2019-07-18"), std::string("2019-07-17")}, 1}, + {{std::string("2019-00-18"), std::string("2019-07-18")}, Null()}, + {{std::string("2019-07-18"), std::string("2019-07-00")}, Null()}}; + + check_function(func_name, input_types, data_set); + } + + { + InputTypeSet input_types = {TypeIndex::Date, TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2019-07-18"), std::string("2019-07-19")}, -1}, + {{std::string("2019-07-18"), std::string("2019-07-17")}, 1}, + {{std::string("2019-00-18"), std::string("2019-07-18")}, Null()}, + {{std::string("2019-07-18"), std::string("2019-07-00")}, Null()}}; + + check_function(func_name, input_types, data_set); + } + + { + InputTypeSet input_types = {TypeIndex::DateTime, TypeIndex::DateV2}; + + DataSet data_set = { + {{std::string("2019-07-18 00:00:00"), std::string("2019-07-19")}, -1}, + {{std::string("2019-07-18 00:00:00"), std::string("2019-07-17")}, 1}, + {{std::string("2019-00-18 00:00:00"), std::string("2019-07-18")}, Null()}, + {{std::string("2019-07-18 00:00:00"), std::string("2019-07-00")}, Null()}}; + + check_function(func_name, input_types, data_set); + } + + { + InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::DateTime}; + + DataSet data_set = { + {{std::string("2019-07-18"), std::string("2019-07-19 00:00:00")}, -1}, + {{std::string("2019-07-18"), std::string("2019-07-17 00:00:00")}, 1}, + {{std::string("2019-00-18"), std::string("2019-07-18 00:00:00")}, Null()}, + {{std::string("2019-07-18"), std::string("2019-07-00 00:00:00")}, Null()}}; + + check_function(func_name, input_types, data_set); + } +} + +TEST(VTimestampFunctionsTest, date_format_v2_test) { + std::string func_name = "date_format"; + + InputTypeSet input_types = {TypeIndex::DateV2, Consted {TypeIndex::String}}; + { + DataSet data_set = {{{std::string("2009-10-04 22:23:00"), std::string("%W %M %Y")}, + std::string("Sunday October 2009")}}; + + check_function(func_name, input_types, data_set); + } + { + DataSet data_set = {{{std::string("2007-10-04 22:23:00"), std::string("%H:%i:%s")}, + std::string("00:00:00")}}; + + check_function(func_name, input_types, data_set); + } + { + DataSet data_set = { + {{std::string("1900-10-04 22:23:00"), std::string("%D %y %a %d %m %b %j")}, + std::string("4th 00 Thu 04 10 Oct 277")}}; + + check_function(func_name, input_types, data_set); + } + { + DataSet data_set = { + {{std::string("1997-10-04 22:23:00"), std::string("%H %k %I %r %T %S %w")}, + std::string("00 0 12 12:00:00 AM 00:00:00 00 6")}}; + + check_function(func_name, input_types, data_set); + } + { + DataSet data_set = {{{std::string("1999-01-01 00:00:00"), std::string("%X %V")}, + std::string("1998 52")}}; + + check_function(func_name, input_types, data_set); + } + { + DataSet data_set = { + {{std::string("2006-06-01 00:00:00"), std::string("%d")}, std::string("01")}}; + + check_function(func_name, input_types, data_set); + } + { + DataSet data_set = { + {{std::string("2006-06-01 00:00:00"), std::string("%%%d")}, std::string("%01")}}; + + check_function(func_name, input_types, data_set); + } +} + +TEST(VTimestampFunctionsTest, years_add_v2_test) { + std::string func_name = "years_add"; + + InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32}; + + DataSet data_set = {{{std::string("2020-05-23"), 5}, str_to_date_v2("2025-05-23", "%Y-%m-%d")}, + {{std::string("2020-05-23"), -5}, str_to_date_v2("2015-05-23", "%Y-%m-%d")}, + {{std::string(""), 5}, Null()}, + {{std::string("2020-05-23"), 8000}, Null()}, + {{Null(), 5}, Null()}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, years_sub_v2_test) { + std::string func_name = "years_sub"; + + InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32}; + + DataSet data_set = {{{std::string("2020-05-23"), 5}, str_to_date_v2("2015-05-23", "%Y-%m-%d")}, + {{std::string("2020-05-23"), -5}, str_to_date_v2("2025-05-23", "%Y-%m-%d")}, + {{std::string(""), 5}, Null()}, + {{std::string("2020-05-23"), 3000}, Null()}, + {{Null(), 5}, Null()}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, months_add_v2_test) { + std::string func_name = "months_add"; + + InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32}; + + DataSet data_set = {{{std::string("2020-10-23"), -4}, str_to_date_v2("2020-06-23", "%Y-%m-%d")}, + {{std::string("2020-05-23"), 4}, str_to_date_v2("2020-09-23", "%Y-%m-%d")}, + {{std::string(""), 4}, Null()}, + {{std::string("2020-05-23"), 10}, str_to_date_v2("2021-03-23", "%Y-%m-%d")}, + {{Null(), 4}, Null()}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, months_sub_v2_test) { + std::string func_name = "months_sub"; + + InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32}; + + DataSet data_set = {{{std::string("2020-05-23"), 4}, str_to_date_v2("2020-01-23", "%Y-%m-%d")}, + {{std::string("2020-05-23"), -4}, str_to_date_v2("2020-09-23", "%Y-%m-%d")}, + {{std::string(""), 4}, Null()}, + {{std::string("2020-05-23"), 10}, str_to_date_v2("2019-07-23", "%Y-%m-%d")}, + {{Null(), 4}, Null()}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, days_add_v2_test) { + std::string func_name = "days_add"; + + InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32}; + + DataSet data_set = {{{std::string("2020-10-23"), -4}, str_to_date_v2("2020-10-19", "%Y-%m-%d")}, + {{std::string("2020-05-23"), 4}, str_to_date_v2("2020-05-27", "%Y-%m-%d")}, + {{std::string(""), 4}, Null()}, + {{std::string("2020-05-23"), 10}, str_to_date_v2("2020-06-02", "%Y-%m-%d")}, + {{Null(), 4}, Null()}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, days_sub_v2_test) { + std::string func_name = "days_sub"; + + InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32}; + + DataSet data_set = {{{std::string("2020-05-23"), 4}, str_to_date_v2("2020-05-19", "%Y-%m-%d")}, + {{std::string("2020-05-23"), -4}, str_to_date_v2("2020-05-27", "%Y-%m-%d")}, + {{std::string(""), 4}, Null()}, + {{std::string("2020-05-23"), 31}, str_to_date_v2("2020-04-22", "%Y-%m-%d")}, + {{Null(), 4}, Null()}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, weeks_add_v2_test) { + std::string func_name = "weeks_add"; + + InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32}; + + DataSet data_set = { + {{std::string("2020-10-23"), 5}, str_to_date_v2("2020-11-27", "%Y-%m-%d")}, + {{std::string("2020-05-23"), -5}, str_to_date_v2("2020-04-18", "%Y-%m-%d")}, + {{std::string(""), 4}, Null()}, + {{std::string("2020-05-23"), 100}, str_to_date_v2("2022-04-23", "%Y-%m-%d")}, + {{Null(), 4}, Null()}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, weeks_sub_v2_test) { + std::string func_name = "weeks_sub"; + + InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32}; + + DataSet data_set = { + {{std::string("2020-05-23"), 5}, str_to_date_v2("2020-04-18", "%Y-%m-%d")}, + {{std::string("2020-05-23"), -5}, str_to_date_v2("2020-06-27", "%Y-%m-%d")}, + {{std::string(""), 4}, Null()}, + {{std::string("2020-05-23"), 100}, str_to_date_v2("2018-06-23", "%Y-%m-%d")}, + {{Null(), 4}, Null()}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, to_days_v2_test) { + std::string func_name = "to_days"; + + InputTypeSet input_types = {TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2021-01-01")}, 738156}, + {{std::string("")}, Null()}, + {{std::string("2021-01-32")}, Null()}, + {{std::string("0000-01-01")}, Null()}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, date_v2_test) { + std::string func_name = "date"; + + InputTypeSet input_types = {TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2021-01-01")}, str_to_date_v2("2021-01-01", "%Y-%m-%d")}, + {{std::string("")}, Null()}, + {{Null()}, Null()}, + {{std::string("0000-01-01")}, Null()}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, week_v2_test) { + std::string func_name = "week"; + + InputTypeSet new_input_types = {TypeIndex::DateV2}; + DataSet new_data_set = {{{std::string("1989-03-21")}, 12}, + {{std::string("")}, Null()}, + {{std::string("9999-12-12")}, 50}}; + + check_function(func_name, new_input_types, new_data_set); +} + +TEST(VTimestampFunctionsTest, yearweek_v2_test) { + std::string func_name = "yearweek"; + + InputTypeSet new_input_types = {TypeIndex::DateV2}; + DataSet new_data_set = {{{std::string("1989-03-21")}, 198912}, + {{std::string("")}, Null()}, + {{std::string("9999-12-12")}, 999950}}; + + check_function(func_name, new_input_types, new_data_set); +} + +TEST(VTimestampFunctionsTest, str_to_date_test) { + std::string func_name = "str_to_date"; + + InputTypeSet input_types = {TypeIndex::String, TypeIndex::String}; + + { + DataSet data_set = {{{std::string("2021-01-01"), std::string("%Y-%m-%d")}, + str_to_date_time("2021-01-01", false)}, + {{std::string("2022-01-03"), std::string("%Y-%m-%d")}, + str_to_date_time("2022-01-03", false)}, + {{std::string("2021-00-01"), std::string("%Y-%m-%d")}, Null()}, + {{std::string("2021-01-00"), std::string("%Y-%m-%d")}, Null()}}; + + check_function(func_name, input_types, data_set); + } +} + +TEST(VTimestampFunctionsTest, from_days_test) { + std::string func_name = "from_days"; + + InputTypeSet input_types = {TypeIndex::Int32}; + + { + DataSet data_set = {{{730669}, str_to_date_time("2000-07-03", false)}, {{0}, Null()}}; + + check_function(func_name, input_types, data_set); + } +} + +TEST(VTimestampFunctionsTest, weekday_v2_test) { + std::string func_name = "weekday"; + + InputTypeSet input_types = {TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2001-02-03")}, 5}, + {{std::string("2019-06-25")}, 1}, + {{std::string("2020-00-01")}, Null()}, + {{std::string("2020-01-00")}, Null()}}; + + check_function(func_name, input_types, data_set); +} + +TEST(VTimestampFunctionsTest, dayname_test) { + std::string func_name = "dayname"; + + { + InputTypeSet input_types = {TypeIndex::DateV2}; + + DataSet data_set = {{{std::string("2007-02-03")}, std::string("Saturday")}, + {{std::string("2020-01-00")}, Null()}}; + + check_function(func_name, input_types, data_set); + } + + { + InputTypeSet input_types = {TypeIndex::DateTime}; + + DataSet data_set = {{{std::string("2007-02-03 00:00:00")}, std::string("Saturday")}, + {{std::string("2020-01-00 00:00:00")}, Null()}}; + + check_function(func_name, input_types, data_set); + } + + { + InputTypeSet input_types = {TypeIndex::Date}; + + DataSet data_set = {{{std::string("2007-02-03")}, std::string("Saturday")}, + {{std::string("2020-01-00")}, Null()}}; + + check_function(func_name, input_types, data_set); + } +} + } // namespace doris::vectorized diff --git a/be/test/vec/runtime/vdatetime_value_test.cpp b/be/test/vec/runtime/vdatetime_value_test.cpp new file mode 100644 index 0000000000..727fdc2737 --- /dev/null +++ b/be/test/vec/runtime/vdatetime_value_test.cpp @@ -0,0 +1,247 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/runtime/vdatetime_value.h" + +#include + +#include +#include + +namespace doris::vectorized { + +TEST(VDateTimeValueTest, date_v2_to_uint32_test) { + uint16_t year = 2022; + uint8_t month = 5; + uint8_t day = 24; + + DateV2Value date_v2; + date_v2.set_time(year, month, day); + + EXPECT_TRUE(date_v2.year() == year); + EXPECT_TRUE(date_v2.month() == month); + EXPECT_TRUE(date_v2.day() == day); + EXPECT_TRUE(date_v2.to_date_uint32() == ((year << 16) | (month << 8) | day)); + EXPECT_TRUE(date_v2.hour() == 0); + EXPECT_TRUE(date_v2.minute() == 0); + EXPECT_TRUE(date_v2.second() == 0); +} + +TEST(VDateTimeValueTest, date_v2_from_uint32_test) { + { + uint16_t year = 2022; + uint8_t month = 5; + uint8_t day = 24; + + DateV2Value date_v2; + date_v2.from_date((uint32_t)((year << 16) | (month << 8) | day)); + + EXPECT_TRUE(date_v2.year() == year); + EXPECT_TRUE(date_v2.month() == month); + EXPECT_TRUE(date_v2.day() == day); + EXPECT_TRUE(date_v2.to_date_uint32() == ((year << 16) | (month << 8) | day)); + EXPECT_TRUE(date_v2.hour() == 0); + EXPECT_TRUE(date_v2.minute() == 0); + EXPECT_TRUE(date_v2.second() == 0); + } + { + uint16_t year = 2022; + uint8_t month = 5; + uint8_t day = 24; + + uint32_t ui32 = (uint32_t)((year << 16) | (month << 8) | day); + auto date_v2 = (DateV2Value&)ui32; + + EXPECT_TRUE(date_v2.year() == year); + EXPECT_TRUE(date_v2.month() == month); + EXPECT_TRUE(date_v2.day() == day); + EXPECT_TRUE(date_v2.to_date_uint32() == ((year << 16) | (month << 8) | day)); + EXPECT_TRUE(date_v2.hour() == 0); + EXPECT_TRUE(date_v2.minute() == 0); + EXPECT_TRUE(date_v2.second() == 0); + } +} + +TEST(VDateTimeValueTest, date_v2_from_date_format_str_test) { + uint16_t year = 2022; + uint8_t month = 5; + uint8_t day = 24; + + { + DateV2Value date_v2; + std::string origin_date = "2022-05-24"; + std::string date_format = "%Y-%m-%d"; + EXPECT_TRUE(date_v2.from_date_format_str(date_format.data(), date_format.size(), + origin_date.data(), origin_date.size())); + + EXPECT_TRUE(date_v2.year() == year); + EXPECT_TRUE(date_v2.month() == month); + EXPECT_TRUE(date_v2.day() == day); + EXPECT_TRUE(date_v2.to_date_uint32() == ((year << 16) | (month << 8) | day)); + EXPECT_TRUE(date_v2.hour() == 0); + EXPECT_TRUE(date_v2.minute() == 0); + EXPECT_TRUE(date_v2.second() == 0); + } + + { + DateV2Value date_v2; + std::string origin_date = "2022-05-24 10:10:00"; + std::string date_format = "%Y-%m-%d"; + EXPECT_TRUE(date_v2.from_date_format_str(date_format.data(), date_format.size(), + origin_date.data(), origin_date.size())); + + EXPECT_TRUE(date_v2.year() == year); + EXPECT_TRUE(date_v2.month() == month); + EXPECT_TRUE(date_v2.day() == day); + EXPECT_TRUE(date_v2.to_date_uint32() == ((year << 16) | (month << 8) | day)); + EXPECT_TRUE(date_v2.hour() == 0); + EXPECT_TRUE(date_v2.minute() == 0); + EXPECT_TRUE(date_v2.second() == 0); + } +} + +TEST(VDateTimeValueTest, date_diff_test) { + { + DateV2Value date_v2_1; + std::string origin_date1 = "2022-05-24"; + std::string date_format1 = "%Y-%m-%d"; + EXPECT_TRUE(date_v2_1.from_date_format_str(date_format1.data(), date_format1.size(), + origin_date1.data(), origin_date1.size())); + + DateV2Value date_v2_2; + std::string origin_date2 = "2022-06-24"; + std::string date_format2 = "%Y-%m-%d"; + EXPECT_TRUE(date_v2_2.from_date_format_str(date_format2.data(), date_format2.size(), + origin_date2.data(), origin_date2.size())); + + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 0); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 1); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31 * 24); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31 * 24 * 60); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31 * 24 * 60 * 60); + } + + { + DateV2Value date_v2_1; + std::string origin_date1 = "2022-05-24"; + std::string date_format1 = "%Y-%m-%d"; + EXPECT_TRUE(date_v2_1.from_date_format_str(date_format1.data(), date_format1.size(), + origin_date1.data(), origin_date1.size())); + + VecDateTimeValue date_v2_2; + std::string origin_date2 = "2022-06-24 00:00:00"; + std::string date_format2 = "%Y-%m-%d %H:%i:%s"; + EXPECT_TRUE(date_v2_2.from_date_format_str(date_format2.data(), date_format2.size(), + origin_date2.data(), origin_date2.size())); + + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 0); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 1); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31 * 24); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31 * 24 * 60); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31 * 24 * 60 * 60); + } + + { + VecDateTimeValue date_v2_1; + std::string origin_date1 = "2022-05-24 00:00:00"; + std::string date_format1 = "%Y-%m-%d %H:%i:%s"; + EXPECT_TRUE(date_v2_1.from_date_format_str(date_format1.data(), date_format1.size(), + origin_date1.data(), origin_date1.size())); + + DateV2Value date_v2_2; + std::string origin_date2 = "2022-06-24"; + std::string date_format2 = "%Y-%m-%d"; + EXPECT_TRUE(date_v2_2.from_date_format_str(date_format2.data(), date_format2.size(), + origin_date2.data(), origin_date2.size())); + + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 0); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 1); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31 * 24); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31 * 24 * 60); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31 * 24 * 60 * 60); + } + + { + DateV2Value date_v2_1; + std::string origin_date1 = "2022-05-24"; + std::string date_format1 = "%Y-%m-%d"; + EXPECT_TRUE(date_v2_1.from_date_format_str(date_format1.data(), date_format1.size(), + origin_date1.data(), origin_date1.size())); + + VecDateTimeValue date_v2_2; + std::string origin_date2 = "2022-06-24 06:00:00"; + std::string date_format2 = "%Y-%m-%d %H:%i:%s"; + EXPECT_TRUE(date_v2_2.from_date_format_str(date_format2.data(), date_format2.size(), + origin_date2.data(), origin_date2.size())); + + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 0); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 1); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31 * 24 + 6); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == (31 * 24 + 6) * 60); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == + (31 * 24 + 6) * 60 * 60); + } + + { + VecDateTimeValue date_v2_1; + std::string origin_date1 = "2022-05-24"; + std::string date_format1 = "%Y-%m-%d"; + EXPECT_TRUE(date_v2_1.from_date_format_str(date_format1.data(), date_format1.size(), + origin_date1.data(), origin_date1.size())); + + VecDateTimeValue date_v2_2; + std::string origin_date2 = "2022-06-24 06:00:00"; + std::string date_format2 = "%Y-%m-%d %H:%i:%s"; + EXPECT_TRUE(date_v2_2.from_date_format_str(date_format2.data(), date_format2.size(), + origin_date2.data(), origin_date2.size())); + + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 0); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 1); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31 * 24 + 6); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == (31 * 24 + 6) * 60); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == + (31 * 24 + 6) * 60 * 60); + } + + { + VecDateTimeValue date_v2_1; + std::string origin_date1 = "2022-05-24 06:00:00"; + std::string date_format1 = "%Y-%m-%d %H:%i:%s"; + EXPECT_TRUE(date_v2_1.from_date_format_str(date_format1.data(), date_format1.size(), + origin_date1.data(), origin_date1.size())); + + VecDateTimeValue date_v2_2; + std::string origin_date2 = "2022-06-24 06:00:00"; + std::string date_format2 = "%Y-%m-%d %H:%i:%s"; + EXPECT_TRUE(date_v2_2.from_date_format_str(date_format2.data(), date_format2.size(), + origin_date2.data(), origin_date2.size())); + + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 0); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 1); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31 * 24); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31 * 24 * 60); + EXPECT_TRUE(datetime_diff(date_v2_1, date_v2_2) == 31 * 24 * 60 * 60); + } +} + +} // namespace doris::vectorized diff --git a/be/test/vec/utils/arrow_column_to_doris_column_test.cpp b/be/test/vec/utils/arrow_column_to_doris_column_test.cpp index 73181a57de..606132ca9d 100644 --- a/be/test/vec/utils/arrow_column_to_doris_column_test.cpp +++ b/be/test/vec/utils/arrow_column_to_doris_column_test.cpp @@ -123,7 +123,8 @@ void test_arrow_to_datetime_column(std::shared_ptr type, ColumnWithTy if constexpr (std::is_same_v) { time_zone = type->timezone(); } - auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, num_elements, time_zone); + auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, column.type, + num_elements, time_zone); ASSERT_EQ(ret.ok(), true); ASSERT_EQ(column.column->size(), counter); MutableColumnPtr data_column = nullptr; @@ -216,7 +217,8 @@ void test_arrow_to_numeric_column(std::shared_ptr type, ColumnWithTyp ASSERT_EQ(column.column->size(), counter); auto array = create_constant_numeric_array(num_elements, arrow_numeric, type, counter); - auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, num_elements, "UTC"); + auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, column.type, + num_elements, "UTC"); ASSERT_EQ(ret.ok(), true); ASSERT_EQ(column.column->size(), counter); MutableColumnPtr data_column = nullptr; @@ -351,7 +353,8 @@ void test_arrow_to_decimal_column(std::shared_ptr type, int128_t arrow_value, int128_t expect_value, size_t& counter) { ASSERT_EQ(column.column->size(), counter); auto array = create_decimal_array(num_elements, arrow_value, type, counter); - auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, num_elements, "UTC"); + auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, column.type, + num_elements, "UTC"); ASSERT_EQ(ret.ok(), true); ASSERT_EQ(column.column->size(), counter); MutableColumnPtr data_column = nullptr; @@ -452,7 +455,8 @@ void test_arrow_to_fixed_binary_column(ColumnWithTypeAndName& column, size_t num ASSERT_EQ(column.column->size(), counter); auto array = create_fixed_size_binary_array(num_elements, value, counter); - auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, num_elements, "UTC"); + auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, column.type, + num_elements, "UTC"); ASSERT_EQ(ret.ok(), true); ASSERT_EQ(column.column->size(), counter); MutableColumnPtr data_column = nullptr; @@ -554,7 +558,8 @@ void test_arrow_to_binary_column(ColumnWithTypeAndName& column, size_t num_eleme ArrowCppType value, size_t& counter) { ASSERT_EQ(column.column->size(), counter); auto array = create_binary_array(num_elements, value, counter); - auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, num_elements, "UTC"); + auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, column.type, + num_elements, "UTC"); ASSERT_EQ(ret.ok(), true); ASSERT_EQ(column.column->size(), counter); MutableColumnPtr data_column = nullptr; @@ -650,8 +655,8 @@ void test_arrow_to_array_column(ColumnWithTypeAndName& column, ASSERT_EQ(column.column->size(), counter); auto array = create_array_array(vec_offsets, null_map, value_type, values, counter); - auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, vec_offsets.size() - 1, - "UTC"); + auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, column.type, + vec_offsets.size() - 1, "UTC"); ASSERT_EQ(ret.ok(), true); ASSERT_EQ(column.column->size(), counter); MutableColumnPtr data_column = nullptr; diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 3b1404fc2e..0d562e1620 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -4652,11 +4652,15 @@ type ::= | KW_DOUBLE {: RESULT = Type.DOUBLE; :} | KW_DATE - {: RESULT = Type.DATE; :} + {: RESULT = ScalarType.createDateType(); :} + | KW_DATETIME LPAREN INTEGER_LITERAL:precision RPAREN + {: RESULT = ScalarType.createDatetimeV2Type(precision.intValue()); :} | KW_DATETIME - {: RESULT = Type.DATETIME; :} + {: RESULT = ScalarType.createDatetimeType(); :} + | KW_TIME LPAREN INTEGER_LITERAL:precision RPAREN + {: RESULT = ScalarType.createTimeV2Type(precision.intValue()); :} | KW_TIME - {: RESULT = Type.TIME; :} + {: RESULT = ScalarType.createTimeType(); :} | KW_BITMAP {: RESULT = Type.BITMAP; :} | KW_QUANTILE_STATE diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java index 5b4ed33e1c..e4f8bc1ea0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java @@ -138,8 +138,7 @@ public class CastExpr extends Expr { private static boolean disableRegisterCastingFunction(Type fromType, Type toType) { // Disable casting from boolean to decimal or datetime or date - if (fromType.isBoolean() - && (toType.equals(Type.DECIMALV2) || toType.equals(Type.DATETIME) || toType.equals(Type.DATE))) { + if (fromType.isBoolean() && (toType.equals(Type.DECIMALV2) || toType.isDateType())) { return true; } @@ -166,7 +165,9 @@ public class CastExpr extends Expr { beClass = "TimeOperators"; } String typeName = Function.getUdfTypeName(toType.getPrimitiveType()); - if (toType.getPrimitiveType() == PrimitiveType.DATE) { + // only refactor date/datetime for vectorized engine. + if (toType.getPrimitiveType() == PrimitiveType.DATE + || toType.getPrimitiveType() == PrimitiveType.DATEV2) { typeName = "date_val"; } String beSymbol = "doris::" + beClass + "::cast_to_" diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java index e98f4c2dfc..b1b471c603 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java @@ -338,11 +338,13 @@ public class ColumnDef { decimalLiteral.checkPrecisionAndScale(scalarType.getScalarPrecision(), scalarType.getScalarScale()); break; case DATE: - new DateLiteral(defaultValue, type); + case DATEV2: + new DateLiteral(defaultValue, DateLiteral.getDefaultDateType(type)); break; case DATETIME: + case DATETIMEV2: if (defaultValueExprDef == null) { - new DateLiteral(defaultValue, type); + new DateLiteral(defaultValue, DateLiteral.getDefaultDateType(type)); } else { if (defaultValueExprDef.getExprName().equals(DefaultValue.NOW)) { break; @@ -360,11 +362,8 @@ public class ColumnDef { } break; case BITMAP: - break; case ARRAY: - break; case MAP: - break; case STRUCT: break; case BOOLEAN: diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java index 9115a25391..74f9c29ad7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java @@ -617,10 +617,13 @@ public class CreateFunctionStmt extends DdlStmt { typeBuilder.setId(Types.PGenericType.TypeId.BITMAP); break; case DATE: + case DATEV2: typeBuilder.setId(Types.PGenericType.TypeId.DATE); break; case DATETIME: + case DATETIMEV2: case TIME: + case TIMEV2: typeBuilder.setId(Types.PGenericType.TypeId.DATETIME); break; case DECIMALV2: diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java index 01cf141c71..bb4eed5615 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java @@ -23,7 +23,6 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; @@ -379,7 +378,7 @@ public class DataDescription { } private static void validateNowFunction(Column mappingColumn) throws AnalysisException { - if (!mappingColumn.getOriginType().equals(Type.DATE) && !mappingColumn.getOriginType().equals(Type.DATETIME)) { + if (!mappingColumn.getOriginType().isDateType()) { throw new AnalysisException("Now() function is only support for DATE/DATETIME column"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java index 24f552e100..e5953ea44d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -21,8 +21,10 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.InvalidFormatException; import org.apache.doris.thrift.TDateLiteral; import org.apache.doris.thrift.TExprNode; @@ -55,8 +57,6 @@ public class DateLiteral extends LiteralExpr { private static final DateLiteral MAX_DATE = new DateLiteral(9999, 12, 31); private static final DateLiteral MIN_DATETIME = new DateLiteral(0000, 1, 1, 0, 0, 0); private static final DateLiteral MAX_DATETIME = new DateLiteral(9999, 12, 31, 23, 59, 59); - public static final DateLiteral UNIX_EPOCH_TIME = new DateLiteral(1970, 01, 01, 00, 00, 00); - private static final int DATEKEY_LENGTH = 8; private static final int MAX_MICROSECOND = 999999; private static final int DATETIME_TO_MINUTE_STRING_LENGTH = 16; @@ -152,7 +152,10 @@ public class DateLiteral extends LiteralExpr { //Date Literal persist type in meta private enum DateLiteralType { DATETIME(0), - DATE(1); + DATE(1), + + DATETIMEV2(2), + DATEV2(3); private final int value; private DateLiteralType(int value) { @@ -193,7 +196,7 @@ public class DateLiteral extends LiteralExpr { analysisDone(); } - public DateLiteral(long unixTimestamp, TimeZone timeZone, Type type) { + public DateLiteral(long unixTimestamp, TimeZone timeZone, Type type) throws AnalysisException { DateTime dt = new DateTime(unixTimestamp, DateTimeZone.forTimeZone(timeZone)); year = dt.getYear(); month = dt.getMonthOfYear(); @@ -206,8 +209,17 @@ public class DateLiteral extends LiteralExpr { minute = 0; second = 0; this.type = Type.DATE; - } else { + } else if (type.equals(Type.DATETIME)) { this.type = Type.DATETIME; + } else if (type.equals(Type.DATEV2)) { + hour = 0; + minute = 0; + second = 0; + this.type = Type.DATEV2; + } else if (type.equals(Type.DATETIMEV2)) { + this.type = Type.DATETIMEV2; + } else { + throw new AnalysisException("Error date literal type : " + type); } } @@ -218,7 +230,23 @@ public class DateLiteral extends LiteralExpr { this.year = year; this.month = month; this.day = day; - this.type = Type.DATE; + try { + this.type = DateLiteral.getDefaultDateType(Type.DATE); + } catch (AnalysisException e) { + this.type = Type.DATE; + } + } + + public DateLiteral(long year, long month, long day, Type type) { + this.hour = hour; + this.minute = minute; + this.second = second; + this.year = year; + this.month = month; + this.day = day; + Preconditions.checkArgument(type.getPrimitiveType().equals(Type.DATE.getPrimitiveType()) + || type.getPrimitiveType().equals(Type.DATEV2.getPrimitiveType())); + this.type = type; } public DateLiteral(long year, long month, long day, long hour, long minute, long second) { @@ -228,7 +256,34 @@ public class DateLiteral extends LiteralExpr { this.year = year; this.month = month; this.day = day; - this.type = Type.DATETIME; + try { + this.type = DateLiteral.getDefaultDateType(Type.DATETIME); + } catch (AnalysisException e) { + this.type = Type.DATETIME; + } + } + + public DateLiteral(long year, long month, long day, long hour, long minute, long second, long microsecond) { + this.hour = hour; + this.minute = minute; + this.second = second; + this.year = year; + this.month = month; + this.day = day; + this.microsecond = microsecond; + this.type = Type.DATETIMEV2; + } + + public DateLiteral(long year, long month, long day, long hour, long minute, long second, Type type) { + this.hour = hour; + this.minute = minute; + this.second = second; + this.year = year; + this.month = month; + this.day = day; + Preconditions.checkArgument(type.getPrimitiveType().equals(Type.DATETIME.getPrimitiveType()) + || type.getPrimitiveType().equals(Type.DATETIMEV2.getPrimitiveType())); + this.type = type; } public DateLiteral(LocalDateTime dateTime, Type type) { @@ -261,7 +316,7 @@ public class DateLiteral extends LiteralExpr { try { Preconditions.checkArgument(type.isDateType()); LocalDateTime dateTime; - if (type.equals(Type.DATE)) { + if (type.equals(Type.DATE) || type.equals(Type.DATEV2)) { if (s.split("-")[0].length() == 2) { dateTime = DATE_FORMATTER_TWO_DIGIT.parseLocalDateTime(s); } else if (s.length() == DATEKEY_LENGTH && !s.contains("-")) { @@ -317,8 +372,10 @@ public class DateLiteral extends LiteralExpr { public boolean isMinValue() { switch (type.getPrimitiveType()) { case DATE: + case DATEV2: return this.getStringValue().compareTo(MIN_DATE.getStringValue()) == 0; case DATETIME: + case DATETIMEV2: return this.getStringValue().compareTo(MIN_DATETIME.getStringValue()) == 0; default: return false; @@ -331,6 +388,11 @@ public class DateLiteral extends LiteralExpr { return year * 16 * 32L + month * 32 + day; } else if (type.equals(Type.DATETIME)) { return (year * 10000 + month * 100 + day) * 1000000L + hour * 10000 + minute * 100 + second; + } else if (type.equals(Type.DATEV2)) { + return (year << 16) | (month << 8) | day; + } else if (type.equals(Type.DATETIMEV2)) { + return (year << 50) | (month << 46) | (day << 41) | (hour << 36) + | (minute << 30) | (second << 24) | microsecond; } else { Preconditions.checkState(false, "invalid date type: " + type); return -1L; @@ -340,6 +402,8 @@ public class DateLiteral extends LiteralExpr { // Date column and Datetime column's hash value is not same. @Override public ByteBuffer getHashValue(PrimitiveType type) { + // This hash value should be computed using new String since precision is introduced to datetime. + // But it is hard to keep compatibility. So I don't change this function here. String value = convertToString(type); ByteBuffer buffer; try { @@ -370,12 +434,25 @@ public class DateLiteral extends LiteralExpr { @Override public String getStringValue() { - return convertToString(type.getPrimitiveType()); + if (type.isDate() || type.isDateV2()) { + return String.format("%04d-%02d-%02d", year, month, day); + } else if (type.isDatetimeV2()) { + String s = String.format("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second); + if (((ScalarType) type).decimalScale() == 0) { + return s; + } + return s + "." + microsecond / (10L * (6 - ((ScalarType) type).decimalScale())); + } else { + return String.format("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second); + } } private String convertToString(PrimitiveType type) { - if (type == PrimitiveType.DATE) { + if (type == PrimitiveType.DATE || type == PrimitiveType.DATEV2) { return String.format("%04d-%02d-%02d", year, month, day); + } else if (type == PrimitiveType.DATETIMEV2) { + return String.format("%04d-%02d-%02d %02d:%02d:%02d.%06d", + year, month, day, hour, minute, second, microsecond); } else { return String.format("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second); } @@ -403,10 +480,13 @@ public class DateLiteral extends LiteralExpr { if (type.equals(targetType)) { return this; } - if (targetType.equals(Type.DATE)) { + if (targetType.equals(Type.DATE) || targetType.equals(Type.DATEV2)) { return new DateLiteral(this.year, this.month, this.day); } else if (targetType.equals(Type.DATETIME)) { return new DateLiteral(this.year, this.month, this.day, this.hour, this.minute, this.second); + } else if (targetType.isDatetimeV2()) { + return new DateLiteral(this.year, this.month, this.day, this.hour, this.minute, this.microsecond, + targetType); } else { throw new AnalysisException("Error date literal type : " + type); } @@ -420,7 +500,11 @@ public class DateLiteral extends LiteralExpr { } public void castToDate() { - this.type = Type.DATE; + if (Config.use_date_v2_by_default) { + this.type = Type.DATEV2; + } else { + this.type = Type.DATE; + } hour = 0; minute = 0; second = 0; @@ -433,31 +517,43 @@ public class DateLiteral extends LiteralExpr { return packedDatetime; } + private long makePackedDatetimeV2() { + return (year << 50) | (month << 46) | (day << 41) | (hour << 36) + | (minute << 30) | (second << 24) | microsecond; + } + @Override public void write(DataOutput out) throws IOException { super.write(out); //set flag bit in meta, 0 is DATETIME and 1 is DATE if (this.type.equals(Type.DATETIME)) { out.writeShort(DateLiteralType.DATETIME.value()); + out.writeLong(makePackedDatetime()); } else if (this.type.equals(Type.DATE)) { out.writeShort(DateLiteralType.DATE.value()); + out.writeLong(makePackedDatetime()); + } else if (this.type.equals(Type.DATETIMEV2)) { + out.writeShort(DateLiteralType.DATETIMEV2.value()); + out.writeLong(makePackedDatetimeV2()); + } else if (this.type.equals(Type.DATEV2)) { + out.writeShort(DateLiteralType.DATEV2.value()); + out.writeLong(makePackedDatetimeV2()); } else { throw new IOException("Error date literal type : " + type); } - out.writeLong(makePackedDatetime()); } private void fromPackedDatetime(long packedTime) { microsecond = (packedTime % (1L << 24)); long ymdhms = (packedTime >> 24); long ymd = ymdhms >> 17; - long hms = ymdhms % (1 << 17); - day = ymd % (1 << 5); long ym = ymd >> 5; month = ym % 13; year = ym / 13; year %= 10000; + + long hms = ymdhms % (1 << 17); second = hms % (1 << 6); minute = (hms >> 6) % (1 << 6); hour = (hms >> 12); @@ -466,6 +562,24 @@ public class DateLiteral extends LiteralExpr { this.type = Type.DATETIME; } + private void fromPackedDatetimeV2(long packedTime) { + microsecond = (packedTime % (1L << 24)); + long ymdhms = (packedTime >> 24); + long ymd = ymdhms >> 17; + day = ymd % (1 << 5); + long ym = ymd >> 5; + month = ym % (1 << 4); + year = ym >> 4; + + long hms = ymdhms % (1 << 17); + second = hms % (1 << 6); + minute = (hms >> 6) % (1 << 6); + hour = (hms >> 12); + // set default date literal type to DATETIME + // date literal read from meta will set type by flag bit; + this.type = Type.DATETIMEV2; + } + public void readFields(DataInput in) throws IOException { super.readFields(in); short dateLiteralType = in.readShort(); @@ -474,6 +588,12 @@ public class DateLiteral extends LiteralExpr { this.type = Type.DATETIME; } else if (dateLiteralType == DateLiteralType.DATE.value()) { this.type = Type.DATE; + } else if (dateLiteralType == DateLiteralType.DATETIMEV2.value()) { + fromPackedDatetime(in.readLong()); + this.type = Type.DATETIMEV2; + } else if (dateLiteralType == DateLiteralType.DATEV2.value()) { + fromPackedDatetime(in.readLong()); + this.type = Type.DATEV2; } else { throw new IOException("Error date literal type : " + type); } @@ -612,17 +732,17 @@ public class DateLiteral extends LiteralExpr { case 'y': // %y Year, numeric (two digits) builder.appendTwoDigitYear(2020); break; + // TODO(Gabriel): support microseconds in date literal case 'f': // %f Microseconds (000000..999999) case 'w': // %w Day of the week (0=Sunday..6=Saturday) case 'U': // %U Week (00..53), where Sunday is the first day of the week case 'u': // %u Week (00..53), where Monday is the first day of the week case 'V': // %V Week (01..53), where Sunday is the first day of the week; used with %X - case 'X': - // %X Year for the week where Sunday is the first day of the week, + case 'X': // %X Year for the week where Sunday is the first day of the week, // numeric, four digits; used with %V case 'D': // %D Day of the month with English suffix (0th, 1st, 2nd, 3rd, …) - throw new AnalysisException( - String.format("%%%s not supported in date format string", character)); + throw new AnalysisException(String.format("%%%s not supported in date format string", + character)); case '%': // %% A literal "%" character builder.appendLiteral('%'); break; @@ -641,9 +761,9 @@ public class DateLiteral extends LiteralExpr { } public LocalDateTime getTimeFormatter() throws AnalysisException { - if (type.equals(Type.DATE)) { + if (type.equals(Type.DATE) || type.equals(Type.DATEV2)) { return DATE_FORMATTER.parseLocalDateTime(getStringValue()); - } else if (type.equals(Type.DATETIME)) { + } else if (type.equals(Type.DATETIME) || type.equals(Type.DATETIMEV2)) { return DATE_TIME_FORMATTER.parseLocalDateTime(getStringValue()); } else { throw new AnalysisException("Not support date literal type"); @@ -706,7 +826,6 @@ public class DateLiteral extends LiteralExpr { private long second; private long microsecond; - @Override public int hashCode() { return 31 * super.hashCode() + Objects.hashCode(unixTimestamp(TimeZone.getDefault())); @@ -963,8 +1082,8 @@ public class DateLiteral extends LiteralExpr { } } else if (format.charAt(fp) != ' ') { if (format.charAt(fp) != value.charAt(vp)) { - throw new InvalidFormatException("Invalid char: " + value.charAt(vp) - + ", expected: " + format.charAt(fp)); + throw new InvalidFormatException("Invalid char: " + value.charAt(vp) + ", expected: " + + format.charAt(fp)); } fp++; vp++; @@ -1041,6 +1160,8 @@ public class DateLiteral extends LiteralExpr { } // Compute timestamp type + // TODO(Gabriel): we still use old version datetime/date and change this to new version when + // we think it's stable enough if (datePartUsed) { if (timePartUsed) { this.type = Type.DATETIME; @@ -1055,6 +1176,31 @@ public class DateLiteral extends LiteralExpr { return 0; } + public int fromDateFormatStr(String format, String value, boolean hasSubVal, Type type) + throws InvalidFormatException { + switch (type.getPrimitiveType()) { + case DATETIME: + case DATE: + return fromDateFormatStr(format, value, hasSubVal); + default: + int val = fromDateFormatStr(format, value, hasSubVal); + convertTypeToV2(); + return val; + } + } + + private void convertTypeToV2() { + switch (type.getPrimitiveType()) { + case DATETIME: + this.type = Type.DATETIMEV2; + break; + case DATE: + this.type = Type.DATEV2; + break; + default: + } + } + private boolean checkRange() { return year > MAX_DATETIME.year || month > MAX_DATETIME.month || day > MAX_DATETIME.day || hour > MAX_DATETIME.hour || minute > MAX_DATETIME.minute || second > MAX_DATETIME.second @@ -1285,4 +1431,38 @@ public class DateLiteral extends LiteralExpr { throw new AnalysisException("Datetime value is out of range: " + dateStr); } } + + public void fromDateStr(String dateStr, Type type) throws AnalysisException { + switch (type.getPrimitiveType()) { + case DATETIME: + case DATE: + fromDateStr(dateStr); + break; + default: + fromDateStr(dateStr); + convertTypeToV2(); + } + } + + public static Type getDefaultDateType(Type type) throws AnalysisException { + switch (type.getPrimitiveType()) { + case DATE: + if (Config.use_date_v2_by_default) { + return Type.DATEV2; + } else { + return Type.DATE; + } + case DATETIME: + if (Config.use_date_v2_by_default) { + return Type.DATETIMEV2; + } else { + return Type.DATETIME; + } + case DATEV2: + case DATETIMEV2: + return type; + default: + throw new AnalysisException("Invalid date type: " + type); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java index f9b79d7ccc..37b405390f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java @@ -20,6 +20,7 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.NotImplementedException; import org.apache.doris.thrift.TExprNode; import org.apache.doris.thrift.TExprNodeType; @@ -129,12 +130,27 @@ public class FloatLiteral extends LiteralExpr { public String getStringValue() { // TODO: Here is weird use float to represent TIME type // rethink whether it is reasonable to use this way - if (type.equals(Type.TIME)) { + if (type.equals(Type.TIME) || type.equals(Type.TIMEV2)) { return timeStrFromFloat(value); } return Double.toString(value); } + public static Type getDefaultTimeType(Type type) throws AnalysisException { + switch (type.getPrimitiveType()) { + case TIME: + if (Config.use_date_v2_by_default) { + return Type.TIMEV2; + } else { + return Type.TIME; + } + case TIMEV2: + return type; + default: + throw new AnalysisException("Invalid time type: " + type); + } + } + @Override public long getLongValue() { return (long) value; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 03c4cbe6af..6b4ee81dc7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -1042,12 +1042,12 @@ public class FunctionCallExpr extends Expr { Expr child1Result = getChild(1).getResultValue(); if (child1Result instanceof StringLiteral) { if (DateLiteral.hasTimePart(((StringLiteral) child1Result).getStringValue())) { - this.type = Type.DATETIME; + this.type = DateLiteral.getDefaultDateType(Type.DATETIME); } else { - this.type = Type.DATE; + this.type = DateLiteral.getDefaultDateType(Type.DATE); } } else { - this.type = Type.DATETIME; + this.type = DateLiteral.getDefaultDateType(Type.DATETIME); } } else { this.type = fn.getReturnType(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java index 6d40c80acb..3d3574dd35 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java @@ -80,6 +80,8 @@ public abstract class LiteralExpr extends Expr implements Comparable=|<=|>|<|!= " + "\"2019-12-02|2019-12-02 14:54:00\""); } - subExpr.setChild(1, ((StringLiteral) subExpr.getChild(1)).castTo(Type.DATETIME)); + subExpr.setChild(1, (subExpr.getChild(1)).castTo( + DateLiteral.getDefaultDateType(Type.DATETIME))); } else { throw new AnalysisException("The columns of TableName/CreateTime/FinishTime/State are supported."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java index a215cc3132..800f26fcaf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java @@ -175,11 +175,11 @@ public class StringLiteral extends LiteralExpr { public LiteralExpr convertToDate(Type targetType) throws AnalysisException { LiteralExpr newLiteral = null; try { - newLiteral = new DateLiteral(value, targetType); + newLiteral = new DateLiteral(value, DateLiteral.getDefaultDateType(targetType)); } catch (AnalysisException e) { if (targetType.isScalarType(PrimitiveType.DATETIME)) { - newLiteral = new DateLiteral(value, Type.DATE); - newLiteral.setType(Type.DATETIME); + newLiteral = new DateLiteral(value, DateLiteral.getDefaultDateType(Type.DATE)); + newLiteral.setType(DateLiteral.getDefaultDateType(Type.DATETIME)); } else { throw e; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java index bdbb4c88fc..21b3eea072 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java @@ -29,8 +29,6 @@ import org.apache.doris.catalog.StructType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; -import com.google.common.base.Preconditions; - import java.util.ArrayList; /** @@ -52,6 +50,14 @@ public class TypeDef implements ParseNode { return new TypeDef(ScalarType.createDecimalV2Type(precision, scale)); } + public static TypeDef createDatetimeV2(int scale) { + return new TypeDef(ScalarType.createDatetimeV2Type(scale)); + } + + public static TypeDef createTimeV2(int scale) { + return new TypeDef(ScalarType.createTimeV2Type(scale)); + } + public static TypeDef createVarchar(int len) { return new TypeDef(ScalarType.createVarchar(len)); } @@ -68,9 +74,9 @@ public class TypeDef implements ParseNode { // Check the max nesting depth before calling the recursive analyze() to avoid // a stack overflow. if (parsedType.exceedsMaxNestingDepth()) { - throw new AnalysisException( - String.format("Type exceeds the maximum nesting depth of %s:\n%s", Type.MAX_NESTING_DEPTH, - parsedType.toSql())); + throw new AnalysisException(String.format( + "Type exceeds the maximum nesting depth of %s:\n%s", + Type.MAX_NESTING_DEPTH, parsedType.toSql())); } analyze(parsedType); isAnalyzed = true; @@ -111,6 +117,10 @@ public class TypeDef implements ParseNode { if (type.isNull()) { throw new AnalysisException("Unsupported data type: " + type.toSql()); } + if (!type.getPrimitiveType().isIntegerType() + && !type.getPrimitiveType().isCharFamily()) { + throw new AnalysisException("Array column just support INT/VARCHAR sub-type"); + } if (type.getPrimitiveType().isStringType() && !type.isAssignedStrLenInColDefinition()) { type.setLength(1); @@ -129,11 +139,9 @@ public class TypeDef implements ParseNode { if (type == PrimitiveType.VARCHAR) { name = "VARCHAR"; maxLen = ScalarType.MAX_VARCHAR_LENGTH; - } else if (type == PrimitiveType.CHAR) { + } else { name = "CHAR"; maxLen = ScalarType.MAX_CHAR_LENGTH; - } else { - Preconditions.checkState(false); return; } int len = scalarType.getLength(); @@ -168,10 +176,25 @@ public class TypeDef implements ParseNode { } break; } + case TIMEV2: + case DATETIMEV2: { + int precision = scalarType.decimalPrecision(); + int scale = scalarType.decimalScale(); + // precision: [1, 27] + if (precision != ScalarType.DATETIME_PRECISION) { + throw new AnalysisException("Precision of Datetime/Time must be " + ScalarType.DATETIME_PRECISION + + "." + " Precision was set to: " + precision + "."); + } + // scale: [0, 9] + if (scale < 0 || scale > 6) { + throw new AnalysisException("Scale of Datetime/Time must between 0 and 6." + + " Scale was set to: " + scale + "."); + } + break; + } case INVALID_TYPE: throw new AnalysisException("Invalid type."); - default: - break; + default: break; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index 02892573bc..a68ca2ce44 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -455,14 +455,6 @@ public class Column implements Writable { || other.getDataType() == PrimitiveType.STRING)) { return; } - - if (this.getPrecision() != other.getPrecision()) { - throw new DdlException("Cannot change precision"); - } - - if (this.getScale() != other.getScale()) { - throw new DdlException("Cannot change scale"); - } } public boolean nameEquals(String otherColName, boolean ignorePrefix) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java index 81bd0e3238..ae82bfeaa8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java @@ -111,8 +111,21 @@ public abstract class ColumnType { schemaChangeMatrix[PrimitiveType.DECIMALV2.ordinal()][PrimitiveType.STRING.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATETIME.ordinal()][PrimitiveType.DATE.ordinal()] = true; - schemaChangeMatrix[PrimitiveType.DATE.ordinal()][PrimitiveType.DATETIME.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATETIME.ordinal()][PrimitiveType.DATEV2.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATE.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATETIME.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATE.ordinal()][PrimitiveType.DATEV2.ordinal()] = true; + + schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.DATE.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATEV2.ordinal()][PrimitiveType.DATETIME.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.DATEV2.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATEV2.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.DATETIME.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATEV2.ordinal()][PrimitiveType.DATE.ordinal()] = true; + + // we should support schema change between different precision + schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true; } static boolean isSchemaChangeAllowed(Type lhs, Type rhs) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java index 9079deaa2d..93c218d0b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java @@ -496,6 +496,7 @@ public class Function implements Writable { return "float_val"; case DOUBLE: case TIME: + case TIMEV2: return "double_val"; case VARCHAR: case CHAR: @@ -506,6 +507,8 @@ public class Function implements Writable { return "string_val"; case DATE: case DATETIME: + case DATEV2: + case DATETIMEV2: return "datetime_val"; case DECIMALV2: return "decimalv2_val"; @@ -535,6 +538,7 @@ public class Function implements Writable { return "FloatVal"; case DOUBLE: case TIME: + case TIMEV2: return "DoubleVal"; case VARCHAR: case CHAR: @@ -545,6 +549,8 @@ public class Function implements Writable { return "StringVal"; case DATE: case DATETIME: + case DATEV2: + case DATETIMEV2: return "DateTimeVal"; case DECIMALV2: return "DecimalV2Val"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java index 3b44365626..d8bcb961e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java @@ -310,6 +310,8 @@ public class PartitionKey implements Comparable, Writable { break; case DATE: case DATETIME: + case DATEV2: + case DATETIMEV2: literal = DateLiteral.read(in); break; case CHAR: diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java index 32dc2bec55..7f01110cba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java @@ -54,6 +54,9 @@ public enum PrimitiveType { HLL("HLL", 16, TPrimitiveType.HLL), BITMAP("BITMAP", 16, TPrimitiveType.OBJECT), QUANTILE_STATE("QUANTILE_STATE", 16, TPrimitiveType.QUANTILE_STATE), + DATEV2("DATEV2", 4, TPrimitiveType.DATEV2), + DATETIMEV2("DATETIMEV2", 8, TPrimitiveType.DATETIMEV2), + TIMEV2("TIMEV2", 8, TPrimitiveType.TIMEV2), // sizeof(CollectionValue) ARRAY("ARRAY", 32, TPrimitiveType.ARRAY), @@ -85,12 +88,15 @@ public enum PrimitiveType { builder.put(NULL_TYPE, DOUBLE); builder.put(NULL_TYPE, DATE); builder.put(NULL_TYPE, DATETIME); + builder.put(NULL_TYPE, DATEV2); + builder.put(NULL_TYPE, DATETIMEV2); builder.put(NULL_TYPE, DECIMALV2); builder.put(NULL_TYPE, CHAR); builder.put(NULL_TYPE, VARCHAR); builder.put(NULL_TYPE, STRING); builder.put(NULL_TYPE, BITMAP); //TODO(weixiang):why null type can cast to bitmap? builder.put(NULL_TYPE, TIME); + builder.put(NULL_TYPE, TIMEV2); // Boolean builder.put(BOOLEAN, BOOLEAN); builder.put(BOOLEAN, TINYINT); @@ -102,6 +108,8 @@ public enum PrimitiveType { builder.put(BOOLEAN, DOUBLE); builder.put(BOOLEAN, DATE); builder.put(BOOLEAN, DATETIME); + builder.put(BOOLEAN, DATEV2); + builder.put(BOOLEAN, DATETIMEV2); builder.put(BOOLEAN, DECIMALV2); builder.put(BOOLEAN, VARCHAR); builder.put(BOOLEAN, STRING); @@ -116,6 +124,8 @@ public enum PrimitiveType { builder.put(TINYINT, DOUBLE); builder.put(TINYINT, DATE); builder.put(TINYINT, DATETIME); + builder.put(TINYINT, DATEV2); + builder.put(TINYINT, DATETIMEV2); builder.put(TINYINT, DECIMALV2); builder.put(TINYINT, VARCHAR); builder.put(TINYINT, STRING); @@ -130,6 +140,8 @@ public enum PrimitiveType { builder.put(SMALLINT, DOUBLE); builder.put(SMALLINT, DATE); builder.put(SMALLINT, DATETIME); + builder.put(SMALLINT, DATEV2); + builder.put(SMALLINT, DATETIMEV2); builder.put(SMALLINT, DECIMALV2); builder.put(SMALLINT, VARCHAR); builder.put(SMALLINT, STRING); @@ -144,6 +156,8 @@ public enum PrimitiveType { builder.put(INT, DOUBLE); builder.put(INT, DATE); builder.put(INT, DATETIME); + builder.put(INT, DATEV2); + builder.put(INT, DATETIMEV2); builder.put(INT, DECIMALV2); builder.put(INT, VARCHAR); builder.put(INT, STRING); @@ -158,6 +172,8 @@ public enum PrimitiveType { builder.put(BIGINT, DOUBLE); builder.put(BIGINT, DATE); builder.put(BIGINT, DATETIME); + builder.put(BIGINT, DATEV2); + builder.put(BIGINT, DATETIMEV2); builder.put(BIGINT, DECIMALV2); builder.put(BIGINT, VARCHAR); builder.put(BIGINT, STRING); @@ -172,6 +188,8 @@ public enum PrimitiveType { builder.put(LARGEINT, DOUBLE); builder.put(LARGEINT, DATE); builder.put(LARGEINT, DATETIME); + builder.put(LARGEINT, DATEV2); + builder.put(LARGEINT, DATETIMEV2); builder.put(LARGEINT, DECIMALV2); builder.put(LARGEINT, VARCHAR); builder.put(LARGEINT, STRING); @@ -186,6 +204,8 @@ public enum PrimitiveType { builder.put(FLOAT, DOUBLE); builder.put(FLOAT, DATE); builder.put(FLOAT, DATETIME); + builder.put(FLOAT, DATEV2); + builder.put(FLOAT, DATETIMEV2); builder.put(FLOAT, DECIMALV2); builder.put(FLOAT, VARCHAR); builder.put(FLOAT, STRING); @@ -200,6 +220,8 @@ public enum PrimitiveType { builder.put(DOUBLE, DOUBLE); builder.put(DOUBLE, DATE); builder.put(DOUBLE, DATETIME); + builder.put(DOUBLE, DATEV2); + builder.put(DOUBLE, DATETIMEV2); builder.put(DOUBLE, DECIMALV2); builder.put(DOUBLE, VARCHAR); builder.put(DOUBLE, STRING); @@ -214,6 +236,8 @@ public enum PrimitiveType { builder.put(DATE, DOUBLE); builder.put(DATE, DATE); builder.put(DATE, DATETIME); + builder.put(DATE, DATEV2); + builder.put(DATE, DATETIMEV2); builder.put(DATE, DECIMALV2); builder.put(DATE, VARCHAR); builder.put(DATE, STRING); @@ -228,9 +252,43 @@ public enum PrimitiveType { builder.put(DATETIME, DOUBLE); builder.put(DATETIME, DATE); builder.put(DATETIME, DATETIME); + builder.put(DATETIME, DATEV2); + builder.put(DATETIME, DATETIMEV2); builder.put(DATETIME, DECIMALV2); builder.put(DATETIME, VARCHAR); builder.put(DATETIME, STRING); + // DateV2 + builder.put(DATEV2, BOOLEAN); + builder.put(DATEV2, TINYINT); + builder.put(DATEV2, SMALLINT); + builder.put(DATEV2, INT); + builder.put(DATEV2, BIGINT); + builder.put(DATEV2, LARGEINT); + builder.put(DATEV2, FLOAT); + builder.put(DATEV2, DOUBLE); + builder.put(DATEV2, DATE); + builder.put(DATEV2, DATETIME); + builder.put(DATEV2, DATEV2); + builder.put(DATEV2, DATETIMEV2); + builder.put(DATEV2, DECIMALV2); + builder.put(DATEV2, VARCHAR); + builder.put(DATEV2, STRING); + // DatetimeV2 + builder.put(DATETIMEV2, BOOLEAN); + builder.put(DATETIMEV2, TINYINT); + builder.put(DATETIMEV2, SMALLINT); + builder.put(DATETIMEV2, INT); + builder.put(DATETIMEV2, BIGINT); + builder.put(DATETIMEV2, LARGEINT); + builder.put(DATETIMEV2, FLOAT); + builder.put(DATETIMEV2, DOUBLE); + builder.put(DATETIMEV2, DATE); + builder.put(DATETIMEV2, DATETIME); + builder.put(DATETIMEV2, DATEV2); + builder.put(DATETIMEV2, DATETIMEV2); + builder.put(DATETIMEV2, DECIMALV2); + builder.put(DATETIMEV2, VARCHAR); + builder.put(DATETIMEV2, STRING); // Char builder.put(CHAR, BOOLEAN); builder.put(CHAR, TINYINT); @@ -243,6 +301,8 @@ public enum PrimitiveType { builder.put(CHAR, DOUBLE); builder.put(CHAR, DATE); builder.put(CHAR, DATETIME); + builder.put(CHAR, DATEV2); + builder.put(CHAR, DATETIMEV2); builder.put(CHAR, DECIMALV2); builder.put(CHAR, VARCHAR); builder.put(CHAR, STRING); @@ -257,6 +317,8 @@ public enum PrimitiveType { builder.put(VARCHAR, DOUBLE); builder.put(VARCHAR, DATE); builder.put(VARCHAR, DATETIME); + builder.put(VARCHAR, DATEV2); + builder.put(VARCHAR, DATETIMEV2); builder.put(VARCHAR, DECIMALV2); builder.put(VARCHAR, VARCHAR); builder.put(VARCHAR, STRING); @@ -272,6 +334,8 @@ public enum PrimitiveType { builder.put(STRING, DOUBLE); builder.put(STRING, DATE); builder.put(STRING, DATETIME); + builder.put(STRING, DATEV2); + builder.put(STRING, DATETIMEV2); builder.put(STRING, DECIMALV2); builder.put(STRING, VARCHAR); builder.put(STRING, STRING); @@ -300,8 +364,14 @@ public enum PrimitiveType { // TIME builder.put(TIME, TIME); + builder.put(TIME, TIMEV2); builder.put(TIME, DOUBLE); + //TIMEV2 + builder.put(TIMEV2, TIME); + builder.put(TIMEV2, TIMEV2); + builder.put(TIMEV2, DOUBLE); + implicitCastMap = builder.build(); } @@ -344,6 +414,9 @@ public enum PrimitiveType { supportedTypes.add(DATE); supportedTypes.add(DATETIME); supportedTypes.add(TIME); + supportedTypes.add(DATEV2); + supportedTypes.add(DATETIMEV2); + supportedTypes.add(TIMEV2); supportedTypes.add(DECIMALV2); supportedTypes.add(BITMAP); supportedTypes.add(ARRAY); @@ -398,14 +471,15 @@ public enum PrimitiveType { compatibilityMatrix[NULL_TYPE.ordinal()][DOUBLE.ordinal()] = DOUBLE; compatibilityMatrix[NULL_TYPE.ordinal()][DATE.ordinal()] = DATE; compatibilityMatrix[NULL_TYPE.ordinal()][DATETIME.ordinal()] = DATETIME; + compatibilityMatrix[NULL_TYPE.ordinal()][DATEV2.ordinal()] = DATEV2; + compatibilityMatrix[NULL_TYPE.ordinal()][DATETIMEV2.ordinal()] = DATETIMEV2; compatibilityMatrix[NULL_TYPE.ordinal()][CHAR.ordinal()] = CHAR; compatibilityMatrix[NULL_TYPE.ordinal()][VARCHAR.ordinal()] = VARCHAR; compatibilityMatrix[NULL_TYPE.ordinal()][STRING.ordinal()] = STRING; compatibilityMatrix[NULL_TYPE.ordinal()][DECIMALV2.ordinal()] = DECIMALV2; compatibilityMatrix[NULL_TYPE.ordinal()][TIME.ordinal()] = TIME; - //TODO(weixiang): bitmap can be null? + compatibilityMatrix[NULL_TYPE.ordinal()][TIMEV2.ordinal()] = TIMEV2; compatibilityMatrix[NULL_TYPE.ordinal()][BITMAP.ordinal()] = BITMAP; - //TODO(weixiang): QUANTILE_STATE can be null? compatibilityMatrix[NULL_TYPE.ordinal()][QUANTILE_STATE.ordinal()] = QUANTILE_STATE; compatibilityMatrix[BOOLEAN.ordinal()][BOOLEAN.ordinal()] = BOOLEAN; @@ -418,11 +492,14 @@ public enum PrimitiveType { compatibilityMatrix[BOOLEAN.ordinal()][DOUBLE.ordinal()] = DOUBLE; compatibilityMatrix[BOOLEAN.ordinal()][DATE.ordinal()] = INVALID_TYPE; compatibilityMatrix[BOOLEAN.ordinal()][DATETIME.ordinal()] = INVALID_TYPE; + compatibilityMatrix[BOOLEAN.ordinal()][DATEV2.ordinal()] = INVALID_TYPE; + compatibilityMatrix[BOOLEAN.ordinal()][DATETIMEV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[BOOLEAN.ordinal()][CHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[BOOLEAN.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[BOOLEAN.ordinal()][STRING.ordinal()] = INVALID_TYPE; compatibilityMatrix[BOOLEAN.ordinal()][DECIMALV2.ordinal()] = DECIMALV2; compatibilityMatrix[BOOLEAN.ordinal()][TIME.ordinal()] = TIME; + compatibilityMatrix[BOOLEAN.ordinal()][TIMEV2.ordinal()] = TIMEV2; compatibilityMatrix[TINYINT.ordinal()][TINYINT.ordinal()] = TINYINT; compatibilityMatrix[TINYINT.ordinal()][SMALLINT.ordinal()] = SMALLINT; @@ -433,11 +510,14 @@ public enum PrimitiveType { compatibilityMatrix[TINYINT.ordinal()][DOUBLE.ordinal()] = DOUBLE; compatibilityMatrix[TINYINT.ordinal()][DATE.ordinal()] = INVALID_TYPE; compatibilityMatrix[TINYINT.ordinal()][DATETIME.ordinal()] = INVALID_TYPE; + compatibilityMatrix[TINYINT.ordinal()][DATEV2.ordinal()] = INVALID_TYPE; + compatibilityMatrix[TINYINT.ordinal()][DATETIMEV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[TINYINT.ordinal()][CHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[TINYINT.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[TINYINT.ordinal()][STRING.ordinal()] = INVALID_TYPE; compatibilityMatrix[TINYINT.ordinal()][DECIMALV2.ordinal()] = DECIMALV2; compatibilityMatrix[TINYINT.ordinal()][TIME.ordinal()] = TIME; + compatibilityMatrix[TINYINT.ordinal()][TIMEV2.ordinal()] = TIMEV2; compatibilityMatrix[SMALLINT.ordinal()][SMALLINT.ordinal()] = SMALLINT; compatibilityMatrix[SMALLINT.ordinal()][INT.ordinal()] = INT; @@ -447,11 +527,14 @@ public enum PrimitiveType { compatibilityMatrix[SMALLINT.ordinal()][DOUBLE.ordinal()] = DOUBLE; compatibilityMatrix[SMALLINT.ordinal()][DATE.ordinal()] = INVALID_TYPE; compatibilityMatrix[SMALLINT.ordinal()][DATETIME.ordinal()] = INVALID_TYPE; + compatibilityMatrix[SMALLINT.ordinal()][DATEV2.ordinal()] = INVALID_TYPE; + compatibilityMatrix[SMALLINT.ordinal()][DATETIMEV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[SMALLINT.ordinal()][CHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[SMALLINT.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[SMALLINT.ordinal()][STRING.ordinal()] = INVALID_TYPE; compatibilityMatrix[SMALLINT.ordinal()][DECIMALV2.ordinal()] = DECIMALV2; compatibilityMatrix[SMALLINT.ordinal()][TIME.ordinal()] = TIME; + compatibilityMatrix[SMALLINT.ordinal()][TIMEV2.ordinal()] = TIMEV2; compatibilityMatrix[INT.ordinal()][INT.ordinal()] = INT; compatibilityMatrix[INT.ordinal()][BIGINT.ordinal()] = BIGINT; @@ -460,11 +543,14 @@ public enum PrimitiveType { compatibilityMatrix[INT.ordinal()][DOUBLE.ordinal()] = DOUBLE; compatibilityMatrix[INT.ordinal()][DATE.ordinal()] = INVALID_TYPE; compatibilityMatrix[INT.ordinal()][DATETIME.ordinal()] = INVALID_TYPE; + compatibilityMatrix[INT.ordinal()][DATEV2.ordinal()] = INVALID_TYPE; + compatibilityMatrix[INT.ordinal()][DATETIMEV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[INT.ordinal()][CHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[INT.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[INT.ordinal()][STRING.ordinal()] = INVALID_TYPE; compatibilityMatrix[INT.ordinal()][DECIMALV2.ordinal()] = DECIMALV2; compatibilityMatrix[INT.ordinal()][TIME.ordinal()] = TIME; + compatibilityMatrix[INT.ordinal()][TIMEV2.ordinal()] = TIMEV2; compatibilityMatrix[BIGINT.ordinal()][BIGINT.ordinal()] = BIGINT; compatibilityMatrix[BIGINT.ordinal()][LARGEINT.ordinal()] = LARGEINT; @@ -472,81 +558,127 @@ public enum PrimitiveType { compatibilityMatrix[BIGINT.ordinal()][DOUBLE.ordinal()] = DOUBLE; compatibilityMatrix[BIGINT.ordinal()][DATE.ordinal()] = INVALID_TYPE; compatibilityMatrix[BIGINT.ordinal()][DATETIME.ordinal()] = INVALID_TYPE; + compatibilityMatrix[BIGINT.ordinal()][DATEV2.ordinal()] = INVALID_TYPE; + compatibilityMatrix[BIGINT.ordinal()][DATETIMEV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[BIGINT.ordinal()][CHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[BIGINT.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[BIGINT.ordinal()][STRING.ordinal()] = INVALID_TYPE; compatibilityMatrix[BIGINT.ordinal()][DECIMALV2.ordinal()] = DECIMALV2; compatibilityMatrix[BIGINT.ordinal()][TIME.ordinal()] = TIME; + compatibilityMatrix[BIGINT.ordinal()][TIMEV2.ordinal()] = TIMEV2; compatibilityMatrix[LARGEINT.ordinal()][LARGEINT.ordinal()] = LARGEINT; compatibilityMatrix[LARGEINT.ordinal()][FLOAT.ordinal()] = DOUBLE; compatibilityMatrix[LARGEINT.ordinal()][DOUBLE.ordinal()] = DOUBLE; compatibilityMatrix[LARGEINT.ordinal()][DATE.ordinal()] = INVALID_TYPE; compatibilityMatrix[LARGEINT.ordinal()][DATETIME.ordinal()] = INVALID_TYPE; + compatibilityMatrix[LARGEINT.ordinal()][DATEV2.ordinal()] = INVALID_TYPE; + compatibilityMatrix[LARGEINT.ordinal()][DATETIMEV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[LARGEINT.ordinal()][CHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[LARGEINT.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[LARGEINT.ordinal()][STRING.ordinal()] = INVALID_TYPE; compatibilityMatrix[LARGEINT.ordinal()][DECIMALV2.ordinal()] = DECIMALV2; compatibilityMatrix[LARGEINT.ordinal()][TIME.ordinal()] = TIME; + compatibilityMatrix[LARGEINT.ordinal()][TIMEV2.ordinal()] = TIMEV2; compatibilityMatrix[FLOAT.ordinal()][FLOAT.ordinal()] = FLOAT; compatibilityMatrix[FLOAT.ordinal()][DOUBLE.ordinal()] = DOUBLE; compatibilityMatrix[FLOAT.ordinal()][DATE.ordinal()] = INVALID_TYPE; compatibilityMatrix[FLOAT.ordinal()][DATETIME.ordinal()] = INVALID_TYPE; + compatibilityMatrix[FLOAT.ordinal()][DATEV2.ordinal()] = INVALID_TYPE; + compatibilityMatrix[FLOAT.ordinal()][DATETIMEV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[FLOAT.ordinal()][CHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[FLOAT.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[FLOAT.ordinal()][STRING.ordinal()] = INVALID_TYPE; compatibilityMatrix[FLOAT.ordinal()][DECIMALV2.ordinal()] = DECIMALV2; compatibilityMatrix[FLOAT.ordinal()][TIME.ordinal()] = TIME; + compatibilityMatrix[FLOAT.ordinal()][TIMEV2.ordinal()] = TIMEV2; compatibilityMatrix[DOUBLE.ordinal()][DOUBLE.ordinal()] = DOUBLE; compatibilityMatrix[DOUBLE.ordinal()][DATE.ordinal()] = INVALID_TYPE; compatibilityMatrix[DOUBLE.ordinal()][DATETIME.ordinal()] = INVALID_TYPE; + compatibilityMatrix[DOUBLE.ordinal()][DATEV2.ordinal()] = INVALID_TYPE; + compatibilityMatrix[DOUBLE.ordinal()][DATETIMEV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[DOUBLE.ordinal()][CHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[DOUBLE.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[DOUBLE.ordinal()][STRING.ordinal()] = INVALID_TYPE; compatibilityMatrix[DOUBLE.ordinal()][DECIMALV2.ordinal()] = DECIMALV2; compatibilityMatrix[DOUBLE.ordinal()][TIME.ordinal()] = TIME; + compatibilityMatrix[DOUBLE.ordinal()][TIMEV2.ordinal()] = TIMEV2; compatibilityMatrix[DATE.ordinal()][DATE.ordinal()] = DATE; compatibilityMatrix[DATE.ordinal()][DATETIME.ordinal()] = DATETIME; + compatibilityMatrix[DATE.ordinal()][DATEV2.ordinal()] = DATEV2; + compatibilityMatrix[DATE.ordinal()][DATETIMEV2.ordinal()] = DATETIMEV2; compatibilityMatrix[DATE.ordinal()][CHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[DATE.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[DATE.ordinal()][STRING.ordinal()] = INVALID_TYPE; compatibilityMatrix[DATE.ordinal()][DECIMALV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[DATE.ordinal()][TIME.ordinal()] = INVALID_TYPE; + compatibilityMatrix[DATE.ordinal()][TIMEV2.ordinal()] = INVALID_TYPE; + + compatibilityMatrix[DATEV2.ordinal()][DATE.ordinal()] = DATEV2; + compatibilityMatrix[DATEV2.ordinal()][DATETIME.ordinal()] = DATETIMEV2; + compatibilityMatrix[DATEV2.ordinal()][DATEV2.ordinal()] = DATEV2; + compatibilityMatrix[DATEV2.ordinal()][DATETIMEV2.ordinal()] = DATETIMEV2; + compatibilityMatrix[DATEV2.ordinal()][CHAR.ordinal()] = INVALID_TYPE; + compatibilityMatrix[DATEV2.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE; + compatibilityMatrix[DATEV2.ordinal()][STRING.ordinal()] = INVALID_TYPE; + compatibilityMatrix[DATEV2.ordinal()][DECIMALV2.ordinal()] = INVALID_TYPE; + compatibilityMatrix[DATEV2.ordinal()][TIME.ordinal()] = INVALID_TYPE; + compatibilityMatrix[DATEV2.ordinal()][TIMEV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[DATETIME.ordinal()][DATETIME.ordinal()] = DATETIME; + compatibilityMatrix[DATETIME.ordinal()][DATETIMEV2.ordinal()] = DATETIMEV2; compatibilityMatrix[DATETIME.ordinal()][CHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[DATETIME.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE; compatibilityMatrix[DATETIME.ordinal()][STRING.ordinal()] = INVALID_TYPE; compatibilityMatrix[DATETIME.ordinal()][DECIMALV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[DATETIME.ordinal()][TIME.ordinal()] = INVALID_TYPE; + compatibilityMatrix[DATETIME.ordinal()][TIMEV2.ordinal()] = INVALID_TYPE; + + compatibilityMatrix[DATETIMEV2.ordinal()][DATETIME.ordinal()] = DATETIMEV2; + compatibilityMatrix[DATETIMEV2.ordinal()][DATETIMEV2.ordinal()] = DATETIMEV2; + compatibilityMatrix[DATETIMEV2.ordinal()][CHAR.ordinal()] = INVALID_TYPE; + compatibilityMatrix[DATETIMEV2.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE; + compatibilityMatrix[DATETIMEV2.ordinal()][STRING.ordinal()] = INVALID_TYPE; + compatibilityMatrix[DATETIMEV2.ordinal()][DECIMALV2.ordinal()] = INVALID_TYPE; + compatibilityMatrix[DATETIMEV2.ordinal()][TIME.ordinal()] = INVALID_TYPE; + compatibilityMatrix[DATETIMEV2.ordinal()][TIMEV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[CHAR.ordinal()][CHAR.ordinal()] = CHAR; compatibilityMatrix[CHAR.ordinal()][VARCHAR.ordinal()] = VARCHAR; compatibilityMatrix[CHAR.ordinal()][STRING.ordinal()] = STRING; compatibilityMatrix[CHAR.ordinal()][DECIMALV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[CHAR.ordinal()][TIME.ordinal()] = INVALID_TYPE; + compatibilityMatrix[CHAR.ordinal()][TIMEV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[VARCHAR.ordinal()][VARCHAR.ordinal()] = VARCHAR; compatibilityMatrix[VARCHAR.ordinal()][STRING.ordinal()] = STRING; compatibilityMatrix[VARCHAR.ordinal()][DECIMALV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[VARCHAR.ordinal()][TIME.ordinal()] = INVALID_TYPE; + compatibilityMatrix[VARCHAR.ordinal()][TIMEV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[STRING.ordinal()][STRING.ordinal()] = STRING; compatibilityMatrix[STRING.ordinal()][DECIMALV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[STRING.ordinal()][TIME.ordinal()] = INVALID_TYPE; + compatibilityMatrix[STRING.ordinal()][TIMEV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[DECIMALV2.ordinal()][DECIMALV2.ordinal()] = DECIMALV2; compatibilityMatrix[DECIMALV2.ordinal()][TIME.ordinal()] = INVALID_TYPE; + compatibilityMatrix[DECIMALV2.ordinal()][TIMEV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[HLL.ordinal()][HLL.ordinal()] = HLL; compatibilityMatrix[HLL.ordinal()][TIME.ordinal()] = INVALID_TYPE; + compatibilityMatrix[HLL.ordinal()][TIMEV2.ordinal()] = INVALID_TYPE; compatibilityMatrix[BITMAP.ordinal()][BITMAP.ordinal()] = BITMAP; compatibilityMatrix[TIME.ordinal()][TIME.ordinal()] = TIME; + compatibilityMatrix[TIME.ordinal()][TIMEV2.ordinal()] = TIMEV2; + + compatibilityMatrix[TIMEV2.ordinal()][TIME.ordinal()] = TIMEV2; + compatibilityMatrix[TIMEV2.ordinal()][TIMEV2.ordinal()] = TIMEV2; compatibilityMatrix[QUANTILE_STATE.ordinal()][QUANTILE_STATE.ordinal()] = QUANTILE_STATE; } @@ -606,12 +738,18 @@ public enum PrimitiveType { return DATE; case DATETIME: return DATETIME; + case DATEV2: + return DATEV2; + case DATETIMEV2: + return DATETIMEV2; case BINARY: return BINARY; case DECIMALV2: return DECIMALV2; case TIME: return TIME; + case TIMEV2: + return TIMEV2; case VARCHAR: return VARCHAR; case STRING: @@ -717,7 +855,11 @@ public enum PrimitiveType { } public boolean isDateType() { - return (this == DATE || this == DATETIME); + return (this == DATE || this == DATETIME || this == DATEV2 || this == DATETIMEV2); + } + + public boolean isDateV2Type() { + return (this == DATEV2 || this == DATETIMEV2); } public boolean isArrayType() { @@ -755,10 +897,13 @@ public enum PrimitiveType { case DOUBLE: return MysqlColType.MYSQL_TYPE_DOUBLE; case TIME: + case TIMEV2: return MysqlColType.MYSQL_TYPE_TIME; case DATE: + case DATEV2: return MysqlColType.MYSQL_TYPE_DATE; - case DATETIME: { + case DATETIME: + case DATETIMEV2: { if (isTimeType) { return MysqlColType.MYSQL_TYPE_TIME; } else { @@ -777,8 +922,10 @@ public enum PrimitiveType { public int getOlapColumnIndexSize() { switch (this) { case DATE: + case DATEV2: return DATE_INDEX_LEN; case DATETIME: + case DATETIMEV2: return DATETIME_INDEX_LEN; case VARCHAR: return VARCHAR_INDEX_LEN; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java index 91d15db09f..5659259812 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java @@ -84,7 +84,7 @@ public class RangePartitionInfo extends PartitionInfo { // generate partitionItemEntryList List> partitionItemEntryList = isFixedPartitionKeyValueType - ? getPartitionItemEntryList(isTemp, false) : getPartitionItemEntryList(isTemp, true); + ? getPartitionItemEntryList(isTemp, false) : getPartitionItemEntryList(isTemp, true); if (isFixedPartitionKeyValueType) { return createNewRangeForFixedPartitionValueType(partKeyDesc, partitionItemEntryList); @@ -139,8 +139,8 @@ public class RangePartitionInfo extends PartitionInfo { private Range createNewRangeForLessThanPartitionValueType(PartitionKey newRangeUpper, Range lastRange, Range currentRange) throws AnalysisException, DdlException { - PartitionKey lowKey = lastRange == null ? PartitionKey.createInfinityPartitionKey(partitionColumns, false) - : lastRange.upperEndpoint(); + PartitionKey lowKey = lastRange == null + ? PartitionKey.createInfinityPartitionKey(partitionColumns, false) : lastRange.upperEndpoint(); // check: [left, right), error if left equal right if (lowKey.compareTo(newRangeUpper) >= 0) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java index e716fb65e2..cb4055e949 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java @@ -153,6 +153,7 @@ public class ScalarFunction extends Function { break; case DOUBLE: case TIME: + case TIMEV2: beFn += "_double_val"; break; case CHAR: @@ -165,6 +166,8 @@ public class ScalarFunction extends Function { break; case DATE: case DATETIME: + case DATEV2: + case DATETIMEV2: beFn += "_datetime_val"; break; case DECIMALV2: @@ -227,6 +230,7 @@ public class ScalarFunction extends Function { break; case DOUBLE: case TIME: + case TIMEV2: beFn.append("_double_val"); break; case CHAR: @@ -237,6 +241,8 @@ public class ScalarFunction extends Function { break; case DATE: case DATETIME: + case DATEV2: + case DATETIMEV2: beFn.append("_datetime_val"); break; case DECIMALV2: diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarType.java index 11202c8a56..601e7d833f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarType.java @@ -17,6 +17,7 @@ package org.apache.doris.catalog; +import org.apache.doris.common.Config; import org.apache.doris.common.io.Text; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.thrift.TColumnType; @@ -47,7 +48,8 @@ import java.util.Objects; * indicating that any decimal type is a subtype of the decimal type. */ public class ScalarType extends Type { - private static final Logger LOG = LogManager.getLogger(ScalarType.class); + // We use a fixed-length decimal type to represent a date time. + public static final int DATETIME_PRECISION = 18; // SQL allows the engine to pick the default precision. We pick the largest // precision that is supported by the smallest decimal type in the BE (4 bytes). @@ -74,6 +76,7 @@ public class ScalarType extends Type { // Hive, mysql, sql server standard. public static final int MAX_PRECISION = 38; + private static final Logger LOG = LogManager.getLogger(ScalarType.class); @SerializedName(value = "type") private final PrimitiveType type; @@ -157,6 +160,12 @@ public class ScalarType extends Type { return DATE; case DATETIME: return DATETIME; + case DATEV2: + return DATEV2; + case DATETIMEV2: + return DEFAULT_DATETIMEV2; + case TIMEV2: + return TIMEV2; case TIME: return TIME; case DECIMALV2: @@ -209,6 +218,10 @@ public class ScalarType extends Type { return DATE; case "DATETIME": return DATETIME; + case "DATEV2": + return DATEV2; + case "DATETIMEV2": + return DATETIMEV2; case "TIME": return TIME; case "DECIMAL": @@ -279,6 +292,53 @@ public class ScalarType extends Type { return type; } + @SuppressWarnings("checkstyle:MissingJavadocMethod") + public static ScalarType createDatetimeV2Type(int scale) { + ScalarType type = new ScalarType(PrimitiveType.DATETIMEV2); + type.precision = DATETIME_PRECISION; + type.scale = scale; + return type; + } + + @SuppressWarnings("checkstyle:MissingJavadocMethod") + public static ScalarType createTimeV2Type(int scale) { + ScalarType type = new ScalarType(PrimitiveType.TIMEV2); + type.precision = DATETIME_PRECISION; + type.scale = scale; + return type; + } + + @SuppressWarnings("checkstyle:MissingJavadocMethod") + public static ScalarType createDatetimeType() { + if (!Config.use_date_v2_by_default) { + return new ScalarType(PrimitiveType.DATETIME); + } + ScalarType type = new ScalarType(PrimitiveType.DATETIMEV2); + type.precision = DATETIME_PRECISION; + type.scale = 0; + return type; + } + + @SuppressWarnings("checkstyle:MissingJavadocMethod") + public static ScalarType createDateType() { + if (Config.use_date_v2_by_default) { + return new ScalarType(PrimitiveType.DATEV2); + } else { + return new ScalarType(PrimitiveType.DATE); + } + } + + @SuppressWarnings("checkstyle:MissingJavadocMethod") + public static ScalarType createTimeType() { + if (!Config.use_date_v2_by_default) { + return new ScalarType(PrimitiveType.TIME); + } + ScalarType type = new ScalarType(PrimitiveType.TIMEV2); + type.precision = DATETIME_PRECISION; + type.scale = 0; + return type; + } + public static ScalarType createVarcharType(int len) { // length checked in analysis ScalarType type = new ScalarType(PrimitiveType.VARCHAR); @@ -329,6 +389,10 @@ public class ScalarType extends Type { return "DECIMAL(*,*)"; } return "DECIMAL(" + precision + "," + scale + ")"; + } else if (type == PrimitiveType.DATETIMEV2) { + return "Datetime(" + scale + ")"; + } else if (type == PrimitiveType.TIMEV2) { + return "Time(" + scale + ")"; } else if (type == PrimitiveType.VARCHAR) { if (isWildcardVarchar()) { return "VARCHAR(*)"; @@ -369,6 +433,12 @@ public class ScalarType extends Type { stringBuilder.append("decimal").append("(`").append(precisionStr).append("`)"); } break; + case DATETIMEV2: + stringBuilder.append("datetime").append("(").append(scale).append(")"); + break; + case TIMEV2: + stringBuilder.append("time").append("(").append(scale).append(")"); + break; case BOOLEAN: return "boolean"; case TINYINT: @@ -385,6 +455,7 @@ public class ScalarType extends Type { case DOUBLE: case DATE: case DATETIME: + case DATEV2: case HLL: case BITMAP: case QUANTILE_STATE: @@ -436,12 +507,14 @@ public class ScalarType extends Type { } public int decimalPrecision() { - Preconditions.checkState(type == PrimitiveType.DECIMALV2); + Preconditions.checkState(type == PrimitiveType.DECIMALV2 || type == PrimitiveType.DATETIMEV2 + || type == PrimitiveType.TIMEV2); return precision; } public int decimalScale() { - Preconditions.checkState(type == PrimitiveType.DECIMALV2); + Preconditions.checkState(type == PrimitiveType.DECIMALV2 || type == PrimitiveType.DATETIMEV2 + || type == PrimitiveType.TIMEV2); return scale; } @@ -515,7 +588,8 @@ public class ScalarType extends Type { || type == PrimitiveType.BIGINT || type == PrimitiveType.FLOAT || type == PrimitiveType.DOUBLE || type == PrimitiveType.DATE || type == PrimitiveType.DATETIME || type == PrimitiveType.DECIMALV2 - || type == PrimitiveType.CHAR; + || type == PrimitiveType.CHAR || type == PrimitiveType.DATEV2 || type == PrimitiveType.DATETIMEV2 + || type == PrimitiveType.TIMEV2; } @Override @@ -585,6 +659,18 @@ public class ScalarType extends Type { return false; } ScalarType other = (ScalarType) o; + if ((this.isDate() && other.isDateV2()) || (this.isDateV2() && other.isDate())) { + return true; + } + if ((this.isDatetime() && other.isDatetimeV2()) || (this.isTime() && other.isTimeV2())) { + return other.decimalScale() == 0; + } + if ((this.isDatetimeV2() && other.isDatetime()) || (this.isTimeV2() && other.isTime())) { + return this.decimalScale() == 0; + } + if ((this.isDatetimeV2() && other.isDatetimeV2()) || (this.isTimeV2() && other.isTimeV2())) { + return this.decimalScale() == other.decimalScale(); + } if (type != other.type) { return false; } @@ -594,7 +680,7 @@ public class ScalarType extends Type { if (type == PrimitiveType.VARCHAR) { return len == other.len; } - if (type == PrimitiveType.DECIMALV2) { + if (type == PrimitiveType.DECIMALV2 || type == PrimitiveType.DATETIMEV2 || type == PrimitiveType.TIMEV2) { return precision == other.precision && scale == other.scale; } return true; @@ -612,6 +698,10 @@ public class ScalarType extends Type { return createDecimalV2TypeInternal(MAX_PRECISION, scale); } else if (isLargeIntType()) { return ScalarType.LARGEINT; + } else if (isDatetimeV2()) { + return createDatetimeV2Type(6); + } else if (isTimeV2()) { + return createTimeV2Type(6); } else { return ScalarType.INVALID; } @@ -623,6 +713,10 @@ public class ScalarType extends Type { return this; } else if (type == PrimitiveType.DECIMALV2) { return createDecimalV2TypeInternal(MAX_PRECISION, scale); + } else if (type == PrimitiveType.DATETIMEV2) { + return createDatetimeV2Type(6); + } else if (type == PrimitiveType.TIMEV2) { + return createTimeV2Type(6); } return createType(PrimitiveType.values()[type.ordinal() + 1]); } @@ -774,12 +868,16 @@ public class ScalarType extends Type { case BIGINT: case TIME: case DATETIME: + // TODO(Gabriel): unify execution engine and storage engine + case TIMEV2: + case DATETIMEV2: return 8; case LARGEINT: case DECIMALV2: return 16; case DOUBLE: return 12; + case DATEV2: case DATE: return 3; case CHAR: @@ -805,7 +903,7 @@ public class ScalarType extends Type { if (type == PrimitiveType.CHAR || type == PrimitiveType.VARCHAR || type == PrimitiveType.HLL) { thrift.setLen(len); } - if (type == PrimitiveType.DECIMALV2) { + if (type == PrimitiveType.DECIMALV2 || type == PrimitiveType.DATETIMEV2 || type == PrimitiveType.TIMEV2) { thrift.setPrecision(precision); thrift.setScale(scale); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java index 9bf27226e1..3a77e2381e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java @@ -17,8 +17,10 @@ package org.apache.doris.catalog; +import org.apache.doris.analysis.DateLiteral; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.Pair; import org.apache.doris.thrift.TColumnType; @@ -44,8 +46,6 @@ import java.util.List; * as abstract methods that subclasses must implement. */ public abstract class Type { - private static final Logger LOG = LogManager.getLogger(Type.class); - // Maximum nesting depth of a type. This limit was determined experimentally by // org.apache.doris.rewrite.FoldConstantsRule.apply generating and scanning // deeply nested Parquet and Avro files. In those experiments, we exceeded @@ -66,17 +66,21 @@ public abstract class Type { public static final ScalarType DOUBLE = new ScalarType(PrimitiveType.DOUBLE); public static final ScalarType DATE = new ScalarType(PrimitiveType.DATE); public static final ScalarType DATETIME = new ScalarType(PrimitiveType.DATETIME); + public static final ScalarType DATEV2 = new ScalarType(PrimitiveType.DATEV2); + public static final ScalarType TIMEV2 = new ScalarType(PrimitiveType.TIMEV2); public static final ScalarType TIME = new ScalarType(PrimitiveType.TIME); public static final ScalarType STRING = new ScalarType(PrimitiveType.STRING); - public static final ScalarType DEFAULT_DECIMALV2 = (ScalarType) - ScalarType.createDecimalV2Type(ScalarType.DEFAULT_PRECISION, - ScalarType.DEFAULT_SCALE); + public static final ScalarType DEFAULT_DECIMALV2 = + ScalarType.createDecimalV2Type(ScalarType.DEFAULT_PRECISION, ScalarType.DEFAULT_SCALE); + public static final ScalarType DEFAULT_DATETIMEV2 = ScalarType.createDatetimeV2Type(0); + public static final ScalarType DATETIMEV2 = DEFAULT_DATETIMEV2; + public static final ScalarType DEFAULT_TIMEV2 = ScalarType.createTimeV2Type(0); public static final ScalarType DECIMALV2 = DEFAULT_DECIMALV2; // (ScalarType) ScalarType.createDecimalTypeInternal(-1, -1); public static final ScalarType DEFAULT_VARCHAR = ScalarType.createVarcharType(-1); public static final ScalarType VARCHAR = ScalarType.createVarcharType(-1); public static final ScalarType HLL = ScalarType.createHllType(); - public static final ScalarType CHAR = (ScalarType) ScalarType.createCharType(-1); + public static final ScalarType CHAR = ScalarType.createCharType(-1); public static final ScalarType BITMAP = new ScalarType(PrimitiveType.BITMAP); public static final ScalarType QUANTILE_STATE = new ScalarType(PrimitiveType.QUANTILE_STATE); // Only used for alias function, to represent any type in function args @@ -85,9 +89,10 @@ public abstract class Type { public static final ArrayType ARRAY = ArrayType.create(); public static final StructType STRUCT = new StructType(); - private static ArrayList integerTypes; - private static ArrayList numericTypes; - private static ArrayList supportedTypes; + private static final Logger LOG = LogManager.getLogger(Type.class); + private static final ArrayList integerTypes; + private static final ArrayList numericTypes; + private static final ArrayList supportedTypes; static { integerTypes = Lists.newArrayList(); @@ -124,8 +129,11 @@ public abstract class Type { supportedTypes.add(CHAR); supportedTypes.add(DATE); supportedTypes.add(DATETIME); + supportedTypes.add(DATEV2); + supportedTypes.add(DATETIMEV2); supportedTypes.add(DECIMALV2); supportedTypes.add(TIME); + supportedTypes.add(TIMEV2); supportedTypes.add(STRING); } @@ -188,6 +196,14 @@ public abstract class Type { return isScalarType(PrimitiveType.DECIMALV2); } + public boolean isDatetimeV2() { + return isScalarType(PrimitiveType.DATETIMEV2); + } + + public boolean isTimeV2() { + return isScalarType(PrimitiveType.TIMEV2); + } + public boolean isWildcardDecimal() { return false; } @@ -287,7 +303,8 @@ public abstract class Type { } public boolean isDateType() { - return isScalarType(PrimitiveType.DATE) || isScalarType(PrimitiveType.DATETIME); + return isScalarType(PrimitiveType.DATE) || isScalarType(PrimitiveType.DATETIME) + || isScalarType(PrimitiveType.DATEV2) || isScalarType(PrimitiveType.DATETIMEV2); } public boolean isDatetime() { @@ -326,6 +343,10 @@ public abstract class Type { return isScalarType(PrimitiveType.DATE); } + public boolean isDateV2() { + return isScalarType(PrimitiveType.DATEV2); + } + /** * Returns true if Impala supports this type in the metdata. It does not mean we * can manipulate data of this type. For tables that contain columns with these @@ -546,6 +567,12 @@ public abstract class Type { return Type.DATETIME; case TIME: return Type.TIME; + case DATEV2: + return Type.DATEV2; + case DATETIMEV2: + return Type.DATETIMEV2; + case TIMEV2: + return Type.TIMEV2; case DECIMALV2: return Type.DECIMALV2; case CHAR: @@ -616,6 +643,14 @@ public abstract class Type { && scalarType.isSetPrecision()); type = ScalarType.createDecimalV2Type(scalarType.getPrecision(), scalarType.getScale()); + } else if (scalarType.getType() == TPrimitiveType.DATETIMEV2) { + Preconditions.checkState(scalarType.isSetPrecision() + && scalarType.isSetScale()); + type = ScalarType.createDatetimeV2Type(scalarType.getScale()); + } else if (scalarType.getType() == TPrimitiveType.TIMEV2) { + Preconditions.checkState(scalarType.isSetPrecision() + && scalarType.isSetScale()); + type = ScalarType.createTimeV2Type(scalarType.getScale()); } else { type = ScalarType.createType( PrimitiveType.fromThrift(scalarType.getType())); @@ -755,6 +790,8 @@ public abstract class Type { case DOUBLE: return 15; case DECIMALV2: + case DATETIMEV2: + case TIMEV2: return t.decimalPrecision(); default: return null; @@ -783,6 +820,8 @@ public abstract class Type { return 7; case DOUBLE: return 15; + case DATETIMEV2: + case TIMEV2: case DECIMALV2: return t.decimalScale(); default: @@ -867,6 +906,9 @@ public abstract class Type { compatibilityMatrix[BOOLEAN.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; compatibilityMatrix[BOOLEAN.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[BOOLEAN.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[BOOLEAN.ordinal()][DATEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[BOOLEAN.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[BOOLEAN.ordinal()][TIMEV2.ordinal()] = PrimitiveType.DOUBLE; compatibilityMatrix[BOOLEAN.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[BOOLEAN.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[BOOLEAN.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -886,6 +928,9 @@ public abstract class Type { compatibilityMatrix[TINYINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; compatibilityMatrix[TINYINT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[TINYINT.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[TINYINT.ordinal()][DATEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[TINYINT.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[TINYINT.ordinal()][TIMEV2.ordinal()] = PrimitiveType.DOUBLE; compatibilityMatrix[TINYINT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[TINYINT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[TINYINT.ordinal()][DECIMALV2.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -904,6 +949,9 @@ public abstract class Type { compatibilityMatrix[SMALLINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; compatibilityMatrix[SMALLINT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[SMALLINT.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[SMALLINT.ordinal()][DATEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[SMALLINT.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[SMALLINT.ordinal()][TIMEV2.ordinal()] = PrimitiveType.DOUBLE; compatibilityMatrix[SMALLINT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[SMALLINT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[SMALLINT.ordinal()][DECIMALV2.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -925,6 +973,9 @@ public abstract class Type { compatibilityMatrix[INT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; compatibilityMatrix[INT.ordinal()][DATE.ordinal()] = PrimitiveType.INT; compatibilityMatrix[INT.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[INT.ordinal()][DATEV2.ordinal()] = PrimitiveType.INT; + compatibilityMatrix[INT.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[INT.ordinal()][TIMEV2.ordinal()] = PrimitiveType.DOUBLE; compatibilityMatrix[INT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[INT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[INT.ordinal()][DECIMALV2.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -948,6 +999,9 @@ public abstract class Type { compatibilityMatrix[BIGINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; compatibilityMatrix[BIGINT.ordinal()][DATE.ordinal()] = PrimitiveType.BIGINT; compatibilityMatrix[BIGINT.ordinal()][DATETIME.ordinal()] = PrimitiveType.BIGINT; + compatibilityMatrix[BIGINT.ordinal()][DATEV2.ordinal()] = PrimitiveType.BIGINT; + compatibilityMatrix[BIGINT.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.BIGINT; + compatibilityMatrix[BIGINT.ordinal()][TIMEV2.ordinal()] = PrimitiveType.DOUBLE; compatibilityMatrix[BIGINT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[BIGINT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[BIGINT.ordinal()][DECIMALV2.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -962,6 +1016,9 @@ public abstract class Type { compatibilityMatrix[LARGEINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; compatibilityMatrix[LARGEINT.ordinal()][DATE.ordinal()] = PrimitiveType.LARGEINT; compatibilityMatrix[LARGEINT.ordinal()][DATETIME.ordinal()] = PrimitiveType.LARGEINT; + compatibilityMatrix[LARGEINT.ordinal()][DATEV2.ordinal()] = PrimitiveType.LARGEINT; + compatibilityMatrix[LARGEINT.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.LARGEINT; + compatibilityMatrix[LARGEINT.ordinal()][TIMEV2.ordinal()] = PrimitiveType.DOUBLE; compatibilityMatrix[LARGEINT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[LARGEINT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[LARGEINT.ordinal()][DECIMALV2.ordinal()] = PrimitiveType.DECIMALV2; @@ -975,6 +1032,9 @@ public abstract class Type { compatibilityMatrix[FLOAT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE; compatibilityMatrix[FLOAT.ordinal()][DATE.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[FLOAT.ordinal()][DATETIME.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[FLOAT.ordinal()][DATEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[FLOAT.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[FLOAT.ordinal()][TIMEV2.ordinal()] = PrimitiveType.DOUBLE; compatibilityMatrix[FLOAT.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[FLOAT.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[FLOAT.ordinal()][DECIMALV2.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -995,34 +1055,73 @@ public abstract class Type { compatibilityMatrix[DOUBLE.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DOUBLE.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DOUBLE.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DOUBLE.ordinal()][DATEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DOUBLE.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.DOUBLE; + compatibilityMatrix[DOUBLE.ordinal()][TIMEV2.ordinal()] = PrimitiveType.DOUBLE; // DATE compatibilityMatrix[DATE.ordinal()][DATETIME.ordinal()] = PrimitiveType.DATETIME; + compatibilityMatrix[DATE.ordinal()][DATEV2.ordinal()] = PrimitiveType.DATEV2; + compatibilityMatrix[DATE.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.DATETIMEV2; compatibilityMatrix[DATE.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATE.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATE.ordinal()][DECIMALV2.ordinal()] = PrimitiveType.DECIMALV2; compatibilityMatrix[DATE.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATE.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATE.ordinal()][TIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATE.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATE.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATE.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + // DATEV2 + compatibilityMatrix[DATEV2.ordinal()][DATE.ordinal()] = PrimitiveType.DATEV2; + compatibilityMatrix[DATEV2.ordinal()][DATETIME.ordinal()] = PrimitiveType.DATETIMEV2; + compatibilityMatrix[DATEV2.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.DATETIMEV2; + compatibilityMatrix[DATEV2.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATEV2.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATEV2.ordinal()][DECIMALV2.ordinal()] = PrimitiveType.DECIMALV2; + compatibilityMatrix[DATEV2.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATEV2.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATEV2.ordinal()][TIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATEV2.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATEV2.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATEV2.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + // DATETIME compatibilityMatrix[DATETIME.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATETIME.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.DATETIMEV2; + compatibilityMatrix[DATETIME.ordinal()][DATEV2.ordinal()] = PrimitiveType.DATETIMEV2; compatibilityMatrix[DATETIME.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATETIME.ordinal()][DECIMALV2.ordinal()] = PrimitiveType.DECIMALV2; compatibilityMatrix[DATETIME.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATETIME.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATETIME.ordinal()][TIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATETIME.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATETIME.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DATETIME.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + // DATETIMEV2 + compatibilityMatrix[DATETIMEV2.ordinal()][CHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATETIMEV2.ordinal()][DATETIME.ordinal()] = PrimitiveType.DATETIMEV2; + compatibilityMatrix[DATETIMEV2.ordinal()][DATEV2.ordinal()] = PrimitiveType.DATETIMEV2; + compatibilityMatrix[DATETIMEV2.ordinal()][VARCHAR.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATETIMEV2.ordinal()][DECIMALV2.ordinal()] = PrimitiveType.DECIMALV2; + compatibilityMatrix[DATETIMEV2.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATETIMEV2.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATETIMEV2.ordinal()][TIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATETIMEV2.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATETIMEV2.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DATETIMEV2.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + // We can convert some but not all string values to timestamps. // CHAR compatibilityMatrix[CHAR.ordinal()][VARCHAR.ordinal()] = PrimitiveType.VARCHAR; + compatibilityMatrix[CHAR.ordinal()][DATEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[CHAR.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[CHAR.ordinal()][DECIMALV2.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[CHAR.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[CHAR.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[CHAR.ordinal()][TIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[CHAR.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[CHAR.ordinal()][STRING.ordinal()] = PrimitiveType.STRING; compatibilityMatrix[CHAR.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -1031,6 +1130,9 @@ public abstract class Type { compatibilityMatrix[VARCHAR.ordinal()][DECIMALV2.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[VARCHAR.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[VARCHAR.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[VARCHAR.ordinal()][TIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[VARCHAR.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[VARCHAR.ordinal()][DATEV2.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[VARCHAR.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[VARCHAR.ordinal()][STRING.ordinal()] = PrimitiveType.STRING; compatibilityMatrix[VARCHAR.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -1038,6 +1140,9 @@ public abstract class Type { //String compatibilityMatrix[STRING.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[STRING.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[STRING.ordinal()][TIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[STRING.ordinal()][DATEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[STRING.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[STRING.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[STRING.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -1045,6 +1150,9 @@ public abstract class Type { // DECIMALV2 compatibilityMatrix[DECIMALV2.ordinal()][HLL.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DECIMALV2.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DECIMALV2.ordinal()][TIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[DECIMALV2.ordinal()][DATEV2.ordinal()] = PrimitiveType.DECIMALV2; + compatibilityMatrix[DECIMALV2.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.DECIMALV2; compatibilityMatrix[DECIMALV2.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DECIMALV2.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[DECIMALV2.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -1052,6 +1160,9 @@ public abstract class Type { // HLL compatibilityMatrix[HLL.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[HLL.ordinal()][TIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[HLL.ordinal()][DATEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[HLL.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[HLL.ordinal()][BITMAP.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[HLL.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[HLL.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; @@ -1059,15 +1170,23 @@ public abstract class Type { // BITMAP compatibilityMatrix[BITMAP.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[BITMAP.ordinal()][TIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[BITMAP.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; compatibilityMatrix[BITMAP.ordinal()][QUANTILE_STATE.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[BITMAP.ordinal()][DATEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[BITMAP.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; //QUANTILE_STATE compatibilityMatrix[QUANTILE_STATE.ordinal()][STRING.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[QUANTILE_STATE.ordinal()][DATEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[QUANTILE_STATE.ordinal()][DATETIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; // TIME why here not??? compatibilityMatrix[TIME.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[TIME.ordinal()][TIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[TIMEV2.ordinal()][TIMEV2.ordinal()] = PrimitiveType.INVALID_TYPE; + compatibilityMatrix[TIMEV2.ordinal()][TIME.ordinal()] = PrimitiveType.INVALID_TYPE; // Check all of the necessary entries that should be filled. // ignore binary and all @@ -1076,25 +1195,14 @@ public abstract class Type { PrimitiveType t1 = PrimitiveType.values()[i]; PrimitiveType t2 = PrimitiveType.values()[j]; // DECIMAL, NULL, and INVALID_TYPE are handled separately. - if (t1 == PrimitiveType.INVALID_TYPE || t2 == PrimitiveType.INVALID_TYPE) { - continue; - } - if (t1 == PrimitiveType.NULL_TYPE || t2 == PrimitiveType.NULL_TYPE) { - continue; - } - if (t1 == PrimitiveType.DECIMALV2 || t2 == PrimitiveType.DECIMALV2) { - continue; - } - if (t1 == PrimitiveType.TIME || t2 == PrimitiveType.TIME) { - continue; - } - if (t1 == PrimitiveType.ARRAY || t2 == PrimitiveType.ARRAY) { - continue; - } - if (t1 == PrimitiveType.MAP || t2 == PrimitiveType.MAP) { - continue; - } - if (t1 == PrimitiveType.STRUCT || t2 == PrimitiveType.STRUCT) { + if (t1 == PrimitiveType.INVALID_TYPE || t2 == PrimitiveType.INVALID_TYPE + || t1 == PrimitiveType.NULL_TYPE || t2 == PrimitiveType.NULL_TYPE + || t1 == PrimitiveType.ARRAY || t2 == PrimitiveType.ARRAY + || t1 == PrimitiveType.DECIMALV2 || t2 == PrimitiveType.DECIMALV2 + || t1 == PrimitiveType.TIME || t2 == PrimitiveType.TIME + || t1 == PrimitiveType.TIMEV2 || t2 == PrimitiveType.TIMEV2 + || t1 == PrimitiveType.MAP || t2 == PrimitiveType.MAP + || t1 == PrimitiveType.STRUCT || t2 == PrimitiveType.STRUCT) { continue; } Preconditions.checkNotNull(compatibilityMatrix[i][j]); @@ -1116,6 +1224,7 @@ public abstract class Type { case DOUBLE: return DOUBLE; case DATE: + case DATEV2: case DATETIME: case TIME: case CHAR: @@ -1124,6 +1233,10 @@ public abstract class Type { case BITMAP: case QUANTILE_STATE: return VARCHAR; + case DATETIMEV2: + return DEFAULT_DATETIMEV2; + case TIMEV2: + return DEFAULT_TIMEV2; case DECIMALV2: return DECIMALV2; case STRING: @@ -1145,7 +1258,7 @@ public abstract class Type { PrimitiveType t1ResultType = t1.getResultType().getPrimitiveType(); PrimitiveType t2ResultType = t2.getResultType().getPrimitiveType(); if (canCompareDate(t1.getPrimitiveType(), t2.getPrimitiveType())) { - return Type.DATETIME; + return getDateComparisonResultType((ScalarType) t1, (ScalarType) t2); } // Following logical is compatible with MySQL. @@ -1182,7 +1295,7 @@ public abstract class Type { return Type.DOUBLE; } - public static boolean canCompareDate(PrimitiveType t1, PrimitiveType t2) { + private static boolean canCompareDate(PrimitiveType t1, PrimitiveType t2) { if (t1.isDateType()) { if (t2.isDateType() || t2.isStringType() || t2.isIntegerType()) { return true; @@ -1198,6 +1311,35 @@ public abstract class Type { } } + private static Type getDateComparisonResultType(ScalarType t1, ScalarType t2) { + try { + if (t1.isDate() && t2.isDate()) { + return DateLiteral.getDefaultDateType(Type.DATE); + } else if ((t1.isDateV2() && t2.isDate()) || t1.isDate() && t2.isDateV2()) { + return Type.DATEV2; + } else if (t1.isDateV2() && t2.isDateV2()) { + return Type.DATEV2; + } else if (t1.isDatetime() && t2.isDatetime()) { + return DateLiteral.getDefaultDateType(Type.DATETIME); + } else if (t1.isDatetime() && t2.isDatetimeV2()) { + return t2; + } else if (t1.isDatetimeV2() && t2.isDatetime()) { + return t1; + } else if (t1.isDatetimeV2() && t2.isDatetimeV2()) { + return t1.decimalScale() > t2.decimalScale() ? t1 : t2; + } else if (t1.isDatetimeV2()) { + return t1; + } else if (t2.isDatetimeV2()) { + return t2; + } else { + return DateLiteral.getDefaultDateType(Type.DATETIME); + } + } catch (AnalysisException ignored) { + LOG.error("Invalid date type: {} and {}", t1, t2); + } + return null; + } + public Type getMaxResolutionType() { Preconditions.checkState(true, "must implemented"); return null; @@ -1211,6 +1353,7 @@ public abstract class Type { case INT: case BIGINT: case DATE: + case DATEV2: case DATETIME: return Type.BIGINT; case LARGEINT: @@ -1223,6 +1366,10 @@ public abstract class Type { case STRING: case HLL: return Type.DOUBLE; + case DATETIMEV2: + return Type.DEFAULT_DATETIMEV2; + case TIMEV2: + return Type.DEFAULT_TIMEV2; case DECIMALV2: return Type.DECIMALV2; default: diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java index 2b473d7ac0..c74f91750d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java @@ -1664,4 +1664,7 @@ public class Config extends ConfigBase { */ @ConfField(mutable = false, masterOnly = true) public static boolean be_rebalancer_fuzzy_test = false; + + @ConfField(mutable = true, masterOnly = true) + public static boolean use_date_v2_by_default = false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java index 5a6c4261ea..72665d62a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java @@ -97,7 +97,20 @@ public class PartitionsProcDir implements ProcDirInterface { long leftVal; long rightVal; if (subExpr.getChild(1) instanceof DateLiteral) { - leftVal = (new DateLiteral((String) element, Type.DATETIME)).getLongValue(); + Type type; + switch (subExpr.getChild(1).getType().getPrimitiveType()) { + case DATE: + case DATETIME: + type = Type.DATETIME; + break; + case DATEV2: + case DATETIMEV2: + type = Type.DATETIMEV2; + break; + default: + throw new AnalysisException("Invalid date type: " + subExpr.getChild(1).getType()); + } + leftVal = (new DateLiteral((String) element, type)).getLongValue(); rightVal = ((DateLiteral) subExpr.getChild(1)).getLongValue(); } else { leftVal = Long.parseLong(element.toString()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/RollupProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/RollupProcDir.java index 28318a87f0..c3d52787c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/RollupProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/RollupProcDir.java @@ -154,7 +154,20 @@ public class RollupProcDir implements ProcDirInterface { return ((StringLiteral) subExpr.getChild(1)).getValue().equals(element); } if (subExpr.getChild(1) instanceof DateLiteral) { - Long leftVal = (new DateLiteral((String) element, Type.DATETIME)).getLongValue(); + Type type; + switch (subExpr.getChild(1).getType().getPrimitiveType()) { + case DATE: + case DATETIME: + type = Type.DATETIME; + break; + case DATEV2: + case DATETIMEV2: + type = Type.DATETIMEV2; + break; + default: + throw new AnalysisException("Invalid date type: " + subExpr.getChild(1).getType()); + } + Long leftVal = (new DateLiteral((String) element, type)).getLongValue(); Long rightVal = ((DateLiteral) subExpr.getChild(1)).getLongValue(); switch (binaryPredicate.getOp()) { case EQ: diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/SchemaChangeProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/SchemaChangeProcDir.java index 04b0441441..a7e171bb21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/SchemaChangeProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/SchemaChangeProcDir.java @@ -73,7 +73,20 @@ public class SchemaChangeProcDir implements ProcDirInterface { return ((StringLiteral) subExpr.getChild(1)).getValue().equals(element); } if (subExpr.getChild(1) instanceof DateLiteral) { - Long leftVal = (new DateLiteral((String) element, Type.DATETIME)).getLongValue(); + Type type; + switch (subExpr.getChild(1).getType().getPrimitiveType()) { + case DATE: + case DATETIME: + type = Type.DATETIME; + break; + case DATEV2: + case DATETIMEV2: + type = Type.DATETIMEV2; + break; + default: + throw new AnalysisException("Invalid date type: " + subExpr.getChild(1).getType()); + } + Long leftVal = (new DateLiteral((String) element, type)).getLongValue(); Long rightVal = ((DateLiteral) subExpr.getChild(1)).getLongValue(); switch (binaryPredicate.getOp()) { case EQ: diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 563706fcca..b5a73778cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -146,7 +146,7 @@ public class PropertyAnalyzer { throw new AnalysisException("Invalid storage medium: " + value); } } else if (key.equalsIgnoreCase(PROPERTIES_STORAGE_COOLDOWN_TIME)) { - DateLiteral dateLiteral = new DateLiteral(value, Type.DATETIME); + DateLiteral dateLiteral = new DateLiteral(value, DateLiteral.getDefaultDateType(Type.DATETIME)); cooldownTimeStamp = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); } else if (key.equalsIgnoreCase(PROPERTIES_REMOTE_STORAGE_POLICY)) { remoteStoragePolicy = value; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java index b2857d26f3..628d7231bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java @@ -70,6 +70,8 @@ public class Util { TYPE_STRING_MAP.put(PrimitiveType.DOUBLE, "double"); TYPE_STRING_MAP.put(PrimitiveType.DATE, "date"); TYPE_STRING_MAP.put(PrimitiveType.DATETIME, "datetime"); + TYPE_STRING_MAP.put(PrimitiveType.DATEV2, "datev2"); + TYPE_STRING_MAP.put(PrimitiveType.DATETIMEV2, "datetimev2"); TYPE_STRING_MAP.put(PrimitiveType.CHAR, "char(%d)"); TYPE_STRING_MAP.put(PrimitiveType.VARCHAR, "varchar(%d)"); TYPE_STRING_MAP.put(PrimitiveType.STRING, "string"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/DorisTypeToType.java b/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/DorisTypeToType.java index a8569befc7..a5d2701ed4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/DorisTypeToType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/DorisTypeToType.java @@ -93,16 +93,18 @@ public class DorisTypeToType extends DorisTypeVisitor { } else if (atomic.getPrimitiveType().equals(PrimitiveType.CHAR) || atomic.getPrimitiveType().equals(PrimitiveType.VARCHAR)) { return Types.StringType.get(); - } else if (atomic.getPrimitiveType().equals(PrimitiveType.DATE)) { + } else if (atomic.getPrimitiveType().equals(PrimitiveType.DATE) + || atomic.getPrimitiveType().equals(PrimitiveType.DATEV2)) { return Types.DateType.get(); - } else if (atomic.getPrimitiveType().equals(PrimitiveType.TIME)) { + } else if (atomic.getPrimitiveType().equals(PrimitiveType.TIME) + || atomic.getPrimitiveType().equals(PrimitiveType.TIMEV2)) { return Types.TimeType.get(); - } else if (atomic.getPrimitiveType().equals(PrimitiveType.DECIMALV2) - || atomic.getPrimitiveType().equals(PrimitiveType.DECIMALV2)) { + } else if (atomic.getPrimitiveType().equals(PrimitiveType.DECIMALV2)) { return Types.DecimalType.of( ((ScalarType) atomic).getScalarPrecision(), ((ScalarType) atomic).getScalarScale()); - } else if (atomic.getPrimitiveType().equals(PrimitiveType.DATETIME)) { + } else if (atomic.getPrimitiveType().equals(PrimitiveType.DATETIME) + || atomic.getPrimitiveType().equals(PrimitiveType.DATETIMEV2)) { return Types.TimestampType.withZone(); } // unsupported type: PrimitiveType.HLL BITMAP BINARY diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/TypeToDorisType.java b/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/TypeToDorisType.java index 926b95c64f..87774fc40b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/TypeToDorisType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/TypeToDorisType.java @@ -17,8 +17,10 @@ package org.apache.doris.external.iceberg.util; +import org.apache.doris.analysis.DateLiteral; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; import org.apache.iceberg.Schema; import org.apache.iceberg.types.TypeUtil; @@ -78,9 +80,17 @@ public class TypeToDorisType extends TypeUtil.SchemaVisitor { Types.DecimalType decimal = (Types.DecimalType) primitive; return ScalarType.createDecimalV2Type(decimal.precision(), decimal.scale()); case DATE: - return Type.DATE; + try { + return DateLiteral.getDefaultDateType(Type.DATE); + } catch (AnalysisException e) { + return Type.DATE; + } case TIMESTAMP: - return Type.DATETIME; + try { + return DateLiteral.getDefaultDateType(Type.DATETIME); + } catch (AnalysisException e) { + return Type.DATETIME; + } case STRING: return Type.STRING; // use varchar diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java index c96879741c..6151229e67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java @@ -174,7 +174,8 @@ public class StatementSubmitter { // index start from 1 for (int i = 1; i <= colNum; ++i) { String type = rs.getMetaData().getColumnTypeName(i); - if ("DATE".equalsIgnoreCase(type) || "DATETIME".equalsIgnoreCase(type)) { + if ("DATE".equalsIgnoreCase(type) || "DATETIME".equalsIgnoreCase(type) + || "DATEV2".equalsIgnoreCase(type) || "DATETIMEV2".equalsIgnoreCase(type)) { row.add(rs.getString(i)); } else { row.add(rs.getObject(i)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java index ce080b8870..649e030a50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java @@ -575,7 +575,9 @@ public class DeleteHandler implements Writable { binaryPredicate.setChild(1, LiteralExpr.create("0", Type.TINYINT)); } } else if (column.getDataType() == PrimitiveType.DATE - || column.getDataType() == PrimitiveType.DATETIME) { + || column.getDataType() == PrimitiveType.DATETIME + || column.getDataType() == PrimitiveType.DATEV2 + || column.getDataType() == PrimitiveType.DATETIMEV2) { DateLiteral dateLiteral = new DateLiteral(value, Type.fromPrimitiveType(column.getDataType())); value = dateLiteral.getStringValue(); binaryPredicate.setChild(1, LiteralExpr.create(value, @@ -591,9 +593,10 @@ public class DeleteHandler implements Writable { try { InPredicate inPredicate = (InPredicate) condition; for (int i = 1; i <= inPredicate.getInElementNum(); i++) { - value = ((LiteralExpr) inPredicate.getChild(i)).getStringValue(); if (column.getDataType() == PrimitiveType.DATE - || column.getDataType() == PrimitiveType.DATETIME) { + || column.getDataType() == PrimitiveType.DATETIME + || column.getDataType() == PrimitiveType.DATEV2 + || column.getDataType() == PrimitiveType.DATETIMEV2) { DateLiteral dateLiteral = new DateLiteral(value, Type.fromPrimitiveType(column.getDataType())); value = dateLiteral.getStringValue(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java index d5c798a622..7f4d26fa58 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java @@ -249,16 +249,20 @@ public class MysqlSerializer { case DOUBLE: return 22; case TIME: - return 10; + case DATEV2: case DATE: return 10; - case DATETIME: { + case DATETIME: + case DATETIMEV2: { if (type.isTimeType()) { return 10; } else { return 19; } } + // todo:It needs to be obtained according to the field length set during the actual creation, + // todo:which is not supported for the time being.default is 255 + // DECIMAL,DECIMALV2,CHAR,VARCHAR: default: return 255; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java index 1ae7845dbe..a482184a4e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java @@ -175,7 +175,9 @@ public class PartitionRange { switch (type.getPrimitiveType()) { case BOOLEAN: case TIME: + case TIMEV2: case DATETIME: + case DATETIMEV2: case FLOAT: case DOUBLE: case DECIMALV2: @@ -186,6 +188,7 @@ public class PartitionRange { LOG.info("PartitionCache not support such key type {}", type.toSql()); return false; case DATE: + case DATEV2: date = getDateValue(expr); keyType = KeyType.DATE; break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/FEFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FEFunctions.java index b2463d71d2..f95cde929e 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/FEFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FEFunctions.java @@ -62,7 +62,8 @@ public class FEFunctions { public static FloatLiteral timeDiff(LiteralExpr first, LiteralExpr second) throws AnalysisException { long firstTimestamp = ((DateLiteral) first).unixTimestamp(TimeUtils.getTimeZone()); long secondTimestamp = ((DateLiteral) second).unixTimestamp(TimeUtils.getTimeZone()); - return new FloatLiteral((double) (firstTimestamp - secondTimestamp) / 1000, Type.TIME); + return new FloatLiteral((double) (firstTimestamp - secondTimestamp) / 1000, + FloatLiteral.getDefaultTimeType(Type.TIME)); } @FEFunction(name = "datediff", argTypes = { "DATETIME", "DATETIME" }, returnType = "INT") @@ -134,6 +135,7 @@ public class FEFunctions { DateLiteral dateLiteral = new DateLiteral(); try { dateLiteral.fromDateFormatStr(fmtLiteral.getStringValue(), date.getStringValue(), false); + dateLiteral.setType(DateLiteral.getDefaultDateType(dateLiteral.getType())); return dateLiteral; } catch (InvalidFormatException e) { e.printStackTrace(); @@ -220,7 +222,8 @@ public class FEFunctions { if (unixTime.getLongValue() < 0) { throw new AnalysisException("unixtime should larger than zero"); } - DateLiteral dl = new DateLiteral(unixTime.getLongValue() * 1000, TimeUtils.getTimeZone(), Type.DATETIME); + DateLiteral dl = new DateLiteral(unixTime.getLongValue() * 1000, TimeUtils.getTimeZone(), + DateLiteral.getDefaultDateType(Type.DATETIME)); return new StringLiteral(dl.getStringValue()); } @@ -230,13 +233,15 @@ public class FEFunctions { if (unixTime.getLongValue() < 0) { throw new AnalysisException("unixtime should larger than zero"); } - DateLiteral dl = new DateLiteral(unixTime.getLongValue() * 1000, TimeUtils.getTimeZone(), Type.DATETIME); + DateLiteral dl = new DateLiteral(unixTime.getLongValue() * 1000, TimeUtils.getTimeZone(), + DateLiteral.getDefaultDateType(Type.DATETIME)); return new StringLiteral(dl.dateFormat(fmtLiteral.getStringValue())); } @FEFunction(name = "now", argTypes = {}, returnType = "DATETIME") public static DateLiteral now() throws AnalysisException { - return new DateLiteral(LocalDateTime.now(DateTimeZone.forTimeZone(TimeUtils.getTimeZone())), Type.DATETIME); + return new DateLiteral(LocalDateTime.now(DateTimeZone.forTimeZone(TimeUtils.getTimeZone())), + DateLiteral.getDefaultDateType(Type.DATETIME)); } @FEFunction(name = "current_timestamp", argTypes = {}, returnType = "DATETIME") @@ -246,13 +251,15 @@ public class FEFunctions { @FEFunction(name = "curdate", argTypes = {}, returnType = "DATE") public static DateLiteral curDate() throws AnalysisException { - return new DateLiteral(LocalDateTime.now(DateTimeZone.forTimeZone(TimeUtils.getTimeZone())), Type.DATE); + return new DateLiteral(LocalDateTime.now(DateTimeZone.forTimeZone(TimeUtils.getTimeZone())), + DateLiteral.getDefaultDateType(Type.DATE)); } @FEFunction(name = "curtime", argTypes = {}, returnType = "TIME") public static FloatLiteral curTime() throws AnalysisException { DateLiteral now = now(); - return new FloatLiteral((double) (now.getHour() * 3600 + now.getMinute() * 60 + now.getSecond()), Type.TIME); + return new FloatLiteral((double) (now.getHour() * 3600 + now.getMinute() * 60 + now.getSecond()), + FloatLiteral.getDefaultTimeType(Type.TIME)); } @FEFunction(name = "current_time", argTypes = {}, returnType = "TIME") @@ -263,7 +270,7 @@ public class FEFunctions { @FEFunction(name = "utc_timestamp", argTypes = {}, returnType = "DATETIME") public static DateLiteral utcTimestamp() throws AnalysisException { return new DateLiteral(LocalDateTime.now(DateTimeZone.forTimeZone(TimeUtils.getOrSystemTimeZone("+00:00"))), - Type.DATETIME); + DateLiteral.getDefaultDateType(Type.DATETIME)); } @FEFunction(name = "yearweek", argTypes = { "DATE" }, returnType = "INT") @@ -541,7 +548,10 @@ public class FEFunctions { @FEFunction(name = "ifnull", argTypes = {"BIGINT", "BIGINT"}, returnType = "BIGINT"), @FEFunction(name = "ifnull", argTypes = {"DATETIME", "DATETIME"}, returnType = "DATETIME"), @FEFunction(name = "ifnull", argTypes = { "DATE", "DATETIME" }, returnType = "DATETIME"), - @FEFunction(name = "ifnull", argTypes = { "DATETIME", "DATE" }, returnType = "DATETIME") + @FEFunction(name = "ifnull", argTypes = { "DATETIME", "DATE" }, returnType = "DATETIME"), + @FEFunction(name = "ifnull", argTypes = {"DATETIMEV2", "DATETIMEV2"}, returnType = "DATETIME"), + @FEFunction(name = "ifnull", argTypes = { "DATEV2", "DATETIMEV2" }, returnType = "DATETIME"), + @FEFunction(name = "ifnull", argTypes = { "DATETIMEV2", "DATEV2" }, returnType = "DATETIME") }) public static LiteralExpr ifNull(LiteralExpr first, LiteralExpr second) throws AnalysisException { return first instanceof NullLiteral ? second : first; diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/RewriteDateLiteralRule.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/RewriteDateLiteralRule.java index cda54130de..cd6ff37872 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/RewriteDateLiteralRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/RewriteDateLiteralRule.java @@ -63,6 +63,7 @@ public class RewriteDateLiteralRule implements ExprRewriteRule { String dateStr = childExpr.getStringValue(); DateLiteral dateLiteral = new DateLiteral(); dateLiteral.fromDateStr(dateStr); + dateLiteral.setType(DateLiteral.getDefaultDateType(dateLiteral.getType())); expr.setChild(1, dateLiteral); } catch (AnalysisException e) { if (clauseType == ExprRewriter.ClauseType.OTHER_CLAUSE) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStats.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStats.java index df3b17899b..bb81e443ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStats.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStats.java @@ -154,6 +154,8 @@ public class ColumnStats { return decimalLiteral; case DATE: case DATETIME: + case DATEV2: + case DATETIMEV2: return new DateLiteral(columnValue, type); case CHAR: case VARCHAR: diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/HadoopLoadPendingTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/HadoopLoadPendingTask.java index 3e40527394..7e5b110403 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/HadoopLoadPendingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/HadoopLoadPendingTask.java @@ -529,9 +529,11 @@ public class HadoopLoadPendingTask extends LoadPendingTask { columnType = "DOUBLE"; break; case DATE: + case DATEV2: columnType = "DATE"; break; case DATETIME: + case DATETIMEV2: columnType = "DATETIME"; break; case CHAR: diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex index ecc4a59b65..f52f8e3c52 100644 --- a/fe/fe-core/src/main/jflex/sql_scanner.flex +++ b/fe/fe-core/src/main/jflex/sql_scanner.flex @@ -160,6 +160,7 @@ import org.apache.doris.qe.SqlModeHelper; keywordMap.put("databases", new Integer(SqlParserSymbols.KW_DATABASES)); keywordMap.put("date", new Integer(SqlParserSymbols.KW_DATE)); keywordMap.put("datetime", new Integer(SqlParserSymbols.KW_DATETIME)); + keywordMap.put("time", new Integer(SqlParserSymbols.KW_TIME)); keywordMap.put("day", new Integer(SqlParserSymbols.KW_DAY)); keywordMap.put("decimal", new Integer(SqlParserSymbols.KW_DECIMAL)); keywordMap.put("decommission", new Integer(SqlParserSymbols.KW_DECOMMISSION)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java index 62dd02d6dc..d06a176091 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java @@ -152,6 +152,22 @@ public class AlterTest { + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + "PROPERTIES('replication_num' = '1');"); + createTable("CREATE TABLE test.tbl6\n" + + "(\n" + + " k1 datetime(3),\n" + + " k2 time(3),\n" + + " v1 int \n," + + " v2 datetime(3)\n" + + ") ENGINE=OLAP\n" + + "UNIQUE KEY (k1,k2)\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values less than('2020-02-01 00:00:00'),\n" + + " PARTITION p2 values less than('2020-03-01 00:00:00')\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');"); + createTable("create external table test.odbc_table\n" + "( `k1` bigint(20) COMMENT \"\",\n" + " `k2` datetime COMMENT \"\",\n" @@ -425,6 +441,86 @@ public class AlterTest { alterTable(stmt, false); } + @Test + public void testAlterDateV2Operations() throws Exception { + String stmt = "alter table test.tbl6 add partition p3 values less than('2020-04-01 00:00:00')," + + "add partition p4 values less than('2020-05-01 00:00:00')"; + alterTable(stmt, true); + + stmt = "alter table test.tbl6 add partition p3 values less than('2020-04-01 00:00:00'), drop partition p4"; + alterTable(stmt, true); + + stmt = "alter table test.tbl6 drop partition p3, drop partition p4"; + alterTable(stmt, true); + + stmt = "alter table test.tbl6 drop partition p3, add column k3 datetime(6)"; + alterTable(stmt, true); + + // no conflict + stmt = "alter table test.tbl6 add column k3 int, add column k4 time(6)"; + alterTable(stmt, false); + waitSchemaChangeJobDone(false); + + stmt = "alter table test.tbl6 add rollup r1 (k1, k2)"; + alterTable(stmt, false); + waitSchemaChangeJobDone(true); + + stmt = "alter table test.tbl6 add rollup r2 (k1, k2), r3 (k1, k2)"; + alterTable(stmt, false); + waitSchemaChangeJobDone(true); + + // enable dynamic partition + // not adding the `start` property so that it won't drop the origin partition p1, p2 and p3 + stmt = "alter table test.tbl6 set (\n" + + "'dynamic_partition.enable' = 'true',\n" + + "'dynamic_partition.time_unit' = 'DAY',\n" + + "'dynamic_partition.end' = '3',\n" + + "'dynamic_partition.prefix' = 'p',\n" + + "'dynamic_partition.buckets' = '3'\n" + + " );"; + alterTable(stmt, false); + Database db = Catalog.getCurrentInternalCatalog().getDbOrMetaException("default_cluster:test"); + OlapTable tbl = (OlapTable) db.getTableOrMetaException("tbl6"); + Assert.assertTrue(tbl.getTableProperty().getDynamicPartitionProperty().getEnable()); + Assert.assertEquals(4, tbl.getIndexIdToSchema().size()); + + // add partition when dynamic partition is enable + stmt = "alter table test.tbl6 add partition p3 values less than('2020-04-01 00:00:00') distributed by" + + " hash(k2) buckets 4 PROPERTIES ('replication_num' = '1')"; + alterTable(stmt, true); + + // add temp partition when dynamic partition is enable + stmt = "alter table test.tbl6 add temporary partition tp3 values less than('2020-04-01 00:00:00') distributed" + + " by hash(k2) buckets 4 PROPERTIES ('replication_num' = '1')"; + alterTable(stmt, false); + Assert.assertEquals(1, tbl.getTempPartitions().size()); + + // disable the dynamic partition + stmt = "alter table test.tbl6 set ('dynamic_partition.enable' = 'false')"; + alterTable(stmt, false); + Assert.assertFalse(tbl.getTableProperty().getDynamicPartitionProperty().getEnable()); + + // add partition when dynamic partition is disable + stmt = "alter table test.tbl6 add partition p3 values less than('2020-04-01 00:00:00') distributed" + + " by hash(k2) buckets 4"; + alterTable(stmt, false); + + // set table's default replication num + Assert.assertEquals((short) 1, tbl.getDefaultReplicaAllocation().getTotalReplicaNum()); + stmt = "alter table test.tbl6 set ('default.replication_num' = '3');"; + alterTable(stmt, false); + Assert.assertEquals((short) 3, tbl.getDefaultReplicaAllocation().getTotalReplicaNum()); + + // set range table's real replication num + Partition p1 = tbl.getPartition("p1"); + Assert.assertEquals(Short.valueOf("1"), Short.valueOf(tbl.getPartitionInfo().getReplicaAllocation(p1.getId()) + .getTotalReplicaNum())); + stmt = "alter table test.tbl6 set ('replication_num' = '3');"; + alterTable(stmt, true); + Assert.assertEquals(Short.valueOf("1"), Short.valueOf(tbl.getPartitionInfo().getReplicaAllocation(p1.getId()) + .getTotalReplicaNum())); + } + // test batch update range partitions' properties @Test public void testBatchUpdatePartitionProperties() throws Exception { @@ -899,6 +995,48 @@ public class AlterTest { alterTable(changeOrderStmt, true); } + @Test + public void testAlterDateV2Schema() throws Exception { + createTable("CREATE TABLE test.unique_partition_datev2\n" + + "(\n" + + " k1 date,\n" + + " k2 datetime(3),\n" + + " k3 datetime,\n" + + " v1 date,\n" + + " v2 datetime(3),\n" + + " v3 datetime,\n" + + " v4 int\n" + + ")\n" + + "UNIQUE KEY(k1, k2, k3)\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + " PARTITION p1 values less than('2020-02-01'),\n" + + " PARTITION p2 values less than('2020-03-01')\n" + + ")\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 3\n" + + "PROPERTIES('replication_num' = '1');"); + + // partition key can not be changed. + String changeOrderStmt = "ALTER TABLE test.unique_partition_datev2 modify column k1 int key null"; + alterTable(changeOrderStmt, true); + changeOrderStmt = "ALTER TABLE test.unique_partition_datev2 modify column k2 int key null"; + alterTable(changeOrderStmt, true); + changeOrderStmt = "ALTER TABLE test.unique_partition_datev2 modify column k3 int key null"; + alterTable(changeOrderStmt, true); + + // partition keys which are date type should be changed between each other. + changeOrderStmt = "ALTER TABLE test.unique_partition_datev2 modify column k2 datetime key null"; + alterTable(changeOrderStmt, false); + waitSchemaChangeJobDone(false); + changeOrderStmt = "ALTER TABLE test.unique_partition_datev2 modify column k3 datetime(3) key null"; + alterTable(changeOrderStmt, false); + waitSchemaChangeJobDone(false); + // Change to another precision datetime + changeOrderStmt = "ALTER TABLE test.unique_partition_datev2 modify column k3 datetime(6) key null"; + alterTable(changeOrderStmt, false); + waitSchemaChangeJobDone(false); + } + private boolean checkAllTabletsExists(List tabletIds) { TabletInvertedIndex invertedIndex = Catalog.getCurrentCatalog().getTabletInvertedIndex(); for (long tabletId : tabletIds) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/AggregateTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/AggregateTest.java index ce5ef14b55..a5b19b8bfc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/AggregateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/AggregateTest.java @@ -38,8 +38,9 @@ public class AggregateTest extends TestWithFeService { dorisAssert = new DorisAssert(); dorisAssert.withDatabase(DB_NAME).useDatabase(DB_NAME); String createTableSQL = "create table " + DB_NAME + "." + TABLE_NAME + " (empid int, name varchar, " - + "deptno int, salary int, commission int, time DATETIME) " - + "distributed by hash(empid) buckets 3 properties('replication_num' = '1');"; + + "deptno int, salary int, commission int, time_col DATETIME, timev2_col " + + " DATETIME(3)) distributed by hash(empid) buckets 3" + + " properties('replication_num' = '1');"; createTable(createTableSQL); } @@ -52,11 +53,13 @@ public class AggregateTest extends TestWithFeService { // NOT support mix distinct, one DistinctAggregationFunction has one column, the other DistinctAggregationFunction has some columns. do { - String query = "select count(distinct empid), count(distinct salary), count(distinct empid, salary) from " + DB_NAME + "." + TABLE_NAME; + String query = "select count(distinct empid), count(distinct salary), " + + "count(distinct empid, salary) from " + DB_NAME + "." + TABLE_NAME; try { UtFrameUtils.parseAndAnalyzeStmt(query, ctx); } catch (AnalysisException e) { - Assert.assertTrue(e.getMessage().contains("The query contains multi count distinct or sum distinct, each can't have multi columns.")); + Assert.assertTrue(e.getMessage().contains( + "The query contains multi count distinct or sum distinct, each can't have multi columns.")); break; } catch (Exception e) { Assert.fail("must be AnalysisException."); @@ -90,19 +93,43 @@ public class AggregateTest extends TestWithFeService { ConnectContext ctx = UtFrameUtils.createDefaultCtx(); // normal. - { // CHECKSTYLE IGNORE THIS LINE - String query = "select empid, window_funnel(1, 'default', time, empid = 1, empid = 2) from " + do { + String query = "select empid, window_funnel(1, 'default', time_col, empid = 1, empid = 2) from " + DB_NAME + "." + TABLE_NAME + " group by empid"; try { UtFrameUtils.parseAndAnalyzeStmt(query, ctx); } catch (Exception e) { Assert.fail("must be AnalysisException."); } - } // CHECKSTYLE IGNORE THIS LINE + } while (false); + + do { + String query = "select empid, window_funnel(1, 'default', timev2_col, empid = 1, empid = 2) from " + + DB_NAME + "." + TABLE_NAME + " group by empid"; + try { + UtFrameUtils.parseAndAnalyzeStmt(query, ctx); + } catch (Exception e) { + Assert.fail("must be AnalysisException."); + } + } while (false); // less argument. do { - String query = "select empid, window_funnel(1, 'default', time) from " + String query = "select empid, window_funnel(1, 'default', time_col) from " + + DB_NAME + "." + TABLE_NAME + " group by empid"; + try { + UtFrameUtils.parseAndAnalyzeStmt(query, ctx); + } catch (AnalysisException e) { + Assert.assertTrue(e.getMessage().contains("function must have at least four params")); + break; + } catch (Exception e) { + Assert.fail("must be AnalysisException."); + } + Assert.fail("must be AnalysisException."); + } while (false); + + do { + String query = "select empid, window_funnel(1, 'default', timev2_col) from " + DB_NAME + "." + TABLE_NAME + " group by empid"; try { UtFrameUtils.parseAndAnalyzeStmt(query, ctx); @@ -117,12 +144,13 @@ public class AggregateTest extends TestWithFeService { // argument with wrong type. do { - String query = "select empid, window_funnel('xx', 'default', time, empid = 1) from " + String query = "select empid, window_funnel('xx', 'default', time_col, empid = 1) from " + DB_NAME + "." + TABLE_NAME + " group by empid"; try { UtFrameUtils.parseAndAnalyzeStmt(query, ctx); } catch (AnalysisException e) { - Assert.assertTrue(e.getMessage().contains("The window params of window_funnel function must be integer")); + Assert.assertTrue( + e.getMessage().contains("The window params of window_funnel function must be integer")); break; } catch (Exception e) { Assert.fail("must be AnalysisException."); @@ -131,12 +159,13 @@ public class AggregateTest extends TestWithFeService { } while (false); do { - String query = "select empid, window_funnel(1, 1, time, empid = 1) from " + String query = "select empid, window_funnel('xx', 'default', timev2_col, empid = 1) from " + DB_NAME + "." + TABLE_NAME + " group by empid"; try { UtFrameUtils.parseAndAnalyzeStmt(query, ctx); } catch (AnalysisException e) { - Assert.assertTrue(e.getMessage().contains("The mode params of window_funnel function must be integer")); + Assert.assertTrue( + e.getMessage().contains("The window params of window_funnel function must be integer")); break; } catch (Exception e) { Assert.fail("must be AnalysisException."); @@ -144,6 +173,35 @@ public class AggregateTest extends TestWithFeService { Assert.fail("must be AnalysisException."); } while (false); + do { + String query = "select empid, window_funnel(1, 1, time_col, empid = 1) from " + + DB_NAME + "." + TABLE_NAME + " group by empid"; + try { + UtFrameUtils.parseAndAnalyzeStmt(query, ctx); + } catch (AnalysisException e) { + Assert.assertTrue( + e.getMessage().contains("The mode params of window_funnel function must be integer")); + break; + } catch (Exception e) { + Assert.fail("must be AnalysisException."); + } + Assert.fail("must be AnalysisException."); + } while (false); + + do { + String query = "select empid, window_funnel(1, 1, timev2_col, empid = 1) from " + + DB_NAME + "." + TABLE_NAME + " group by empid"; + try { + UtFrameUtils.parseAndAnalyzeStmt(query, ctx); + } catch (AnalysisException e) { + Assert.assertTrue( + e.getMessage().contains("The mode params of window_funnel function must be integer")); + break; + } catch (Exception e) { + Assert.fail("must be AnalysisException."); + } + Assert.fail("must be AnalysisException."); + } while (false); do { String query = "select empid, window_funnel(1, '1', empid, '1') from " @@ -151,7 +209,8 @@ public class AggregateTest extends TestWithFeService { try { UtFrameUtils.parseAndAnalyzeStmt(query, ctx); } catch (AnalysisException e) { - Assert.assertTrue(e.getMessage().contains("The 3rd param of window_funnel function must be DATE or DATETIME")); + Assert.assertTrue( + e.getMessage().contains("The 3rd param of window_funnel function must be DATE or DATETIME")); break; } catch (Exception e) { Assert.fail("must be AnalysisException."); @@ -160,12 +219,28 @@ public class AggregateTest extends TestWithFeService { } while (false); do { - String query = "select empid, window_funnel(1, '1', time, '1') from " + String query = "select empid, window_funnel(1, '1', time_col, '1') from " + DB_NAME + "." + TABLE_NAME + " group by empid"; try { UtFrameUtils.parseAndAnalyzeStmt(query, ctx); } catch (AnalysisException e) { - Assert.assertTrue(e.getMessage().contains("The 4th and subsequent params of window_funnel function must be boolean")); + Assert.assertTrue(e.getMessage().contains( + "The 4th and subsequent params of window_funnel function must be boolean")); + break; + } catch (Exception e) { + Assert.fail("must be AnalysisException."); + } + Assert.fail("must be AnalysisException."); + } while (false); + + do { + String query = "select empid, window_funnel(1, '1', timev2_col, '1') from " + + DB_NAME + "." + TABLE_NAME + " group by empid"; + try { + UtFrameUtils.parseAndAnalyzeStmt(query, ctx); + } catch (AnalysisException e) { + Assert.assertTrue(e.getMessage().contains( + "The 4th and subsequent params of window_funnel function must be boolean")); break; } catch (Exception e) { Assert.fail("must be AnalysisException."); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ComparisonPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ComparisonPredicateTest.java index c13b31909e..ba7b943ff8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ComparisonPredicateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ComparisonPredicateTest.java @@ -125,4 +125,46 @@ public class ComparisonPredicateTest { Assert.assertFalse(range.hasLowerBound()); } + @Test + public void testConvertToRangeForDateV2() { + SlotRef slotRef = new SlotRef(new TableName("db1", "tb1"), "k1"); + LiteralExpr dateExpr = new DateLiteral(2022, 5, 19, Type.DATE); + LiteralExpr dateV2Expr = new DateLiteral(2022, 5, 19, Type.DATEV2); + BinaryPredicate binaryPredicate1 = new BinaryPredicate(BinaryPredicate.Operator.LE, slotRef, dateExpr); + BinaryPredicate binaryPredicate2 = new BinaryPredicate(BinaryPredicate.Operator.LE, slotRef, dateV2Expr); + Range range1 = binaryPredicate1.convertToRange(); + Range range2 = binaryPredicate2.convertToRange(); + Assert.assertEquals(dateExpr, range1.upperEndpoint()); + Assert.assertEquals(dateV2Expr, range1.upperEndpoint()); + Assert.assertEquals(BoundType.CLOSED, range1.upperBoundType()); + Assert.assertFalse(range1.hasLowerBound()); + Assert.assertEquals(dateExpr, range2.upperEndpoint()); + Assert.assertEquals(dateV2Expr, range2.upperEndpoint()); + Assert.assertEquals(BoundType.CLOSED, range2.upperBoundType()); + Assert.assertFalse(range2.hasLowerBound()); + } + + @Test + public void testConvertToRangeForDateTimeV2() { + SlotRef slotRef = new SlotRef(new TableName("db1", "tb1"), "k1"); + LiteralExpr dateTimeExpr = new DateLiteral(2022, 5, 19, 0, 0, 0, Type.DATETIME); + LiteralExpr dateTimeV2Expr1 = new DateLiteral(2022, 5, 19, 0, 0, 0, Type.DEFAULT_DATETIMEV2); + LiteralExpr dateTimeV2Expr2 = new DateLiteral(2022, 5, 19, 0, 0, 0, ScalarType.createDatetimeV2Type(6)); + BinaryPredicate binaryPredicate1 = new BinaryPredicate(BinaryPredicate.Operator.LE, slotRef, dateTimeExpr); + BinaryPredicate binaryPredicate2 = new BinaryPredicate(BinaryPredicate.Operator.LE, slotRef, dateTimeV2Expr1); + Range range1 = binaryPredicate1.convertToRange(); + Range range2 = binaryPredicate2.convertToRange(); + + Assert.assertEquals(dateTimeExpr, range1.upperEndpoint()); + Assert.assertEquals(dateTimeV2Expr1, range1.upperEndpoint()); + Assert.assertEquals(dateTimeV2Expr2, range1.upperEndpoint()); + Assert.assertEquals(BoundType.CLOSED, range1.upperBoundType()); + Assert.assertFalse(range1.hasLowerBound()); + + Assert.assertEquals(dateTimeExpr, range2.upperEndpoint()); + Assert.assertEquals(dateTimeV2Expr1, range2.upperEndpoint()); + Assert.assertEquals(BoundType.CLOSED, range2.upperBoundType()); + Assert.assertFalse(range2.hasLowerBound()); + Assert.assertEquals(dateTimeV2Expr2, range2.upperEndpoint()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java index 64b55c43d7..db2dbd797d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java @@ -17,6 +17,7 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.InvalidFormatException; @@ -163,4 +164,184 @@ public class DateLiteralTest { } Assert.assertFalse(hasException); } + + @Test + public void testTwoDigitYearForDateV2() { + boolean hasException = false; + try { + DateLiteral literal = new DateLiteral("1997-10-07", Type.DATEV2); + Assert.assertEquals(1997, literal.getYear()); + + DateLiteral literal2 = new DateLiteral("97-10-07", Type.DATEV2); + Assert.assertEquals(1997, literal2.getYear()); + + DateLiteral literal3 = new DateLiteral("0097-10-07", Type.DATEV2); + Assert.assertEquals(97, literal3.getYear()); + + DateLiteral literal4 = new DateLiteral("99-10-07", Type.DATEV2); + Assert.assertEquals(1999, literal4.getYear()); + + DateLiteral literal5 = new DateLiteral("70-10-07", Type.DATEV2); + Assert.assertEquals(1970, literal5.getYear()); + + DateLiteral literal6 = new DateLiteral("69-10-07", Type.DATEV2); + Assert.assertEquals(2069, literal6.getYear()); + + DateLiteral literal7 = new DateLiteral("00-10-07", Type.DATEV2); + Assert.assertEquals(2000, literal7.getYear()); + + } catch (AnalysisException e) { + e.printStackTrace(); + hasException = true; + } + Assert.assertFalse(hasException); + } + + @Test + public void testDateFormatForDateV2() { + boolean hasException = false; + try { + DateLiteral literal = new DateLiteral("1997-10-7", Type.DATEV2); + Assert.assertEquals(1997, literal.getYear()); + + literal = new DateLiteral("2021-06-1", Type.DATEV2); + Assert.assertEquals(2021, literal.getYear()); + Assert.assertEquals(6, literal.getMonth()); + Assert.assertEquals(1, literal.getDay()); + + literal = new DateLiteral("2022-6-01", Type.DATEV2); + Assert.assertEquals(2022, literal.getYear()); + Assert.assertEquals(6, literal.getMonth()); + Assert.assertEquals(1, literal.getDay()); + + literal = new DateLiteral("2023-6-1", Type.DATEV2); + Assert.assertEquals(2023, literal.getYear()); + Assert.assertEquals(6, literal.getMonth()); + Assert.assertEquals(1, literal.getDay()); + + literal = new DateLiteral("20230601", Type.DATEV2); + Assert.assertEquals(2023, literal.getYear()); + Assert.assertEquals(6, literal.getMonth()); + Assert.assertEquals(1, literal.getDay()); + } catch (AnalysisException e) { + e.printStackTrace(); + hasException = true; + } + Assert.assertFalse(hasException); + } + + @Test + public void testParseDateTimeToHourORMinuteForDateV2() throws Exception { + String s = "2020-12-13 12:13:14"; + Type type = Type.DATETIMEV2; + DateLiteral literal = new DateLiteral(s, type); + Assert.assertTrue(literal.toSql().contains("2020-12-13 12:13:14")); + s = "2020-12-13 12:13"; + literal = new DateLiteral(s, type); + Assert.assertTrue(literal.toSql().contains("2020-12-13 12:13:00")); + s = "2020-12-13 12"; + literal = new DateLiteral(s, type); + Assert.assertTrue(literal.toSql().contains("2020-12-13 12:00:00")); + } + + @Test + public void testUncheckedCastToForDateV2() { + boolean hasException = false; + try { + // DATEV2 -> DATE/DATETIME/DATETIMEV2 + DateLiteral literal = new DateLiteral("1997-10-07", Type.DATEV2); + Expr castToExpr = literal.uncheckedCastTo(Type.DATE); + Assert.assertTrue(castToExpr instanceof DateLiteral); + Assert.assertEquals(castToExpr.type, Type.DATE); + castToExpr = literal.uncheckedCastTo(Type.DATETIME); + Assert.assertTrue(castToExpr instanceof DateLiteral); + Assert.assertEquals(castToExpr.type, Type.DATETIME); + castToExpr = literal.uncheckedCastTo(Type.DATETIMEV2); + Assert.assertTrue(castToExpr instanceof DateLiteral); + Assert.assertEquals(castToExpr.type, Type.DATETIMEV2); + + // DATE -> DATEV2/DATETIME/DATETIMEV2 + literal = new DateLiteral("1997-10-07", Type.DATE); + castToExpr = literal.uncheckedCastTo(Type.DATEV2); + Assert.assertTrue(castToExpr instanceof DateLiteral); + Assert.assertEquals(castToExpr.type, Type.DATEV2); + castToExpr = literal.uncheckedCastTo(Type.DATETIME); + Assert.assertTrue(castToExpr instanceof DateLiteral); + Assert.assertEquals(castToExpr.type, Type.DATETIME); + castToExpr = literal.uncheckedCastTo(Type.DATETIMEV2); + Assert.assertTrue(castToExpr instanceof DateLiteral); + Assert.assertEquals(castToExpr.type, Type.DATETIMEV2); + + // DATETIME -> DATEV2/DATE/DATETIMEV2 + literal = new DateLiteral("1997-10-07 12:23:23", Type.DATETIME); + castToExpr = literal.uncheckedCastTo(Type.DATEV2); + Assert.assertTrue(castToExpr instanceof DateLiteral); + Assert.assertEquals(castToExpr.type, Type.DATEV2); + castToExpr = literal.uncheckedCastTo(Type.DATE); + Assert.assertTrue(castToExpr instanceof DateLiteral); + Assert.assertEquals(castToExpr.type, Type.DATE); + castToExpr = literal.uncheckedCastTo(Type.DATETIMEV2); + Assert.assertTrue(castToExpr instanceof DateLiteral); + Assert.assertEquals(castToExpr.type, Type.DATETIMEV2); + + // DATETIMEV2 -> DATEV2/DATE/DATETIME + literal = new DateLiteral("1997-10-07 12:23:23", Type.DATETIMEV2); + castToExpr = literal.uncheckedCastTo(Type.DATEV2); + Assert.assertTrue(castToExpr instanceof DateLiteral); + Assert.assertEquals(castToExpr.type, Type.DATEV2); + castToExpr = literal.uncheckedCastTo(Type.DATE); + Assert.assertTrue(castToExpr instanceof DateLiteral); + Assert.assertEquals(castToExpr.type, Type.DATE); + castToExpr = literal.uncheckedCastTo(Type.DATETIME); + Assert.assertTrue(castToExpr instanceof DateLiteral); + Assert.assertEquals(castToExpr.type, Type.DATETIME); + Type t = ScalarType.createDatetimeV2Type(6); + castToExpr = literal.uncheckedCastTo(t); + Assert.assertTrue(castToExpr instanceof DateLiteral); + Assert.assertEquals(castToExpr.type, t); + } catch (AnalysisException e) { + e.printStackTrace(); + hasException = true; + } + Assert.assertFalse(hasException); + } + + @Test + public void testCheckDateForDateV2() throws AnalysisException { + boolean hasException = false; + try { + DateLiteral dateLiteral = new DateLiteral(); + dateLiteral.fromDateFormatStr("%Y%m%d", "19971007", false, Type.DATEV2); + Assert.assertFalse(Deencapsulation.invoke(dateLiteral, "checkDate")); + + dateLiteral.fromDateFormatStr("%Y%m%d", "19970007", false, Type.DATEV2); + Assert.assertFalse(Deencapsulation.invoke(dateLiteral, "checkDate")); + + dateLiteral.fromDateFormatStr("%Y%m%d", "19971000", false, Type.DATEV2); + Assert.assertFalse(Deencapsulation.invoke(dateLiteral, "checkDate")); + + dateLiteral.fromDateFormatStr("%Y%m%d", "20000229", false, Type.DATEV2); + Assert.assertFalse(Deencapsulation.invoke(dateLiteral, "checkDate")); + + } catch (InvalidFormatException e) { + e.printStackTrace(); + hasException = true; + } + Assert.assertFalse(hasException); + } + + @Test + public void testCheckRangeForDateV2() { + boolean hasException = false; + try { + DateLiteral dateLiteral = new DateLiteral(); + dateLiteral.fromDateFormatStr("%Y%m%d%H%i%s%f", "20201209123456123456", false, Type.DATETIMEV2); + Assert.assertFalse(Deencapsulation.invoke(dateLiteral, "checkRange")); + + } catch (InvalidFormatException e) { + e.printStackTrace(); + hasException = true; + } + Assert.assertFalse(hasException); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java index c8fcd9793c..ad461d93dc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java @@ -17,6 +17,7 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; @@ -166,6 +167,10 @@ public class ExprTest { Expr r2 = new DateLiteral(2020, 10, 21); Expr r3 = new DateLiteral(2020, 10, 22); Expr r4 = new DateLiteral(2020, 10, 23); + Expr r5 = new DateLiteral(2020, 10, 23, Type.DATEV2); + Expr r6 = new DateLiteral(2020, 10, 23, 0, 0, 0, Type.DATETIME); + Expr r7 = new DateLiteral(2020, 10, 23, 0, 0, 0, Type.DATETIMEV2); + Expr r8 = new DateLiteral(2020, 10, 23, 0, 0, 0, ScalarType.createDatetimeV2Type(3)); //list1 equal list2 List list1 = new ArrayList<>(); @@ -173,9 +178,17 @@ public class ExprTest { list1.add(r1); list1.add(r2); list1.add(r3); + list1.add(r5); + list1.add(r6); + list1.add(r7); + list1.add(r8); list2.add(r1); list2.add(r2); list2.add(r3); + list2.add(r5); + list2.add(r6); + list2.add(r7); + list2.add(r8); Assert.assertTrue(Expr.equalSets(list1, list2)); //list3 not equal list4 diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/InPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/InPredicateTest.java index 3b688f9069..d2f32e37c9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/InPredicateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/InPredicateTest.java @@ -17,6 +17,7 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import com.clearspring.analytics.util.Lists; @@ -106,4 +107,84 @@ public class InPredicateTest { Assert.assertEquals(1, inPredicate2.getListChildren().size()); Assert.assertTrue(inPredicate2.contains(literalChild3)); } + + @Test + public void testIntersectionWithDateV2() throws AnalysisException { + SlotRef slotRef1 = new SlotRef(new TableName("db1", "tb1"), "k1"); + LiteralExpr literalChild1 = new DateLiteral(2022, 5, 19, Type.DATE); + LiteralExpr literalChild2 = new DateLiteral(2022, 5, 20, Type.DATEV2); + List literalChildren1 = Lists.newArrayList(); + literalChildren1.add(literalChild1); + literalChildren1.add(literalChild2); + InPredicate inPredicate1 = new InPredicate(slotRef1, literalChildren1, false); + + SlotRef slotRef2 = new SlotRef(new TableName("db1", "tb1"), "k1"); + LiteralExpr literalChild3 = new DateLiteral(2022, 5, 19, Type.DATEV2); + LiteralExpr literalChild4 = new DateLiteral(2022, 5, 21, Type.DATE); + List literalChildren2 = Lists.newArrayList(); + literalChildren2.add(literalChild3); + literalChildren2.add(literalChild4); + InPredicate inPredicate2 = new InPredicate(slotRef2, literalChildren2, false); + + // check result + InPredicate intersection = inPredicate1.intersection(inPredicate2); + Assert.assertEquals(slotRef1, intersection.getChild(0)); + Assert.assertTrue(intersection.isLiteralChildren()); + Assert.assertEquals(1, intersection.getListChildren().size()); + Assert.assertEquals(literalChild1, intersection.getChild(1)); + + // keep origin predicate + Assert.assertTrue(inPredicate1.isLiteralChildren()); + Assert.assertEquals(2, inPredicate1.getListChildren().size()); + Assert.assertTrue(inPredicate1.contains(literalChild1)); + Assert.assertTrue(inPredicate1.contains(literalChild2)); + Assert.assertTrue(inPredicate2.isLiteralChildren()); + Assert.assertEquals(2, inPredicate2.getListChildren().size()); + Assert.assertTrue(inPredicate2.contains(literalChild3)); + Assert.assertTrue(inPredicate2.contains(literalChild4)); + } + + /* + InPredicate1: k1 in (1,2) + InPredicate2: k1 in (1) + Union: k1 in (1,2) + */ + @Test + public void testUnionWithDateV2() throws AnalysisException { + SlotRef slotRef1 = new SlotRef(new TableName("db1", "tb1"), "k1"); + LiteralExpr literalChild1 = new DateLiteral(2022, 5, 19, Type.DATE); + LiteralExpr literalChild2 = new DateLiteral(2022, 5, 20, Type.DATEV2); + LiteralExpr literalChild3 = new DateLiteral(2022, 5, 20, 0, 0, 0, Type.DATETIME); + List literalChildren1 = Lists.newArrayList(); + literalChildren1.add(literalChild1); + literalChildren1.add(literalChild2); + literalChildren1.add(literalChild3); + InPredicate inPredicate1 = new InPredicate(slotRef1, literalChildren1, false); + + SlotRef slotRef2 = new SlotRef(new TableName("db1", "tb1"), "k1"); + LiteralExpr literalChild4 = new DateLiteral(2022, 5, 19, Type.DATE); + LiteralExpr literalChild5 = new DateLiteral(2022, 5, 20, 0, 0, 0, Type.DATETIMEV2); + List literalChildren2 = Lists.newArrayList(); + literalChildren2.add(literalChild4); + literalChildren2.add(literalChild5); + InPredicate inPredicate2 = new InPredicate(slotRef2, literalChildren2, false); + + // check result + InPredicate union = inPredicate1.union(inPredicate2); + Assert.assertEquals(slotRef1, union.getChild(0)); + Assert.assertTrue(union.isLiteralChildren()); + Assert.assertEquals(2, union.getListChildren().size()); + Assert.assertTrue(union.getListChildren().contains(literalChild1)); + Assert.assertTrue(union.getListChildren().contains(literalChild2)); + Assert.assertTrue(union.getListChildren().contains(literalChild5)); + + // keep origin predicate + Assert.assertTrue(inPredicate1.isLiteralChildren()); + Assert.assertEquals(3, inPredicate1.getListChildren().size()); + Assert.assertTrue(inPredicate1.contains(literalChild1)); + Assert.assertTrue(inPredicate1.contains(literalChild2)); + Assert.assertTrue(inPredicate2.isLiteralChildren()); + Assert.assertEquals(2, inPredicate2.getListChildren().size()); + Assert.assertTrue(inPredicate2.contains(literalChild3)); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/LiteralExprCompareTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/LiteralExprCompareTest.java index 4a0f556ea5..baf07d8a2d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/LiteralExprCompareTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/LiteralExprCompareTest.java @@ -61,12 +61,24 @@ public class LiteralExprCompareTest { Assert.fail(); } + @Test(expected = AnalysisException.class) + public void dateV2Format1Test() throws AnalysisException { + new DateLiteral("2015-02-15 12:12:12", ScalarType.DATEV2); + Assert.fail(); + } + @Test(expected = AnalysisException.class) public void dateFormat2Test() throws AnalysisException { new DateLiteral("2015-02-15", ScalarType.DATETIME); Assert.fail(); } + @Test(expected = AnalysisException.class) + public void dateV2Format2Test() throws AnalysisException { + new DateLiteral("2015-02-15", ScalarType.DATETIMEV2); + Assert.fail(); + } + @Test public void dateTest() throws AnalysisException { LiteralExpr date1 = new DateLiteral("2015-02-15", ScalarType.DATE); @@ -121,6 +133,114 @@ public class LiteralExprCompareTest { Assert.assertTrue(!minDatetime1.equals(datetime1) && -1 == minDatetime1.compareLiteral(datetime1)); } + @Test + public void dateV2Test() throws AnalysisException { + LiteralExpr date1 = new DateLiteral("2015-02-15", ScalarType.DATEV2); + LiteralExpr date1Same = new DateLiteral("2015-02-15", ScalarType.DATEV2); + LiteralExpr date1Large = new DateLiteral("2015-02-16", ScalarType.DATEV2); + LiteralExpr datetime1 = new DateLiteral("2015-02-15 13:14:00", ScalarType.DATETIMEV2); + LiteralExpr datetime1Same = new DateLiteral("2015-02-15 13:14:00", ScalarType.DATETIMEV2); + LiteralExpr datetime1Large = new DateLiteral("2015-02-15 13:14:15", ScalarType.DATETIMEV2); + + // infinity + LiteralExpr maxDate1 = new DateLiteral(ScalarType.DATEV2, true); + LiteralExpr maxDate1Same = new DateLiteral(ScalarType.DATEV2, true); + LiteralExpr minDate1 = new DateLiteral(ScalarType.DATEV2, false); + LiteralExpr minDate1Same = new DateLiteral(ScalarType.DATEV2, false); + LiteralExpr maxDatetime1 = new DateLiteral(ScalarType.DATETIMEV2, true); + LiteralExpr maxDatetime1Same = new DateLiteral(ScalarType.DATETIMEV2, true); + LiteralExpr minDatetime1 = new DateLiteral(ScalarType.DATETIMEV2, false); + LiteralExpr minDatetime1Same = new DateLiteral(ScalarType.DATETIMEV2, false); + LiteralExpr date8 = new DateLiteral("9999-12-31", ScalarType.DATEV2); + LiteralExpr date9 = new DateLiteral("9999-12-31 23:59:59", ScalarType.DATETIMEV2); + LiteralExpr date10 = new DateLiteral("0000-01-01", ScalarType.DATEV2); + LiteralExpr date11 = new DateLiteral("0000-01-01 00:00:00", ScalarType.DATETIMEV2); + + Assert.assertTrue(date1.equals(date1Same) && date1.compareLiteral(date1Same) == 0); + Assert.assertTrue(date1.equals(date1Same) && date1.compareLiteral(date1Same) == 0); + Assert.assertTrue(datetime1.equals(datetime1Same) && datetime1.compareLiteral(datetime1Same) == 0); + Assert.assertTrue(datetime1.equals(datetime1) && datetime1.compareLiteral(datetime1) == 0); + + // value compare + Assert.assertTrue(!date1Large.equals(date1Same) && 1 == date1Large.compareLiteral(date1Same)); + Assert.assertTrue(!datetime1Large.equals(datetime1Same) && 1 == datetime1Large.compareLiteral(datetime1Same)); + Assert.assertTrue(!datetime1Same.equals(datetime1Large) && -1 == datetime1Same.compareLiteral(datetime1Large)); + + // infinity + Assert.assertTrue(maxDate1.equals(maxDate1) && maxDate1.compareLiteral(maxDate1) == 0); + Assert.assertTrue(maxDate1.equals(maxDate1Same) && maxDate1.compareLiteral(maxDate1Same) == 0); + Assert.assertTrue(minDate1.equals(minDate1) && minDate1.compareLiteral(minDate1) == 0); + Assert.assertTrue(minDate1.equals(minDate1Same) && minDate1.compareLiteral(minDate1Same) == 0); + Assert.assertTrue(maxDatetime1.equals(maxDatetime1) && maxDatetime1.compareLiteral(maxDatetime1) == 0); + Assert.assertTrue(maxDatetime1.equals(maxDatetime1Same) && maxDatetime1.compareLiteral(maxDatetime1Same) == 0); + Assert.assertTrue(minDatetime1.equals(minDatetime1) && minDatetime1.compareLiteral(minDatetime1) == 0); + Assert.assertTrue(minDatetime1.equals(minDatetime1Same) && minDatetime1.compareLiteral(minDatetime1Same) == 0); + + Assert.assertTrue(maxDate1.equals(date8) && maxDate1.compareLiteral(date8) == 0); + Assert.assertTrue(minDate1.equals(date10) && minDate1.compareLiteral(date10) == 0); + Assert.assertTrue(maxDatetime1.equals(date9) && maxDatetime1.compareLiteral(date9) == 0); + Assert.assertTrue(minDatetime1.equals(date11) && minDatetime1.compareLiteral(date11) == 0); + + Assert.assertTrue(!maxDate1.equals(date1) && 1 == maxDate1.compareLiteral(date1)); + Assert.assertTrue(!minDate1.equals(date1) && -1 == minDate1.compareLiteral(date1)); + Assert.assertTrue(!maxDatetime1.equals(datetime1) && 1 == maxDatetime1.compareLiteral(datetime1)); + Assert.assertTrue(!minDatetime1.equals(datetime1) && -1 == minDatetime1.compareLiteral(datetime1)); + } + + @Test + public void dateCompatibilityTest() throws AnalysisException { + LiteralExpr date1 = new DateLiteral("2015-02-15", ScalarType.DATEV2); + LiteralExpr date1Same = new DateLiteral("2015-02-15", ScalarType.DATEV2); + LiteralExpr date1Large = new DateLiteral("2015-02-16", ScalarType.DATEV2); + LiteralExpr datetime1 = new DateLiteral("2015-02-15 13:14:00", ScalarType.DATETIMEV2); + LiteralExpr datetime1Same = new DateLiteral("2015-02-15 13:14:00", ScalarType.DATETIMEV2); + LiteralExpr datetime1Large = new DateLiteral("2015-02-15 13:14:15", ScalarType.DATETIMEV2); + + // infinity + LiteralExpr maxDate1 = new DateLiteral(ScalarType.DATE, true); + LiteralExpr maxDate1Same = new DateLiteral(ScalarType.DATE, true); + LiteralExpr minDate1 = new DateLiteral(ScalarType.DATE, false); + LiteralExpr minDate1Same = new DateLiteral(ScalarType.DATE, false); + LiteralExpr maxDatetime1 = new DateLiteral(ScalarType.DATETIME, true); + LiteralExpr maxDatetime1Same = new DateLiteral(ScalarType.DATETIME, true); + LiteralExpr minDatetime1 = new DateLiteral(ScalarType.DATETIME, false); + LiteralExpr minDatetime1Same = new DateLiteral(ScalarType.DATETIME, false); + LiteralExpr date8 = new DateLiteral("9999-12-31", ScalarType.DATEV2); + LiteralExpr date9 = new DateLiteral("9999-12-31 23:59:59", ScalarType.DATETIMEV2); + LiteralExpr date10 = new DateLiteral("0000-01-01", ScalarType.DATEV2); + LiteralExpr date11 = new DateLiteral("0000-01-01 00:00:00", ScalarType.DATETIMEV2); + + Assert.assertTrue(date1.equals(date1Same) && date1.compareLiteral(date1Same) == 0); + Assert.assertTrue(date1.equals(date1Same) && date1.compareLiteral(date1Same) == 0); + Assert.assertTrue(datetime1.equals(datetime1Same) && datetime1.compareLiteral(datetime1Same) == 0); + Assert.assertTrue(datetime1.equals(datetime1) && datetime1.compareLiteral(datetime1) == 0); + + // value compare + Assert.assertTrue(!date1Large.equals(date1Same) && 1 == date1Large.compareLiteral(date1Same)); + Assert.assertTrue(!datetime1Large.equals(datetime1Same) && 1 == datetime1Large.compareLiteral(datetime1Same)); + Assert.assertTrue(!datetime1Same.equals(datetime1Large) && -1 == datetime1Same.compareLiteral(datetime1Large)); + + // infinity + Assert.assertTrue(maxDate1.equals(maxDate1) && maxDate1.compareLiteral(maxDate1) == 0); + Assert.assertTrue(maxDate1.equals(maxDate1Same) && maxDate1.compareLiteral(maxDate1Same) == 0); + Assert.assertTrue(minDate1.equals(minDate1) && minDate1.compareLiteral(minDate1) == 0); + Assert.assertTrue(minDate1.equals(minDate1Same) && minDate1.compareLiteral(minDate1Same) == 0); + Assert.assertTrue(maxDatetime1.equals(maxDatetime1) && maxDatetime1.compareLiteral(maxDatetime1) == 0); + Assert.assertTrue(maxDatetime1.equals(maxDatetime1Same) && maxDatetime1.compareLiteral(maxDatetime1Same) == 0); + Assert.assertTrue(minDatetime1.equals(minDatetime1) && minDatetime1.compareLiteral(minDatetime1) == 0); + Assert.assertTrue(minDatetime1.equals(minDatetime1Same) && minDatetime1.compareLiteral(minDatetime1Same) == 0); + + Assert.assertTrue(maxDate1.equals(date8) && maxDate1.compareLiteral(date8) == 0); + Assert.assertTrue(minDate1.equals(date10) && minDate1.compareLiteral(date10) == 0); + Assert.assertTrue(maxDatetime1.equals(date9) && maxDatetime1.compareLiteral(date9) == 0); + Assert.assertTrue(minDatetime1.equals(date11) && minDatetime1.compareLiteral(date11) == 0); + + Assert.assertTrue(!maxDate1.equals(date1) && 1 == maxDate1.compareLiteral(date1)); + Assert.assertTrue(!minDate1.equals(date1) && -1 == minDate1.compareLiteral(date1)); + Assert.assertTrue(!maxDatetime1.equals(datetime1) && 1 == maxDatetime1.compareLiteral(datetime1)); + Assert.assertTrue(!minDatetime1.equals(datetime1) && -1 == minDatetime1.compareLiteral(datetime1)); + } + @Test public void decimalTest() throws AnalysisException { LiteralExpr decimal1 = new DecimalLiteral("1.23456"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/QueryStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/QueryStmtTest.java index de0d525daa..e33efbfba8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/QueryStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/QueryStmtTest.java @@ -21,7 +21,6 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.UserException; -import org.apache.doris.common.VecNotImplException; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.rewrite.FoldConstantsRule; @@ -62,6 +61,7 @@ public class QueryStmtTest { + " `citycode` smallint(6) NULL COMMENT \"\",\n" + " `username` varchar(32) NULL DEFAULT \"\" COMMENT \"\",\n" + " `workDateTime` datetime NOT NULL COMMENT \"\",\n" + + " `workDateTimeV2` datetime NOT NULL COMMENT \"\",\n" + " `pv` bigint(20) NULL DEFAULT \"0\" COMMENT \"\"\n" + ") ENGINE=OLAP\n" + "UNIQUE KEY(`siteid`, `citycode`, `username`)\n" @@ -220,16 +220,33 @@ public class QueryStmtTest { Assert.assertEquals(1, constMap.size()); // expr in subquery associate with column in grandparent level - sql = "WITH aa AS\n" + " (SELECT DATE_FORMAT(workDateTime, '%Y-%m') mon,\n" + " siteid\n" + sql = "WITH aa AS\n" + + " (SELECT DATE_FORMAT(workDateTime, '%Y-%m') mon,\n" + + " DATE_FORMAT(workDateTimeV2, '%Y-%m') mon1,\n" + + " siteid\n" + " FROM db1.table1\n" + " WHERE workDateTime >= concat(year(now())-1, '-01-01 00:00:00')\n" - + " AND workDateTime < now()\n" + " GROUP BY siteid,\n" - + " DATE_FORMAT(workDateTime, '%Y-%m')),\n" + " bb AS\n" - + " (SELECT mon,\n" + " count(DISTINCT siteid) total\n" - + " FROM aa\n" + " GROUP BY mon),\n" + " cc AS\n" - + " (SELECT mon,\n" + " count(DISTINCT siteid) num\n" - + " FROM aa\n" + " GROUP BY mon)\n" + "SELECT bb.mon,\n" - + " round(cc.num / bb.total, 4) rate\n" + "FROM bb\n" + "LEFT JOIN cc ON cc.mon = bb.mon\n" + + " AND workDateTimeV2 >= concat(year(now())-1, '-01-01 00:00:00')\n" + + " AND workDateTimeV2 >= concat(year(now())-1, '-01-01 00:00:00.000000')\n" + + " AND workDateTime < now()\n" + + " AND workDateTimeV2 < now()\n" + + " GROUP BY siteid,\n" + + " DATE_FORMAT(workDateTime, '%Y-%m'),\n" + + " DATE_FORMAT(workDateTimeV2, '%Y-%m')),\n" + + " bb AS\n" + + " (SELECT mon,\n" + + " count(DISTINCT siteid) total\n" + + " FROM aa\n" + + " GROUP BY mon),\n" + + " cc AS\n" + + " (SELECT mon,\n" + + " count(DISTINCT siteid) num\n" + + " FROM aa\n" + + " GROUP BY mon)\n" + + "SELECT bb.mon,\n" + + " round(cc.num / bb.total, 4) rate\n" + + "FROM bb\n" + + "LEFT JOIN cc ON cc.mon = bb.mon\n" + "ORDER BY mon;"; // When disable vec engine, this sql can be analyzed successfully. @@ -241,17 +258,10 @@ public class QueryStmtTest { stmt = (QueryStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, ctx); exprsMap.clear(); stmt.collectExprs(exprsMap); - Assert.assertEquals(18, exprsMap.size()); + Assert.assertEquals(24, exprsMap.size()); constMap.clear(); constMap = getConstantExprMap(exprsMap, analyzer); - Assert.assertEquals(4, constMap.size()); - } else { - try { - UtFrameUtils.parseAndAnalyzeStmt(sql, ctx); - Assert.fail(); - } catch (VecNotImplException e) { - Assert.assertTrue(e.getMessage().contains("could not be changed to nullable")); - } + Assert.assertEquals(10, constMap.size()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/RangeCompareTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/RangeCompareTest.java index 0eb1606511..721db3e458 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/RangeCompareTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/RangeCompareTest.java @@ -232,4 +232,20 @@ public class RangeCompareTest { } } + @Test + public void testMergeRangeWithDifferentType2() throws AnalysisException { + LiteralExpr lowerBoundOfRange1 = new DecimalLiteral("3.0"); + Range range1 = Range.lessThan(lowerBoundOfRange1); + LiteralExpr upperBoundOfRange2 = new DateLiteral("2021-01-01", Type.DATEV2); + Range range2 = Range.atLeast(upperBoundOfRange2); + RangeSet rangeSet = TreeRangeSet.create(); + rangeSet.add(range1); + try { + rangeSet.add(range2); + Assert.fail(); + } catch (ClassCastException e) { + System.out.println(e); + } + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTypeTest.java index 8c8644fc09..9130a540f6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTypeTest.java @@ -116,6 +116,76 @@ public class ColumnTypeTest { Assert.assertNotEquals(type.getType(), type4.getType()); } + @Test + public void testDatetimeV2() throws AnalysisException { + TypeDef type = TypeDef.createDatetimeV2(3); + type.analyze(null); + Assert.assertEquals("datetime(3)", type.toString()); + Assert.assertEquals(PrimitiveType.DATETIMEV2, type.getType().getPrimitiveType()); + Assert.assertEquals(ScalarType.DATETIME_PRECISION, ((ScalarType) type.getType()).getScalarPrecision()); + Assert.assertEquals(3, ((ScalarType) type.getType()).getScalarScale()); + + // equal type + TypeDef type2 = TypeDef.createDatetimeV2(3); + Assert.assertEquals(type.getType(), type2.getType()); + + // different type + TypeDef type3 = TypeDef.createDatetimeV2(6); + Assert.assertNotEquals(type.getType(), type3.getType()); + type3 = TypeDef.createDatetimeV2(0); + Assert.assertNotEquals(type.getType(), type3.getType()); + + // different type + TypeDef type4 = TypeDef.create(PrimitiveType.BIGINT); + Assert.assertNotEquals(type.getType(), type4.getType()); + + TypeDef type5 = TypeDef.createDatetimeV2(0); + TypeDef type6 = TypeDef.create(PrimitiveType.DATETIME); + Assert.assertEquals(type5.getType(), type6.getType()); + Assert.assertNotEquals(type.getType(), type6.getType()); + } + + @Test + public void testDateV2() throws AnalysisException { + TypeDef type = TypeDef.create(PrimitiveType.DATE); + TypeDef type2 = TypeDef.create(PrimitiveType.DATEV2); + type.analyze(null); + Assert.assertEquals(type.getType(), type2.getType()); + + // different type + TypeDef type3 = TypeDef.createDatetimeV2(6); + Assert.assertNotEquals(type2.getType(), type3.getType()); + } + + @Test + public void testTimeV2() throws AnalysisException { + TypeDef type = TypeDef.createTimeV2(3); + type.analyze(null); + Assert.assertEquals("time(3)", type.toString()); + Assert.assertEquals(PrimitiveType.TIMEV2, type.getType().getPrimitiveType()); + Assert.assertEquals(ScalarType.DATETIME_PRECISION, ((ScalarType) type.getType()).getScalarPrecision()); + Assert.assertEquals(3, ((ScalarType) type.getType()).getScalarScale()); + + // equal type + TypeDef type2 = TypeDef.createTimeV2(3); + Assert.assertEquals(type.getType(), type2.getType()); + + // different type + TypeDef type3 = TypeDef.createTimeV2(6); + Assert.assertNotEquals(type.getType(), type3.getType()); + type3 = TypeDef.createTimeV2(0); + Assert.assertNotEquals(type.getType(), type3.getType()); + + // different type + TypeDef type4 = TypeDef.create(PrimitiveType.BIGINT); + Assert.assertNotEquals(type.getType(), type4.getType()); + + TypeDef type5 = TypeDef.createTimeV2(0); + TypeDef type6 = TypeDef.create(PrimitiveType.TIME); + Assert.assertEquals(type5.getType(), type6.getType()); + Assert.assertNotEquals(type.getType(), type6.getType()); + } + @Test(expected = AnalysisException.class) public void testDecimalPreFail() throws AnalysisException { TypeDef type = TypeDef.createDecimal(28, 3); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RangePartitionInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RangePartitionInfoTest.java index d45a82e2f3..2db7338b8e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RangePartitionInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RangePartitionInfoTest.java @@ -375,4 +375,71 @@ public class RangePartitionInfoTest { partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, partitionId++, false); } } + + @Test(expected = AnalysisException.class) + public void testFixedRange7() throws DdlException, AnalysisException { + //add columns + Column k1 = new Column("k1", new ScalarType(PrimitiveType.DATEV2), true, null, "", ""); + Column k2 = new Column("k2", new ScalarType(PrimitiveType.INT), true, null, "", ""); + Column k3 = new Column("k3", new ScalarType(PrimitiveType.INT), true, null, "", ""); + partitionColumns.add(k1); + partitionColumns.add(k2); + partitionColumns.add(k3); + + //add RangePartitionDescs + PartitionKeyDesc p1 = PartitionKeyDesc.createLessThan(Lists.newArrayList(new PartitionValue("2019-02-01"), + new PartitionValue("100"), new PartitionValue("200"))); + PartitionKeyDesc p2 = PartitionKeyDesc.createFixed(Lists.newArrayList(new PartitionValue("2020-02-01"), + new PartitionValue("100"), new PartitionValue("200")), + Lists.newArrayList(new PartitionValue("2020-03-01"))); + PartitionKeyDesc p3 = PartitionKeyDesc.createLessThan(Lists.newArrayList(new PartitionValue("2021-02-01"))); + + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", p1, null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p2", p2, null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p3", p3, null)); + partitionInfo = new RangePartitionInfo(partitionColumns); + PartitionKeyValueType partitionKeyValueType = PartitionKeyValueType.INVALID; + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + // check partitionType + if (partitionKeyValueType == PartitionKeyValueType.INVALID) { + partitionKeyValueType = singlePartitionDesc.getPartitionKeyDesc().getPartitionType(); + } else if (partitionKeyValueType != singlePartitionDesc.getPartitionKeyDesc().getPartitionType()) { + throw new AnalysisException("You can only use one of these methods to create partitions"); + } + singlePartitionDesc.analyze(partitionColumns.size(), null); + partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, 20000L, false); + } + } + + @Test (expected = DdlException.class) + public void testFixedRange8() throws DdlException, AnalysisException { + //add columns + int columns = 2; + Column k1 = new Column("k1", new ScalarType(PrimitiveType.DATEV2), true, null, "", ""); + partitionColumns.add(k1); + + //add RangePartitionDescs + PartitionKeyDesc p1 = PartitionKeyDesc.createFixed( + Lists.newArrayList(new PartitionValue("2021-06-01")), + Lists.newArrayList(new PartitionValue("2021-06-02"))); + + PartitionKeyDesc p2 = PartitionKeyDesc.createFixed( + Lists.newArrayList(new PartitionValue("2021-07-01")), + Lists.newArrayList(new PartitionValue("2021-08-01"))); + + PartitionKeyDesc p3 = PartitionKeyDesc.createFixed( + Lists.newArrayList(new PartitionValue("2021-06-01")), + Lists.newArrayList(new PartitionValue("2021-07-01"))); + + singlePartitionDescs.add(new SinglePartitionDesc(false, "p1", p1, null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p2", p2, null)); + singlePartitionDescs.add(new SinglePartitionDesc(false, "p3", p3, null)); + partitionInfo = new RangePartitionInfo(partitionColumns); + + long partitionId = 20000L; + for (SinglePartitionDesc singlePartitionDesc : singlePartitionDescs) { + singlePartitionDesc.analyze(columns, null); + partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, partitionId++, false); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RecoverTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RecoverTest.java index 15d6d3854e..7313e4d07b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RecoverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RecoverTest.java @@ -41,7 +41,7 @@ import java.util.UUID; public class RecoverTest { - private static String runningDir = "fe/mocked/RecoverTest/" + UUID.randomUUID().toString() + "/"; + private static String runningDir = "fe/mocked/RecoverTest/" + UUID.randomUUID() + "/"; private static ConnectContext connectContext; @@ -60,7 +60,8 @@ public class RecoverTest { } private static void createDb(String db) throws Exception { - CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt("create database " + db, connectContext); + CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt( + "create database " + db, connectContext); Catalog.getCurrentCatalog().createDb(createDbStmt); } @@ -75,7 +76,8 @@ public class RecoverTest { } private static void dropTable(String db, String tbl) throws Exception { - DropTableStmt dropTableStmt = (DropTableStmt) UtFrameUtils.parseAndAnalyzeStmt("drop table " + db + "." + tbl, connectContext); + DropTableStmt dropTableStmt = (DropTableStmt) UtFrameUtils.parseAndAnalyzeStmt( + "drop table " + db + "." + tbl, connectContext); Catalog.getCurrentCatalog().dropTable(dropTableStmt); } @@ -86,12 +88,14 @@ public class RecoverTest { } private static void recoverDb(String db) throws Exception { - RecoverDbStmt recoverDbStmt = (RecoverDbStmt) UtFrameUtils.parseAndAnalyzeStmt("recover database " + db, connectContext); + RecoverDbStmt recoverDbStmt = (RecoverDbStmt) UtFrameUtils.parseAndAnalyzeStmt( + "recover database " + db, connectContext); Catalog.getCurrentCatalog().recoverDatabase(recoverDbStmt); } private static void recoverTable(String db, String tbl) throws Exception { - RecoverTableStmt recoverTableStmt = (RecoverTableStmt) UtFrameUtils.parseAndAnalyzeStmt("recover table " + db + "." + tbl, connectContext); + RecoverTableStmt recoverTableStmt = (RecoverTableStmt) UtFrameUtils.parseAndAnalyzeStmt( + "recover table " + db + "." + tbl, connectContext); Catalog.getCurrentCatalog().recoverTable(recoverTableStmt); } @@ -136,15 +140,13 @@ public class RecoverTest { + " `use_time` double SUM NOT NULL COMMENT \"\",\n" + " `start_times` bigint(20) SUM NOT NULL COMMENT \"\"\n" + ") ENGINE=OLAP\n" - + "AGGREGATE KEY(`event_date`, `app_name`, `package_name`, `age`," - + " `gender`, `level`, `city`, `model`, `brand`, `hours`)\n" - + "COMMENT \"OLAP\"\n" + + "AGGREGATE KEY(`event_date`, `app_name`, `package_name`, `age`, `gender`, `level`, `city`, \n" + + " `model`, `brand`, `hours`) COMMENT \"OLAP\"\n" + "PARTITION BY RANGE(`event_date`)\n" + "(PARTITION p1 VALUES [('2020-02-27'), ('2020-03-02')),\n" + "PARTITION p2 VALUES [('2020-03-02'), ('2020-03-07')))\n" - + "DISTRIBUTED BY HASH(`event_date`, `app_name`, `package_name`," - + " `age`, `gender`, `level`, `city`, `model`, `brand`, `hours`) BUCKETS 1\n" - + "PROPERTIES (\n" + + "DISTRIBUTED BY HASH(`event_date`, `app_name`, `package_name`, `age`, `gender`, `level`, `city`, \n" + + " `model`, `brand`, `hours`) BUCKETS 1 PROPERTIES (\n" + " \"replication_num\" = \"1\"\n" + ");"); @@ -186,15 +188,13 @@ public class RecoverTest { + " `use_time` double SUM NOT NULL COMMENT \"\",\n" + " `start_times` bigint(20) SUM NOT NULL COMMENT \"\"\n" + ") ENGINE=OLAP\n" - + "AGGREGATE KEY(`event_date`, `app_name`, `package_name`," - + " `age`, `gender`, `level`, `city`, `model`, `brand`, `hours`)\n" - + "COMMENT \"OLAP\"\n" + + "AGGREGATE KEY(`event_date`, `app_name`, `package_name`, `age`, `gender`, `level`, \n" + + " `city`, `model`, `brand`, `hours`) COMMENT \"OLAP\"\n" + "PARTITION BY RANGE(`event_date`)\n" + "(PARTITION p1 VALUES [('2020-02-27'), ('2020-03-02')),\n" + "PARTITION p2 VALUES [('2020-03-02'), ('2020-03-07')))\n" - + "DISTRIBUTED BY HASH(`event_date`, `app_name`, `package_name`," - + " `age`, `gender`, `level`, `city`, `model`, `brand`, `hours`) BUCKETS 1\n" - + "PROPERTIES (\n" + + "DISTRIBUTED BY HASH(`event_date`, `app_name`, `package_name`, `age`, `gender`, `level`, `city`, \n" + + " `model`, `brand`, `hours`) BUCKETS 1 PROPERTIES (\n" + " \"replication_num\" = \"1\"\n" + ");"); Assert.assertTrue(checkDbExist("test")); @@ -215,4 +215,96 @@ public class RecoverTest { Assert.assertTrue(checkPartitionExist("test", "table1", "p1")); } + @Test + public void testRecover2() throws Exception { + createDb("test2"); + createTable("CREATE TABLE test2.`table2` (\n" + + " `event_date` datetime(3) NOT NULL COMMENT \"\",\n" + + " `app_name` varchar(64) NOT NULL COMMENT \"\",\n" + + " `package_name` varchar(64) NOT NULL COMMENT \"\",\n" + + " `age` varchar(32) NOT NULL COMMENT \"\",\n" + + " `gender` varchar(32) NOT NULL COMMENT \"\",\n" + + " `level` varchar(64) NOT NULL COMMENT \"\",\n" + + " `city` varchar(64) NOT NULL COMMENT \"\",\n" + + " `model` varchar(64) NOT NULL COMMENT \"\",\n" + + " `brand` varchar(64) NOT NULL COMMENT \"\",\n" + + " `hours` varchar(16) NOT NULL COMMENT \"\",\n" + + " `use_num` int(11) SUM NOT NULL COMMENT \"\",\n" + + " `use_time` double SUM NOT NULL COMMENT \"\",\n" + + " `start_times` bigint(20) SUM NOT NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "AGGREGATE KEY(`event_date`, `app_name`, `package_name`, `age`, `gender`, `level`, \n" + + "`city`, `model`, `brand`, `hours`) COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE(`event_date`)\n" + + "(PARTITION p1 VALUES [('2020-02-27 00:00:00'), ('2020-03-02 00:00:00')),\n" + + "PARTITION p2 VALUES [('2020-03-02 00:00:00'), ('2020-03-07 00:00:00')))\n" + + "DISTRIBUTED BY HASH(`event_date`, `app_name`, `package_name`, `age`, `gender`, `level`, \n" + + " `city`, `model`, `brand`, `hours`) BUCKETS 1 PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ");"); + + Assert.assertTrue(checkDbExist("test2")); + Assert.assertTrue(checkTableExist("test2", "table2")); + + dropDb("test2"); + Assert.assertFalse(checkDbExist("test2")); + Assert.assertFalse(checkTableExist("test2", "table2")); + + recoverDb("test2"); + Assert.assertTrue(checkDbExist("test2")); + Assert.assertTrue(checkTableExist("test2", "table2")); + + dropTable("test2", "table2"); + Assert.assertTrue(checkDbExist("test2")); + Assert.assertFalse(checkTableExist("test2", "table2")); + + recoverTable("test2", "table2"); + Assert.assertTrue(checkDbExist("test2")); + Assert.assertTrue(checkTableExist("test2", "table2")); + + dropTable("test2", "table2"); + Assert.assertTrue(checkDbExist("test2")); + Assert.assertFalse(checkTableExist("test2", "table2")); + + createTable("CREATE TABLE test2.`table2` (\n" + + " `event_date` datetime(3) NOT NULL COMMENT \"\",\n" + + " `app_name` varchar(64) NOT NULL COMMENT \"\",\n" + + " `package_name` varchar(64) NOT NULL COMMENT \"\",\n" + + " `age` varchar(32) NOT NULL COMMENT \"\",\n" + + " `gender` varchar(32) NOT NULL COMMENT \"\",\n" + + " `level` varchar(64) NOT NULL COMMENT \"\",\n" + + " `city` varchar(64) NOT NULL COMMENT \"\",\n" + + " `model` varchar(64) NOT NULL COMMENT \"\",\n" + + " `brand` varchar(64) NOT NULL COMMENT \"\",\n" + + " `hours` varchar(16) NOT NULL COMMENT \"\",\n" + + " `use_num` int(11) SUM NOT NULL COMMENT \"\",\n" + + " `use_time` double SUM NOT NULL COMMENT \"\",\n" + + " `start_times` bigint(20) SUM NOT NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "AGGREGATE KEY(`event_date`, `app_name`, `package_name`, `age`, `gender`, `level`, `city`, \n" + + " `model`, `brand`, `hours`) COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE(`event_date`)\n" + + "(PARTITION p1 VALUES [('2020-02-27 00:00:00'), ('2020-03-02 00:00:00')),\n" + + "PARTITION p2 VALUES [('2020-03-02 00:00:00'), ('2020-03-07 00:00:00')))\n" + + "DISTRIBUTED BY HASH(`event_date`, `app_name`, `package_name`, `age`, `gender`, `level`, `city`, \n" + + " `model`, `brand`, `hours`) BUCKETS 1 PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ");"); + Assert.assertTrue(checkDbExist("test2")); + Assert.assertTrue(checkTableExist("test2", "table2")); + + try { + recoverTable("test2", "table2"); + Assert.fail("should not recover succeed"); + } catch (DdlException e) { + e.printStackTrace(); + } + + Assert.assertTrue(checkPartitionExist("test2", "table2", "p1")); + dropPartition("test2", "table2", "p1"); + Assert.assertFalse(checkPartitionExist("test2", "table2", "p1")); + + recoverPartition("test2", "table2", "p1"); + Assert.assertTrue(checkPartitionExist("test2", "table2", "p1")); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/ConstantExpressTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/ConstantExpressTest.java index e651166e91..d73a106ae6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/ConstantExpressTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/ConstantExpressTest.java @@ -133,6 +133,14 @@ public class ConstantExpressTest { testConstantExpressResult( "select cast ('2020-01-20' as date);", "'2020-01-20'"); + + testConstantExpressResult( + "select cast ('2020-01-20 00:00:00' as datetime);", + "'2020-01-20 00:00:00'"); + + testConstantExpressResult( + "select cast ('2020-01-20 00:00:00' as datetime(0));", + "'2020-01-20 00:00:00'"); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java index 49f64d6e02..cf418a6830 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java @@ -63,20 +63,20 @@ public class MaterializedViewFunctionTest { @Before public void beforeMethod() throws Exception { - String createTableSQL = "create table " + HR_DB_NAME + "." + EMPS_TABLE_NAME + " (time date, empid int, name varchar, " - + "deptno int, salary int, commission int) partition by range (time) " - + "(partition p1 values less than MAXVALUE) " - + "distributed by hash(time) buckets 3 properties('replication_num' = '1');"; + String createTableSQL = "create table " + HR_DB_NAME + "." + EMPS_TABLE_NAME + " (time_col date, empid int, " + + "name varchar, deptno int, salary int, commission int) partition by range (time_col) " + + "(partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3" + + " properties('replication_num' = '1');"; dorisAssert.withTable(createTableSQL); createTableSQL = "create table " + HR_DB_NAME + "." + DEPTS_TABLE_NAME - + " (time date, deptno int, name varchar, cost int) partition by range (time) " + + " (time_col date, deptno int, name varchar, cost int) partition by range (time_col) " + "(partition p1 values less than MAXVALUE) " - + "distributed by hash(time) buckets 3 properties('replication_num' = '1');"; + + "distributed by hash(time_col) buckets 3 properties('replication_num' = '1');"; dorisAssert.withTable(createTableSQL); createTableSQL = "create table " + HR_DB_NAME + "." + USER_TAG_TABLE_NAME - + " (time date, user_id int, user_name varchar(20), tag_id int) partition by range (time) " + + " (time_col date, user_id int, user_name varchar(20), tag_id int) partition by range (time_col) " + " (partition p1 values less than MAXVALUE) " - + "distributed by hash(time) buckets 3 properties('replication_num' = '1');"; + + "distributed by hash(time_col) buckets 3 properties('replication_num' = '1');"; dorisAssert.withTable(createTableSQL); } @@ -434,8 +434,8 @@ public class MaterializedViewFunctionTest { @Test public void testQueryOnStar() throws Exception { - String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select time, deptno, empid, name, " - + "salary, commission from " + EMPS_TABLE_NAME + " order by time, deptno, empid;"; + String createEmpsMVsql = "create materialized view " + EMPS_MV_NAME + " as select time_col, deptno," + + "empid, name, salary, commission from " + EMPS_TABLE_NAME + " order by time_col, deptno, empid;"; String query = "select * from " + EMPS_TABLE_NAME + " where deptno = 1"; dorisAssert.withMaterializedView(createEmpsMVsql).query(query).explainContains(QUERY_USE_EMPS_MV); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java index 8d9a115094..4163112aa7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java @@ -69,7 +69,7 @@ public class QueryPlanTest extends TestWithFeService { createTable("create table test.test1\n" + "(\n" + " query_id varchar(48) comment \"Unique query id\",\n" - + " time datetime not null comment \"Query start time\",\n" + + " time_col datetime not null comment \"Query start time\",\n" + " client_ip varchar(32) comment \"Client IP\",\n" + " user varchar(64) comment \"User name\",\n" + " db varchar(96) comment \"Database of this query\",\n" @@ -83,7 +83,7 @@ public class QueryPlanTest extends TestWithFeService { + " frontend_ip varchar(32) comment \"Frontend ip of executing this statement\",\n" + " stmt varchar(2048) comment \"The original statement, trimed if longer than 2048 bytes\"\n" + ")\n" - + "partition by range(time) ()\n" + + "partition by range(time_col) ()\n" + "distributed by hash(query_id) buckets 1\n" + "properties(\n" + " \"dynamic_partition.time_unit\" = \"DAY\",\n" @@ -239,15 +239,14 @@ public class QueryPlanTest extends TestWithFeService { + " `use_time` double SUM NOT NULL COMMENT \"\",\n" + " `start_times` bigint(20) SUM NOT NULL COMMENT \"\"\n" + ") ENGINE=OLAP\n" - + "AGGREGATE KEY(`event_date`, `app_name`, `package_name`, `age`, `gender`, `level`, `city`, `model`, `brand`, `hours`)\n" - + - "COMMENT \"OLAP\"\n" + + "AGGREGATE KEY(`event_date`, `app_name`, `package_name`, `age`, `gender`, `level`, " + + "`city`, `model`, `brand`, `hours`) COMMENT \"OLAP\"\n" + "PARTITION BY RANGE(`event_date`)\n" + "(PARTITION p_20200301 VALUES [('2020-02-27'), ('2020-03-02')),\n" + "PARTITION p_20200306 VALUES [('2020-03-02'), ('2020-03-07')))\n" - + "DISTRIBUTED BY HASH(`event_date`, `app_name`, `package_name`, `age`, `gender`, `level`, `city`, `model`, `brand`, `hours`) BUCKETS 1\n" - + - "PROPERTIES (\n" + + "DISTRIBUTED BY HASH(`event_date`, `app_name`, `package_name`, `age`, `gender`, `level`, " + + "`city`, `model`, `brand`, `hours`) BUCKETS 1\n" + + "PROPERTIES (\n" + " \"replication_num\" = \"1\"\n" + ");"); @@ -391,7 +390,8 @@ public class QueryPlanTest extends TestWithFeService { createView("create view test.tbl_null_column_view AS SELECT *,NULL as add_column FROM test.test1;"); - createView("create view test.function_view AS SELECT query_id, client_ip, concat(user, db) as concat FROM test.test1;"); + createView("create view test.function_view AS SELECT query_id, client_ip, concat(user, db) as" + + " concat FROM test.test1;"); createTable("create table test.tbl_using_a\n" + "(\n" @@ -418,7 +418,8 @@ public class QueryPlanTest extends TestWithFeService { @Test public void testFunctionViewGroupingSet() throws Exception { - String queryStr = "select query_id, client_ip, concat from test.function_view group by rollup(query_id, client_ip, concat);"; + String queryStr = "select query_id, client_ip, concat from test.function_view group by rollup(" + + "query_id, client_ip, concat);"; assertSQLPlanOrErrorMsgContains(queryStr, "repeat: repeat 3 lines [[], [0], [0, 1], [0, 1, 2, 3]]"); } @@ -698,8 +699,7 @@ public class QueryPlanTest extends TestWithFeService { + "(app_name,package_name,age,gender,level,city,model,brand,hours,use_num,use_time,start_times)\n" + "SET\n" + "(event_date = default_value('2020-03-06'))) \n" - + "PROPERTIES ( 'max_filter_ratio'='0.0001' );\n" - + ""; + + "PROPERTIES ( 'max_filter_ratio'='0.0001' );\n"; LoadStmt loadStmt = (LoadStmt) parseAndAnalyzeStmt(loadStr); Catalog.getCurrentCatalog().getLoadManager().createLoadJobV1FromStmt(loadStmt, EtlJobType.HADOOP, System.currentTimeMillis()); @@ -868,81 +868,102 @@ public class QueryPlanTest extends TestWithFeService { String caseWhenSql = "select " + "case when date_format(now(),'%H%i') < 123 then 1 else 0 end as col " + "from test.test1 " - + "where time = case when date_format(now(),'%H%i') < 123 then date_format(date_sub(now(),2),'%Y%m%d') else date_format(date_sub(now(),1),'%Y%m%d') end"; - Assert.assertTrue(!StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + caseWhenSql), "CASE WHEN")); + + "where time_col = case when date_format(now(),'%H%i') < 123 then date_format(date_sub(" + + "now(),2),'%Y%m%d') else date_format(date_sub(now(),1),'%Y%m%d') end"; + Assert.assertTrue(!StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + caseWhenSql), + "CASE WHEN")); // test 1: case when then // 1.1 multi when in on `case when` and can be converted to constants String sql11 = "select case when false then 2 when true then 3 else 0 end as col11;"; - Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql11), "constant exprs: \n 3")); + Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql11), + "constant exprs: \n 3")); // 1.2 multi `when expr` in on `case when` ,`when expr` can not be converted to constants - String sql121 = "select case when false then 2 when substr(k7,2,1) then 3 else 0 end as col121 from test.baseall"; + String sql121 = "select case when false then 2 when substr(k7,2,1) then 3 else 0 end as col121 from" + + " test.baseall"; Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql121), "OUTPUT EXPRS:CASE WHEN substr(`k7`, 2, 1) THEN 3 ELSE 0 END")); // 1.2.2 when expr which can not be converted to constants in the first - String sql122 = "select case when substr(k7,2,1) then 2 when false then 3 else 0 end as col122 from test.baseall"; + String sql122 = "select case when substr(k7,2,1) then 2 when false then 3 else 0 end as col122" + + " from test.baseall"; Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql122), "OUTPUT EXPRS:CASE WHEN substr(`k7`, 2, 1) THEN 2 WHEN FALSE THEN 3 ELSE 0 END")); // 1.2.3 test return `then expr` in the middle String sql124 = "select case when false then 1 when true then 2 when false then 3 else 'other' end as col124"; - Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql124), "constant exprs: \n '2'")); + Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql124), + "constant exprs: \n '2'")); // 1.3 test return null String sql3 = "select case when false then 2 end as col3"; - Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql3), "constant exprs: \n NULL")); + Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql3), + "constant exprs: \n NULL")); // 1.3.1 test return else expr String sql131 = "select case when false then 2 when false then 3 else 4 end as col131"; - Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql131), "constant exprs: \n 4")); + Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql131), + "constant exprs: \n 4")); // 1.4 nest `case when` and can be converted to constants String sql14 = "select case when (case when true then true else false end) then 2 when false then 3 else 0 end as col"; - Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql14), "constant exprs: \n 2")); + Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql14), + "constant exprs: \n 2")); // 1.5 nest `case when` and can not be converted to constants - String sql15 = "select case when case when substr(k7,2,1) then true else false end then 2 when false then 3 else 0 end as col from test.baseall"; + String sql15 = "select case when case when substr(k7,2,1) then true else false end then 2 when false then 3" + + " else 0 end as col from test.baseall"; Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql15), - "OUTPUT EXPRS:CASE WHEN CASE WHEN substr(`k7`, 2, 1) THEN TRUE ELSE FALSE END THEN 2 WHEN FALSE THEN 3 ELSE 0 END")); + "OUTPUT EXPRS:CASE WHEN CASE WHEN substr(`k7`, 2, 1) THEN TRUE ELSE FALSE END THEN 2" + + " WHEN FALSE THEN 3 ELSE 0 END")); // 1.6 test when expr is null String sql16 = "select case when null then 1 else 2 end as col16;"; - Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql16), "constant exprs: \n 2")); + Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql16), + "constant exprs: \n 2")); // test 2: case xxx when then // 2.1 test equal String sql2 = "select case 1 when 1 then 'a' when 2 then 'b' else 'other' end as col2;"; - Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql2), "constant exprs: \n 'a'")); + Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql2), + "constant exprs: \n 'a'")); // 2.1.2 test not equal String sql212 = "select case 'a' when 1 then 'a' when 'a' then 'b' else 'other' end as col212;"; - Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql212), "constant exprs: \n 'b'")); + Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql212), + "constant exprs: \n 'b'")); // 2.2 test return null String sql22 = "select case 'a' when 1 then 'a' when 'b' then 'b' end as col22;"; - Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql22), "constant exprs: \n NULL")); + Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql22), + "constant exprs: \n NULL")); // 2.2.2 test return else String sql222 = "select case 1 when 2 then 'a' when 3 then 'b' else 'other' end as col222;"; - Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql222), "constant exprs: \n 'other'")); + Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql222), + "constant exprs: \n 'other'")); // 2.3 test can not convert to constant,middle when expr is not constant - String sql23 = "select case 'a' when 'b' then 'a' when substr(k7,2,1) then 2 when false then 3 else 0 end as col23 from test.baseall"; + String sql23 = "select case 'a' when 'b' then 'a' when substr(k7,2,1) then 2 when false then 3" + + " else 0 end as col23 from test.baseall"; String a = getSQLPlanOrErrorMsg("explain " + sql23); Assert.assertTrue(StringUtils.containsIgnoreCase(a, "OUTPUT EXPRS:CASE 'a' WHEN substr(`k7`, 2, 1) THEN '2' WHEN '0' THEN '3' ELSE '0' END")); // 2.3.1 first when expr is not constant - String sql231 = "select case 'a' when substr(k7,2,1) then 2 when 1 then 'a' when false then 3 else 0 end as col231 from test.baseall"; + String sql231 = "select case 'a' when substr(k7,2,1) then 2 when 1 then 'a' when false then 3 else 0 end" + + " as col231 from test.baseall"; Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql231), - "OUTPUT EXPRS:CASE 'a' WHEN substr(`k7`, 2, 1) THEN '2' WHEN '1' THEN 'a' WHEN '0' THEN '3' ELSE '0' END")); + "OUTPUT EXPRS:CASE 'a' WHEN substr(`k7`, 2, 1) THEN '2' WHEN '1' THEN 'a' WHEN '0'" + + " THEN '3' ELSE '0' END")); // 2.3.2 case expr is not constant - String sql232 = "select case k1 when substr(k7,2,1) then 2 when 1 then 'a' when false then 3 else 0 end as col232 from test.baseall"; + String sql232 = "select case k1 when substr(k7,2,1) then 2 when 1 then 'a' when false then 3 else 0 end" + + " as col232 from test.baseall"; Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql232), - "OUTPUT EXPRS:CASE `k1` WHEN substr(`k7`, 2, 1) THEN '2' WHEN '1' THEN 'a' WHEN '0' THEN '3' ELSE '0' END")); + "OUTPUT EXPRS:CASE `k1` WHEN substr(`k7`, 2, 1) THEN '2' WHEN '1' THEN 'a' " + + "WHEN '0' THEN '3' ELSE '0' END")); // 3.1 test float,float in case expr String sql31 = "select case cast(100 as float) when 1 then 'a' when 2 then 'b' else 'other' end as col31;"; @@ -951,15 +972,18 @@ public class QueryPlanTest extends TestWithFeService { // 4.1 test null in case expr return else String sql41 = "select case null when 1 then 'a' when 2 then 'b' else 'other' end as col41"; - Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql41), "constant exprs: \n 'other'")); + Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql41), + "constant exprs: \n 'other'")); // 4.1.2 test null in case expr return null String sql412 = "select case null when 1 then 'a' when 2 then 'b' end as col41"; - Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql412), "constant exprs: \n NULL")); + Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql412), + "constant exprs: \n NULL")); // 4.2.1 test null in when expr String sql421 = "select case 'a' when null then 'a' else 'other' end as col421"; - Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql421), "constant exprs: \n 'other'")); + Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql421), + "constant exprs: \n 'other'")); // 5.1 test same type in then expr and else expr String sql51 = "select case when 132 then k7 else 'all' end as col51 from test.baseall group by col51"; @@ -977,12 +1001,14 @@ public class QueryPlanTest extends TestWithFeService { "OUTPUT EXPRS: `k1`")); // 5.4 test return CastExpr with other SlotRef in selectListItem - String sql54 = "select k2, case when 2 < 1 then 'all' else k1 end as col54, k7 from test.baseall group by k2, col54, k7"; + String sql54 = "select k2, case when 2 < 1 then 'all' else k1 end as col54, k7 from test.baseall" + + " group by k2, col54, k7"; Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql54), "OUTPUT EXPRS: `k2` | `k1` | `k7`")); // 5.5 test return CastExpr> with other SlotRef in selectListItem - String sql55 = "select case when 2 < 1 then 'all' else cast(k1 as int) end as col55, k7 from test.baseall group by col55, k7"; + String sql55 = "select case when 2 < 1 then 'all' else cast(k1 as int) end as col55, k7 from" + + " test.baseall group by col55, k7"; Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain " + sql55), "OUTPUT EXPRS: CAST(`k1` AS INT) | `k7`")); } @@ -1012,7 +1038,8 @@ public class QueryPlanTest extends TestWithFeService { @Test public void testConstInParitionPrune() throws Exception { FeConstants.runningUnitTest = true; - String queryStr = "explain select * from (select 'aa' as kk1, sum(id) from test.join1 where dt = 9 group by kk1)tt where kk1 in ('aa');"; + String queryStr = "explain select * from (select 'aa' as kk1, sum(id) from test.join1 where dt = 9" + + " group by kk1)tt where kk1 in ('aa');"; String explainString = getSQLPlanOrErrorMsg(queryStr); FeConstants.runningUnitTest = false; Assert.assertTrue(explainString.contains("partitions=1/1")); @@ -1037,7 +1064,8 @@ public class QueryPlanTest extends TestWithFeService { public void testColocateJoin() throws Exception { FeConstants.runningUnitTest = true; - String queryStr = "explain select * from test.colocate1 t1, test.colocate2 t2 where t1.k1 = t2.k1 and t1.k2 = t2.k2 and t1.k3 = t2.k3"; + String queryStr = "explain select * from test.colocate1 t1, test.colocate2 t2 where t1.k1 = t2.k1 and" + + " t1.k2 = t2.k2 and t1.k3 = t2.k3"; String explainString = getSQLPlanOrErrorMsg(queryStr); Assert.assertTrue(explainString.contains(ColocatePlanTest.COLOCATE_ENABLE)); @@ -1106,15 +1134,15 @@ public class QueryPlanTest extends TestWithFeService { } // single partition - String queryStr = "explain select * from test.jointest t1, test.bucket_shuffle1 t2" - + " where t1.k1 = t2.k1 and t1.k1 = t2.k2"; + String queryStr = "explain select * from test.jointest t1, test.bucket_shuffle1 t2 where t1.k1 = t2.k1" + + " and t1.k1 = t2.k2"; String explainString = getSQLPlanOrErrorMsg(queryStr); Assert.assertTrue(explainString.contains("BUCKET_SHFFULE")); Assert.assertTrue(explainString.contains("BUCKET_SHFFULE_HASH_PARTITIONED: `t1`.`k1`, `t1`.`k1`")); // not bucket shuffle join do not support different type - queryStr = "explain select * from test.jointest t1, test.bucket_shuffle1 t2" - + " where cast (t1.k1 as tinyint) = t2.k1 and t1.k1 = t2.k2"; + queryStr = "explain select * from test.jointest t1, test.bucket_shuffle1 t2 where cast (t1.k1 as tinyint)" + + " = t2.k1 and t1.k1 = t2.k2"; explainString = getSQLPlanOrErrorMsg(queryStr); Assert.assertTrue(!explainString.contains("BUCKET_SHFFULE")); @@ -1124,37 +1152,37 @@ public class QueryPlanTest extends TestWithFeService { Assert.assertTrue(!explainString.contains("BUCKET_SHFFULE")); // multi partition, should not be bucket shuffle join - queryStr = "explain select * from test.jointest t1, test.bucket_shuffle2 t2" - + " where t1.k1 = t2.k1 and t1.k1 = t2.k2"; + queryStr = "explain select * from test.jointest t1, test.bucket_shuffle2 t2 where t1.k1 = t2.k1" + + " and t1.k1 = t2.k2"; explainString = getSQLPlanOrErrorMsg(queryStr); Assert.assertTrue(!explainString.contains("BUCKET_SHFFULE")); // left table is colocate table, should be bucket shuffle - queryStr = "explain select * from test.colocate1 t1, test.bucket_shuffle2 t2" - + " where t1.k1 = t2.k1 and t1.k1 = t2.k2"; + queryStr = "explain select * from test.colocate1 t1, test.bucket_shuffle2 t2 where t1.k1 = t2.k1" + + " and t1.k1 = t2.k2"; explainString = getSQLPlanOrErrorMsg(queryStr); Assert.assertTrue(!explainString.contains("BUCKET_SHFFULE")); // support recurse of bucket shuffle join - queryStr = "explain select * from test.jointest t1 join test.bucket_shuffle1 t2" - + " on t1.k1 = t2.k1 and t1.k1 = t2.k2 join test.colocate1 t3" - + " on t2.k1 = t3.k1 and t2.k2 = t3.k2"; + // TODO: support the UT in the future + queryStr = "explain select * from test.jointest t1 join test.bucket_shuffle1 t2 on t1.k1 = t2.k1 and" + + " t1.k1 = t2.k2 join test.colocate1 t3 on t2.k1 = t3.k1 and t2.k2 = t3.k2"; explainString = getSQLPlanOrErrorMsg(queryStr); Assert.assertTrue(explainString.contains("BUCKET_SHFFULE_HASH_PARTITIONED: `t1`.`k1`, `t1`.`k1`")); Assert.assertTrue(explainString.contains("BUCKET_SHFFULE_HASH_PARTITIONED: `t3`.`k1`, `t3`.`k2`")); // support recurse of bucket shuffle because t4 join t2 and join column name is same as t2 distribute column name - queryStr = "explain select * from test.jointest t1 join test.bucket_shuffle1 t2" - + " on t1.k1 = t2.k1 and t1.k1 = t2.k2 join test.colocate1 t3" - + " on t2.k1 = t3.k1 join test.jointest t4 on t4.k1 = t2.k1 and t4.k1 = t2.k2"; + queryStr = "explain select * from test.jointest t1 join test.bucket_shuffle1 t2 on t1.k1 = t2.k1 and" + + " t1.k1 = t2.k2 join test.colocate1 t3 on t2.k1 = t3.k1 join test.jointest t4 on t4.k1 = t2.k1 and" + + " t4.k1 = t2.k2"; explainString = getSQLPlanOrErrorMsg(queryStr); Assert.assertTrue(explainString.contains("BUCKET_SHFFULE_HASH_PARTITIONED: `t1`.`k1`, `t1`.`k1`")); Assert.assertTrue(explainString.contains("BUCKET_SHFFULE_HASH_PARTITIONED: `t4`.`k1`, `t4`.`k1`")); // some column name in join expr t3 join t4 and t1 distribute column name, so should not be bucket shuffle join - queryStr = "explain select * from test.jointest t1 join test.bucket_shuffle1 t2" - + " on t1.k1 = t2.k1 and t1.k1 = t2.k2 join test.colocate1 t3" - + " on t2.k1 = t3.k1 join test.jointest t4 on t4.k1 = t3.k1 and t4.k2 = t3.k2"; + queryStr = "explain select * from test.jointest t1 join test.bucket_shuffle1 t2 on t1.k1 = t2.k1 and t1.k1 =" + + " t2.k2 join test.colocate1 t3 on t2.k1 = t3.k1 join test.jointest t4 on t4.k1 = t3.k1 and" + + " t4.k2 = t3.k2"; explainString = getSQLPlanOrErrorMsg(queryStr); Assert.assertTrue(explainString.contains("BUCKET_SHFFULE_HASH_PARTITIONED: `t1`.`k1`, `t1`.`k1`")); Assert.assertTrue(!explainString.contains("BUCKET_SHFFULE_HASH_PARTITIONED: `t4`.`k1`, `t4`.`k1`")); @@ -1362,7 +1390,8 @@ public class QueryPlanTest extends TestWithFeService { explainString = getSQLPlanOrErrorMsg(queryStr); Assert.assertFalse(explainString.contains("runtime filter")); - queryStr = "explain select * from jointest as a where k1 = (select count(1) from jointest as b where a.k1 = b.k1);"; + queryStr = "explain select * from jointest as a where k1 = (select count(1) from jointest as b" + + " where a.k1 = b.k1);"; Deencapsulation.setField(connectContext.getSessionVariable(), "runtimeFilterMode", "GLOBAL"); Deencapsulation.setField(connectContext.getSessionVariable(), "runtimeFilterType", 15); explainString = getSQLPlanOrErrorMsg(queryStr); @@ -1411,8 +1440,10 @@ public class QueryPlanTest extends TestWithFeService { Deencapsulation.setField(connectContext.getSessionVariable(), "runtimeFilterType", 7); explainString = getSQLPlanOrErrorMsg(queryStr); - Assert.assertTrue(explainString.contains("runtime filters: RF000[in] <- `t1`.`k1`, RF001[bloom] <- `t1`.`k1`, RF002[min_max] <- `t1`.`k1`")); - Assert.assertTrue(explainString.contains("runtime filters: RF000[in] -> `t2`.`k1`, RF001[bloom] -> `t2`.`k1`, RF002[min_max] -> `t2`.`k1`")); + Assert.assertTrue(explainString.contains("runtime filters: RF000[in] <- `t1`.`k1`, RF001[bloom] <- `t1`.`k1`," + + " RF002[min_max] <- `t1`.`k1`")); + Assert.assertTrue(explainString.contains("runtime filters: RF000[in] -> `t2`.`k1`, RF001[bloom] -> `t2`.`k1`," + + " RF002[min_max] -> `t2`.`k1`")); Deencapsulation.setField(connectContext.getSessionVariable(), "runtimeFilterType", 8); explainString = getSQLPlanOrErrorMsg(queryStr); @@ -1535,19 +1566,19 @@ public class QueryPlanTest extends TestWithFeService { public void testLeadAndLagFunction() throws Exception { connectContext.setDatabase("default_cluster:test"); - String queryStr = "explain select time, lead(query_time, 1, NULL) over () as time2 from test.test1"; + String queryStr = "explain select time_col, lead(query_time, 1, NULL) over () as time2 from test.test1"; String explainString = getSQLPlanOrErrorMsg(queryStr); Assert.assertTrue(explainString.contains("lead(`query_time`, 1, NULL)")); - queryStr = "explain select time, lead(query_time, 1, 2) over () as time2 from test.test1"; + queryStr = "explain select time_col, lead(query_time, 1, 2) over () as time2 from test.test1"; explainString = getSQLPlanOrErrorMsg(queryStr); Assert.assertTrue(explainString.contains("lead(`query_time`, 1, 2)")); - queryStr = "explain select time, lead(time, 1, '2020-01-01 00:00:00') over () as time2 from test.test1"; + queryStr = "explain select time_col, lead(time_col, 1, '2020-01-01 00:00:00') over () as time2 from test.test1"; explainString = getSQLPlanOrErrorMsg(queryStr); - Assert.assertTrue(explainString.contains("lead(`time`, 1, '2020-01-01 00:00:00')")); + Assert.assertTrue(explainString.contains("lead(`time_col`, 1, '2020-01-01 00:00:00')")); - queryStr = "explain select time, lag(query_time, 1, 2) over () as time2 from test.test1"; + queryStr = "explain select time_col, lag(query_time, 1, 2) over () as time2 from test.test1"; explainString = getSQLPlanOrErrorMsg(queryStr); Assert.assertTrue(explainString.contains("lag(`query_time`, 1, 2)")); } @@ -1974,6 +2005,10 @@ public class QueryPlanTest extends TestWithFeService { rewriteDateLiteralRuleTest.testWithIntFormatDate(); rewriteDateLiteralRuleTest.testWithInvalidFormatDate(); rewriteDateLiteralRuleTest.testWithStringFormatDate(); + rewriteDateLiteralRuleTest.testWithDoubleFormatDateV2(); + rewriteDateLiteralRuleTest.testWithIntFormatDateV2(); + rewriteDateLiteralRuleTest.testWithInvalidFormatDateV2(); + rewriteDateLiteralRuleTest.testWithStringFormatDateV2(); rewriteDateLiteralRuleTest.after(); } // --end-- diff --git a/fe/fe-core/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java b/fe/fe-core/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java index e0f27e74a8..5060fa67f3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java @@ -81,20 +81,67 @@ public class FEFunctionsTest { } } + @Test + public void unixtimestampV2Test() { + try { + IntLiteral timestamp = FEFunctions.unixTimestamp(new DateLiteral("2018-01-01", Type.DATEV2)); + Assert.assertEquals(1514736000, timestamp.getValue()); + timestamp = FEFunctions.unixTimestamp(new DateLiteral("1970-01-01 08:00:00", Type.DATETIMEV2)); + Assert.assertEquals(0, timestamp.getValue()); + timestamp = FEFunctions.unixTimestamp(new DateLiteral("1970-01-01 00:00:00", Type.DATETIMEV2)); + Assert.assertEquals(0, timestamp.getValue()); + timestamp = FEFunctions.unixTimestamp(new DateLiteral("1969-01-01 00:00:00", Type.DATETIMEV2)); + Assert.assertEquals(0, timestamp.getValue()); + timestamp = FEFunctions.unixTimestamp(new DateLiteral("2038-01-19 03:14:07", Type.DATETIMEV2)); + // CST time zone + Assert.assertEquals(Integer.MAX_VALUE - 8 * 3600, timestamp.getValue()); + + } catch (AnalysisException e) { + e.printStackTrace(); + Assert.fail(); + } + } + @Test public void dateDiffTest() throws AnalysisException { - IntLiteral actualResult = FEFunctions.dateDiff(new DateLiteral("2010-11-30 23:59:59", Type.DATETIME), new DateLiteral("2010-12-31", Type.DATE)); + IntLiteral actualResult = FEFunctions.dateDiff(new DateLiteral("2010-11-30 23:59:59", Type.DATETIME), + new DateLiteral("2010-12-31", Type.DATE)); IntLiteral expectedResult = new IntLiteral(-31); Assert.assertEquals(expectedResult, actualResult); - actualResult = FEFunctions.dateDiff(new DateLiteral("2010-11-30 23:59:50", Type.DATETIME), new DateLiteral("2010-12-30 23:59:59", Type.DATETIME)); + actualResult = FEFunctions.dateDiff(new DateLiteral("2010-11-30 23:59:50", Type.DATETIME), + new DateLiteral("2010-12-30 23:59:59", Type.DATETIME)); + expectedResult = new IntLiteral(-30); + Assert.assertEquals(expectedResult, actualResult); + } + + @Test + public void dateV2DiffTest() throws AnalysisException { + IntLiteral actualResult = FEFunctions.dateDiff(new DateLiteral("2010-11-30 23:59:59", Type.DATETIMEV2), + new DateLiteral("2010-12-31", Type.DATEV2)); + IntLiteral expectedResult = new IntLiteral(-31); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.dateDiff(new DateLiteral("2010-11-30 23:59:59", Type.DATETIMEV2), + new DateLiteral("2010-12-31", Type.DATE)); + expectedResult = new IntLiteral(-31); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.dateDiff(new DateLiteral("2010-11-30 23:59:50", Type.DATETIMEV2), + new DateLiteral("2010-12-30 23:59:59", Type.DATETIMEV2)); + expectedResult = new IntLiteral(-30); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.dateDiff(new DateLiteral("2010-11-30 23:59:50", Type.DATETIMEV2), + new DateLiteral("2010-12-30 23:59:59", Type.DATETIME)); expectedResult = new IntLiteral(-30); Assert.assertEquals(expectedResult, actualResult); } @Test public void dateAddTest() throws AnalysisException { - DateLiteral actualResult = FEFunctions.dateAdd(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(1)); + DateLiteral actualResult = FEFunctions.dateAdd(new DateLiteral("2018-08-08", Type.DATE), + new IntLiteral(1)); DateLiteral expectedResult = new DateLiteral("2018-08-09 00:00:00", Type.DATETIME); Assert.assertEquals(expectedResult, actualResult); @@ -103,6 +150,33 @@ public class FEFunctionsTest { Assert.assertEquals(expectedResult, actualResult); } + @Test + public void dateV2AddTest() throws AnalysisException { + DateLiteral actualResult = FEFunctions.dateAdd(new DateLiteral("2018-08-08", Type.DATEV2), new IntLiteral(1)); + DateLiteral expectedResult = new DateLiteral("2018-08-09 00:00:00", Type.DATETIMEV2); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.dateAdd(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(1)); + expectedResult = new DateLiteral("2018-08-09 00:00:00", Type.DATETIMEV2); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.dateAdd(new DateLiteral("2018-08-08", Type.DATEV2), new IntLiteral(1)); + expectedResult = new DateLiteral("2018-08-09 00:00:00", Type.DATETIME); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.dateAdd(new DateLiteral("2018-08-08", Type.DATEV2), new IntLiteral(-1)); + expectedResult = new DateLiteral("2018-08-07 00:00:00", Type.DATETIMEV2); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.dateAdd(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(-1)); + expectedResult = new DateLiteral("2018-08-07 00:00:00", Type.DATETIMEV2); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.dateAdd(new DateLiteral("2018-08-08", Type.DATEV2), new IntLiteral(-1)); + expectedResult = new DateLiteral("2018-08-07 00:00:00", Type.DATETIME); + Assert.assertEquals(expectedResult, actualResult); + } + @Test public void addDateTest() throws AnalysisException { DateLiteral actualResult = FEFunctions.addDate(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(1)); @@ -115,6 +189,34 @@ public class FEFunctionsTest { } + @Test + public void addDateV2Test() throws AnalysisException { + DateLiteral actualResult = FEFunctions.addDate(new DateLiteral("2018-08-08", Type.DATEV2), new IntLiteral(1)); + DateLiteral expectedResult = new DateLiteral("2018-08-09 00:00:00", Type.DATETIMEV2); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.addDate(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(1)); + expectedResult = new DateLiteral("2018-08-09 00:00:00", Type.DATETIMEV2); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.addDate(new DateLiteral("2018-08-08", Type.DATEV2), new IntLiteral(1)); + expectedResult = new DateLiteral("2018-08-09 00:00:00", Type.DATETIME); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.addDate(new DateLiteral("2018-08-08", Type.DATEV2), new IntLiteral(-1)); + expectedResult = new DateLiteral("2018-08-07 00:00:00", Type.DATETIMEV2); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.addDate(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(-1)); + expectedResult = new DateLiteral("2018-08-07 00:00:00", Type.DATETIMEV2); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.addDate(new DateLiteral("2018-08-08", Type.DATEV2), new IntLiteral(-1)); + expectedResult = new DateLiteral("2018-08-07 00:00:00", Type.DATETIME); + Assert.assertEquals(expectedResult, actualResult); + + } + @Test public void daysAddTest() throws AnalysisException { DateLiteral actualResult = FEFunctions.daysAdd(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(1)); @@ -126,6 +228,21 @@ public class FEFunctionsTest { Assert.assertEquals(expectedResult, actualResult); } + @Test + public void daysAddTest2() throws AnalysisException { + DateLiteral actualResult = FEFunctions.daysAdd(new DateLiteral("2018-08-08", Type.DATEV2), new IntLiteral(1)); + DateLiteral expectedResult = new DateLiteral("2018-08-09", Type.DATEV2); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.daysAdd(new DateLiteral("2018-08-08", Type.DATEV2), new IntLiteral(-1)); + expectedResult = new DateLiteral("2018-08-07", Type.DATEV2); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.daysAdd(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(-1)); + expectedResult = new DateLiteral("2018-08-07", Type.DATEV2); + Assert.assertEquals(expectedResult, actualResult); + } + @Test public void fromUnixTimeTest() throws AnalysisException { StringLiteral actualResult = FEFunctions.fromUnixTime(new IntLiteral(100000)); @@ -168,10 +285,53 @@ public class FEFunctionsTest { Assert.assertEquals("January", FEFunctions.dateFormat(testDate, new StringLiteral("%M")).getStringValue()); Assert.assertEquals("01", FEFunctions.dateFormat(testDate, new StringLiteral("%m")).getStringValue()); Assert.assertEquals("PM", FEFunctions.dateFormat(testDate, new StringLiteral("%p")).getStringValue()); - Assert.assertEquals("01:04:05 PM", FEFunctions.dateFormat(testDate, new StringLiteral("%r")).getStringValue()); + Assert.assertEquals("01:04:05 PM", FEFunctions.dateFormat(testDate, + new StringLiteral("%r")).getStringValue()); Assert.assertEquals("05", FEFunctions.dateFormat(testDate, new StringLiteral("%S")).getStringValue()); Assert.assertEquals("05", FEFunctions.dateFormat(testDate, new StringLiteral("%s")).getStringValue()); - Assert.assertEquals("13:04:05", FEFunctions.dateFormat(testDate, new StringLiteral("%T")).getStringValue()); + Assert.assertEquals("13:04:05", FEFunctions.dateFormat(testDate, + new StringLiteral("%T")).getStringValue()); + Assert.assertEquals("02", FEFunctions.dateFormat(testDate, new StringLiteral("%v")).getStringValue()); + Assert.assertEquals("Tuesday", FEFunctions.dateFormat(testDate, new StringLiteral("%W")).getStringValue()); + Assert.assertEquals("2001", FEFunctions.dateFormat(testDate, new StringLiteral("%Y")).getStringValue()); + Assert.assertEquals("01", FEFunctions.dateFormat(testDate, new StringLiteral("%y")).getStringValue()); + Assert.assertEquals("%", FEFunctions.dateFormat(testDate, new StringLiteral("%%")).getStringValue()); + Assert.assertEquals("foo", FEFunctions.dateFormat(testDate, new StringLiteral("foo")).getStringValue()); + Assert.assertEquals("g", FEFunctions.dateFormat(testDate, new StringLiteral("%g")).getStringValue()); + Assert.assertEquals("4", FEFunctions.dateFormat(testDate, new StringLiteral("%4")).getStringValue()); + Assert.assertEquals("2001 02", FEFunctions.dateFormat(testDate, + new StringLiteral("%x %v")).getStringValue()); + } catch (AnalysisException e) { + e.printStackTrace(); + } + } + + @Test + public void dateV2FormatUtilTest() { + try { + Locale.setDefault(Locale.ENGLISH); + DateLiteral testDate = new DateLiteral("2001-01-09 13:04:05", Type.DATETIMEV2); + Assert.assertEquals("Tue", FEFunctions.dateFormat(testDate, new StringLiteral("%a")).getStringValue()); + Assert.assertEquals("Jan", FEFunctions.dateFormat(testDate, new StringLiteral("%b")).getStringValue()); + Assert.assertEquals("1", FEFunctions.dateFormat(testDate, new StringLiteral("%c")).getStringValue()); + Assert.assertEquals("09", FEFunctions.dateFormat(testDate, new StringLiteral("%d")).getStringValue()); + Assert.assertEquals("9", FEFunctions.dateFormat(testDate, new StringLiteral("%e")).getStringValue()); + Assert.assertEquals("13", FEFunctions.dateFormat(testDate, new StringLiteral("%H")).getStringValue()); + Assert.assertEquals("01", FEFunctions.dateFormat(testDate, new StringLiteral("%h")).getStringValue()); + Assert.assertEquals("01", FEFunctions.dateFormat(testDate, new StringLiteral("%I")).getStringValue()); + Assert.assertEquals("04", FEFunctions.dateFormat(testDate, new StringLiteral("%i")).getStringValue()); + Assert.assertEquals("009", FEFunctions.dateFormat(testDate, new StringLiteral("%j")).getStringValue()); + Assert.assertEquals("13", FEFunctions.dateFormat(testDate, new StringLiteral("%k")).getStringValue()); + Assert.assertEquals("1", FEFunctions.dateFormat(testDate, new StringLiteral("%l")).getStringValue()); + Assert.assertEquals("January", FEFunctions.dateFormat(testDate, new StringLiteral("%M")).getStringValue()); + Assert.assertEquals("01", FEFunctions.dateFormat(testDate, new StringLiteral("%m")).getStringValue()); + Assert.assertEquals("PM", FEFunctions.dateFormat(testDate, new StringLiteral("%p")).getStringValue()); + Assert.assertEquals("01:04:05 PM", FEFunctions.dateFormat(testDate, + new StringLiteral("%r")).getStringValue()); + Assert.assertEquals("05", FEFunctions.dateFormat(testDate, new StringLiteral("%S")).getStringValue()); + Assert.assertEquals("05", FEFunctions.dateFormat(testDate, new StringLiteral("%s")).getStringValue()); + Assert.assertEquals("13:04:05", FEFunctions.dateFormat(testDate, + new StringLiteral("%T")).getStringValue()); Assert.assertEquals("02", FEFunctions.dateFormat(testDate, new StringLiteral("%v")).getStringValue()); Assert.assertEquals("Tuesday", FEFunctions.dateFormat(testDate, new StringLiteral("%W")).getStringValue()); Assert.assertEquals("2001", FEFunctions.dateFormat(testDate, new StringLiteral("%Y")).getStringValue()); @@ -181,6 +341,8 @@ public class FEFunctionsTest { Assert.assertEquals("g", FEFunctions.dateFormat(testDate, new StringLiteral("%g")).getStringValue()); Assert.assertEquals("4", FEFunctions.dateFormat(testDate, new StringLiteral("%4")).getStringValue()); Assert.assertEquals("2001 02", FEFunctions.dateFormat(testDate, new StringLiteral("%x %v")).getStringValue()); + Assert.assertEquals("2001 02", FEFunctions.dateFormat(testDate, + new StringLiteral("%x %v")).getStringValue()); } catch (AnalysisException e) { e.printStackTrace(); } @@ -189,28 +351,55 @@ public class FEFunctionsTest { @Test public void dateParseTest() { try { - Assert.assertEquals("2019-05-09 00:00:00", FEFunctions.dateParse(new StringLiteral("2019-05-09"), new StringLiteral("%Y-%m-%d %H:%i:%s")).getStringValue()); - Assert.assertEquals("2013-05-10", FEFunctions.dateParse(new StringLiteral("2013,05,10"), new StringLiteral("%Y,%m,%d")).getStringValue()); - Assert.assertEquals("2013-05-17 00:35:10", FEFunctions.dateParse(new StringLiteral("2013-05-17 12:35:10"), new StringLiteral("%Y-%m-%d %h:%i:%s")).getStringValue()); - Assert.assertEquals("2013-05-17 00:35:10", FEFunctions.dateParse(new StringLiteral("2013-05-17 00:35:10"), new StringLiteral("%Y-%m-%d %H:%i:%s")).getStringValue()); - Assert.assertEquals("2013-05-17 00:35:10", FEFunctions.dateParse(new StringLiteral("2013-05-17 12:35:10 AM"), new StringLiteral("%Y-%m-%d %h:%i:%s %p")).getStringValue()); - Assert.assertEquals("2013-05-17 12:35:10", FEFunctions.dateParse(new StringLiteral("2013-05-17 12:35:10 PM"), new StringLiteral("%Y-%m-%d %h:%i:%s %p")).getStringValue()); - Assert.assertEquals("2013-05-17 23:35:10", FEFunctions.dateParse(new StringLiteral("abc 2013-05-17 fff 23:35:10 xyz"), new StringLiteral("abc %Y-%m-%d fff %H:%i:%s xyz")).getStringValue()); - Assert.assertEquals("2016-01-28 23:45:46", FEFunctions.dateParse(new StringLiteral("28-JAN-16 11.45.46 PM"), new StringLiteral("%d-%b-%y %l.%i.%s %p")).getStringValue()); - Assert.assertEquals("2019-05-09", FEFunctions.dateParse(new StringLiteral("2019/May/9"), new StringLiteral("%Y/%b/%d")).getStringValue()); - Assert.assertEquals("2019-05-09", FEFunctions.dateParse(new StringLiteral("2019,129"), new StringLiteral("%Y,%j")).getStringValue()); - Assert.assertEquals("2019-05-09", FEFunctions.dateParse(new StringLiteral("2019,19,Thursday"), new StringLiteral("%x,%v,%W")).getStringValue()); - Assert.assertEquals("2019-05-09 12:10:45", FEFunctions.dateParse(new StringLiteral("12:10:45-20190509"), new StringLiteral("%T-%Y%m%d")).getStringValue()); - Assert.assertEquals("2019-05-09 09:10:45", FEFunctions.dateParse(new StringLiteral("20190509-9:10:45"), new StringLiteral("%Y%m%d-%k:%i:%S")).getStringValue()); - Assert.assertEquals("0000-00-20", FEFunctions.dateParse(new StringLiteral("2013-05-17"), new StringLiteral("%D")).getStringValue()); - Assert.assertEquals("0000-00-00", FEFunctions.dateParse(new StringLiteral("2013-05-17"), new StringLiteral("%U")).getStringValue()); - Assert.assertEquals("0000-00-00", FEFunctions.dateParse(new StringLiteral("2013-05-17"), new StringLiteral("%u")).getStringValue()); - Assert.assertEquals("0000-00-00", FEFunctions.dateParse(new StringLiteral("2013-05-17"), new StringLiteral("%V")).getStringValue()); - Assert.assertEquals("0000-00-00", FEFunctions.dateParse(new StringLiteral("2013-05-17"), new StringLiteral("%w")).getStringValue()); - Assert.assertEquals("0000-00-00", FEFunctions.dateParse(new StringLiteral("2013-05-17"), new StringLiteral("%x")).getStringValue()); - Assert.assertEquals("0000-00-00", FEFunctions.dateParse(new StringLiteral("2013-05-17"), new StringLiteral("%X")).getStringValue()); - Assert.assertEquals("2013-05-17 20:07:05", FEFunctions.dateParse(new StringLiteral("2013-05-17 08:07:05 PM"), new StringLiteral("%Y-%m-%d %r")).getStringValue()); - Assert.assertEquals("2013-05-17 08:07:05", FEFunctions.dateParse(new StringLiteral("2013-05-17 08:07:05"), new StringLiteral("%Y-%m-%d %T")).getStringValue()); + Assert.assertEquals("2019-05-09 00:00:00", FEFunctions.dateParse(new StringLiteral("2019-05-09"), + new StringLiteral("%Y-%m-%d %H:%i:%s")).getStringValue()); + Assert.assertEquals("2013-05-10", FEFunctions.dateParse(new StringLiteral("2013,05,10"), + new StringLiteral("%Y,%m,%d")).getStringValue()); + Assert.assertEquals("2013-05-17 00:35:10", FEFunctions.dateParse( + new StringLiteral("2013-05-17 12:35:10"), + new StringLiteral("%Y-%m-%d %h:%i:%s")).getStringValue()); + Assert.assertEquals("2013-05-17 00:35:10", FEFunctions.dateParse(new StringLiteral("2013-05-17 00:35:10"), + new StringLiteral("%Y-%m-%d %H:%i:%s")).getStringValue()); + Assert.assertEquals("2013-05-17 00:35:10", FEFunctions.dateParse( + new StringLiteral("2013-05-17 12:35:10 AM"), + new StringLiteral("%Y-%m-%d %h:%i:%s %p")).getStringValue()); + Assert.assertEquals("2013-05-17 12:35:10", FEFunctions.dateParse( + new StringLiteral("2013-05-17 12:35:10 PM"), + new StringLiteral("%Y-%m-%d %h:%i:%s %p")).getStringValue()); + Assert.assertEquals("2013-05-17 23:35:10", FEFunctions.dateParse( + new StringLiteral("abc 2013-05-17 fff 23:35:10 xyz"), + new StringLiteral("abc %Y-%m-%d fff %H:%i:%s xyz")).getStringValue()); + Assert.assertEquals("2016-01-28 23:45:46", FEFunctions.dateParse( + new StringLiteral("28-JAN-16 11.45.46 PM"), + new StringLiteral("%d-%b-%y %l.%i.%s %p")).getStringValue()); + Assert.assertEquals("2019-05-09", FEFunctions.dateParse(new StringLiteral("2019/May/9"), + new StringLiteral("%Y/%b/%d")).getStringValue()); + Assert.assertEquals("2019-05-09", FEFunctions.dateParse(new StringLiteral("2019,129"), + new StringLiteral("%Y,%j")).getStringValue()); + Assert.assertEquals("2019-05-09", FEFunctions.dateParse(new StringLiteral("2019,19,Thursday"), + new StringLiteral("%x,%v,%W")).getStringValue()); + Assert.assertEquals("2019-05-09 12:10:45", FEFunctions.dateParse(new StringLiteral("12:10:45-20190509"), + new StringLiteral("%T-%Y%m%d")).getStringValue()); + Assert.assertEquals("2019-05-09 09:10:45", FEFunctions.dateParse(new StringLiteral("20190509-9:10:45"), + new StringLiteral("%Y%m%d-%k:%i:%S")).getStringValue()); + Assert.assertEquals("0000-00-20", FEFunctions.dateParse(new StringLiteral("2013-05-17"), + new StringLiteral("%D")).getStringValue()); + Assert.assertEquals("0000-00-00", FEFunctions.dateParse(new StringLiteral("2013-05-17"), + new StringLiteral("%U")).getStringValue()); + Assert.assertEquals("0000-00-00", FEFunctions.dateParse(new StringLiteral("2013-05-17"), + new StringLiteral("%u")).getStringValue()); + Assert.assertEquals("0000-00-00", FEFunctions.dateParse(new StringLiteral("2013-05-17"), + new StringLiteral("%V")).getStringValue()); + Assert.assertEquals("0000-00-00", FEFunctions.dateParse(new StringLiteral("2013-05-17"), + new StringLiteral("%w")).getStringValue()); + Assert.assertEquals("0000-00-00", FEFunctions.dateParse(new StringLiteral("2013-05-17"), + new StringLiteral("%x")).getStringValue()); + Assert.assertEquals("0000-00-00", FEFunctions.dateParse(new StringLiteral("2013-05-17"), + new StringLiteral("%X")).getStringValue()); + Assert.assertEquals("2013-05-17 20:07:05", FEFunctions.dateParse( + new StringLiteral("2013-05-17 08:07:05 PM"), new StringLiteral("%Y-%m-%d %r")).getStringValue()); + Assert.assertEquals("2013-05-17 08:07:05", FEFunctions.dateParse(new StringLiteral("2013-05-17 08:07:05"), + new StringLiteral("%Y-%m-%d %T")).getStringValue()); } catch (AnalysisException e) { e.printStackTrace(); Assert.fail("Junit test dateParse fail"); @@ -235,6 +424,33 @@ public class FEFunctionsTest { Assert.assertEquals(expectedResult, actualResult); } + @Test + public void dateV2SubTest() throws AnalysisException { + DateLiteral actualResult = FEFunctions.dateSub(new DateLiteral("2018-08-08", Type.DATEV2), new IntLiteral(1)); + DateLiteral expectedResult = new DateLiteral("2018-08-07", Type.DATEV2); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.dateSub(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(1)); + expectedResult = new DateLiteral("2018-08-07", Type.DATEV2); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.dateSub(new DateLiteral("2018-08-08", Type.DATEV2), new IntLiteral(1)); + expectedResult = new DateLiteral("2018-08-07", Type.DATE); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.dateSub(new DateLiteral("2018-08-08", Type.DATEV2), new IntLiteral(-1)); + expectedResult = new DateLiteral("2018-08-09", Type.DATEV2); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.dateSub(new DateLiteral("2018-08-08", Type.DATEV2), new IntLiteral(-1)); + expectedResult = new DateLiteral("2018-08-09", Type.DATE); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.dateSub(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(-1)); + expectedResult = new DateLiteral("2018-08-09", Type.DATEV2); + Assert.assertEquals(expectedResult, actualResult); + } + @Test public void yearTest() throws AnalysisException { IntLiteral actualResult = FEFunctions.year(new DateLiteral("2018-08-08", Type.DATE)); @@ -246,6 +462,25 @@ public class FEFunctionsTest { Assert.assertEquals(expectedResult, actualResult); } + @Test + public void yearTest2() throws AnalysisException { + IntLiteral actualResult = FEFunctions.year(new DateLiteral("2018-08-08", Type.DATE)); + IntLiteral expectedResult = new IntLiteral(2018, Type.INT); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.year(new DateLiteral("2018-08-08", Type.DATEV2)); + expectedResult = new IntLiteral(2018, Type.INT); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.year(new DateLiteral("1970-01-02 11:46:40", Type.DATETIME)); + expectedResult = new IntLiteral(1970, Type.INT); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.year(new DateLiteral("1970-01-02 11:46:40", Type.DATETIMEV2)); + expectedResult = new IntLiteral(1970, Type.INT); + Assert.assertEquals(expectedResult, actualResult); + } + @Test public void monthTest() throws AnalysisException { IntLiteral actualResult = FEFunctions.month(new DateLiteral("2018-08-08", Type.DATE)); @@ -257,6 +492,17 @@ public class FEFunctionsTest { Assert.assertEquals(expectedResult, actualResult); } + @Test + public void monthTest2() throws AnalysisException { + IntLiteral actualResult = FEFunctions.month(new DateLiteral("2018-08-08", Type.DATEV2)); + IntLiteral expectedResult = new IntLiteral(8, Type.INT); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.month(new DateLiteral("1970-01-02 11:46:40", Type.DATETIMEV2)); + expectedResult = new IntLiteral(1, Type.INT); + Assert.assertEquals(expectedResult, actualResult); + } + @Test public void dayTest() throws AnalysisException { IntLiteral actualResult = FEFunctions.day(new DateLiteral("2018-08-08", Type.DATE)); @@ -268,6 +514,17 @@ public class FEFunctionsTest { Assert.assertEquals(expectedResult, actualResult); } + @Test + public void dayTest2() throws AnalysisException { + IntLiteral actualResult = FEFunctions.day(new DateLiteral("2018-08-08", Type.DATEV2)); + IntLiteral expectedResult = new IntLiteral(8, Type.INT); + Assert.assertEquals(expectedResult, actualResult); + + actualResult = FEFunctions.day(new DateLiteral("1970-01-02 11:46:40", Type.DATETIMEV2)); + expectedResult = new IntLiteral(2, Type.INT); + Assert.assertEquals(expectedResult, actualResult); + } + @Test public void floorTest() throws AnalysisException { IntLiteral actualResult = FEFunctions.floor(new FloatLiteral(3.3)); @@ -315,7 +572,9 @@ public class FEFunctionsTest { @Test public void addBigIntTest() throws AnalysisException { - LargeIntLiteral actualResult = FEFunctions.addBigInt(new LargeIntLiteral("170141183460469231731687303715884105727"), new LargeIntLiteral("-170141183460469231731687303715884105728")); + LargeIntLiteral actualResult = FEFunctions.addBigInt( + new LargeIntLiteral("170141183460469231731687303715884105727"), + new LargeIntLiteral("-170141183460469231731687303715884105728")); LargeIntLiteral expectedResult = new LargeIntLiteral("-1"); Assert.assertEquals(expectedResult, actualResult); @@ -348,7 +607,8 @@ public class FEFunctionsTest { @Test public void subtractDecimalV2Test() throws AnalysisException { - DecimalLiteral actualResult = FEFunctions.subtractDecimalV2(new DecimalLiteral("2.2"), new DecimalLiteral("3.3")); + DecimalLiteral actualResult = FEFunctions.subtractDecimalV2(new DecimalLiteral("2.2"), + new DecimalLiteral("3.3")); DecimalLiteral expectedResult = new DecimalLiteral("-1.1"); Assert.assertEquals(expectedResult, actualResult); @@ -359,7 +619,9 @@ public class FEFunctionsTest { @Test public void subtractBigIntTest() throws AnalysisException { - LargeIntLiteral actualResult = FEFunctions.subtractBigInt(new LargeIntLiteral("170141183460469231731687303715884105727"), new LargeIntLiteral("170141183460469231731687303715884105728")); + LargeIntLiteral actualResult = FEFunctions.subtractBigInt( + new LargeIntLiteral("170141183460469231731687303715884105727"), + new LargeIntLiteral("170141183460469231731687303715884105728")); LargeIntLiteral expectedResult = new LargeIntLiteral("-1"); Assert.assertEquals(expectedResult, actualResult); @@ -400,7 +662,8 @@ public class FEFunctionsTest { @Test public void multiplyDecimalV2Test() throws AnalysisException { - DecimalLiteral actualResult = FEFunctions.multiplyDecimalV2(new DecimalLiteral("1.1"), new DecimalLiteral("1.0")); + DecimalLiteral actualResult = FEFunctions.multiplyDecimalV2(new DecimalLiteral("1.1"), + new DecimalLiteral("1.0")); DecimalLiteral expectedResult = new DecimalLiteral("1.1"); Assert.assertEquals(expectedResult, actualResult); @@ -415,7 +678,8 @@ public class FEFunctionsTest { @Test public void multiplyBigIntTest() throws AnalysisException { - LargeIntLiteral actualResult = FEFunctions.multiplyBigInt(new LargeIntLiteral("-170141183460469231731687303715884105727"), new LargeIntLiteral("1")); + LargeIntLiteral actualResult = FEFunctions.multiplyBigInt( + new LargeIntLiteral("-170141183460469231731687303715884105727"), new LargeIntLiteral("1")); LargeIntLiteral expectedResult = new LargeIntLiteral("-170141183460469231731687303715884105727"); Assert.assertEquals(expectedResult, actualResult); @@ -441,7 +705,8 @@ public class FEFunctionsTest { @Test public void divideDecimalV2Test() throws AnalysisException { - DecimalLiteral actualResult = FEFunctions.divideDecimalV2(new DecimalLiteral("1.1"), new DecimalLiteral("1.0")); + DecimalLiteral actualResult = FEFunctions.divideDecimalV2(new DecimalLiteral("1.1"), + new DecimalLiteral("1.0")); DecimalLiteral expectedResult = new DecimalLiteral("1.1"); Assert.assertEquals(expectedResult, actualResult); @@ -460,6 +725,16 @@ public class FEFunctionsTest { Assert.assertEquals(2419200, FEFunctions.timeDiff(d3, d2).getLongValue()); } + @Test + public void timeDiffTest2() throws AnalysisException { + DateLiteral d1 = new DateLiteral("1019-02-28 00:00:00", Type.DATETIMEV2); + DateLiteral d2 = new DateLiteral("2019-02-28 00:00:00", Type.DATETIME); + DateLiteral d3 = new DateLiteral("2019-03-28 00:00:00", Type.DATETIMEV2); + Assert.assertEquals(31556995543L, FEFunctions.timeDiff(d2, d1).getLongValue()); + Assert.assertEquals(31559414743L, FEFunctions.timeDiff(d3, d1).getLongValue()); + Assert.assertEquals(2419200, FEFunctions.timeDiff(d3, d2).getLongValue()); + } + @Test public void timeNowTest() throws AnalysisException { String curTimeString = FEFunctions.curTime().toSqlImpl().replace("'", ""); @@ -511,4 +786,69 @@ public class FEFunctionsTest { Assert.assertEquals(new DateLiteral("2019-11-10 23:59:59", Type.DATETIME), FEFunctions.secondsSub(dateLiteral, new IntLiteral(1))); } + + @Test + public void dateV2PlusAndSubTest() throws AnalysisException { + DateLiteral dateLiteral = new DateLiteral("2019-11-11 00:00:00", Type.DATETIMEV2); + + Assert.assertEquals(new DateLiteral("2020-11-11 00:00:00", Type.DATETIME), + FEFunctions.yearsAdd(dateLiteral, new IntLiteral(1))); + Assert.assertEquals(new DateLiteral("2020-11-11 00:00:00", Type.DATETIMEV2), + FEFunctions.yearsAdd(dateLiteral, new IntLiteral(1))); + + Assert.assertEquals(new DateLiteral("2018-11-11 00:00:00", Type.DATETIME), + FEFunctions.yearsSub(dateLiteral, new IntLiteral(1))); + Assert.assertEquals(new DateLiteral("2018-11-11 00:00:00", Type.DATETIMEV2), + FEFunctions.yearsSub(dateLiteral, new IntLiteral(1))); + + Assert.assertEquals(new DateLiteral("2019-12-11 00:00:00", Type.DATETIME), + FEFunctions.monthsAdd(dateLiteral, new IntLiteral(1))); + Assert.assertEquals(new DateLiteral("2019-12-11 00:00:00", Type.DATETIMEV2), + FEFunctions.monthsAdd(dateLiteral, new IntLiteral(1))); + + Assert.assertEquals(new DateLiteral("2019-10-11 00:00:00", Type.DATETIME), + FEFunctions.monthsSub(dateLiteral, new IntLiteral(1))); + Assert.assertEquals(new DateLiteral("2019-10-11 00:00:00", Type.DATETIMEV2), + FEFunctions.monthsSub(dateLiteral, new IntLiteral(1))); + + Assert.assertEquals(new DateLiteral("2019-11-12 00:00:00", Type.DATETIME), + FEFunctions.daysAdd(dateLiteral, new IntLiteral(1))); + Assert.assertEquals(new DateLiteral("2019-11-12 00:00:00", Type.DATETIMEV2), + FEFunctions.daysAdd(dateLiteral, new IntLiteral(1))); + + Assert.assertEquals(new DateLiteral("2019-11-10 00:00:00", Type.DATETIME), + FEFunctions.daysSub(dateLiteral, new IntLiteral(1))); + Assert.assertEquals(new DateLiteral("2019-11-10 00:00:00", Type.DATETIMEV2), + FEFunctions.daysSub(dateLiteral, new IntLiteral(1))); + + Assert.assertEquals(new DateLiteral("2019-11-11 01:00:00", Type.DATETIME), + FEFunctions.hoursAdd(dateLiteral, new IntLiteral(1))); + Assert.assertEquals(new DateLiteral("2019-11-11 01:00:00", Type.DATETIMEV2), + FEFunctions.hoursAdd(dateLiteral, new IntLiteral(1))); + + Assert.assertEquals(new DateLiteral("2019-11-10 23:00:00", Type.DATETIME), + FEFunctions.hoursSub(dateLiteral, new IntLiteral(1))); + Assert.assertEquals(new DateLiteral("2019-11-10 23:00:00", Type.DATETIMEV2), + FEFunctions.hoursSub(dateLiteral, new IntLiteral(1))); + + Assert.assertEquals(new DateLiteral("2019-11-11 00:01:00", Type.DATETIME), + FEFunctions.minutesAdd(dateLiteral, new IntLiteral(1))); + Assert.assertEquals(new DateLiteral("2019-11-11 00:01:00", Type.DATETIMEV2), + FEFunctions.minutesAdd(dateLiteral, new IntLiteral(1))); + + Assert.assertEquals(new DateLiteral("2019-11-10 23:59:00", Type.DATETIME), + FEFunctions.minutesSub(dateLiteral, new IntLiteral(1))); + Assert.assertEquals(new DateLiteral("2019-11-10 23:59:00", Type.DATETIMEV2), + FEFunctions.minutesSub(dateLiteral, new IntLiteral(1))); + + Assert.assertEquals(new DateLiteral("2019-11-11 00:00:01", Type.DATETIME), + FEFunctions.secondsAdd(dateLiteral, new IntLiteral(1))); + Assert.assertEquals(new DateLiteral("2019-11-11 00:00:01", Type.DATETIMEV2), + FEFunctions.secondsAdd(dateLiteral, new IntLiteral(1))); + + Assert.assertEquals(new DateLiteral("2019-11-10 23:59:59", Type.DATETIME), + FEFunctions.secondsSub(dateLiteral, new IntLiteral(1))); + Assert.assertEquals(new DateLiteral("2019-11-10 23:59:59", Type.DATETIMEV2), + FEFunctions.secondsSub(dateLiteral, new IntLiteral(1))); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java b/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java index 68610859f0..f91568d514 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java @@ -28,6 +28,7 @@ public class RewriteDateLiteralRuleTest { private DorisAssert dorisAssert; private static final String DB_NAME = "rewritedaterule"; private static final String TABLE_NAME_1 = "tb1"; + private static final String TABLE_NAME_2 = "tb2"; public void before(ConnectContext ctx) throws Exception { FeConstants.default_scheduler_interval_millisecond = 10; @@ -38,6 +39,10 @@ public class RewriteDateLiteralRuleTest { + " (k1 datetime, k2 int) " + "distributed by hash(k2) buckets 3 properties('replication_num' = '1');"; dorisAssert.withTable(createTableSQL); + createTableSQL = "create table " + DB_NAME + "." + TABLE_NAME_2 + + " (k1 datetime(3), k2 int) " + + "distributed by hash(k2) buckets 3 properties('replication_num' = '1');"; + dorisAssert.withTable(createTableSQL); } public void after() throws Exception { @@ -56,6 +61,18 @@ public class RewriteDateLiteralRuleTest { Assert.assertTrue(planString.contains("`k1` > '2021-03-01 22:33:44'")); } + public void testWithIntFormatDateV2() throws Exception { + String query = "select * from " + DB_NAME + ".tb2 where k1 > 20210301"; + String planString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00'")); + query = "select k1 > 20210301 from " + DB_NAME + ".tb2"; + planString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00'")); + query = "select k1 > 20210301223344 from " + DB_NAME + ".tb2"; + planString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 22:33:44'")); + } + public void testWithStringFormatDate() throws Exception { String query = "select * from " + DB_NAME + ".tb1 where k1 > '2021030112334455'"; String planString = dorisAssert.query(query).explainQuery(); @@ -102,6 +119,52 @@ public class RewriteDateLiteralRuleTest { Assert.assertTrue(planString.contains("`k1` > '2012-03-01 11:22:00'")); } + public void testWithStringFormatDateV2() throws Exception { + String query = "select * from " + DB_NAME + ".tb2 where k1 > '2021030112334455'"; + String planString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 12:33:44'")); + + query = "select k1 > '20210301' from " + DB_NAME + ".tb2"; + planString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00'")); + + query = "select k1 > '20210301233234.34' from " + DB_NAME + ".tb2"; + planString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 23:32:34'")); + + query = "select * from " + DB_NAME + ".tb2 where k1 > '2021-03-01'"; + planString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00'")); + + query = "select k1 > '2021-03-01 11:22:33' from " + DB_NAME + ".tb2"; + planString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 11:22:33'")); + + query = "select k1 > '2021-03-01 16:22:33' from " + DB_NAME + ".tb2"; + planString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 16:22:33'")); + + query = "select k1 > '2021-03-01 11:22' from " + DB_NAME + ".tb2"; + planString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 11:22:00'")); + + query = "select k1 > '20210301T221133' from " + DB_NAME + ".tb2"; + planString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 22:11:33'")); + + query = "select k1 > '2021-03-01dd 11:22' from " + DB_NAME + ".tb2"; + planString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00'")); + + query = "select k1 > '80-03-01 11:22' from " + DB_NAME + ".tb2"; + planString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(planString.contains("`k1` > '1980-03-01 11:22:00'")); + + query = "select k1 > '12-03-01 11:22' from " + DB_NAME + ".tb2"; + planString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(planString.contains("`k1` > '2012-03-01 11:22:00'")); + } + public void testWithDoubleFormatDate() throws Exception { String query = "select * from " + DB_NAME + ".tb1 where k1 > 20210301.22"; String planString = dorisAssert.query(query).explainQuery(); @@ -112,6 +175,16 @@ public class RewriteDateLiteralRuleTest { Assert.assertTrue(planString.contains("`k1` > 2.021033122E7")); } + public void testWithDoubleFormatDateV2() throws Exception { + String query = "select * from " + DB_NAME + ".tb2 where k1 > 20210301.22"; + String planString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(planString.contains("`k1` > 2.021030122E7")); + + query = "select k1 > 20210331.22 from " + DB_NAME + ".tb2"; + planString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(planString.contains("`k1` > 2.021033122E7")); + } + public void testWithInvalidFormatDate() throws Exception { String query = "select * from " + DB_NAME + ".tb1 where k1 > '2021030125334455'"; try { @@ -145,4 +218,38 @@ public class RewriteDateLiteralRuleTest { plainString = dorisAssert.query(query).explainQuery(); Assert.assertTrue(plainString.contains("NULL")); } + + public void testWithInvalidFormatDateV2() throws Exception { + String query = "select * from " + DB_NAME + ".tb2 where k1 > '2021030125334455'"; + try { + dorisAssert.query(query).explainQuery(); + } catch (AnalysisException e) { + Assert.assertTrue(e.getMessage().contains( + "Incorrect datetime value: '2021030125334455' in expression: `k1` > '2021030125334455'")); + } + + query = "select k1 > '2021030125334455' from " + DB_NAME + ".tb2"; + String plainString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(plainString.contains("NULL")); + + query = "select * from " + DB_NAME + ".tb2 where k1 > '2021-03-32 23:33:55'"; + try { + dorisAssert.query(query).explainQuery(); + } catch (AnalysisException e) { + Assert.assertTrue(e.getMessage().contains( + "Incorrect datetime value: '2021-03-32 23:33:55' in expression: `k1` > '2021-03-32 23:33:55'")); + } + + query = "select * from " + DB_NAME + ".tb2 where k1 > '2021-03- 03 23:33:55'"; + try { + dorisAssert.query(query).explainQuery(); + } catch (AnalysisException e) { + Assert.assertTrue(e.getMessage().contains( + "Incorrect datetime value: '2021-03- 03 23:33:55' in expression: `k1` > '2021-03- 03 23:33:55'")); + } + + query = "select k1 > '2021-03- 03 23:33:55' from " + DB_NAME + ".tb2"; + plainString = dorisAssert.query(query).explainQuery(); + Assert.assertTrue(plainString.contains("NULL")); + } } diff --git a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/SparkRDDAggregator.java b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/SparkRDDAggregator.java index 57548443ba..9e3f31dfc7 100644 --- a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/SparkRDDAggregator.java +++ b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/SparkRDDAggregator.java @@ -75,6 +75,8 @@ public abstract class SparkRDDAggregator implements Serializable { case "decimalv2": case "date": case "datetime": + case "datev2": + case "datetimev2": return new NumberMaxAggregator(); case "char": case "varchar": @@ -96,6 +98,8 @@ public abstract class SparkRDDAggregator implements Serializable { case "decimalv2": case "date": case "datetime": + case "datev2": + case "datetimev2": return new NumberMinAggregator(); case "char": case "varchar": diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index 51f48f1a2a..f7013880d4 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -378,6 +378,7 @@ enum PColumnType { COLUMN_TYPE_DECIMAL = 12; COLUMN_TYPE_DECIMALV2 = 13; COLUMN_TYPE_STRING = 14; + COLUMN_TYPE_DATEV2 = 15; } message PMinMaxFilter { diff --git a/gensrc/proto/types.proto b/gensrc/proto/types.proto index 7a95288517..7655255e47 100644 --- a/gensrc/proto/types.proto +++ b/gensrc/proto/types.proto @@ -99,6 +99,7 @@ message PGenericType { DECIMAL128 = 25; BYTES = 26; NOTHING = 27; + DATEV2 = 28; UNKNOWN = 999; } required TypeId id = 2; diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index 1d6bc8c683..b9f6c61ef5 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -84,6 +84,9 @@ enum TPrimitiveType { STRING, ALL, QUANTILE_STATE, + DATEV2, + DATETIMEV2, + TIMEV2, } enum TTypeNodeType { diff --git a/regression-test/suites/correctness/test_last_value_window.groovy b/regression-test/suites/correctness/test_last_value_window.groovy index 274cdec46d..fab1543d2b 100644 --- a/regression-test/suites/correctness/test_last_value_window.groovy +++ b/regression-test/suites/correctness/test_last_value_window.groovy @@ -23,10 +23,10 @@ suite("test_last_value_window") { sql """ CREATE TABLE ${tableName} ( `myday` INT, - `time` VARCHAR(40) NOT NULL, + `time_col` VARCHAR(40) NOT NULL, `state` INT ) ENGINE=OLAP - DUPLICATE KEY(`myday`,time,state) + DUPLICATE KEY(`myday`,time_col,state) COMMENT "OLAP" DISTRIBUTED BY HASH(`myday`) BUCKETS 2 PROPERTIES ( @@ -46,6 +46,6 @@ suite("test_last_value_window") { // not_vectorized sql """ set enable_vectorized_engine = false; """ - qt_select_default """ select *,last_value(state) over(partition by myday order by time) from ${tableName}; """ + qt_select_default """ select *,last_value(state) over(partition by myday order by time_col) from ${tableName}; """ } \ No newline at end of file