diff --git a/be/src/exec/olap_common.h b/be/src/exec/olap_common.h index 5bd06d8d54..7a58645b74 100644 --- a/be/src/exec/olap_common.h +++ b/be/src/exec/olap_common.h @@ -79,7 +79,7 @@ std::string cast_to_string(T value, int scale) { template class ColumnValueRange { public: - using CppType = typename VecPrimitiveTypeTraits::CppType; + using CppType = typename PrimitiveTypeTraits::CppType; using IteratorType = typename std::set::iterator; ColumnValueRange(); @@ -1094,7 +1094,7 @@ bool ColumnValueRange::has_intersection(ColumnValueRange Status OlapScanKeys::extend_scan_key(ColumnValueRange& range, int32_t max_scan_key_num, bool* exact_value, bool* eos) { - using CppType = typename VecPrimitiveTypeTraits::CppType; + using CppType = typename PrimitiveTypeTraits::CppType; using ConstIterator = typename std::set::const_iterator; // 1. clear ScanKey if some column range is empty diff --git a/be/src/exec/text_converter.cpp b/be/src/exec/text_converter.cpp index f278e273ed..95232678ea 100644 --- a/be/src/exec/text_converter.cpp +++ b/be/src/exec/text_converter.cpp @@ -253,7 +253,7 @@ bool TextConverter::_write_data(const TypeDescriptor& type_desc, } case TYPE_DECIMAL32: { StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - int32_t value = StringParser::string_to_decimal( + int32_t value = StringParser::string_to_decimal( data, len, type_desc.precision, type_desc.scale, &result); if (result != StringParser::PARSE_SUCCESS) { parse_result = StringParser::PARSE_FAILURE; @@ -266,7 +266,7 @@ bool TextConverter::_write_data(const TypeDescriptor& type_desc, } case TYPE_DECIMAL64: { StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - int64_t value = StringParser::string_to_decimal( + int64_t value = StringParser::string_to_decimal( data, len, type_desc.precision, type_desc.scale, &result); if (result != StringParser::PARSE_SUCCESS) { parse_result = StringParser::PARSE_FAILURE; @@ -279,9 +279,8 @@ bool TextConverter::_write_data(const TypeDescriptor& type_desc, } case TYPE_DECIMAL128I: { StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - vectorized::Int128 value = - StringParser::string_to_decimal( - data, len, type_desc.precision, type_desc.scale, &result); + vectorized::Int128 value = StringParser::string_to_decimal( + data, len, type_desc.precision, type_desc.scale, &result); if (result != StringParser::PARSE_SUCCESS) { parse_result = StringParser::PARSE_FAILURE; break; diff --git a/be/src/exprs/create_predicate_function.h b/be/src/exprs/create_predicate_function.h index 4dbd1be7e2..aefe8e3f03 100644 --- a/be/src/exprs/create_predicate_function.h +++ b/be/src/exprs/create_predicate_function.h @@ -49,13 +49,12 @@ public: using Set = std::conditional_t< std::is_same_v, StringSet<>, HybridSet::CppType, N>>>; + FixedContainer::CppType, N>>>; return new Set(); } else { using Set = std::conditional_t< std::is_same_v, StringSet<>, - HybridSet::CppType>>>; + HybridSet::CppType>>>; return new Set(); } } diff --git a/be/src/exprs/hybrid_set.h b/be/src/exprs/hybrid_set.h index c158b9ab52..8084d76ce2 100644 --- a/be/src/exprs/hybrid_set.h +++ b/be/src/exprs/hybrid_set.h @@ -261,8 +261,8 @@ bool check_hybrid_set(const HybridSetBase* column) { } template ::CppType>, - typename _ColumnType = typename VecPrimitiveTypeTraits::ColumnType> + typename _ContainerType = DynamicContainer::CppType>, + typename _ColumnType = typename PrimitiveTypeTraits::ColumnType> class HybridSet : public HybridSetBase { public: using ContainerType = _ContainerType; diff --git a/be/src/olap/key_coder.h b/be/src/olap/key_coder.h index 87c2748688..30d33cd3fa 100644 --- a/be/src/olap/key_coder.h +++ b/be/src/olap/key_coder.h @@ -34,6 +34,7 @@ #include "olap/olap_common.h" #include "olap/types.h" #include "util/slice.h" +#include "vec/core/types.h" namespace doris { @@ -80,8 +81,11 @@ template class KeyCoderTraits {}; template -class KeyCoderTraits::CppType>::value>::type> { +class KeyCoderTraits< + field_type, + typename std::enable_if< + std::is_integral::CppType>::value || + vectorized::IsDecimalNumber::CppType>>::type> { public: using CppType = typename CppTypeTraits::CppType; using UnsignedCppType = typename CppTypeTraits::UnsignedCppType; diff --git a/be/src/olap/predicate_creator.h b/be/src/olap/predicate_creator.h index 93702ac101..362484000e 100644 --- a/be/src/olap/predicate_creator.h +++ b/be/src/olap/predicate_creator.h @@ -96,8 +96,8 @@ private: static CppType convert(const TabletColumn& column, const std::string& condition) { StringParser::ParseResult result = StringParser::ParseResult::PARSE_SUCCESS; // return CppType value cast from int128_t - return StringParser::string_to_decimal( - condition.data(), condition.size(), column.precision(), column.frac(), &result); + return StringParser::string_to_decimal(condition.data(), condition.size(), + column.precision(), column.frac(), &result); } }; diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 03f29a6451..7a7eec9578 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -336,7 +336,6 @@ void ColumnReader::_parse_zone_map(const ZoneMapPB& zone_map, WrapperField* min_ max_value_container->from_string(zone_map.max()); } // for compatible original Cond eval logic - // TODO(hkp): optimize OlapCond if (zone_map.has_null()) { // for compatible, if exist null, original logic treat null as min min_value_container->set_null(); diff --git a/be/src/olap/types.h b/be/src/olap/types.h index edd925ea98..9ef6522524 100644 --- a/be/src/olap/types.h +++ b/be/src/olap/types.h @@ -798,11 +798,11 @@ struct BaseFieldtypeTraits : public CppTypeTraits { static inline void direct_copy_may_cut(void* dest, const void* src) { direct_copy(dest, src); } static inline void set_to_max(void* buf) { - set_cpp_type_value(buf, std::numeric_limits::max()); + set_cpp_type_value(buf, type_limit::max()); } static inline void set_to_min(void* buf) { - set_cpp_type_value(buf, std::numeric_limits::min()); + set_cpp_type_value(buf, type_limit::min()); } static std::string to_string(const void* src) { @@ -1027,24 +1027,16 @@ struct FieldTypeTraits static Status from_string(void* buf, const std::string& scan_key, const int precision, const int scale) { StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - int32_t value = StringParser::string_to_decimal( + int32_t value = StringParser::string_to_decimal( scan_key.c_str(), scan_key.size(), 9, scale, &result); if (result == StringParser::PARSE_FAILURE) { return Status::Error( "FieldTypeTraits::from_string meet PARSE_FAILURE"); } - *reinterpret_cast(buf) = (int32_t)value; + *reinterpret_cast(buf) = value; return Status::OK(); } - static void set_to_max(void* buf) { - CppType* data = reinterpret_cast(buf); - *data = 999999999; - } - static void set_to_min(void* buf) { - CppType* data = reinterpret_cast(buf); - *data = -999999999; - } }; template <> @@ -1053,23 +1045,15 @@ struct FieldTypeTraits static Status from_string(void* buf, const std::string& scan_key, const int precision, const int scale) { StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - int64_t value = StringParser::string_to_decimal( + int64_t value = StringParser::string_to_decimal( scan_key.c_str(), scan_key.size(), 18, scale, &result); if (result == StringParser::PARSE_FAILURE) { return Status::Error( "FieldTypeTraits::from_string meet PARSE_FAILURE"); } - *reinterpret_cast(buf) = (int64_t)value; + *reinterpret_cast(buf) = value; return Status::OK(); } - static void set_to_max(void* buf) { - CppType* data = reinterpret_cast(buf); - *data = 999999999999999999ll; - } - static void set_to_min(void* buf) { - CppType* data = reinterpret_cast(buf); - *data = -999999999999999999ll; - } }; template <> @@ -1078,7 +1062,7 @@ struct FieldTypeTraits static Status from_string(void* buf, const std::string& scan_key, const int precision, const int scale) { StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - int128_t value = StringParser::string_to_decimal( + int128_t value = StringParser::string_to_decimal( scan_key.c_str(), scan_key.size(), 38, scale, &result); if (result == StringParser::PARSE_FAILURE) { return Status::Error( @@ -1093,16 +1077,6 @@ struct FieldTypeTraits fmt::format_to(buffer, "{}", value); return std::string(buffer.data(), buffer.size()); } - static void set_to_max(void* buf) { - *reinterpret_cast(buf) = - static_cast(999999999999999999ll) * 100000000000000000ll * 1000ll + - static_cast(99999999999999999ll) * 1000ll + 999ll; - } - static void set_to_min(void* buf) { - *reinterpret_cast(buf) = - -(static_cast(999999999999999999ll) * 100000000000000000ll * 1000ll + - static_cast(99999999999999999ll) * 1000ll + 999ll); - } }; template <> diff --git a/be/src/runtime/decimalv2_value.cpp b/be/src/runtime/decimalv2_value.cpp index ca50b2d643..6155372ae6 100644 --- a/be/src/runtime/decimalv2_value.cpp +++ b/be/src/runtime/decimalv2_value.cpp @@ -356,8 +356,8 @@ int DecimalV2Value::parse_from_str(const char* decimal_str, int32_t length) { int32_t error = E_DEC_OK; StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - _value = StringParser::string_to_decimal(decimal_str, length, - PRECISION, SCALE, &result); + _value = StringParser::string_to_decimal(decimal_str, length, PRECISION, SCALE, + &result); if (!config::allow_invalid_decimalv2_literal && result != StringParser::PARSE_SUCCESS) { error = E_DEC_BAD_NUM; } else if (config::allow_invalid_decimalv2_literal && result == StringParser::PARSE_FAILURE) { diff --git a/be/src/runtime/decimalv2_value.h b/be/src/runtime/decimalv2_value.h index daf72d3e24..6472ad4275 100644 --- a/be/src/runtime/decimalv2_value.h +++ b/be/src/runtime/decimalv2_value.h @@ -49,6 +49,7 @@ enum DecimalRoundMode { HALF_UP = 1, HALF_EVEN = 2, CEILING = 3, FLOOR = 4, TRUN class DecimalV2Value { public: + using NativeType = __int128_t; friend DecimalV2Value operator+(const DecimalV2Value& v1, const DecimalV2Value& v2); friend DecimalV2Value operator-(const DecimalV2Value& v1, const DecimalV2Value& v2); friend DecimalV2Value operator*(const DecimalV2Value& v1, const DecimalV2Value& v2); diff --git a/be/src/runtime/primitive_type.h b/be/src/runtime/primitive_type.h index a3745f8ce8..5a4c05ef40 100644 --- a/be/src/runtime/primitive_type.h +++ b/be/src/runtime/primitive_type.h @@ -187,17 +187,17 @@ struct PrimitiveTypeTraits { }; template <> struct PrimitiveTypeTraits { - using CppType = int32_t; + using CppType = vectorized::Decimal32; using ColumnType = vectorized::ColumnDecimal; }; template <> struct PrimitiveTypeTraits { - using CppType = int64_t; + using CppType = vectorized::Decimal64; using ColumnType = vectorized::ColumnDecimal; }; template <> struct PrimitiveTypeTraits { - using CppType = __int128_t; + using CppType = vectorized::Decimal128I; using ColumnType = vectorized::ColumnDecimal; }; template <> @@ -265,25 +265,6 @@ struct PredicatePrimitiveTypeTraits { using PredicateFieldType = uint64_t; }; -// used for VInPredicate. VInPredicate should use vectorized data type -template -struct VecPrimitiveTypeTraits { - using CppType = typename PrimitiveTypeTraits::CppType; - using ColumnType = typename PrimitiveTypeTraits::ColumnType; -}; - -template <> -struct VecPrimitiveTypeTraits { - using CppType = vectorized::VecDateTimeValue; - using ColumnType = vectorized::ColumnVector; -}; - -template <> -struct VecPrimitiveTypeTraits { - using CppType = vectorized::VecDateTimeValue; - using ColumnType = vectorized::ColumnVector; -}; - template concept HaveCppType = requires() { sizeof(typename Traits::CppType); }; diff --git a/be/src/runtime/type_limit.h b/be/src/runtime/type_limit.h index dad1846bff..4d9fd5e646 100644 --- a/be/src/runtime/type_limit.h +++ b/be/src/runtime/type_limit.h @@ -43,26 +43,32 @@ struct type_limit { template <> struct type_limit { - static vectorized::Int32 min() { return -999999999; } - static vectorized::Int32 max() { return 999999999; } + static vectorized::Decimal32 max() { return 999999999; } + static vectorized::Decimal32 min() { return -max(); } }; template <> struct type_limit { - static vectorized::Int64 min() { return -999999999999999999; } - static vectorized::Int64 max() { return 999999999999999999; } + static vectorized::Decimal64 max() { return int64_t(999999999999999999ll); } + static vectorized::Decimal64 min() { return -max(); } }; template <> struct type_limit { - static vectorized::Int128 min() { - return -(static_cast(999999999999999999ll) * 100000000000000000ll * 1000ll + - static_cast(99999999999999999ll) * 1000ll + 999ll); + static vectorized::Decimal128I max() { + return (static_cast(999999999999999999ll) * 100000000000000000ll * 1000ll + + static_cast(99999999999999999ll) * 1000ll + 999ll); } - static vectorized::Int128 max() { - return static_cast(999999999999999999ll) * 100000000000000000ll * 1000ll + - static_cast(99999999999999999ll) * 1000ll + 999ll; + static vectorized::Decimal128I min() { return -max(); } +}; + +template <> +struct type_limit { + static vectorized::Decimal128 max() { + return (static_cast(999999999999999999ll) * 100000000000000000ll * 1000ll + + static_cast(99999999999999999ll) * 1000ll + 999ll); } + static vectorized::Decimal128 min() { return -max(); } }; template <> diff --git a/be/src/util/string_parser.hpp b/be/src/util/string_parser.hpp index a59d6bf7de..08990b2de0 100644 --- a/be/src/util/string_parser.hpp +++ b/be/src/util/string_parser.hpp @@ -39,12 +39,12 @@ #include "common/compiler_util.h" // IWYU pragma: keep #include "common/status.h" #include "runtime/large_int_value.h" +#include "runtime/primitive_type.h" #include "vec/common/int_exp.h" #include "vec/data_types/data_type_decimal.h" namespace doris { namespace vectorized { -struct Int128I; template struct Decimal; } // namespace vectorized @@ -158,7 +158,7 @@ public: return string_to_bool_internal(s + i, len - i, result); } - template + template ::CppType::NativeType> static inline T string_to_decimal(const char* s, int len, int type_precision, int type_scale, ParseResult* result); @@ -642,10 +642,8 @@ T StringParser::string_to_decimal(const char* s, int len, int type_precision, in value = (value * 10) + (c - '0'); // Benchmarks are faster with parenthesis... } else { *result = StringParser::PARSE_OVERFLOW; - value = is_negative ? vectorized::min_decimal_value>( - type_precision) - : vectorized::max_decimal_value>( - type_precision); + value = is_negative ? type_limit::min() + : type_limit::max(); return value; } DCHECK(value >= 0); // For some reason //DCHECK_GE doesn't work with __int128. @@ -669,11 +667,7 @@ T StringParser::string_to_decimal(const char* s, int len, int type_precision, in return 0; } *result = StringParser::PARSE_SUCCESS; - if constexpr (std::is_same_v) { - value *= get_scale_multiplier<__int128>(type_scale - scale); - } else { - value *= get_scale_multiplier(type_scale - scale); - } + value *= get_scale_multiplier(type_scale - scale); return is_negative ? T(-value) : T(value); } @@ -695,10 +689,8 @@ T StringParser::string_to_decimal(const char* s, int len, int type_precision, in if (!found_dot && max_digit < (precision - scale)) { // parse_overflow should only happen when the digit part reached the max *result = StringParser::PARSE_OVERFLOW; - value = is_negative ? vectorized::min_decimal_value>( - type_precision) - : vectorized::max_decimal_value>( - type_precision); + value = is_negative ? type_limit>::min() + : type_limit>::max(); return value; } // keep a rounding precision to round the decimal value @@ -722,11 +714,7 @@ T StringParser::string_to_decimal(const char* s, int len, int type_precision, in return 0; } *result = StringParser::PARSE_SUCCESS; - if constexpr (std::is_same_v) { - value *= get_scale_multiplier<__int128>(type_scale - scale); - } else { - value *= get_scale_multiplier(type_scale - scale); - } + value *= get_scale_multiplier(type_scale - scale); return is_negative ? T(-value) : T(value); } @@ -739,11 +727,7 @@ T StringParser::string_to_decimal(const char* s, int len, int type_precision, in // scale must be set to 0 and the value set to 100 which means a precision of 3. precision += exponent - scale; - if constexpr (std::is_same_v) { - value *= get_scale_multiplier<__int128>(exponent - scale); - } else { - value *= get_scale_multiplier(exponent - scale); - } + value *= get_scale_multiplier(exponent - scale); scale = 0; } else { // Ex: 100e-4, the scale must be set to 4 but no adjustment to the value is needed, @@ -766,12 +750,7 @@ T StringParser::string_to_decimal(const char* s, int len, int type_precision, in *result = StringParser::PARSE_UNDERFLOW; int shift = scale - type_scale; if (shift > 0) { - T divisor; - if constexpr (std::is_same_v) { - divisor = get_scale_multiplier<__int128>(shift); - } else { - divisor = get_scale_multiplier(shift); - } + T divisor = get_scale_multiplier(shift); if (LIKELY(divisor > 0)) { T remainder = value % divisor; value /= divisor; @@ -789,11 +768,7 @@ T StringParser::string_to_decimal(const char* s, int len, int type_precision, in } if (type_scale > scale) { - if constexpr (std::is_same_v) { - value *= get_scale_multiplier<__int128>(type_scale - scale); - } else { - value *= get_scale_multiplier(type_scale - scale); - } + value *= get_scale_multiplier(type_scale - scale); } return is_negative ? T(-value) : T(value); diff --git a/be/src/vec/aggregate_functions/aggregate_function_avg.h b/be/src/vec/aggregate_functions/aggregate_function_avg.h index c30c6db6b5..88704a96fd 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_avg.h +++ b/be/src/vec/aggregate_functions/aggregate_function_avg.h @@ -58,7 +58,7 @@ namespace doris::vectorized { template struct AggregateFunctionAvgData { - T sum = 0; + T sum {}; UInt64 count = 0; AggregateFunctionAvgData& operator=(const AggregateFunctionAvgData& src) { @@ -145,7 +145,7 @@ public: } void reset(AggregateDataPtr place) const override { - this->data(place).sum = 0; + this->data(place).sum = {}; this->data(place).count = 0; } diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_max.h b/be/src/vec/aggregate_functions/aggregate_function_min_max.h index dcc58204f8..794b169434 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_min_max.h +++ b/be/src/vec/aggregate_functions/aggregate_function_min_max.h @@ -175,11 +175,10 @@ template struct SingleValueDataDecimal { private: using Self = SingleValueDataDecimal; - using Type = typename NativeType::Type; bool has_value = false; /// We need to remember if at least one value has been passed. This is necessary for AggregateFunctionIf. - Type value; + T value; public: SingleValueDataDecimal() = default; diff --git a/be/src/vec/aggregate_functions/aggregate_function_product.h b/be/src/vec/aggregate_functions/aggregate_function_product.h index ba3f74d626..ca1c7c9fae 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_product.h +++ b/be/src/vec/aggregate_functions/aggregate_function_product.h @@ -98,7 +98,7 @@ struct AggregateFunctionProductData { Decimal128 get() const { return product; } - void reset(Decimal128 value) { product = std::move(value); } + void reset(Decimal128 value) { product = value; } }; template @@ -139,7 +139,7 @@ public: void reset(AggregateDataPtr place) const override { if constexpr (IsDecimalNumber) { - this->data(place).reset(T(1 * multiplier)); + this->data(place).reset(multiplier); } else { this->data(place).reset(1); } diff --git a/be/src/vec/columns/column_decimal.cpp b/be/src/vec/columns/column_decimal.cpp index 2c98e1193b..7c9a8cf084 100644 --- a/be/src/vec/columns/column_decimal.cpp +++ b/be/src/vec/columns/column_decimal.cpp @@ -454,8 +454,8 @@ void ColumnDecimal::replicate(const uint32_t* __restrict indexs, size_t targe template void ColumnDecimal::get_extremes(Field& min, Field& max) const { if (data.size() == 0) { - min = NearestFieldType(0, scale); - max = NearestFieldType(0, scale); + min = NearestFieldType(T(), scale); + max = NearestFieldType(T(), scale); return; } diff --git a/be/src/vec/columns/column_decimal.h b/be/src/vec/columns/column_decimal.h index aae9ab94d1..87a8cddc99 100644 --- a/be/src/vec/columns/column_decimal.h +++ b/be/src/vec/columns/column_decimal.h @@ -204,7 +204,7 @@ public: bool get_bool(size_t n) const override { return bool(data[n]); } Int64 get_int(size_t n) const override { return Int64(data[n] * scale); } UInt64 get64(size_t n) const override; - bool is_default_at(size_t n) const override { return data[n] == 0; } + bool is_default_at(size_t n) const override { return data[n].value == 0; } void clear() override { data.clear(); } diff --git a/be/src/vec/columns/predicate_column.h b/be/src/vec/columns/predicate_column.h index 2cbf871804..f9acd7af63 100644 --- a/be/src/vec/columns/predicate_column.h +++ b/be/src/vec/columns/predicate_column.h @@ -21,6 +21,7 @@ #include "olap/decimal12.h" #include "olap/uint24.h" +#include "runtime/define_primitive_type.h" #include "runtime/primitive_type.h" #include "vec/columns/column.h" #include "vec/columns/column_decimal.h" @@ -44,6 +45,7 @@ private: PredicateColumnType(const size_t n) : data(n) {} friend class COWHelper>; using T = typename PredicatePrimitiveTypeTraits::PredicateFieldType; + using ColumnType = typename PrimitiveTypeTraits::ColumnType; PredicateColumnType(const PredicateColumnType& src) : data(src.data.begin(), src.data.end()) {} @@ -60,15 +62,7 @@ private: } void insert_date_to_res_column(const uint16_t* sel, size_t sel_size, - vectorized::ColumnVector* res_ptr) { - for (size_t i = 0; i < sel_size; i++) { - VecDateTimeValue date = VecDateTimeValue::create_from_olap_date(get_date_at(sel[i])); - res_ptr->insert_data(reinterpret_cast(&date), 0); - } - } - - void insert_date32_to_res_column(const uint16_t* sel, size_t sel_size, - vectorized::ColumnVector* res_ptr) { + ColumnVector* res_ptr) { res_ptr->reserve(sel_size); auto& res_data = res_ptr->get_data(); @@ -82,17 +76,15 @@ private: } void insert_datetime_to_res_column(const uint16_t* sel, size_t sel_size, - vectorized::ColumnVector* res_ptr) { + ColumnVector* res_ptr) { for (size_t i = 0; i < sel_size; i++) { uint64_t value = data[sel[i]]; - vectorized::VecDateTimeValue datetime = - VecDateTimeValue::create_from_olap_datetime(value); + VecDateTimeValue datetime = VecDateTimeValue::create_from_olap_datetime(value); res_ptr->insert_data(reinterpret_cast(&datetime), 0); } } - void insert_string_to_res_column(const uint16_t* sel, size_t sel_size, - vectorized::ColumnString* res_ptr) { + void insert_string_to_res_column(const uint16_t* sel, size_t sel_size, ColumnString* res_ptr) { StringRef refs[sel_size]; size_t length = 0; for (size_t i = 0; i < sel_size; i++) { @@ -108,7 +100,7 @@ private: } void insert_decimal_to_res_column(const uint16_t* sel, size_t sel_size, - vectorized::ColumnDecimal* res_ptr) { + ColumnDecimal* res_ptr) { for (size_t i = 0; i < sel_size; i++) { uint16_t n = sel[i]; auto& dv = reinterpret_cast(data[n]); @@ -117,9 +109,9 @@ private: } } - template + template typename ColumnContainer> void insert_default_value_res_column(const uint16_t* sel, size_t sel_size, - vectorized::ColumnVector* res_ptr) { + ColumnContainer* res_ptr) { static_assert(std::is_same_v); auto& res_data = res_ptr->get_data(); DCHECK(res_data.empty()); @@ -131,8 +123,7 @@ private: res_data.set_end_ptr(y + sel_size); } - void insert_byte_to_res_column(const uint16_t* sel, size_t sel_size, - vectorized::IColumn* res_ptr) { + void insert_byte_to_res_column(const uint16_t* sel, size_t sel_size, IColumn* res_ptr) { for (size_t i = 0; i < sel_size; i++) { uint16_t n = sel[i]; char* ch_val = reinterpret_cast(&data[n]); @@ -226,7 +217,7 @@ public: insert_string_value(data_ptr, length); } else if constexpr (std::is_same_v) { insert_decimal_value(data_ptr, length); - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { insert_in_copy_way(data_ptr, length); } else { insert_default_type(data_ptr, length); @@ -251,7 +242,7 @@ public: void insert_many_fix_len_data(const char* data_ptr, size_t num) override { if constexpr (std::is_same_v) { insert_many_in_copy_way(data_ptr, num); - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { insert_many_in_copy_way(data_ptr, num); } else if constexpr (std::is_same_v) { // here is unreachable, just for compilation to be able to pass @@ -466,74 +457,32 @@ public: } Status filter_by_selector(const uint16_t* sel, size_t sel_size, IColumn* col_ptr) override { - if constexpr (std::is_same_v) { - insert_string_to_res_column(sel, sel_size, - reinterpret_cast(col_ptr)); + ColumnType* column = assert_cast(col_ptr); + if constexpr (std::is_same_v, ColumnType>) { + insert_default_value_res_column(sel, sel_size, column); + } else if constexpr (std::is_same_v, ColumnType>) { + insert_default_value_res_column(sel, sel_size, column); + } else if constexpr (std::is_same_v) { + insert_string_to_res_column(sel, sel_size, column); } else if constexpr (std::is_same_v) { - insert_decimal_to_res_column( - sel, sel_size, - reinterpret_cast*>(col_ptr)); - } else if constexpr (std::is_same_v) { - insert_default_value_res_column( - sel, sel_size, - reinterpret_cast*>(col_ptr)); - } else if constexpr (std::is_same_v) { - insert_default_value_res_column( - sel, sel_size, - reinterpret_cast*>(col_ptr)); - } else if constexpr (std::is_same_v) { - insert_default_value_res_column( - sel, sel_size, - reinterpret_cast*>(col_ptr)); - } else if constexpr (std::is_same_v) { - insert_default_value_res_column( - sel, sel_size, - reinterpret_cast*>(col_ptr)); - } else if constexpr (std::is_same_v) { - insert_default_value_res_column( - sel, sel_size, - reinterpret_cast*>( - col_ptr)); - } else if constexpr (std::is_same_v) { - insert_default_value_res_column( - sel, sel_size, - reinterpret_cast*>( - col_ptr)); + insert_decimal_to_res_column(sel, sel_size, column); } else if constexpr (std::is_same_v) { - if (const vectorized::ColumnVector* date_col = - check_and_get_column>( - const_cast(col_ptr))) { - insert_default_value_res_column( - sel, sel_size, const_cast*>(date_col)); + if constexpr (Type == TYPE_DATETIMEV2) { + insert_default_value_res_column(sel, sel_size, column); } else { - insert_datetime_to_res_column( - sel, sel_size, reinterpret_cast*>(col_ptr)); + insert_datetime_to_res_column(sel, sel_size, column); } - } else if constexpr (std::is_same_v) { - insert_date_to_res_column(sel, sel_size, - reinterpret_cast*>(col_ptr)); } else if constexpr (std::is_same_v) { - if (const vectorized::ColumnVector* date_col = - check_and_get_column>( - const_cast(col_ptr))) { - // a trick type judge, need refactor it. - insert_date32_to_res_column(sel, sel_size, - const_cast*>(date_col)); + if constexpr (Type == TYPE_DATEV2) { + insert_default_value_res_column(sel, sel_size, column); } else { - insert_default_value_res_column( - sel, sel_size, - reinterpret_cast*>( - col_ptr)); + insert_date_to_res_column(sel, sel_size, column); } - } else if constexpr (std::is_same_v) { - insert_default_value_res_column( - sel, sel_size, - reinterpret_cast*>( - col_ptr)); } else if (std::is_same_v) { insert_byte_to_res_column(sel, sel_size, col_ptr); } else { - return Status::NotSupported("not supported output type in predicate_column"); + return Status::NotSupported("not supported output type in predicate_column, type={}", + type_to_string(Type)); } return Status::OK(); } diff --git a/be/src/vec/common/field_visitors.h b/be/src/vec/common/field_visitors.h index 7a37499ff6..d33dbce570 100644 --- a/be/src/vec/common/field_visitors.h +++ b/be/src/vec/common/field_visitors.h @@ -24,12 +24,8 @@ #include "vec/core/accurate_comparison.h" #include "vec/core/field.h" -class SipHash; - namespace doris::vectorized { -UInt128 string_to_uuid(const String&); - /** StaticVisitor (and its descendants) - class with overloaded operator() for all types of fields. * You could call visitor for field using function 'apply_visitor'. * Also "binary visitor" is supported - its operator() takes two arguments. @@ -131,402 +127,4 @@ typename std::decay_t::ResultType apply_visitor(Visitor&& visitor, F1&& } } -/** Prints Field as literal in SQL query */ -class FieldVisitorToString : public StaticVisitor { -public: - String operator()(const Null& x) const; - String operator()(const UInt64& x) const; - String operator()(const UInt128& x) const; - String operator()(const Int64& x) const; - String operator()(const Float64& x) const; - String operator()(const String& x) const; - String operator()(const Array& x) const; - String operator()(const Tuple& x) const; - String operator()(const DecimalField& x) const; - String operator()(const DecimalField& x) const; - String operator()(const DecimalField& x) const; - String operator()(const DecimalField& x) const; - String operator()(const AggregateFunctionStateData& x) const; -}; - -/** Print readable and unique text dump of field type and value. */ -class FieldVisitorDump : public StaticVisitor { -public: - String operator()(const Null& x) const; - String operator()(const UInt64& x) const; - String operator()(const UInt128& x) const; - String operator()(const Int64& x) const; - String operator()(const Float64& x) const; - String operator()(const String& x) const; - String operator()(const Array& x) const; - String operator()(const Tuple& x) const; - String operator()(const DecimalField& x) const; - String operator()(const DecimalField& x) const; - String operator()(const DecimalField& x) const; - String operator()(const DecimalField& x) const; - String operator()(const AggregateFunctionStateData& x) const; -}; - -/** Converts numeric value of any type to specified type. */ -template -class FieldVisitorConvertToNumber : public StaticVisitor { -public: - T operator()(const Null&) const { - LOG(FATAL) << "Cannot convert NULL to " << demangle(typeid(T).name()); - return {}; - } - - T operator()(const String&) const { - LOG(FATAL) << "Cannot convert String to " << demangle(typeid(T).name()); - return {}; - } - - T operator()(const Array&) const { - LOG(FATAL) << "Cannot convert Array to " << demangle(typeid(T).name()); - return {}; - } - - T operator()(const Tuple&) const { - LOG(FATAL) << "Cannot convert Tuple to " << demangle(typeid(T).name()); - return {}; - } - - T operator()(const UInt64& x) const { return x; } - T operator()(const Int64& x) const { return x; } - T operator()(const Float64& x) const { return x; } - - T operator()(const UInt128&) const { - LOG(FATAL) << "Cannot convert UInt128 to " << demangle(typeid(T).name()); - return {}; - } - - template - T operator()(const DecimalField& x) const { - if constexpr (std::is_floating_point_v) - return static_cast(x.get_value()) / x.get_scale_multiplier(); - else { - return x.get_value() / x.get_scale_multiplier(); - } - } - - T operator()(const AggregateFunctionStateData&) const { - LOG(FATAL) << "Cannot convert AggregateFunctionStateData to " << demangle(typeid(T).name()); - return {}; - } -}; - -/** Updates SipHash by type and value of Field */ -class FieldVisitorHash : public StaticVisitor<> { -private: - SipHash& hash; - -public: - FieldVisitorHash(SipHash& hash_); - - void operator()(const Null& x) const; - void operator()(const UInt64& x) const; - void operator()(const UInt128& x) const; - void operator()(const Int64& x) const; - void operator()(const Float64& x) const; - void operator()(const String& x) const; - void operator()(const Array& x) const; - void operator()(const Tuple& x) const; - void operator()(const DecimalField& x) const; - void operator()(const DecimalField& x) const; - void operator()(const DecimalField& x) const; - void operator()(const DecimalField& x) const; - void operator()(const AggregateFunctionStateData& x) const; -}; - -template -constexpr bool is_decimalField() { - return false; -} -template <> -constexpr bool is_decimalField>() { - return true; -} -template <> -constexpr bool is_decimalField>() { - return true; -} -template <> -constexpr bool is_decimalField>() { - return true; -} -template <> -constexpr bool is_decimalField>() { - return true; -} - -/** More precise comparison, used for index. - * Differs from Field::operator< and Field::operator== in that it also compares values of different types. - * Comparison rules are same as in FunctionsComparison (to be consistent with expression evaluation in query). - */ -class FieldVisitorAccurateEquals : public StaticVisitor { -public: - bool operator()(const UInt64& l, const Null& r) const { return cant_compare(l, r); } - bool operator()(const UInt64& l, const UInt64& r) const { return l == r; } - bool operator()(const UInt64& l, const UInt128& r) const { return cant_compare(l, r); } - bool operator()(const UInt64& l, const Int64& r) const { return accurate::equalsOp(l, r); } - bool operator()(const UInt64& l, const Float64& r) const { return accurate::equalsOp(l, r); } - bool operator()(const UInt64& l, const String& r) const { return cant_compare(l, r); } - bool operator()(const UInt64& l, const Array& r) const { return cant_compare(l, r); } - bool operator()(const UInt64& l, const Tuple& r) const { return cant_compare(l, r); } - bool operator()(const UInt64& l, const AggregateFunctionStateData& r) const { - return cant_compare(l, r); - } - - bool operator()(const Int64& l, const Null& r) const { return cant_compare(l, r); } - bool operator()(const Int64& l, const UInt64& r) const { return accurate::equalsOp(l, r); } - bool operator()(const Int64& l, const UInt128& r) const { return cant_compare(l, r); } - bool operator()(const Int64& l, const Int64& r) const { return l == r; } - bool operator()(const Int64& l, const Float64& r) const { return accurate::equalsOp(l, r); } - bool operator()(const Int64& l, const String& r) const { return cant_compare(l, r); } - bool operator()(const Int64& l, const Array& r) const { return cant_compare(l, r); } - bool operator()(const Int64& l, const Tuple& r) const { return cant_compare(l, r); } - bool operator()(const Int64& l, const AggregateFunctionStateData& r) const { - return cant_compare(l, r); - } - - bool operator()(const Float64& l, const Null& r) const { return cant_compare(l, r); } - bool operator()(const Float64& l, const UInt64& r) const { return accurate::equalsOp(l, r); } - bool operator()(const Float64& l, const UInt128& r) const { return cant_compare(l, r); } - bool operator()(const Float64& l, const Int64& r) const { return accurate::equalsOp(l, r); } - bool operator()(const Float64& l, const Float64& r) const { return l == r; } - bool operator()(const Float64& l, const String& r) const { return cant_compare(l, r); } - bool operator()(const Float64& l, const Array& r) const { return cant_compare(l, r); } - bool operator()(const Float64& l, const Tuple& r) const { return cant_compare(l, r); } - bool operator()(const Float64& l, const AggregateFunctionStateData& r) const { - return cant_compare(l, r); - } - - template - bool operator()(const Null&, const T&) const { - return std::is_same_v; - } - - template - bool operator()(const String& l, const T& r) const { - if constexpr (std::is_same_v) return l == r; - if constexpr (std::is_same_v) return string_to_uuid(l) == r; - return cant_compare(l, r); - } - - template - bool operator()(const UInt128& l, const T& r) const { - if constexpr (std::is_same_v) return l == r; - if constexpr (std::is_same_v) return l == string_to_uuid(r); - return cant_compare(l, r); - } - - template - bool operator()(const Array& l, const T& r) const { - if constexpr (std::is_same_v) return l == r; - return cant_compare(l, r); - } - - template - bool operator()(const Tuple& l, const T& r) const { - if constexpr (std::is_same_v) return l == r; - return cant_compare(l, r); - } - - template - bool operator()(const DecimalField& l, const U& r) const { - if constexpr (is_decimalField()) return l == r; - if constexpr (std::is_same_v || std::is_same_v) - return l == DecimalField(r, 0); - return cant_compare(l, r); - } - - template - bool operator()(const UInt64& l, const DecimalField& r) const { - return DecimalField(l, 0) == r; - } - template - bool operator()(const Int64& l, const DecimalField& r) const { - return DecimalField(l, 0) == r; - } - template - bool operator()(const Float64& l, const DecimalField& r) const { - return cant_compare(l, r); - } - - template - bool operator()(const AggregateFunctionStateData& l, const T& r) const { - if constexpr (std::is_same_v) return l == r; - return cant_compare(l, r); - } - -private: - template - bool cant_compare(const T&, const U&) const { - if constexpr (std::is_same_v) return false; - LOG(FATAL) << fmt::format("Cannot compare {} with {}", demangle(typeid(T).name()), - demangle(typeid(U).name())); - } -}; - -class FieldVisitorAccurateLess : public StaticVisitor { -public: - bool operator()(const UInt64& l, const Null& r) const { return cant_compare(l, r); } - bool operator()(const UInt64& l, const UInt64& r) const { return l < r; } - bool operator()(const UInt64& l, const UInt128& r) const { return cant_compare(l, r); } - bool operator()(const UInt64& l, const Int64& r) const { return accurate::lessOp(l, r); } - bool operator()(const UInt64& l, const Float64& r) const { return accurate::lessOp(l, r); } - bool operator()(const UInt64& l, const String& r) const { return cant_compare(l, r); } - bool operator()(const UInt64& l, const Array& r) const { return cant_compare(l, r); } - bool operator()(const UInt64& l, const Tuple& r) const { return cant_compare(l, r); } - bool operator()(const UInt64& l, const AggregateFunctionStateData& r) const { - return cant_compare(l, r); - } - - bool operator()(const Int64& l, const Null& r) const { return cant_compare(l, r); } - bool operator()(const Int64& l, const UInt64& r) const { return accurate::lessOp(l, r); } - bool operator()(const Int64& l, const UInt128& r) const { return cant_compare(l, r); } - bool operator()(const Int64& l, const Int64& r) const { return l < r; } - bool operator()(const Int64& l, const Float64& r) const { return accurate::lessOp(l, r); } - bool operator()(const Int64& l, const String& r) const { return cant_compare(l, r); } - bool operator()(const Int64& l, const Array& r) const { return cant_compare(l, r); } - bool operator()(const Int64& l, const Tuple& r) const { return cant_compare(l, r); } - bool operator()(const Int64& l, const AggregateFunctionStateData& r) const { - return cant_compare(l, r); - } - - bool operator()(const Float64& l, const Null& r) const { return cant_compare(l, r); } - bool operator()(const Float64& l, const UInt64& r) const { return accurate::lessOp(l, r); } - bool operator()(const Float64& l, const UInt128& r) const { return cant_compare(l, r); } - bool operator()(const Float64& l, const Int64& r) const { return accurate::lessOp(l, r); } - bool operator()(const Float64& l, const Float64& r) const { return l < r; } - bool operator()(const Float64& l, const String& r) const { return cant_compare(l, r); } - bool operator()(const Float64& l, const Array& r) const { return cant_compare(l, r); } - bool operator()(const Float64& l, const Tuple& r) const { return cant_compare(l, r); } - bool operator()(const Float64& l, const AggregateFunctionStateData& r) const { - return cant_compare(l, r); - } - - template - bool operator()(const Null&, const T&) const { - return !std::is_same_v; - } - - template - bool operator()(const String& l, const T& r) const { - if constexpr (std::is_same_v) return l < r; - if constexpr (std::is_same_v) return string_to_uuid(l) < r; - return cant_compare(l, r); - } - - template - bool operator()(const UInt128& l, const T& r) const { - if constexpr (std::is_same_v) return l < r; - if constexpr (std::is_same_v) return l < string_to_uuid(r); - return cant_compare(l, r); - } - - template - bool operator()(const Array& l, const T& r) const { - if constexpr (std::is_same_v) return l < r; - return cant_compare(l, r); - } - - template - bool operator()(const Tuple& l, const T& r) const { - if constexpr (std::is_same_v) return l < r; - return cant_compare(l, r); - } - - template - bool operator()(const DecimalField& l, const U& r) const { - if constexpr (is_decimalField()) - return l < r; - else if constexpr (std::is_same_v || std::is_same_v) - return l < DecimalField(r, 0); - return cant_compare(l, r); - } - - template - bool operator()(const UInt64& l, const DecimalField& r) const { - return DecimalField(l, 0) < r; - } - template - bool operator()(const Int64& l, const DecimalField& r) const { - return DecimalField(l, 0) < r; - } - template - bool operator()(const Float64&, const DecimalField&) const { - return false; - } - - template - bool operator()(const AggregateFunctionStateData& l, const T& r) const { - return cant_compare(l, r); - } - -private: - template - bool cant_compare(const T&, const U&) const { - LOG(FATAL) << fmt::format("Cannot compare {} with {}", demangle(typeid(T).name()), - demangle(typeid(U).name())); - return false; - } -}; - -/** Implements `+=` operation. - * Returns false if the result is zero. - */ -class FieldVisitorSum : public StaticVisitor { -private: - const Field& rhs; - -public: - explicit FieldVisitorSum(const Field& rhs_) : rhs(rhs_) {} - - bool operator()(UInt64& x) const { - x += get(rhs); - return x != 0; - } - bool operator()(Int64& x) const { - x += get(rhs); - return x != 0; - } - bool operator()(Float64& x) const { - x += get(rhs); - return x != 0; - } - - bool operator()(Null&) const { - LOG(FATAL) << "Cannot sum Nulls"; - return false; - } - - bool operator()(String&) const { - LOG(FATAL) << "Cannot sum Strings"; - return false; - } - bool operator()(Array&) const { - LOG(FATAL) << "Cannot sum Arrays"; - return false; - } - bool operator()(Tuple&) const { - LOG(FATAL) << "Cannot sum Tuples"; - return false; - } - bool operator()(UInt128&) const { - LOG(FATAL) << "Cannot sum UUIDs"; - return false; - } - bool operator()(AggregateFunctionStateData&) const { - LOG(FATAL) << "Cannot sum AggregateFunctionStates"; - return false; - } - - template - bool operator()(DecimalField& x) const { - x += get>(rhs); - return x.get_value() != 0; - } -}; - } // namespace doris::vectorized diff --git a/be/src/vec/common/hash_table/hash.h b/be/src/vec/common/hash_table/hash.h index 46df8a4c72..3c7df75b0a 100644 --- a/be/src/vec/common/hash_table/hash.h +++ b/be/src/vec/common/hash_table/hash.h @@ -91,13 +91,6 @@ struct DefaultHash { size_t operator()(T key) const { return default_hash64(key); } }; -template <> -struct DefaultHash { - size_t operator()(doris::vectorized::Int128I key) const { - return default_hash64(key); - } -}; - template <> struct DefaultHash : public doris::StringRefHash {}; diff --git a/be/src/vec/common/sip_hash.h b/be/src/vec/common/sip_hash.h index bc20e66357..0d69be721f 100644 --- a/be/src/vec/common/sip_hash.h +++ b/be/src/vec/common/sip_hash.h @@ -169,7 +169,6 @@ public: } } - /// NOTE: std::has_unique_object_representations is only available since clang 6. As of Mar 2017 we still use clang 5 sometimes. template requires std::is_standard_layout_v void update(const T& x) { diff --git a/be/src/vec/core/types.h b/be/src/vec/core/types.h index 4bec3619d8..1e88e32420 100644 --- a/be/src/vec/core/types.h +++ b/be/src/vec/core/types.h @@ -286,8 +286,6 @@ using DateTime = Int64; using DateV2 = UInt32; using DateTimeV2 = UInt64; -struct Int128I {}; - template inline constexpr T decimal_scale_multiplier(UInt32 scale); template <> @@ -380,6 +378,8 @@ struct Decimal { return *this; } + auto operator<=>(const Decimal& x) const { return value <=> x.value; } + static constexpr int max_string_length() { constexpr auto precision = std::is_same_v @@ -501,14 +501,13 @@ struct Decimal { T value; }; -template <> -struct Decimal : public Decimal { +struct Decimal128I : public Decimal { using NativeType = Int128; - Decimal() = default; + Decimal128I() = default; #define DECLARE_NUMERIC_CTOR(TYPE) \ - Decimal(const TYPE& value_) : Decimal(value_) {} + Decimal128I(const TYPE& value_) : Decimal(value_) {} DECLARE_NUMERIC_CTOR(Int128) DECLARE_NUMERIC_CTOR(Int32) @@ -520,7 +519,7 @@ struct Decimal : public Decimal { #undef DECLARE_NUMERIC_CTOR template - Decimal(const Decimal& x) { + Decimal128I(const Decimal& x) { value = x; } }; @@ -528,7 +527,6 @@ struct Decimal : public Decimal { using Decimal32 = Decimal; using Decimal64 = Decimal; using Decimal128 = Decimal; -using Decimal128I = Decimal; template <> struct TypeName { @@ -736,7 +734,7 @@ struct std::hash { template <> struct std::hash { - size_t operator()(const doris::vectorized::Decimal& x) const { + size_t operator()(const doris::vectorized::Decimal128I& x) const { return std::hash()(x.value >> 64) ^ std::hash()( x.value & std::numeric_limits::max()); diff --git a/be/src/vec/data_types/data_type_decimal.cpp b/be/src/vec/data_types/data_type_decimal.cpp index deb559f75b..41fc668db5 100644 --- a/be/src/vec/data_types/data_type_decimal.cpp +++ b/be/src/vec/data_types/data_type_decimal.cpp @@ -87,7 +87,7 @@ void DataTypeDecimal::to_string(const IColumn& column, size_t row_num, template Status DataTypeDecimal::from_string(ReadBuffer& rb, IColumn* column) const { auto& column_data = static_cast(*column).get_data(); - T val = 0; + T val {}; if (!read_decimal_text_impl::get_primitive_type(), T>( val, rb, precision, scale)) { return Status::InvalidArgument("parse decimal fail, string: '{}', primitive type: '{}'", @@ -142,7 +142,7 @@ void DataTypeDecimal::to_pb_column_meta(PColumnMeta* col_meta) const { template Field DataTypeDecimal::get_default() const { - return DecimalField(T(0), scale); + return DecimalField(T(), scale); } template MutableColumnPtr DataTypeDecimal::create_column() const { @@ -158,7 +158,7 @@ MutableColumnPtr DataTypeDecimal::create_column() const { template bool DataTypeDecimal::parse_from_string(const std::string& str, T* res) const { StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - *res = StringParser::string_to_decimal::get_primitive_type(), __int128>( + res->value = StringParser::string_to_decimal::get_primitive_type()>( str.c_str(), str.size(), precision, scale, &result); return result == StringParser::PARSE_SUCCESS; } @@ -210,49 +210,6 @@ Decimal128I DataTypeDecimal::get_scale_multiplier(UInt32 scale) { return common::exp10_i128(scale); } -template -typename T::NativeType max_decimal_value(UInt32 precision) { - return 0; -} -template <> -Int32 max_decimal_value(UInt32 precision) { - return 999999999 / DataTypeDecimal::get_scale_multiplier( - (UInt32)(max_decimal_precision() - precision)); -} -template <> -Int64 max_decimal_value(UInt32 precision) { - return 999999999999999999 / DataTypeDecimal::get_scale_multiplier( - (UInt64)max_decimal_precision() - precision); -} -template <> -Int128 max_decimal_value(UInt32 precision) { - return (static_cast(999999999999999999ll) * 100000000000000000ll * 1000ll + - static_cast(99999999999999999ll) * 1000ll + 999ll) / - DataTypeDecimal::get_scale_multiplier( - (UInt64)max_decimal_precision() - precision); -} - -template -typename T::NativeType min_decimal_value(UInt32 precision) { - return 0; -} -template <> -Int32 min_decimal_value(UInt32 precision) { - return -999999999 / DataTypeDecimal::get_scale_multiplier( - (UInt32)max_decimal_precision() - precision); -} -template <> -Int64 min_decimal_value(UInt32 precision) { - return -999999999999999999 / DataTypeDecimal::get_scale_multiplier( - (UInt64)max_decimal_precision() - precision); -} -template <> -Int128 min_decimal_value(UInt32 precision) { - return -(static_cast(999999999999999999ll) * 100000000000000000ll * 1000ll + - static_cast(99999999999999999ll) * 1000ll + 999ll) / - DataTypeDecimal::get_scale_multiplier( - (UInt64)max_decimal_precision() - precision); -} /// Explicit template instantiations. template class DataTypeDecimal; template class DataTypeDecimal; diff --git a/be/src/vec/data_types/data_type_decimal.h b/be/src/vec/data_types/data_type_decimal.h index 138f277c3f..bc2ac4ab68 100644 --- a/be/src/vec/data_types/data_type_decimal.h +++ b/be/src/vec/data_types/data_type_decimal.h @@ -38,6 +38,7 @@ #include "common/status.h" #include "olap/olap_common.h" #include "runtime/define_primitive_type.h" +#include "runtime/type_limit.h" #include "serde/data_type_decimal_serde.h" #include "util/binary_cast.hpp" #include "vec/columns/column_decimal.h" @@ -92,24 +93,6 @@ template constexpr typename T::NativeType max_decimal_value() { return 0; } -template <> -constexpr Int32 max_decimal_value() { - return 999999999; -} -template <> -constexpr Int64 max_decimal_value() { - return 999999999999999999; -} -template <> -constexpr Int128 max_decimal_value() { - return static_cast(999999999999999999ll) * 100000000000000000ll * 1000ll + - static_cast(99999999999999999ll) * 1000ll + 999ll; -} -template <> -constexpr Int128 max_decimal_value() { - return static_cast(999999999999999999ll) * 100000000000000000ll * 1000ll + - static_cast(99999999999999999ll) * 1000ll + 999ll; -} DataTypePtr create_decimal(UInt64 precision, UInt64 scale, bool use_v2); @@ -267,19 +250,19 @@ public: T fractional_part(T x) const { if (scale == 0) { - return 0; + return T(); } - if (x < T(0)) { - x *= T(-1); + if (x < T()) { + x *= -1; } return x % get_scale_multiplier(); } - T max_whole_value() const { return get_scale_multiplier(max_precision() - scale) - T(1); } + T max_whole_value() const { return get_scale_multiplier(max_precision() - scale) - 1; } bool can_store_whole(T x) const { T max = max_whole_value(); - if (x > max || x < -max) { + if (x > max || x < T(-max)) { return false; } return true; @@ -420,21 +403,18 @@ ToDataType::FieldType convert_decimals(const typename FromDataType::FieldType& v Decimal128I, std::conditional_t<(sizeof(FromFieldType) > sizeof(ToFieldType)), FromFieldType, ToFieldType>>; - using MaxNativeType = typename MaxFieldType::NativeType; - MaxNativeType converted_value; + MaxFieldType converted_value; if (scale_to > scale_from) { converted_value = DataTypeDecimal::get_scale_multiplier(scale_to - scale_from); - if (common::mul_overflow(static_cast(value), converted_value, - converted_value)) { + if (common::mul_overflow(static_cast(value).value, converted_value.value, + converted_value.value)) { if (overflow_flag) { *overflow_flag = 1; } - VLOG_DEBUG << "Decimal convert overflow"; - return converted_value < 0 - ? std::numeric_limits::min() - : std::numeric_limits::max(); + return converted_value < MaxFieldType() ? type_limit::min() + : type_limit::max(); } } else { converted_value = @@ -442,19 +422,16 @@ ToDataType::FieldType convert_decimals(const typename FromDataType::FieldType& v } if constexpr (sizeof(FromFieldType) > sizeof(ToFieldType)) { - if (converted_value < std::numeric_limits::min()) { + if (converted_value < FromFieldType(type_limit::min())) { if (overflow_flag) { *overflow_flag = 1; } - VLOG_DEBUG << "Decimal convert overflow"; - return std::numeric_limits::min(); - } - if (converted_value > std::numeric_limits::max()) { + return type_limit::min(); + } else if (converted_value > FromFieldType(type_limit::max())) { if (overflow_flag) { *overflow_flag = 1; } - VLOG_DEBUG << "Decimal convert overflow"; - return std::numeric_limits::max(); + return type_limit::max(); } } @@ -490,10 +467,8 @@ void convert_decimal_cols( if (overflow_flag) { overflow_flag[i] = 1; } - VLOG_DEBUG << "Decimal convert overflow"; - vec_to[i] = - res < 0 ? std::numeric_limits::min() - : std::numeric_limits::max(); + vec_to[i] = res < 0 ? type_limit::min() + : type_limit::max(); } else { vec_to[i] = res; } @@ -505,7 +480,7 @@ void convert_decimal_cols( MaxNativeType multiplier = DataTypeDecimal::get_scale_multiplier(scale_from - scale_to); for (size_t i = 0; i < sz; i++) { - if (vec_from[i] >= 0) { + if (vec_from[i] >= FromFieldType(0)) { vec_to[i] = (vec_from[i] + multiplier / 2) / multiplier; } else { vec_to[i] = (vec_from[i] - multiplier / 2) / multiplier; @@ -515,19 +490,16 @@ void convert_decimal_cols( if constexpr (sizeof(FromFieldType) > sizeof(ToFieldType)) { for (size_t i = 0; i < sz; i++) { - if (vec_to[i] < std::numeric_limits::min()) { + if (vec_to[i] < type_limit::min()) { if (overflow_flag) { *overflow_flag = 1; } - VLOG_DEBUG << "Decimal convert overflow"; - vec_to[i] = std::numeric_limits::min(); - } - if (vec_to[i] > std::numeric_limits::max()) { + vec_to[i] = type_limit::min(); + } else if (vec_to[i] > type_limit::max()) { if (overflow_flag) { *overflow_flag = 1; } - VLOG_DEBUG << "Decimal convert overflow"; - vec_to[i] = std::numeric_limits::max(); + vec_to[i] = type_limit::min(); } } } @@ -552,25 +524,11 @@ ToDataType::FieldType convert_from_decimal(const typename FromDataType::FieldTyp if constexpr (sizeof(FromFieldType) > sizeof(ToFieldType) || !std::numeric_limits::is_signed) { - if constexpr (std::numeric_limits::is_signed) { - if (converted_value < std::numeric_limits::min()) { - VLOG_DEBUG << "Decimal convert overflow"; - return std::numeric_limits::min(); - } - if (converted_value > std::numeric_limits::max()) { - VLOG_DEBUG << "Decimal convert overflow"; - return std::numeric_limits::max(); - } - } else { - using CastIntType = - std::conditional_t, Int128, Int64>; - - if (converted_value < 0 || - converted_value > - static_cast(std::numeric_limits::max())) { - VLOG_DEBUG << "Decimal convert overflow"; - return std::numeric_limits::max(); - } + if (converted_value < FromFieldType(type_limit::min())) { + return FromFieldType(type_limit::min()); + } + if (converted_value > FromFieldType(type_limit::max())) { + return FromFieldType(type_limit::max()); } } return converted_value; @@ -582,7 +540,7 @@ template ToDataType::FieldType convert_to_decimal(const typename FromDataType::FieldType& value, UInt32 scale, UInt8* overflow_flag) { using FromFieldType = typename FromDataType::FieldType; - using ToNativeType = typename ToDataType::FieldType::NativeType; + using ToFieldType = typename ToDataType::FieldType; if constexpr (std::is_floating_point_v) { if (!std::isfinite(value)) { @@ -590,25 +548,23 @@ ToDataType::FieldType convert_to_decimal(const typename FromDataType::FieldType& *overflow_flag = 1; } VLOG_DEBUG << "Decimal convert overflow. Cannot convert infinity or NaN to decimal"; - return value < 0 ? std::numeric_limits::min() - : std::numeric_limits::max(); + return value < 0 ? type_limit::min() : type_limit::max(); } FromFieldType out; out = value * ToDataType::get_scale_multiplier(scale); - if (out <= static_cast(std::numeric_limits::min())) { + if (out <= static_cast(type_limit::min())) { if (overflow_flag) { *overflow_flag = 1; } - VLOG_DEBUG << "Decimal convert overflow. Float is out of Decimal range"; - return std::numeric_limits::min(); + return type_limit::min(); } - if (out >= static_cast(std::numeric_limits::max())) { + if (out >= static_cast(type_limit::max())) { if (overflow_flag) { *overflow_flag = 1; } VLOG_DEBUG << "Decimal convert overflow. Float is out of Decimal range"; - return std::numeric_limits::max(); + return type_limit::max(); } return out; } else { @@ -621,10 +577,4 @@ ToDataType::FieldType convert_to_decimal(const typename FromDataType::FieldType& } } -template -typename T::NativeType max_decimal_value(UInt32 precision); - -template -typename T::NativeType min_decimal_value(UInt32 precision); - } // namespace doris::vectorized diff --git a/be/src/vec/data_types/serde/data_type_decimal_serde.cpp b/be/src/vec/data_types/serde/data_type_decimal_serde.cpp index e60fad97d2..cc4ee16d94 100644 --- a/be/src/vec/data_types/serde/data_type_decimal_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_decimal_serde.cpp @@ -69,7 +69,7 @@ template Status DataTypeDecimalSerDe::deserialize_one_cell_from_text(IColumn& column, Slice& slice, const FormatOptions& options) const { auto& column_data = assert_cast&>(column).get_data(); - T val = 0; + T val = {}; if (ReadBuffer rb(slice.data, slice.size); !read_decimal_text_impl(val, rb, precision, scale)) { return Status::InvalidArgument("parse decimal fail, string: '{}', primitive type: '{}'", @@ -103,7 +103,7 @@ void DataTypeDecimalSerDe::write_column_to_arrow(const IColumn& column, const checkArrowStatus(builder.Append(value), column.get_name(), array_builder->type()->name()); } - } else if constexpr (std::is_same_v>) { + } else if constexpr (std::is_same_v) { std::shared_ptr s_decimal_ptr = std::make_shared(38, col.get_scale()); for (size_t i = start; i < end; ++i) { diff --git a/be/src/vec/data_types/serde/data_type_decimal_serde.h b/be/src/vec/data_types/serde/data_type_decimal_serde.h index 9836bb2f3e..f6826d6b44 100644 --- a/be/src/vec/data_types/serde/data_type_decimal_serde.h +++ b/be/src/vec/data_types/serde/data_type_decimal_serde.h @@ -119,7 +119,7 @@ Status DataTypeDecimalSerDe::write_column_to_pb(const IColumn& column, PValue auto ptype = result.mutable_type(); if constexpr (std::is_same_v>) { ptype->set_id(PGenericType::DECIMAL128); - } else if constexpr (std::is_same_v>) { + } else if constexpr (std::is_same_v) { ptype->set_id(PGenericType::DECIMAL128I); } else if constexpr (std::is_same_v>) { ptype->set_id(PGenericType::INT32); @@ -138,12 +138,12 @@ Status DataTypeDecimalSerDe::write_column_to_pb(const IColumn& column, PValue template Status DataTypeDecimalSerDe::read_column_from_pb(IColumn& column, const PValues& arg) const { - if constexpr (std::is_same_v> || std::is_same_v> || + if constexpr (std::is_same_v> || std::is_same_v || std::is_same_v> || std::is_same_v>) { column.resize(arg.bytes_value_size()); auto& data = reinterpret_cast&>(column).get_data(); for (int i = 0; i < arg.bytes_value_size(); ++i) { - data[i] = *(int128_t*)(arg.bytes_value(i).c_str()); + data[i] = *(T*)(arg.bytes_value(i).c_str()); } return Status::OK(); } @@ -161,7 +161,7 @@ void DataTypeDecimalSerDe::write_one_cell_to_jsonb(const IColumn& column, Jso Decimal128::NativeType val = *reinterpret_cast(data_ref.data); result.writeInt128(val); - } else if constexpr (std::is_same_v>) { + } else if constexpr (std::is_same_v) { Decimal128I::NativeType val = *reinterpret_cast(data_ref.data); result.writeInt128(val); @@ -182,7 +182,7 @@ void DataTypeDecimalSerDe::read_one_cell_from_jsonb(IColumn& column, auto& col = reinterpret_cast&>(column); if constexpr (std::is_same_v>) { col.insert_value(static_cast(arg)->val()); - } else if constexpr (std::is_same_v>) { + } else if constexpr (std::is_same_v) { col.insert_value(static_cast(arg)->val()); } else if constexpr (std::is_same_v>) { col.insert_value(static_cast(arg)->val()); diff --git a/be/src/vec/exec/format/format_common.h b/be/src/vec/exec/format/format_common.h index 42d31b3aa9..bea3322eba 100644 --- a/be/src/vec/exec/format/format_common.h +++ b/be/src/vec/exec/format/format_common.h @@ -38,8 +38,7 @@ struct DecimalScaleParams { return common::exp10_i32(n); } else if constexpr (std::is_same_v) { return common::exp10_i64(n); - } else if constexpr (std::is_same_v || - std::is_same_v) { + } else if constexpr (std::is_same_v) { return common::exp10_i128(n); } else { return DecimalPrimitiveType(1); diff --git a/be/src/vec/exec/jni_connector.h b/be/src/vec/exec/jni_connector.h index f7779736c2..1cadc37a1b 100644 --- a/be/src/vec/exec/jni_connector.h +++ b/be/src/vec/exec/jni_connector.h @@ -353,7 +353,7 @@ private: template void _parse_value_range(const ColumnValueRange& col_val_range, const std::string& column_name) { - using CppType = typename VecPrimitiveTypeTraits::CppType; + using CppType = typename PrimitiveTypeTraits::CppType; if (col_val_range.is_fixed_value_range()) { ScanPredicate in_predicate(column_name); diff --git a/be/src/vec/exec/scan/vscan_node.cpp b/be/src/vec/exec/scan/vscan_node.cpp index e65bdab093..580015d067 100644 --- a/be/src/vec/exec/scan/vscan_node.cpp +++ b/be/src/vec/exec/scan/vscan_node.cpp @@ -1152,7 +1152,7 @@ Status VScanNode::_normalize_match_predicate(VExpr* expr, VExprContext* expr_ctx if (temp_pdt != PushDownType::UNACCEPTABLE) { DCHECK(slot_ref_child >= 0); if (value.data != nullptr) { - using CppType = typename VecPrimitiveTypeTraits::CppType; + using CppType = typename PrimitiveTypeTraits::CppType; if constexpr (T == TYPE_CHAR || T == TYPE_VARCHAR || T == TYPE_STRING || T == TYPE_HLL) { auto val = StringRef(value.data, value.size); @@ -1214,10 +1214,10 @@ Status VScanNode::_change_value_range(ColumnValueRange& temp_rang (PrimitiveType == TYPE_BOOLEAN) || (PrimitiveType == TYPE_DATEV2)) { if constexpr (IsFixed) { func(temp_range, - reinterpret_cast::CppType*>(value)); + reinterpret_cast::CppType*>(value)); } else { func(temp_range, to_olap_filter_type(fn_name, slot_ref_child), - reinterpret_cast::CppType*>(value)); + reinterpret_cast::CppType*>(value)); } } else { static_assert(always_false_v); diff --git a/be/src/vec/functions/array/function_array_index.h b/be/src/vec/functions/array/function_array_index.h index 6da1381d9a..c7cf7bcc99 100644 --- a/be/src/vec/functions/array/function_array_index.h +++ b/be/src/vec/functions/array/function_array_index.h @@ -224,68 +224,8 @@ private: const IColumn& right_column, const UInt8* right_nested_null_map, const UInt8* outer_null_map) { - if (check_column(right_column)) { - return _execute_number( - offsets, nested_null_map, nested_column, right_column, right_nested_null_map, - outer_null_map); - } else if (check_column(right_column)) { - return _execute_number( - offsets, nested_null_map, nested_column, right_column, right_nested_null_map, - outer_null_map); - } else if (check_column(right_column)) { - return _execute_number( - offsets, nested_null_map, nested_column, right_column, right_nested_null_map, - outer_null_map); - } else if (check_column(right_column)) { - return _execute_number( - offsets, nested_null_map, nested_column, right_column, right_nested_null_map, - outer_null_map); - } else if (check_column(right_column)) { - return _execute_number( - offsets, nested_null_map, nested_column, right_column, right_nested_null_map, - outer_null_map); - } else if (check_column(right_column)) { - return _execute_number( - offsets, nested_null_map, nested_column, right_column, right_nested_null_map, - outer_null_map); - } else if (check_column(right_column)) { - return _execute_number( - offsets, nested_null_map, nested_column, right_column, right_nested_null_map, - outer_null_map); - } else if (check_column(right_column)) { - return _execute_number( - offsets, nested_null_map, nested_column, right_column, right_nested_null_map, - outer_null_map); - } else if (right_column.is_date_type()) { - return _execute_number( - offsets, nested_null_map, nested_column, right_column, right_nested_null_map, - outer_null_map); - } else if (right_column.is_datetime_type()) { - return _execute_number( - offsets, nested_null_map, nested_column, right_column, right_nested_null_map, - outer_null_map); - } else if (check_column(right_column)) { - return _execute_number( - offsets, nested_null_map, nested_column, right_column, right_nested_null_map, - outer_null_map); - } else if (check_column(right_column)) { - return _execute_number( - offsets, nested_null_map, nested_column, right_column, right_nested_null_map, - outer_null_map); - } else if (check_column(right_column)) { - return _execute_number( - offsets, nested_null_map, nested_column, right_column, right_nested_null_map, - outer_null_map); - } else if (check_column(right_column)) { - return _execute_number( - offsets, nested_null_map, nested_column, right_column, right_nested_null_map, - outer_null_map); - } else if (check_column(right_column)) { - return _execute_number( - offsets, nested_null_map, nested_column, right_column, right_nested_null_map, - outer_null_map); - } else if (check_column(right_column)) { - return _execute_number( + if (check_column(right_column)) { + return _execute_number( offsets, nested_null_map, nested_column, right_column, right_nested_null_map, outer_null_map); } diff --git a/be/src/vec/functions/array/function_array_remove.h b/be/src/vec/functions/array/function_array_remove.h index 065f348fa8..fdda43de43 100644 --- a/be/src/vec/functions/array/function_array_remove.h +++ b/be/src/vec/functions/array/function_array_remove.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include "common/status.h" @@ -253,53 +254,8 @@ private: ColumnPtr _execute_number_expanded(const ColumnArray::Offsets64& offsets, const IColumn& nested_column, const IColumn& right_column, const UInt8* nested_null_map) { - if (check_column(right_column)) { - return _execute_number(offsets, nested_column, - right_column, nested_null_map); - } else if (check_column(right_column)) { - return _execute_number(offsets, nested_column, - right_column, nested_null_map); - } else if (check_column(right_column)) { - return _execute_number(offsets, nested_column, - right_column, nested_null_map); - } else if (check_column(right_column)) { - return _execute_number(offsets, nested_column, - right_column, nested_null_map); - } else if (right_column.is_date_type()) { - return _execute_number(offsets, nested_column, - right_column, nested_null_map); - } else if (right_column.is_datetime_type()) { - return _execute_number(offsets, nested_column, - right_column, nested_null_map); - } else if (check_column(right_column)) { - return _execute_number(offsets, nested_column, - right_column, nested_null_map); - } else if (check_column(right_column)) { - return _execute_number( - offsets, nested_column, right_column, nested_null_map); - } else if (check_column(right_column)) { - return _execute_number(offsets, nested_column, - right_column, nested_null_map); - } else if (check_column(right_column)) { - return _execute_number(offsets, nested_column, - right_column, nested_null_map); - } else if (check_column(right_column)) { - return _execute_number(offsets, nested_column, - right_column, nested_null_map); - } else if (check_column(right_column)) { - return _execute_number(offsets, nested_column, - right_column, nested_null_map); - } else if (check_column(right_column)) { - return _execute_number( - offsets, nested_column, right_column, nested_null_map); - } else if (check_column(right_column)) { - return _execute_number( - offsets, nested_column, right_column, nested_null_map); - } else if (check_column(right_column)) { - return _execute_number( - offsets, nested_column, right_column, nested_null_map); - } else if (check_column(right_column)) { - return _execute_number( + if (check_column(right_column)) { + return _execute_number( offsets, nested_column, right_column, nested_null_map); } return nullptr; diff --git a/be/src/vec/functions/function_binary_arithmetic.h b/be/src/vec/functions/function_binary_arithmetic.h index 8db58b072f..0b92898302 100644 --- a/be/src/vec/functions/function_binary_arithmetic.h +++ b/be/src/vec/functions/function_binary_arithmetic.h @@ -459,7 +459,6 @@ private: NativeResultType res; // TODO handle overflow gracefully if (Op::template apply(a, b, res)) { - LOG(WARNING) << "Decimal math overflow"; res = max_decimal_value(); } return res; diff --git a/be/src/vec/functions/function_case.h b/be/src/vec/functions/function_case.h index ed3eae425b..c3592e5ef3 100644 --- a/be/src/vec/functions/function_case.h +++ b/be/src/vec/functions/function_case.h @@ -322,7 +322,7 @@ public: // set default value for (int i = 0; i < rows_count; i++) { - result_raw_data[i] = 0; + result_raw_data[i] = {}; } // some types had simd automatically, but some not. diff --git a/be/src/vec/functions/math.cpp b/be/src/vec/functions/math.cpp index 34a126971a..298a765aa1 100644 --- a/be/src/vec/functions/math.cpp +++ b/be/src/vec/functions/math.cpp @@ -179,9 +179,9 @@ struct AbsImpl { static inline ResultType apply(A a) { if constexpr (IsDecimalNumber) - return a < 0 ? A(-a) : a; + return a < A(0) ? A(-a) : a; else if constexpr (std::is_integral_v && std::is_signed_v) - return a < 0 ? static_cast(~a) + 1 : a; + return a < A(0) ? static_cast(~a) + 1 : a; else if constexpr (std::is_integral_v && std::is_unsigned_v) return static_cast(a); else if constexpr (std::is_floating_point_v) diff --git a/be/src/vec/io/io_helper.h b/be/src/vec/io/io_helper.h index 5f07d8ced1..7159f402e4 100644 --- a/be/src/vec/io/io_helper.h +++ b/be/src/vec/io/io_helper.h @@ -63,13 +63,12 @@ void write_text(Decimal value, UInt32 scale, std::ostream& ostr) { } } - using Type = std::conditional_t, int128_t, T>; - Type whole_part = value; + T whole_part = value; if (scale) { - whole_part = value / decimal_scale_multiplier(scale); + whole_part = value / decimal_scale_multiplier(scale); } - if constexpr (std::is_same_v || std::is_same_v) { + if constexpr (std::is_same_v) { ostr << int128_to_string(whole_part); } else { ostr << whole_part; @@ -352,17 +351,17 @@ bool read_decimal_text_impl(T& x, ReadBuffer& buf, UInt32 precision, UInt32 scal if constexpr (!std::is_same_v) { StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - x.value = StringParser::string_to_decimal( - (const char*)buf.position(), buf.count(), precision, scale, &result); + x.value = StringParser::string_to_decimal

((const char*)buf.position(), buf.count(), + precision, scale, &result); // only to match the is_all_read() check to prevent return null buf.position() = buf.end(); return result == StringParser::PARSE_SUCCESS || result == StringParser::PARSE_UNDERFLOW; } else { StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - x.value = StringParser::string_to_decimal( - buf.position(), buf.count(), DecimalV2Value::PRECISION, DecimalV2Value::SCALE, - &result); + x.value = StringParser::string_to_decimal(buf.position(), buf.count(), + DecimalV2Value::PRECISION, + DecimalV2Value::SCALE, &result); // only to match the is_all_read() check to prevent return null buf.position() = buf.end(); diff --git a/be/src/vec/sink/vtablet_block_convertor.cpp b/be/src/vec/sink/vtablet_block_convertor.cpp index 95dc0a4e85..7c7300b9b3 100644 --- a/be/src/vec/sink/vtablet_block_convertor.cpp +++ b/be/src/vec/sink/vtablet_block_convertor.cpp @@ -131,33 +131,6 @@ DecimalV2Value OlapTableBlockConvertor::_get_decimalv2_min_or_max(const TypeDesc return value; } -template -DecimalType OlapTableBlockConvertor::_get_decimalv3_min_or_max(const TypeDescriptor& type) { - std::map* pmap; - if constexpr (std::is_same_v) { - pmap = IsMin ? &_min_decimal32_val : &_max_decimal32_val; - } else if constexpr (std::is_same_v) { - pmap = IsMin ? &_min_decimal64_val : &_max_decimal64_val; - } else { - pmap = IsMin ? &_min_decimal128_val : &_max_decimal128_val; - } - - // found - auto iter = pmap->find(type.precision); - if (iter != pmap->end()) { - return iter->second; - } - - typename DecimalType::NativeType value; - if constexpr (IsMin) { - value = vectorized::min_decimal_value(type.precision); - } else { - value = vectorized::max_decimal_value(type.precision); - } - pmap->emplace(type.precision, value); - return value; -} - Status OlapTableBlockConvertor::_validate_column(RuntimeState* state, const TypeDescriptor& type, bool is_nullable, vectorized::ColumnPtr column, size_t slot_index, bool* stop_processing, @@ -293,12 +266,11 @@ Status OlapTableBlockConvertor::_validate_column(RuntimeState* state, const Type break; } case TYPE_DECIMAL32: { -#define CHECK_VALIDATION_FOR_DECIMALV3(ColumnDecimalType, DecimalType) \ - auto column_decimal = const_cast*>( \ - assert_cast*>( \ - real_column_ptr.get())); \ - const auto& max_decimal = _get_decimalv3_min_or_max(type); \ - const auto& min_decimal = _get_decimalv3_min_or_max(type); \ +#define CHECK_VALIDATION_FOR_DECIMALV3(DecimalType) \ + auto column_decimal = const_cast*>( \ + assert_cast*>(real_column_ptr.get())); \ + const auto& max_decimal = type_limit::max(); \ + const auto& min_decimal = type_limit::min(); \ for (size_t j = 0; j < column->size(); ++j) { \ auto row = rows ? (*rows)[j] : j; \ if (row == last_invalid_row) { \ @@ -320,17 +292,18 @@ Status OlapTableBlockConvertor::_validate_column(RuntimeState* state, const Type } \ } \ } - CHECK_VALIDATION_FOR_DECIMALV3(Decimal32, Decimal32); + CHECK_VALIDATION_FOR_DECIMALV3(vectorized::Decimal32); break; } case TYPE_DECIMAL64: { - CHECK_VALIDATION_FOR_DECIMALV3(Decimal64, Decimal64); + CHECK_VALIDATION_FOR_DECIMALV3(vectorized::Decimal64); break; } case TYPE_DECIMAL128I: { - CHECK_VALIDATION_FOR_DECIMALV3(Decimal128I, Decimal128); + CHECK_VALIDATION_FOR_DECIMALV3(vectorized::Decimal128I); break; } +#undef CHECK_VALIDATION_FOR_DECIMALV3 case TYPE_ARRAY: { const auto column_array = assert_cast(real_column_ptr.get()); diff --git a/be/src/vec/sink/vtablet_block_convertor.h b/be/src/vec/sink/vtablet_block_convertor.h index bfc7b3b5d9..335e876284 100644 --- a/be/src/vec/sink/vtablet_block_convertor.h +++ b/be/src/vec/sink/vtablet_block_convertor.h @@ -62,9 +62,6 @@ private: template DecimalV2Value _get_decimalv2_min_or_max(const TypeDescriptor& type); - template - DecimalType _get_decimalv3_min_or_max(const TypeDescriptor& type); - Status _validate_column(RuntimeState* state, const TypeDescriptor& type, bool is_nullable, vectorized::ColumnPtr column, size_t slot_index, bool* stop_processing, fmt::memory_buffer& error_prefix, diff --git a/be/test/vec/aggregate_functions/agg_min_max_test.cpp b/be/test/vec/aggregate_functions/agg_min_max_test.cpp index c62b5205be..aaff93d5e6 100644 --- a/be/test/vec/aggregate_functions/agg_min_max_test.cpp +++ b/be/test/vec/aggregate_functions/agg_min_max_test.cpp @@ -104,7 +104,7 @@ TEST_P(AggMinMaxTest, min_max_decimal_test) { // Check result. ColumnDecimal128 ans(0, 9); agg_function->insert_result_into(place, ans); - EXPECT_EQ(min_max_type == "min" ? 0 : agg_test_batch_size - 1, ans.get_element(0)); + EXPECT_EQ(min_max_type == "min" ? 0 : agg_test_batch_size - 1, ans.get_element(0).value); agg_function->destroy(place); auto dst = agg_function->create_serialize_column(); @@ -120,7 +120,7 @@ TEST_P(AggMinMaxTest, min_max_decimal_test) { } for (size_t i = 0; i != agg_test_batch_size; ++i) { - EXPECT_EQ(i, result.get_element(i)); + EXPECT_EQ(i, result.get_element(i).value); } } diff --git a/be/test/vec/core/block_spill_test.cpp b/be/test/vec/core/block_spill_test.cpp index 721b9d16c8..e59ef83287 100644 --- a/be/test/vec/core/block_spill_test.cpp +++ b/be/test/vec/core/block_spill_test.cpp @@ -358,7 +358,7 @@ TEST_F(TestBlockSpill, TestDecimal) { (vectorized::ColumnDecimal>*)column.get(); for (size_t j = 0; j < batch_size; ++j) { __int128_t value = (j + i * batch_size) * (pow(10, 9) + pow(10, 8)); - EXPECT_EQ(real_column->get_element(j), value); + EXPECT_EQ(real_column->get_element(j).value, value); } } @@ -370,7 +370,7 @@ TEST_F(TestBlockSpill, TestDecimal) { auto column = block_read.get_by_position(0).column; auto* real_column = (vectorized::ColumnDecimal>*)column.get(); - EXPECT_EQ(real_column->get_element(0), batch_size * 3 * (pow(10, 9) + pow(10, 8))); + EXPECT_EQ(real_column->get_element(0).value, batch_size * 3 * (pow(10, 9) + pow(10, 8))); } TEST_F(TestBlockSpill, TestDecimalNullable) { int batch_size = 3; // rows in a block @@ -419,7 +419,7 @@ TEST_F(TestBlockSpill, TestDecimalNullable) { ASSERT_TRUE(real_column->is_null_at(j)); } else { __int128_t value = (j + i * batch_size) * (pow(10, 9) + pow(10, 8)); - EXPECT_EQ(decimal_col.get_element(j), value); + EXPECT_EQ(decimal_col.get_element(j).value, value); } } } @@ -434,7 +434,7 @@ TEST_F(TestBlockSpill, TestDecimalNullable) { const auto& decimal_col = (vectorized::ColumnDecimal< vectorized::Decimal>&)(real_column->get_nested_column()); - EXPECT_EQ(decimal_col.get_element(0), batch_size * 3 * (pow(10, 9) + pow(10, 8))); + EXPECT_EQ(decimal_col.get_element(0).value, batch_size * 3 * (pow(10, 9) + pow(10, 8))); } std::string convert_bitmap_to_string(BitmapValue& bitmap); TEST_F(TestBlockSpill, TestBitmap) { diff --git a/be/test/vec/data_types/from_string_test.cpp b/be/test/vec/data_types/from_string_test.cpp index e3d3b5bd5d..67e2cc0214 100644 --- a/be/test/vec/data_types/from_string_test.cpp +++ b/be/test/vec/data_types/from_string_test.cpp @@ -16,7 +16,6 @@ // under the License. #include "gtest/gtest_pred_impl.h" -#include "olap/types.h" // for TypeInfo #include "olap/wrapper_field.h" #include "vec/columns/column.h" #include "vec/core/field.h" diff --git a/be/test/vec/data_types/serde/data_type_serde_arrow_test.cpp b/be/test/vec/data_types/serde/data_type_serde_arrow_test.cpp index 3c5191c1cb..2bf47ca135 100644 --- a/be/test/vec/data_types/serde/data_type_serde_arrow_test.cpp +++ b/be/test/vec/data_types/serde/data_type_serde_arrow_test.cpp @@ -181,10 +181,10 @@ void serialize_and_deserialize_arrow_test() { } Int32 val; StringParser::ParseResult result = StringParser::PARSE_SUCCESS; - i % 2 == 0 ? val = StringParser::string_to_decimal( + i % 2 == 0 ? val = StringParser::string_to_decimal( "1234567.56", 11, type_desc.precision, type_desc.scale, &result) - : val = StringParser::string_to_decimal( + : val = StringParser::string_to_decimal( "-1234567.56", 12, type_desc.precision, type_desc.scale, &result); EXPECT_TRUE(result == StringParser::PARSE_SUCCESS); @@ -217,7 +217,7 @@ void serialize_and_deserialize_arrow_test() { StringParser::ParseResult result = StringParser::PARSE_SUCCESS; std::string decimal_string = i % 2 == 0 ? "-123456789012.123456" : "123456789012.123456"; - val = StringParser::string_to_decimal( + val = StringParser::string_to_decimal( decimal_string.c_str(), decimal_string.size(), type_desc.precision, type_desc.scale, &result); EXPECT_TRUE(result == StringParser::PARSE_SUCCESS); diff --git a/be/test/vec/function/function_array_index_test.cpp b/be/test/vec/function/function_array_index_test.cpp index 497826019d..2d5c70b5f9 100644 --- a/be/test/vec/function/function_array_index_test.cpp +++ b/be/test/vec/function/function_array_index_test.cpp @@ -20,7 +20,6 @@ #include "common/status.h" #include "function_test_util.h" -#include "gtest/gtest_pred_impl.h" #include "testutil/any_type.h" #include "vec/core/field.h" #include "vec/core/types.h" @@ -46,11 +45,11 @@ TEST(function_array_index_test, array_contains) { check_function(func_name, input_types, data_set); } - // array_contains(Array, Int8) + // array_contains(Array, Int8) { - InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Int32, TypeIndex::Int8}; + InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Int8, TypeIndex::Int8}; - Array vec = {Int32(1), Int32(2), Int32(3)}; + Array vec = {Int8(1), Int8(2), Int8(3)}; DataSet data_set = {{{vec, Int8(2)}, UInt8(1)}, {{vec, Int8(4)}, UInt8(0)}, {{Null(), Int8(1)}, Null()}, @@ -59,11 +58,11 @@ TEST(function_array_index_test, array_contains) { check_function(func_name, input_types, data_set); } - // array_contains(Array, Int64) + // array_contains(Array, Int64) { - InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Int8, TypeIndex::Int64}; + InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Int64, TypeIndex::Int64}; - Array vec = {Int8(1), Int8(2), Int8(3)}; + Array vec = {Int64(1), Int64(2), Int64(3)}; DataSet data_set = {{{vec, Int64(2)}, UInt8(1)}, {{vec, Int64(4)}, UInt8(0)}, {{Null(), Int64(1)}, Null()}, @@ -187,28 +186,28 @@ TEST(function_array_index_test, array_position) { check_function(func_name, input_types, data_set); } - // array_position(Array, Int8) + // array_position(Array, Int32) { - InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Int32, TypeIndex::Int8}; + InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Int32, TypeIndex::Int32}; Array vec = {Int32(1), Int32(2), Int32(3)}; - DataSet data_set = {{{vec, Int8(2)}, Int64(2)}, - {{vec, Int8(4)}, Int64(0)}, - {{Null(), Int8(1)}, Null()}, - {{empty_arr, Int8(1)}, Int64(0)}}; + DataSet data_set = {{{vec, Int32(2)}, Int64(2)}, + {{vec, Int32(4)}, Int64(0)}, + {{Null(), Int32(1)}, Null()}, + {{empty_arr, Int32(1)}, Int64(0)}}; check_function(func_name, input_types, data_set); } - // array_position(Array, Int64) + // array_position(Array, Int8) { - InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Int8, TypeIndex::Int64}; + InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Int8, TypeIndex::Int8}; Array vec = {Int8(1), Int8(2), Int8(3)}; - DataSet data_set = {{{vec, Int64(2)}, Int64(2)}, - {{vec, Int64(4)}, Int64(0)}, - {{Null(), Int64(1)}, Null()}, - {{empty_arr, Int64(1)}, Int64(0)}}; + DataSet data_set = {{{vec, Int8(2)}, Int64(2)}, + {{vec, Int8(4)}, Int64(0)}, + {{Null(), Int8(1)}, Null()}, + {{empty_arr, Int8(1)}, Int64(0)}}; check_function(func_name, input_types, data_set); } diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions_by_literal.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions_by_literal.groovy index 4e1adafac4..f2ac1da429 100644 --- a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions_by_literal.groovy +++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions_by_literal.groovy @@ -43,7 +43,7 @@ suite("test_array_functions_by_literal") { qt_sql "select array_contains([1,2,3], 4)" qt_sql "select array_contains([1,2,3,NULL], 1)" qt_sql "select array_contains([1,2,3,NULL], NULL)" - qt_sql "select array_contains([], 1)" + qt_sql "select array_contains([], true)" qt_sql "select array_contains([], NULL)" qt_sql "select array_contains(NULL, 1)" qt_sql "select array_contains(NULL, NULL)" @@ -58,11 +58,11 @@ suite("test_array_functions_by_literal") { qt_sql "select array_position([1,2,3], 4)" qt_sql "select array_position([NULL,2,3], 2)" qt_sql "select array_position([NULL,2,3], NULL)" - qt_sql "select array_position([], 1)" + qt_sql "select array_position([], true)" qt_sql "select array_position([], NULL)" qt_sql "select array_position(NULL, 1)" qt_sql "select array_position(NULL, NULL)" - qt_sql "select array_position([null], 0)" + qt_sql "select array_position([null], true)" qt_sql "select array_position([0], null)" qt_sql "select array_position([null, '1'], '')" qt_sql "select array_position([''], null)"