From fbe5a7c244d609b49fb4760b92f1738226d23b77 Mon Sep 17 00:00:00 2001 From: TengJianPing <18241664+jacktengg@users.noreply.github.com> Date: Mon, 18 Dec 2023 10:54:25 +0800 Subject: [PATCH] [improvement](decimalv2) support check overflow for decimalv2 arithmetics (#28456) --- be/src/vec/core/block.cpp | 2 +- .../functions/function_binary_arithmetic.h | 194 ++++++------ be/src/vec/functions/multiply.cpp | 27 +- .../decimalv2/test_decimalv2_overflow.out | 62 +++- .../decimalv2/test_decimalv2_overflow2.out | 61 ++++ .../decimalv3/test_decimalv3_overflow.out | 3 + .../decimalv2/test_decimalv2_overflow.groovy | 269 ++++++++++++++++ .../decimalv2/test_decimalv2_overflow2.groovy | 288 ++++++++++++++++++ .../decimalv3/test_decimalv3_overflow.groovy | 4 +- 9 files changed, 795 insertions(+), 115 deletions(-) create mode 100644 regression-test/data/datatype_p0/decimalv2/test_decimalv2_overflow2.out create mode 100644 regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow2.groovy diff --git a/be/src/vec/core/block.cpp b/be/src/vec/core/block.cpp index 195134d029..5ca921a53d 100644 --- a/be/src/vec/core/block.cpp +++ b/be/src/vec/core/block.cpp @@ -497,7 +497,7 @@ std::string Block::dump_data(size_t begin, size_t row_limit) const { // content for (size_t row_num = begin; row_num < rows() && row_num < row_limit + begin; ++row_num) { for (size_t i = 0; i < columns(); ++i) { - if (data[i].column->empty()) { + if (!data[i].column || data[i].column->empty()) { out << std::setfill(' ') << std::setw(1) << "|" << std::setw(headers_size[i]) << std::right; continue; diff --git a/be/src/vec/functions/function_binary_arithmetic.h b/be/src/vec/functions/function_binary_arithmetic.h index 1cc67804ec..217118d7e6 100644 --- a/be/src/vec/functions/function_binary_arithmetic.h +++ b/be/src/vec/functions/function_binary_arithmetic.h @@ -244,10 +244,10 @@ private: typename ArrayC::value_type* c, size_t size, const ResultType& max_result_number, const ResultType& scale_diff_multiplier) { - // TODO: handle overflow of decimalv2 + static_assert(OpTraits::is_plus_minus || OpTraits::is_multiply); if constexpr (OpTraits::is_multiply && IsDecimalV2 && IsDecimalV2 && IsDecimalV2) { - Op::vector_vector(a, b, c, size); + Op::template vector_vector(a, b, c, size); } else { bool need_adjust_scale = scale_diff_multiplier.value > 1; std::visit( @@ -264,33 +264,31 @@ private: /// null_map for divide and mod static void vector_vector(const typename Traits::ArrayA::value_type* __restrict a, const typename Traits::ArrayB::value_type* __restrict b, - typename ArrayC::value_type* c, NullMap& null_map, size_t size) { + typename ArrayC::value_type* c, NullMap& null_map, size_t size, + const ResultType& max_result_number) { + static_assert(OpTraits::is_division || OpTraits::is_mod); if constexpr (IsDecimalV2 || IsDecimalV2) { - /// default: use it if no return before for (size_t i = 0; i < size; ++i) { - c[i] = typename ArrayC::value_type(apply(a[i], b[i], null_map[i])); + c[i] = typename ArrayC::value_type( + apply(a[i], b[i], null_map[i], max_result_number)); } } else if constexpr (OpTraits::is_division && (IsDecimalNumber || IsDecimalNumber)) { for (size_t i = 0; i < size; ++i) { if constexpr (IsDecimalNumber && IsDecimalNumber) { c[i] = typename ArrayC::value_type( - apply_scaled_div(a[i].value, b[i].value, null_map[i])); + apply(a[i].value, b[i].value, null_map[i], max_result_number)); } else if constexpr (IsDecimalNumber) { c[i] = typename ArrayC::value_type( - apply_scaled_div(a[i].value, b[i], null_map[i])); + apply(a[i].value, b[i], null_map[i], max_result_number)); } else { c[i] = typename ArrayC::value_type( - apply_scaled_div(a[i], b[i].value, null_map[i])); + apply(a[i], b[i].value, null_map[i], max_result_number)); } } - } else if constexpr ((OpTraits::is_multiply || OpTraits::is_plus_minus) && - (IsDecimalNumber || IsDecimalNumber)) { - for (size_t i = 0; i < size; ++i) { - null_map[i] = apply_op_safely(a[i], b[i], c[i].value); - } } else { for (size_t i = 0; i < size; ++i) { - c[i] = typename ArrayC::value_type(apply(a[i], b[i], null_map[i])); + c[i] = typename ArrayC::value_type( + apply(a[i], b[i], null_map[i], max_result_number)); } } } @@ -299,15 +297,8 @@ private: typename ArrayC::value_type* c, size_t size, const ResultType& max_result_number, const ResultType& scale_diff_multiplier) { - if constexpr (OpTraits::is_division && IsDecimalNumber) { - for (size_t i = 0; i < size; ++i) { - // code never executed???? - c[i] = typename ArrayC::value_type(apply_scaled_div(a[i], b, a)); - } - return; - } + static_assert(!OpTraits::is_division); - /// default: use it if no return before bool need_adjust_scale = scale_diff_multiplier.value > 1; std::visit( [&](auto need_adjust_scale) { @@ -320,19 +311,17 @@ private: } static void vector_constant(const typename Traits::ArrayA::value_type* __restrict a, B b, - typename ArrayC::value_type* c, NullMap& null_map, size_t size) { + typename ArrayC::value_type* c, NullMap& null_map, size_t size, + const ResultType& max_result_number) { + static_assert(OpTraits::is_division || OpTraits::is_mod); if constexpr (OpTraits::is_division && IsDecimalNumber) { for (size_t i = 0; i < size; ++i) { - c[i] = typename ArrayC::value_type(apply_scaled_div(a[i], b.value, null_map[i])); - } - } else if constexpr ((OpTraits::is_multiply || OpTraits::is_plus_minus) && - (IsDecimalNumber || IsDecimalNumber)) { - for (size_t i = 0; i < size; ++i) { - null_map[i] = apply_op_safely(a[i], b, c[i].value); + c[i] = typename ArrayC::value_type( + apply(a[i], b.value, null_map[i], max_result_number)); } } else { for (size_t i = 0; i < size; ++i) { - c[i] = typename ArrayC::value_type(apply(a[i], b, null_map[i])); + c[i] = typename ArrayC::value_type(apply(a[i], b, null_map[i], max_result_number)); } } } @@ -341,39 +330,29 @@ private: typename ArrayC::value_type* c, size_t size, const ResultType& max_result_number, const ResultType& scale_diff_multiplier) { - if constexpr (IsDecimalV2 || IsDecimalV2) { - DecimalV2Value da(a); - for (size_t i = 0; i < size; ++i) { - c[i] = typename ArrayC::value_type( - Op::template apply(da, DecimalV2Value(b[i])).value()); - } - } else { - bool need_adjust_scale = scale_diff_multiplier.value > 1; - std::visit( - [&](auto need_adjust_scale) { - for (size_t i = 0; i < size; ++i) { - c[i] = typename ArrayC::value_type(apply( - a, b[i], max_result_number, scale_diff_multiplier)); - } - }, - make_bool_variant(need_adjust_scale)); - } + bool need_adjust_scale = scale_diff_multiplier.value > 1; + std::visit( + [&](auto need_adjust_scale) { + for (size_t i = 0; i < size; ++i) { + c[i] = typename ArrayC::value_type(apply( + a, b[i], max_result_number, scale_diff_multiplier)); + } + }, + make_bool_variant(need_adjust_scale)); } static void constant_vector(A a, const typename Traits::ArrayB::value_type* __restrict b, - typename ArrayC::value_type* c, NullMap& null_map, size_t size) { + typename ArrayC::value_type* c, NullMap& null_map, size_t size, + const ResultType& max_result_number) { + static_assert(OpTraits::is_division || OpTraits::is_mod); if constexpr (OpTraits::is_division && IsDecimalNumber) { for (size_t i = 0; i < size; ++i) { - c[i] = typename ArrayC::value_type(apply_scaled_div(a, b[i].value, null_map[i])); - } - } else if constexpr ((OpTraits::is_multiply || OpTraits::is_plus_minus) && - (IsDecimalNumber || IsDecimalNumber)) { - for (size_t i = 0; i < size; ++i) { - null_map[i] = apply_op_safely(a, b[i], c[i].value); + c[i] = typename ArrayC::value_type( + apply(a, b[i].value, null_map[i], max_result_number)); } } else { for (size_t i = 0; i < size; ++i) { - c[i] = typename ArrayC::value_type(apply(a, b[i], null_map[i])); + c[i] = typename ArrayC::value_type(apply(a, b[i], null_map[i], max_result_number)); } } } @@ -383,20 +362,17 @@ private: return ResultType(apply(a, b, max_result_number, scale_diff_multiplier)); } - static ResultType constant_constant(A a, B b, UInt8& is_null) { + static ResultType constant_constant(A a, B b, UInt8& is_null, + const ResultType& max_result_number) { + static_assert(OpTraits::is_division || OpTraits::is_mod); if constexpr (OpTraits::is_division && IsDecimalNumber) { if constexpr (IsDecimalNumber) { - return ResultType(apply_scaled_div(a.value, b.value, is_null)); + return ResultType(apply(a.value, b.value, is_null, max_result_number)); } else { - return ResultType(apply_scaled_div(a, b.value, is_null)); + return ResultType(apply(a, b.value, is_null, max_result_number)); } - } else if constexpr ((OpTraits::is_multiply || OpTraits::is_plus_minus) && - (IsDecimalNumber || IsDecimalNumber)) { - NativeResultType res; - is_null = apply_op_safely(a, b, res); - return ResultType(res); } else { - return ResultType(apply(a, b, is_null)); + return ResultType(apply(a, b, is_null, max_result_number)); } } @@ -408,13 +384,13 @@ public: 1, assert_cast&>(*res_data_type).get_scale()); if constexpr (check_overflow && !is_to_null_type && - ((!OpTraits::is_multiply && !OpTraits::is_plus_minus) || IsDecimalV2 || - IsDecimalV2)) { + ((!OpTraits::is_multiply && !OpTraits::is_plus_minus))) { LOG(FATAL) << "Invalid function type!"; return column_result; } else if constexpr (is_to_null_type) { auto null_map = ColumnUInt8::create(1, 0); - column_result->get_element(0) = constant_constant(a, b, null_map->get_element(0)); + column_result->get_element(0) = + constant_constant(a, b, null_map->get_element(0), max_result_number); return ColumnNullable::create(std::move(column_result), std::move(null_map)); } else { column_result->get_element(0) = @@ -434,14 +410,13 @@ public: DCHECK(column_left_ptr != nullptr); if constexpr (check_overflow && !is_to_null_type && - ((!OpTraits::is_multiply && !OpTraits::is_plus_minus) || IsDecimalV2 || - IsDecimalV2)) { + ((!OpTraits::is_multiply && !OpTraits::is_plus_minus))) { LOG(FATAL) << "Invalid function type!"; return column_result; } else if constexpr (is_to_null_type) { auto null_map = ColumnUInt8::create(column_left->size(), 0); vector_constant(column_left_ptr->get_data().data(), b, column_result->get_data().data(), - null_map->get_data(), column_left->size()); + null_map->get_data(), column_left->size(), max_result_number); return ColumnNullable::create(std::move(column_result), std::move(null_map)); } else { vector_constant(column_left_ptr->get_data().data(), b, column_result->get_data().data(), @@ -461,15 +436,14 @@ public: DCHECK(column_right_ptr != nullptr); if constexpr (check_overflow && !is_to_null_type && - ((!OpTraits::is_multiply && !OpTraits::is_plus_minus) || IsDecimalV2 || - IsDecimalV2)) { + ((!OpTraits::is_multiply && !OpTraits::is_plus_minus))) { LOG(FATAL) << "Invalid function type!"; return column_result; } else if constexpr (is_to_null_type) { auto null_map = ColumnUInt8::create(column_right->size(), 0); constant_vector(a, column_right_ptr->get_data().data(), column_result->get_data().data(), null_map->get_data(), - column_right->size()); + column_right->size(), max_result_number); return ColumnNullable::create(std::move(column_result), std::move(null_map)); } else { constant_vector(a, column_right_ptr->get_data().data(), @@ -492,15 +466,15 @@ public: DCHECK(column_left_ptr != nullptr && column_right_ptr != nullptr); if constexpr (check_overflow && !is_to_null_type && - ((!OpTraits::is_multiply && !OpTraits::is_plus_minus) || IsDecimalV2 || - IsDecimalV2)) { + ((!OpTraits::is_multiply && !OpTraits::is_plus_minus))) { LOG(FATAL) << "Invalid function type!"; return column_result; } else if constexpr (is_to_null_type) { + // function divide, modulo and pmod auto null_map = ColumnUInt8::create(column_result->size(), 0); vector_vector(column_left_ptr->get_data().data(), column_right_ptr->get_data().data(), column_result->get_data().data(), null_map->get_data(), - column_left->size()); + column_left->size(), max_result_number); return ColumnNullable::create(std::move(column_result), std::move(null_map)); } else { vector_vector(column_left_ptr->get_data().data(), column_right_ptr->get_data().data(), @@ -516,11 +490,18 @@ private: static ALWAYS_INLINE NativeResultType apply(NativeResultType a, NativeResultType b, const ResultType& max_result_number, const ResultType& scale_diff_multiplier) { - // TODO: handle overflow of decimalv2 + static_assert(OpTraits::is_plus_minus || OpTraits::is_multiply); if constexpr (IsDecimalV2 || IsDecimalV2) { // Now, Doris only support decimal +-*/ decimal. - // overflow in consider in operator - return Op::template apply(DecimalV2Value(a), DecimalV2Value(b)).value(); + if constexpr (check_overflow) { + auto res = Op::template apply(DecimalV2Value(a), DecimalV2Value(b)).value(); + if (res > max_result_number.value || res < -max_result_number.value) { + throw Exception(ErrorCode::ARITHMETIC_OVERFLOW_ERRROR, "Arithmetic overflow"); + } + return res; + } else { + return Op::template apply(DecimalV2Value(a), DecimalV2Value(b)).value(); + } } else { NativeResultType res; if constexpr (OpTraits::can_overflow && check_overflow) { @@ -588,11 +569,34 @@ private: /// null_map for divide and mod static ALWAYS_INLINE NativeResultType apply(NativeResultType a, NativeResultType b, - UInt8& is_null) { + UInt8& is_null, + const ResultType& max_result_number) { + static_assert(OpTraits::is_division || OpTraits::is_mod); if constexpr (IsDecimalV2 || IsDecimalV2) { DecimalV2Value l(a); DecimalV2Value r(b); auto ans = Op::template apply(l, r, is_null); + using ANS_TYPE = std::decay_t; + if constexpr (check_overflow && OpTraits::is_division) { + if constexpr (std::is_same_v) { + if (ans.value() > max_result_number.value || + ans.value() < -max_result_number.value) { + throw Exception(ErrorCode::ARITHMETIC_OVERFLOW_ERRROR, + "Arithmetic overflow"); + } + } else if constexpr (IsDecimalNumber) { + if (ans.value > max_result_number.value || + ans.value < -max_result_number.value) { + throw Exception(ErrorCode::ARITHMETIC_OVERFLOW_ERRROR, + "Arithmetic overflow"); + } + } else { + if (ans > max_result_number.value || ans < -max_result_number.value) { + throw Exception(ErrorCode::ARITHMETIC_OVERFLOW_ERRROR, + "Arithmetic overflow"); + } + } + } NativeResultType result {}; memcpy(&result, &ans, std::min(sizeof(result), sizeof(ans))); return result; @@ -600,25 +604,6 @@ private: return Op::template apply(a, b, is_null); } } - - static NativeResultType apply_scaled_div(NativeResultType a, NativeResultType b, - UInt8& is_null) { - if constexpr (OpTraits::is_division) { - return apply(a, b, is_null); - } - } - - static NativeResultType apply_scaled_mod(NativeResultType a, NativeResultType b, - UInt8& is_null) { - return apply(a, b, is_null); - } - - static ALWAYS_INLINE UInt8 apply_op_safely(NativeResultType a, NativeResultType b, - NativeResultType& c) { - if constexpr (OpTraits::is_multiply || OpTraits::is_plus_minus) { - return Op::template apply(a, b, c); - } - } }; /// Used to indicate undefined operation @@ -979,11 +964,14 @@ public: (IsDataTypeDecimal || IsDataTypeDecimal))) { if (check_overflow_for_decimal) { - if constexpr ((IsDecimalV2 || - IsDecimalV2)&&!is_to_null_type) { + // !is_to_null_type: plus, minus, multiply, + // pow, bitxor, bitor, bitand + // if check_overflow and params are decimal types: + // for functions pow, bitxor, bitor, bitand, return error + if constexpr (IsDataTypeDecimal && !is_to_null_type && + !OpTraits::is_multiply && !OpTraits::is_plus_minus) { status = Status::Error( - "cannot check overflow with decimalv2"); + "cannot check overflow with decimal for function {}", name); return false; } auto column_result = ConstOrVectorAdapter< diff --git a/be/src/vec/functions/multiply.cpp b/be/src/vec/functions/multiply.cpp index 359e3131c3..7965391099 100644 --- a/be/src/vec/functions/multiply.cpp +++ b/be/src/vec/functions/multiply.cpp @@ -48,10 +48,17 @@ struct MultiplyImpl { return a * b; } + /* + select 999999999999999999999999999 * 999999999999999999999999999; + 999999999999999999999999998000000000.000000000000000001 54 digits + */ + template static void vector_vector(const ColumnDecimal128::Container::value_type* __restrict a, const ColumnDecimal128::Container::value_type* __restrict b, ColumnDecimal128::Container::value_type* c, size_t size) { int8 sgn[size]; + auto max = DecimalV2Value::get_max_decimal(); + auto min = DecimalV2Value::get_min_decimal(); for (int i = 0; i < size; i++) { sgn[i] = ((DecimalV2Value(a[i]).value() > 0) && (DecimalV2Value(b[i]).value() > 0)) || @@ -64,14 +71,20 @@ struct MultiplyImpl { } for (int i = 0; i < size; i++) { - int128_t i128_mul_result; - if (common::mul_overflow(DecimalV2Value(a[i]).value(), DecimalV2Value(b[i]).value(), - i128_mul_result)) { - VLOG_DEBUG << "Decimal multiply overflow"; - c[i] = (sgn[i] == -1) ? -DecimalV2Value::MAX_DECIMAL_VALUE - : DecimalV2Value::MAX_DECIMAL_VALUE; - } else { + if constexpr (check_overflow) { + int128_t i128_mul_result; + if (common::mul_overflow(DecimalV2Value(a[i]).value(), DecimalV2Value(b[i]).value(), + i128_mul_result)) { + throw Exception(ErrorCode::ARITHMETIC_OVERFLOW_ERRROR, "Arithmetic overflow"); + } c[i] = (i128_mul_result - sgn[i]) / DecimalV2Value::ONE_BILLION + sgn[i]; + if (c[i].value > max.value() || c[i].value < min.value()) { + throw Exception(ErrorCode::ARITHMETIC_OVERFLOW_ERRROR, "Arithmetic overflow"); + } + } else { + c[i] = (DecimalV2Value(a[i]).value() * DecimalV2Value(b[i]).value() - sgn[i]) / + DecimalV2Value::ONE_BILLION + + sgn[i]; } } } diff --git a/regression-test/data/datatype_p0/decimalv2/test_decimalv2_overflow.out b/regression-test/data/datatype_p0/decimalv2/test_decimalv2_overflow.out index 8375fe82db..02f0d00fb2 100644 --- a/regression-test/data/datatype_p0/decimalv2/test_decimalv2_overflow.out +++ b/regression-test/data/datatype_p0/decimalv2/test_decimalv2_overflow.out @@ -1,6 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !sql1 -- -9223372036854775807.999999999 +-214151953057016418.148439877 -- !decimalv2_calc_overflow -- 123456789000.123456 12345678.12345678 1524157779151043787.117829481 @@ -28,3 +28,63 @@ -- !except2 -- 9999999.999999999 +-- !add_overflow1 -- +999999999999999999.999999999 + +-- !add_overflow2 -- +999999999999999999.999999998 1E-9 999999999999999999.999999999 + +-- !add_overflow3 -- +1E-9 999999999999999999.999999998 999999999999999999.999999999 + +-- !add_overflow3 -- +999999999999999999.999999998 1E-9 999999999999999999.999999999 + +-- !sub_overflow1 -- +-999999999999999999.999999999 + +-- !sub_overflow2 -- +-999999999999999999.999999998 1E-9 -999999999999999999.999999999 + +-- !sub_overflow3 -- +1E-9 -999999999999999999.999999998 999999999999999999.999999999 + +-- !sub_overflow3 -- +-999999999999999999.999999998 1E-9 -999999999999999999.999999999 + +-- !multi_overflow1 -- +999999999999999999.999999999 + +-- !multi_overflow2 -- +999999999999999999.999999999 999999999999999999.999999999 + +-- !multi_overflow3 -- +999999999999999999.999999999 999999999999999999.999999999 + +-- !multi_overflow4 -- +999999999999999999.999999999 1.000000000 999999999999999999.999999999 + +-- !div_overflow1 -- +99999999999999999.999999999 0.100000000 999999999999999999.999999990 + +-- !div_overflow2 -- +999999999999999999.999999990 + +-- !div_overflow3 -- +99999999999999999.999999999 0.1 999999999999999999.999999990 + +-- !div_overflow4 -- +999999999999999999.999999990 + +-- !mod1 -- +99999999999999999.999999999 0.100000000 0.099999999 + +-- !mod2 -- +0.099999999 + +-- !mod3 -- +99999999999999999.999999999 0.1 0.099999999 + +-- !mod4 -- +0.099999999 + diff --git a/regression-test/data/datatype_p0/decimalv2/test_decimalv2_overflow2.out b/regression-test/data/datatype_p0/decimalv2/test_decimalv2_overflow2.out new file mode 100644 index 0000000000..ecce20f1b2 --- /dev/null +++ b/regression-test/data/datatype_p0/decimalv2/test_decimalv2_overflow2.out @@ -0,0 +1,61 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !add_overflow1 -- +999999999999999999.999999999 + +-- !add_overflow2 -- +999999999999999999.999999998 1E-9 999999999999999999.999999999 + +-- !add_overflow3 -- +1E-9 999999999999999999.999999998 999999999999999999.999999999 + +-- !add_overflow3 -- +999999999999999999.999999998 1E-9 999999999999999999.999999999 + +-- !sub_overflow1 -- +-999999999999999999.999999999 + +-- !sub_overflow2 -- +-999999999999999999.999999998 1E-9 -999999999999999999.999999999 + +-- !sub_overflow3 -- +1E-9 -999999999999999999.999999998 999999999999999999.999999999 + +-- !sub_overflow3 -- +-999999999999999999.999999998 1E-9 -999999999999999999.999999999 + +-- !multi_overflow1 -- +999999999999999999.999999999 + +-- !multi_overflow2 -- +999999999999999999.999999999 999999999999999999.999999999000000000 + +-- !multi_overflow3 -- +999999999999999999.999999999 999999999999999999.999999999000000000 + +-- !multi_overflow4 -- +999999999999999999.999999999 1.000000000 999999999999999999.999999999 + +-- !div_overflow1 -- +99999999999999999.999999999 0.100000000 999999999999999999.999999990 + +-- !div_overflow2 -- +999999999999999999.999999990 + +-- !div_overflow3 -- +99999999999999999.999999999 0.1 999999999999999999.9999999900000 + +-- !div_overflow4 -- +999999999999999999.999999990 + +-- !mod1 -- +99999999999999999.999999999 0.100000000 0.099999999 + +-- !mod2 -- +0.099999999 + +-- !mod3 -- +99999999999999999.999999999 0.1 0.099999999 + +-- !mod4 -- +0.099999999 + diff --git a/regression-test/data/datatype_p0/decimalv3/test_decimalv3_overflow.out b/regression-test/data/datatype_p0/decimalv3/test_decimalv3_overflow.out index f865fb7280..06cf127c68 100644 --- a/regression-test/data/datatype_p0/decimalv3/test_decimalv3_overflow.out +++ b/regression-test/data/datatype_p0/decimalv3/test_decimalv3_overflow.out @@ -1,4 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this +-- !sql1 -- +7379840948334924036536518953815.011655 + -- !decimal32_overflow1 -- 1 1234567.8 1234567.8 9 9999999.9 89999999.1 diff --git a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow.groovy b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow.groovy index 932bb8376c..2b23bf2d5d 100644 --- a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow.groovy +++ b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow.groovy @@ -123,4 +123,273 @@ suite("test_decimalv2_overflow", "nonConcurrent") { qt_except2 """ select * from (select * from test_decimalv2_tb2 except select * from test_decimalv2_tb1) t order by 1; """ + + sql """ set check_overflow_for_decimal=true; """ + def prepare_decimalv2_overflow_test = { + sql """ + drop TABLE if exists decimalv2_overflow_test; + """ + sql """ + CREATE TABLE decimalv2_overflow_test ( + k1 decimalv2(27, 9), + k2 decimalv2(27, 9) + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`k1`) + PROPERTIES ( + "replication_num" = "1" + ); + """ + } + // max int128: 170141183460469231731687303715884105727, 39 digits + + // add + prepare_decimalv2_overflow_test() + sql """ + insert into decimalv2_overflow_test values(999999999999999999.999999998, 0.000000001); + """ + // const + const + qt_add_overflow1 """ + select cast(999999999999999999.999999998 as decimalv2(27,9)) + cast(0.000000001 as decimalv2(27,9)); + """ + // vector + const + qt_add_overflow2 """ + select k1, 0.000000001, k1 + cast(0.000000001 as decimalv2(27,9)) from decimalv2_overflow_test order by 1,2,3; + """ + // const + vector + qt_add_overflow3 """ + select 0.000000001, k1, cast(0.000000001 as decimalv2(27,9)) + k1 from decimalv2_overflow_test order by 1,2,3; + """ + // vector + vector + qt_add_overflow3 """ + select k1, k2, k1 + k2 from decimalv2_overflow_test order by 1,2,3; + """ + + // add overflow + prepare_decimalv2_overflow_test() + sql """ + insert into decimalv2_overflow_test values(999999999999999999.999999999, 0.000000001); + """ + test { + sql """ + select cast(999999999999999999.999999999 as decimalv2(27,9)) + cast(0.000000001 as decimalv2(27,9)); + """ + exception "Arithmetic overflow" + } + test { + sql """ + select k1, 0.000000001, k1 + cast(0.000000001 as decimalv2(27,9)) from decimalv2_overflow_test order by 1,2,3; + """ + exception "Arithmetic overflow" + } + test { + sql """ + select 0.000000001, k1, cast(0.000000001 as decimalv2(27,9)) + k1 from decimalv2_overflow_test order by 1,2,3; + """ + exception "Arithmetic overflow" + } + test { + sql """ + select k1, k2, k1 + k2 from decimalv2_overflow_test order by 1,2,3; + """ + exception "Arithmetic overflow" + } + + // sub + prepare_decimalv2_overflow_test() + sql """ + insert into decimalv2_overflow_test values(-999999999999999999.999999998, 0.000000001); + """ + // const - const + qt_sub_overflow1 """ + select cast(-999999999999999999.999999998 as decimalv2(27,9)) - cast(0.000000001 as decimalv2(27,9)); + """ + // vector - const + qt_sub_overflow2 """ + select k1, 0.000000001, k1 - cast(0.000000001 as decimalv2(27,9)) from decimalv2_overflow_test order by 1,2,3; + """ + // const - vector + qt_sub_overflow3 """ + select 0.000000001, k1, cast(0.000000001 as decimalv2(27,9)) - k1 from decimalv2_overflow_test order by 1,2,3; + """ + // vector - vector + qt_sub_overflow3 """ + select k1, k2, k1 - k2 from decimalv2_overflow_test order by 1,2,3; + """ + + // sub overflow + prepare_decimalv2_overflow_test() + sql """ + insert into decimalv2_overflow_test values(-999999999999999999.999999999, 0.000000001); + """ + test { + sql """ + select cast(-999999999999999999.999999999 as decimalv2(27,9)) - cast(0.000000001 as decimalv2(27,9)); + """ + exception "Arithmetic overflow" + } + test { + sql """ + select k1, 0.000000001, k1 - cast(0.000000001 as decimalv2(27,9)) from decimalv2_overflow_test order by 1,2,3; + """ + exception "Arithmetic overflow" + } + test { + sql """ + select 0.000000001, k1, cast(0.100000000 as decimalv2(27,9)) - k1 from decimalv2_overflow_test order by 1,2,3; + """ + exception "Arithmetic overflow" + } + test { + sql """ + select k1, k2, k1 - k2 from decimalv2_overflow_test order by 1,2,3; + """ + exception "Arithmetic overflow" + } + + // multiply + prepare_decimalv2_overflow_test() + sql """ + insert into decimalv2_overflow_test values(999999999999999999.999999999, 1.0); + """ + + // const * const + qt_multi_overflow1 """ + select cast(999999999999999999.999999999 as decimalv2(27,9)) * cast(1.0 as decimalv2(27,9)); + """ + // vector * const + qt_multi_overflow2 """ + select k1, k1 * 1.000000000 from decimalv2_overflow_test order by 1, 2; + """ + // const * vector + qt_multi_overflow3 """ + select k1, 1.000000000 * k1 from decimalv2_overflow_test order by 1, 2; + """ + // vector * vector + qt_multi_overflow4 """ + select k1, k2, k1 * k2 from decimalv2_overflow_test order by 1, 2; + """ + + prepare_decimalv2_overflow_test() + sql """ + insert into decimalv2_overflow_test values(999999999999999999.999999999, 1.1); + """ + test { + sql """ + select cast(999999999999999999.999999999 as decimalv2(27,9)) * cast(1.1 as decimalv2(27,9)); + """ + exception "Arithmetic overflow" + } + test { + sql """ + select cast(999999999999999999.999999999 as decimalv2(27,9)) * cast(999999999999999999.999999999 as decimalv2(27,9)); + """ + exception "Arithmetic overflow" + } + test { + sql """ + select k1, k1 * 1.1 from decimalv2_overflow_test order by 1, 2; + """ + exception "Arithmetic overflow" + } + test { + sql """ + select k1, 1.1 * k1 from decimalv2_overflow_test order by 1, 2; + """ + exception "Arithmetic overflow" + } + test { + sql """ + select k1, k2, k1 * k2 from decimalv2_overflow_test order by 1, 2; + """ + exception "Arithmetic overflow" + } + + // divide + prepare_decimalv2_overflow_test() + sql """ + insert into decimalv2_overflow_test values(99999999999999999.999999999, 0.1); + """ + qt_div_overflow1 """ + select k1, k2, k1 / k2 from decimalv2_overflow_test order by 1, 2; + """ + qt_div_overflow2 """ + select cast(99999999999999999.999999999 as decimalv2(27,9)) / cast(0.1 as decimalv2(27,9)); + """ + qt_div_overflow3 """ + select k1, 0.1, k1 / 0.1 from decimalv2_overflow_test order by 1, 2, 3; + """ + qt_div_overflow4 """ + select cast(99999999999999999.999999999 as decimalv2(27,9)) / k2 from decimalv2_overflow_test order by 1; + """ + + // divide overflow + prepare_decimalv2_overflow_test() + sql """ + insert into decimalv2_overflow_test values(999999999999999999.999999999, 0.1); + """ + test { + sql """ + select k1, k2, k1 / k2 from decimalv2_overflow_test; + """ + exception "Arithmetic overflow" + } + test { + sql """ + select cast(999999999999999999.999999999 as decimalv2(27,9)) / cast(0.1 as decimalv2(27,9)); + """ + exception "Arithmetic overflow" + } + test { + sql """ + select cast(999999999999999999.999999999 as decimalv2(27,9)) / cast(0.000000001 as decimalv2(27,9)); + """ + exception "Arithmetic overflow" + } + test { + sql """ + select k1, 0.1, k1 / 0.1 from decimalv2_overflow_test; + """ + exception "Arithmetic overflow" + } + test { + sql """ + select cast(999999999999999999.999999999 as decimalv2(27,9)) / k2 from decimalv2_overflow_test; + """ + exception "Arithmetic overflow" + } + + // mod + prepare_decimalv2_overflow_test() + sql """ + insert into decimalv2_overflow_test values(99999999999999999.999999999, 0.1); + """ + qt_mod1 """ + select k1, k2, k1 % k2 from decimalv2_overflow_test order by 1, 2; + """ + qt_mod2 """ + select cast(99999999999999999.999999999 as decimalv2(27,9)) % cast(0.1 as decimalv2(27,9)); + """ + qt_mod3 """ + select k1, 0.1, k1 % 0.1 from decimalv2_overflow_test order by 1, 2, 3; + """ + qt_mod4 """ + select cast(99999999999999999.999999999 as decimalv2(27,9)) % k2 from decimalv2_overflow_test order by 1; + """ + + // TODO + // decimalv2 +-*/ integer + // integer +-*/ decimalv2 + + // decimalv2 +-*/ decimalv3 + // decimalv3 +-*/ decimalv2 + + // decimalv2 mod, pmod + + // decimalv2 largeint + + // decimalv3 +-*/ integer + // integer +-*/ decimalv3 + + /// Decimal Real is not supported (traditional DBs convert Decimal Real to Real) + // {decimalv2 | decimalv3} +-*/ {float, double} } diff --git a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow2.groovy b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow2.groovy new file mode 100644 index 0000000000..ad6dea6765 --- /dev/null +++ b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow2.groovy @@ -0,0 +1,288 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_decimalv2_overflow2") { + sql """ set check_overflow_for_decimal=true; """ + def prepare_decimalv2_overflow_test2 = { + sql """ + drop TABLE if exists decimalv2_overflow_test2; + """ + sql """ + CREATE TABLE decimalv2_overflow_test2 ( + k1 decimalv2(27, 9), + k2 decimalv2(27, 9) + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`k1`) + PROPERTIES ( + "replication_num" = "1" + ); + """ + } + // max int128: 170141183460469231731687303715884105727, 39 digits + + // add + prepare_decimalv2_overflow_test2() + sql """ + insert into decimalv2_overflow_test2 values(999999999999999999.999999998, 0.000000001); + """ + // const + const + qt_add_overflow1 """ + select /*+SET_VAR(enable_fold_constant_by_be=true)*/ cast(999999999999999999.999999998 as decimalv2(27,9)) + cast(0.000000001 as decimalv2(27,9)); + """ + // vector + const + qt_add_overflow2 """ + select k1, 0.000000001, k1 + cast(0.000000001 as decimalv2(27,9)) from decimalv2_overflow_test2 order by 1,2,3; + """ + // const + vector + qt_add_overflow3 """ + select 0.000000001, k1, cast(0.000000001 as decimalv2(27,9)) + k1 from decimalv2_overflow_test2 order by 1,2,3; + """ + // vector + vector + qt_add_overflow3 """ + select k1, k2, k1 + k2 from decimalv2_overflow_test2 order by 1,2,3; + """ + + // add overflow + prepare_decimalv2_overflow_test2() + sql """ + insert into decimalv2_overflow_test2 values(999999999999999999.999999999, 0.000000001); + """ + test { + sql """ + select /*+SET_VAR(enable_fold_constant_by_be=true)*/ cast(999999999999999999.999999999 as decimalv2(27,9)) + cast(0.000000001 as decimalv2(27,9)); + """ + exception "Arithmetic overflow" + } + test { + sql """ + select k1, 0.000000001, k1 + cast(0.000000001 as decimalv2(27,9)) from decimalv2_overflow_test2 order by 1,2,3; + """ + exception "Arithmetic overflow" + } + test { + sql """ + select 0.000000001, k1, cast(0.000000001 as decimalv2(27,9)) + k1 from decimalv2_overflow_test2 order by 1,2,3; + """ + exception "Arithmetic overflow" + } + test { + sql """ + select k1, k2, k1 + k2 from decimalv2_overflow_test2 order by 1,2,3; + """ + exception "Arithmetic overflow" + } + + // sub + prepare_decimalv2_overflow_test2() + sql """ + insert into decimalv2_overflow_test2 values(-999999999999999999.999999998, 0.000000001); + """ + // const - const + qt_sub_overflow1 """ + select /*+SET_VAR(enable_fold_constant_by_be=true)*/ cast(-999999999999999999.999999998 as decimalv2(27,9)) - cast(0.000000001 as decimalv2(27,9)); + """ + // vector - const + qt_sub_overflow2 """ + select k1, 0.000000001, k1 - cast(0.000000001 as decimalv2(27,9)) from decimalv2_overflow_test2 order by 1,2,3; + """ + // const - vector + qt_sub_overflow3 """ + select 0.000000001, k1, cast(0.000000001 as decimalv2(27,9)) - k1 from decimalv2_overflow_test2 order by 1,2,3; + """ + // vector - vector + qt_sub_overflow3 """ + select k1, k2, k1 - k2 from decimalv2_overflow_test2 order by 1,2,3; + """ + + // sub overflow + prepare_decimalv2_overflow_test2() + sql """ + insert into decimalv2_overflow_test2 values(-999999999999999999.999999999, 0.000000001); + """ + test { + sql """ + select /*+SET_VAR(enable_fold_constant_by_be=true)*/ cast(-999999999999999999.999999999 as decimalv2(27,9)) - cast(0.000000001 as decimalv2(27,9)); + """ + exception "Arithmetic overflow" + } + test { + sql """ + select k1, 0.000000001, k1 - cast(0.000000001 as decimalv2(27,9)) from decimalv2_overflow_test2 order by 1,2,3; + """ + exception "Arithmetic overflow" + } + test { + sql """ + select 0.000000001, k1, cast(0.100000000 as decimalv2(27,9)) - k1 from decimalv2_overflow_test2 order by 1,2,3; + """ + exception "Arithmetic overflow" + } + test { + sql """ + select k1, k2, k1 - k2 from decimalv2_overflow_test2 order by 1,2,3; + """ + exception "Arithmetic overflow" + } + + // multiply + prepare_decimalv2_overflow_test2() + sql """ + insert into decimalv2_overflow_test2 values(999999999999999999.999999999, 1.0); + """ + + // const * const + qt_multi_overflow1 """ + select /*+SET_VAR(enable_fold_constant_by_be=true)*/ cast(999999999999999999.999999999 as decimalv2(27,9)) * cast(1.0 as decimalv2(27,9)); + """ + // vector * const + qt_multi_overflow2 """ + select k1, k1 * 1.000000000 from decimalv2_overflow_test2 order by 1, 2; + """ + // const * vector + qt_multi_overflow3 """ + select k1, 1.000000000 * k1 from decimalv2_overflow_test2 order by 1, 2; + """ + // vector * vector + qt_multi_overflow4 """ + select k1, k2, k1 * k2 from decimalv2_overflow_test2 order by 1, 2; + """ + + prepare_decimalv2_overflow_test2() + sql """ + insert into decimalv2_overflow_test2 values(999999999999999999.999999999, 1.1); + """ + test { + sql """ + select /*+SET_VAR(enable_fold_constant_by_be=true)*/ cast(999999999999999999.999999999 as decimalv2(27,9)) * cast(1.1 as decimalv2(27,9)); + """ + exception "Arithmetic overflow" + } + test { + sql """ + select /*+SET_VAR(enable_fold_constant_by_be=true)*/ cast(999999999999999999.999999999 as decimalv2(27,9)) * cast(999999999999999999.999999999 as decimalv2(27,9)); + """ + exception "Arithmetic overflow" + } + test { + sql """ + select k1, k1 * cast(1.1 as decimalv2(27,9)) from decimalv2_overflow_test2 order by 1, 2; + """ + exception "Arithmetic overflow" + } + test { + sql """ + select k1, cast(1.1 as decimalv2(27,9)) * k1 from decimalv2_overflow_test2 order by 1, 2; + """ + exception "Arithmetic overflow" + } + test { + sql """ + select k1, k2, k1 * k2 from decimalv2_overflow_test2 order by 1, 2; + """ + exception "Arithmetic overflow" + } + + // divide + prepare_decimalv2_overflow_test2() + sql """ + insert into decimalv2_overflow_test2 values(99999999999999999.999999999, 0.1); + """ + qt_div_overflow1 """ + select k1, k2, k1 / k2 from decimalv2_overflow_test2 order by 1, 2; + """ + qt_div_overflow2 """ + select /*+SET_VAR(enable_fold_constant_by_be=true)*/ cast(99999999999999999.999999999 as decimalv2(27,9)) / cast(0.1 as decimalv2(27,9)); + """ + qt_div_overflow3 """ + select k1, 0.1, k1 / 0.1 from decimalv2_overflow_test2 order by 1, 2, 3; + """ + qt_div_overflow4 """ + select cast(99999999999999999.999999999 as decimalv2(27,9)) / k2 from decimalv2_overflow_test2 order by 1; + """ + + // divide overflow + prepare_decimalv2_overflow_test2() + sql """ + insert into decimalv2_overflow_test2 values(999999999999999999.999999999, 0.1); + """ + test { + sql """ + select k1, k2, k1 / k2 from decimalv2_overflow_test2; + """ + exception "Arithmetic overflow" + } + test { + sql """ + select /*+SET_VAR(enable_fold_constant_by_be=true)*/ cast(999999999999999999.999999999 as decimalv2(27,9)) / cast(0.1 as decimalv2(27,9)); + """ + exception "Arithmetic overflow" + } + test { + sql """ + select /*+SET_VAR(enable_fold_constant_by_be=true)*/ cast(999999999999999999.999999999 as decimalv2(27,9)) / cast(0.000000001 as decimalv2(27,9)); + """ + exception "Arithmetic overflow" + } + test { + sql """ + select k1, 0.1, k1 / cast(0.1 as decimalv2(27,9)) from decimalv2_overflow_test2; + """ + exception "Arithmetic overflow" + } + test { + sql """ + select cast(999999999999999999.999999999 as decimalv2(27,9)) / k2 from decimalv2_overflow_test2; + """ + exception "Arithmetic overflow" + } + + // mod + prepare_decimalv2_overflow_test2() + sql """ + insert into decimalv2_overflow_test2 values(99999999999999999.999999999, 0.1); + """ + qt_mod1 """ + select k1, k2, k1 % k2 from decimalv2_overflow_test2 order by 1, 2; + """ + qt_mod2 """ + select cast(99999999999999999.999999999 as decimalv2(27,9)) % cast(0.1 as decimalv2(27,9)); + """ + qt_mod3 """ + select k1, 0.1, k1 % 0.1 from decimalv2_overflow_test2 order by 1, 2, 3; + """ + qt_mod4 """ + select cast(99999999999999999.999999999 as decimalv2(27,9)) % k2 from decimalv2_overflow_test2 order by 1; + """ + + + // TODO + // decimalv2 +-*/ integer + // integer +-*/ decimalv2 + + // decimalv2 +-*/ decimalv3 + // decimalv3 +-*/ decimalv2 + + // decimalv2 mod, pmod + + // decimalv2 largeint + + // decimalv3 +-*/ integer + // integer +-*/ decimalv3 + + /// Decimal Real is not supported (traditional DBs convert Decimal Real to Real) + // {decimalv2 | decimalv3} +-*/ {float, double} +} diff --git a/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_overflow.groovy b/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_overflow.groovy index 3e0e094421..f371c4c812 100644 --- a/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_overflow.groovy +++ b/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_overflow.groovy @@ -41,9 +41,7 @@ suite("test_decimalv3_overflow") { ); """ sql "insert into ${tblName2} values(705091149953414452.46)" - // qt_sql1 """ select c2 / 10000 * c1 from ${tblName1}, ${tblName2}; """ - - sql """ select c2 / 10000 * c1 from ${tblName1}, ${tblName2}; """ + qt_sql1 """ select c2 / 10000 * c1 from ${tblName1}, ${tblName2}; """ //======================================= // decimal32