From 3328a65b75c9678d8ad24633225e577fd088e8c4 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Thu, 20 Apr 2023 11:02:53 +0800 Subject: [PATCH] [Fix](mutli-catalog) Use decimal v3 type to fix decimal loss issue in multi-catalog module. (#18835) Fix decimal v3 precision loss issues in the multi-catalog module. Now it will use decimal v3 to represent decimal type in the multi-catalog module. Regression Test: `test_load_with_decimal.groovy` --- be/src/runtime/types.h | 18 + .../vec/exec/format/json/new_json_reader.cpp | 5 +- be/src/vec/exec/format/orc/vorc_reader.cpp | 4 +- .../vec/exec/format/parquet/schema_desc.cpp | 15 +- be/src/vec/exec/format/parquet/schema_desc.h | 2 +- .../java/org/apache/doris/catalog/Column.java | 4 + .../catalog/HiveMetaStoreClientHelper.java | 2 +- .../ExternalFileTableValuedFunction.java | 5 +- .../table_valued_function/test_hdfs_tvf.out | 82 +- .../hive/test_tvf_p2.out | 58 +- .../data/load_p0/stream_load/test_decimal.csv | 5 + .../load_p0/stream_load/test_decimal.json | 7 + .../data/load_p0/stream_load/test_decimal.orc | Bin 0 -> 1262 bytes .../load_p0/stream_load/test_decimal.parquet | Bin 0 -> 1402 bytes .../stream_load/test_load_with_decimal.out | 29 + .../multi_catalog_query/hive_catalog_orc.out | 2674 ++++++++--------- .../hive_catalog_parquet.out | 2674 ++++++++--------- .../stream_load/test_load_with_decimal.groovy | 90 + 18 files changed, 2918 insertions(+), 2756 deletions(-) create mode 100644 regression-test/data/load_p0/stream_load/test_decimal.csv create mode 100644 regression-test/data/load_p0/stream_load/test_decimal.json create mode 100644 regression-test/data/load_p0/stream_load/test_decimal.orc create mode 100644 regression-test/data/load_p0/stream_load/test_decimal.parquet create mode 100644 regression-test/data/load_p0/stream_load/test_load_with_decimal.out create mode 100644 regression-test/suites/load_p0/stream_load/test_load_with_decimal.groovy diff --git a/be/src/runtime/types.h b/be/src/runtime/types.h index 82df1d01d6..5950fe55d7 100644 --- a/be/src/runtime/types.h +++ b/be/src/runtime/types.h @@ -126,6 +126,24 @@ struct TypeDescriptor { return ret; } + static TypeDescriptor create_decimalv3_type(int precision, int scale) { + DCHECK_LE(precision, MAX_PRECISION); + DCHECK_LE(scale, MAX_SCALE); + DCHECK_GE(precision, 0); + DCHECK_LE(scale, precision); + TypeDescriptor ret; + if (precision <= MAX_DECIMAL4_PRECISION) { + ret.type = TYPE_DECIMAL32; + } else if (precision <= MAX_DECIMAL8_PRECISION) { + ret.type = TYPE_DECIMAL64; + } else { + ret.type = TYPE_DECIMAL128I; + } + ret.precision = precision; + ret.scale = scale; + return ret; + } + static TypeDescriptor from_thrift(const TTypeDesc& t) { int idx = 0; TypeDescriptor result(t.types, &idx); diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp index 41bede7c4f..3668cd51f5 100644 --- a/be/src/vec/exec/format/json/new_json_reader.cpp +++ b/be/src/vec/exec/format/json/new_json_reader.cpp @@ -913,8 +913,11 @@ Status NewJsonReader::_write_data_to_column(rapidjson::Value::ConstValueIterator wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%" PRIu64, value->GetUint64()); } else if (value->IsInt64()) { wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%" PRId64, value->GetInt64()); + } else if (value->IsFloat() || value->IsDouble()) { + auto end = fmt::format_to(tmp_buf, "{}", value->GetDouble()); + wbytes = end - tmp_buf; } else { - wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%f", value->GetDouble()); + return Status::InternalError("It should not here."); } str_value = tmp_buf; break; diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp index 6644121b00..475b76631a 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.cpp +++ b/be/src/vec/exec/format/orc/vorc_reader.cpp @@ -623,8 +623,8 @@ TypeDescriptor OrcReader::_convert_to_doris_type(const orc::Type* orc_type) { case orc::TypeKind::TIMESTAMP: return TypeDescriptor(PrimitiveType::TYPE_DATETIMEV2); case orc::TypeKind::DECIMAL: - // TODO: using decimal v3 instead - return TypeDescriptor(PrimitiveType::TYPE_DECIMALV2); + return TypeDescriptor::create_decimalv3_type(orc_type->getPrecision(), + orc_type->getScale()); case orc::TypeKind::DATE: return TypeDescriptor(PrimitiveType::TYPE_DATEV2); case orc::TypeKind::VARCHAR: diff --git a/be/src/vec/exec/format/parquet/schema_desc.cpp b/be/src/vec/exec/format/parquet/schema_desc.cpp index 0cf8bef696..60e6b640ac 100644 --- a/be/src/vec/exec/format/parquet/schema_desc.cpp +++ b/be/src/vec/exec/format/parquet/schema_desc.cpp @@ -177,7 +177,7 @@ TypeDescriptor FieldDescriptor::get_doris_type(const tparquet::SchemaElement& ph if (physical_schema.__isset.logicalType) { type = convert_to_doris_type(physical_schema.logicalType); } else if (physical_schema.__isset.converted_type) { - type = convert_to_doris_type(physical_schema.converted_type); + type = convert_to_doris_type(physical_schema); } // use physical type instead if (type.type == INVALID_TYPE) { @@ -218,7 +218,8 @@ TypeDescriptor FieldDescriptor::convert_to_doris_type(tparquet::LogicalType logi if (logicalType.__isset.STRING) { type = TypeDescriptor(TYPE_STRING); } else if (logicalType.__isset.DECIMAL) { - type = TypeDescriptor(TYPE_DECIMALV2); + type = TypeDescriptor::create_decimalv3_type(logicalType.DECIMAL.precision, + logicalType.DECIMAL.scale); } else if (logicalType.__isset.DATE) { type = TypeDescriptor(TYPE_DATEV2); } else if (logicalType.__isset.INTEGER) { @@ -245,14 +246,16 @@ TypeDescriptor FieldDescriptor::convert_to_doris_type(tparquet::LogicalType logi return type; } -TypeDescriptor FieldDescriptor::convert_to_doris_type(tparquet::ConvertedType::type convertedType) { +TypeDescriptor FieldDescriptor::convert_to_doris_type( + const tparquet::SchemaElement& physical_schema) { TypeDescriptor type; - switch (convertedType) { + switch (physical_schema.converted_type) { case tparquet::ConvertedType::type::UTF8: type = TypeDescriptor(TYPE_STRING); break; case tparquet::ConvertedType::type::DECIMAL: - type = TypeDescriptor(TYPE_DECIMALV2); + type = TypeDescriptor::create_decimalv3_type(physical_schema.precision, + physical_schema.scale); break; case tparquet::ConvertedType::type::DATE: type = TypeDescriptor(TYPE_DATEV2); @@ -288,7 +291,7 @@ TypeDescriptor FieldDescriptor::convert_to_doris_type(tparquet::ConvertedType::t type = TypeDescriptor(TYPE_BIGINT); break; default: - LOG(WARNING) << "Not supported parquet ConvertedType: " << convertedType; + LOG(WARNING) << "Not supported parquet ConvertedType: " << physical_schema.converted_type; type = TypeDescriptor(INVALID_TYPE); break; } diff --git a/be/src/vec/exec/format/parquet/schema_desc.h b/be/src/vec/exec/format/parquet/schema_desc.h index ad6b8fb293..fb61ad918a 100644 --- a/be/src/vec/exec/format/parquet/schema_desc.h +++ b/be/src/vec/exec/format/parquet/schema_desc.h @@ -86,7 +86,7 @@ private: TypeDescriptor convert_to_doris_type(tparquet::LogicalType logicalType); - TypeDescriptor convert_to_doris_type(tparquet::ConvertedType::type convertedType); + TypeDescriptor convert_to_doris_type(const tparquet::SchemaElement& physical_schema); TypeDescriptor get_doris_type(const tparquet::SchemaElement& physical_schema); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index 0b96d0f4df..d6a5b1b576 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -144,6 +144,10 @@ public class Column implements Writable, GsonPostProcessable { this(name, ScalarType.createType(dataType), isAllowNull); } + public Column(String name, PrimitiveType dataType, int len, int precision, int scale, boolean isAllowNull) { + this(name, ScalarType.createType(dataType, len, precision, scale), isAllowNull); + } + public Column(String name, Type type, boolean isAllowNull) { this(name, type, false, null, isAllowNull, null, ""); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java index 0bbfbee81e..e3cb598f24 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java @@ -797,7 +797,7 @@ public class HiveMetaStoreClientHelper { if (match.find()) { scale = Integer.parseInt(match.group(1)); } - return ScalarType.createDecimalType(precision, scale); + return ScalarType.createDecimalV3Type(precision, scale); } return Type.UNSUPPORTED; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index 8d64ed476c..e50c6caa45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -380,7 +380,9 @@ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctio // only support ScalarType. PScalarType scalarType = typeNode.getScalarType(); TPrimitiveType tPrimitiveType = TPrimitiveType.findByValue(scalarType.getType()); - columns.add(new Column(colName, PrimitiveType.fromThrift(tPrimitiveType), true)); + columns.add(new Column(colName, PrimitiveType.fromThrift(tPrimitiveType), + scalarType.getLen() <= 0 ? -1 : scalarType.getLen(), scalarType.getPrecision(), + scalarType.getScale(), true)); } } } @@ -426,3 +428,4 @@ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctio .setFileScanRange(ByteString.copyFrom(new TSerializer().serialize(fileScanRange))).build(); } } + diff --git a/regression-test/data/correctness_p0/table_valued_function/test_hdfs_tvf.out b/regression-test/data/correctness_p0/table_valued_function/test_hdfs_tvf.out index ec4ead39c8..12f1866162 100644 --- a/regression-test/data/correctness_p0/table_valued_function/test_hdfs_tvf.out +++ b/regression-test/data/correctness_p0/table_valued_function/test_hdfs_tvf.out @@ -176,48 +176,48 @@ 10 nami 18 -- !parquet -- -1 Supplier#000000001 N kD4on9OM Ipw3,gf0JBoQDd7tgrzrddZ 17 27-918-335-1736 5755 each slyly above the careful -2 Supplier#000000002 89eJ5ksX3ImxJQBvxObC, 5 15-679-861-2259 4032 slyly bold instructions. idle dependen -3 Supplier#000000003 q1,G3Pj6OjIuUYfUoH18BFTKP5aU9bEV3 1 11-383-516-1199 4192 blithely silent requests after the express dependencies are sl -4 Supplier#000000004 Bk7ah4CK8SYQTepEmvMkkgMwg 15 25-843-787-7479 4641 riously even requests above the exp -5 Supplier#000000005 Gcdm2rJRzl5qlTVzc 11 21-151-690-3663 -283 . slyly regular pinto bea -6 Supplier#000000006 tQxuVm7s7CnK 14 24-696-997-4969 1365 final accounts. regular dolphins use against the furiously ironic decoys. -7 Supplier#000000007 s,4TicNGB4uO6PaSqNBUq 23 33-990-965-2201 6820 s unwind silently furiously regular courts. final requests are deposits. requests wake quietly blit -8 Supplier#000000008 9Sq4bBH2FQEmaFOocY45sRTxo6yuoG 17 27-498-742-3860 7627 al pinto beans. asymptotes haggl -9 Supplier#000000009 1KhUgZegwM3ua7dsYmekYBsK 10 20-403-398-8662 5302 s. unusual, even requests along the furiously regular pac -10 Supplier#000000010 Saygah3gYWMp72i PY 24 34-852-489-8585 3891 ing waters. regular requests ar -11 Supplier#000000011 JfwTs,LZrV, M,9C 18 28-613-996-1505 3393 y ironic packages. slyly ironic accounts affix furiously; ironically unusual excuses across the flu -12 Supplier#000000012 aLIW q0HYd 8 18-179-925-7181 1432 al packages nag alongside of the bold instructions. express, daring accounts -13 Supplier#000000013 HK71HQyWoqRWOX8GI FpgAifW,2PoH 3 13-727-620-7813 9107 requests engage regularly instructions. furiously special requests ar -14 Supplier#000000014 EXsnO5pTNj4iZRm 15 25-656-247-5058 9189 l accounts boost. fluffily bold warhorses wake -15 Supplier#000000015 olXVbNBfVzRqgokr1T,Ie 8 18-453-357-6394 308 across the furiously regular platelets wake even deposits. quickly express she -16 Supplier#000000016 YjP5C55zHDXL7LalK27zfQnwejdpin4AMpvh 22 32-822-502-4215 2972 ously express ideas haggle quickly dugouts? fu -17 Supplier#000000017 c2d,ESHRSkK3WYnxpgw6aOqN0q 19 29-601-884-9219 1687 eep against the furiously bold ideas. fluffily bold packa -18 Supplier#000000018 PGGVE5PWAMwKDZw 16 26-729-551-1115 7040 accounts snooze slyly furiously bold -19 Supplier#000000019 edZT3es,nBFD8lBXTGeTl 24 34-278-310-2731 6150 refully final foxes across the dogged theodolites sleep slyly abou -20 Supplier#000000020 iybAE,RmTymrZVYaFZva2SH,j 3 13-715-945-6730 530 n, ironic ideas would nag blithely about the slyly regular accounts. silent, expr +1 Supplier#000000001 N kD4on9OM Ipw3,gf0JBoQDd7tgrzrddZ 17 27-918-335-1736 5755.94 each slyly above the careful +2 Supplier#000000002 89eJ5ksX3ImxJQBvxObC, 5 15-679-861-2259 4032.68 slyly bold instructions. idle dependen +3 Supplier#000000003 q1,G3Pj6OjIuUYfUoH18BFTKP5aU9bEV3 1 11-383-516-1199 4192.40 blithely silent requests after the express dependencies are sl +4 Supplier#000000004 Bk7ah4CK8SYQTepEmvMkkgMwg 15 25-843-787-7479 4641.08 riously even requests above the exp +5 Supplier#000000005 Gcdm2rJRzl5qlTVzc 11 21-151-690-3663 -283.84 . slyly regular pinto bea +6 Supplier#000000006 tQxuVm7s7CnK 14 24-696-997-4969 1365.79 final accounts. regular dolphins use against the furiously ironic decoys. +7 Supplier#000000007 s,4TicNGB4uO6PaSqNBUq 23 33-990-965-2201 6820.35 s unwind silently furiously regular courts. final requests are deposits. requests wake quietly blit +8 Supplier#000000008 9Sq4bBH2FQEmaFOocY45sRTxo6yuoG 17 27-498-742-3860 7627.85 al pinto beans. asymptotes haggl +9 Supplier#000000009 1KhUgZegwM3ua7dsYmekYBsK 10 20-403-398-8662 5302.37 s. unusual, even requests along the furiously regular pac +10 Supplier#000000010 Saygah3gYWMp72i PY 24 34-852-489-8585 3891.91 ing waters. regular requests ar +11 Supplier#000000011 JfwTs,LZrV, M,9C 18 28-613-996-1505 3393.08 y ironic packages. slyly ironic accounts affix furiously; ironically unusual excuses across the flu +12 Supplier#000000012 aLIW q0HYd 8 18-179-925-7181 1432.69 al packages nag alongside of the bold instructions. express, daring accounts +13 Supplier#000000013 HK71HQyWoqRWOX8GI FpgAifW,2PoH 3 13-727-620-7813 9107.22 requests engage regularly instructions. furiously special requests ar +14 Supplier#000000014 EXsnO5pTNj4iZRm 15 25-656-247-5058 9189.82 l accounts boost. fluffily bold warhorses wake +15 Supplier#000000015 olXVbNBfVzRqgokr1T,Ie 8 18-453-357-6394 308.56 across the furiously regular platelets wake even deposits. quickly express she +16 Supplier#000000016 YjP5C55zHDXL7LalK27zfQnwejdpin4AMpvh 22 32-822-502-4215 2972.26 ously express ideas haggle quickly dugouts? fu +17 Supplier#000000017 c2d,ESHRSkK3WYnxpgw6aOqN0q 19 29-601-884-9219 1687.81 eep against the furiously bold ideas. fluffily bold packa +18 Supplier#000000018 PGGVE5PWAMwKDZw 16 26-729-551-1115 7040.82 accounts snooze slyly furiously bold +19 Supplier#000000019 edZT3es,nBFD8lBXTGeTl 24 34-278-310-2731 6150.38 refully final foxes across the dogged theodolites sleep slyly abou +20 Supplier#000000020 iybAE,RmTymrZVYaFZva2SH,j 3 13-715-945-6730 530.82 n, ironic ideas would nag blithely about the slyly regular accounts. silent, expr -- !orc -- -1 goldenrod lavender spring chocolate lace Manufacturer#1 Brand#13 PROMO BURNISHED COPPER 7 JUMBO PKG 901 ly. slyly ironi -2 blush thistle blue yellow saddle Manufacturer#1 Brand#13 LARGE BRUSHED BRASS 1 LG CASE 902 lar accounts amo -3 spring green yellow purple cornsilk Manufacturer#4 Brand#42 STANDARD POLISHED BRASS 21 WRAP CASE 903 egular deposits hag -4 cornflower chocolate smoke green pink Manufacturer#3 Brand#34 SMALL PLATED BRASS 14 MED DRUM 904 p furiously r -5 forest brown coral puff cream Manufacturer#3 Brand#32 STANDARD POLISHED TIN 15 SM PKG 905 wake carefully -6 bisque cornflower lawn forest magenta Manufacturer#2 Brand#24 PROMO PLATED STEEL 4 MED BAG 906 sual a -7 moccasin green thistle khaki floral Manufacturer#1 Brand#11 SMALL PLATED COPPER 45 SM BAG 907 lyly. ex -8 misty lace thistle snow royal Manufacturer#4 Brand#44 PROMO BURNISHED TIN 41 LG DRUM 908 eposi -9 thistle dim navajo dark gainsboro Manufacturer#4 Brand#43 SMALL BURNISHED STEEL 12 WRAP CASE 909 ironic foxe -10 linen pink saddle puff powder Manufacturer#5 Brand#54 LARGE BURNISHED STEEL 44 LG CAN 910 ithely final deposit -11 spring maroon seashell almond orchid Manufacturer#2 Brand#25 STANDARD BURNISHED NICKEL 43 WRAP BOX 911 ng gr -12 cornflower wheat orange maroon ghost Manufacturer#3 Brand#33 MEDIUM ANODIZED STEEL 25 JUMBO CASE 912 quickly -13 ghost olive orange rosy thistle Manufacturer#5 Brand#55 MEDIUM BURNISHED NICKEL 1 JUMBO PACK 913 osits. -14 khaki seashell rose cornsilk navajo Manufacturer#1 Brand#13 SMALL POLISHED STEEL 28 JUMBO BOX 914 kages c -15 blanched honeydew sky turquoise medium Manufacturer#1 Brand#15 LARGE ANODIZED BRASS 45 LG CASE 915 usual ac -16 deep sky turquoise drab peach Manufacturer#3 Brand#32 PROMO PLATED TIN 2 MED PACK 916 unts a -17 indian navy coral pink deep Manufacturer#4 Brand#43 ECONOMY BRUSHED STEEL 16 LG BOX 917 regular accounts -18 turquoise indian lemon lavender misty Manufacturer#1 Brand#11 SMALL BURNISHED STEEL 42 JUMBO PACK 918 s cajole slyly a -19 chocolate navy tan deep brown Manufacturer#2 Brand#23 SMALL ANODIZED NICKEL 33 WRAP BOX 919 pending acc -20 ivory navy honeydew sandy midnight Manufacturer#1 Brand#12 LARGE POLISHED NICKEL 48 MED BAG 920 are across the asympt +1 goldenrod lavender spring chocolate lace Manufacturer#1 Brand#13 PROMO BURNISHED COPPER 7 JUMBO PKG 901.00 ly. slyly ironi +2 blush thistle blue yellow saddle Manufacturer#1 Brand#13 LARGE BRUSHED BRASS 1 LG CASE 902.00 lar accounts amo +3 spring green yellow purple cornsilk Manufacturer#4 Brand#42 STANDARD POLISHED BRASS 21 WRAP CASE 903.00 egular deposits hag +4 cornflower chocolate smoke green pink Manufacturer#3 Brand#34 SMALL PLATED BRASS 14 MED DRUM 904.00 p furiously r +5 forest brown coral puff cream Manufacturer#3 Brand#32 STANDARD POLISHED TIN 15 SM PKG 905.00 wake carefully +6 bisque cornflower lawn forest magenta Manufacturer#2 Brand#24 PROMO PLATED STEEL 4 MED BAG 906.00 sual a +7 moccasin green thistle khaki floral Manufacturer#1 Brand#11 SMALL PLATED COPPER 45 SM BAG 907.00 lyly. ex +8 misty lace thistle snow royal Manufacturer#4 Brand#44 PROMO BURNISHED TIN 41 LG DRUM 908.00 eposi +9 thistle dim navajo dark gainsboro Manufacturer#4 Brand#43 SMALL BURNISHED STEEL 12 WRAP CASE 909.00 ironic foxe +10 linen pink saddle puff powder Manufacturer#5 Brand#54 LARGE BURNISHED STEEL 44 LG CAN 910.01 ithely final deposit +11 spring maroon seashell almond orchid Manufacturer#2 Brand#25 STANDARD BURNISHED NICKEL 43 WRAP BOX 911.01 ng gr +12 cornflower wheat orange maroon ghost Manufacturer#3 Brand#33 MEDIUM ANODIZED STEEL 25 JUMBO CASE 912.01 quickly +13 ghost olive orange rosy thistle Manufacturer#5 Brand#55 MEDIUM BURNISHED NICKEL 1 JUMBO PACK 913.01 osits. +14 khaki seashell rose cornsilk navajo Manufacturer#1 Brand#13 SMALL POLISHED STEEL 28 JUMBO BOX 914.01 kages c +15 blanched honeydew sky turquoise medium Manufacturer#1 Brand#15 LARGE ANODIZED BRASS 45 LG CASE 915.01 usual ac +16 deep sky turquoise drab peach Manufacturer#3 Brand#32 PROMO PLATED TIN 2 MED PACK 916.01 unts a +17 indian navy coral pink deep Manufacturer#4 Brand#43 ECONOMY BRUSHED STEEL 16 LG BOX 917.01 regular accounts +18 turquoise indian lemon lavender misty Manufacturer#1 Brand#11 SMALL BURNISHED STEEL 42 JUMBO PACK 918.01 s cajole slyly a +19 chocolate navy tan deep brown Manufacturer#2 Brand#23 SMALL ANODIZED NICKEL 33 WRAP BOX 919.01 pending acc +20 ivory navy honeydew sandy midnight Manufacturer#1 Brand#12 LARGE POLISHED NICKEL 48 MED BAG 920.02 are across the asympt -- !json -- 1 beijing 2345671 @@ -293,6 +293,6 @@ s_name TEXT Yes false \N NONE s_address TEXT Yes false \N NONE s_nationkey INT Yes false \N NONE s_phone TEXT Yes false \N NONE -s_acctbal DECIMAL(9, 0) Yes false \N NONE +s_acctbal DECIMAL(12, 2) Yes false \N NONE s_comment TEXT Yes false \N NONE diff --git a/regression-test/data/external_table_emr_p2/hive/test_tvf_p2.out b/regression-test/data/external_table_emr_p2/hive/test_tvf_p2.out index 7f94b13974..6ab3cc9f3e 100644 --- a/regression-test/data/external_table_emr_p2/hive/test_tvf_p2.out +++ b/regression-test/data/external_table_emr_p2/hive/test_tvf_p2.out @@ -1,32 +1,32 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !eof_check -- -2451718 \N 9242 \N \N 2886 \N 4 250 1374252 18 \N \N \N 0 1513 1435 \N \N 0 \N 1588 78 -\N \N 14846 1945858 \N 1015 \N 4 581 2383831 \N \N 5 1 0 110 \N \N \N 0 110 \N -213 -\N 50835 25618 1166535 \N 1748 \N 4 \N 2880907 7 \N 17 \N \N 115 \N 125 1 \N 115 \N \N -2452195 45280 29385 1298621 1649018 1815 \N 4 \N 3379765 24 73 \N \N 0 2617 1770 3399 \N 0 \N 2826 \N -2451488 53117 31945 \N 8644 \N \N 4 783 4877135 100 \N \N \N \N \N 3450 \N \N \N 565 581 -2885 -\N 53900 35887 702626 \N 2568 \N 4 \N 2381514 \N \N \N 0 \N \N \N \N 1 \N 19 20 -357 -\N 53985 38881 760602 289764 \N \N 4 227 3377513 68 75 \N \N \N 5833 \N \N \N \N 524 \N -4588 -\N \N 51685 1833943 \N \N \N 4 \N 1879197 \N \N \N \N 0 46 163 \N \N 0 \N 49 -116 -\N \N 62073 \N 287578 \N \N 4 990 16264789 90 91 \N \N 0 6381 8247 \N \N 0 6381 \N \N -\N 34914 64259 167395 897626 \N \N 4 327 19379058 15 \N \N 51 \N \N 1480 \N \N \N \N \N -707 -\N 70509 100949 \N \N \N \N 4 185 2381361 35 1 \N \N 0 \N 41 \N \N 0 \N 82 33 -2452489 74165 103575 \N 1359778 \N \N 4 \N 2383538 1 \N 23 0 \N 0 15 23 \N \N 0 \N -14 -2451253 \N 111502 246668 \N \N \N 4 \N 2881367 \N \N \N 21 0 \N \N 1218 74 0 \N 999 -49 -2451093 \N 121339 \N \N \N \N 4 894 19379088 11 92 \N \N \N \N \N 1364 9 \N 305 314 \N -2452592 \N 126241 1673449 1682209 \N \N 4 \N 16872613 \N 4 7 4 0 20 20 \N \N 0 \N 21 0 -2451985 \N 128921 \N \N \N \N 4 \N 17767165 \N \N \N \N \N \N \N \N 55 \N 929 \N -1765 -\N \N 137292 1348279 1465562 6398 \N 4 922 372083 \N \N 130 65 \N 3772 3772 7544 \N \N \N 4036 \N -\N \N 146095 494197 \N 2565 \N 4 352 17767165 \N \N 24 22 776 \N 1877 2158 \N 776 1165 \N \N -\N \N 158078 889184 \N \N \N 4 \N 369514 \N 31 48 31 0 501 \N \N 20 0 \N \N \N -\N \N 159005 397184 1836520 3856 \N 4 \N 371981 \N \N \N 0 0 0 \N 208 0 0 0 0 -163 -\N 70509 160172 \N 1578153 5827 \N 4 \N 2381361 32 \N \N 1 \N 56 3037 \N \N \N 56 61 \N -\N \N 166592 1688819 891341 \N \N 4 535 1879015 38 77 125 56 \N 2148 \N \N 107 \N \N 2256 \N -2452394 45881 167121 1409858 294280 \N \N 4 752 2381343 63 \N \N \N \N \N 2271 4473 \N \N 2460 \N \N -\N 58568 174392 742421 \N \N \N 4 104 19379109 100 75 93 \N \N \N \N \N 54 \N \N 5494 -2125 -2451460 71227 178990 \N \N \N \N 4 \N 2878825 22 59 \N 10 0 \N 1317 \N 9 0 239 \N \N -\N \N 196777 \N \N \N \N 4 954 2381557 84 \N 4 \N \N \N 216 \N \N \N 142 \N -74 -\N \N 198073 \N 1537282 3742 \N 4 \N 3878103 31 \N \N 7 \N 239 \N 615 \N \N \N \N -370 -2451656 \N 200558 \N \N 1066 \N 4 \N 2381557 \N 79 \N 94 0 \N \N 12956 481 0 \N 8514 1248 -\N \N 203791 \N 1655274 6679 \N 4 \N 3379960 71 \N 96 45 \N 3214 3525 6840 160 \N \N \N \N +2451718 \N 9242 \N \N 2886 \N 4 250 1374252 18 \N \N \N 0.00 1513.26 1435.14 \N \N 0.00 \N 1588.92 78.12 +\N \N 14846 1945858 \N 1015 \N 4 581 2383831 \N \N 5.53 1.54 0.00 110.88 \N \N \N 0.00 110.88 \N -213.12 +\N 50835 25618 1166535 \N 1748 \N 4 \N 2880907 7 \N 17.89 \N \N 115.15 \N 125.23 1.15 \N 115.15 \N \N +2452195 45280 29385 1298621 1649018 1815 \N 4 \N 3379765 24 73.77 \N \N 0.00 2617.20 1770.48 3399.12 \N 0.00 \N 2826.57 \N +2451488 53117 31945 \N 8644 \N \N 4 783 4877135 100 \N \N \N \N \N 3450.00 \N \N \N 565.00 581.95 -2885.00 +\N 53900 35887 702626 \N 2568 \N 4 \N 2381514 \N \N \N 0.59 \N \N \N \N 1.36 \N 19.47 20.83 -357.06 +\N 53985 38881 760602 289764 \N \N 4 227 3377513 68 75.20 \N \N \N 5833.04 \N \N \N \N 524.98 \N -4588.62 +\N \N 51685 1833943 \N \N \N 4 \N 1879197 \N \N \N \N 0.00 46.40 163.04 \N \N 0.00 \N 49.64 -116.64 +\N \N 62073 \N 287578 \N \N 4 990 16264789 90 91.64 \N \N 0.00 6381.90 8247.60 \N \N 0.00 6381.90 \N \N +\N 34914 64259 167395 897626 \N \N 4 327 19379058 15 \N \N 51.50 \N \N 1480.05 \N \N \N \N \N -707.55 +\N 70509 100949 \N \N \N \N 4 185 2381361 35 1.18 \N \N 0.00 \N 41.30 \N \N 0.00 \N 82.02 33.95 +2452489 74165 103575 \N 1359778 \N \N 4 \N 2383538 1 \N 23.08 0.46 \N 0.46 15.29 23.08 \N \N 0.46 \N -14.83 +2451253 \N 111502 246668 \N \N \N 4 \N 2881367 \N \N \N 21.53 0.00 \N \N 1218.19 74.06 0.00 \N 999.85 -49.02 +2451093 \N 121339 \N \N \N \N 4 894 19379088 11 92.56 \N \N \N \N \N 1364.33 9.16 \N 305.60 314.76 \N +2452592 \N 126241 1673449 1682209 \N \N 4 \N 16872613 \N 4.14 7.32 4.02 0.00 20.10 20.70 \N \N 0.00 \N 21.90 -0.60 +2451985 \N 128921 \N \N \N \N 4 \N 17767165 \N \N \N \N \N \N \N \N 55.75 \N 929.22 \N -1765.28 +\N \N 137292 1348279 1465562 6398 \N 4 922 372083 \N \N 130.08 65.04 \N 3772.32 3772.32 7544.64 \N \N \N 4036.38 \N +\N \N 146095 494197 \N 2565 \N 4 352 17767165 \N \N 24.81 22.32 776.73 \N 1877.46 2158.47 \N 776.73 1165.11 \N \N +\N \N 158078 889184 \N \N \N 4 \N 369514 \N 31.92 48.19 31.32 0.00 501.12 \N \N 20.04 0.00 \N \N \N +\N \N 159005 397184 1836520 3856 \N 4 \N 371981 \N \N \N 0.00 0.00 0.00 \N 208.88 0.00 0.00 0.00 0.00 -163.20 +\N 70509 160172 \N 1578153 5827 \N 4 \N 2381361 32 \N \N 1.77 \N 56.64 3037.76 \N \N \N 56.64 61.17 \N +\N \N 166592 1688819 891341 \N \N 4 535 1879015 38 77.10 125.67 56.55 \N 2148.90 \N \N 107.44 \N \N 2256.34 \N +2452394 45881 167121 1409858 294280 \N \N 4 752 2381343 63 \N \N \N \N \N 2271.15 4473.63 \N \N 2460.15 \N \N +\N 58568 174392 742421 \N \N \N 4 104 19379109 100 75.65 93.80 \N \N \N \N \N 54.40 \N \N 5494.40 -2125.00 +2451460 71227 178990 \N \N \N \N 4 \N 2878825 22 59.88 \N 10.88 0.00 \N 1317.36 \N 9.57 0.00 239.36 \N \N +\N \N 196777 \N \N \N \N 4 954 2381557 84 \N 4.97 \N \N \N 216.72 \N \N \N 142.39 \N -74.33 +\N \N 198073 \N 1537282 3742 \N 4 \N 3878103 31 \N \N 7.74 \N 239.94 \N 615.97 \N \N \N \N -370.14 +2451656 \N 200558 \N \N 1066 \N 4 \N 2381557 \N 79.81 \N 94.50 0.00 \N \N 12956.55 481.95 0.00 \N 8514.45 1248.65 +\N \N 203791 \N 1655274 6679 \N 4 \N 3379960 71 \N 96.34 45.27 \N 3214.17 3525.86 6840.14 160.70 \N \N \N \N diff --git a/regression-test/data/load_p0/stream_load/test_decimal.csv b/regression-test/data/load_p0/stream_load/test_decimal.csv new file mode 100644 index 0000000000..1e182970a2 --- /dev/null +++ b/regression-test/data/load_p0/stream_load/test_decimal.csv @@ -0,0 +1,5 @@ +1,1.1234,12.123456,123.123456789876,12,1234.123456789,123 +2,1234.1234,123456789123.123456,12345678912345678912345678.123456789876,123456789,123456789123456789.12345678,987654321 +3,1234,123456789123,12345678912345678912345678,123456789,123456789123456789,987654321 +4,NULL,NULL,123.123456789876,12,NULL,123 +5,1.1234,12.123456,NULL,NULL,1234.123456789,NULL diff --git a/regression-test/data/load_p0/stream_load/test_decimal.json b/regression-test/data/load_p0/stream_load/test_decimal.json new file mode 100644 index 0000000000..b52821ef44 --- /dev/null +++ b/regression-test/data/load_p0/stream_load/test_decimal.json @@ -0,0 +1,7 @@ +[ + {"id": 1, "decimal_col1": 1.1234, "decimal_col2": 12.123456, "decimal_col3": 123.123456789876, "decimal_col4": 12, "decimal_col5": 1234.123456789, "decimal_col6": 123}, + {"id": 2, "decimal_col1": 1234.1234, "decimal_col2": 123456789123.123456, "decimal_col3": 12345678912345678912345678.123456789876, "decimal_col4": 123456789, "decimal_col5": 123456789123456789.123456780, "decimal_col6": 987654321}, + {"id": 3, "decimal_col1": 1234.0000, "decimal_col2": 123456789123.000000, "decimal_col3": 12345678912345678912345678.000000000000, "decimal_col4": 123456789, "decimal_col5": 123456789123456789.000000000, "decimal_col6": 987654321}, + {"id": 4, "decimal_col1": null, "decimal_col2": null, "decimal_col3": 123.123456789876, "decimal_col4": 12, "decimal_col5": null, "decimal_col6": 123}, + {"id": 5, "decimal_col1": 1.1234, "decimal_col2": 12.123456, "decimal_col3": null, "decimal_col4": null, "decimal_col5": 1234.123456789, "decimal_col6": null} +] diff --git a/regression-test/data/load_p0/stream_load/test_decimal.orc b/regression-test/data/load_p0/stream_load/test_decimal.orc new file mode 100644 index 0000000000000000000000000000000000000000..eb7d1bc9ca7a543afc644914ea6f27beec8c9615 GIT binary patch literal 1262 zcmeYdau#G@;9?VE;b0A5&}LxZ66RuNU|$+ik+hNi~bH*7I57Y=IwY#^b*we$*He}H5fn_JbfXKV^7X_;v$f(JMlLKqkx zUwR5MMUj@aO+RH}V{hkVW9G#SZVXin43E1WgA^}x;1g5M z*RumDv=vCqO3O>sRN+lOBv}xtDGAXMT+D57_{@PLKId=TVQBHuWs+bF16c-kvZ5-V zn6ji^nzM4UqMtW+v2n0)va++Lp0jsxa56Ku0!IQ111AH+0Tw1ERtARuFJu`Qj;v>V zvi}kHfi;h~kE~~8Wnf_B088mHFl7IznzK9cwxN2c+gh zGy}tnt^4NP{b&B->--~suIxVe{>`C@e~R85mZzy!rmh9mRxIf11vIy>xaPGeiz-l8^v{ z5ZELs28Q3qeq>yHzV%JQBWY!jIgAXq7#K>=?B;4P;Awq$%WS1MOR3e4-79}EHBEm1 z-{V+vSo^2R{Y%zfvtlWEdz&jkIn3-~ig8}phK*g_x;NHNy(^l%!TO<$Vbsy*3IA*6 z8C;%RU3T8$g(Pq69ah`en)th-+_AO$cZlv6j2x+YZy^o?k>vTPJzyh_vd7 z&U$=ij(v*hSrpYZ8r{Jyo1H?doK{Z;6Bq;m9N-hrPV?!1qi_pSf$ z>v>i^mOa*Mie?nd&|wys#woFuu~SEBQbf$8J4tRf-NBP&EjvS59-RwT7Ff)}!aC=S zb^h#CsvS3E)JmKs*9WBKML+5ZHNE*^)`oNU{jN^E#<}A6Hfsn@xolF9Z5)BOkN=ytq8kZRj*qgpEGy4ZQO9B9zT?HNh literal 0 HcmV?d00001 diff --git a/regression-test/data/load_p0/stream_load/test_decimal.parquet b/regression-test/data/load_p0/stream_load/test_decimal.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a0163057c0ce7206ef649696c8ab2f4fbfc2c65a GIT binary patch literal 1402 zcmWG=3^EjD5H%4s(GlemWe{Z(<&cqJVP#-okYHhCU|G8(-H7xNw5+JP%3=E9&;kAvT zFk{3aHbVuGl*6=^te6B*0@aAkbyoK>HiBJO4Ko%L2-P5A5N2Rts1{|C?D62v$Sg~Z zFHX)#%}o^LV-RIhWs+dZOkoh^6Xg(PlHf^6P0q|s%!yCV&oNZx66FvT07(g;Ng1hf ziSmf5fTR@Aq>NR$L?uLhKvL)iny7+Orw~X=0!^2xDwn8`s2NBKU6&bDml&6%jAV@( zhm0tbq$wK%ha|||q6}hOVttHaeT-@xHh9v!q>LzoD1#^{xvNcMlo92F8iC?puwi*( zd1})bZO{@vvi=q(?D~zs`sXo<&0|z+VFHg>R zJ9ZPzz$SuR^ne{?9t$YBf?ZUN>>{ya9AaDyk}?tk