From 78b0fec33a6cafff1dcf7807a598fa39761c5a56 Mon Sep 17 00:00:00 2001 From: Tiewei Fang <43782773+BePPPower@users.noreply.github.com> Date: Wed, 13 Dec 2023 11:55:27 +0800 Subject: [PATCH] [Fix](Outfile) Support export nested complex type data to orc file format (#28182) --- .../serde/data_type_array_serde.cpp | 18 +- .../data_types/serde/data_type_map_serde.cpp | 22 +- .../serde/data_type_nullable_serde.cpp | 7 +- .../serde/data_type_struct_serde.cpp | 18 +- be/src/vec/runtime/vorc_transformer.cpp | 69 ++- be/src/vec/runtime/vorc_transformer.h | 5 + .../apache/doris/analysis/OutFileClause.java | 17 - .../csv/test_outfile_csv_one_nested_type.out | 225 +++++++++ .../orc/test_outfile_orc_one_nested_type.out | 225 +++++++++ .../test_outfile_parquet_one_nested_type.out | 225 +++++++++ .../test_outfile_csv_one_nested_type.groovy | 447 ++++++++++++++++++ .../test_outfile_orc_one_nested_type.groovy | 447 ++++++++++++++++++ ...est_outfile_parquet_one_nested_type.groovy | 447 ++++++++++++++++++ 13 files changed, 2109 insertions(+), 63 deletions(-) create mode 100644 regression-test/data/export_p0/outfile/csv/test_outfile_csv_one_nested_type.out create mode 100644 regression-test/data/export_p0/outfile/orc/test_outfile_orc_one_nested_type.out create mode 100644 regression-test/data/export_p0/outfile/parquet/test_outfile_parquet_one_nested_type.out create mode 100644 regression-test/suites/export_p0/outfile/csv/test_outfile_csv_one_nested_type.groovy create mode 100644 regression-test/suites/export_p0/outfile/orc/test_outfile_orc_one_nested_type.groovy create mode 100644 regression-test/suites/export_p0/outfile/parquet/test_outfile_parquet_one_nested_type.groovy diff --git a/be/src/vec/data_types/serde/data_type_array_serde.cpp b/be/src/vec/data_types/serde/data_type_array_serde.cpp index 7aa5ab78cc..c372c46c0f 100644 --- a/be/src/vec/data_types/serde/data_type_array_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_array_serde.cpp @@ -361,24 +361,18 @@ Status DataTypeArraySerDe::write_column_to_orc(const std::string& timezone, cons const NullMap* null_map, orc::ColumnVectorBatch* orc_col_batch, int start, int end, std::vector& buffer_list) const { - orc::ListVectorBatch* cur_batch = dynamic_cast(orc_col_batch); + auto* cur_batch = dynamic_cast(orc_col_batch); cur_batch->offsets[0] = 0; - const ColumnArray& array_col = assert_cast(column); + const auto& array_col = assert_cast(column); const IColumn& nested_column = array_col.get_data(); - auto& offsets = array_col.get_offsets(); - - cur_batch->elements->resize(nested_column.size()); + const auto& offsets = array_col.get_offsets(); for (size_t row_id = start; row_id < end; row_id++) { size_t offset = offsets[row_id - 1]; size_t next_offset = offsets[row_id]; - - if (cur_batch->notNull[row_id] == 1) { - static_cast(nested_serde->write_column_to_orc(timezone, nested_column, nullptr, - cur_batch->elements.get(), offset, - next_offset, buffer_list)); - } - + RETURN_IF_ERROR(nested_serde->write_column_to_orc(timezone, nested_column, nullptr, + cur_batch->elements.get(), offset, + next_offset, buffer_list)); cur_batch->offsets[row_id + 1] = next_offset; } cur_batch->elements->numElements = nested_column.size(); diff --git a/be/src/vec/data_types/serde/data_type_map_serde.cpp b/be/src/vec/data_types/serde/data_type_map_serde.cpp index 8d86e100ed..c682062471 100644 --- a/be/src/vec/data_types/serde/data_type_map_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_map_serde.cpp @@ -483,29 +483,23 @@ Status DataTypeMapSerDe::write_column_to_orc(const std::string& timezone, const const NullMap* null_map, orc::ColumnVectorBatch* orc_col_batch, int start, int end, std::vector& buffer_list) const { - orc::MapVectorBatch* cur_batch = dynamic_cast(orc_col_batch); + auto* cur_batch = dynamic_cast(orc_col_batch); cur_batch->offsets[0] = 0; - auto& map_column = assert_cast(column); + const auto& map_column = assert_cast(column); const ColumnArray::Offsets64& offsets = map_column.get_offsets(); const IColumn& nested_keys_column = map_column.get_keys(); const IColumn& nested_values_column = map_column.get_values(); - - cur_batch->keys->resize(nested_keys_column.size()); - cur_batch->elements->resize(nested_values_column.size()); - for (size_t row_id = start; row_id < end; row_id++) { size_t offset = offsets[row_id - 1]; size_t next_offset = offsets[row_id]; - if (cur_batch->notNull[row_id] == 1) { - static_cast(key_serde->write_column_to_orc(timezone, nested_keys_column, nullptr, - cur_batch->keys.get(), offset, - next_offset, buffer_list)); - static_cast(value_serde->write_column_to_orc(timezone, nested_values_column, - nullptr, cur_batch->elements.get(), - offset, next_offset, buffer_list)); - } + RETURN_IF_ERROR(key_serde->write_column_to_orc(timezone, nested_keys_column, nullptr, + cur_batch->keys.get(), offset, next_offset, + buffer_list)); + RETURN_IF_ERROR(value_serde->write_column_to_orc(timezone, nested_values_column, nullptr, + cur_batch->elements.get(), offset, + next_offset, buffer_list)); cur_batch->offsets[row_id + 1] = next_offset; } diff --git a/be/src/vec/data_types/serde/data_type_nullable_serde.cpp b/be/src/vec/data_types/serde/data_type_nullable_serde.cpp index 02886b8de4..6bdadfe23a 100644 --- a/be/src/vec/data_types/serde/data_type_nullable_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_nullable_serde.cpp @@ -320,7 +320,6 @@ Status DataTypeNullableSerDe::write_column_to_orc(const std::string& timezone, std::vector& buffer_list) const { const auto& column_nullable = assert_cast(column); orc_col_batch->hasNulls = true; - auto& null_map_tmp = column_nullable.get_null_map_data(); auto orc_null_map = revert_null_map(&null_map_tmp, start, end); // orc_col_batch->notNull.data() must add 'start' (+ start), @@ -329,9 +328,9 @@ Status DataTypeNullableSerDe::write_column_to_orc(const std::string& timezone, // because orc_null_map begins at start and only has (end - start) elements memcpy(orc_col_batch->notNull.data() + start, orc_null_map.data(), end - start); - static_cast(nested_serde->write_column_to_orc( - timezone, column_nullable.get_nested_column(), &column_nullable.get_null_map_data(), - orc_col_batch, start, end, buffer_list)); + RETURN_IF_ERROR(nested_serde->write_column_to_orc(timezone, column_nullable.get_nested_column(), + &column_nullable.get_null_map_data(), + orc_col_batch, start, end, buffer_list)); return Status::OK(); } diff --git a/be/src/vec/data_types/serde/data_type_struct_serde.cpp b/be/src/vec/data_types/serde/data_type_struct_serde.cpp index 6d7cdb81bd..9ddda64b46 100644 --- a/be/src/vec/data_types/serde/data_type_struct_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_struct_serde.cpp @@ -400,22 +400,12 @@ Status DataTypeStructSerDe::write_column_to_orc(const std::string& timezone, con int end, std::vector& buffer_list) const { orc::StructVectorBatch* cur_batch = dynamic_cast(orc_col_batch); - const ColumnStruct& struct_col = assert_cast(column); for (size_t row_id = start; row_id < end; row_id++) { - if (cur_batch->notNull[row_id] == 1) { - for (int i = 0; i < struct_col.tuple_size(); ++i) { - static_cast(elemSerDeSPtrs[i]->write_column_to_orc( - timezone, struct_col.get_column(i), nullptr, cur_batch->fields[i], row_id, - row_id + 1, buffer_list)); - } - } else { - // This else is necessary - // because we must set notNull when cur_batch->notNull[row_id] == 0 - for (int j = 0; j < struct_col.tuple_size(); ++j) { - cur_batch->fields[j]->hasNulls = true; - cur_batch->fields[j]->notNull[row_id] = 0; - } + for (int i = 0; i < struct_col.tuple_size(); ++i) { + RETURN_IF_ERROR(elemSerDeSPtrs[i]->write_column_to_orc( + timezone, struct_col.get_column(i), nullptr, cur_batch->fields[i], row_id, + row_id + 1, buffer_list)); } } diff --git a/be/src/vec/runtime/vorc_transformer.cpp b/be/src/vec/runtime/vorc_transformer.cpp index 0339a4467b..be52b672aa 100644 --- a/be/src/vec/runtime/vorc_transformer.cpp +++ b/be/src/vec/runtime/vorc_transformer.cpp @@ -24,6 +24,7 @@ #include #include +#include "common/status.h" #include "io/fs/file_writer.h" #include "orc/Int128.hh" #include "orc/MemoryPool.hh" @@ -48,6 +49,9 @@ #include "vec/common/string_ref.h" #include "vec/core/column_with_type_and_name.h" #include "vec/core/types.h" +#include "vec/data_types/data_type_array.h" +#include "vec/data_types/data_type_map.h" +#include "vec/data_types/data_type_struct.h" #include "vec/exprs/vexpr.h" #include "vec/exprs/vexpr_context.h" #include "vec/runtime/vdatetime_value.h" @@ -160,10 +164,12 @@ Status VOrcTransformer::write(const Block& block) { size_t sz = block.rows(); auto row_batch = _create_row_batch(sz); - orc::StructVectorBatch* root = dynamic_cast(row_batch.get()); + auto* root = dynamic_cast(row_batch.get()); try { for (size_t i = 0; i < block.columns(); i++) { - auto& raw_column = block.get_by_position(i).column; + const auto& col = block.get_by_position(i); + const auto& raw_column = col.column; + RETURN_IF_ERROR(_resize_row_batch(col.type, *raw_column, root->fields[i])); RETURN_IF_ERROR(_serdes[i]->write_column_to_orc( _state->timezone(), *raw_column, nullptr, root->fields[i], 0, sz, buffer_list)); } @@ -178,4 +184,63 @@ Status VOrcTransformer::write(const Block& block) { return Status::OK(); } +Status VOrcTransformer::_resize_row_batch(const DataTypePtr& type, const IColumn& column, + orc::ColumnVectorBatch* orc_col_batch) { + auto real_type = remove_nullable(type); + WhichDataType which(real_type); + + if (which.is_struct()) { + auto* struct_batch = dynamic_cast(orc_col_batch); + const auto& struct_col = + column.is_nullable() + ? assert_cast( + assert_cast(column).get_nested_column()) + : assert_cast(column); + int idx = 0; + for (auto* child : struct_batch->fields) { + const IColumn& child_column = struct_col.get_column(idx); + child->resize(child_column.size()); + auto child_type = assert_cast(real_type.get()) + ->get_element(idx); + ++idx; + RETURN_IF_ERROR(_resize_row_batch(child_type, child_column, child)); + } + } else if (which.is_map()) { + auto* map_batch = dynamic_cast(orc_col_batch); + const auto& map_column = + column.is_nullable() + ? assert_cast( + assert_cast(column).get_nested_column()) + : assert_cast(column); + + // key of map + const IColumn& nested_keys_column = map_column.get_keys(); + map_batch->keys->resize(nested_keys_column.size()); + auto key_type = + assert_cast(real_type.get())->get_key_type(); + RETURN_IF_ERROR(_resize_row_batch(key_type, nested_keys_column, map_batch->keys.get())); + + // value of map + const IColumn& nested_values_column = map_column.get_values(); + map_batch->elements->resize(nested_values_column.size()); + auto value_type = + assert_cast(real_type.get())->get_value_type(); + RETURN_IF_ERROR( + _resize_row_batch(value_type, nested_values_column, map_batch->elements.get())); + } else if (which.is_array()) { + auto* list_batch = dynamic_cast(orc_col_batch); + const auto& array_col = + column.is_nullable() + ? assert_cast( + assert_cast(column).get_nested_column()) + : assert_cast(column); + const IColumn& nested_column = array_col.get_data(); + list_batch->elements->resize(nested_column.size()); + auto child_type = + assert_cast(real_type.get())->get_nested_type(); + RETURN_IF_ERROR(_resize_row_batch(child_type, nested_column, list_batch->elements.get())); + } + return Status::OK(); +} + } // namespace doris::vectorized diff --git a/be/src/vec/runtime/vorc_transformer.h b/be/src/vec/runtime/vorc_transformer.h index bc2ff05c88..4b8ea178ca 100644 --- a/be/src/vec/runtime/vorc_transformer.h +++ b/be/src/vec/runtime/vorc_transformer.h @@ -90,6 +90,11 @@ public: private: std::unique_ptr _create_row_batch(size_t sz); + // The size of subtypes of a complex type may be different from + // the size of the complex type itself, + // so we need to resize the subtype of a complex type + Status _resize_row_batch(const DataTypePtr& type, const IColumn& column, + orc::ColumnVectorBatch* orc_col_batch); doris::io::FileWriter* _file_writer = nullptr; std::unique_ptr _output_stream; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java index 7d41b25e2d..236788ce32 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java @@ -325,13 +325,6 @@ public class OutFileClause { break; case STRUCT: { StructType structType = (StructType) dorisType; - ArrayList fields = structType.getFields(); - for (StructField field : fields) { - if (!(field.getType() instanceof ScalarType)) { - throw new AnalysisException("currently ORC outfile do not support field type: " - + field.getType().toSql() + " for STRUCT"); - } - } StringBuilder sb = new StringBuilder(); sb.append("struct<"); @@ -350,11 +343,6 @@ public class OutFileClause { } case MAP: { MapType mapType = (MapType) dorisType; - if ((!(mapType.getKeyType() instanceof ScalarType) - || !(mapType.getValueType() instanceof ScalarType))) { - throw new AnalysisException("currently ORC outfile do not support data type: MAP<" - + mapType.getKeyType().toSql() + "," + mapType.getValueType().toSql() + ">"); - } StringBuilder sb = new StringBuilder(); sb.append("map<") .append(dorisTypeToOrcTypeMap(mapType.getKeyType())) @@ -365,11 +353,6 @@ public class OutFileClause { break; } case ARRAY: { - Type itemType = ((ArrayType) dorisType).getItemType(); - if (!(itemType instanceof ScalarType)) { - throw new AnalysisException("currently ORC outfile do not support data type: ARRAY<" - + itemType.toSql() + ">"); - } StringBuilder sb = new StringBuilder(); ArrayType arrayType = (ArrayType) dorisType; sb.append("array<") diff --git a/regression-test/data/export_p0/outfile/csv/test_outfile_csv_one_nested_type.out b/regression-test/data/export_p0/outfile/csv/test_outfile_csv_one_nested_type.out new file mode 100644 index 0000000000..490ab79add --- /dev/null +++ b/regression-test/data/export_p0/outfile/csv/test_outfile_csv_one_nested_type.out @@ -0,0 +1,225 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_base1 -- +1 doris1 {"s_info": {"s_id": 1, "s_name": "sn1", "s_address": "sa1"}} +2 doris2 {"s_info": {"s_id": 2, "s_name": "sn2", "s_address": "sa2"}} +3 doris3 {"s_info": null} +4 doris4 \N +5 doris5 {"s_info": {"s_id": 5, "s_name": "sn5", "s_address": "sa5"}} +6 doris6 {"s_info": {"s_id": 6, "s_name": "sn6", "s_address": "sa6"}} +7 \N {"s_info": {"s_id": 7, "s_name": "sn7", "s_address": "sa7"}} +8 \N \N + +-- !select_load1 -- +1 doris1 {"s_info": {"s_id": 1, "s_name": "sn1", "s_address": "sa1"}} +2 doris2 {"s_info": {"s_id": 2, "s_name": "sn2", "s_address": "sa2"}} +3 doris3 {"s_info": null} +4 doris4 \N +5 doris5 {"s_info": {"s_id": 5, "s_name": "sn5", "s_address": "sa5"}} +6 doris6 {"s_info": {"s_id": 6, "s_name": "sn6", "s_address": "sa6"}} +7 \N {"s_info": {"s_id": 7, "s_name": "sn7", "s_address": "sa7"}} +8 \N \N + +-- !select_base2 -- +1 doris1 {"s_info": {"s_id": 1, "s_name": "sn1", "s_address": "sa1"}} +2 doris2 {"s_info": {"s_id": 2, "s_name": "sn2", "s_address": "sa2"}} +3 doris3 {"s_info": null} +5 doris5 {"s_info": {"s_id": 5, "s_name": "sn5", "s_address": "sa5"}} +6 doris6 {"s_info": {"s_id": 6, "s_name": "sn6", "s_address": "sa6"}} +7 \N {"s_info": {"s_id": 7, "s_name": "sn7", "s_address": "sa7"}} + +-- !select_load2 -- +1 doris1 {"s_info": {"s_id": 1, "s_name": "sn1", "s_address": "sa1"}} +2 doris2 {"s_info": {"s_id": 2, "s_name": "sn2", "s_address": "sa2"}} +3 doris3 {"s_info": null} +5 doris5 {"s_info": {"s_id": 5, "s_name": "sn5", "s_address": "sa5"}} +6 doris6 {"s_info": {"s_id": 6, "s_name": "sn6", "s_address": "sa6"}} +7 \N {"s_info": {"s_id": 7, "s_name": "sn7", "s_address": "sa7"}} + +-- !select_base3 -- +1 doris1 {"l_info": ["doris1", "nereids1", "doris-nereids-1"]} +3 doris3 {"l_info": null} +4 doris4 \N +5 doris5 {"l_info": ["doris2", null, "nereids2"]} +7 \N {"l_info": [null, "null", "doris3"]} +8 \N \N +9 \N {"l_info": ["sn7", "sa7", "sn8", "sa8", "sn9", "sa9", "sn10", "sa10"]} + +-- !select_load3 -- +1 doris1 {"l_info": ["doris1", "nereids1", "doris-nereids-1"]} +3 doris3 {"l_info": null} +4 doris4 \N +5 doris5 {"l_info": ["doris2", null, "nereids2"]} +7 \N {"l_info": [null, "null", "doris3"]} +8 \N \N +9 \N {"l_info": ["sn7", "sa7", "sn8", "sa8", "sn9", "sa9", "sn10", "sa10"]} + +-- !select_base4 -- +1 doris1 {"m_info": {"a":100, "b":111}} +2 doris2 {"m_info": {"a":200, "b":222}} +3 doris3 {"m_info": null} +4 doris4 \N +5 doris5 {"m_info": {"a":null, "b":333, "c":399, "d":399999999999999}} +6 doris6 {"m_info": {"null":100, "b":null}} +7 \N {"m_info": {"null":null, "null":null}} +8 \N \N + +-- !select_load4 -- +1 doris1 {"m_info": {"a":100, "b":111}} +2 doris2 {"m_info": {"a":200, "b":222}} +3 doris3 {"m_info": null} +4 doris4 \N +5 doris5 {"m_info": {"a":null, "b":333, "c":399, "d":399999999999999}} +6 doris6 {"m_info": {"null":100, "b":null}} +7 \N {"m_info": {"null":null, "null":null}} +8 \N \N + +-- !select_base5 -- +1 doris1 [{"i_info": 1, "s_info": "doris1"}, {"i_info": 2, "s_info": "nereids1"}, {"i_info": 3, "s_info": "doris-nereids-1"}] +2 doris2 [{"i_info": 4, "s_info": "doris-nereids-4"}] +3 doris3 [] +4 doris4 [null, null, {"i_info": 5, "s_info": "doris-nereids-5"}] +5 doris5 [{"i_info": 6, "s_info": "doris7"}, null, null] +6 doris6 [null, null, null] +7 \N \N +8 \N [{"i_info": 8, "s_info": "doris8"}] +9 \N [{"i_info": 9, "s_info": "doris9"}, {"i_info": 10, "s_info": "doris10"}] + +-- !select_load5 -- +1 doris1 [{"i_info": 1, "s_info": "doris1"}, {"i_info": 2, "s_info": "nereids1"}, {"i_info": 3, "s_info": "doris-nereids-1"}] +2 doris2 [{"i_info": 4, "s_info": "doris-nereids-4"}] +3 doris3 [] +4 doris4 [null, null, {"i_info": 5, "s_info": "doris-nereids-5"}] +5 doris5 [{"i_info": 6, "s_info": "doris7"}, null, null] +6 doris6 [null, null, null] +7 \N \N +8 \N [{"i_info": 8, "s_info": "doris8"}] +9 \N [{"i_info": 9, "s_info": "doris9"}, {"i_info": 10, "s_info": "doris10"}] + +-- !select_base6 -- +1 doris1 [{"i_info": 1, "s_info": "doris1"}, {"i_info": 2, "s_info": "nereids1"}, {"i_info": 3, "s_info": "doris-nereids-1"}] +2 doris2 [{"i_info": 4, "s_info": "doris-nereids-4"}] +3 doris3 [] +4 doris4 [null, null, {"i_info": 5, "s_info": "doris-nereids-5"}] +5 doris5 [{"i_info": 6, "s_info": "doris7"}, null, null] +6 doris6 [null, null, null] +8 \N [{"i_info": 8, "s_info": "doris8"}] +9 \N [{"i_info": 9, "s_info": "doris9"}, {"i_info": 10, "s_info": "doris10"}] + +-- !select_load6 -- +1 doris1 [{"i_info": 1, "s_info": "doris1"}, {"i_info": 2, "s_info": "nereids1"}, {"i_info": 3, "s_info": "doris-nereids-1"}] +2 doris2 [{"i_info": 4, "s_info": "doris-nereids-4"}] +3 doris3 [] +4 doris4 [null, null, {"i_info": 5, "s_info": "doris-nereids-5"}] +5 doris5 [{"i_info": 6, "s_info": "doris7"}, null, null] +6 doris6 [null, null, null] +8 \N [{"i_info": 8, "s_info": "doris8"}] +9 \N [{"i_info": 9, "s_info": "doris9"}, {"i_info": 10, "s_info": "doris10"}] + +-- !select_base7 -- +1 doris1 [["doris", "null"], ["basdakljdoaidsjowqd", "asdqqqqsdafdorisdoris"], ["a", "b", "c"]] +2 doris2 [["null"]] +3 doris3 [] +4 doris4 [["xx", "yy", "nereids"], ["doris", "null"]] +5 doris5 \N +6 doris6 [null, ["xx", null, "nereids"], null] +7 \N \N +8 \N [[], ["abc"], [], null] + +-- !select_load7 -- +1 doris1 [["doris", "null"], ["basdakljdoaidsjowqd", "asdqqqqsdafdorisdoris"], ["a", "b", "c"]] +2 doris2 [["null"]] +3 doris3 [] +4 doris4 [["xx", "yy", "nereids"], ["doris", "null"]] +5 doris5 \N +6 doris6 [null, ["xx", null, "nereids"], null] +7 \N \N +8 \N [[], ["abc"], [], null] + +-- !select_base8 -- +1 doris1 [{"a":100, "b":111}, {"a":200, "b":222}, {"a":null, "b":333, "c":399, "d":399999999999999}] +2 doris2 [{"doris":200, "nereids":222}] +3 doris3 [] +4 doris4 [null, null, {"a":100, "b":111}] +5 doris5 \N +6 doris6 [null, null, {"nereids":222}, null] +7 \N \N +8 \N [null, null] +9 \N [{"doris":200, "nereids":222}, null, null] + +-- !select_load8 -- +1 doris1 [{"a":100, "b":111}, {"a":200, "b":222}, {"a":null, "b":333, "c":399, "d":399999999999999}] +2 doris2 [{"doris":200, "nereids":222}] +3 doris3 [] +4 doris4 [null, null, {"a":100, "b":111}] +5 doris5 \N +6 doris6 [null, null, {"nereids":222}, null] +7 \N \N +8 \N [null, null] +9 \N [{"doris":200, "nereids":222}, null, null] + +-- !select_base9 -- +1 doris1 {"a":{"s_info": "doris", "l_info": 18}, "b":{"s_info": "nereids", "l_info": 20}, "c":{"s_info": "nereids", "l_info": 21}} +2 doris2 {"xx":null, "a":{"s_info": "doris", "l_info": 18}} +3 doris3 {"dd":{"s_info": null, "l_info": null}} +4 doris4 \N +5 doris5 {"doris-nereids":{"s_info": "nereids", "l_info": null}, "yyzz":{"s_info": null, "l_info": 999999}} +7 \N {"null":null, "null":null} +8 \N \N + +-- !select_load9 -- +1 doris1 {"a":{"s_info": "doris", "l_info": 18}, "b":{"s_info": "nereids", "l_info": 20}, "c":{"s_info": "nereids", "l_info": 21}} +2 doris2 {"xx":null, "a":{"s_info": "doris", "l_info": 18}} +3 doris3 {"dd":{"s_info": null, "l_info": null}} +4 doris4 \N +5 doris5 {"doris-nereids":{"s_info": "nereids", "l_info": null}, "yyzz":{"s_info": null, "l_info": 999999}} +7 \N {"null":null, "null":null} +8 \N \N + +-- !select_base10 -- +1 doris1 {"a":{"s_info": "doris", "l_info": 18}, "b":{"s_info": "nereids", "l_info": 20}, "c":{"s_info": "nereids", "l_info": 21}} +2 doris2 {"xx":null, "a":{"s_info": "doris", "l_info": 18}} +3 doris3 {"dd":{"s_info": null, "l_info": null}} +5 doris5 {"doris-nereids":{"s_info": "nereids", "l_info": null}, "yyzz":{"s_info": null, "l_info": 999999}} +7 \N {"null":null, "null":null} + +-- !select_load10 -- +1 doris1 {"a":{"s_info": "doris", "l_info": 18}, "b":{"s_info": "nereids", "l_info": 20}, "c":{"s_info": "nereids", "l_info": 21}} +2 doris2 {"xx":null, "a":{"s_info": "doris", "l_info": 18}} +3 doris3 {"dd":{"s_info": null, "l_info": null}} +5 doris5 {"doris-nereids":{"s_info": "nereids", "l_info": null}, "yyzz":{"s_info": null, "l_info": 999999}} +7 \N {"null":null, "null":null} + +-- !select_base11 -- +1 doris1 {"a":{"a":100, "b":111}, "b":{"a":200, "b":222, "c":333}, "c":{"nereids":99}} +2 doris2 {"xx":null, "yy":{"a":null, "b":333, "c":399, "d":399999999999999}} +3 doris3 {"dd":null, "qq":null, "ww":{"null":100, "b":null}} +4 doris4 \N +5 doris5 {"doris-nereids":{"nereids":null}, "yyzz":{"xx":null, "doris":999999}} +7 \N {"doris":{"null":null, "null":null}, "nereids":{"a":200, "b":222, "c":333}} +8 \N \N + +-- !select_load11 -- +1 doris1 {"a":{"a":100, "b":111}, "b":{"a":200, "b":222, "c":333}, "c":{"nereids":99}} +2 doris2 {"xx":null, "yy":{"a":null, "b":333, "c":399, "d":399999999999999}} +3 doris3 {"dd":null, "qq":null, "ww":{"null":100, "b":null}} +4 doris4 \N +5 doris5 {"doris-nereids":{"nereids":null}, "yyzz":{"xx":null, "doris":999999}} +7 \N {"doris":{"null":null, "null":null}, "nereids":{"a":200, "b":222, "c":333}} +8 \N \N + +-- !select_base12 -- +1 doris1 {"a":["doris", "nereids", "zzz"], "b":["a", "b", "c"], "c":["qwe", "doris-nereids", "pppppppp"]} +2 doris2 {"a":["doris", null, "zzz"], "b":["a", "b", null], "c":[null, "doris-nereids", "pppppppp"]} +3 doris3 {"a":[null, null, "zzz"], "b":[null, null], "c":[], "d":null, "e":[null]} +4 doris4 \N +5 doris5 {"a":null, "b":[], "null":[null, "doris-nereids", "pppppppp"]} +6 \N \N + +-- !select_load12 -- +1 doris1 {"a":["doris", "nereids", "zzz"], "b":["a", "b", "c"], "c":["qwe", "doris-nereids", "pppppppp"]} +2 doris2 {"a":["doris", null, "zzz"], "b":["a", "b", null], "c":[null, "doris-nereids", "pppppppp"]} +3 doris3 {"a":[null, null, "zzz"], "b":[null, null], "c":[], "d":null, "e":[null]} +4 doris4 \N +5 doris5 {"a":null, "b":[], "null":[null, "doris-nereids", "pppppppp"]} +6 \N \N + diff --git a/regression-test/data/export_p0/outfile/orc/test_outfile_orc_one_nested_type.out b/regression-test/data/export_p0/outfile/orc/test_outfile_orc_one_nested_type.out new file mode 100644 index 0000000000..e12a17bda8 --- /dev/null +++ b/regression-test/data/export_p0/outfile/orc/test_outfile_orc_one_nested_type.out @@ -0,0 +1,225 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_base1 -- +1 doris1 {"s_info": {"s_id": 1, "s_name": "sn1", "s_address": "sa1"}} +2 doris2 {"s_info": {"s_id": 2, "s_name": "sn2", "s_address": "sa2"}} +3 doris3 {"s_info": null} +4 doris4 \N +5 doris5 {"s_info": {"s_id": 5, "s_name": "sn5", "s_address": "sa5"}} +6 doris6 {"s_info": {"s_id": 6, "s_name": "sn6", "s_address": "sa6"}} +7 \N {"s_info": {"s_id": 7, "s_name": "sn7", "s_address": "sa7"}} +8 \N \N + +-- !select_load1 -- +1 doris1 {"s_info": {"s_id": 1, "s_name": "sn1", "s_address": "sa1"}} +2 doris2 {"s_info": {"s_id": 2, "s_name": "sn2", "s_address": "sa2"}} +3 doris3 {"s_info": null} +4 doris4 \N +5 doris5 {"s_info": {"s_id": 5, "s_name": "sn5", "s_address": "sa5"}} +6 doris6 {"s_info": {"s_id": 6, "s_name": "sn6", "s_address": "sa6"}} +7 \N {"s_info": {"s_id": 7, "s_name": "sn7", "s_address": "sa7"}} +8 \N \N + +-- !select_base2 -- +1 doris1 {"s_info": {"s_id": 1, "s_name": "sn1", "s_address": "sa1"}} +2 doris2 {"s_info": {"s_id": 2, "s_name": "sn2", "s_address": "sa2"}} +3 doris3 {"s_info": null} +5 doris5 {"s_info": {"s_id": 5, "s_name": "sn5", "s_address": "sa5"}} +6 doris6 {"s_info": {"s_id": 6, "s_name": "sn6", "s_address": "sa6"}} +7 \N {"s_info": {"s_id": 7, "s_name": "sn7", "s_address": "sa7"}} + +-- !select_load2 -- +1 doris1 {"s_info": {"s_id": 1, "s_name": "sn1", "s_address": "sa1"}} +2 doris2 {"s_info": {"s_id": 2, "s_name": "sn2", "s_address": "sa2"}} +3 doris3 {"s_info": null} +5 doris5 {"s_info": {"s_id": 5, "s_name": "sn5", "s_address": "sa5"}} +6 doris6 {"s_info": {"s_id": 6, "s_name": "sn6", "s_address": "sa6"}} +7 \N {"s_info": {"s_id": 7, "s_name": "sn7", "s_address": "sa7"}} + +-- !select_base3 -- +1 doris1 {"l_info": ["doris1", "nereids1", "doris-nereids-1"]} +3 doris3 {"l_info": null} +4 doris4 \N +5 doris5 {"l_info": ["doris2", null, "nereids2"]} +7 \N {"l_info": [null, "null", "doris3"]} +8 \N \N +9 \N {"l_info": ["sn7", "sa7", "sn8", "sa8", "sn9", "sa9", "sn10", "sa10"]} + +-- !select_load3 -- +1 doris1 {"l_info": ["doris1", "nereids1", "doris-nereids-1"]} +3 doris3 {"l_info": null} +4 doris4 \N +5 doris5 {"l_info": ["doris2", null, "nereids2"]} +7 \N {"l_info": [null, "null", "doris3"]} +8 \N \N +9 \N {"l_info": ["sn7", "sa7", "sn8", "sa8", "sn9", "sa9", "sn10", "sa10"]} + +-- !select_base4 -- +1 doris1 {"m_info": {"a":100, "b":111}} +2 doris2 {"m_info": {"a":200, "b":222}} +3 doris3 {"m_info": null} +4 doris4 \N +5 doris5 {"m_info": {"a":null, "b":333, "c":399, "d":399999999999999}} +6 doris6 {"m_info": {"null":100, "b":null}} +7 \N {"m_info": {"null":null, "null":null}} +8 \N \N + +-- !select_load4 -- +1 doris1 {"m_info": {"a":"100", "b":"111"}} +2 doris2 {"m_info": {"a":"200", "b":"222"}} +3 doris3 {"m_info": null} +4 doris4 \N +5 doris5 {"m_info": {"a":null, "b":"333", "c":"399", "d":"399999999999999"}} +6 doris6 {"m_info": {"null":"100", "b":null}} +7 \N {"m_info": {"null":null, "null":null}} +8 \N \N + +-- !select_base5 -- +1 doris1 [{"i_info": 1, "s_info": "doris1"}, {"i_info": 2, "s_info": "nereids1"}, {"i_info": 3, "s_info": "doris-nereids-1"}] +2 doris2 [{"i_info": 4, "s_info": "doris-nereids-4"}] +3 doris3 [] +4 doris4 [null, null, {"i_info": 5, "s_info": "doris-nereids-5"}] +5 doris5 [{"i_info": 6, "s_info": "doris7"}, null, null] +6 doris6 [null, null, null] +7 \N \N +8 \N [{"i_info": 8, "s_info": "doris8"}] +9 \N [{"i_info": 9, "s_info": "doris9"}, {"i_info": 10, "s_info": "doris10"}] + +-- !select_load5 -- +1 doris1 [{"i_info": 1, "s_info": "doris1"}, {"i_info": 2, "s_info": "nereids1"}, {"i_info": 3, "s_info": "doris-nereids-1"}] +2 doris2 [{"i_info": 4, "s_info": "doris-nereids-4"}] +3 doris3 [] +4 doris4 [null, null, {"i_info": 5, "s_info": "doris-nereids-5"}] +5 doris5 [{"i_info": 6, "s_info": "doris7"}, null, null] +6 doris6 [null, null, null] +7 \N \N +8 \N [{"i_info": 8, "s_info": "doris8"}] +9 \N [{"i_info": 9, "s_info": "doris9"}, {"i_info": 10, "s_info": "doris10"}] + +-- !select_base6 -- +1 doris1 [{"i_info": 1, "s_info": "doris1"}, {"i_info": 2, "s_info": "nereids1"}, {"i_info": 3, "s_info": "doris-nereids-1"}] +2 doris2 [{"i_info": 4, "s_info": "doris-nereids-4"}] +3 doris3 [] +4 doris4 [null, null, {"i_info": 5, "s_info": "doris-nereids-5"}] +5 doris5 [{"i_info": 6, "s_info": "doris7"}, null, null] +6 doris6 [null, null, null] +8 \N [{"i_info": 8, "s_info": "doris8"}] +9 \N [{"i_info": 9, "s_info": "doris9"}, {"i_info": 10, "s_info": "doris10"}] + +-- !select_load6 -- +1 doris1 [{"i_info": 1, "s_info": "doris1"}, {"i_info": 2, "s_info": "nereids1"}, {"i_info": 3, "s_info": "doris-nereids-1"}] +2 doris2 [{"i_info": 4, "s_info": "doris-nereids-4"}] +3 doris3 [] +4 doris4 [null, null, {"i_info": 5, "s_info": "doris-nereids-5"}] +5 doris5 [{"i_info": 6, "s_info": "doris7"}, null, null] +6 doris6 [null, null, null] +8 \N [{"i_info": 8, "s_info": "doris8"}] +9 \N [{"i_info": 9, "s_info": "doris9"}, {"i_info": 10, "s_info": "doris10"}] + +-- !select_base7 -- +1 doris1 [["doris", "null"], ["basdakljdoaidsjowqd", "asdqqqqsdafdorisdoris"], ["a", "b", "c"]] +2 doris2 [["null"]] +3 doris3 [] +4 doris4 [["xx", "yy", "nereids"], ["doris", "null"]] +5 doris5 \N +6 doris6 [null, ["xx", null, "nereids"], null] +7 \N \N +8 \N [[], ["abc"], [], null] + +-- !select_load7 -- +1 doris1 [["doris", "null"], ["basdakljdoaidsjowqd", "asdqqqqsdafdorisdoris"], ["a", "b", "c"]] +2 doris2 [["null"]] +3 doris3 [] +4 doris4 [["xx", "yy", "nereids"], ["doris", "null"]] +5 doris5 \N +6 doris6 [null, ["xx", null, "nereids"], null] +7 \N \N +8 \N [[], ["abc"], [], null] + +-- !select_base8 -- +1 doris1 [{"a":100, "b":111}, {"a":200, "b":222}, {"a":null, "b":333, "c":399, "d":399999999999999}] +2 doris2 [{"doris":200, "nereids":222}] +3 doris3 [] +4 doris4 [null, null, {"a":100, "b":111}] +5 doris5 \N +6 doris6 [null, null, {"nereids":222}, null] +7 \N \N +8 \N [null, null] +9 \N [{"doris":200, "nereids":222}, null, null] + +-- !select_load8 -- +1 doris1 [{"a":"100", "b":"111"}, {"a":"200", "b":"222"}, {"a":null, "b":"333", "c":"399", "d":"399999999999999"}] +2 doris2 [{"doris":"200", "nereids":"222"}] +3 doris3 [] +4 doris4 [null, null, {"a":"100", "b":"111"}] +5 doris5 \N +6 doris6 [null, null, {"nereids":"222"}, null] +7 \N \N +8 \N [null, null] +9 \N [{"doris":"200", "nereids":"222"}, null, null] + +-- !select_base9 -- +1 doris1 {"a":{"s_info": "doris", "l_info": 18}, "b":{"s_info": "nereids", "l_info": 20}, "c":{"s_info": "nereids", "l_info": 21}} +2 doris2 {"xx":null, "a":{"s_info": "doris", "l_info": 18}} +3 doris3 {"dd":{"s_info": null, "l_info": null}} +4 doris4 \N +5 doris5 {"doris-nereids":{"s_info": "nereids", "l_info": null}, "yyzz":{"s_info": null, "l_info": 999999}} +7 \N {"null":null, "null":null} +8 \N \N + +-- !select_load9 -- +1 doris1 {"a":{"s_info": "doris", "l_info": "18"}, "b":{"s_info": "nereids", "l_info": "20"}, "c":{"s_info": "nereids", "l_info": "21"}} +2 doris2 {"xx":null, "a":{"s_info": "doris", "l_info": "18"}} +3 doris3 {"dd":{"s_info": null, "l_info": null}} +4 doris4 \N +5 doris5 {"doris-nereids":{"s_info": "nereids", "l_info": null}, "yyzz":{"s_info": null, "l_info": "999999"}} +7 \N {"null":null, "null":null} +8 \N \N + +-- !select_base10 -- +1 doris1 {"a":{"s_info": "doris", "l_info": 18}, "b":{"s_info": "nereids", "l_info": 20}, "c":{"s_info": "nereids", "l_info": 21}} +2 doris2 {"xx":null, "a":{"s_info": "doris", "l_info": 18}} +3 doris3 {"dd":{"s_info": null, "l_info": null}} +5 doris5 {"doris-nereids":{"s_info": "nereids", "l_info": null}, "yyzz":{"s_info": null, "l_info": 999999}} +7 \N {"null":null, "null":null} + +-- !select_load10 -- +1 doris1 {"a":{"s_info": "doris", "l_info": "18"}, "b":{"s_info": "nereids", "l_info": "20"}, "c":{"s_info": "nereids", "l_info": "21"}} +2 doris2 {"xx":null, "a":{"s_info": "doris", "l_info": "18"}} +3 doris3 {"dd":{"s_info": null, "l_info": null}} +5 doris5 {"doris-nereids":{"s_info": "nereids", "l_info": null}, "yyzz":{"s_info": null, "l_info": "999999"}} +7 \N {"null":null, "null":null} + +-- !select_base11 -- +1 doris1 {"a":{"a":100, "b":111}, "b":{"a":200, "b":222, "c":333}, "c":{"nereids":99}} +2 doris2 {"xx":null, "yy":{"a":null, "b":333, "c":399, "d":399999999999999}} +3 doris3 {"dd":null, "qq":null, "ww":{"null":100, "b":null}} +4 doris4 \N +5 doris5 {"doris-nereids":{"nereids":null}, "yyzz":{"xx":null, "doris":999999}} +7 \N {"doris":{"null":null, "null":null}, "nereids":{"a":200, "b":222, "c":333}} +8 \N \N + +-- !select_load11 -- +1 doris1 {"a":{"a":"100", "b":"111"}, "b":{"a":"200", "b":"222", "c":"333"}, "c":{"nereids":"99"}} +2 doris2 {"xx":null, "yy":{"a":null, "b":"333", "c":"399", "d":"399999999999999"}} +3 doris3 {"dd":null, "qq":null, "ww":{"null":"100", "b":null}} +4 doris4 \N +5 doris5 {"doris-nereids":{"nereids":null}, "yyzz":{"xx":null, "doris":"999999"}} +7 \N {"doris":{"null":null, "null":null}, "nereids":{"a":"200", "b":"222", "c":"333"}} +8 \N \N + +-- !select_base12 -- +1 doris1 {"a":["doris", "nereids", "zzz"], "b":["a", "b", "c"], "c":["qwe", "doris-nereids", "pppppppp"]} +2 doris2 {"a":["doris", null, "zzz"], "b":["a", "b", null], "c":[null, "doris-nereids", "pppppppp"]} +3 doris3 {"a":[null, null, "zzz"], "b":[null, null], "c":[], "d":null, "e":[null]} +4 doris4 \N +5 doris5 {"a":null, "b":[], "null":[null, "doris-nereids", "pppppppp"]} +6 \N \N + +-- !select_load12 -- +1 doris1 {"a":["doris", "nereids", "zzz"], "b":["a", "b", "c"], "c":["qwe", "doris-nereids", "pppppppp"]} +2 doris2 {"a":["doris", null, "zzz"], "b":["a", "b", null], "c":[null, "doris-nereids", "pppppppp"]} +3 doris3 {"a":[null, null, "zzz"], "b":[null, null], "c":[], "d":null, "e":[null]} +4 doris4 \N +5 doris5 {"a":null, "b":[], "null":[null, "doris-nereids", "pppppppp"]} +6 \N \N + diff --git a/regression-test/data/export_p0/outfile/parquet/test_outfile_parquet_one_nested_type.out b/regression-test/data/export_p0/outfile/parquet/test_outfile_parquet_one_nested_type.out new file mode 100644 index 0000000000..e12a17bda8 --- /dev/null +++ b/regression-test/data/export_p0/outfile/parquet/test_outfile_parquet_one_nested_type.out @@ -0,0 +1,225 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_base1 -- +1 doris1 {"s_info": {"s_id": 1, "s_name": "sn1", "s_address": "sa1"}} +2 doris2 {"s_info": {"s_id": 2, "s_name": "sn2", "s_address": "sa2"}} +3 doris3 {"s_info": null} +4 doris4 \N +5 doris5 {"s_info": {"s_id": 5, "s_name": "sn5", "s_address": "sa5"}} +6 doris6 {"s_info": {"s_id": 6, "s_name": "sn6", "s_address": "sa6"}} +7 \N {"s_info": {"s_id": 7, "s_name": "sn7", "s_address": "sa7"}} +8 \N \N + +-- !select_load1 -- +1 doris1 {"s_info": {"s_id": 1, "s_name": "sn1", "s_address": "sa1"}} +2 doris2 {"s_info": {"s_id": 2, "s_name": "sn2", "s_address": "sa2"}} +3 doris3 {"s_info": null} +4 doris4 \N +5 doris5 {"s_info": {"s_id": 5, "s_name": "sn5", "s_address": "sa5"}} +6 doris6 {"s_info": {"s_id": 6, "s_name": "sn6", "s_address": "sa6"}} +7 \N {"s_info": {"s_id": 7, "s_name": "sn7", "s_address": "sa7"}} +8 \N \N + +-- !select_base2 -- +1 doris1 {"s_info": {"s_id": 1, "s_name": "sn1", "s_address": "sa1"}} +2 doris2 {"s_info": {"s_id": 2, "s_name": "sn2", "s_address": "sa2"}} +3 doris3 {"s_info": null} +5 doris5 {"s_info": {"s_id": 5, "s_name": "sn5", "s_address": "sa5"}} +6 doris6 {"s_info": {"s_id": 6, "s_name": "sn6", "s_address": "sa6"}} +7 \N {"s_info": {"s_id": 7, "s_name": "sn7", "s_address": "sa7"}} + +-- !select_load2 -- +1 doris1 {"s_info": {"s_id": 1, "s_name": "sn1", "s_address": "sa1"}} +2 doris2 {"s_info": {"s_id": 2, "s_name": "sn2", "s_address": "sa2"}} +3 doris3 {"s_info": null} +5 doris5 {"s_info": {"s_id": 5, "s_name": "sn5", "s_address": "sa5"}} +6 doris6 {"s_info": {"s_id": 6, "s_name": "sn6", "s_address": "sa6"}} +7 \N {"s_info": {"s_id": 7, "s_name": "sn7", "s_address": "sa7"}} + +-- !select_base3 -- +1 doris1 {"l_info": ["doris1", "nereids1", "doris-nereids-1"]} +3 doris3 {"l_info": null} +4 doris4 \N +5 doris5 {"l_info": ["doris2", null, "nereids2"]} +7 \N {"l_info": [null, "null", "doris3"]} +8 \N \N +9 \N {"l_info": ["sn7", "sa7", "sn8", "sa8", "sn9", "sa9", "sn10", "sa10"]} + +-- !select_load3 -- +1 doris1 {"l_info": ["doris1", "nereids1", "doris-nereids-1"]} +3 doris3 {"l_info": null} +4 doris4 \N +5 doris5 {"l_info": ["doris2", null, "nereids2"]} +7 \N {"l_info": [null, "null", "doris3"]} +8 \N \N +9 \N {"l_info": ["sn7", "sa7", "sn8", "sa8", "sn9", "sa9", "sn10", "sa10"]} + +-- !select_base4 -- +1 doris1 {"m_info": {"a":100, "b":111}} +2 doris2 {"m_info": {"a":200, "b":222}} +3 doris3 {"m_info": null} +4 doris4 \N +5 doris5 {"m_info": {"a":null, "b":333, "c":399, "d":399999999999999}} +6 doris6 {"m_info": {"null":100, "b":null}} +7 \N {"m_info": {"null":null, "null":null}} +8 \N \N + +-- !select_load4 -- +1 doris1 {"m_info": {"a":"100", "b":"111"}} +2 doris2 {"m_info": {"a":"200", "b":"222"}} +3 doris3 {"m_info": null} +4 doris4 \N +5 doris5 {"m_info": {"a":null, "b":"333", "c":"399", "d":"399999999999999"}} +6 doris6 {"m_info": {"null":"100", "b":null}} +7 \N {"m_info": {"null":null, "null":null}} +8 \N \N + +-- !select_base5 -- +1 doris1 [{"i_info": 1, "s_info": "doris1"}, {"i_info": 2, "s_info": "nereids1"}, {"i_info": 3, "s_info": "doris-nereids-1"}] +2 doris2 [{"i_info": 4, "s_info": "doris-nereids-4"}] +3 doris3 [] +4 doris4 [null, null, {"i_info": 5, "s_info": "doris-nereids-5"}] +5 doris5 [{"i_info": 6, "s_info": "doris7"}, null, null] +6 doris6 [null, null, null] +7 \N \N +8 \N [{"i_info": 8, "s_info": "doris8"}] +9 \N [{"i_info": 9, "s_info": "doris9"}, {"i_info": 10, "s_info": "doris10"}] + +-- !select_load5 -- +1 doris1 [{"i_info": 1, "s_info": "doris1"}, {"i_info": 2, "s_info": "nereids1"}, {"i_info": 3, "s_info": "doris-nereids-1"}] +2 doris2 [{"i_info": 4, "s_info": "doris-nereids-4"}] +3 doris3 [] +4 doris4 [null, null, {"i_info": 5, "s_info": "doris-nereids-5"}] +5 doris5 [{"i_info": 6, "s_info": "doris7"}, null, null] +6 doris6 [null, null, null] +7 \N \N +8 \N [{"i_info": 8, "s_info": "doris8"}] +9 \N [{"i_info": 9, "s_info": "doris9"}, {"i_info": 10, "s_info": "doris10"}] + +-- !select_base6 -- +1 doris1 [{"i_info": 1, "s_info": "doris1"}, {"i_info": 2, "s_info": "nereids1"}, {"i_info": 3, "s_info": "doris-nereids-1"}] +2 doris2 [{"i_info": 4, "s_info": "doris-nereids-4"}] +3 doris3 [] +4 doris4 [null, null, {"i_info": 5, "s_info": "doris-nereids-5"}] +5 doris5 [{"i_info": 6, "s_info": "doris7"}, null, null] +6 doris6 [null, null, null] +8 \N [{"i_info": 8, "s_info": "doris8"}] +9 \N [{"i_info": 9, "s_info": "doris9"}, {"i_info": 10, "s_info": "doris10"}] + +-- !select_load6 -- +1 doris1 [{"i_info": 1, "s_info": "doris1"}, {"i_info": 2, "s_info": "nereids1"}, {"i_info": 3, "s_info": "doris-nereids-1"}] +2 doris2 [{"i_info": 4, "s_info": "doris-nereids-4"}] +3 doris3 [] +4 doris4 [null, null, {"i_info": 5, "s_info": "doris-nereids-5"}] +5 doris5 [{"i_info": 6, "s_info": "doris7"}, null, null] +6 doris6 [null, null, null] +8 \N [{"i_info": 8, "s_info": "doris8"}] +9 \N [{"i_info": 9, "s_info": "doris9"}, {"i_info": 10, "s_info": "doris10"}] + +-- !select_base7 -- +1 doris1 [["doris", "null"], ["basdakljdoaidsjowqd", "asdqqqqsdafdorisdoris"], ["a", "b", "c"]] +2 doris2 [["null"]] +3 doris3 [] +4 doris4 [["xx", "yy", "nereids"], ["doris", "null"]] +5 doris5 \N +6 doris6 [null, ["xx", null, "nereids"], null] +7 \N \N +8 \N [[], ["abc"], [], null] + +-- !select_load7 -- +1 doris1 [["doris", "null"], ["basdakljdoaidsjowqd", "asdqqqqsdafdorisdoris"], ["a", "b", "c"]] +2 doris2 [["null"]] +3 doris3 [] +4 doris4 [["xx", "yy", "nereids"], ["doris", "null"]] +5 doris5 \N +6 doris6 [null, ["xx", null, "nereids"], null] +7 \N \N +8 \N [[], ["abc"], [], null] + +-- !select_base8 -- +1 doris1 [{"a":100, "b":111}, {"a":200, "b":222}, {"a":null, "b":333, "c":399, "d":399999999999999}] +2 doris2 [{"doris":200, "nereids":222}] +3 doris3 [] +4 doris4 [null, null, {"a":100, "b":111}] +5 doris5 \N +6 doris6 [null, null, {"nereids":222}, null] +7 \N \N +8 \N [null, null] +9 \N [{"doris":200, "nereids":222}, null, null] + +-- !select_load8 -- +1 doris1 [{"a":"100", "b":"111"}, {"a":"200", "b":"222"}, {"a":null, "b":"333", "c":"399", "d":"399999999999999"}] +2 doris2 [{"doris":"200", "nereids":"222"}] +3 doris3 [] +4 doris4 [null, null, {"a":"100", "b":"111"}] +5 doris5 \N +6 doris6 [null, null, {"nereids":"222"}, null] +7 \N \N +8 \N [null, null] +9 \N [{"doris":"200", "nereids":"222"}, null, null] + +-- !select_base9 -- +1 doris1 {"a":{"s_info": "doris", "l_info": 18}, "b":{"s_info": "nereids", "l_info": 20}, "c":{"s_info": "nereids", "l_info": 21}} +2 doris2 {"xx":null, "a":{"s_info": "doris", "l_info": 18}} +3 doris3 {"dd":{"s_info": null, "l_info": null}} +4 doris4 \N +5 doris5 {"doris-nereids":{"s_info": "nereids", "l_info": null}, "yyzz":{"s_info": null, "l_info": 999999}} +7 \N {"null":null, "null":null} +8 \N \N + +-- !select_load9 -- +1 doris1 {"a":{"s_info": "doris", "l_info": "18"}, "b":{"s_info": "nereids", "l_info": "20"}, "c":{"s_info": "nereids", "l_info": "21"}} +2 doris2 {"xx":null, "a":{"s_info": "doris", "l_info": "18"}} +3 doris3 {"dd":{"s_info": null, "l_info": null}} +4 doris4 \N +5 doris5 {"doris-nereids":{"s_info": "nereids", "l_info": null}, "yyzz":{"s_info": null, "l_info": "999999"}} +7 \N {"null":null, "null":null} +8 \N \N + +-- !select_base10 -- +1 doris1 {"a":{"s_info": "doris", "l_info": 18}, "b":{"s_info": "nereids", "l_info": 20}, "c":{"s_info": "nereids", "l_info": 21}} +2 doris2 {"xx":null, "a":{"s_info": "doris", "l_info": 18}} +3 doris3 {"dd":{"s_info": null, "l_info": null}} +5 doris5 {"doris-nereids":{"s_info": "nereids", "l_info": null}, "yyzz":{"s_info": null, "l_info": 999999}} +7 \N {"null":null, "null":null} + +-- !select_load10 -- +1 doris1 {"a":{"s_info": "doris", "l_info": "18"}, "b":{"s_info": "nereids", "l_info": "20"}, "c":{"s_info": "nereids", "l_info": "21"}} +2 doris2 {"xx":null, "a":{"s_info": "doris", "l_info": "18"}} +3 doris3 {"dd":{"s_info": null, "l_info": null}} +5 doris5 {"doris-nereids":{"s_info": "nereids", "l_info": null}, "yyzz":{"s_info": null, "l_info": "999999"}} +7 \N {"null":null, "null":null} + +-- !select_base11 -- +1 doris1 {"a":{"a":100, "b":111}, "b":{"a":200, "b":222, "c":333}, "c":{"nereids":99}} +2 doris2 {"xx":null, "yy":{"a":null, "b":333, "c":399, "d":399999999999999}} +3 doris3 {"dd":null, "qq":null, "ww":{"null":100, "b":null}} +4 doris4 \N +5 doris5 {"doris-nereids":{"nereids":null}, "yyzz":{"xx":null, "doris":999999}} +7 \N {"doris":{"null":null, "null":null}, "nereids":{"a":200, "b":222, "c":333}} +8 \N \N + +-- !select_load11 -- +1 doris1 {"a":{"a":"100", "b":"111"}, "b":{"a":"200", "b":"222", "c":"333"}, "c":{"nereids":"99"}} +2 doris2 {"xx":null, "yy":{"a":null, "b":"333", "c":"399", "d":"399999999999999"}} +3 doris3 {"dd":null, "qq":null, "ww":{"null":"100", "b":null}} +4 doris4 \N +5 doris5 {"doris-nereids":{"nereids":null}, "yyzz":{"xx":null, "doris":"999999"}} +7 \N {"doris":{"null":null, "null":null}, "nereids":{"a":"200", "b":"222", "c":"333"}} +8 \N \N + +-- !select_base12 -- +1 doris1 {"a":["doris", "nereids", "zzz"], "b":["a", "b", "c"], "c":["qwe", "doris-nereids", "pppppppp"]} +2 doris2 {"a":["doris", null, "zzz"], "b":["a", "b", null], "c":[null, "doris-nereids", "pppppppp"]} +3 doris3 {"a":[null, null, "zzz"], "b":[null, null], "c":[], "d":null, "e":[null]} +4 doris4 \N +5 doris5 {"a":null, "b":[], "null":[null, "doris-nereids", "pppppppp"]} +6 \N \N + +-- !select_load12 -- +1 doris1 {"a":["doris", "nereids", "zzz"], "b":["a", "b", "c"], "c":["qwe", "doris-nereids", "pppppppp"]} +2 doris2 {"a":["doris", null, "zzz"], "b":["a", "b", null], "c":[null, "doris-nereids", "pppppppp"]} +3 doris3 {"a":[null, null, "zzz"], "b":[null, null], "c":[], "d":null, "e":[null]} +4 doris4 \N +5 doris5 {"a":null, "b":[], "null":[null, "doris-nereids", "pppppppp"]} +6 \N \N + diff --git a/regression-test/suites/export_p0/outfile/csv/test_outfile_csv_one_nested_type.groovy b/regression-test/suites/export_p0/outfile/csv/test_outfile_csv_one_nested_type.groovy new file mode 100644 index 0000000000..783f6c126d --- /dev/null +++ b/regression-test/suites/export_p0/outfile/csv/test_outfile_csv_one_nested_type.groovy @@ -0,0 +1,447 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_outfile_csv_one_nested_type", "p0") { + // open nereids + sql """ set enable_nereids_planner=true """ + sql """ set enable_fallback_to_original_planner=false """ + + String ak = getS3AK() + String sk = getS3SK() + String s3_endpoint = getS3Endpoint() + String region = getS3Region() + String bucket = context.config.otherConfigs.get("s3BucketName"); + + def export_table_name = "outfile_csv_one_nested_type_export_test" + def outFilePath = "${bucket}/outfile/csv/nested_complex_type/exp_" + def outfile_format = "csv" + + def create_table = {table_name, struct_field -> + sql """ DROP TABLE IF EXISTS ${table_name} """ + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `name` STRING COMMENT "用户年龄", + ${struct_field} + ) + DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); + """ + } + + def outfile_to_S3 = { + // select ... into outfile ... + def res = sql """ + SELECT * FROM ${export_table_name} t ORDER BY user_id + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS ${outfile_format} + PROPERTIES ( + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${region}", + "s3.secret_key"="${sk}", + "s3.access_key" = "${ak}" + ); + """ + + return res[0][3] + } + + // 1. test NULL STRUCT_STRUCT + try { + def struct_field_define = "`ss_info` STRUCT> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {{1, 'sn1', 'sa1'}}); """ + sql """ insert into ${export_table_name} values (2, 'doris2', {{2, 'sn2', 'sa2'}}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {null}); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {{5, 'sn5', 'sa5'}}); """ + sql """ insert into ${export_table_name} values (6, 'doris6', {{6, 'sn6', 'sa6'}}); """ + sql """ insert into ${export_table_name} values (7, null, {{7, 'sn7', 'sa7'}}); """ + sql """ insert into ${export_table_name} values (8, null, null); """ + + + // test base data + qt_select_base1 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load1 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 2. test NOT NULL STRUCT_STRUCT + try { + def struct_field_define = "`ss_info` STRUCT> NOT NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {{1, 'sn1', 'sa1'}}); """ + sql """ insert into ${export_table_name} values (2, 'doris2', {{2, 'sn2', 'sa2'}}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {null}); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {{5, 'sn5', 'sa5'}}); """ + sql """ insert into ${export_table_name} values (6, 'doris6', {{6, 'sn6', 'sa6'}}); """ + sql """ insert into ${export_table_name} values (7, null, {{7, 'sn7', 'sa7'}}); """ + + // test base data + qt_select_base2 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load2 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 3. test NULL STRUCT_ARRAY + try { + def struct_field_define = "`ss_info` STRUCT> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {['doris1', 'nereids1', 'doris-nereids-1']}); """ + // sql """ insert into ${export_table_name} values (2, 'doris2', {[]}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {null}); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {['doris2', null, 'nereids2']}); """ + // sql """ insert into ${export_table_name} values (6, 'doris6', {[null, null, null]}); """ + sql """ insert into ${export_table_name} values (7, null, {[null, 'null', 'doris3']}); """ + sql """ insert into ${export_table_name} values (8, null, null); """ + sql """ insert into ${export_table_name} values (9, null, {['sn7', 'sa7', 'sn8', 'sa8', 'sn9', 'sa9', 'sn10', 'sa10']}); """ + + + // test base data + qt_select_base3 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load3 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 4. test NULL STRUCT_MAP + try { + def struct_field_define = "`ss_info` STRUCT> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {{'a': 100, 'b': 111}}); """ + sql """ insert into ${export_table_name} values (2, 'doris2', {{'a': 200, 'b': 222}}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {null}); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {{'a': null, 'b': 333, 'c':399, 'd':399999999999999}}); """ + sql """ insert into ${export_table_name} values (6, 'doris6', {{'null': 100, 'b': null}}); """ + sql """ insert into ${export_table_name} values (7, null, {{'null': null, 'null':null}}); """ + sql """ insert into ${export_table_name} values (8, null, null); """ + + + // test base data + qt_select_base4 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load4 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 5. test NULL ARRAY_STRUCT + try { + def struct_field_define = "`ss_info` ARRAY> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', [{1, 'doris1'}, {2, 'nereids1'}, {3, 'doris-nereids-1'}]); """ + sql """ insert into ${export_table_name} values (2, 'doris2', [{4, 'doris-nereids-4'}]); """ + sql """ insert into ${export_table_name} values (3, 'doris3', []); """ + sql """ insert into ${export_table_name} values (4, 'doris4', [null, null , {5, 'doris-nereids-5'}]); """ + sql """ insert into ${export_table_name} values (5, 'doris5', [{6, 'doris7'}, null, null]); """ + sql """ insert into ${export_table_name} values (6, 'doris6', [null, null, null]); """ + sql """ insert into ${export_table_name} values (7, null, null); """ + sql """ insert into ${export_table_name} values (8, null, [{8, 'doris8'}]); """ + sql """ insert into ${export_table_name} values (9, null, [{9, 'doris9'}, {10, 'doris10'}]); """ + + + // test base data + qt_select_base5 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load5 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + // 6. test NOT NULL ARRAY_STRUCT + try { + def struct_field_define = "`ss_info` ARRAY> NOT NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', [{1, 'doris1'}, {2, 'nereids1'}, {3, 'doris-nereids-1'}]); """ + sql """ insert into ${export_table_name} values (2, 'doris2', [{4, 'doris-nereids-4'}]); """ + sql """ insert into ${export_table_name} values (3, 'doris3', []); """ + sql """ insert into ${export_table_name} values (4, 'doris4', [null, null , {5, 'doris-nereids-5'}]); """ + sql """ insert into ${export_table_name} values (5, 'doris5', [{6, 'doris7'}, null, null]); """ + sql """ insert into ${export_table_name} values (6, 'doris6', [null, null, null]); """ + sql """ insert into ${export_table_name} values (8, null, [{8, 'doris8'}]); """ + sql """ insert into ${export_table_name} values (9, null, [{9, 'doris9'}, {10, 'doris10'}]); """ + + + // test base data + qt_select_base6 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load6 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + // 7. test NULL ARRAY_ARRAY + try { + def struct_field_define = "`ss_info` ARRAY> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', [["doris", "null"], ["basdakljdoaidsjowqd", "asdqqqqsdafdorisdoris"], ["a", "b", "c"]]); """ + sql """ insert into ${export_table_name} values (2, 'doris2', [["null"]]); """ + sql """ insert into ${export_table_name} values (3, 'doris3', []); """ + sql """ insert into ${export_table_name} values (4, 'doris4', [["xx", "yy", "nereids"], ["doris", "null"]]); """ + sql """ insert into ${export_table_name} values (5, 'doris5', null); """ + sql """ insert into ${export_table_name} values (6, 'doris6', [null, ["xx", null, "nereids"], null]); """ + sql """ insert into ${export_table_name} values (7, null, null); """ + sql """ insert into ${export_table_name} values (8, null, [[], ["abc"], [], null]); """ + + + // test base data + qt_select_base7 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load7 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + // 8. test NULL ARRAY_MAP + try { + def struct_field_define = "`ss_info` ARRAY> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', [{'a': 100, 'b': 111}, {'a': 200, 'b': 222}, {'a': null, 'b': 333, 'c':399, 'd':399999999999999}]); """ + sql """ insert into ${export_table_name} values (2, 'doris2', [{'doris': 200, 'nereids': 222}]); """ + sql """ insert into ${export_table_name} values (3, 'doris3', []); """ + sql """ insert into ${export_table_name} values (4, 'doris4', [null, null, {'a': 100, 'b': 111}]); """ + sql """ insert into ${export_table_name} values (5, 'doris5', null); """ + sql """ insert into ${export_table_name} values (6, 'doris6', [null, null, {'nereids': 222}, null]); """ + sql """ insert into ${export_table_name} values (7, null, null); """ + sql """ insert into ${export_table_name} values (8, null, [null, null]); """ + sql """ insert into ${export_table_name} values (9, null, [{'doris': 200, 'nereids': 222}, null, null]); """ + + + // test base data + qt_select_base8 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load8 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 9. test NULL MAP_STRUCT + try { + def struct_field_define = "`ms_info` MAP> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {'a': {'doris', 18}, 'b':{'nereids', 20}, 'c':{'nereids', 21}}); """ + sql """ insert into ${export_table_name} values (2, 'doris2', {'xx': null, 'a': {'doris', 18}}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {'dd': {null, null}}); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {'doris-nereids': {'nereids', null}, 'yyzz': {null, 999999}}); """ + sql """ insert into ${export_table_name} values (7, null, {'null': null, 'null':null}); """ + sql """ insert into ${export_table_name} values (8, null, null); """ + + // test base data + qt_select_base9 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load9 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + // 10. test NOT NULL MAP_STRUCT + try { + def struct_field_define = "`ms_info` MAP> NOT NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {'a': {'doris', 18}, 'b':{'nereids', 20}, 'c':{'nereids', 21}}); """ + sql """ insert into ${export_table_name} values (2, 'doris2', {'xx': null, 'a': {'doris', 18}}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {'dd': {null, null}}); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {'doris-nereids': {'nereids', null}, 'yyzz': {null, 999999}}); """ + sql """ insert into ${export_table_name} values (7, null, {'null': null, 'null':null}); """ + + // test base data + qt_select_base10 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load10 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + // 11. test NULL MAP_MAP + try { + def struct_field_define = "`ms_info` MAP> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {'a': {'a': 100, 'b': 111}, 'b': {'a': 200, 'b': 222, 'c': 333}, 'c':{'nereids': 99}}); """ + sql """ insert into ${export_table_name} values (2, 'doris2', {'xx': null, 'yy': {'a': null, 'b': 333, 'c':399, 'd':399999999999999}}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {'dd': null, 'qq': null, 'ww': {'null': 100, 'b': null}}); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {'doris-nereids': {'nereids': null}, 'yyzz': {'xx': null, 'doris': 999999}}); """ + sql """ insert into ${export_table_name} values (7, null, {'doris': {'null': null, 'null':null}, 'nereids': {'a': 200, 'b': 222, 'c': 333}}); """ + sql """ insert into ${export_table_name} values (8, null, null); """ + + // test base data + qt_select_base11 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load11 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 12. test NULL MAP_LIST + try { + def struct_field_define = "`ms_info` MAP> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {'a': ['doris', 'nereids', 'zzz'], 'b': ['a', 'b', 'c'], 'c': ['qwe', 'doris-nereids', 'pppppppp']}); """ + sql """ insert into ${export_table_name} values (2, 'doris2', {'a': ['doris', null, 'zzz'], 'b': ['a', 'b', null], 'c': [null, 'doris-nereids', 'pppppppp']}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {'a': [null, null, 'zzz'], 'b': [null, null], 'c': [], 'd': null, 'e': [null]}); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {'a': null, 'b': [], 'null': [null, 'doris-nereids', 'pppppppp']}); """ + sql """ insert into ${export_table_name} values (6, null, null); """ + + // test base data + qt_select_base12 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load12 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } +} \ No newline at end of file diff --git a/regression-test/suites/export_p0/outfile/orc/test_outfile_orc_one_nested_type.groovy b/regression-test/suites/export_p0/outfile/orc/test_outfile_orc_one_nested_type.groovy new file mode 100644 index 0000000000..96809c98c9 --- /dev/null +++ b/regression-test/suites/export_p0/outfile/orc/test_outfile_orc_one_nested_type.groovy @@ -0,0 +1,447 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_outfile_orc_one_nested_type", "p0") { + // open nereids + sql """ set enable_nereids_planner=true """ + sql """ set enable_fallback_to_original_planner=false """ + + String ak = getS3AK() + String sk = getS3SK() + String s3_endpoint = getS3Endpoint() + String region = getS3Region() + String bucket = context.config.otherConfigs.get("s3BucketName"); + + def export_table_name = "outfile_orc_one_nested_type_export_test" + def outFilePath = "${bucket}/outfile/orc/nested_complex_type/exp_" + def outfile_format = "orc" + + def create_table = {table_name, struct_field -> + sql """ DROP TABLE IF EXISTS ${table_name} """ + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `name` STRING COMMENT "用户年龄", + ${struct_field} + ) + DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); + """ + } + + def outfile_to_S3 = { + // select ... into outfile ... + def res = sql """ + SELECT * FROM ${export_table_name} t ORDER BY user_id + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS ${outfile_format} + PROPERTIES ( + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${region}", + "s3.secret_key"="${sk}", + "s3.access_key" = "${ak}" + ); + """ + + return res[0][3] + } + + // 1. test NULL STRUCT_STRUCT + try { + def struct_field_define = "`ss_info` STRUCT> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', struct(struct(1, 'sn1', 'sa1'))); """ + sql """ insert into ${export_table_name} values (2, 'doris2', struct(struct(2, 'sn2', 'sa2'))); """ + sql """ insert into ${export_table_name} values (3, 'doris3', struct(null)); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', struct(struct(5, 'sn5', 'sa5'))); """ + sql """ insert into ${export_table_name} values (6, 'doris6', struct(struct(6, 'sn6', 'sa6'))); """ + sql """ insert into ${export_table_name} values (7, null, struct(struct(7, 'sn7', 'sa7'))); """ + sql """ insert into ${export_table_name} values (8, null, null); """ + + + // test base data + qt_select_base1 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load1 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 2. test NOT NULL STRUCT_STRUCT + try { + def struct_field_define = "`ss_info` STRUCT> NOT NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', struct(struct(1, 'sn1', 'sa1'))); """ + sql """ insert into ${export_table_name} values (2, 'doris2', struct(struct(2, 'sn2', 'sa2'))); """ + sql """ insert into ${export_table_name} values (3, 'doris3', struct(null)); """ + sql """ insert into ${export_table_name} values (5, 'doris5', struct(struct(5, 'sn5', 'sa5'))); """ + sql """ insert into ${export_table_name} values (6, 'doris6', struct(struct(6, 'sn6', 'sa6'))); """ + sql """ insert into ${export_table_name} values (7, null, struct(struct(7, 'sn7', 'sa7'))); """ + + // test base data + qt_select_base2 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load2 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 3. test NULL STRUCT_ARRAY + try { + def struct_field_define = "`ss_info` STRUCT> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', struct(['doris1', 'nereids1', 'doris-nereids-1'])); """ + // sql """ insert into ${export_table_name} values (2, 'doris2', {[]}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', struct(null)); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', struct(['doris2', null, 'nereids2'])); """ + // sql """ insert into ${export_table_name} values (6, 'doris6', {[null, null, null]}); """ + sql """ insert into ${export_table_name} values (7, null, struct([null, 'null', 'doris3'])); """ + sql """ insert into ${export_table_name} values (8, null, null); """ + sql """ insert into ${export_table_name} values (9, null, struct(['sn7', 'sa7', 'sn8', 'sa8', 'sn9', 'sa9', 'sn10', 'sa10'])); """ + + + // test base data + qt_select_base3 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load3 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 4. test NULL STRUCT_MAP + try { + def struct_field_define = "`ss_info` STRUCT> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', struct({'a': 100, 'b': 111})); """ + sql """ insert into ${export_table_name} values (2, 'doris2', struct({'a': 200, 'b': 222})); """ + sql """ insert into ${export_table_name} values (3, 'doris3', struct(null)); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', struct({'a': null, 'b': 333, 'c':399, 'd':399999999999999})); """ + sql """ insert into ${export_table_name} values (6, 'doris6', struct({'null': 100, 'b': null})); """ + sql """ insert into ${export_table_name} values (7, null, struct({'null': null, 'null':null})); """ + sql """ insert into ${export_table_name} values (8, null, null); """ + + + // test base data + qt_select_base4 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load4 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 5. test NULL ARRAY_STRUCT + try { + def struct_field_define = "`ss_info` ARRAY> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', array(struct(1, 'doris1'), struct(2, 'nereids1'), struct(3, 'doris-nereids-1'))); """ + sql """ insert into ${export_table_name} values (2, 'doris2', array(struct(4, 'doris-nereids-4'))); """ + sql """ insert into ${export_table_name} values (3, 'doris3', array()); """ + sql """ insert into ${export_table_name} values (4, 'doris4', array(null, null, struct(5, 'doris-nereids-5'))); """ + sql """ insert into ${export_table_name} values (5, 'doris5', array(struct(6, 'doris7'), null, null)); """ + sql """ insert into ${export_table_name} values (6, 'doris6', array(null, null, null)); """ + sql """ insert into ${export_table_name} values (7, null, null); """ + sql """ insert into ${export_table_name} values (8, null, array(struct(8, 'doris8'))); """ + sql """ insert into ${export_table_name} values (9, null, array(struct(9, 'doris9'), struct(10, 'doris10'))); """ + + + // test base data + qt_select_base5 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load5 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + // 6. test NOT NULL ARRAY_STRUCT + try { + def struct_field_define = "`ss_info` ARRAY> NOT NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', array(struct(1, 'doris1'), struct(2, 'nereids1'), struct(3, 'doris-nereids-1'))); """ + sql """ insert into ${export_table_name} values (2, 'doris2', array(struct(4, 'doris-nereids-4'))); """ + sql """ insert into ${export_table_name} values (3, 'doris3', array()); """ + sql """ insert into ${export_table_name} values (4, 'doris4', array(null, null, struct(5, 'doris-nereids-5'))); """ + sql """ insert into ${export_table_name} values (5, 'doris5', array(struct(6, 'doris7'), null, null)); """ + sql """ insert into ${export_table_name} values (6, 'doris6', array(null, null, null)); """ + sql """ insert into ${export_table_name} values (8, null, array(struct(8, 'doris8'))); """ + sql """ insert into ${export_table_name} values (9, null, array(struct(9, 'doris9'), struct(10, 'doris10'))); """ + + + // test base data + qt_select_base6 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load6 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + // 7. test NULL ARRAY_ARRAY + try { + def struct_field_define = "`ss_info` ARRAY> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', [["doris", "null"], ["basdakljdoaidsjowqd", "asdqqqqsdafdorisdoris"], ["a", "b", "c"]]); """ + sql """ insert into ${export_table_name} values (2, 'doris2', [["null"]]); """ + sql """ insert into ${export_table_name} values (3, 'doris3', []); """ + sql """ insert into ${export_table_name} values (4, 'doris4', [["xx", "yy", "nereids"], ["doris", "null"]]); """ + sql """ insert into ${export_table_name} values (5, 'doris5', null); """ + sql """ insert into ${export_table_name} values (6, 'doris6', [null, ["xx", null, "nereids"], null]); """ + sql """ insert into ${export_table_name} values (7, null, null); """ + sql """ insert into ${export_table_name} values (8, null, [[], ["abc"], [], null]); """ + + + // test base data + qt_select_base7 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load7 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + // 8. test NULL ARRAY_MAP + try { + def struct_field_define = "`ss_info` ARRAY> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', [{'a': 100, 'b': 111}, {'a': 200, 'b': 222}, {'a': null, 'b': 333, 'c':399, 'd':399999999999999}]); """ + sql """ insert into ${export_table_name} values (2, 'doris2', [{'doris': 200, 'nereids': 222}]); """ + sql """ insert into ${export_table_name} values (3, 'doris3', []); """ + sql """ insert into ${export_table_name} values (4, 'doris4', [null, null, {'a': 100, 'b': 111}]); """ + sql """ insert into ${export_table_name} values (5, 'doris5', null); """ + sql """ insert into ${export_table_name} values (6, 'doris6', [null, null, {'nereids': 222}, null]); """ + sql """ insert into ${export_table_name} values (7, null, null); """ + sql """ insert into ${export_table_name} values (8, null, [null, null]); """ + sql """ insert into ${export_table_name} values (9, null, [{'doris': 200, 'nereids': 222}, null, null]); """ + + + // test base data + qt_select_base8 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load8 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 9. test NULL MAP_STRUCT + try { + def struct_field_define = "`ms_info` MAP> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', map('a', struct('doris', 18), 'b', struct('nereids', 20), 'c', struct('nereids', 21))); """ + sql """ insert into ${export_table_name} values (2, 'doris2', map('xx', null, 'a', struct('doris', 18))); """ + sql """ insert into ${export_table_name} values (3, 'doris3', map('dd', struct(null, null))); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {'doris-nereids': {'nereids', null}, 'yyzz': {null, 999999}}); """ + sql """ insert into ${export_table_name} values (7, null, map('null', null, 'null',null)); """ + sql """ insert into ${export_table_name} values (8, null, null); """ + + // test base data + qt_select_base9 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load9 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + // 10. test NOT NULL MAP_STRUCT + try { + def struct_field_define = "`ms_info` MAP> NOT NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', map('a', struct('doris', 18), 'b', struct('nereids', 20), 'c', struct('nereids', 21))); """ + sql """ insert into ${export_table_name} values (2, 'doris2', map('xx', null, 'a', struct('doris', 18))); """ + sql """ insert into ${export_table_name} values (3, 'doris3', map('dd', struct(null, null))); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {'doris-nereids': {'nereids', null}, 'yyzz': {null, 999999}}); """ + sql """ insert into ${export_table_name} values (7, null, map('null', null, 'null',null)); """ + + // test base data + qt_select_base10 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load10 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + // 11. test NULL MAP_MAP + try { + def struct_field_define = "`ms_info` MAP> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {'a': {'a': 100, 'b': 111}, 'b': {'a': 200, 'b': 222, 'c': 333}, 'c':{'nereids': 99}}); """ + sql """ insert into ${export_table_name} values (2, 'doris2', {'xx': null, 'yy': {'a': null, 'b': 333, 'c':399, 'd':399999999999999}}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {'dd': null, 'qq': null, 'ww': {'null': 100, 'b': null}}); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {'doris-nereids': {'nereids': null}, 'yyzz': {'xx': null, 'doris': 999999}}); """ + sql """ insert into ${export_table_name} values (7, null, {'doris': {'null': null, 'null':null}, 'nereids': {'a': 200, 'b': 222, 'c': 333}}); """ + sql """ insert into ${export_table_name} values (8, null, null); """ + + // test base data + qt_select_base11 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load11 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 12. test NULL MAP_LIST + try { + def struct_field_define = "`ms_info` MAP> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {'a': ['doris', 'nereids', 'zzz'], 'b': ['a', 'b', 'c'], 'c': ['qwe', 'doris-nereids', 'pppppppp']}); """ + sql """ insert into ${export_table_name} values (2, 'doris2', {'a': ['doris', null, 'zzz'], 'b': ['a', 'b', null], 'c': [null, 'doris-nereids', 'pppppppp']}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {'a': [null, null, 'zzz'], 'b': [null, null], 'c': [], 'd': null, 'e': [null]}); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {'a': null, 'b': [], 'null': [null, 'doris-nereids', 'pppppppp']}); """ + sql """ insert into ${export_table_name} values (6, null, null); """ + + // test base data + qt_select_base12 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load12 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } +} \ No newline at end of file diff --git a/regression-test/suites/export_p0/outfile/parquet/test_outfile_parquet_one_nested_type.groovy b/regression-test/suites/export_p0/outfile/parquet/test_outfile_parquet_one_nested_type.groovy new file mode 100644 index 0000000000..23153d69a8 --- /dev/null +++ b/regression-test/suites/export_p0/outfile/parquet/test_outfile_parquet_one_nested_type.groovy @@ -0,0 +1,447 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_outfile_parquet_one_nested_type", "p0") { + // open nereids + sql """ set enable_nereids_planner=true """ + sql """ set enable_fallback_to_original_planner=false """ + + String ak = getS3AK() + String sk = getS3SK() + String s3_endpoint = getS3Endpoint() + String region = getS3Region() + String bucket = context.config.otherConfigs.get("s3BucketName"); + + def export_table_name = "outfile_parquet_one_nested_type_export_test" + def outFilePath = "${bucket}/outfile/parquet/nested_complex_type/exp_" + def outfile_format = "parquet" + + def create_table = {table_name, struct_field -> + sql """ DROP TABLE IF EXISTS ${table_name} """ + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `name` STRING COMMENT "用户年龄", + ${struct_field} + ) + DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); + """ + } + + def outfile_to_S3 = { + // select ... into outfile ... + def res = sql """ + SELECT * FROM ${export_table_name} t ORDER BY user_id + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS ${outfile_format} + PROPERTIES ( + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${region}", + "s3.secret_key"="${sk}", + "s3.access_key" = "${ak}" + ); + """ + + return res[0][3] + } + + // 1. test NULL STRUCT_STRUCT + try { + def struct_field_define = "`ss_info` STRUCT> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {{1, 'sn1', 'sa1'}}); """ + sql """ insert into ${export_table_name} values (2, 'doris2', {{2, 'sn2', 'sa2'}}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {null}); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {{5, 'sn5', 'sa5'}}); """ + sql """ insert into ${export_table_name} values (6, 'doris6', {{6, 'sn6', 'sa6'}}); """ + sql """ insert into ${export_table_name} values (7, null, {{7, 'sn7', 'sa7'}}); """ + sql """ insert into ${export_table_name} values (8, null, null); """ + + + // test base data + qt_select_base1 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load1 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 2. test NOT NULL STRUCT_STRUCT + try { + def struct_field_define = "`ss_info` STRUCT> NOT NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {{1, 'sn1', 'sa1'}}); """ + sql """ insert into ${export_table_name} values (2, 'doris2', {{2, 'sn2', 'sa2'}}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {null}); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {{5, 'sn5', 'sa5'}}); """ + sql """ insert into ${export_table_name} values (6, 'doris6', {{6, 'sn6', 'sa6'}}); """ + sql """ insert into ${export_table_name} values (7, null, {{7, 'sn7', 'sa7'}}); """ + + // test base data + qt_select_base2 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load2 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 3. test NULL STRUCT_ARRAY + try { + def struct_field_define = "`ss_info` STRUCT> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {['doris1', 'nereids1', 'doris-nereids-1']}); """ + // sql """ insert into ${export_table_name} values (2, 'doris2', {[]}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {null}); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {['doris2', null, 'nereids2']}); """ + // sql """ insert into ${export_table_name} values (6, 'doris6', {[null, null, null]}); """ + sql """ insert into ${export_table_name} values (7, null, {[null, 'null', 'doris3']}); """ + sql """ insert into ${export_table_name} values (8, null, null); """ + sql """ insert into ${export_table_name} values (9, null, {['sn7', 'sa7', 'sn8', 'sa8', 'sn9', 'sa9', 'sn10', 'sa10']}); """ + + + // test base data + qt_select_base3 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load3 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 4. test NULL STRUCT_MAP + try { + def struct_field_define = "`ss_info` STRUCT> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {{'a': 100, 'b': 111}}); """ + sql """ insert into ${export_table_name} values (2, 'doris2', {{'a': 200, 'b': 222}}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {null}); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {{'a': null, 'b': 333, 'c':399, 'd':399999999999999}}); """ + sql """ insert into ${export_table_name} values (6, 'doris6', {{'null': 100, 'b': null}}); """ + sql """ insert into ${export_table_name} values (7, null, {{'null': null, 'null':null}}); """ + sql """ insert into ${export_table_name} values (8, null, null); """ + + + // test base data + qt_select_base4 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load4 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 5. test NULL ARRAY_STRUCT + try { + def struct_field_define = "`ss_info` ARRAY> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', [{1, 'doris1'}, {2, 'nereids1'}, {3, 'doris-nereids-1'}]); """ + sql """ insert into ${export_table_name} values (2, 'doris2', [{4, 'doris-nereids-4'}]); """ + sql """ insert into ${export_table_name} values (3, 'doris3', []); """ + sql """ insert into ${export_table_name} values (4, 'doris4', [null, null , {5, 'doris-nereids-5'}]); """ + sql """ insert into ${export_table_name} values (5, 'doris5', [{6, 'doris7'}, null, null]); """ + sql """ insert into ${export_table_name} values (6, 'doris6', [null, null, null]); """ + sql """ insert into ${export_table_name} values (7, null, null); """ + sql """ insert into ${export_table_name} values (8, null, [{8, 'doris8'}]); """ + sql """ insert into ${export_table_name} values (9, null, [{9, 'doris9'}, {10, 'doris10'}]); """ + + + // test base data + qt_select_base5 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load5 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + // 6. test NOT NULL ARRAY_STRUCT + try { + def struct_field_define = "`ss_info` ARRAY> NOT NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', [{1, 'doris1'}, {2, 'nereids1'}, {3, 'doris-nereids-1'}]); """ + sql """ insert into ${export_table_name} values (2, 'doris2', [{4, 'doris-nereids-4'}]); """ + sql """ insert into ${export_table_name} values (3, 'doris3', []); """ + sql """ insert into ${export_table_name} values (4, 'doris4', [null, null , {5, 'doris-nereids-5'}]); """ + sql """ insert into ${export_table_name} values (5, 'doris5', [{6, 'doris7'}, null, null]); """ + sql """ insert into ${export_table_name} values (6, 'doris6', [null, null, null]); """ + sql """ insert into ${export_table_name} values (8, null, [{8, 'doris8'}]); """ + sql """ insert into ${export_table_name} values (9, null, [{9, 'doris9'}, {10, 'doris10'}]); """ + + + // test base data + qt_select_base6 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load6 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + // 7. test NULL ARRAY_ARRAY + try { + def struct_field_define = "`ss_info` ARRAY> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', [["doris", "null"], ["basdakljdoaidsjowqd", "asdqqqqsdafdorisdoris"], ["a", "b", "c"]]); """ + sql """ insert into ${export_table_name} values (2, 'doris2', [["null"]]); """ + sql """ insert into ${export_table_name} values (3, 'doris3', []); """ + sql """ insert into ${export_table_name} values (4, 'doris4', [["xx", "yy", "nereids"], ["doris", "null"]]); """ + sql """ insert into ${export_table_name} values (5, 'doris5', null); """ + sql """ insert into ${export_table_name} values (6, 'doris6', [null, ["xx", null, "nereids"], null]); """ + sql """ insert into ${export_table_name} values (7, null, null); """ + sql """ insert into ${export_table_name} values (8, null, [[], ["abc"], [], null]); """ + + + // test base data + qt_select_base7 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load7 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + // 8. test NULL ARRAY_MAP + try { + def struct_field_define = "`ss_info` ARRAY> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', [{'a': 100, 'b': 111}, {'a': 200, 'b': 222}, {'a': null, 'b': 333, 'c':399, 'd':399999999999999}]); """ + sql """ insert into ${export_table_name} values (2, 'doris2', [{'doris': 200, 'nereids': 222}]); """ + sql """ insert into ${export_table_name} values (3, 'doris3', []); """ + sql """ insert into ${export_table_name} values (4, 'doris4', [null, null, {'a': 100, 'b': 111}]); """ + sql """ insert into ${export_table_name} values (5, 'doris5', null); """ + sql """ insert into ${export_table_name} values (6, 'doris6', [null, null, {'nereids': 222}, null]); """ + sql """ insert into ${export_table_name} values (7, null, null); """ + sql """ insert into ${export_table_name} values (8, null, [null, null]); """ + sql """ insert into ${export_table_name} values (9, null, [{'doris': 200, 'nereids': 222}, null, null]); """ + + + // test base data + qt_select_base8 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load8 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 9. test NULL MAP_STRUCT + try { + def struct_field_define = "`ms_info` MAP> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {'a': {'doris', 18}, 'b':{'nereids', 20}, 'c':{'nereids', 21}}); """ + sql """ insert into ${export_table_name} values (2, 'doris2', {'xx': null, 'a': {'doris', 18}}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {'dd': {null, null}}); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {'doris-nereids': {'nereids', null}, 'yyzz': {null, 999999}}); """ + sql """ insert into ${export_table_name} values (7, null, {'null': null, 'null':null}); """ + sql """ insert into ${export_table_name} values (8, null, null); """ + + // test base data + qt_select_base9 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load9 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + // 10. test NOT NULL MAP_STRUCT + try { + def struct_field_define = "`ms_info` MAP> NOT NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {'a': {'doris', 18}, 'b':{'nereids', 20}, 'c':{'nereids', 21}}); """ + sql """ insert into ${export_table_name} values (2, 'doris2', {'xx': null, 'a': {'doris', 18}}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {'dd': {null, null}}); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {'doris-nereids': {'nereids', null}, 'yyzz': {null, 999999}}); """ + sql """ insert into ${export_table_name} values (7, null, {'null': null, 'null':null}); """ + + // test base data + qt_select_base10 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load10 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + // 11. test NULL MAP_MAP + try { + def struct_field_define = "`ms_info` MAP> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {'a': {'a': 100, 'b': 111}, 'b': {'a': 200, 'b': 222, 'c': 333}, 'c':{'nereids': 99}}); """ + sql """ insert into ${export_table_name} values (2, 'doris2', {'xx': null, 'yy': {'a': null, 'b': 333, 'c':399, 'd':399999999999999}}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {'dd': null, 'qq': null, 'ww': {'null': 100, 'b': null}}); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {'doris-nereids': {'nereids': null}, 'yyzz': {'xx': null, 'doris': 999999}}); """ + sql """ insert into ${export_table_name} values (7, null, {'doris': {'null': null, 'null':null}, 'nereids': {'a': 200, 'b': 222, 'c': 333}}); """ + sql """ insert into ${export_table_name} values (8, null, null); """ + + // test base data + qt_select_base11 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load11 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } + + // 12. test NULL MAP_LIST + try { + def struct_field_define = "`ms_info` MAP> NULL" + // create table to export data + create_table(export_table_name, struct_field_define) + + // insert data + sql """ insert into ${export_table_name} values (1, 'doris1', {'a': ['doris', 'nereids', 'zzz'], 'b': ['a', 'b', 'c'], 'c': ['qwe', 'doris-nereids', 'pppppppp']}); """ + sql """ insert into ${export_table_name} values (2, 'doris2', {'a': ['doris', null, 'zzz'], 'b': ['a', 'b', null], 'c': [null, 'doris-nereids', 'pppppppp']}); """ + sql """ insert into ${export_table_name} values (3, 'doris3', {'a': [null, null, 'zzz'], 'b': [null, null], 'c': [], 'd': null, 'e': [null]}); """ + sql """ insert into ${export_table_name} values (4, 'doris4', null); """ + sql """ insert into ${export_table_name} values (5, 'doris5', {'a': null, 'b': [], 'null': [null, 'doris-nereids', 'pppppppp']}); """ + sql """ insert into ${export_table_name} values (6, null, null); """ + + // test base data + qt_select_base12 """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + def outfile_url = outfile_to_S3() + + qt_select_load12 """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4, outfile_url.length() - 1)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } finally { + } +}