From f65876d80305565989019589222e076e112abfb0 Mon Sep 17 00:00:00 2001 From: amory Date: Thu, 22 Feb 2024 20:01:19 +0800 Subject: [PATCH] [Feature](explode) support explode map type (#30151) --- .../table_function/table_function_factory.cpp | 2 + .../vec/exprs/table_function/vexplode_map.cpp | 125 +++++++++++++++++ .../vec/exprs/table_function/vexplode_map.h | 68 +++++++++ be/src/vec/functions/function_fake.cpp | 15 ++ .../table-functions/explode_map.md | 131 +++++++++++++++++ .../table-functions/explode_map.md | 132 ++++++++++++++++++ .../org/apache/doris/nereids/DorisParser.g4 | 2 +- .../BuiltinTableGeneratingFunctions.java | 4 + .../org/apache/doris/catalog/FunctionSet.java | 1 + .../nereids/parser/LogicalPlanBuilder.java | 12 +- .../rules/analysis/BindExpression.java | 41 +++++- .../functions/generator/ExplodeMap.java | 80 +++++++++++ .../functions/generator/ExplodeMapOuter.java | 80 +++++++++++ .../expressions/literal/StructLiteral.java | 13 +- .../TableGeneratingFunctionVisitor.java | 10 ++ .../trees/plans/logical/LogicalGenerate.java | 30 +++- .../table_function/explode_map.out | 51 +++++++ .../table_function/explode_map.groovy | 55 ++++++++ 18 files changed, 836 insertions(+), 16 deletions(-) create mode 100644 be/src/vec/exprs/table_function/vexplode_map.cpp create mode 100644 be/src/vec/exprs/table_function/vexplode_map.h create mode 100644 docs/en/docs/sql-manual/sql-functions/table-functions/explode_map.md create mode 100644 docs/zh-CN/docs/sql-manual/sql-functions/table-functions/explode_map.md create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeMap.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeMapOuter.java create mode 100644 regression-test/data/nereids_p0/sql_functions/table_function/explode_map.out create mode 100644 regression-test/suites/nereids_p0/sql_functions/table_function/explode_map.groovy diff --git a/be/src/vec/exprs/table_function/table_function_factory.cpp b/be/src/vec/exprs/table_function/table_function_factory.cpp index bafe47b69e..e42c0a27fd 100644 --- a/be/src/vec/exprs/table_function/table_function_factory.cpp +++ b/be/src/vec/exprs/table_function/table_function_factory.cpp @@ -24,6 +24,7 @@ #include "vec/exprs/table_function/vexplode.h" #include "vec/exprs/table_function/vexplode_bitmap.h" #include "vec/exprs/table_function/vexplode_json_array.h" +#include "vec/exprs/table_function/vexplode_map.h" #include "vec/exprs/table_function/vexplode_numbers.h" #include "vec/exprs/table_function/vexplode_split.h" #include "vec/utils/util.hpp" @@ -61,6 +62,7 @@ const std::unordered_map()}, + {"explode_map", TableFunctionCreator {}}, {"explode", TableFunctionCreator {}}}; Status TableFunctionFactory::get_fn(const std::string& fn_name_raw, ObjectPool* pool, diff --git a/be/src/vec/exprs/table_function/vexplode_map.cpp b/be/src/vec/exprs/table_function/vexplode_map.cpp new file mode 100644 index 0000000000..923316fd28 --- /dev/null +++ b/be/src/vec/exprs/table_function/vexplode_map.cpp @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/exprs/table_function/vexplode_map.h" + +#include + +#include +#include + +#include "common/status.h" +#include "vec/columns/column.h" +#include "vec/common/string_ref.h" +#include "vec/core/block.h" +#include "vec/core/column_with_type_and_name.h" +#include "vec/exprs/vexpr.h" +#include "vec/exprs/vexpr_context.h" + +namespace doris::vectorized { + +VExplodeMapTableFunction::VExplodeMapTableFunction() { + _fn_name = "vexplode_map"; +} + +bool extract_column_map_info(const IColumn& src, ColumnMapExecutionData& data) { + const IColumn* map_col = &src; + // extract array nullable info + if (src.is_nullable()) { + const auto& null_col = reinterpret_cast(src); + // map column's nullmap + data.map_nullmap_data = null_col.get_null_map_data().data(); + map_col = null_col.get_nested_column_ptr().get(); + } + + if (data.map_col = check_and_get_column(map_col); !data.map_col) { + return false; + } + + data.offsets_ptr = &data.map_col->get_offsets(); + return true; +} + +Status VExplodeMapTableFunction::process_init(Block* block, RuntimeState* state) { + CHECK(_expr_context->root()->children().size() == 1) + << "VExplodeMapTableFunction only support 1 child but has " + << _expr_context->root()->children().size(); + + int value_column_idx = -1; + RETURN_IF_ERROR(_expr_context->root()->children()[0]->execute(_expr_context.get(), block, + &value_column_idx)); + + _collection_column = + block->get_by_position(value_column_idx).column->convert_to_full_column_if_const(); + + if (!extract_column_map_info(*_collection_column, _map_detail)) { + return Status::NotSupported("column type {} not supported now, only support array or map", + block->get_by_position(value_column_idx).column->get_name()); + } + + return Status::OK(); +} + +void VExplodeMapTableFunction::process_row(size_t row_idx) { + DCHECK(row_idx < _collection_column->size()); + TableFunction::process_row(row_idx); + + if (!_map_detail.map_nullmap_data || !_map_detail.map_nullmap_data[row_idx]) { + _collection_offset = (*_map_detail.offsets_ptr)[row_idx - 1]; + _cur_size = (*_map_detail.offsets_ptr)[row_idx] - _collection_offset; + } +} + +void VExplodeMapTableFunction::process_close() { + _collection_column = nullptr; + _map_detail.reset(); + _collection_offset = 0; +} + +void VExplodeMapTableFunction::get_value(MutableColumnPtr& column) { + // now we only support map column explode to struct column + size_t pos = _collection_offset + _cur_offset; + // if current is empty map row, also append a default value + if (current_empty()) { + column->insert_default(); + return; + } + ColumnStruct* ret = nullptr; + // this _is_nullable is whole output column's nullable + if (_is_nullable) { + // make map kv value into struct + ret = assert_cast( + assert_cast(column.get())->get_nested_column_ptr().get()); + assert_cast( + assert_cast(column.get())->get_null_map_column_ptr().get()) + ->insert_default(); + } else if (column->is_column_struct()) { + ret = assert_cast(column.get()); + } else { + throw Exception(ErrorCode::INTERNAL_ERROR, + "only support map column explode to struct column"); + } + if (!ret || ret->tuple_size() != 2) { + throw Exception( + ErrorCode::INTERNAL_ERROR, + "only support map column explode to two column, but given: ", ret->tuple_size()); + } + ret->get_column(0).insert_from(_map_detail.map_col->get_keys(), pos); + ret->get_column(1).insert_from(_map_detail.map_col->get_values(), pos); +} + +} // namespace doris::vectorized diff --git a/be/src/vec/exprs/table_function/vexplode_map.h b/be/src/vec/exprs/table_function/vexplode_map.h new file mode 100644 index 0000000000..770c06e3bb --- /dev/null +++ b/be/src/vec/exprs/table_function/vexplode_map.h @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "vec/columns/column_map.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_array.h" +#include "vec/data_types/data_type_map.h" +#include "vec/exprs/table_function/table_function.h" +#include "vec/functions/array/function_array_utils.h" + +namespace doris::vectorized { +class Block; +} // namespace doris::vectorized + +namespace doris::vectorized { + +struct ColumnMapExecutionData { +public: + void reset() { + map_nullmap_data = nullptr; + map_col = nullptr; + offsets_ptr = nullptr; + } + + const UInt8* map_nullmap_data = nullptr; + const ColumnMap* map_col = nullptr; + const ColumnArray::Offsets64* offsets_ptr = nullptr; +}; + +class VExplodeMapTableFunction : public TableFunction { + ENABLE_FACTORY_CREATOR(VExplodeMapTableFunction); + +public: + VExplodeMapTableFunction(); + + ~VExplodeMapTableFunction() override = default; + + Status process_init(Block* block, RuntimeState* state) override; + void process_row(size_t row_idx) override; + void process_close() override; + void get_value(MutableColumnPtr& column) override; + +private: + ColumnPtr _collection_column; + ColumnMapExecutionData _map_detail; + size_t _collection_offset; // start offset of array[row_idx] +}; + +} // namespace doris::vectorized diff --git a/be/src/vec/functions/function_fake.cpp b/be/src/vec/functions/function_fake.cpp index 26896c028c..2d8e258d52 100644 --- a/be/src/vec/functions/function_fake.cpp +++ b/be/src/vec/functions/function_fake.cpp @@ -27,9 +27,11 @@ #include "vec/aggregate_functions/aggregate_function.h" #include "vec/data_types/data_type_array.h" #include "vec/data_types/data_type_jsonb.h" +#include "vec/data_types/data_type_map.h" #include "vec/data_types/data_type_nullable.h" #include "vec/data_types/data_type_number.h" #include "vec/data_types/data_type_string.h" +#include "vec/data_types/data_type_struct.h" #include "vec/exprs/table_function/table_function.h" #include "vec/functions/function_helpers.h" #include "vec/functions/simple_function_factory.h" @@ -56,6 +58,18 @@ struct FunctionExplode { static std::string get_error_msg() { return "Fake function do not support execute"; } }; +// explode map: make map k,v as struct field +struct FunctionExplodeMap { + static DataTypePtr get_return_type_impl(const DataTypes& arguments) { + DCHECK(is_map(arguments[0])) << arguments[0]->get_name() << " not supported"; + DataTypes fieldTypes(2); + fieldTypes[0] = check_and_get_data_type(arguments[0].get())->get_key_type(); + fieldTypes[1] = check_and_get_data_type(arguments[0].get())->get_value_type(); + return make_nullable(std::make_shared(fieldTypes)); + } + static std::string get_error_msg() { return "Fake function do not support execute"; } +}; + struct FunctionEsquery { static DataTypePtr get_return_type_impl(const DataTypes& arguments) { return FunctionFakeBaseImpl::get_return_type_impl(arguments); @@ -97,6 +111,7 @@ void register_function_fake(SimpleFunctionFactory& factory) { register_function(factory, "esquery"); register_table_function_expand_outer(factory, "explode"); + register_table_function_expand_outer(factory, "explode_map"); register_table_function_expand_outer_default(factory, "explode_split"); register_table_function_expand_outer_default(factory, "explode_numbers"); diff --git a/docs/en/docs/sql-manual/sql-functions/table-functions/explode_map.md b/docs/en/docs/sql-manual/sql-functions/table-functions/explode_map.md new file mode 100644 index 0000000000..7c5cae597c --- /dev/null +++ b/docs/en/docs/sql-manual/sql-functions/table-functions/explode_map.md @@ -0,0 +1,131 @@ +--- +{ + "title": "EXPLODE_MAP", + "language": "en" +} +--- + + + +## explode + +### description + +Table functions must be used in conjunction with Lateral View, support multi conjunction with Lateral View,support new optimizer only. + +explode map column to rows. `explode_map_outer` will return NULL, while `map` is NULL or empty. +`explode_map` and `explode_map_outer` both keep the nested NULL elements of map. + +#### syntax +```sql +explode_map(expr) +explode_map_outer(expr) +``` + +### example +``` +mysql> set enable_vectorized_engine = true +mysql> SET enable_nereids_planner=true +mysql> SET enable_fallback_to_original_planner=false + +mysql> CREATE TABLE IF NOT EXISTS `sdu`( + `id` INT NULL, + `name` TEXT NULL, + `score` MAP NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); +Query OK, 0 rows affected (0.15 sec) + +mysql> insert into sdu values (0, "zhangsan", {"Chinese":"80","Math":"60","English":"90"}), (1, "lisi", {"null":null}), (2, "wangwu", {"Chinese":"88","Math":"90","English":"96"}), (3, "lisi2", {null:null}), (4, "amory", NULL); +Query OK, 5 rows affected (0.23 sec) +{'label':'label_9b35d9d9d59147f5_bffb974881ed2133', 'status':'VISIBLE', 'txnId':'4005'} + +mysql> select * from sdu order by id; ++------+----------+-----------------------------------------+ +| id | name | score | ++------+----------+-----------------------------------------+ +| 0 | zhangsan | {"Chinese":80, "Math":60, "English":90} | +| 1 | lisi | {"null":null} | +| 2 | wangwu | {"Chinese":88, "Math":90, "English":96} | +| 3 | lisi2 | {null:null} | +| 4 | amory | NULL | ++------+----------+-----------------------------------------+ + +mysql> select name, k,v from sdu lateral view explode_map(score) tmp as k,v; ++----------+---------+------+ +| name | k | v | ++----------+---------+------+ +| zhangsan | Chinese | 80 | +| zhangsan | Math | 60 | +| zhangsan | English | 90 | +| lisi | null | NULL | +| wangwu | Chinese | 88 | +| wangwu | Math | 90 | +| wangwu | English | 96 | +| lisi2 | NULL | NULL | ++----------+---------+------+ + +mysql> select name, k,v from sdu lateral view explode_map_outer(score) tmp as k,v; ++----------+---------+------+ +| name | k | v | ++----------+---------+------+ +| zhangsan | Chinese | 80 | +| zhangsan | Math | 60 | +| zhangsan | English | 90 | +| lisi | null | NULL | +| wangwu | Chinese | 88 | +| wangwu | Math | 90 | +| wangwu | English | 96 | +| lisi2 | NULL | NULL | +| amory | NULL | NULL | ++----------+---------+------+ + +mysql> select name, k,v,k1,v1 from sdu lateral view explode_map_outer(score) tmp as k,v lateral view explode_map(score) tmp2 as k1,v1; ++----------+---------+------+---------+------+ +| name | k | v | k1 | v1 | ++----------+---------+------+---------+------+ +| zhangsan | Chinese | 80 | Chinese | 80 | +| zhangsan | Chinese | 80 | Math | 60 | +| zhangsan | Chinese | 80 | English | 90 | +| zhangsan | Math | 60 | Chinese | 80 | +| zhangsan | Math | 60 | Math | 60 | +| zhangsan | Math | 60 | English | 90 | +| zhangsan | English | 90 | Chinese | 80 | +| zhangsan | English | 90 | Math | 60 | +| zhangsan | English | 90 | English | 90 | +| lisi | null | NULL | null | NULL | +| wangwu | Chinese | 88 | Chinese | 88 | +| wangwu | Chinese | 88 | Math | 90 | +| wangwu | Chinese | 88 | English | 96 | +| wangwu | Math | 90 | Chinese | 88 | +| wangwu | Math | 90 | Math | 90 | +| wangwu | Math | 90 | English | 96 | +| wangwu | English | 96 | Chinese | 88 | +| wangwu | English | 96 | Math | 90 | +| wangwu | English | 96 | English | 96 | +| lisi2 | NULL | NULL | NULL | NULL | ++----------+---------+------+---------+------+ +``` + +### keywords +EXPLODE_MAP,EXPLODE_MAP_OUTER,MAP \ No newline at end of file diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/explode_map.md b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/explode_map.md new file mode 100644 index 0000000000..2cc7c30dbe --- /dev/null +++ b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/explode_map.md @@ -0,0 +1,132 @@ +--- +{ +"title": "EXPLODE_MAP", +"language": "zh-CN" +} +--- + + + +## explode_map + +### description + +表函数，需配合 Lateral View 使用, 可以支持多个 Lateral view, 仅仅支持新优化器。 + +将 map 列展开成多行。当 map 为NULL或者为空时，`explode_map_outer` 返回NULL。 +`explode_map` 和 `explode_map_outer` 均会返回 map 内部的NULL元素。 + +#### syntax +```sql +explode_map(expr) +explode_map_outer(expr) +``` + +### example + +``` +mysql> set enable_vectorized_engine = true +mysql> SET enable_nereids_planner=true +mysql> SET enable_fallback_to_original_planner=false + +mysql> CREATE TABLE IF NOT EXISTS `sdu`( + `id` INT NULL, + `name` TEXT NULL, + `score` MAP NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); +Query OK, 0 rows affected (0.15 sec) + +mysql> insert into sdu values (0, "zhangsan", {"Chinese":"80","Math":"60","English":"90"}), (1, "lisi", {"null":null}), (2, "wangwu", {"Chinese":"88","Math":"90","English":"96"}), (3, "lisi2", {null:null}), (4, "amory", NULL); +Query OK, 5 rows affected (0.23 sec) +{'label':'label_9b35d9d9d59147f5_bffb974881ed2133', 'status':'VISIBLE', 'txnId':'4005'} + +mysql> select * from sdu order by id; ++------+----------+-----------------------------------------+ +| id | name | score | ++------+----------+-----------------------------------------+ +| 0 | zhangsan | {"Chinese":80, "Math":60, "English":90} | +| 1 | lisi | {"null":null} | +| 2 | wangwu | {"Chinese":88, "Math":90, "English":96} | +| 3 | lisi2 | {null:null} | +| 4 | amory | NULL | ++------+----------+-----------------------------------------+ + +mysql> select name, k,v from sdu lateral view explode_map(score) tmp as k,v; ++----------+---------+------+ +| name | k | v | ++----------+---------+------+ +| zhangsan | Chinese | 80 | +| zhangsan | Math | 60 | +| zhangsan | English | 90 | +| lisi | null | NULL | +| wangwu | Chinese | 88 | +| wangwu | Math | 90 | +| wangwu | English | 96 | +| lisi2 | NULL | NULL | ++----------+---------+------+ + +mysql> select name, k,v from sdu lateral view explode_map_outer(score) tmp as k,v; ++----------+---------+------+ +| name | k | v | ++----------+---------+------+ +| zhangsan | Chinese | 80 | +| zhangsan | Math | 60 | +| zhangsan | English | 90 | +| lisi | null | NULL | +| wangwu | Chinese | 88 | +| wangwu | Math | 90 | +| wangwu | English | 96 | +| lisi2 | NULL | NULL | +| amory | NULL | NULL | ++----------+---------+------+ + +mysql> select name, k,v,k1,v1 from sdu lateral view explode_map_outer(score) tmp as k,v lateral view explode_map(score) tmp2 as k1,v1; ++----------+---------+------+---------+------+ +| name | k | v | k1 | v1 | ++----------+---------+------+---------+------+ +| zhangsan | Chinese | 80 | Chinese | 80 | +| zhangsan | Chinese | 80 | Math | 60 | +| zhangsan | Chinese | 80 | English | 90 | +| zhangsan | Math | 60 | Chinese | 80 | +| zhangsan | Math | 60 | Math | 60 | +| zhangsan | Math | 60 | English | 90 | +| zhangsan | English | 90 | Chinese | 80 | +| zhangsan | English | 90 | Math | 60 | +| zhangsan | English | 90 | English | 90 | +| lisi | null | NULL | null | NULL | +| wangwu | Chinese | 88 | Chinese | 88 | +| wangwu | Chinese | 88 | Math | 90 | +| wangwu | Chinese | 88 | English | 96 | +| wangwu | Math | 90 | Chinese | 88 | +| wangwu | Math | 90 | Math | 90 | +| wangwu | Math | 90 | English | 96 | +| wangwu | English | 96 | Chinese | 88 | +| wangwu | English | 96 | Math | 90 | +| wangwu | English | 96 | English | 96 | +| lisi2 | NULL | NULL | NULL | NULL | ++----------+---------+------+---------+------+ +``` + +### keywords +EXPLODE_MAP,EXPLODE_MAP_OUTER,MAP diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index a10f204b0f..73933fd86b 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -394,7 +394,7 @@ updateAssignmentSeq lateralView : LATERAL VIEW functionName=identifier LEFT_PAREN (expression (COMMA expression)*)? RIGHT_PAREN - tableName=identifier AS columnName=identifier + tableName=identifier AS columnNames+=identifier (COMMA columnNames+=identifier)* ; queryOrganization diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java index cdcebc4be9..6fab5cfc04 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java @@ -28,6 +28,8 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeJso import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeJsonArrayJsonOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeJsonArrayString; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeJsonArrayStringOuter; +import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeMap; +import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeMapOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeNumbers; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeNumbersOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeOuter; @@ -48,6 +50,8 @@ public class BuiltinTableGeneratingFunctions implements FunctionHelper { public final List tableGeneratingFunctions = ImmutableList.of( tableGenerating(Explode.class, "explode"), tableGenerating(ExplodeOuter.class, "explode_outer"), + tableGenerating(ExplodeMap.class, "explode_map"), + tableGenerating(ExplodeMapOuter.class, "explode_map_outer"), tableGenerating(ExplodeNumbers.class, "explode_numbers"), tableGenerating(ExplodeNumbersOuter.class, "explode_numbers_outer"), tableGenerating(ExplodeBitmap.class, "explode_bitmap"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index c06e0dcd50..babd697680 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -1931,6 +1931,7 @@ public class FunctionSet { addTableFunctionWithCombinator(EXPLODE, Type.WILDCARD_DECIMAL, Function.NullableMode.ALWAYS_NULLABLE, Lists.newArrayList(new ArrayType(Type.WILDCARD_DECIMAL)), false, "_ZN5doris19DummyTableFunctions7explodeEPN9doris_udf15FunctionContextERKNS1_13CollectionValE"); + } public boolean isAggFunctionName(String name) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 2a6a602b71..550785999f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1023,14 +1023,22 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { return plan; } String generateName = ctx.tableName.getText(); - String columnName = ctx.columnName.getText(); + // if later view explode map type, we need to add a project to convert map to struct + String columnName = ctx.columnNames.get(0).getText(); + List expandColumnNames = Lists.newArrayList(); + if (ctx.columnNames.size() > 1) { + columnName = ConnectContext.get() != null + ? ConnectContext.get().getStatementContext().generateColumnName() : "expand_cols"; + expandColumnNames = ctx.columnNames.stream() + .map(RuleContext::getText).collect(ImmutableList.toImmutableList()); + } String functionName = ctx.functionName.getText(); List arguments = ctx.expression().stream() .map(this::typedVisit) .collect(ImmutableList.toImmutableList()); Function unboundFunction = new UnboundFunction(functionName, arguments); return new LogicalGenerate<>(ImmutableList.of(unboundFunction), - ImmutableList.of(new UnboundSlot(generateName, columnName)), plan); + ImmutableList.of(new UnboundSlot(generateName, columnName)), ImmutableList.of(expandColumnNames), plan); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index df910ecc56..f31f1f15e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -53,8 +53,11 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunctio import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction; import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda; +import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement; import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; +import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; @@ -82,6 +85,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation; import org.apache.doris.nereids.trees.plans.logical.UsingJoin; import org.apache.doris.nereids.trees.plans.visitor.InferPlanOutputAlias; import org.apache.doris.nereids.types.BooleanType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.qe.ConnectContext; @@ -93,6 +98,7 @@ import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; @@ -587,7 +593,7 @@ public class BindExpression implements AnalysisRuleFactory { // we need to do cast before set operation, because we maybe use these slot to do shuffle // so, we must cast it before shuffle to get correct hash code. List> childrenProjections = setOperation.collectChildrenProjections(); - ImmutableList.Builder> childrenOutputs = ImmutableList.builder(); + Builder> childrenOutputs = ImmutableList.builder(); Builder newChildren = ImmutableList.builder(); for (int i = 0; i < childrenProjections.size(); i++) { Plan newChild; @@ -608,14 +614,15 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_GENERATE_SLOT.build( - logicalGenerate().thenApply(ctx -> { + logicalGenerate().when(AbstractPlan::canBind).thenApply(ctx -> { LogicalGenerate generate = ctx.root; List boundSlotGenerators = bindSlot(generate.getGenerators(), generate.child(), ctx.cascadesContext); List boundFunctionGenerators = boundSlotGenerators.stream() .map(f -> bindTableGeneratingFunction((UnboundFunction) f, ctx.root, ctx.cascadesContext)) .collect(Collectors.toList()); - Builder slotBuilder = ImmutableList.builder(); + ImmutableList.Builder slotBuilder = ImmutableList.builder(); + List expandAlias = Lists.newArrayList(); for (int i = 0; i < generate.getGeneratorOutput().size(); i++) { Function generator = boundFunctionGenerators.get(i); UnboundSlot slot = (UnboundSlot) generate.getGeneratorOutput().get(i); @@ -624,8 +631,34 @@ public class BindExpression implements AnalysisRuleFactory { Slot boundSlot = new SlotReference(slot.getNameParts().get(1), generator.getDataType(), generator.nullable(), ImmutableList.of(slot.getNameParts().get(0))); slotBuilder.add(boundSlot); + // the boundSlot may has two situation: + // 1. the expandColumnsAlias is not empty, we should use make boundSlot expand to multi alias + // 2. the expandColumnsAlias is empty, we should use origin boundSlot + if (generate.getExpandColumnAlias() != null && i < generate.getExpandColumnAlias().size() + && !CollectionUtils.isEmpty(generate.getExpandColumnAlias().get(i))) { + // if the alias is not empty, we should bind it with struct_element as child expr with alias + // struct_element(#expand_col#k, #k) as #k + // struct_element(#expand_col#v, #v) as #v + List fields = ((StructType) boundSlot.getDataType()).getFields(); + for (int idx = 0; idx < fields.size(); ++idx) { + expandAlias.add(new Alias(new StructElement( + boundSlot, new StringLiteral(fields.get(idx).getName())), + generate.getExpandColumnAlias().get(i).get(idx))); + } + } } - return new LogicalGenerate<>(boundFunctionGenerators, slotBuilder.build(), generate.child()); + LogicalGenerate ret = new LogicalGenerate<>( + boundFunctionGenerators, slotBuilder.build(), generate.child()); + if (expandAlias.size() > 0) { + // we need a project to deal with explode(map) to struct with field alias + // project should contains: generator.child slot + expandAlias + List allProjectSlots = generate.child().getOutput().stream() + .map(NamedExpression.class::cast) + .collect(Collectors.toList()); + allProjectSlots.addAll(expandAlias); + return new LogicalProject<>(allProjectSlots, ret); + } + return ret; }) ), RuleType.BINDING_UNBOUND_TVF_RELATION_FUNCTION.build( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeMap.java new file mode 100644 index 0000000000..068c2220d1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeMap.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.generator; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.MapType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * explode({"amory":1, "doris": 2}) generate two column and two lines with: + * key column: amory, doris + * value column: 1, 2 + */ +public class ExplodeMap extends TableGeneratingFunction implements UnaryExpression, AlwaysNullable { + + /** + * constructor with 1 argument. + */ + public ExplodeMap(Expression arg) { + super("explode_map", arg); + } + + /** + * withChildren. + */ + @Override + public ExplodeMap withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new ExplodeMap(children.get(0)); + } + + @Override + public void checkLegalityBeforeTypeCoercion() { + if (!(child().getDataType() instanceof MapType)) { + throw new AnalysisException("only support map type for explode_map function but got " + + child().getDataType()); + } + } + + @Override + public List getSignatures() { + return ImmutableList.of( + FunctionSignature.ret(new StructType(ImmutableList.of( + new StructField("col1", ((MapType) child().getDataType()).getKeyType(), true, ""), + new StructField("col2", ((MapType) child().getDataType()).getValueType(), true, "")))) + .args(child().getDataType()) + ); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitExplodeMap(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeMapOuter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeMapOuter.java new file mode 100644 index 0000000000..a98aa5e850 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeMapOuter.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.generator; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.MapType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import java.util.List; + +/** + * explode({"amory":1, "doris": 2}) generate two column and two lines with: + * key column: amory, doris + * value column: 1, 2 + */ +public class ExplodeMapOuter extends TableGeneratingFunction implements UnaryExpression, AlwaysNullable { + + /** + * constructor with 1 argument. + */ + public ExplodeMapOuter(Expression arg) { + super("explode_map_outer", arg); + } + + /** + * withChildren. + */ + @Override + public ExplodeMapOuter withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new ExplodeMapOuter(children.get(0)); + } + + @Override + public void checkLegalityBeforeTypeCoercion() { + if (!(child().getDataType() instanceof MapType)) { + throw new AnalysisException("only support map type for explode_map function but got " + + child().getDataType()); + } + } + + @Override + public List getSignatures() { + return ImmutableList.of( + FunctionSignature.ret(StructLiteral.constructStructType( + Lists.newArrayList(((MapType) child().getDataType()).getKeyType(), + ((MapType) child().getDataType()).getValueType()))) + .args(child().getDataType()) + ); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitExplodeMapOuter(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java index 4b4200c233..2fd2186f49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java @@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.expressions.literal; import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.StructField; @@ -30,6 +31,7 @@ import com.google.common.collect.ImmutableList; import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; /** * struct literal @@ -139,11 +141,16 @@ public class StructLiteral extends Literal { return visitor.visitStructLiteral(this, context); } - public static StructType computeDataType(List fields) { + public static StructType constructStructType(List fieldTypes) { ImmutableList.Builder structFields = ImmutableList.builder(); - for (int i = 0; i < fields.size(); i++) { - structFields.add(new StructField("col" + (i + 1), fields.get(i).getDataType(), true, "")); + for (int i = 0; i < fieldTypes.size(); i++) { + structFields.add(new StructField("col" + (i + 1), fieldTypes.get(i), true, "")); } return new StructType(structFields.build()); } + + public static StructType computeDataType(List fields) { + List fieldTypes = fields.stream().map(ExpressionTrait::getDataType).collect(Collectors.toList()); + return constructStructType(fieldTypes); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java index 5046e3c205..4e4c8ab2bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java @@ -28,6 +28,8 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeJso import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeJsonArrayJsonOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeJsonArrayString; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeJsonArrayStringOuter; +import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeMap; +import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeMapOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeNumbers; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeNumbersOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeOuter; @@ -49,6 +51,14 @@ public interface TableGeneratingFunctionVisitor { return visitTableGeneratingFunction(explodeOuter, context); } + default R visitExplodeMap(ExplodeMap explode, C context) { + return visitTableGeneratingFunction(explode, context); + } + + default R visitExplodeMapOuter(ExplodeMapOuter explodeOuter, C context) { + return visitTableGeneratingFunction(explodeOuter, context); + } + default R visitExplodeNumbers(ExplodeNumbers explodeNumbers, C context) { return visitTableGeneratingFunction(explodeNumbers, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java index 2bb5d79b48..a26ce99d6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java @@ -47,17 +47,25 @@ public class LogicalGenerate extends LogicalUnary generators; private final List generatorOutput; + // mapping with function. + private final List> expandColumnAlias; public LogicalGenerate(List generators, List generatorOutput, CHILD_TYPE child) { - this(generators, generatorOutput, Optional.empty(), Optional.empty(), child); + this(generators, generatorOutput, ImmutableList.of(), Optional.empty(), Optional.empty(), child); } - public LogicalGenerate(List generators, List generatorOutput, + public LogicalGenerate(List generators, List generatorOutput, List> expandColumnAlias, + CHILD_TYPE child) { + this(generators, generatorOutput, expandColumnAlias, Optional.empty(), Optional.empty(), child); + } + + public LogicalGenerate(List generators, List generatorOutput, List> expandColumnAlias, Optional groupExpression, Optional logicalProperties, CHILD_TYPE child) { super(PlanType.LOGICAL_GENERATE, groupExpression, logicalProperties, child); this.generators = ImmutableList.copyOf(generators); this.generatorOutput = ImmutableList.copyOf(generatorOutput); + this.expandColumnAlias = ImmutableList.copyOf(expandColumnAlias); } public List getGenerators() { @@ -68,10 +76,19 @@ public class LogicalGenerate extends LogicalUnary> getExpandColumnAlias() { + return expandColumnAlias; + } + + public LogicalGenerate withExpandColumnAlias(List> expandColumnAlias) { + return new LogicalGenerate<>(generators, generatorOutput, expandColumnAlias, + Optional.empty(), Optional.of(getLogicalProperties()), child()); + } + @Override public LogicalGenerate withChildren(List children) { Preconditions.checkArgument(children.size() == 1); - return new LogicalGenerate<>(generators, generatorOutput, children.get(0)); + return new LogicalGenerate<>(generators, generatorOutput, expandColumnAlias, children.get(0)); } @Override @@ -93,13 +110,13 @@ public class LogicalGenerate extends LogicalUnary(generators, newGeneratorOutput, + return new LogicalGenerate<>(generators, newGeneratorOutput, expandColumnAlias, Optional.empty(), Optional.of(getLogicalProperties()), child()); } @Override public LogicalGenerate withGroupExpression(Optional groupExpression) { - return new LogicalGenerate<>(generators, generatorOutput, + return new LogicalGenerate<>(generators, generatorOutput, expandColumnAlias, groupExpression, Optional.of(getLogicalProperties()), child()); } @@ -107,7 +124,8 @@ public class LogicalGenerate extends LogicalUnary groupExpression, Optional logicalProperties, List children) { Preconditions.checkArgument(children.size() == 1); - return new LogicalGenerate<>(generators, generatorOutput, groupExpression, logicalProperties, children.get(0)); + return new LogicalGenerate<>(generators, generatorOutput, expandColumnAlias, + groupExpression, logicalProperties, children.get(0)); } @Override diff --git a/regression-test/data/nereids_p0/sql_functions/table_function/explode_map.out b/regression-test/data/nereids_p0/sql_functions/table_function/explode_map.out new file mode 100644 index 0000000000..295b908d7f --- /dev/null +++ b/regression-test/data/nereids_p0/sql_functions/table_function/explode_map.out @@ -0,0 +1,51 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +0 zhangsan {"Chinese":80, "Math":60, "English":90} +1 lisi {"null":null} +2 wangwu {"Chinese":88, "Math":90, "English":96} +3 lisi2 {null:null} +4 amory \N + +-- !explode_sql -- +lisi null \N +lisi2 \N \N +wangwu Chinese 88 +wangwu English 96 +wangwu Math 90 +zhangsan Chinese 80 +zhangsan English 90 +zhangsan Math 60 + +-- !explode_outer_sql -- +amory \N \N +lisi null \N +lisi2 \N \N +wangwu Chinese 88 +wangwu English 96 +wangwu Math 90 +zhangsan Chinese 80 +zhangsan English 90 +zhangsan Math 60 + +-- !explode_sql_multi -- +lisi null \N null \N +lisi2 \N \N \N \N +wangwu Chinese 88 Chinese 88 +wangwu Chinese 88 English 96 +wangwu Chinese 88 Math 90 +wangwu English 96 Chinese 88 +wangwu English 96 English 96 +wangwu English 96 Math 90 +wangwu Math 90 Chinese 88 +wangwu Math 90 English 96 +wangwu Math 90 Math 90 +zhangsan Chinese 80 Chinese 80 +zhangsan Chinese 80 English 90 +zhangsan Chinese 80 Math 60 +zhangsan English 90 Chinese 80 +zhangsan English 90 English 90 +zhangsan English 90 Math 60 +zhangsan Math 60 Chinese 80 +zhangsan Math 60 English 90 +zhangsan Math 60 Math 60 + diff --git a/regression-test/suites/nereids_p0/sql_functions/table_function/explode_map.groovy b/regression-test/suites/nereids_p0/sql_functions/table_function/explode_map.groovy new file mode 100644 index 0000000000..98656df99b --- /dev/null +++ b/regression-test/suites/nereids_p0/sql_functions/table_function/explode_map.groovy @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("explode_map") { + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + sql """ DROP TABLE IF EXISTS sdu """ + sql """ + CREATE TABLE IF NOT EXISTS `sdu`( + `id` INT NULL, + `name` TEXT NULL, + `score` MAP NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + + // insert values + sql """ insert into sdu values (0, "zhangsan", {"Chinese":"80","Math":"60","English":"90"}); """ + sql """ insert into sdu values (1, "lisi", {"null":null}); """ + sql """ insert into sdu values (2, "wangwu", {"Chinese":"88","Math":"90","English":"96"}); """ + sql """ insert into sdu values (3, "lisi2", {null:null}); """ + sql """ insert into sdu values (4, "amory", NULL); """ + + qt_sql """ select * from sdu order by id; """ + order_qt_explode_sql """ select name, k,v from sdu lateral view explode_map(score) tmp as k,v order by id;""" + order_qt_explode_outer_sql """ select name, k,v from sdu lateral view explode_map_outer(score) tmp as k,v order by id; """ + + // multi lateral view + order_qt_explode_sql_multi """ select name, k,v,k1,v1 from sdu lateral view explode_map_outer(score) tmp as k,v lateral view explode_map(score) tmp2 as k1,v1 order by id;""" + + // Old planner do not support explode_map + sql "SET enable_nereids_planner=false;" + test { + sql "select name, k,v from sdu lateral view explode_map(score) tmp as k,v;" + exception "errCode = 2," + } +}