From 85080ee3c30111fbf9140e7df24fccf22c371952 Mon Sep 17 00:00:00 2001 From: zhangstar333 <87313068+zhangstar333@users.noreply.github.com> Date: Wed, 15 Mar 2023 10:51:29 +0800 Subject: [PATCH] [vectorized](function) support array_map function (#17581) --- be/src/runtime/define_primitive_type.h | 25 +-- be/src/runtime/primitive_type.cpp | 6 + be/src/vec/CMakeLists.txt | 1 + be/src/vec/data_types/data_type_factory.cpp | 2 + .../exprs/lambda_function/lambda_function.h | 41 +++++ .../lambda_function/lambda_function_factory.h | 65 +++++++ .../lambda_function/varray_map_function.cpp | 153 ++++++++++++++++ be/src/vec/exprs/vcolumn_ref.h | 75 ++++++++ be/src/vec/exprs/vexpr.cpp | 15 ++ be/src/vec/exprs/vexpr.h | 2 + be/src/vec/exprs/vlambda_function_call_expr.h | 86 +++++++++ be/src/vec/exprs/vlambda_function_expr.h | 43 +++++ .../array-functions/array_map.md | 168 +++++++++++++++++ docs/sidebars.json | 1 + .../array-functions/array_map.md | 169 ++++++++++++++++++ .../apache/doris/catalog/PrimitiveType.java | 1 + .../org/apache/doris/catalog/ScalarType.java | 5 + .../java/org/apache/doris/catalog/Type.java | 10 +- fe/fe-core/src/main/cup/sql_parser.cup | 17 +- .../apache/doris/analysis/ColumnRefExpr.java | 111 ++++++++++++ .../java/org/apache/doris/analysis/Expr.java | 28 +++ .../analysis/LambdaFunctionCallExpr.java | 107 +++++++++++ .../doris/analysis/LambdaFunctionExpr.java | 135 ++++++++++++++ .../apache/doris/catalog/ScalarFunction.java | 3 + fe/fe-core/src/main/jflex/sql_scanner.flex | 3 + gensrc/script/doris_builtins_functions.py | 1 + gensrc/thrift/Exprs.thrift | 12 ++ gensrc/thrift/Types.thrift | 3 +- .../test_array_map_function.out | 94 ++++++++++ .../test_array_map_function.groovy | 68 +++++++ 30 files changed, 1435 insertions(+), 15 deletions(-) create mode 100644 be/src/vec/exprs/lambda_function/lambda_function.h create mode 100644 be/src/vec/exprs/lambda_function/lambda_function_factory.h create mode 100644 be/src/vec/exprs/lambda_function/varray_map_function.cpp create mode 100644 be/src/vec/exprs/vcolumn_ref.h create mode 100644 be/src/vec/exprs/vlambda_function_call_expr.h create mode 100644 be/src/vec/exprs/vlambda_function_expr.h create mode 100644 docs/en/docs/sql-manual/sql-functions/array-functions/array_map.md create mode 100644 docs/zh-CN/docs/sql-manual/sql-functions/array-functions/array_map.md create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnRefExpr.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionCallExpr.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionExpr.java create mode 100644 regression-test/data/query_p0/sql_functions/array_functions/test_array_map_function.out create mode 100644 regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy diff --git a/be/src/runtime/define_primitive_type.h b/be/src/runtime/define_primitive_type.h index 8ce3da99f5..a8e7ba3fae 100644 --- a/be/src/runtime/define_primitive_type.h +++ b/be/src/runtime/define_primitive_type.h @@ -43,18 +43,19 @@ enum PrimitiveType { TYPE_HLL, /* 19 */ TYPE_DECIMALV2, /* 20 */ - TYPE_TIME, /* 21 */ - TYPE_OBJECT, /* 22 */ - TYPE_STRING, /* 23 */ - TYPE_QUANTILE_STATE, /* 24 */ - TYPE_DATEV2, /* 25 */ - TYPE_DATETIMEV2, /* 26 */ - TYPE_TIMEV2, /* 27 */ - TYPE_DECIMAL32, /* 28 */ - TYPE_DECIMAL64, /* 29 */ - TYPE_DECIMAL128I, /* 30 */ - TYPE_JSONB, /* 31 */ - TYPE_VARIANT /* 32 */ + TYPE_TIME, /* 21 */ + TYPE_OBJECT, /* 22 */ + TYPE_STRING, /* 23 */ + TYPE_QUANTILE_STATE, /* 24 */ + TYPE_DATEV2, /* 25 */ + TYPE_DATETIMEV2, /* 26 */ + TYPE_TIMEV2, /* 27 */ + TYPE_DECIMAL32, /* 28 */ + TYPE_DECIMAL64, /* 29 */ + TYPE_DECIMAL128I, /* 30 */ + TYPE_JSONB, /* 31 */ + TYPE_VARIANT, /* 32 */ + TYPE_LAMBDA_FUNCTION, /* 33 */ }; } diff --git a/be/src/runtime/primitive_type.cpp b/be/src/runtime/primitive_type.cpp index 7085dce8e7..4c55c29dc1 100644 --- a/be/src/runtime/primitive_type.cpp +++ b/be/src/runtime/primitive_type.cpp @@ -158,6 +158,8 @@ PrimitiveType thrift_to_type(TPrimitiveType::type ttype) { case TPrimitiveType::STRUCT: return TYPE_STRUCT; + case TPrimitiveType::LAMBDA_FUNCTION: + return TYPE_LAMBDA_FUNCTION; default: return INVALID_TYPE; @@ -258,6 +260,8 @@ TPrimitiveType::type to_thrift(PrimitiveType ptype) { case TYPE_STRUCT: return TPrimitiveType::STRUCT; + case TYPE_LAMBDA_FUNCTION: + return TPrimitiveType::LAMBDA_FUNCTION; default: return TPrimitiveType::INVALID_TYPE; @@ -358,6 +362,8 @@ std::string type_to_string(PrimitiveType t) { case TYPE_STRUCT: return "STRUCT"; + case TYPE_LAMBDA_FUNCTION: + return "LAMBDA_FUNCTION TYPE"; default: return ""; diff --git a/be/src/vec/CMakeLists.txt b/be/src/vec/CMakeLists.txt index 289bf3ca10..aa9f875385 100644 --- a/be/src/vec/CMakeLists.txt +++ b/be/src/vec/CMakeLists.txt @@ -160,6 +160,7 @@ set(VEC_FILES exprs/table_function/vexplode_split.cpp exprs/table_function/vexplode_numbers.cpp exprs/table_function/vexplode_bitmap.cpp + exprs/lambda_function/varray_map_function.cpp functions/array/function_array_index.cpp functions/array/function_array_element.cpp functions/array/function_array_register.cpp diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp index ec1eea5637..fa49886eba 100644 --- a/be/src/vec/data_types/data_type_factory.cpp +++ b/be/src/vec/data_types/data_type_factory.cpp @@ -23,6 +23,7 @@ #include #include "data_type_time.h" +#include "runtime/define_primitive_type.h" #include "vec/data_types/data_type_hll.h" #include "vec/data_types/data_type_object.h" @@ -141,6 +142,7 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo case TYPE_CHAR: case TYPE_VARCHAR: case TYPE_BINARY: + case TYPE_LAMBDA_FUNCTION: nested = std::make_shared(); break; case TYPE_JSONB: diff --git a/be/src/vec/exprs/lambda_function/lambda_function.h b/be/src/vec/exprs/lambda_function/lambda_function.h new file mode 100644 index 0000000000..a1eb173725 --- /dev/null +++ b/be/src/vec/exprs/lambda_function/lambda_function.h @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "vec/core/block.h" +#include "vec/exprs/vexpr_context.h" + +namespace doris::vectorized { +class VExpr; +class LambdaFunction { +public: + virtual ~LambdaFunction() = default; + + virtual std::string get_name() const = 0; + + virtual doris::Status execute(VExprContext* context, doris::vectorized::Block* block, + int* result_column_id, DataTypePtr result_type, + const std::vector& children) = 0; +}; + +using LambdaFunctionPtr = std::shared_ptr; + +} // namespace doris::vectorized diff --git a/be/src/vec/exprs/lambda_function/lambda_function_factory.h b/be/src/vec/exprs/lambda_function/lambda_function_factory.h new file mode 100644 index 0000000000..f46c2d1d7f --- /dev/null +++ b/be/src/vec/exprs/lambda_function/lambda_function_factory.h @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "vec/exprs/lambda_function/lambda_function.h" + +namespace doris::vectorized { + +class LambdaFunctionFactory; + +void register_function_array_map(LambdaFunctionFactory& factory); + +class LambdaFunctionFactory { + using Creator = std::function; + using FunctionCreators = phmap::flat_hash_map; + +public: + void register_function(const std::string& name, const Creator& ptr) { + function_creators[name] = ptr; + } + + template + void register_function() { + register_function(Function::name, &Function::create); + } + + LambdaFunctionPtr get_function(const std::string& name) { + auto iter = function_creators.find(name); + if (iter != function_creators.end()) { + return iter->second(); + } + LOG(WARNING) << fmt::format("Function signature {} is not found", name); + return nullptr; + } + +private: + FunctionCreators function_creators; + +public: + static LambdaFunctionFactory& instance() { + static std::once_flag oc; + static LambdaFunctionFactory instance; + std::call_once(oc, []() { register_function_array_map(instance); }); + return instance; + } +}; +} // namespace doris::vectorized diff --git a/be/src/vec/exprs/lambda_function/varray_map_function.cpp b/be/src/vec/exprs/lambda_function/varray_map_function.cpp new file mode 100644 index 0000000000..c3a26f18b1 --- /dev/null +++ b/be/src/vec/exprs/lambda_function/varray_map_function.cpp @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "common/status.h" +#include "vec/columns/column_array.h" +#include "vec/core/block.h" +#include "vec/data_types/data_type_array.h" +#include "vec/exprs/lambda_function/lambda_function.h" +#include "vec/exprs/lambda_function/lambda_function_factory.h" +#include "vec/exprs/vexpr.h" +#include "vec/exprs/vexpr_context.h" +#include "vec/utils/util.hpp" + +namespace doris::vectorized { + +class ArrayMapFunction : public LambdaFunction { +public: + ~ArrayMapFunction() override = default; + + static constexpr auto name = "array_map"; + + static LambdaFunctionPtr create() { return std::make_shared(); } + + std::string get_name() const override { return name; } + + doris::Status execute(VExprContext* context, doris::vectorized::Block* block, + int* result_column_id, DataTypePtr result_type, + const std::vector& children) override { + ///* array_map(lambda,arg1,arg2,.....) */// + + //1. child[1:end]->execute(src_block) + doris::vectorized::ColumnNumbers arguments(children.size() - 1); + for (int i = 1; i < children.size(); ++i) { + int column_id = -1; + RETURN_IF_ERROR(children[i]->execute(context, block, &column_id)); + arguments[i - 1] = column_id; + } + + // used for save column array outside null map + auto outside_null_map = + ColumnUInt8::create(block->get_by_position(arguments[0]) + .column->convert_to_full_column_if_const() + ->size(), + 0); + // offset column + MutableColumnPtr array_column_offset; + int nested_array_column_rows = 0; + + //2. get the result column from executed expr, and the needed is nested column of array + Block lambda_block; + for (int i = 0; i < arguments.size(); ++i) { + const auto& array_column_type_name = block->get_by_position(arguments[i]); + auto column_array = array_column_type_name.column; + column_array = column_array->convert_to_full_column_if_const(); + auto type_array = array_column_type_name.type; + + if (type_array->is_nullable()) { + // get the nullmap of nullable column + const auto& column_array_nullmap = + assert_cast(*array_column_type_name.column) + .get_null_map_column(); + + // get the array column from nullable column + column_array = + assert_cast(array_column_type_name.column.get()) + ->get_nested_column_ptr(); + + // get the nested type from nullable type + type_array = assert_cast(array_column_type_name.type.get()) + ->get_nested_type(); + + // need to union nullmap from all columns + VectorizedUtils::update_null_map(outside_null_map->get_data(), + column_array_nullmap.get_data()); + } + + // here is the array column + const ColumnArray& col_array = assert_cast(*column_array); + const auto& col_type = assert_cast(*type_array); + if (i == 0) { + nested_array_column_rows = col_array.get_data_ptr()->size(); + auto& off_data = assert_cast( + col_array.get_offsets_column()); + array_column_offset = off_data.clone_resized(col_array.get_offsets_column().size()); + } else { + // select array_map((x,y)->x+y,c_array1,[0,1,2,3]) from array_test2; + // c_array1: [0,1,2,3,4,5,6,7,8,9] + if (nested_array_column_rows != col_array.get_data_ptr()->size()) { + return Status::InternalError( + "in array map function, the input column nested column data rows are " + "not equal, the first size is {}, but with {}th size is {}.", + nested_array_column_rows, i + 1, col_array.get_data_ptr()->size()); + } + } + + // insert the data column to the new block + ColumnWithTypeAndName data_column {col_array.get_data_ptr(), col_type.get_nested_type(), + "R" + array_column_type_name.name}; + lambda_block.insert(std::move(data_column)); + } + + //3. child[0]->execute(new_block) + RETURN_IF_ERROR(children[0]->execute(context, &lambda_block, result_column_id)); + + auto res_col = lambda_block.get_by_position(*result_column_id) + .column->convert_to_full_column_if_const(); + auto res_type = lambda_block.get_by_position(*result_column_id).type; + auto res_name = lambda_block.get_by_position(*result_column_id).name; + + //4. get the result column after execution, reassemble it into a new array column, and return. + ColumnWithTypeAndName result_arr; + if (res_type->is_nullable()) { + result_arr = {ColumnNullable::create( + ColumnArray::create(res_col, std::move(array_column_offset)), + std::move(outside_null_map)), + result_type, res_name}; + + } else { + // need to create the nested column null map for column array + auto nested_null_map = ColumnUInt8::create(res_col->size(), 0); + result_arr = {ColumnNullable::create( + ColumnArray::create(ColumnNullable::create( + res_col, std::move(nested_null_map)), + std::move(array_column_offset)), + std::move(outside_null_map)), + result_type, res_name}; + } + block->insert(std::move(result_arr)); + *result_column_id = block->columns() - 1; + return Status::OK(); + } +}; + +void register_function_array_map(doris::vectorized::LambdaFunctionFactory& factory) { + factory.register_function(); +} +} // namespace doris::vectorized diff --git a/be/src/vec/exprs/vcolumn_ref.h b/be/src/vec/exprs/vcolumn_ref.h new file mode 100644 index 0000000000..b5985fe82c --- /dev/null +++ b/be/src/vec/exprs/vcolumn_ref.h @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once +#include "runtime/runtime_state.h" +#include "vec/exprs/vexpr.h" +#include "vec/functions/function.h" + +namespace doris { +namespace vectorized { +class VColumnRef final : public VExpr { +public: + //this is different of slotref is using slot_id find a column_id + //slotref: need to find the equal id in tuple, then return column_id, the plan of FE is very important + //columnref: is columnid = slotid, not used to find, so you should know this column placed in block + VColumnRef(const doris::TExprNode& node) + : VExpr(node), + _column_id(node.column_ref.column_id), + _column_name(node.column_ref.column_name) {} + + doris::Status prepare(doris::RuntimeState* state, const doris::RowDescriptor& desc, + VExprContext* context) override { + RETURN_IF_ERROR_OR_PREPARED(VExpr::prepare(state, desc, context)); + DCHECK_EQ(_children.size(), 0); + if (_column_id < 0) { + return Status::InternalError( + "VColumnRef have invalid slot id: {}, _column_name: {}, desc: {}", _column_id, + _column_name, desc.debug_string()); + } + return Status::OK(); + } + + doris::Status execute(VExprContext* context, doris::vectorized::Block* block, + int* result_column_id) override { + *result_column_id = _column_id; + return Status::OK(); + } + + VExpr* clone(doris::ObjectPool* pool) const override { + return pool->add(new VColumnRef(*this)); + } + + bool is_constant() const override { return false; } + + int column_id() const { return _column_id; } + + const std::string& expr_name() const override { return _column_name; } + + std::string debug_string() const override { + std::stringstream out; + out << "VColumnRef(slot_id: " << _column_id << ",column_name: " << _column_name + << VExpr::debug_string() << ")"; + return out.str(); + } + +private: + int _column_id; + std::string _column_name; +}; +} // namespace vectorized +} // namespace doris diff --git a/be/src/vec/exprs/vexpr.cpp b/be/src/vec/exprs/vexpr.cpp index d6cf7cbf72..b21fa4db67 100644 --- a/be/src/vec/exprs/vexpr.cpp +++ b/be/src/vec/exprs/vexpr.cpp @@ -27,10 +27,13 @@ #include "vec/exprs/varray_literal.h" #include "vec/exprs/vcase_expr.h" #include "vec/exprs/vcast_expr.h" +#include "vec/exprs/vcolumn_ref.h" #include "vec/exprs/vcompound_pred.h" #include "vec/exprs/vectorized_fn_call.h" #include "vec/exprs/vin_predicate.h" #include "vec/exprs/vinfo_func.h" +#include "vec/exprs/vlambda_function_call_expr.h" +#include "vec/exprs/vlambda_function_expr.h" #include "vec/exprs/vliteral.h" #include "vec/exprs/vmap_literal.h" #include "vec/exprs/vruntimefilter_wrapper.h" @@ -146,10 +149,22 @@ Status VExpr::create_expr(doris::ObjectPool* pool, const doris::TExprNode& texpr *expr = pool->add(new VSlotRef(texpr_node)); break; } + case doris::TExprNodeType::COLUMN_REF: { + *expr = pool->add(new VColumnRef(texpr_node)); + break; + } case doris::TExprNodeType::COMPOUND_PRED: { *expr = pool->add(new VcompoundPred(texpr_node)); break; } + case doris::TExprNodeType::LAMBDA_FUNCTION_EXPR: { + *expr = pool->add(new VLambdaFunctionExpr(texpr_node)); + break; + } + case doris::TExprNodeType::LAMBDA_FUNCTION_CALL_EXPR: { + *expr = pool->add(new VLambdaFunctionCallExpr(texpr_node)); + break; + } case doris::TExprNodeType::ARITHMETIC_EXPR: case doris::TExprNodeType::BINARY_PRED: case doris::TExprNodeType::FUNCTION_CALL: diff --git a/be/src/vec/exprs/vexpr.h b/be/src/vec/exprs/vexpr.h index 41b4006539..2a3b24892a 100644 --- a/be/src/vec/exprs/vexpr.h +++ b/be/src/vec/exprs/vexpr.h @@ -103,6 +103,8 @@ public: TExprOpcode::type op() const { return _opcode; } void add_child(VExpr* expr) { _children.push_back(expr); } + VExpr* get_child(int i) const { return _children[i]; } + int get_num_children() const { return _children.size(); } static Status create_expr_tree(ObjectPool* pool, const TExpr& texpr, VExprContext** ctx); diff --git a/be/src/vec/exprs/vlambda_function_call_expr.h b/be/src/vec/exprs/vlambda_function_call_expr.h new file mode 100644 index 0000000000..f2957ee29e --- /dev/null +++ b/be/src/vec/exprs/vlambda_function_call_expr.h @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "common/status.h" +#include "fmt/format.h" +#include "fmt/ranges.h" +#include "vec/exprs/lambda_function/lambda_function.h" +#include "vec/exprs/lambda_function/lambda_function_factory.h" +#include "vec/exprs/vexpr.h" +#include "vec/exprs/vlambda_function_expr.h" + +namespace doris::vectorized { + +class VLambdaFunctionCallExpr : public VExpr { +public: + VLambdaFunctionCallExpr(const TExprNode& node) : VExpr(node) {} + ~VLambdaFunctionCallExpr() override = default; + + VExpr* clone(ObjectPool* pool) const override { + return pool->add(new VLambdaFunctionCallExpr(*this)); + } + + doris::Status prepare(doris::RuntimeState* state, const doris::RowDescriptor& desc, + VExprContext* context) override { + RETURN_IF_ERROR_OR_PREPARED(VExpr::prepare(state, desc, context)); + + std::vector child_expr_name; + for (auto child : _children) { + child_expr_name.emplace_back(child->expr_name()); + } + _expr_name = fmt::format("{}({})", _fn.name.function_name, child_expr_name); + + _lambda_function = LambdaFunctionFactory::instance().get_function(_fn.name.function_name); + if (_lambda_function == nullptr) { + return Status::InternalError("Lambda Function {} is not implemented.", + _fn.name.function_name); + } + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + + Status execute(VExprContext* context, doris::vectorized::Block* block, + int* result_column_id) override { + return _lambda_function->execute(context, block, result_column_id, _data_type, _children); + } + + std::string debug_string() const override { + std::stringstream out; + out << "VLambdaFunctionCallExpr["; + out << _expr_name; + out << "]{"; + bool first = true; + for (VExpr* input_expr : children()) { + if (first) { + first = false; + } else { + out << ","; + } + out << "\n" << input_expr->debug_string(); + } + out << "}"; + return out.str(); + } + +private: + std::string _expr_name; + LambdaFunctionPtr _lambda_function; +}; +} // namespace doris::vectorized diff --git a/be/src/vec/exprs/vlambda_function_expr.h b/be/src/vec/exprs/vlambda_function_expr.h new file mode 100644 index 0000000000..af561a2213 --- /dev/null +++ b/be/src/vec/exprs/vlambda_function_expr.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once +#include "common/global_types.h" +#include "vec/exprs/vexpr.h" +#include "vec/functions/function.h" + +namespace doris::vectorized { +class VLambdaFunctionExpr final : public VExpr { +public: + VLambdaFunctionExpr(const TExprNode& node) : VExpr(node) {} + ~VLambdaFunctionExpr() override = default; + + doris::Status execute(VExprContext* context, doris::vectorized::Block* block, + int* result_column_id) override { + return get_child(0)->execute(context, block, result_column_id); + } + + VExpr* clone(doris::ObjectPool* pool) const override { + return pool->add(new VLambdaFunctionExpr(*this)); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const std::string _expr_name = "vlambda_function_expr"; +}; +} // namespace doris::vectorized diff --git a/docs/en/docs/sql-manual/sql-functions/array-functions/array_map.md b/docs/en/docs/sql-manual/sql-functions/array-functions/array_map.md new file mode 100644 index 0000000000..ce3ff2cb17 --- /dev/null +++ b/docs/en/docs/sql-manual/sql-functions/array-functions/array_map.md @@ -0,0 +1,168 @@ +--- +{ + "title": "array_map", + "language": "en" +} +--- + + + +## array_map + + + +array_map(lambda,array,....) + + + +### description + +Use a lambda expression as the input parameter to calculate the corresponding expression for the internal data of other input ARRAY parameters. +The number of parameters entered in the lambda expression is 1 or more, which must be consistent with the number of input array columns. +The scalar functions can be executed in lambda, and aggregate functions are not supported. + +``` +array_map(x->x, array1); +array_map(x->(x+2), array1); +array_map(x->(abs(x)-2), array1); + +array_map((x,y)->(x = y), array1, array2); +array_map((x,y)->(power(x,2)+y), array1, array2); +array_map((x,y,z)->(abs(x)+y*z), array1, array2, array3); +``` + +### example + +```shell + +mysql [test]>select *, array_map(x->x,[1,2,3]) from array_test2 order by id; ++------+-----------------+-------------------------+----------------------------------------+ +| id | c_array1 | c_array2 | array_map([x] -> x(0), ARRAY(1, 2, 3)) | ++------+-----------------+-------------------------+----------------------------------------+ +| 1 | [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [1, 2, 3] | +| 2 | [6, 7, 8] | [10, 12, 13] | [1, 2, 3] | +| 3 | [1] | [-100] | [1, 2, 3] | +| 4 | NULL | NULL | [1, 2, 3] | ++------+-----------------+-------------------------+----------------------------------------+ +4 rows in set (0.02 sec) + +mysql [test]>select *, array_map(x->x+2,[1,2,3]) from array_test2 order by id; ++------+-----------------+-------------------------+--------------------------------------------+ +| id | c_array1 | c_array2 | array_map([x] -> x(0) + 2, ARRAY(1, 2, 3)) | ++------+-----------------+-------------------------+--------------------------------------------+ +| 1 | [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [3, 4, 5] | +| 2 | [6, 7, 8] | [10, 12, 13] | [3, 4, 5] | +| 3 | [1] | [-100] | [3, 4, 5] | +| 4 | NULL | NULL | [3, 4, 5] | ++------+-----------------+-------------------------+--------------------------------------------+ +4 rows in set (0.02 sec) + +mysql [test]>select c_array1, c_array2, array_map(x->x,[1,2,3]) from array_test2 order by id; ++-----------------+-------------------------+----------------------------------------+ +| c_array1 | c_array2 | array_map([x] -> x(0), ARRAY(1, 2, 3)) | ++-----------------+-------------------------+----------------------------------------+ +| [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [1, 2, 3] | +| [6, 7, 8] | [10, 12, 13] | [1, 2, 3] | +| [1] | [-100] | [1, 2, 3] | +| NULL | NULL | [1, 2, 3] | ++-----------------+-------------------------+----------------------------------------+ +4 rows in set (0.01 sec) + +mysql [test]>select c_array1, c_array2, array_map(x->power(x,2),[1,2,3]) from array_test2 order by id; ++-----------------+-------------------------+----------------------------------------------------+ +| c_array1 | c_array2 | array_map([x] -> power(x(0), 2.0), ARRAY(1, 2, 3)) | ++-----------------+-------------------------+----------------------------------------------------+ +| [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [1, 4, 9] | +| [6, 7, 8] | [10, 12, 13] | [1, 4, 9] | +| [1] | [-100] | [1, 4, 9] | +| NULL | NULL | [1, 4, 9] | ++-----------------+-------------------------+----------------------------------------------------+ + +mysql [test]>select c_array1, c_array2, array_map((x,y)->x+y,c_array1,c_array2) from array_test2 order by id; ++-----------------+-------------------------+----------------------------------------------------------+ +| c_array1 | c_array2 | array_map([x, y] -> x(0) + y(1), `c_array1`, `c_array2`) | ++-----------------+-------------------------+----------------------------------------------------------+ +| [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [11, 22, -37, 84, -95] | +| [6, 7, 8] | [10, 12, 13] | [16, 19, 21] | +| [1] | [-100] | [-99] | +| NULL | NULL | NULL | ++-----------------+-------------------------+----------------------------------------------------------+ +4 rows in set (0.02 sec) + +mysql [test]>select c_array1, c_array2, array_map((x,y)->power(x,2)+y,c_array1, c_array2) from array_test2 order by id; ++-----------------+-------------------------+----------------------------------------------------------------------+ +| c_array1 | c_array2 | array_map([x, y] -> power(x(0), 2.0) + y(1), `c_array1`, `c_array2`) | ++-----------------+-------------------------+----------------------------------------------------------------------+ +| [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [11, 24, -31, 96, -75] | +| [6, 7, 8] | [10, 12, 13] | [46, 61, 77] | +| [1] | [-100] | [-99] | +| NULL | NULL | NULL | ++-----------------+-------------------------+----------------------------------------------------------------------+ +4 rows in set (0.03 sec) + +mysql [test]>select *,array_map(x->x=3,c_array1) from array_test2 order by id; ++------+-----------------+-------------------------+----------------------------------------+ +| id | c_array1 | c_array2 | array_map([x] -> x(0) = 3, `c_array1`) | ++------+-----------------+-------------------------+----------------------------------------+ +| 1 | [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [0, 0, 1, 0, 0] | +| 2 | [6, 7, 8] | [10, 12, 13] | [0, 0, 0] | +| 3 | [1] | [-100] | [0] | +| 4 | NULL | NULL | NULL | ++------+-----------------+-------------------------+----------------------------------------+ +4 rows in set (0.02 sec) + +mysql [test]>select *,array_map(x->x>3,c_array1) from array_test2 order by id; ++------+-----------------+-------------------------+----------------------------------------+ +| id | c_array1 | c_array2 | array_map([x] -> x(0) > 3, `c_array1`) | ++------+-----------------+-------------------------+----------------------------------------+ +| 1 | [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [0, 0, 0, 1, 1] | +| 2 | [6, 7, 8] | [10, 12, 13] | [1, 1, 1] | +| 3 | [1] | [-100] | [0] | +| 4 | NULL | NULL | NULL | ++------+-----------------+-------------------------+----------------------------------------+ +4 rows in set (0.02 sec) + +mysql [test]>select *,array_map((x,y)->x>y,c_array1,c_array2) from array_test2 order by id; ++------+-----------------+-------------------------+----------------------------------------------------------+ +| id | c_array1 | c_array2 | array_map([x, y] -> x(0) > y(1), `c_array1`, `c_array2`) | ++------+-----------------+-------------------------+----------------------------------------------------------+ +| 1 | [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [0, 0, 1, 0, 1] | +| 2 | [6, 7, 8] | [10, 12, 13] | [0, 0, 0] | +| 3 | [1] | [-100] | [1] | +| 4 | NULL | NULL | NULL | ++------+-----------------+-------------------------+----------------------------------------------------------+ +4 rows in set (0.02 sec) + +mysql [test]>select array_map(x->cast(x as string), c_array1) from test_array_map_function; ++-----------------+-------------------------------------------------------+ +| c_array1 | array_map([x] -> CAST(x(0) AS CHARACTER), `c_array1`) | ++-----------------+-------------------------------------------------------+ +| [1, 2, 3, 4, 5] | ['1', '2', '3', '4', '5'] | +| [6, 7, 8] | ['6', '7', '8'] | +| [] | [] | +| NULL | NULL | ++-----------------+-------------------------------------------------------+ +4 rows in set (0.01 sec) +``` + +### keywords + +ARRAY,MAP,ARRAY_MAP + diff --git a/docs/sidebars.json b/docs/sidebars.json index 7ad255cbf5..19635becc4 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -273,6 +273,7 @@ "sql-manual/sql-functions/array-functions/array", "sql-manual/sql-functions/array-functions/array_max", "sql-manual/sql-functions/array-functions/array_min", + "sql-manual/sql-functions/array-functions/array_map", "sql-manual/sql-functions/array-functions/array_avg", "sql-manual/sql-functions/array-functions/array_sum", "sql-manual/sql-functions/array-functions/array_size", diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/array-functions/array_map.md b/docs/zh-CN/docs/sql-manual/sql-functions/array-functions/array_map.md new file mode 100644 index 0000000000..6080c7c51f --- /dev/null +++ b/docs/zh-CN/docs/sql-manual/sql-functions/array-functions/array_map.md @@ -0,0 +1,169 @@ +--- +{ + "title": "array_map", + "language": "zh-CN" +} +--- + + + +## array_map + + + +array_map(lambda,array1,array2....) + + + +### description + +使用一个lambda表达式作为输入参数，对其他的输入ARRAY参数的内部数据做对应表达式计算。 +在lambda表达式中输入的参数为1个或多个，必须和后面的输入array列数量一致。 +在lambda中可以执行合法的标量函数，不支持聚合函数等。 + +``` +array_map(x->x, array1); +array_map(x->(x+2), array1); +array_map(x->(abs(x)-2), array1); + +array_map((x,y)->(x = y), array1, array2); +array_map((x,y)->(power(x,2)+y), array1, array2); +array_map((x,y,z)->(abs(x)+y*z), array1, array2, array3); +``` + +### example + +```shell + +mysql [test]>select *, array_map(x->x,[1,2,3]) from array_test2 order by id; ++------+-----------------+-------------------------+----------------------------------------+ +| id | c_array1 | c_array2 | array_map([x] -> x(0), ARRAY(1, 2, 3)) | ++------+-----------------+-------------------------+----------------------------------------+ +| 1 | [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [1, 2, 3] | +| 2 | [6, 7, 8] | [10, 12, 13] | [1, 2, 3] | +| 3 | [1] | [-100] | [1, 2, 3] | +| 4 | NULL | NULL | [1, 2, 3] | ++------+-----------------+-------------------------+----------------------------------------+ +4 rows in set (0.02 sec) + +mysql [test]>select *, array_map(x->x+2,[1,2,3]) from array_test2 order by id; ++------+-----------------+-------------------------+--------------------------------------------+ +| id | c_array1 | c_array2 | array_map([x] -> x(0) + 2, ARRAY(1, 2, 3)) | ++------+-----------------+-------------------------+--------------------------------------------+ +| 1 | [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [3, 4, 5] | +| 2 | [6, 7, 8] | [10, 12, 13] | [3, 4, 5] | +| 3 | [1] | [-100] | [3, 4, 5] | +| 4 | NULL | NULL | [3, 4, 5] | ++------+-----------------+-------------------------+--------------------------------------------+ +4 rows in set (0.02 sec) + +mysql [test]>select c_array1, c_array2, array_map(x->x,[1,2,3]) from array_test2 order by id; ++-----------------+-------------------------+----------------------------------------+ +| c_array1 | c_array2 | array_map([x] -> x(0), ARRAY(1, 2, 3)) | ++-----------------+-------------------------+----------------------------------------+ +| [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [1, 2, 3] | +| [6, 7, 8] | [10, 12, 13] | [1, 2, 3] | +| [1] | [-100] | [1, 2, 3] | +| NULL | NULL | [1, 2, 3] | ++-----------------+-------------------------+----------------------------------------+ +4 rows in set (0.01 sec) + +mysql [test]>select c_array1, c_array2, array_map(x->power(x,2),[1,2,3]) from array_test2 order by id; ++-----------------+-------------------------+----------------------------------------------------+ +| c_array1 | c_array2 | array_map([x] -> power(x(0), 2.0), ARRAY(1, 2, 3)) | ++-----------------+-------------------------+----------------------------------------------------+ +| [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [1, 4, 9] | +| [6, 7, 8] | [10, 12, 13] | [1, 4, 9] | +| [1] | [-100] | [1, 4, 9] | +| NULL | NULL | [1, 4, 9] | ++-----------------+-------------------------+----------------------------------------------------+ + +mysql [test]>select c_array1, c_array2, array_map((x,y)->x+y,c_array1,c_array2) from array_test2 order by id; ++-----------------+-------------------------+----------------------------------------------------------+ +| c_array1 | c_array2 | array_map([x, y] -> x(0) + y(1), `c_array1`, `c_array2`) | ++-----------------+-------------------------+----------------------------------------------------------+ +| [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [11, 22, -37, 84, -95] | +| [6, 7, 8] | [10, 12, 13] | [16, 19, 21] | +| [1] | [-100] | [-99] | +| NULL | NULL | NULL | ++-----------------+-------------------------+----------------------------------------------------------+ +4 rows in set (0.02 sec) + +mysql [test]>select c_array1, c_array2, array_map((x,y)->power(x,2)+y,c_array1, c_array2) from array_test2 order by id; ++-----------------+-------------------------+----------------------------------------------------------------------+ +| c_array1 | c_array2 | array_map([x, y] -> power(x(0), 2.0) + y(1), `c_array1`, `c_array2`) | ++-----------------+-------------------------+----------------------------------------------------------------------+ +| [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [11, 24, -31, 96, -75] | +| [6, 7, 8] | [10, 12, 13] | [46, 61, 77] | +| [1] | [-100] | [-99] | +| NULL | NULL | NULL | ++-----------------+-------------------------+----------------------------------------------------------------------+ +4 rows in set (0.03 sec) + +mysql [test]>select *,array_map(x->x=3,c_array1) from array_test2 order by id; ++------+-----------------+-------------------------+----------------------------------------+ +| id | c_array1 | c_array2 | array_map([x] -> x(0) = 3, `c_array1`) | ++------+-----------------+-------------------------+----------------------------------------+ +| 1 | [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [0, 0, 1, 0, 0] | +| 2 | [6, 7, 8] | [10, 12, 13] | [0, 0, 0] | +| 3 | [1] | [-100] | [0] | +| 4 | NULL | NULL | NULL | ++------+-----------------+-------------------------+----------------------------------------+ +4 rows in set (0.02 sec) + +mysql [test]>select *,array_map(x->x>3,c_array1) from array_test2 order by id; ++------+-----------------+-------------------------+----------------------------------------+ +| id | c_array1 | c_array2 | array_map([x] -> x(0) > 3, `c_array1`) | ++------+-----------------+-------------------------+----------------------------------------+ +| 1 | [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [0, 0, 0, 1, 1] | +| 2 | [6, 7, 8] | [10, 12, 13] | [1, 1, 1] | +| 3 | [1] | [-100] | [0] | +| 4 | NULL | NULL | NULL | ++------+-----------------+-------------------------+----------------------------------------+ +4 rows in set (0.02 sec) + +mysql [test]>select *,array_map((x,y)->x>y,c_array1,c_array2) from array_test2 order by id; ++------+-----------------+-------------------------+----------------------------------------------------------+ +| id | c_array1 | c_array2 | array_map([x, y] -> x(0) > y(1), `c_array1`, `c_array2`) | ++------+-----------------+-------------------------+----------------------------------------------------------+ +| 1 | [1, 2, 3, 4, 5] | [10, 20, -40, 80, -100] | [0, 0, 1, 0, 1] | +| 2 | [6, 7, 8] | [10, 12, 13] | [0, 0, 0] | +| 3 | [1] | [-100] | [1] | +| 4 | NULL | NULL | NULL | ++------+-----------------+-------------------------+----------------------------------------------------------+ +4 rows in set (0.02 sec) + +mysql [test]>select array_map(x->cast(x as string), c_array1) from test_array_map_function; ++-----------------+-------------------------------------------------------+ +| c_array1 | array_map([x] -> CAST(x(0) AS CHARACTER), `c_array1`) | ++-----------------+-------------------------------------------------------+ +| [1, 2, 3, 4, 5] | ['1', '2', '3', '4', '5'] | +| [6, 7, 8] | ['6', '7', '8'] | +| [] | [] | +| NULL | NULL | ++-----------------+-------------------------------------------------------+ +4 rows in set (0.01 sec) + +``` + +### keywords + +ARRAY,MAP,ARRAY_MAP + diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java index fdcbfaedef..ed3ce8f85e 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java @@ -62,6 +62,7 @@ public enum PrimitiveType { DATEV2("DATEV2", 4, TPrimitiveType.DATEV2), DATETIMEV2("DATETIMEV2", 8, TPrimitiveType.DATETIMEV2), TIMEV2("TIMEV2", 8, TPrimitiveType.TIMEV2), + LAMBDA_FUNCTION("LAMBDA_FUNCTION", 16, TPrimitiveType.LAMBDA_FUNCTION), // sizeof(CollectionValue) ARRAY("ARRAY", 32, TPrimitiveType.ARRAY), diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java index f44aff6909..ec0a18635a 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java @@ -180,6 +180,8 @@ public class ScalarType extends Type { return BITMAP; case QUANTILE_STATE: return QUANTILE_STATE; + case LAMBDA_FUNCTION: + return LAMBDA_FUNCTION; case DATE: return DATE; case DATETIME: @@ -246,6 +248,8 @@ public class ScalarType extends Type { return BITMAP; case "QUANTILE_STATE": return QUANTILE_STATE; + case "LAMBDA_FUNCTION": + return LAMBDA_FUNCTION; case "DATE": return DATE; case "DATETIME": @@ -619,6 +623,7 @@ public class ScalarType extends Type { case BITMAP: case VARIANT: case QUANTILE_STATE: + case LAMBDA_FUNCTION: stringBuilder.append(type.toString().toLowerCase()); break; case STRING: diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index c70a9d91d3..7a59f30663 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -102,6 +102,7 @@ public abstract class Type { public static final ScalarType CHAR = ScalarType.createCharType(-1); public static final ScalarType BITMAP = new ScalarType(PrimitiveType.BITMAP); public static final ScalarType QUANTILE_STATE = new ScalarType(PrimitiveType.QUANTILE_STATE); + public static final ScalarType LAMBDA_FUNCTION = new ScalarType(PrimitiveType.LAMBDA_FUNCTION); // Only used for alias function, to represent any type in function args public static final ScalarType ALL = new ScalarType(PrimitiveType.ALL); public static final MapType MAP = new MapType(); @@ -405,6 +406,10 @@ public abstract class Type { return isScalarType(PrimitiveType.QUANTILE_STATE); } + public boolean isLambdaFunctionType() { + return isScalarType(PrimitiveType.LAMBDA_FUNCTION); + } + public boolean isObjectStored() { return isHllType() || isBitmapType() || isQuantileStateType(); } @@ -810,6 +815,8 @@ public abstract class Type { return Type.QUANTILE_STATE; case VARIANT: return new VariantType(); + case LAMBDA_FUNCTION: + return Type.LAMBDA_FUNCTION; default: return null; } @@ -1585,7 +1592,8 @@ public abstract class Type { || t1 == PrimitiveType.STRUCT || t2 == PrimitiveType.STRUCT || t1 == PrimitiveType.TEMPLATE || t2 == PrimitiveType.TEMPLATE || t1 == PrimitiveType.UNSUPPORTED || t2 == PrimitiveType.UNSUPPORTED - || t1 == PrimitiveType.VARIANT || t2 == PrimitiveType.VARIANT) { + || t1 == PrimitiveType.VARIANT || t2 == PrimitiveType.VARIANT + || t1 == PrimitiveType.LAMBDA_FUNCTION || t2 == PrimitiveType.LAMBDA_FUNCTION) { continue; } Preconditions.checkNotNull(compatibilityMatrix[i][j]); diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index fb05c5af66..b3cb0e13b9 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -624,7 +624,7 @@ terminal String KW_LINES, KW_IGNORE; -terminal COMMA, COLON, DOT, DOTDOTDOT, AT, STAR, LPAREN, RPAREN, SEMICOLON, LBRACKET, RBRACKET, LBRACE, RBRACE, DIVIDE, MOD, ADD, SUBTRACT, PLACEHOLDER; +terminal COMMA, COLON, DOT, DOTDOTDOT, AT, STAR, LPAREN, RPAREN, SEMICOLON, LBRACKET, RBRACKET, LBRACE, RBRACE, DIVIDE, MOD, ADD, SUBTRACT, PLACEHOLDER, ARROW; terminal BITAND, BITOR, BITXOR, BITNOT; terminal EQUAL, NOT, LESSTHAN, GREATERTHAN, SET_VAR; terminal COMMENTED_PLAN_HINT_START, COMMENTED_PLAN_HINT_END; @@ -6031,6 +6031,21 @@ function_call_expr ::= RESULT = new FunctionCallExpr(fn_name, params); } :} + | function_name:fn_name LPAREN ident:id ARROW expr:e COMMA function_params:params RPAREN + {: + List exprs = params.exprs(); + LambdaFunctionExpr lambda = new LambdaFunctionExpr(e, id, exprs); + exprs.add(lambda); + RESULT = new LambdaFunctionCallExpr(fn_name, exprs); + :} + | function_name:fn_name LPAREN LPAREN ident:id COMMA ident_list:idList RPAREN ARROW expr:e COMMA function_params:params RPAREN + {: + List exprs = params.exprs(); + idList.add(0, id); + LambdaFunctionExpr lambda = new LambdaFunctionExpr(e, idList, exprs); + exprs.add(lambda); + RESULT = new LambdaFunctionCallExpr(fn_name, exprs); + :} ; array_literal ::= diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnRefExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnRefExpr.java new file mode 100644 index 0000000000..88c652f6f5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnRefExpr.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.common.AnalysisException; +import org.apache.doris.thrift.TColumnRef; +import org.apache.doris.thrift.TExprNode; +import org.apache.doris.thrift.TExprNodeType; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class ColumnRefExpr extends Expr { + private static final Logger LOG = LogManager.getLogger(ColumnRefExpr.class); + private String columnName; + private int columnId; + private boolean isNullable; + + public ColumnRefExpr() { + super(); + } + + public ColumnRefExpr(int columnId, String columnName, boolean isNullable) { + super(); + this.columnId = columnId; + this.columnName = columnName; + this.isNullable = isNullable; + } + + public ColumnRefExpr(ColumnRefExpr rhs) { + super(rhs); + this.columnId = rhs.columnId; + this.columnName = rhs.columnName; + this.isNullable = rhs.isNullable; + } + + public String getName() { + return columnName; + } + + public void setName(String name) { + this.columnName = name; + } + + public int getcolumnId() { + return columnId; + } + + public void setcolumnId(int id) { + this.columnId = id; + } + + @Override + public boolean isNullable() { + return isNullable; + } + + public void setNullable(boolean nullable) { + this.isNullable = nullable; + } + + @Override + protected void analyzeImpl(Analyzer analyzer) throws AnalysisException { + if (columnId < 0) { + throw new AnalysisException("the columnId is invalid : " + columnId); + } + } + + @Override + protected String toSqlImpl() { + return columnName + "(" + columnId + ")"; + } + + @Override + protected void toThrift(TExprNode msg) { + msg.node_type = TExprNodeType.COLUMN_REF; + TColumnRef columnRef = new TColumnRef(); + columnRef.setColumnId(columnId); + columnRef.setColumnName(columnName); + msg.column_ref = columnRef; + } + + @Override + public Expr clone() { + return new ColumnRefExpr(this); + } + + @Override + protected boolean isConstantImpl() { + return false; + } + + public String debugString() { + return columnName + " (" + columnId + ")id"; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java index da71acf8c3..ef8b2e3fd2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -1698,6 +1698,34 @@ public abstract class Expr extends TreeNode implements ParseNode, Cloneabl return null; } + /* + * this function is used be lambda function to find lambda argument. + * and replace a new ColumnRefExpr + */ + public void replaceExpr(String colName, ColumnRefExpr slotRefs, List slotExpr) { + for (int i = 0; i < children.size(); ++i) { + children.get(i).replaceExpr(colName, slotRefs, slotExpr); + if (children.get(i).findSlotRefByName(colName)) { + slotExpr.add(slotRefs); + setChild(i, slotRefs); + break; + } + } + } + + private boolean findSlotRefByName(String colName) { + if (this instanceof SlotRef) { + SlotRef slot = (SlotRef) this; + if (slot.getColumnName() != null && slot.getColumnName().equals(colName)) { + return true; + } + } + return false; + } + + + + /** * Looks up in the catalog the builtin for 'name' and 'argTypes'. * Returns null if the function is not found. diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionCallExpr.java new file mode 100644 index 0000000000..18a2a1f144 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionCallExpr.java @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.ArrayType; +import org.apache.doris.catalog.Function; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.thrift.TExprNode; +import org.apache.doris.thrift.TExprNodeType; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; + +public class LambdaFunctionCallExpr extends FunctionCallExpr { + public static final ImmutableSet LAMBDA_FUNCTION_SET = new ImmutableSortedSet.Builder( + String.CASE_INSENSITIVE_ORDER).add("array_map").build(); + + private static final Logger LOG = LogManager.getLogger(LambdaFunctionCallExpr.class); + + public LambdaFunctionCallExpr(String functionName, List params) { + super(functionName, params); + } + + public LambdaFunctionCallExpr(FunctionName functionName, List params) { + super(functionName, params); + } + + public LambdaFunctionCallExpr(LambdaFunctionCallExpr other) { + super(other); + } + + @Override + public Expr clone() { + return new LambdaFunctionCallExpr(this); + } + + @Override + public void analyzeImpl(Analyzer analyzer) throws AnalysisException { + FunctionName fnName = getFnName(); + FunctionParams fnParams = getFnParams(); + if (!LAMBDA_FUNCTION_SET.contains(fnName.getFunction().toLowerCase())) { + throw new AnalysisException( + "Function {} maybe not in the LAMBDA_FUNCTION_SET, should check the implement" + fnName + .getFunction()); + } + + int childSize = this.children.size(); + Type[] argTypes = new Type[childSize]; + for (int i = 0; i < childSize; ++i) { + this.children.get(i).analyze(analyzer); + argTypes[i] = this.children.get(i).getType(); + } + + if (fnName.getFunction().equalsIgnoreCase("array_map")) { + if (fnParams.exprs() == null || fnParams.exprs().size() < 2) { + throw new AnalysisException("The " + fnName.getFunction() + " function must have at least two params"); + } + + // change the lambda expr to the first args position + if (getChild(childSize - 1) instanceof LambdaFunctionExpr) { + Type lastType = argTypes[childSize - 1]; + Expr lastChild = getChild(childSize - 1); + for (int i = childSize - 1; i > 0; --i) { + argTypes[i] = getChild(i - 1).getType(); + this.setChild(i, getChild(i - 1)); + } + argTypes[0] = lastType; + this.setChild(0, lastChild); + } + + fn = getBuiltinFunction(fnName.getFunction(), argTypes, + Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); + Expr lambda = this.children.get(0); + if (fn == null) { + LOG.warn("fn {} not exists", this.toSqlImpl()); + throw new AnalysisException(getFunctionNotFoundError(collectChildReturnTypes())); + } + fn.setReturnType(ArrayType.create(lambda.getChild(0).getType(), true)); + } + this.type = fn.getReturnType(); + } + + @Override + protected void toThrift(TExprNode msg) { + msg.node_type = TExprNodeType.LAMBDA_FUNCTION_CALL_EXPR; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionExpr.java new file mode 100644 index 0000000000..5ead8c8aef --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionExpr.java @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.ArrayType; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.thrift.TExprNode; +import org.apache.doris.thrift.TExprNodeType; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; + +public class LambdaFunctionExpr extends Expr { + private static final Logger LOG = LogManager.getLogger(LambdaFunctionExpr.class); + private ArrayList names = new ArrayList<>(); + private ArrayList slotExpr = new ArrayList<>(); + private ArrayList params = new ArrayList<>(); + private int columnId = 0; + + public LambdaFunctionExpr(Expr e, String arg, List params) { + this.names.add(arg); + this.slotExpr.add(e); + this.params.addAll(params); + columnId = 0; + this.setType(Type.LAMBDA_FUNCTION); + } + + public LambdaFunctionExpr(Expr e, ArrayList args, List params) { + this.names.addAll(args); + this.slotExpr.add(e); + this.params.addAll(params); + columnId = 0; + this.setType(Type.LAMBDA_FUNCTION); + } + + public LambdaFunctionExpr(LambdaFunctionExpr rhs) { + super(rhs); + this.names.addAll(rhs.names); + this.slotExpr.addAll(rhs.slotExpr); + this.params.addAll(rhs.params); + this.columnId = rhs.columnId; + } + + @Override + protected void analyzeImpl(Analyzer analyzer) throws AnalysisException { + if (names.size() != params.size()) { + throw new AnalysisException("Lambda argument size: is " + names.size() + " but input params size is " + + params.size()); + } + if (this.children.size() == 0) { + this.children.add(slotExpr.get(0)); + } + // the first is lambda + int size = slotExpr.size(); + for (int i = size - 1; i < names.size(); ++i) { + Expr param = params.get(i); + Type paramType = param.getType(); + if (!paramType.isArrayType()) { + throw new AnalysisException( + "The lambda function of params must be array type, now " + (i + 1) + "th is " + + paramType.toString()); + } + // this ColumnRefExpr record the unique columnId, which is used for BE + // so could insert nested column by order. + ColumnRefExpr column = new ColumnRefExpr(); + column.setName(names.get(i)); + column.setcolumnId(columnId); + column.setNullable(true); + column.setType(((ArrayType) paramType).getItemType()); + columnId = columnId + 1; + replaceExpr(names.get(i), column, slotExpr); + } + if (slotExpr.size() != params.size() + 1) { + String msg = new String(); + for (Expr s : slotExpr) { + msg = msg + s.debugString() + " ,"; + } + throw new AnalysisException( + "Lambda columnref size: is " + (slotExpr.size() - 1) + " but input params size is " + + params.size() + ". the replaceExpr of columnref is " + msg); + } + this.children.get(0).analyze(analyzer); + } + + @Override + protected String toSqlImpl() { + return String.format("%s -> %s", names.toString(), getChild(0).toSql()); + } + + @Override + protected void toThrift(TExprNode msg) { + msg.setNodeType(TExprNodeType.LAMBDA_FUNCTION_EXPR); + } + + @Override + public Expr clone() { + return new LambdaFunctionExpr(this); + } + + public ArrayList getNames() { + return names; + } + + public ArrayList getSlotExprs() { + return slotExpr; + } + + public boolean isNullable() { + for (int i = 1; i < slotExpr.size(); ++i) { + if (slotExpr.get(i).isNullable()) { + return true; + } + } + return false; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java index 3f7038a025..5e3906d93b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java @@ -256,6 +256,9 @@ public class ScalarFunction extends Function { case JSONB: beFn.append("_jsonb_val"); break; + case LAMBDA_FUNCTION: + beFn.append("_lambda_function"); + break; case DATE: case DATETIME: case DATEV2: diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex index 28da455e57..4ef69468af 100644 --- a/fe/fe-core/src/main/jflex/sql_scanner.flex +++ b/fe/fe-core/src/main/jflex/sql_scanner.flex @@ -499,6 +499,7 @@ import org.apache.doris.qe.SqlModeHelper; // add non-keyword tokens tokenIdMap.put(new Integer(SqlParserSymbols.IDENT), "IDENTIFIER"); tokenIdMap.put(new Integer(SqlParserSymbols.COMMA), "COMMA"); + tokenIdMap.put(new Integer(SqlParserSymbols.ARROW), "->"); tokenIdMap.put(new Integer(SqlParserSymbols.BITNOT), "~"); tokenIdMap.put(new Integer(SqlParserSymbols.LPAREN), "("); tokenIdMap.put(new Integer(SqlParserSymbols.RPAREN), ")"); @@ -535,6 +536,7 @@ import org.apache.doris.qe.SqlModeHelper; tokenIdMap.put(new Integer(SqlParserSymbols.BITXOR), "^"); tokenIdMap.put(new Integer(SqlParserSymbols.NUMERIC_OVERFLOW), "NUMERIC OVERFLOW"); tokenIdMap.put(new Integer(SqlParserSymbols.PLACEHOLDER), "?"); + } public static boolean isKeyword(Integer tokenId) { @@ -642,6 +644,7 @@ EndOfLineComment = "--" !({HintContent}|{ContainsLineTerminator}) {LineTerminato %% "..." { return newToken(SqlParserSymbols.DOTDOTDOT, null); } +"->" { return newToken(SqlParserSymbols.ARROW, null); } // single-character tokens "," { return newToken(SqlParserSymbols.COMMA, null); } diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index f39f9f065e..d0c82f3fc6 100644 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -584,6 +584,7 @@ visible_functions = [ [['array_popfront'], 'ARRAY_DECIMAL128', ['ARRAY_DECIMAL128'], ''], [['array_popfront'], 'ARRAY_VARCHAR', ['ARRAY_VARCHAR'], ''], [['array_popfront'], 'ARRAY_STRING', ['ARRAY_STRING'], ''], + [['array_map'], 'ARRAY', ['LAMBDA_FUNCTION', 'ARRAY', '...'], ''], [['array_pushfront'], 'ARRAY_BOOLEAN', ['ARRAY_BOOLEAN', 'BOOLEAN'], 'ALWAYS_NULLABLE'], [['array_pushfront'], 'ARRAY_TINYINT', ['ARRAY_TINYINT', 'TINYINT'], 'ALWAYS_NULLABLE'], diff --git a/gensrc/thrift/Exprs.thrift b/gensrc/thrift/Exprs.thrift index 010cb450e6..9c6720a53f 100644 --- a/gensrc/thrift/Exprs.thrift +++ b/gensrc/thrift/Exprs.thrift @@ -69,6 +69,11 @@ enum TExprNodeType { // for schema change SCHEMA_CHANGE_EXPR, + // for lambda function expr + LAMBDA_FUNCTION_EXPR, + LAMBDA_FUNCTION_CALL_EXPR, + // for column_ref expr + COLUMN_REF, } //enum TAggregationOp { @@ -155,6 +160,11 @@ struct TSlotRef { 3: optional i32 col_unique_id } +struct TColumnRef { + 1: optional Types.TSlotId column_id + 2: optional string column_name +} + struct TStringLiteral { 1: required string value; } @@ -223,6 +233,8 @@ struct TExprNode { 30: optional TJsonLiteral json_literal 31: optional TSchemaChangeExpr schema_change_expr + + 32: optional TColumnRef column_ref } // A flattened representation of a tree of Expr nodes, obtained by depth-first diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index da4d727265..ed4e587413 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -92,7 +92,8 @@ enum TPrimitiveType { DECIMAL128I, JSONB, VARIANT, - UNSUPPORTED + UNSUPPORTED, + LAMBDA_FUNCTION } enum TTypeNodeType { diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_map_function.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_map_function.out new file mode 100644 index 0000000000..b864ad04d2 --- /dev/null +++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_map_function.out @@ -0,0 +1,94 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_1 -- +1 [1, 2, 3, 4, 5] [10, 20, -40, 80, -100] +2 [6, 7, 8] [10, 12, 13] +3 [1] [-100] +4 \N \N + +-- !select_2 -- +1 [1, 2, 3, 4, 5] [10, 20, -40, 80, -100] [1, 2, 3] +2 [6, 7, 8] [10, 12, 13] [1, 2, 3] +3 [1] [-100] [1, 2, 3] +4 \N \N [1, 2, 3] + +-- !select_3 -- +1 [1, 2, 3, 4, 5] [10, 20, -40, 80, -100] [3, 4, 5] +2 [6, 7, 8] [10, 12, 13] [3, 4, 5] +3 [1] [-100] [3, 4, 5] +4 \N \N [3, 4, 5] + +-- !select_4 -- +[1, 2, 3, 4, 5] [10, 20, -40, 80, -100] [1, 2, 3] +[6, 7, 8] [10, 12, 13] [1, 2, 3] +[1] [-100] [1, 2, 3] +\N \N [1, 2, 3] + +-- !select_5 -- +[1, 2, 3, 4, 5] [10, 20, -40, 80, -100] [1, 4, 9] +[6, 7, 8] [10, 12, 13] [1, 4, 9] +[1] [-100] [1, 4, 9] +\N \N [1, 4, 9] + +-- !select_6 -- +[1, 2, 3, 4, 5] [10, 20, -40, 80, -100] [1, 2, 3, 4, 5] +[6, 7, 8] [10, 12, 13] [6, 7, 8] +[1] [-100] [1] +\N \N \N + +-- !select_7 -- +[1, 2, 3, 4, 5] [10, 20, -40, 80, -100] [3, 4, 5, 6, 7] +[6, 7, 8] [10, 12, 13] [8, 9, 10] +[1] [-100] [3] +\N \N \N + +-- !select_8 -- +[1, 2, 3, 4, 5] [10, 20, -40, 80, -100] [1, 4, 9, 16, 25] +[6, 7, 8] [10, 12, 13] [36, 49, 64] +[1] [-100] [1] +\N \N \N + +-- !select_9 -- +[1, 2, 3, 4, 5] [10, 20, -40, 80, -100] [11, 22, -37, 84, -95] +[6, 7, 8] [10, 12, 13] [16, 19, 21] +[1] [-100] [-99] +\N \N \N + +-- !select_10 -- +[1, 2, 3, 4, 5] [10, 20, -40, 80, -100] [11, 24, -31, 96, -75] +[6, 7, 8] [10, 12, 13] [46, 61, 77] +[1] [-100] [-99] +\N \N \N + +-- !select_11 -- +1 [1, 2, 3, 4, 5] [10, 20, -40, 80, -100] [0, 0, 1, 0, 0] +2 [6, 7, 8] [10, 12, 13] [0, 0, 0] +3 [1] [-100] [0] +4 \N \N \N + +-- !select_12 -- +1 [1, 2, 3, 4, 5] [10, 20, -40, 80, -100] [0, 0, 0, 1, 1] +2 [6, 7, 8] [10, 12, 13] [1, 1, 1] +3 [1] [-100] [0] +4 \N \N \N + +-- !select_13 -- +1 [1, 2, 3, 4, 5] [10, 20, -40, 80, -100] [0, 0, 1, 0, 1] +2 [6, 7, 8] [10, 12, 13] [0, 0, 0] +3 [1] [-100] [1] +4 \N \N \N + +-- !select_14 -- +[] + +-- !select_15 -- +[NULL] + +-- !select_16 -- +[1] + +-- !select_17 -- +[1, 0, 0] + +-- !select_18 -- +[NULL, 1, 2] + diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy new file mode 100644 index 0000000000..3d90d3f541 --- /dev/null +++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_array_map_function") { + + def tableName = "array_test2" + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE IF NOT EXISTS `${tableName}` ( + `id` int(11) NULL, + `c_array1` array NULL, + `c_array2` array NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2" + ) + """ + + + sql """INSERT INTO ${tableName} values + (1, [1,2,3,4,5], [10,20,-40,80,-100]), + (2, [6,7,8],[10,12,13]), (3, [1],[-100]), (4, null,null) + """ + qt_select_1 "select * from ${tableName} order by id;" + + + qt_select_2 "select *, array_map(x->x,[1,2,3]) from ${tableName} order by id;" + qt_select_3 "select *, array_map(x->x+2,[1,2,3]) from ${tableName} order by id;" + qt_select_4 "select c_array1, c_array2, array_map(x->x,[1,2,3]) from array_test2 order by id;" + qt_select_5 "select c_array1, c_array2, array_map(x->power(x,2),[1,2,3]) from array_test2 order by id;" + + qt_select_6 "select c_array1, c_array2, array_map(x->x,c_array1) from array_test2 order by id;" + qt_select_7 "select c_array1, c_array2, array_map(x->x+2,c_array1) from array_test2 order by id;" + qt_select_8 "select c_array1, c_array2, array_map(x->power(x,2),c_array1) from array_test2 order by id;" + + qt_select_9 "select c_array1, c_array2, array_map((x,y)->x+y,c_array1,c_array2) from array_test2 order by id;" + qt_select_10 "select c_array1, c_array2, array_map((x,y)->power(x,2)+y,c_array1, c_array2) from array_test2 order by id;" + + qt_select_11 "select *,array_map(x->x=3,c_array1) from array_test2 order by id;" + qt_select_12 "select *,array_map(x->x>3,c_array1) from array_test2 order by id;" + qt_select_13 "select *,array_map((x,y)->x>y,c_array1,c_array2) from array_test2 order by id;" + + qt_select_14 "select array_map(x -> x,[]);" + qt_select_15 "select array_map(x -> x,[null]);" + qt_select_16 "select array_map(x -> x,[1]);" + qt_select_17 "select array_map(x -> x is null, [null, 1, 2]);" + qt_select_18 "select array_map(x -> abs(x), [null, 1, 2]);" + + + sql "DROP TABLE IF EXISTS ${tableName}" +} \ No newline at end of file