From 4ea0d6c5faa778d0443cbf57b3b51aa3eb4cc7e8 Mon Sep 17 00:00:00 2001 From: bobhan1 <68336428+bobhan1@users.noreply.github.com> Date: Wed, 8 Mar 2023 13:57:38 +0800 Subject: [PATCH] [feature](array_function) add support for array_popfront (#17416) --- be/src/vec/CMakeLists.txt | 1 + .../array/function_array_popfront.cpp | 84 +++++++++++++++++++ .../array/function_array_register.cpp | 2 + .../array-functions/array_popfront.md | 64 ++++++++++++++ docs/sidebars.json | 1 + .../array-functions/array_popfront.md | 61 ++++++++++++++ .../doris/analysis/FunctionCallExpr.java | 1 + gensrc/script/doris_builtins_functions.py | 19 +++++ .../array_functions/test_array_functions.out | 66 +++++++++++++++ .../test_array_functions_by_literal.out | 33 ++++++++ .../test_array_functions.groovy | 6 ++ .../test_array_functions_by_literal.groovy | 13 +++ 12 files changed, 351 insertions(+) create mode 100644 be/src/vec/functions/array/function_array_popfront.cpp create mode 100644 docs/en/docs/sql-manual/sql-functions/array-functions/array_popfront.md create mode 100644 docs/zh-CN/docs/sql-manual/sql-functions/array-functions/array_popfront.md diff --git a/be/src/vec/CMakeLists.txt b/be/src/vec/CMakeLists.txt index 35ffd17ab8..d9e897e67b 100644 --- a/be/src/vec/CMakeLists.txt +++ b/be/src/vec/CMakeLists.txt @@ -177,6 +177,7 @@ set(VEC_FILES functions/array/function_array_range.cpp functions/array/function_array_compact.cpp functions/array/function_array_popback.cpp + functions/array/function_array_popfront.cpp functions/array/function_array_constructor.cpp functions/array/function_array_with_constant.cpp functions/array/function_array_apply.cpp diff --git a/be/src/vec/functions/array/function_array_popfront.cpp b/be/src/vec/functions/array/function_array_popfront.cpp new file mode 100644 index 0000000000..8c4ed89db4 --- /dev/null +++ b/be/src/vec/functions/array/function_array_popfront.cpp @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#include + +#include "vec/columns/column_array.h" +#include "vec/columns/column_string.h" +#include "vec/common/string_ref.h" +#include "vec/data_types/data_type_array.h" +#include "vec/data_types/data_type_number.h" +#include "vec/functions/array/function_array_utils.h" +#include "vec/functions/function.h" +#include "vec/functions/simple_function_factory.h" + +namespace doris::vectorized { + +class FunctionArrayPopfront : public IFunction { +public: + static constexpr auto name = "array_popfront"; + static FunctionPtr create() { return std::make_shared(); } + + /// Get function name. + String get_name() const override { return name; } + + bool is_variadic() const override { return false; } + + size_t get_number_of_arguments() const override { return 1; } + + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + DCHECK(is_array(arguments[0])) + << "First argument for function: " << name + << " should be DataTypeArray but it has type " << arguments[0]->get_name() << "."; + return arguments[0]; + } + + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + size_t result, size_t input_rows_count) override { + auto array_column = + block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); + // extract src array column + ColumnArrayExecutionData src; + if (!extract_column_array_info(*array_column, src)) { + return Status::RuntimeError( + fmt::format("execute failed, unsupported types for function {}({})", get_name(), + block.get_by_position(arguments[0]).type->get_name())); + } + // prepare dst array column + bool is_nullable = src.nested_nullmap_data ? true : false; + ColumnArrayMutableData dst = create_mutable_data(src.nested_col, is_nullable); + dst.offsets_ptr->reserve(input_rows_count); + // start from 2 + auto offset_column = ColumnInt64::create(array_column->size(), 2); + // len - 1 + auto length_column = ColumnInt64::create(); + for (size_t row = 0; row < src.offsets_ptr->size(); ++row) { + size_t off = (*src.offsets_ptr)[row - 1]; + size_t len = (*src.offsets_ptr)[row] - off; + length_column->insert_value(len - 1); + } + slice_array(dst, src, *offset_column, length_column.get()); + ColumnPtr res_column = assemble_column_array(dst); + block.replace_by_position(result, std::move(res_column)); + return Status::OK(); + } +}; + +void register_function_array_popfront(SimpleFunctionFactory& factory) { + factory.register_function(); +} + +} // namespace doris::vectorized diff --git a/be/src/vec/functions/array/function_array_register.cpp b/be/src/vec/functions/array/function_array_register.cpp index e553a613bf..1fe76b30be 100644 --- a/be/src/vec/functions/array/function_array_register.cpp +++ b/be/src/vec/functions/array/function_array_register.cpp @@ -39,6 +39,7 @@ void register_function_array_enumerate(SimpleFunctionFactory&); void register_function_array_range(SimpleFunctionFactory&); void register_function_array_compact(SimpleFunctionFactory&); void register_function_array_popback(SimpleFunctionFactory&); +void register_function_array_popfront(SimpleFunctionFactory&); void register_function_array_with_constant(SimpleFunctionFactory&); void register_function_array_constructor(SimpleFunctionFactory&); void register_function_array_apply(SimpleFunctionFactory&); @@ -62,6 +63,7 @@ void register_function_array(SimpleFunctionFactory& factory) { register_function_array_range(factory); register_function_array_compact(factory); register_function_array_popback(factory); + register_function_array_popfront(factory); register_function_array_with_constant(factory); register_function_array_constructor(factory); register_function_array_apply(factory); diff --git a/docs/en/docs/sql-manual/sql-functions/array-functions/array_popfront.md b/docs/en/docs/sql-manual/sql-functions/array-functions/array_popfront.md new file mode 100644 index 0000000000..33e36a7e20 --- /dev/null +++ b/docs/en/docs/sql-manual/sql-functions/array-functions/array_popfront.md @@ -0,0 +1,64 @@ +--- +{ + "title": "array_popfront", + "language": "en" +} +--- + + + +## array_popfront + + + +array_popfront + + + +### description + +#### Syntax + +``` +ARRAY array_popfront(ARRAY arr) +``` + +Remove the first element from array. + +### notice + +`Only supported in vectorized engine` + +### example + +``` +mysql> set enable_vectorized_engine=true; + +mysql> select array_popfront(['test', NULL, 'value']); ++-----------------------------------------------------+ +| array_popfront(ARRAY('test', NULL, 'value')) | ++-----------------------------------------------------+ +| [NULL, value] | ++-----------------------------------------------------+ + +``` + +### keywords + +ARRAY,POPFRONT,ARRAY_POPFRONT + diff --git a/docs/sidebars.json b/docs/sidebars.json index 8cdc234983..4636aea9ba 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -292,6 +292,7 @@ "sql-manual/sql-functions/array-functions/array_with_constant", "sql-manual/sql-functions/array-functions/array_enumerate", "sql-manual/sql-functions/array-functions/array_popback", + "sql-manual/sql-functions/array-functions/array_popfront", "sql-manual/sql-functions/array-functions/array_compact", "sql-manual/sql-functions/array-functions/array_concat", "sql-manual/sql-functions/array-functions/arrays_overlap", diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/array-functions/array_popfront.md b/docs/zh-CN/docs/sql-manual/sql-functions/array-functions/array_popfront.md new file mode 100644 index 0000000000..c95c8752e8 --- /dev/null +++ b/docs/zh-CN/docs/sql-manual/sql-functions/array-functions/array_popfront.md @@ -0,0 +1,61 @@ +--- +{ + "title": "array_popfront", + "language": "zh-CN" +} +--- + + + +## array_popfront + + + +array_popfront + + +### description + +#### Syntax + +``` +ARRAY array_popfront(ARRAY arr) +``` + +返回移除第一个元素后的数组，如果输入参数为NULL，则返回NULL + +### notice + +`仅支持向量化引擎中使用` + +### example + +``` +mysql> set enable_vectorized_engine=true; + +mysql> select array_popfront(['test', NULL, 'value']); ++-----------------------------------------------------+ +| array_popfront(ARRAY('test', NULL, 'value')) | ++-----------------------------------------------------+ +| [NULL, value] | ++-----------------------------------------------------+ +``` + +### keywords + +ARRAY,POPFRONT,ARRAY_POPFRONT diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 48fb9a7ec6..866064f786 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -1490,6 +1490,7 @@ public class FunctionCallExpr extends Expr { || fnName.getFunction().equalsIgnoreCase("array_compact") || fnName.getFunction().equalsIgnoreCase("array_slice") || fnName.getFunction().equalsIgnoreCase("array_popback") + || fnName.getFunction().equalsIgnoreCase("array_popfront") || fnName.getFunction().equalsIgnoreCase("reverse") || fnName.getFunction().equalsIgnoreCase("%element_slice%") || fnName.getFunction().equalsIgnoreCase("array_except") diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 7c504c0585..9e0b32fc6a 100644 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -547,6 +547,25 @@ visible_functions = [ [['array_popback'], 'ARRAY_VARCHAR', ['ARRAY_VARCHAR'], ''], [['array_popback'], 'ARRAY_STRING', ['ARRAY_STRING'], ''], + [['array_popfront'], 'ARRAY_BOOLEAN', ['ARRAY_BOOLEAN'], ''], + [['array_popfront'], 'ARRAY_TINYINT', ['ARRAY_TINYINT'], ''], + [['array_popfront'], 'ARRAY_SMALLINT', ['ARRAY_SMALLINT'], ''], + [['array_popfront'], 'ARRAY_INT', ['ARRAY_INT'], ''], + [['array_popfront'], 'ARRAY_BIGINT', ['ARRAY_BIGINT'], ''], + [['array_popfront'], 'ARRAY_LARGEINT', ['ARRAY_LARGEINT'], ''], + [['array_popfront'], 'ARRAY_DATETIME', ['ARRAY_DATETIME'], ''], + [['array_popfront'], 'ARRAY_DATE', ['ARRAY_DATE'], ''], + [['array_popfront'], 'ARRAY_DATETIMEV2', ['ARRAY_DATETIMEV2'], ''], + [['array_popfront'], 'ARRAY_DATEV2', ['ARRAY_DATEV2'], ''], + [['array_popfront'], 'ARRAY_FLOAT', ['ARRAY_FLOAT'], ''], + [['array_popfront'], 'ARRAY_DOUBLE', ['ARRAY_DOUBLE'], ''], + [['array_popfront'], 'ARRAY_DECIMALV2', ['ARRAY_DECIMALV2'], ''], + [['array_popfront'], 'ARRAY_DECIMAL32', ['ARRAY_DECIMAL32'], ''], + [['array_popfront'], 'ARRAY_DECIMAL64', ['ARRAY_DECIMAL64'], ''], + [['array_popfront'], 'ARRAY_DECIMAL128', ['ARRAY_DECIMAL128'], ''], + [['array_popfront'], 'ARRAY_VARCHAR', ['ARRAY_VARCHAR'], ''], + [['array_popfront'], 'ARRAY_STRING', ['ARRAY_STRING'], ''], + [['array_with_constant'], 'ARRAY_BOOLEAN', ['BIGINT', 'BOOLEAN'], 'ALWAYS_NOT_NULLABLE'], [['array_with_constant'], 'ARRAY_TINYINT', ['BIGINT', 'TINYINT'], 'ALWAYS_NOT_NULLABLE'], [['array_with_constant'], 'ARRAY_SMALLINT', ['BIGINT','SMALLINT'], 'ALWAYS_NOT_NULLABLE'], diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions.out index aacc7ee18f..9c06c4e41e 100644 --- a/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions.out +++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions.out @@ -538,6 +538,72 @@ 8 \N 9 \N +-- !select -- +1 [2, 3] +2 [] +3 [] +4 [2, 3, 4, 5, 4, 3, 2, 1] +5 [] +6 [2, 3, 4, 5, 4, 3, 2, 1] +7 [9, NULL, 10, NULL] +8 [2, 3, 3, 4, 4, NULL] +9 [2, 3] + +-- !select -- +1 [] +2 [] +3 [] +4 \N +5 \N +6 \N +7 \N +8 ['hi', 'hello'] +9 [] + +-- !select -- +1 [] +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 [] +9 [2015-03-13, 2015-03-14] + +-- !select -- +1 [] +2 \N +3 \N +4 \N +5 \N +6 \N +7 \N +8 [] +9 [2015-03-13 12:36:38] + +-- !select -- +1 [2023-02-06] +2 [2023-01-06] +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N + +-- !select -- +1 [2022-08-31 12:00:00.999] +2 [2022-01-31 12:00:00.999] +3 \N +4 \N +5 \N +6 \N +7 \N +8 \N +9 \N + -- !select -- 1 [1, 1, 1] 2 [2, 2, 2] diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions_by_literal.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions_by_literal.out index 450c1f1139..8a024f3a4a 100644 --- a/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions_by_literal.out +++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions_by_literal.out @@ -449,6 +449,39 @@ false -- !sql -- [2023-02-06, 2023-02-05] +-- !sql -- +[2, 3, 4, 5, 6] + +-- !sql -- +[] + +-- !sql -- +\N + +-- !sql -- +[2, 3, 4, 5] + +-- !sql -- +[2, 3, 4, NULL] + +-- !sql -- +['2', '3', '4', '5', '6'] + +-- !sql -- +['2', '3', '4', '5', '6'] + +-- !sql -- +['2', '3', '4', '5', NULL] + +-- !sql -- +[2023-02-05, 2023-02-07, 2023-02-05] + +-- !sql -- +[2023-02-06, 2023-02-05, 2023-02-07, 2023-02-05] + +-- !sql -- +[2023-02-04 23:07:34.999, 2023-02-07 22:07:34.999, 2023-02-04 23:07:34.999] + -- !sql -- 1_2_3 diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions.groovy index 5ce3c7467b..f4d2d3be66 100644 --- a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions.groovy +++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions.groovy @@ -99,6 +99,12 @@ suite("test_array_functions") { qt_select "SELECT k1, array_popback(k7) from ${tableName} ORDER BY k1" qt_select "SELECT k1, array_popback(k8) from ${tableName} ORDER BY k1" qt_select "SELECT k1, array_popback(k10) from ${tableName} ORDER BY k1" + qt_select "SELECT k1, array_popfront(k2) from ${tableName} ORDER BY k1" + qt_select "SELECT k1, array_popfront(k5) from ${tableName} ORDER BY k1" + qt_select "SELECT k1, array_popfront(k6) from ${tableName} ORDER BY k1" + qt_select "SELECT k1, array_popfront(k7) from ${tableName} ORDER BY k1" + qt_select "SELECT k1, array_popfront(k8) from ${tableName} ORDER BY k1" + qt_select "SELECT k1, array_popfront(k10) from ${tableName} ORDER BY k1" qt_select "SELECT k1, array_with_constant(3, k1) from ${tableName} ORDER BY k1" qt_select "SELECT k1, array_with_constant(10, null) from ${tableName} ORDER BY k1" qt_select "SELECT k1, array_with_constant(2, 'a') from ${tableName} ORDER BY k1" diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions_by_literal.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions_by_literal.groovy index b73ca03c8e..073a34b120 100644 --- a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions_by_literal.groovy +++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions_by_literal.groovy @@ -189,6 +189,19 @@ suite("test_array_functions_by_literal") { qt_sql "select (array(cast ('2023-02-06 22:07:34.999' as datetimev2(3)),cast ('2023-02-04 23:07:34.999' as datetimev2(3))))[1:2]" qt_sql "select (array(cast ('2023-02-06' as datev2), cast ('2023-02-05' as datev2)))[1:2]" + // array_popfront function + qt_sql "select array_popfront([1,2,3,4,5,6])" + qt_sql "select array_popfront([])" + qt_sql "select array_popfront(null)" + qt_sql "select array_popfront([null,2,3,4,5])" + qt_sql "select array_popfront([1,2,3,4,null])" + qt_sql "select array_popfront(['1','2','3','4','5','6'])" + qt_sql "select array_popfront([null,'2','3','4','5','6'])" + qt_sql "select array_popfront(['1','2','3','4','5',null])" + qt_sql "select array_popfront(array(cast ('2023-02-06' as datev2), cast ('2023-02-05' as datev2), cast ('2023-02-07' as datev2), cast ('2023-02-05' as datev2)))" + qt_sql "select array_popfront(array(null, cast ('2023-02-06' as datev2), cast ('2023-02-05' as datev2), cast ('2023-02-07' as datev2), cast ('2023-02-05' as datev2)))" + qt_sql "select array_popfront(array(cast ('2023-02-06 22:07:34.999' as datetimev2(3)),cast ('2023-02-04 23:07:34.999' as datetimev2(3)), cast ('2023-02-07 22:07:34.999' as datetimev2(3)),cast ('2023-02-04 23:07:34.999' as datetimev2(3))))" + // array_join function qt_sql "select array_join([1, 2, 3], '_')" qt_sql "select array_join(['1', '2', '3', null], '_')"