diff --git a/be/src/agent/be_exec_version_manager.h b/be/src/agent/be_exec_version_manager.h index ae055e9339..fafa764267 100644 --- a/be/src/agent/be_exec_version_manager.h +++ b/be/src/agent/be_exec_version_manager.h @@ -40,6 +40,7 @@ public: } static int get_newest_version() { return max_be_exec_version; } + static int get_min_version() { return min_be_exec_version; } private: static const int max_be_exec_version; @@ -72,7 +73,7 @@ private: * h. "now": ALWAYS_NOT_NULLABLE -> DEPEND_ON_ARGUMENTS * i. change FunctionIsIPAddressInRange from AlwaysNotNullable to DependOnArguments. controlled by individual session variable. */ -constexpr inline int BeExecVersionManager::max_be_exec_version = 7; +constexpr inline int BeExecVersionManager::max_be_exec_version = 6; constexpr inline int BeExecVersionManager::min_be_exec_version = 0; /// functional diff --git a/be/src/vec/exprs/table_function/table_function_factory.cpp b/be/src/vec/exprs/table_function/table_function_factory.cpp index 18855fa756..6a7f44ffe9 100644 --- a/be/src/vec/exprs/table_function/table_function_factory.cpp +++ b/be/src/vec/exprs/table_function/table_function_factory.cpp @@ -63,11 +63,12 @@ const std::unordered_map {}}, {"explode_json_object", TableFunctionCreator {}}, {"posexplode", TableFunctionCreator {}}, - {"explode", TableFunctionCreator {}}, - {"explode_old", TableFunctionCreator {}}}; + {"explode_v2", TableFunctionCreator {}}, + {"explode_v1", TableFunctionCreator {}}}; -const std::unordered_map TableFunctionFactory::_function_to_replace = { - {"explode", "explode_old"}}; +// key is alias name ---> function name +std::unordered_map TableFunctionFactory::_function_alias = { + {"explode", "explode_v1"}}; Status TableFunctionFactory::get_fn(const std::string& fn_name_raw, ObjectPool* pool, TableFunction** fn, @@ -76,7 +77,9 @@ Status TableFunctionFactory::get_fn(const std::string& fn_name_raw, ObjectPool* std::string fn_name_real = is_outer ? remove_suffix(fn_name_raw, COMBINATOR_SUFFIX_OUTER) : fn_name_raw; - temporary_function_update(be_version, fn_name_real); + if (_function_alias.contains(fn_name_real)) { + fn_name_real = _function_alias[fn_name_real]; + } auto fn_iterator = _function_map.find(fn_name_real); if (fn_iterator != _function_map.end()) { diff --git a/be/src/vec/exprs/table_function/table_function_factory.h b/be/src/vec/exprs/table_function/table_function_factory.h index 478348f26e..87c2c2fd4b 100644 --- a/be/src/vec/exprs/table_function/table_function_factory.h +++ b/be/src/vec/exprs/table_function/table_function_factory.h @@ -39,19 +39,8 @@ public: const static std::unordered_map()>> _function_map; - /// @TEMPORARY: for be_exec_version=7. replace function to old version. - const static std::unordered_map _function_to_replace; - const static int NEWEST_VERSION_EXPLODE_MULTI_PARAM = 7; - -private: - /// @TEMPORARY: for be_exec_version=7 - static void temporary_function_update(int fe_version_now, std::string& name) { - // replace if fe is old version. - if (fe_version_now < NEWEST_VERSION_EXPLODE_MULTI_PARAM && - _function_to_replace.find(name) != _function_to_replace.end()) { - name = _function_to_replace.at(name); - } - } + // alias name ---> function name + static std::unordered_map _function_alias; }; } // namespace vectorized } // namespace doris diff --git a/be/src/vec/exprs/table_function/vexplode.cpp b/be/src/vec/exprs/table_function/vexplode.cpp index cfd1260c29..a25ae4ded8 100644 --- a/be/src/vec/exprs/table_function/vexplode.cpp +++ b/be/src/vec/exprs/table_function/vexplode.cpp @@ -33,7 +33,7 @@ namespace doris::vectorized { VExplodeTableFunction::VExplodeTableFunction() { - _fn_name = "vexplode"; + _fn_name = "vexplode_v1"; } Status VExplodeTableFunction::process_init(Block* block, RuntimeState* state) { diff --git a/be/src/vec/exprs/table_function/vexplode_v2.cpp b/be/src/vec/exprs/table_function/vexplode_v2.cpp index b4d4881fca..2096e00e6c 100644 --- a/be/src/vec/exprs/table_function/vexplode_v2.cpp +++ b/be/src/vec/exprs/table_function/vexplode_v2.cpp @@ -40,7 +40,7 @@ namespace doris::vectorized { #include "vec/columns/column_struct.h" VExplodeV2TableFunction::VExplodeV2TableFunction() { - _fn_name = "vexplode"; + _fn_name = "vexplode_v2"; } Status VExplodeV2TableFunction::process_init(Block* block, RuntimeState* state) { diff --git a/be/src/vec/functions/function_fake.cpp b/be/src/vec/functions/function_fake.cpp index 1bde70c876..bbdcc76724 100644 --- a/be/src/vec/functions/function_fake.cpp +++ b/be/src/vec/functions/function_fake.cpp @@ -152,14 +152,6 @@ void register_table_function_expand(SimpleFunctionFactory& factory, const std::s factory.register_function>(name + suffix); }; -template -void register_table_alternative_function_expand(SimpleFunctionFactory& factory, - const std::string& name, - const std::string& suffix) { - factory.register_alternative_function>(name); - factory.register_alternative_function>(name + suffix); -}; - template void register_table_function_expand_default(SimpleFunctionFactory& factory, const std::string& name, const std::string& suffix) { @@ -169,28 +161,11 @@ void register_table_function_expand_default(SimpleFunctionFactory& factory, cons name + suffix); }; -template -void register_table_alternative_function_expand_default(SimpleFunctionFactory& factory, - const std::string& name, - const std::string& suffix) { - factory.register_alternative_function< - FunctionFake>>(name); - factory.register_alternative_function< - FunctionFake>>(name + suffix); -}; - template void register_table_function_expand_outer(SimpleFunctionFactory& factory, const std::string& name) { register_table_function_expand(factory, name, COMBINATOR_SUFFIX_OUTER); }; -template -void register_table_alternative_function_expand_outer(SimpleFunctionFactory& factory, - const std::string& name) { - register_table_alternative_function_expand(factory, name, - COMBINATOR_SUFFIX_OUTER); -}; - template void register_table_function_expand_outer_default(SimpleFunctionFactory& factory, const std::string& name) { @@ -198,13 +173,6 @@ void register_table_function_expand_outer_default(SimpleFunctionFactory& factory COMBINATOR_SUFFIX_OUTER); }; -template -void register_table_alternative_function_expand_outer_default(SimpleFunctionFactory& factory, - const std::string& name) { - register_table_alternative_function_expand_default( - factory, name, COMBINATOR_SUFFIX_OUTER); -}; - template void register_table_function_with_impl(SimpleFunctionFactory& factory, const std::string& name, const std::string& suffix = "") { @@ -214,8 +182,10 @@ void register_table_function_with_impl(SimpleFunctionFactory& factory, const std void register_function_fake(SimpleFunctionFactory& factory) { register_function(factory, "esquery"); - register_table_function_expand_outer(factory, "explode"); - register_table_alternative_function_expand_outer(factory, "explode"); + register_table_function_expand_outer(factory, "explode_v2"); + register_table_function_expand_outer(factory, "explode_v1"); + factory.register_alias("explode_v1", "explode"); + factory.register_alias("explode_v1_outer", "explode_outer"); register_table_function_expand_outer(factory, "explode_map"); diff --git a/be/src/vec/functions/simple_function_factory.h b/be/src/vec/functions/simple_function_factory.h index 92ae50d168..c86d29aaa6 100644 --- a/be/src/vec/functions/simple_function_factory.h +++ b/be/src/vec/functions/simple_function_factory.h @@ -115,9 +115,6 @@ class SimpleFunctionFactory { /// @TEMPORARY: for be_exec_version=4 constexpr static int NEWEST_VERSION_FUNCTION_SUBSTITUTE = 4; - /// @TEMPORARY: for be_exec_version=7. - constexpr static int NEWEST_VERSION_EXPLODE_MULTI_PARAM = 7; - public: void register_function(const std::string& name, const Creator& ptr) { DataTypes types = ptr()->get_variadic_argument_types(); @@ -148,14 +145,6 @@ public: register_function(name, &createDefaultFunction); } - /// @TEMPORARY: for be_exec_version=7 - template - void register_alternative_function(std::string name) { - static std::string suffix {"_old"}; - function_to_replace[name] = name + suffix; - register_function(name + suffix, &createDefaultFunction); - } - void register_alias(const std::string& name, const std::string& alias) { function_alias[alias] = name; } @@ -219,7 +208,7 @@ private: FunctionCreators function_creators; FunctionIsVariadic function_variadic_set; std::unordered_map function_alias; - /// @TEMPORARY: for be_exec_version=7. replace function to old version. + /// @TEMPORARY: for be_exec_version=3. replace function to old version. std::unordered_map function_to_replace; template @@ -227,10 +216,10 @@ private: return std::make_shared(Function::create()); } - /// @TEMPORARY: for be_exec_version=7 + /// @TEMPORARY: for be_exec_version=3 void temporary_function_update(int fe_version_now, std::string& name) { // replace if fe is old version. - if (fe_version_now < NEWEST_VERSION_EXPLODE_MULTI_PARAM && + if (fe_version_now < NEWEST_VERSION_FUNCTION_SUBSTITUTE && function_to_replace.find(name) != function_to_replace.end()) { name = function_to_replace[name]; } diff --git a/be/test/vec/utils/be_exec_version_test.cpp b/be/test/vec/utils/be_exec_version_test.cpp new file mode 100644 index 0000000000..c0e1117dab --- /dev/null +++ b/be/test/vec/utils/be_exec_version_test.cpp @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include + +#include "agent/be_exec_version_manager.h" + +namespace doris::vectorized { +TEST(be_exec_version_test, empty_buckets) { + // YOU MUST NOT CHANGE THIS TEST ! + // if you want to change be_exec_version, you should know what you are doing + EXPECT_EQ(BeExecVersionManager::get_newest_version(), 6); + EXPECT_EQ(BeExecVersionManager::get_min_version(), 0); +} + +} // namespace doris::vectorized diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 4b2711bcc9..6c81062175 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1952,7 +1952,7 @@ public class Config extends ConfigBase { * Max data version of backends serialize block. */ @ConfField(mutable = false) - public static int max_be_exec_version = 7; + public static int max_be_exec_version = 6; /** * Min data version of backends serialize block. diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java index 01c5f72cd1..06123ffb74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java @@ -35,8 +35,10 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeMap import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeNumbers; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeNumbersOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeOuter; +import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeOuterV2; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplit; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplitOuter; +import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeV2; import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplode; import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplodeOuter; @@ -55,8 +57,10 @@ import java.util.Set; */ public class BuiltinTableGeneratingFunctions implements FunctionHelper { public final List tableGeneratingFunctions = ImmutableList.of( - tableGenerating(Explode.class, "explode"), - tableGenerating(ExplodeOuter.class, "explode_outer"), + tableGenerating(Explode.class, "explode_v1"), + tableGenerating(ExplodeV2.class, "explode_v2", "explode"), + tableGenerating(ExplodeOuter.class, "explode_v1_outer"), + tableGenerating(ExplodeOuterV2.class, "explode_v2_outer", "explode_outer"), tableGenerating(ExplodeMap.class, "explode_map"), tableGenerating(ExplodeMapOuter.class, "explode_map_outer"), tableGenerating(ExplodeJsonObject.class, "explode_json_object"), @@ -80,7 +84,8 @@ public class BuiltinTableGeneratingFunctions implements FunctionHelper { ); public static final ImmutableSet RETURN_MULTI_COLUMNS_FUNCTIONS = new ImmutableSortedSet.Builder( - String.CASE_INSENSITIVE_ORDER).add("explode").add("explode_outer").build(); + String.CASE_INSENSITIVE_ORDER).add("explode_v2").add("explode_v2_outer").add("explode").add("explode_outer") + .build(); public Set getReturnManyColumnFunctions() { return RETURN_MULTI_COLUMNS_FUNCTIONS; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/Explode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/Explode.java index 22b80c7f6d..a09ccf3366 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/Explode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/Explode.java @@ -20,34 +20,31 @@ package org.apache.doris.nereids.trees.expressions.functions.generator; import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; -import org.apache.doris.nereids.trees.expressions.functions.ComputePrecision; -import org.apache.doris.nereids.trees.expressions.functions.CustomSignature; -import org.apache.doris.nereids.trees.expressions.functions.SearchSignature; +import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.ArrayType; -import org.apache.doris.nereids.types.DataType; -import org.apache.doris.nereids.types.NullType; -import org.apache.doris.nereids.types.StructField; -import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.coercion.AnyDataType; +import org.apache.doris.nereids.types.coercion.FollowToAnyDataType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import java.util.ArrayList; import java.util.List; /** - * explode([1, 2, 3]), generate three rows include 1, 2 and 3. - * explode([1, 2, 3], [4, 5, 6]) generates two columns and three rows - * where the first column contains 1, 2, 3, and the second column contains 4, 5, 6. + * explode([1, 2, 3]), generate 3 lines include 1, 2 and 3. */ -public class Explode extends TableGeneratingFunction implements CustomSignature, ComputePrecision, AlwaysNullable { +public class Explode extends TableGeneratingFunction implements BinaryExpression, AlwaysNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(new FollowToAnyDataType(0)).args(ArrayType.of(new AnyDataType(0))) + ); /** - * constructor with one or more argument. + * constructor with 1 argument. */ - public Explode(Expression[] args) { - super("explode", args); + public Explode(Expression arg) { + super("explode_v1", arg); } /** @@ -55,43 +52,17 @@ public class Explode extends TableGeneratingFunction implements CustomSignature, */ @Override public Explode withChildren(List children) { - Preconditions.checkArgument(!children.isEmpty()); - return new Explode(children.toArray(new Expression[0])); + Preconditions.checkArgument(children.size() == 1); + return new Explode(children.get(0)); } @Override - public FunctionSignature computePrecision(FunctionSignature signature) { - return signature; - } - - @Override - public FunctionSignature customSignature() { - List arguments = new ArrayList<>(); - ImmutableList.Builder structFields = ImmutableList.builder(); - for (int i = 0; i < children.size(); i++) { - if (children.get(i).getDataType().isNullType()) { - arguments.add(ArrayType.of(NullType.INSTANCE)); - structFields.add( - new StructField("col" + (i + 1), NullType.INSTANCE, true, "")); - } else if (children.get(i).getDataType().isArrayType()) { - structFields.add( - new StructField("col" + (i + 1), - ((ArrayType) (children.get(i)).getDataType()).getItemType(), true, "")); - arguments.add(children.get(i).getDataType()); - } else { - SearchSignature.throwCanNotFoundFunctionException(this.getName(), getArguments()); - } - } - return FunctionSignature.of(new StructType(structFields.build()), arguments); + public List getSignatures() { + return SIGNATURES; } @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitExplode(this, context); } - - @Override - public FunctionSignature searchSignature(List signatures) { - return super.searchSignature(signatures); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeOuter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeOuter.java index cbcc2eaa1c..e7697cc123 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeOuter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeOuter.java @@ -20,34 +20,31 @@ package org.apache.doris.nereids.trees.expressions.functions.generator; import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; -import org.apache.doris.nereids.trees.expressions.functions.ComputePrecision; -import org.apache.doris.nereids.trees.expressions.functions.CustomSignature; -import org.apache.doris.nereids.trees.expressions.functions.SearchSignature; +import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.ArrayType; -import org.apache.doris.nereids.types.DataType; -import org.apache.doris.nereids.types.NullType; -import org.apache.doris.nereids.types.StructField; -import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.coercion.AnyDataType; +import org.apache.doris.nereids.types.coercion.FollowToAnyDataType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import java.util.ArrayList; import java.util.List; /** - * explode_outer([1, 2, 3]), generate three rows include 1, 2 and 3. - * explode_outer([1, 2, 3], [4, 5, 6]) generates two columns and three rows - * where the first column contains 1, 2, 3, and the second column contains 4, 5, 6. + * explode([1, 2, 3]), generate 3 lines include 1, 2 and 3. */ -public class ExplodeOuter extends TableGeneratingFunction implements CustomSignature, ComputePrecision, AlwaysNullable { +public class ExplodeOuter extends TableGeneratingFunction implements BinaryExpression, AlwaysNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(new FollowToAnyDataType(0)).args(ArrayType.of(new AnyDataType(0))) + ); /** - * constructor with one or more argument. + * constructor with 1 argument. */ - public ExplodeOuter(Expression[] args) { - super("explode_outer", args); + public ExplodeOuter(Expression arg) { + super("explode_v2_outer", arg); } /** @@ -55,43 +52,17 @@ public class ExplodeOuter extends TableGeneratingFunction implements CustomSigna */ @Override public ExplodeOuter withChildren(List children) { - Preconditions.checkArgument(!children.isEmpty()); - return new ExplodeOuter(children.toArray(new Expression[0])); + Preconditions.checkArgument(children.size() == 1); + return new ExplodeOuter(children.get(0)); } @Override - public FunctionSignature computePrecision(FunctionSignature signature) { - return signature; - } - - @Override - public FunctionSignature customSignature() { - List arguments = new ArrayList<>(); - ImmutableList.Builder structFields = ImmutableList.builder(); - for (int i = 0; i < children.size(); i++) { - if (children.get(i).getDataType().isNullType()) { - arguments.add(ArrayType.of(NullType.INSTANCE)); - structFields.add( - new StructField("col" + (i + 1), NullType.INSTANCE, true, "")); - } else if (children.get(i).getDataType().isArrayType()) { - structFields.add( - new StructField("col" + (i + 1), - ((ArrayType) (children.get(i)).getDataType()).getItemType(), true, "")); - arguments.add(children.get(i).getDataType()); - } else { - SearchSignature.throwCanNotFoundFunctionException(this.getName(), getArguments()); - } - } - return FunctionSignature.of(new StructType(structFields.build()), arguments); + public List getSignatures() { + return SIGNATURES; } @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitExplodeOuter(this, context); } - - @Override - public FunctionSignature searchSignature(List signatures) { - return super.searchSignature(signatures); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeOuterV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeOuterV2.java new file mode 100644 index 0000000000..bca1f78ad5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeOuterV2.java @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.generator; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.functions.ComputePrecision; +import org.apache.doris.nereids.trees.expressions.functions.CustomSignature; +import org.apache.doris.nereids.trees.expressions.functions.SearchSignature; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.NullType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.List; + +/** + * explode_outer([1, 2, 3]), generate three rows include 1, 2 and 3. + * explode_outer([1, 2, 3], [4, 5, 6]) generates two columns and three rows + * where the first column contains 1, 2, 3, and the second column contains 4, 5, 6. + */ +public class ExplodeOuterV2 extends TableGeneratingFunction + implements CustomSignature, ComputePrecision, AlwaysNullable { + + /** + * constructor with one or more argument. + */ + public ExplodeOuterV2(Expression[] args) { + super("explode_v2_outer", args); + } + + /** + * withChildren. + */ + @Override + public ExplodeOuterV2 withChildren(List children) { + Preconditions.checkArgument(!children.isEmpty()); + return new ExplodeOuterV2(children.toArray(new Expression[0])); + } + + @Override + public FunctionSignature computePrecision(FunctionSignature signature) { + return signature; + } + + @Override + public FunctionSignature customSignature() { + List arguments = new ArrayList<>(); + ImmutableList.Builder structFields = ImmutableList.builder(); + for (int i = 0; i < children.size(); i++) { + if (children.get(i).getDataType().isNullType()) { + arguments.add(ArrayType.of(NullType.INSTANCE)); + structFields.add( + new StructField("col" + (i + 1), NullType.INSTANCE, true, "")); + } else if (children.get(i).getDataType().isArrayType()) { + structFields.add( + new StructField("col" + (i + 1), + ((ArrayType) (children.get(i)).getDataType()).getItemType(), true, "")); + arguments.add(children.get(i).getDataType()); + } else { + SearchSignature.throwCanNotFoundFunctionException(this.getName(), getArguments()); + } + } + return FunctionSignature.of(new StructType(structFields.build()), arguments); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitExplodeOuterV2(this, context); + } + + @Override + public FunctionSignature searchSignature(List signatures) { + return super.searchSignature(signatures); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeV2.java new file mode 100644 index 0000000000..fc70ab0fbd --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeV2.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.generator; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.functions.ComputePrecision; +import org.apache.doris.nereids.trees.expressions.functions.CustomSignature; +import org.apache.doris.nereids.trees.expressions.functions.SearchSignature; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.NullType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.List; + +/** + * explode([1, 2, 3]), generate three rows include 1, 2 and 3. + * explode([1, 2, 3], [4, 5, 6]) generates two columns and three rows + * where the first column contains 1, 2, 3, and the second column contains 4, 5, 6. + */ +public class ExplodeV2 extends TableGeneratingFunction implements CustomSignature, ComputePrecision, AlwaysNullable { + + /** + * constructor with one or more argument. + */ + public ExplodeV2(Expression[] args) { + super("explode_v2", args); + } + + /** + * withChildren. + */ + @Override + public ExplodeV2 withChildren(List children) { + Preconditions.checkArgument(!children.isEmpty()); + return new ExplodeV2(children.toArray(new Expression[0])); + } + + @Override + public FunctionSignature computePrecision(FunctionSignature signature) { + return signature; + } + + @Override + public FunctionSignature customSignature() { + List arguments = new ArrayList<>(); + ImmutableList.Builder structFields = ImmutableList.builder(); + for (int i = 0; i < children.size(); i++) { + if (children.get(i).getDataType().isNullType()) { + arguments.add(ArrayType.of(NullType.INSTANCE)); + structFields.add( + new StructField("col" + (i + 1), NullType.INSTANCE, true, "")); + } else if (children.get(i).getDataType().isArrayType()) { + structFields.add( + new StructField("col" + (i + 1), + ((ArrayType) (children.get(i)).getDataType()).getItemType(), true, "")); + arguments.add(children.get(i).getDataType()); + } else { + SearchSignature.throwCanNotFoundFunctionException(this.getName(), getArguments()); + } + } + return FunctionSignature.of(new StructType(structFields.build()), arguments); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitExplodeV2(this, context); + } + + @Override + public FunctionSignature searchSignature(List signatures) { + return super.searchSignature(signatures); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java index 103078f8f7..d9ce00894a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java @@ -35,8 +35,10 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeMap import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeNumbers; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeNumbersOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeOuter; +import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeOuterV2; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplit; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplitOuter; +import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeV2; import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplode; import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplodeOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; @@ -55,6 +57,14 @@ public interface TableGeneratingFunctionVisitor { return visitTableGeneratingFunction(explodeOuter, context); } + default R visitExplodeV2(ExplodeV2 explode, C context) { + return visitTableGeneratingFunction(explode, context); + } + + default R visitExplodeOuterV2(ExplodeOuterV2 explodeOuter, C context) { + return visitTableGeneratingFunction(explodeOuter, context); + } + default R visitExplodeMap(ExplodeMap explode, C context) { return visitTableGeneratingFunction(explode, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/FunctionAlias.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FunctionAlias.java index 878319ac56..7b8e32e532 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/FunctionAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FunctionAlias.java @@ -42,7 +42,8 @@ public final class FunctionAlias implements ExprRewriteRule { .put("inet6_aton", "ipv6_string_to_num_or_null").put("lcase", "lower").put("add_months", "months_add") .put("current_timestamp", "now").put("localtime", "now").put("localtimestamp", "now").put("nvl", "ifnull") .put("rand", "random").put("sha", "sha1").put("substr", "substring").put("ucase", "upper") - .put("approx_count_distinct", "ndv").build(); + .put("approx_count_distinct", "ndv").put("explode", "explode_v2").put("explode_outer", "explode_v2_outer") + .build(); @Override public Expr apply(Expr expr, Analyzer analyzer, ExprRewriter.ClauseType clauseType) throws AnalysisException { diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/ConfigTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/ConfigTest.java new file mode 100644 index 0000000000..5af44ec2ed --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/common/ConfigTest.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common; + +import org.junit.Test; +import org.junit.jupiter.api.Assertions; + +public class ConfigTest { + @Test + public void test() { + System.out.println(Config.max_be_exec_version); + System.out.println(Config.min_be_exec_version); + System.out.println(Config.be_exec_version); + // YOU MUST NOT CHANGE THIS TEST ! + // if you want to change be_exec_version, you should know what you are doing + Assertions.assertEquals(6, Config.max_be_exec_version); + Assertions.assertEquals(0, Config.min_be_exec_version); + Assertions.assertEquals(6, Config.be_exec_version); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeOuterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeOuterTest.java index fd381a4e7d..5fb3ea046d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeOuterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeOuterTest.java @@ -43,7 +43,7 @@ public class ExplodeOuterTest { // build explode_outer(array, array) expression Expression[] args = {SlotReference.of("int", ArrayType.of(IntegerType.INSTANCE)), SlotReference.of("str", ArrayType.of(StringType.INSTANCE))}; - ExplodeOuter explode = new ExplodeOuter(args); + ExplodeOuterV2 explode = new ExplodeOuterV2(args); // check signature List signatures = explode.getSignatures(); @@ -65,7 +65,7 @@ public class ExplodeOuterTest { public void testGetSignaturesWithNull() { // build explode(null, array) expression Expression[] args = { SlotReference.of("null", NullType.INSTANCE), SlotReference.of("int", ArrayType.of(IntegerType.INSTANCE))}; - ExplodeOuter explode = new ExplodeOuter(args); + ExplodeOuterV2 explode = new ExplodeOuterV2(args); // check signature List signatures = explode.getSignatures(); @@ -87,7 +87,7 @@ public class ExplodeOuterTest { public void testGetSignaturesWithInvalidArgument() { // build explode_outer(int) Expression[] args = { SlotReference.of("int", IntegerType.INSTANCE) }; - ExplodeOuter explode = new ExplodeOuter(args); + ExplodeOuterV2 explode = new ExplodeOuterV2(args); Assertions.assertThrows(AnalysisException.class, explode::getSignatures); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeTest.java index 0cb61d6171..3434742124 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeTest.java @@ -43,7 +43,7 @@ public class ExplodeTest { // build explode(array, array) expression Expression[] args = {SlotReference.of("int", ArrayType.of(IntegerType.INSTANCE)), SlotReference.of("str", ArrayType.of(StringType.INSTANCE))}; - Explode explode = new Explode(args); + ExplodeV2 explode = new ExplodeV2(args); // check signature List signatures = explode.getSignatures(); @@ -65,7 +65,7 @@ public class ExplodeTest { public void testGetSignaturesWithNull() { // build explode(null, array) expression Expression[] args = { SlotReference.of("null", NullType.INSTANCE), SlotReference.of("int", ArrayType.of(IntegerType.INSTANCE))}; - Explode explode = new Explode(args); + ExplodeV2 explode = new ExplodeV2(args); // check signature List signatures = explode.getSignatures(); @@ -87,7 +87,7 @@ public class ExplodeTest { public void testGetSignaturesWithInvalidArgument() { // build explode(int) Expression[] args = { SlotReference.of("int", IntegerType.INSTANCE) }; - Explode explode = new Explode(args); + ExplodeV2 explode = new ExplodeV2(args); Assertions.assertThrows(AnalysisException.class, explode::getSignatures); }