From f2292a3b1d047cd001c2c6fb8dea5df08b2a1a9b Mon Sep 17 00:00:00 2001 From: camby <104178625@qq.com> Date: Tue, 16 Aug 2022 14:41:57 +0800 Subject: [PATCH] [Enhancement](array-type) enable_array_type flag update (#11785) Co-authored-by: cambyzju --- .../aggregate-functions/collect_list.md | 1 - .../aggregate-functions/collect_set.md | 1 - .../sql-functions/string-functions/reverse.md | 3 --- .../sql-functions/table-functions/explode.md | 1 - .../sql-manual/sql-reference/Data-Types/ARRAY.md | 4 +--- .../aggregate-functions/collect_list.md | 1 - .../aggregate-functions/collect_set.md | 1 - .../sql-functions/string-functions/reverse.md | 3 --- .../sql-functions/table-functions/explode.md | 1 - .../sql-manual/sql-reference/Data-Types/ARRAY.md | 4 +--- .../java/org/apache/doris/catalog/ArrayType.java | 4 ++-- .../main/java/org/apache/doris/common/Config.java | 6 ++++++ .../java/org/apache/doris/qe/SessionVariable.java | 13 ------------- .../org/apache/doris/analysis/ColumnDefTest.java | 3 ++- .../apache/doris/analysis/InsertArrayStmtTest.java | 3 ++- .../org/apache/doris/catalog/CreateTableTest.java | 2 +- .../suites/compaction/test_compaction_array.groovy | 2 +- .../suites/load/broker_load/test_array_load.groovy | 2 +- .../test_aggregate_collect.groovy | 2 +- .../query/sql_functions/array_functions/sql/q01.sql | 2 +- .../query/sql_functions/array_functions/sql/q02.sql | 2 +- .../array_functions/test_array_functions.groovy | 2 +- .../test_array_functions_by_literal.groovy | 2 +- .../test_array_functions_with_where.groovy | 2 +- .../test_cast_array_functions_by_literal.groovy | 2 +- .../sql_functions/table_function/explode.groovy | 2 +- 26 files changed, 25 insertions(+), 46 deletions(-) diff --git a/docs/en/docs/sql-manual/sql-functions/aggregate-functions/collect_list.md b/docs/en/docs/sql-manual/sql-functions/aggregate-functions/collect_list.md index a27b540c78..eb2a732387 100644 --- a/docs/en/docs/sql-manual/sql-functions/aggregate-functions/collect_list.md +++ b/docs/en/docs/sql-manual/sql-functions/aggregate-functions/collect_list.md @@ -43,7 +43,6 @@ Only supported in vectorized engine ``` mysql> set enable_vectorized_engine=true; -mysql> set enable_array_type = true; mysql> select k1,k2,k3 from collect_test order by k1; +------+------------+-------+ diff --git a/docs/en/docs/sql-manual/sql-functions/aggregate-functions/collect_set.md b/docs/en/docs/sql-manual/sql-functions/aggregate-functions/collect_set.md index 8c3d617b3e..ddf9ee0410 100644 --- a/docs/en/docs/sql-manual/sql-functions/aggregate-functions/collect_set.md +++ b/docs/en/docs/sql-manual/sql-functions/aggregate-functions/collect_set.md @@ -43,7 +43,6 @@ Only supported in vectorized engine ``` mysql> set enable_vectorized_engine=true; -mysql> set enable_array_type = true; mysql> select k1,k2,k3 from collect_test order by k1; +------+------------+-------+ diff --git a/docs/en/docs/sql-manual/sql-functions/string-functions/reverse.md b/docs/en/docs/sql-manual/sql-functions/string-functions/reverse.md index a349f7f673..797041bff5 100644 --- a/docs/en/docs/sql-manual/sql-functions/string-functions/reverse.md +++ b/docs/en/docs/sql-manual/sql-functions/string-functions/reverse.md @@ -55,9 +55,6 @@ mysql> SELECT REVERSE('你好'); | 好你 | +------------------+ 1 row in set (0.00 sec) -``` - -mysql> set enable_array_type=true; mysql> set enable_vectorized_engine=true; diff --git a/docs/en/docs/sql-manual/sql-functions/table-functions/explode.md b/docs/en/docs/sql-manual/sql-functions/table-functions/explode.md index 128076d4f2..dbed88df4b 100644 --- a/docs/en/docs/sql-manual/sql-functions/table-functions/explode.md +++ b/docs/en/docs/sql-manual/sql-functions/table-functions/explode.md @@ -43,7 +43,6 @@ explode_outer(expr) ### example ``` mysql> set enable_vectorized_engine = true -mysql> set enable_array_type = true mysql> select e1 from (select 1 k1) as t lateral view explode([1,2,3]) tmp1 as e1; +------+ diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Types/ARRAY.md b/docs/en/docs/sql-manual/sql-reference/Data-Types/ARRAY.md index 8a0ddb88ea..81775340d6 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Types/ARRAY.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Types/ARRAY.md @@ -40,15 +40,13 @@ DATETIME, CHAR, VARCHAR, STRING ``` ### notice -please open `enable_array_type` before use ARRAY +We should add config `enable_array_type=true` inside fe.conf before use ARRAY ### example Create table example: ``` -mysql> set enable_array_type=true; - mysql> CREATE TABLE `array_test` ( `id` int(11) NULL COMMENT "", `c_array` ARRAY NULL COMMENT "" diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/aggregate-functions/collect_list.md b/docs/zh-CN/docs/sql-manual/sql-functions/aggregate-functions/collect_list.md index eaf56bf959..37ee1010d4 100644 --- a/docs/zh-CN/docs/sql-manual/sql-functions/aggregate-functions/collect_list.md +++ b/docs/zh-CN/docs/sql-manual/sql-functions/aggregate-functions/collect_list.md @@ -43,7 +43,6 @@ under the License. ``` mysql> set enable_vectorized_engine=true; -mysql> set enable_array_type = true; mysql> select k1,k2,k3 from collect_test order by k1; +------+------------+-------+ diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/aggregate-functions/collect_set.md b/docs/zh-CN/docs/sql-manual/sql-functions/aggregate-functions/collect_set.md index ccc734cea8..ccaeee793c 100644 --- a/docs/zh-CN/docs/sql-manual/sql-functions/aggregate-functions/collect_set.md +++ b/docs/zh-CN/docs/sql-manual/sql-functions/aggregate-functions/collect_set.md @@ -42,7 +42,6 @@ under the License. ``` mysql> set enable_vectorized_engine=true; -mysql> set enable_array_type = true; mysql> select k1,k2,k3 from collect_test order by k1; +------+------------+-------+ diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/string-functions/reverse.md b/docs/zh-CN/docs/sql-manual/sql-functions/string-functions/reverse.md index b234fede61..37a8bdad4f 100644 --- a/docs/zh-CN/docs/sql-manual/sql-functions/string-functions/reverse.md +++ b/docs/zh-CN/docs/sql-manual/sql-functions/string-functions/reverse.md @@ -55,9 +55,6 @@ mysql> SELECT REVERSE('你好'); | 好你 | +------------------+ 1 row in set (0.00 sec) -``` - -mysql> set enable_array_type=true; mysql> set enable_vectorized_engine=true; diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/explode.md b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/explode.md index e5da64bd4c..adc02d1842 100644 --- a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/explode.md +++ b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/explode.md @@ -44,7 +44,6 @@ explode_outer(expr) ``` mysql> set enable_vectorized_engine = true -mysql> set enable_array_type = true mysql> select e1 from (select 1 k1) as t lateral view explode([1,2,3]) tmp1 as e1; +------+ diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/ARRAY.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/ARRAY.md index 3a6277b6cc..753234e98d 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/ARRAY.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/ARRAY.md @@ -41,15 +41,13 @@ DATETIME, CHAR, VARCHAR, STRING ### notice -使用时需要先打开`enable_array_type`开关 +使用前需要在fe.conf中添加`enable_array_type=true`配置项 ### example 建表示例如下： ``` -mysql> set enable_array_type=true; - mysql> CREATE TABLE `array_test` ( `id` int(11) NULL COMMENT "", `c_array` ARRAY NULL COMMENT "" diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java index 4e1441fdbe..e8fc4c2812 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java @@ -17,7 +17,7 @@ package org.apache.doris.catalog; -import org.apache.doris.qe.ConnectContext; +import org.apache.doris.common.Config; import org.apache.doris.thrift.TColumnType; import org.apache.doris.thrift.TTypeDesc; import org.apache.doris.thrift.TTypeNode; @@ -155,7 +155,7 @@ public class ArrayType extends Type { @Override public boolean isSupported() { - if (!ConnectContext.get().getSessionVariable().isEnableArrayType()) { + if (!Config.enable_array_type) { return false; } return !itemType.isNull(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java index e316ab9d79..b1f2eff13a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java @@ -1732,4 +1732,10 @@ public class Config extends ConfigBase { */ @ConfField public static String s3_compatible_object_storages = "s3,oss,cos,bos"; + + /** + * Support complex data type ARRAY. + */ + @ConfField(mutable = true, masterOnly = true) + public static boolean enable_array_type = false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 6a44207d42..9017b6d5e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -195,8 +195,6 @@ public class SessionVariable implements Serializable, Writable { public static final String TRIM_TAILING_SPACES_FOR_EXTERNAL_TABLE_QUERY = "trim_tailing_spaces_for_external_table_query"; - static final String ENABLE_ARRAY_TYPE = "enable_array_type"; - public static final String ENABLE_NEREIDS_PLANNER = "enable_nereids_planner"; public static final String ENABLE_NEREIDS_REORDER_TO_ELIMINATE_CROSS_JOIN = @@ -497,9 +495,6 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = ENABLE_PROJECTION) private boolean enableProjection = true; - @VariableMgr.VarAttr(name = ENABLE_ARRAY_TYPE) - private boolean enableArrayType = false; - /** * as the new optimizer is not mature yet, use this var * to control whether to use new optimizer, remove it when @@ -1036,14 +1031,6 @@ public class SessionVariable implements Serializable, Writable { this.disableJoinReorder = disableJoinReorder; } - public boolean isEnableArrayType() { - return enableArrayType; - } - - public void setEnableArrayType(boolean enableArrayType) { - this.enableArrayType = enableArrayType; - } - /** * Nereids only support vectorized engine. * diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java index f18e11b222..f699b33ee6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java @@ -25,6 +25,7 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.qe.ConnectContext; import mockit.Mock; @@ -46,6 +47,7 @@ public class ColumnDefTest { stringCol = new TypeDef(ScalarType.createChar(10)); floatCol = new TypeDef(ScalarType.createType(PrimitiveType.FLOAT)); booleanCol = new TypeDef(ScalarType.createType(PrimitiveType.BOOLEAN)); + Config.enable_array_type = true; ctx = new ConnectContext(null); new MockUp() { @@ -133,7 +135,6 @@ public class ColumnDefTest { @Test public void testArray() throws AnalysisException { - ctx.getSessionVariable().setEnableArrayType(true); TypeDef typeDef = new TypeDef(new ArrayType(Type.INT)); ColumnDef columnDef = new ColumnDef("array", typeDef, false, null, true, DefaultValue.NOT_SET, ""); Column column = columnDef.toColumn(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/InsertArrayStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/InsertArrayStmtTest.java index b6fbca8430..6ef0918478 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/InsertArrayStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/InsertArrayStmtTest.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.qe.ConnectContext; @@ -43,7 +44,7 @@ public class InsertArrayStmtTest { public static void setUp() throws Exception { UtFrameUtils.createDorisCluster(RUNNING_DIR); connectContext = UtFrameUtils.createDefaultCtx(); - connectContext.getSessionVariable().setEnableArrayType(true); + Config.enable_array_type = true; createDatabase("create database test;"); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java index c1697fd82e..90e6d0e890 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java @@ -45,6 +45,7 @@ public class CreateTableTest { @BeforeClass public static void beforeClass() throws Exception { Config.disable_storage_medium_check = true; + Config.enable_array_type = true; UtFrameUtils.createDorisCluster(runningDir); // create connect context @@ -560,7 +561,6 @@ public class CreateTableTest { @Test public void testCreateTableWithArrayType() throws Exception { - ConnectContext.get().getSessionVariable().setEnableArrayType(true); ExceptionChecker.expectThrowsNoException(() -> { createTable("create table test.table1(k1 INT, k2 Array) duplicate key (k1) " + "distributed by hash(k1) buckets 1 properties('replication_num' = '1');"); diff --git a/regression-test/suites/compaction/test_compaction_array.groovy b/regression-test/suites/compaction/test_compaction_array.groovy index 2acc89d684..c04e711e45 100644 --- a/regression-test/suites/compaction/test_compaction_array.groovy +++ b/regression-test/suites/compaction/test_compaction_array.groovy @@ -21,7 +21,7 @@ suite("test_array_compaction") { def tableName = "tbl_test_array_compaction" // open enable_array_type - sql """ set enable_array_type = true """ + sql "ADMIN SET FRONTEND CONFIG ('enable_array_type' = 'true')" // array functions only supported in vectorized engine sql """ set enable_vectorized_engine = true """ diff --git a/regression-test/suites/load/broker_load/test_array_load.groovy b/regression-test/suites/load/broker_load/test_array_load.groovy index 51d6859e92..7935a27830 100644 --- a/regression-test/suites/load/broker_load/test_array_load.groovy +++ b/regression-test/suites/load/broker_load/test_array_load.groovy @@ -21,7 +21,7 @@ suite("test_array_load", "load") { def create_test_table = {testTablex, enable_vectorized_flag -> // multi-line sql - sql """ set enable_array_type = true """ + sql "ADMIN SET FRONTEND CONFIG ('enable_array_type' = 'true')" if (enable_vectorized_flag) { sql """ set enable_vectorized_engine = true """ diff --git a/regression-test/suites/query/sql_functions/aggregate_functions/test_aggregate_collect.groovy b/regression-test/suites/query/sql_functions/aggregate_functions/test_aggregate_collect.groovy index 064b3d9fac..9179133b98 100644 --- a/regression-test/suites/query/sql_functions/aggregate_functions/test_aggregate_collect.groovy +++ b/regression-test/suites/query/sql_functions/aggregate_functions/test_aggregate_collect.groovy @@ -17,7 +17,7 @@ suite("test_aggregate_collect", "query") { sql "set enable_vectorized_engine = true" - sql "set enable_array_type = true;" + sql "ADMIN SET FRONTEND CONFIG ('enable_array_type' = 'true')" def tableName = "collect_test" sql "DROP TABLE IF EXISTS ${tableName}" diff --git a/regression-test/suites/query/sql_functions/array_functions/sql/q01.sql b/regression-test/suites/query/sql_functions/array_functions/sql/q01.sql index c8122195f3..3c994fffb8 100644 --- a/regression-test/suites/query/sql_functions/array_functions/sql/q01.sql +++ b/regression-test/suites/query/sql_functions/array_functions/sql/q01.sql @@ -1,4 +1,4 @@ -set enable_array_type = true; +ADMIN SET FRONTEND CONFIG ('enable_array_type' = 'true'); set enable_vectorized_engine=true; DROP TABLE IF EXISTS array_element_test; CREATE TABLE array_element_test (x int, arr array, id int) ENGINE = Olap DUPLICATE KEY(x) DISTRIBUTED BY HASH(x) BUCKETS 1 PROPERTIES("replication_num" = "1");; diff --git a/regression-test/suites/query/sql_functions/array_functions/sql/q02.sql b/regression-test/suites/query/sql_functions/array_functions/sql/q02.sql index 0f84c40e9a..6ff2068c51 100644 --- a/regression-test/suites/query/sql_functions/array_functions/sql/q02.sql +++ b/regression-test/suites/query/sql_functions/array_functions/sql/q02.sql @@ -1,4 +1,4 @@ -set enable_array_type = true; +ADMIN SET FRONTEND CONFIG ('enable_array_type' = 'true'); set enable_vectorized_engine=true; SELECT [1, 2]; SELECT [1.0, 2]; diff --git a/regression-test/suites/query/sql_functions/array_functions/test_array_functions.groovy b/regression-test/suites/query/sql_functions/array_functions/test_array_functions.groovy index d5abee4374..766f0396bc 100644 --- a/regression-test/suites/query/sql_functions/array_functions/test_array_functions.groovy +++ b/regression-test/suites/query/sql_functions/array_functions/test_array_functions.groovy @@ -18,7 +18,7 @@ suite("test_array_functions", "query") { def tableName = "tbl_test_array_functions" // open enable_array_type - sql """ set enable_array_type = true """ + sql "ADMIN SET FRONTEND CONFIG ('enable_array_type' = 'true')" // array functions only supported in vectorized engine sql """ set enable_vectorized_engine = true """ diff --git a/regression-test/suites/query/sql_functions/array_functions/test_array_functions_by_literal.groovy b/regression-test/suites/query/sql_functions/array_functions/test_array_functions_by_literal.groovy index 47230d32c0..3314359547 100644 --- a/regression-test/suites/query/sql_functions/array_functions/test_array_functions_by_literal.groovy +++ b/regression-test/suites/query/sql_functions/array_functions/test_array_functions_by_literal.groovy @@ -17,7 +17,7 @@ suite("test_array_functions_by_literal", "all") { sql "set enable_vectorized_engine = true" - sql "set enable_array_type = true" + sql "ADMIN SET FRONTEND CONFIG ('enable_array_type' = 'true')" // array_contains function qt_sql "select array_contains([1,2,3], 1)" diff --git a/regression-test/suites/query/sql_functions/array_functions/test_array_functions_with_where.groovy b/regression-test/suites/query/sql_functions/array_functions/test_array_functions_with_where.groovy index a9b06828b4..1a52934c50 100644 --- a/regression-test/suites/query/sql_functions/array_functions/test_array_functions_with_where.groovy +++ b/regression-test/suites/query/sql_functions/array_functions/test_array_functions_with_where.groovy @@ -18,7 +18,7 @@ suite("test_array_functions_with_where", "query") { def tableName = "tbl_test_array_functions_with_where" // open enable_array_type - sql """ set enable_array_type = true """ + sql "ADMIN SET FRONTEND CONFIG ('enable_array_type' = 'true')" // array functions only supported in vectorized engine sql """ set enable_vectorized_engine = true """ diff --git a/regression-test/suites/query/sql_functions/array_functions/test_cast_array_functions_by_literal.groovy b/regression-test/suites/query/sql_functions/array_functions/test_cast_array_functions_by_literal.groovy index e046987202..899f17d93d 100644 --- a/regression-test/suites/query/sql_functions/array_functions/test_cast_array_functions_by_literal.groovy +++ b/regression-test/suites/query/sql_functions/array_functions/test_cast_array_functions_by_literal.groovy @@ -17,7 +17,7 @@ suite("test_cast_array_functions", "query") { // open enable_array_type - sql """ set enable_array_type = true """ + sql "ADMIN SET FRONTEND CONFIG ('enable_array_type' = 'true')" // array functions only supported in vectorized engine sql """ set enable_vectorized_engine = true """ test { diff --git a/regression-test/suites/query/sql_functions/table_function/explode.groovy b/regression-test/suites/query/sql_functions/table_function/explode.groovy index 7b46193c4b..5d2643baa6 100644 --- a/regression-test/suites/query/sql_functions/table_function/explode.groovy +++ b/regression-test/suites/query/sql_functions/table_function/explode.groovy @@ -18,7 +18,7 @@ suite("explode") { // vectorized sql """ set enable_vectorized_engine = true """ - sql """ set enable_array_type = true """ + sql "ADMIN SET FRONTEND CONFIG ('enable_array_type' = 'true')" qt_explode """ select e1 from (select 1 k1) as t lateral view explode([1,2,3]) tmp1 as e1; """ qt_explode_outer """ select e1 from (select 1 k1) as t lateral view explode_outer([1,2,3]) tmp1 as e1; """