database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Improve](config)delete confused config for nested complex type (#29988)

Browse Source
This commit is contained in:
amory
2024-01-17 17:41:40 +08:00
committed by yiguolei
parent 143df9f414
commit ade720470d
18 changed files with 4 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
fe/fe-common/src/main/java/org/apache/doris/common/Config.java
View File

@ -2089,17 +2089,6 @@ public class Config extends ConfigBase {
@ConfField(mutable = true)
public static boolean disable_datev1 = true;
/**
* Now we not fully support array/struct/map nesting complex type in many situation,
* so just disable creating nesting complex data type when create table.
* We can make it able after we fully support
*/
@ConfField(mutable = true, masterOnly = true, description = {
"当前默认设置为 true，不支持建表时创建复杂类型(array/struct/map)嵌套复杂类型, 仅支持array类型自身嵌套。",
"Now default set to true, not support create complex type(array/struct/map) nested complex type "
+ "when we create table, only support array type nested array"})
public static boolean disable_nested_complex_type = true;
/*
* This variable indicates the number of digits by which to increase the scale
* of the result of division operations performed with the `/` operator. The

11
fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
View File

@ -28,7 +28,6 @@ import org.apache.doris.catalog.StructField;
import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.thrift.TColumnDesc;
import org.apache.doris.thrift.TPrimitiveType;
@ -130,18 +129,10 @@ public class TypeDef implements ParseNode {
Type itemType = ((ArrayType) type).getItemType();
if (itemType instanceof ScalarType) {
analyzeNestedType(type, (ScalarType) itemType);
} else if (Config.disable_nested_complex_type && !(itemType instanceof ArrayType)) {
// now we can array nesting array
throw new AnalysisException("Unsupported data type: ARRAY<" + itemType.toSql() + ">");
}
}
if (type.isMapType()) {
MapType mt = (MapType) type;
if (Config.disable_nested_complex_type && (!(mt.getKeyType() instanceof ScalarType)
|| !(mt.getValueType() instanceof ScalarType))) {
throw new AnalysisException("Unsupported data type: MAP<" + mt.getKeyType().toSql() + ","
+ mt.getValueType().toSql() + ">");
}
if (mt.getKeyType() instanceof ScalarType) {
analyzeNestedType(type, (ScalarType) mt.getKeyType());
}
@ -160,8 +151,6 @@ public class TypeDef implements ParseNode {
throw new AnalysisException("Duplicate field name "
+ field.getName() + " in struct " + type.toSql());
}
} else if (Config.disable_nested_complex_type) {
throw new AnalysisException("Unsupported field type: " + fieldType.toSql() + " for STRUCT");
}
}
}

14
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
View File

@ -24,7 +24,6 @@ import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.Config;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.BigIntType;
@ -392,21 +391,11 @@ public class ColumnDefinition {
Type itemType = ((org.apache.doris.catalog.ArrayType) catalogType).getItemType();
if (itemType instanceof ScalarType) {
validateNestedType(catalogType, (ScalarType) itemType);
} else if (Config.disable_nested_complex_type
&& !(itemType instanceof org.apache.doris.catalog.ArrayType)) {
// now we can array nesting array
throw new AnalysisException(
"Unsupported data type: ARRAY<" + itemType.toSql() + ">");
}
}
if (catalogType.isMapType()) {
org.apache.doris.catalog.MapType mt =
(org.apache.doris.catalog.MapType) catalogType;
if (Config.disable_nested_complex_type && (!(mt.getKeyType() instanceof ScalarType)
|| !(mt.getValueType() instanceof ScalarType))) {
throw new AnalysisException("Unsupported data type: MAP<"
+ mt.getKeyType().toSql() + "," + mt.getValueType().toSql() + ">");
}
if (mt.getKeyType() instanceof ScalarType) {
validateNestedType(catalogType, (ScalarType) mt.getKeyType());
}
@ -426,9 +415,6 @@ public class ColumnDefinition {
throw new AnalysisException("Duplicate field name " + field.getName()
+ " in struct " + catalogType.toSql());
}
} else if (Config.disable_nested_complex_type) {
throw new AnalysisException(
"Unsupported field type: " + fieldType.toSql() + " for STRUCT");
}
}
}

3
regression-test/data/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.out
View File

@ -2,9 +2,6 @@
-- !sql --
0
-- !sql --
0
-- !sql --
2 {1:[0.35815922932906263, 0.0011899152357573994, 0.28749219850167373, 0.93512930168283781, 0.1552584991620739, 0.73308976093672584, 0.52815960653805338, 0.92936404769642733, 0.89084215295591418, 0.21986459138832559], 2:[0.25076205844319044, 0.54003619330849928, 0.70661164863002113, 0.99472899095144263, 0.078314941019622886, 0.44206845606243961, 0.30857433265707379, 0.048661247184825784, 0.76954870938240083, 0.27253204080482074], 3:[0.687619207224633, 0.81614795840545351, 0.88663481647216158, 0.38735602510379774, 0.80082741637394084, 0.39139397466930215, 0.81747099649629329, 0.17689152877700398, 0.293284479800614, 0.4831799656491037], 4:[0.91830437289915945, 0.073632718338386915, 0.77375510933714242, 0.16816732397102252, 0.43277238052192457, 0.44208980569126877, 0.79186698174595249, 0.16564847296516494, 0.0094630056071454138, 0.35501997374061678], 5:[0.46246293630195257, 0.068748494114144409, 0.99666952760980676, 0.19382320901198535, 0.13487672467906897, 0.33765716511811372, 0.94048529885220122, 0.49731102964787222, 0.32349099284636484, 0.48597339317230759], 6:[0.085073215003979574, 0.28401725721857973, 0.20014761214390719, 0.20524657901039456, 0.11392467553698971, 0.24438834044276414, 0.49910871763701081, 0.33495152703896491, 0.46452858927426788, 0.390595446136895], 7:[0.20961408068486054, 0.67796351365684493, 0.088745389947127551, 0.12660368488966578, 0.505554324463536, 0.27743440100778682, 0.13385620029678247, 0.59106155590166665, 0.10571357063523412, 0.042619186963761324], 8:[0.68279814520834636, 0.94813249012442957, 0.94054706738722227, 0.47041093695670555, 0.58451395465060019, 0.16449033849974781, 0.14272344659751923, 0.66091032176672571, 0.4394562483212725, 0.16905962729691759]} {"G72Qcx-UBjfOr1-Pis-7YGo":{"UsPMIs-ipxhEnU-1EG-RJpe":0.22741640776012562, "gcn7fm-ILbMhQ6-fcz-TndT":0.36370276228098763, "o8dgBn-1bM26Wz-8SJ-xW6u":0.55671646501523719, "wav6ZA-780SwLJ-Vj3-KCv5":0.8665187582647581}}

2
regression-test/suites/datatype_p0/nested_types/base_cases/three_level_nestedtypes_with_s3data.groovy
View File

@ -20,7 +20,7 @@ import org.apache.commons.lang3.StringUtils
suite("three_level_nestedtypes_with_s3data") {
sql """set enable_nereids_planner=false"""
sql """ set enable_fallback_to_original_planner=true;"""
sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false');"""
// this test case aim to test one-level nested type with s3 data

1
regression-test/suites/datatype_p0/nested_types/base_cases/two_level_nestedtypes_with_s3data.groovy
View File

@ -20,7 +20,6 @@ import org.apache.commons.lang3.StringUtils
suite("two_level_nestedtypes_with_s3data") {
sql """set enable_nereids_planner=false"""
sql """ set enable_fallback_to_original_planner=true;"""
sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false');"""
// this test case aim to test one-level nested type with s3 data

2
regression-test/suites/datatype_p0/nested_types/load.groovy
View File

@ -17,7 +17,7 @@
suite("load") {
// ddl begin
sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
sql """set enable_nereids_planner=false"""
def dataFile = """test_scalar_types_100.csv"""

1
regression-test/suites/datatype_p0/nested_types/query/map_functions/test_basic_map_function.groovy
View File

@ -16,7 +16,6 @@
// under the License.
suite("test_basic_map_function", "p0") {
sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false'); """
sql """set enable_nereids_planner=false"""
// ============ sum(map-value) ============
qt_sql """ SELECT "sum-map-value" """

1
regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_count.groovy
View File

@ -20,7 +20,6 @@ import org.apache.commons.lang3.StringUtils
suite("test_nested_type_with_count") {
// this test case aim to test nested type with old planner
sql """set enable_nereids_planner=false"""
sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false');"""
def table_names = ["test_array_one_level", "test_map_one_level", "test_struct_one_level"]

1
regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_literal.groovy
View File

@ -25,7 +25,6 @@ suite("test_nested_types_insert_into_with_literal", "p0") {
// old planner does not support cast empty
sql 'set enable_nereids_planner=true'
sql 'set enable_fallback_to_original_planner=false'
sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
def table_names = [
"two_level_array_array_a",

1
regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_s3.groovy
View File

@ -24,7 +24,6 @@ suite("test_nested_types_insert_into_with_s3", "p0") {
sql 'use regression_test_datatype_p0_nested_types'
sql 'set enable_nereids_planner=false'
sql 'set max_allowed_packet=4194304'
sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
String ak = getS3AK()
String sk = getS3SK()

1
regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_csv_insert_into_with_s3.groovy
View File

@ -25,7 +25,6 @@ suite("test_nestedtypes_csv_insert_into_with_s3", "p0") {
sql 'set enable_nereids_planner=false'
sql 'set max_allowed_packet=4194304'
sql 'set topn_opt_limit_threshold=10000'
sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
String ak = getS3AK()
String sk = getS3SK()

1
regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy
View File

@ -22,7 +22,6 @@ import org.codehaus.groovy.runtime.IOGroovyMethods
suite("test_nestedtypes_insert_into_select", "p0") {
sql "set enable_nereids_planner=false"
sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
// create array struct
sql "DROP TABLE IF EXISTS ast;"

1
regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.groovy
View File

@ -25,7 +25,6 @@ suite("test_nestedtypes_json_insert_into_with_s3", "p0") {
sql 'set enable_nereids_planner=false'
sql 'set max_allowed_packet=4194304'
sql 'set topn_opt_limit_threshold=10000'
sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
String ak = getS3AK()
String sk = getS3SK()

2
regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy
View File

@ -43,7 +43,7 @@ suite("test_local_tvf_with_complex_type_insertinto_doris", "p0") {
}
}
qt_sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""
// create doris table
sql """ DROP TABLE IF EXISTS ${table_name} """

1
regression-test/suites/load_p0/stream_load/test_load_with_map_nested_array.groovy
View File

@ -17,7 +17,6 @@
suite("test_load_with_map_nested_array", "p0") {
def tableName = "test_load_with_map_nested_array"
sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false'); """
sql """ DROP TABLE IF EXISTS ${tableName} """
sql """

2
regression-test/suites/query_p0/aggregate/map_agg_nested_insert_doris.groovy
View File

@ -20,7 +20,7 @@ suite("map_agg_nested_insert_doris", "p0") {
def tb_doris = "test_map_agg_nested_insert_target"
sql "DROP TABLE IF EXISTS `${tb_base}`;"
sql "DROP TABLE IF EXISTS `${tb_doris}`;"
sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false'); """
sql """
CREATE TABLE `${tb_base}` (

56
regression-test/suites/query_p0/show/test_nested_complex_switch.groovy
View File

@ -137,63 +137,7 @@ suite("test_nested_complex_switch", "query") {
sql "DROP TABLE IF EXISTS ${testTable_m}"
sql "DROP TABLE IF EXISTS ${testTable_a}"
sql "DROP TABLE IF EXISTS ${testTable_s}"
sql "ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'true')"
// map
test {
sql sql_m_s
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: MAP<TEXT,STRUCT<f1:TINYINT>>"
}
test {
sql sql_m_a
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: MAP<ARRAY<INT>,TEXT>"
}
test {
sql sql_m_m
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: MAP<TEXT,MAP<TEXT,INT>>"
}
// array
test {
sql sql_a_s
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: ARRAY<STRUCT<f1:TINYINT>>"
}
test {
sql sql_a_m
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: ARRAY<MAP<TEXT,INT>>"
}
// struct
test {
sql sql_s_s
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported"
}
test {
sql sql_s_a
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported"
}
test {
sql sql_s_m
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported"
}
} finally {
try_sql("DROP TABLE IF EXISTS ${testTable_m}")
try_sql("DROP TABLE IF EXISTS ${testTable_a}")
try_sql("DROP TABLE IF EXISTS ${testTable_s}")
}
try {
sql "DROP TABLE IF EXISTS ${testTable_m}"
sql "DROP TABLE IF EXISTS ${testTable_a}"
sql "DROP TABLE IF EXISTS ${testTable_s}"
sql "ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"
// map