database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Improve](complex-type)add switch for array/struct/map nesting complex type (#19928)

Browse Source 
Now we not support array/map/struct nesting each other for many action in be , If we do prohibit it in fe, we will meet many undefined action in be , so I just add switch to prohibit nesting complex type . When we fully support , can make it able.
Issue Number: close #xxx
This commit is contained in:
amory
2023-05-24 11:39:53 +08:00
committed by GitHub
parent cf7a74f6ec
commit b4669eaeba
3 changed files with 271 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fe/fe-common/src/main/java/org/apache/doris/common/Config.java
View File

@ -1932,6 +1932,13 @@ public class Config extends ConfigBase {
@ConfField(mutable = true)
public static boolean disable_datev1 = true;
/**
* Now we not fully support array/struct/map nesting complex type in many situation,
* so just disable creating nesting complex data type when create table.
* We can make it able after we fully support
*/
@ConfField(mutable = true)
public static boolean disable_nested_complex_type = true;
/*
* "max_instance_num" is used to set the maximum concurrency. When the value set
* by "parallel_fragment_exec_instance_num" is greater than "max_instance_num",

23
fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
View File

@ -28,6 +28,7 @@ import org.apache.doris.catalog.StructField;
import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.thrift.TColumnDesc;
import org.apache.doris.thrift.TPrimitiveType;
@ -123,17 +124,29 @@ public class TypeDef implements ParseNode {
}
if (type.isComplexType()) {
// now we not support array / map / struct nesting complex type
if (type.isArrayType()) {
Type itemType = ((ArrayType) type).getItemType();
if (itemType instanceof ScalarType) {
analyzeNestedType(type, (ScalarType) itemType);
} else if (Config.disable_nested_complex_type && !(itemType instanceof ArrayType)) {
// now we can array nesting array
throw new AnalysisException("Unsupported data type: ARRAY<" + itemType.toSql() + ">");
}
}
if (type.isMapType()) {
ScalarType keyType = (ScalarType) ((MapType) type).getKeyType();
ScalarType valueType = (ScalarType) ((MapType) type).getKeyType();
analyzeNestedType(type, keyType);
analyzeNestedType(type, valueType);
MapType mt = (MapType) type;
if (Config.disable_nested_complex_type && (!(mt.getKeyType() instanceof ScalarType)
|| !(mt.getValueType() instanceof ScalarType))) {
throw new AnalysisException("Unsupported data type: MAP<" + mt.getKeyType().toSql() + ","
+ mt.getValueType().toSql() + ">");
}
if (mt.getKeyType() instanceof ScalarType) {
analyzeNestedType(type, (ScalarType) mt.getKeyType());
}
if (mt.getValueType() instanceof ScalarType) {
analyzeNestedType(type, (ScalarType) mt.getValueType());
}
}
if (type.isStructType()) {
ArrayList<StructField> fields = ((StructType) type).getFields();
@ -146,6 +159,8 @@ public class TypeDef implements ParseNode {
throw new AnalysisException("Duplicate field name "
+ field.getName() + " in struct " + type.toSql());
}
} else if (Config.disable_nested_complex_type) {
throw new AnalysisException("Unsupported field type: " + fieldType.toSql() + " for STRUCT");
}
}
}

245
regression-test/suites/query_p0/show/test_nested_complex_switch.groovy Normal file
View File

@ -0,0 +1,245 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_nested_complex_switch", "query") {
// define a sql table
def testTable_m = "test_nested_complex_switch_map"
def testTable_a = "test_nested_complex_switch_array"
def testTable_s = "test_nested_complex_switch_struct"
def sql_m_s = """CREATE TABLE IF NOT EXISTS ${testTable_m} (
`k1` INT(11) NULL,
`k2` MAP<STRING,STRUCT<F1:TINYINT(4)>>
) ENGINE=OLAP
DUPLICATE KEY(`k1`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
"storage_format" = "V2",
"disable_auto_compaction" = "false"
)"""
def sql_m_a = """CREATE TABLE IF NOT EXISTS ${testTable_m} (
`k1` INT(11) NULL,
`k2` MAP<ARRAY<INT>, STRING>
) ENGINE=OLAP
DUPLICATE KEY(`k1`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
"storage_format" = "V2",
"disable_auto_compaction" = "false"
)"""
def sql_m_m = """CREATE TABLE IF NOT EXISTS ${testTable_m} (
`k1` INT(11) NULL,
`k2` MAP<STRING,MAP<STRING, INT>>
) ENGINE=OLAP
DUPLICATE KEY(`k1`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
"storage_format" = "V2",
"disable_auto_compaction" = "false"
)"""
def sql_a_s = """CREATE TABLE IF NOT EXISTS ${testTable_a} (
`k1` INT(11) NULL,
`k2` ARRAY<STRUCT<F1:TINYINT(4)>>
) ENGINE=OLAP
DUPLICATE KEY(`k1`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
"storage_format" = "V2",
"disable_auto_compaction" = "false"
)"""
def sql_a_m = """CREATE TABLE IF NOT EXISTS ${testTable_a} (
`k1` INT(11) NULL,
`k2` ARRAY<MAP<STRING, INT>>
) ENGINE=OLAP
DUPLICATE KEY(`k1`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
"storage_format" = "V2",
"disable_auto_compaction" = "false"
)"""
def sql_s_s = """CREATE TABLE IF NOT EXISTS ${testTable_s} (
`k1` INT(11) NULL,
`k2` STRUCT<F1:STRUCT<F11:BOOLEAN>,F2:TINYINT(4),F3:ARRAY<INT(11)>>
) ENGINE=OLAP
DUPLICATE KEY(`k1`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
"storage_format" = "V2",
"disable_auto_compaction" = "false"
)"""
def sql_s_a = """CREATE TABLE IF NOT EXISTS ${testTable_s} (
`k1` INT(11) NULL,
`k2` STRUCT<F2:TINYINT(4),F3:ARRAY<INT(11)>>
) ENGINE=OLAP
DUPLICATE KEY(`k1`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
"storage_format" = "V2",
"disable_auto_compaction" = "false"
)"""
def sql_s_m = """CREATE TABLE IF NOT EXISTS ${testTable_s} (
`k1` INT(11) NULL,
`k2` STRUCT<F1:TINYINT(4), F2:MAP<BOOLEAN,TINYINT(4)>>
) ENGINE=OLAP
DUPLICATE KEY(`k1`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
"storage_format" = "V2",
"disable_auto_compaction" = "false"
)"""
try {
sql "DROP TABLE IF EXISTS ${testTable_m}"
sql "DROP TABLE IF EXISTS ${testTable_a}"
sql "DROP TABLE IF EXISTS ${testTable_s}"
sql "ADMIN SET FRONTEND CONFIG ('enable_map_type' = 'true')"
sql "ADMIN SET FRONTEND CONFIG ('enable_struct_type' = 'true')"
sql "ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'true')"
// map
test {
sql sql_m_s
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: MAP<text,STRUCT<f1:tinyint(4)>>"
}
test {
sql sql_m_a
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: MAP<array<int(11)>,text>"
}
test {
sql sql_m_m
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: MAP<text,MAP<text,int(11)>>"
}
// array
test {
sql sql_a_s
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: ARRAY<STRUCT<f1:tinyint(4)>>"
}
test {
sql sql_a_m
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported data type: ARRAY<MAP<text,int(11)>>"
}
// struct
test {
sql sql_s_s
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported field type: STRUCT<f11:boolean> for STRUCT"
}
test {
sql sql_s_a
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported field type: array<int(11)> for STRUCT"
}
test {
sql sql_s_m
exception "java.sql.SQLException: errCode = 2, detailMessage = Unsupported field type: MAP<boolean,tinyint(4)> for STRUCT"
}
} finally {
try_sql("DROP TABLE IF EXISTS ${testTable_m}")
try_sql("DROP TABLE IF EXISTS ${testTable_a}")
try_sql("DROP TABLE IF EXISTS ${testTable_s}")
}
try {
sql "DROP TABLE IF EXISTS ${testTable_m}"
sql "DROP TABLE IF EXISTS ${testTable_a}"
sql "DROP TABLE IF EXISTS ${testTable_s}"
sql "ADMIN SET FRONTEND CONFIG ('enable_map_type' = 'true')"
sql "ADMIN SET FRONTEND CONFIG ('enable_struct_type' = 'true')"
sql "ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"
// map
test {
sql sql_m_s
}
test {
sql sql_m_a
}
test {
sql sql_m_m
}
// array
test {
sql sql_a_s
}
test {
sql sql_a_m
}
// struct
test {
sql sql_s_s
}
test {
sql sql_s_a
}
test {
sql sql_s_m
}
} finally {
try_sql("DROP TABLE IF EXISTS ${testTable_m}")
try_sql("DROP TABLE IF EXISTS ${testTable_a}")
try_sql("DROP TABLE IF EXISTS ${testTable_s}")
}
}