database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](ctas) fix NPE when ctas with old planner and varchar issue (#39744) (#39871)

Browse Source 
bp #39744
This commit is contained in:
Mingyu Chen
2024-08-24 09:24:47 +08:00
committed by GitHub
parent b9da934b16
commit de2e8f0ae6
4 changed files with 60 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View File

@ -1307,8 +1307,9 @@ public class InternalCatalog implements CatalogIf<Database> {
if (resultExpr.getSrcSlotRef() != null
&& resultExpr.getSrcSlotRef().getTable() != null
&& !resultExpr.getSrcSlotRef().getTable().isManagedTable()) {
if (createTableStmt.getPartitionDesc().inIdentifierPartitions(
resultExpr.getSrcSlotRef().getColumnName())
if ((createTableStmt.getPartitionDesc() != null
&& createTableStmt.getPartitionDesc().inIdentifierPartitions(
resultExpr.getSrcSlotRef().getColumnName()))
|| (createTableStmt.getDistributionDesc() != null
&& createTableStmt.getDistributionDesc().inDistributionColumns(
resultExpr.getSrcSlotRef().getColumnName()))) {

17
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java
View File

@ -123,6 +123,8 @@ public class CreateTableCommand extends Command implements ForwardWithSync {
Slot s = slots.get(i);
DataType dataType = s.getDataType().conversion();
if (i == 0 && dataType.isStringType()) {
// first column of olap table can not be string type.
// So change it to varchar type.
dataType = VarcharType.createVarcharType(ScalarType.MAX_VARCHAR_LENGTH);
} else {
dataType = TypeCoercionUtils.replaceSpecifiedType(dataType,
@ -135,13 +137,21 @@ public class CreateTableCommand extends Command implements ForwardWithSync {
if (createTableInfo.getPartitionTableInfo().inIdentifierPartitions(s.getName())
|| (createTableInfo.getDistribution() != null
&& createTableInfo.getDistribution().inDistributionColumns(s.getName()))) {
// String type can not be used in partition/distributed column
// String type can not be used in partition/distributed column,
// so we replace it to varchar
dataType = TypeCoercionUtils.replaceSpecifiedType(dataType,
CharacterType.class, VarcharType.MAX_VARCHAR_TYPE);
} else {
dataType = TypeCoercionUtils.replaceSpecifiedType(dataType,
CharacterType.class, StringType.INSTANCE);
if (i == 0) {
// first column of olap table can not be string type.
// So change it to varchar type.
dataType = TypeCoercionUtils.replaceSpecifiedType(dataType,
CharacterType.class, VarcharType.MAX_VARCHAR_TYPE);
} else {
// change varchar/char column from external table to string type
dataType = TypeCoercionUtils.replaceSpecifiedType(dataType,
CharacterType.class, StringType.INSTANCE);
}
}
}
} else {
@ -212,3 +222,4 @@ public class CreateTableCommand extends Command implements ForwardWithSync {
return createTableInfo;
}
}

18
regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out
View File

@ -433,3 +433,21 @@ doris
-- !sql --
-- !sql --
int_u bigint Yes true \N
text varchar(65533) Yes true \N
t2 text Yes false \N NONE
-- !sql --
varchar varchar(65533) Yes true \N
int_u bigint Yes false \N NONE
-- !sql --
int_u bigint Yes true \N
text varchar(65533) Yes true \N
t2 varchar(65533) Yes false \N NONE
-- !sql --
varchar varchar(65533) Yes true \N
int_u bigint Yes false \N NONE

25
regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy
View File

@ -23,6 +23,7 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-java-8.0.25.jar"
// String driver_url = "mysql-connector-java-8.0.25.jar"
if (enabled != null && enabled.equalsIgnoreCase("true")) {
String user = "test_jdbc_user";
String pwd = '123456';
@ -614,6 +615,30 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc
order_qt_sql """select * from mysql_conjuncts.doris_test.text_push where pk <=7;"""
// test create table as select
sql """use internal.${internal_db_name}"""
sql """drop table if exists ctas_partition_text_1"""
sql """drop table if exists ctas_partition_text_2"""
sql """drop table if exists ctas_partition_text_3"""
sql """drop table if exists ctas_partition_text_4"""
sql """set enable_nereids_planner=true"""
// 1. test text type column as distribution col
sql """create table ctas_partition_text_1 distributed by hash(text) buckets 1 properties("replication_num" = "1") as select int_u, text, text as t2 from mysql_conjuncts.doris_test.all_types;"""
qt_sql """desc ctas_partition_text_1"""
// 2. test varchar type column as first col
sql """create table ctas_partition_text_2 distributed by hash(int_u) buckets 1 properties("replication_num" = "1") as select varchar, int_u from mysql_conjuncts.doris_test.all_types;"""
qt_sql """desc ctas_partition_text_2"""
// ctas logic is different between new and old planner.
// so need to test both.
// the old planner's test can be removed once the old planner is removed.
sql """set enable_nereids_planner=false"""
// 1. test text type column as distribution col
sql """create table ctas_partition_text_3 distributed by hash(text) buckets 1 properties("replication_num" = "1") as select int_u, text, text as t2 from mysql_conjuncts.doris_test.all_types;"""
qt_sql """desc ctas_partition_text_3"""
// 2. test varchar type column as first col
sql """create table ctas_partition_text_4 distributed by hash(int_u) buckets 1 properties("replication_num" = "1") as select varchar, int_u from mysql_conjuncts.doris_test.all_types;"""
qt_sql """desc ctas_partition_text_4"""
sql """drop catalog if exists mysql_conjuncts;"""
}
}