From 22f85be71266679a6ca4de7657af99cbe7f2ef4e Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Fri, 17 May 2024 22:52:43 +0800 Subject: [PATCH] [fix](hive-ctas) support create hive table with full quolified name (#34984) Before, when executing `create table hive.db.table as select` to create table in hive catalog, if current catalog is not hive catalog, the default engine name will be filled with `olap`, which is wrong. This PR will fill the default engine name base on specified catalog. --- .../plans/commands/info/CreateTableInfo.java | 17 ++- .../hive/ddl/test_hive_ctas.out | 121 +++++++++++------- .../hive/ddl/test_hive_ctas.groovy | 14 ++ 3 files changed, 103 insertions(+), 49 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index 585d1da0b1..3d4607517d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -40,6 +40,7 @@ import org.apache.doris.common.util.AutoBucketUtils; import org.apache.doris.common.util.InternalDatabaseUtil; import org.apache.doris.common.util.ParseUtil; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.es.EsUtil; import org.apache.doris.datasource.hive.HMSExternalCatalog; @@ -548,16 +549,21 @@ public class CreateTableInfo { } private void paddingEngineName(String ctlName, ConnectContext ctx) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(ctlName)); if (Strings.isNullOrEmpty(engineName)) { - if (InternalCatalog.INTERNAL_CATALOG_NAME.equals(ctlName)) { + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(ctlName); + if (catalog == null) { + throw new AnalysisException("Unknown catalog: " + ctlName); + } + + if (catalog instanceof InternalCatalog) { engineName = "olap"; - } else if (ctx.getCurrentCatalog() instanceof HMSExternalCatalog) { + } else if (catalog instanceof HMSExternalCatalog) { engineName = "hive"; - } else if (ctx.getCurrentCatalog() instanceof IcebergExternalCatalog) { + } else if (catalog instanceof IcebergExternalCatalog) { engineName = "iceberg"; } else { - // set to olap by default - engineName = "olap"; + throw new AnalysisException("Current catalog does not support create table: " + ctlName); } } } @@ -782,3 +788,4 @@ public class CreateTableInfo { this.isExternal = isExternal; } } + diff --git a/regression-test/data/external_table_p0/hive/ddl/test_hive_ctas.out b/regression-test/data/external_table_p0/hive/ddl/test_hive_ctas.out index 39d711b2df..d9fa227a59 100644 --- a/regression-test/data/external_table_p0/hive/ddl/test_hive_ctas.out +++ b/regression-test/data/external_table_p0/hive/ddl/test_hive_ctas.out @@ -39,51 +39,13 @@ \N 11 value_for_pt1 \N 22 value_for_pt11 --- !ctas_types_01 -- -true 127 32767 2147483647 9223372036854775807 default 22.12345 3.141592653 99999.9999 default default 2023-05-29 2023-05-29T23:19:34 +-- !qualified_table1 -- +11 value_for_pt1 +22 value_for_pt11 --- !ctas_types_02 -- -true 127 32767 2147483647 default 22.12345 3.141592653 99999.9999 default - --- !ctas_01 -- -2 -3 - --- !ctas_02 -- -2 -3 - --- !ctas_03 -- -22 value_for_pt11 value_for_pt22 - --- !ctas_04 -- -22 value_for_pt11 value_for_pt22 - --- !ctas_05 -- -11 value_for_pt1 value_for_pt2 -22 value_for_pt11 value_for_pt22 - --- !ctas_06 -- -11 value_for_pt1 value_for_pt2 -22 value_for_pt11 value_for_pt22 - --- !ctas_ex01 -- -2 -3 - --- !ctas_ex02 -- -11 value_for_pt1 value_for_pt2 -22 value_for_pt11 \N -22 value_for_pt11 value_for_pt22 - --- !ctas_03 -- -\N another string value for col2 -\N string value for col2 -\N yet another string value for col2 - --- !ctas_04 -- -\N 11 value_for_pt1 -\N 22 value_for_pt11 +-- !qualified_table2 -- +11 value_for_pt1 +22 value_for_pt11 -- !ctas_types_01 -- true 127 32767 2147483647 9223372036854775807 default 22.12345 3.141592653 99999.9999 default default 2023-05-29 2023-05-29T23:19:34 @@ -131,6 +93,14 @@ true 127 32767 2147483647 default 22.12345 3.141592653 99999.9999 default \N 11 value_for_pt1 \N 22 value_for_pt11 +-- !qualified_table1 -- +11 value_for_pt1 +22 value_for_pt11 + +-- !qualified_table2 -- +11 value_for_pt1 +22 value_for_pt11 + -- !ctas_types_01 -- true 127 32767 2147483647 9223372036854775807 default 22.12345 3.141592653 99999.9999 default default 2023-05-29 2023-05-29T23:19:34 @@ -177,8 +147,71 @@ true 127 32767 2147483647 default 22.12345 3.141592653 99999.9999 default \N 11 value_for_pt1 \N 22 value_for_pt11 +-- !qualified_table1 -- +11 value_for_pt1 +22 value_for_pt11 + +-- !qualified_table2 -- +11 value_for_pt1 +22 value_for_pt11 + -- !ctas_types_01 -- true 127 32767 2147483647 9223372036854775807 default 22.12345 3.141592653 99999.9999 default default 2023-05-29 2023-05-29T23:19:34 -- !ctas_types_02 -- true 127 32767 2147483647 default 22.12345 3.141592653 99999.9999 default + +-- !ctas_01 -- +2 +3 + +-- !ctas_02 -- +2 +3 + +-- !ctas_03 -- +22 value_for_pt11 value_for_pt22 + +-- !ctas_04 -- +22 value_for_pt11 value_for_pt22 + +-- !ctas_05 -- +11 value_for_pt1 value_for_pt2 +22 value_for_pt11 value_for_pt22 + +-- !ctas_06 -- +11 value_for_pt1 value_for_pt2 +22 value_for_pt11 value_for_pt22 + +-- !ctas_ex01 -- +2 +3 + +-- !ctas_ex02 -- +11 value_for_pt1 value_for_pt2 +22 value_for_pt11 \N +22 value_for_pt11 value_for_pt22 + +-- !ctas_03 -- +\N another string value for col2 +\N string value for col2 +\N yet another string value for col2 + +-- !ctas_04 -- +\N 11 value_for_pt1 +\N 22 value_for_pt11 + +-- !qualified_table1 -- +11 value_for_pt1 +22 value_for_pt11 + +-- !qualified_table2 -- +11 value_for_pt1 +22 value_for_pt11 + +-- !ctas_types_01 -- +true 127 32767 2147483647 9223372036854775807 default 22.12345 3.141592653 99999.9999 default default 2023-05-29 2023-05-29T23:19:34 + +-- !ctas_types_02 -- +true 127 32767 2147483647 default 22.12345 3.141592653 99999.9999 default + diff --git a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ctas.groovy b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ctas.groovy index 3bead6d0a3..d7d9b33c47 100644 --- a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ctas.groovy +++ b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ctas.groovy @@ -392,6 +392,20 @@ suite("test_hive_ctas", "p0,external,hive,external_docker,external_docker_hive") exception "errCode = 2, detailMessage = insert into cols should be corresponding to the query output" } sql """ DROP TABLE IF EXISTS ${catalog_name}.test_no_err.ctas_o2 """ + + // test ctas with qualified table name + sql """drop table if exists ${catalog_name}.test_no_err.qualified_table1""" + sql """use internal.test_ctas_olap""" + sql """create table ${catalog_name}.test_no_err.qualified_table1 as SELECT col1,pt1 as col2 FROM ${catalog_name}.test_ctas.part_ctas_src WHERE col1>0;""" + order_qt_qualified_table1 """select * from ${catalog_name}.test_no_err.qualified_table1""" + + sql """drop table if exists ${catalog_name}.test_no_err.qualified_table2""" + sql """switch ${catalog_name}""" + sql """create table test_no_err.qualified_table2 as SELECT col1,pt1 as col2 FROM ${catalog_name}.test_ctas.part_ctas_src WHERE col1>0;""" + order_qt_qualified_table2 """select * from ${catalog_name}.test_no_err.qualified_table2""" + + sql """drop table if exists ${catalog_name}.test_no_err.qualified_table1""" + sql """drop table if exists ${catalog_name}.test_no_err.qualified_table2""" sql """ DROP DATABASE IF EXISTS test_no_err """ } finally {