From a32a2d8bb4030b620ddc0af87a906e39e627aea7 Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Wed, 28 Aug 2024 21:45:03 +0800 Subject: [PATCH] [bugfix](iceberg)Restrictions on creating a database (#39641) (#40051) bp #39641 Co-authored-by: wuwenchi --- .../iceberg/IcebergGlueExternalCatalog.java | 2 +- .../iceberg/IcebergHMSExternalCatalog.java | 2 +- .../iceberg/IcebergHadoopExternalCatalog.java | 2 +- .../iceberg/IcebergMetadataOps.java | 5 ++ .../iceberg/IcebergRestExternalCatalog.java | 2 +- .../iceberg/test_iceberg_show_create.groovy | 86 ++++++++++++------- 6 files changed, 65 insertions(+), 34 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java index 9f3e05757c..ffe48e68a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java @@ -58,7 +58,7 @@ public class IcebergGlueExternalCatalog extends IcebergExternalCatalog { catalogProperties.get(S3Properties.Env.ENDPOINT)); catalogProperties.putIfAbsent(S3FileIOProperties.ENDPOINT, endpoint); - glueCatalog.initialize(icebergCatalogType, catalogProperties); + glueCatalog.initialize(getName(), catalogProperties); catalog = glueCatalog; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java index 34e6f0c187..c147506493 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java @@ -44,7 +44,7 @@ public class IcebergHMSExternalCatalog extends IcebergExternalCatalog { Map catalogProperties = catalogProperty.getProperties(); String metastoreUris = catalogProperty.getOrDefault(HMSProperties.HIVE_METASTORE_URIS, ""); catalogProperties.put(CatalogProperties.URI, metastoreUris); - hiveCatalog.initialize(icebergCatalogType, catalogProperties); + hiveCatalog.initialize(getName(), catalogProperties); catalog = hiveCatalog; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java index dffdc454e1..5053e129ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java @@ -61,7 +61,7 @@ public class IcebergHadoopExternalCatalog extends IcebergExternalCatalog { String warehouse = catalogProperty.getHadoopProperties().get(CatalogProperties.WAREHOUSE_LOCATION); hadoopCatalog.setConf(conf); catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse); - hadoopCatalog.initialize(icebergCatalogType, catalogProperties); + hadoopCatalog.initialize(getName(), catalogProperties); catalog = hadoopCatalog; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index dd4792715e..36654a8e67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -106,6 +106,11 @@ public class IcebergMetadataOps implements ExternalMetadataOps { ErrorReport.reportDdlException(ErrorCode.ERR_DB_CREATE_EXISTS, dbName); } } + String icebergCatalogType = dorisCatalog.getIcebergCatalogType(); + if (!properties.isEmpty() && !IcebergExternalCatalog.ICEBERG_HMS.equals(icebergCatalogType)) { + throw new DdlException( + "Not supported: create database with properties for iceberg catalog type: " + icebergCatalogType); + } nsCatalog.createNamespace(Namespace.of(dbName), properties); dorisCatalog.onRefreshCache(true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java index 77a6a7404e..908a4fa9e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java @@ -47,7 +47,7 @@ public class IcebergRestExternalCatalog extends IcebergExternalCatalog { Configuration conf = replaceS3Properties(getConfiguration()); - catalog = CatalogUtil.buildIcebergCatalog(icebergCatalogType, + catalog = CatalogUtil.buildIcebergCatalog(getName(), convertToRestCatalogProperties(), conf); } diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy index 8065998fa7..23705c5494 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy @@ -22,11 +22,58 @@ suite("test_iceberg_show_create", "p0,external,doris,external_docker,external_do return } + String catalog_name = "test_iceberg_show_create" + String hivePrefix = "hive2"; + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") + String iceberg_catalog_name = "test_iceberg_write_partitions_iceberg_${hivePrefix}" + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String default_fs = "hdfs://${externalEnvIp}:${hdfs_port}" + String warehouse = "${default_fs}/warehouse" + + sql """drop catalog if exists ${catalog_name}""" + sql """create catalog if not exists ${catalog_name} properties ( + 'type'='iceberg', + 'iceberg.catalog.type'='hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}', + 'fs.defaultFS' = '${default_fs}', + 'warehouse' = '${warehouse}', + 'use_meta_cache' = 'true' + );""" + + sql """ switch ${catalog_name} """ + + String db1 = "test_db1" + String db2 = "test_db2" + String tb1 = "test_tb1" + + sql """ drop table if exists ${db1}.${tb1} """ + sql """ drop database if exists ${db1} """ + sql """ drop database if exists ${db2} """ + + sql """ create database ${db1} properties ('location'='${warehouse}/other_location') """ + sql """ create database ${db2} """ + + String result = "" + result = sql "show create database ${db1}" + logger.info("${result}") + assertTrue(result.toString().containsIgnoreCase("${warehouse}/other_location")) + + result = sql "show create database ${db2}" + logger.info("${result}") + assertTrue(result.toString().containsIgnoreCase("${warehouse}/${db2}")) + + sql """ create table ${db1}.${tb1} (id int) """ + result = sql "show create table ${db1}.${tb1}" + logger.info("${result}") + assertTrue(result.toString().containsIgnoreCase("${warehouse}/other_location/${tb1}")) + + sql """ drop table ${db1}.${tb1} """ + sql """ drop database ${db1} """ + sql """ drop database ${db2} """ + String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") String minio_port = context.config.otherConfigs.get("iceberg_minio_port") - String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") - String catalog_name = "test_iceberg_show_create" - sql """drop catalog if exists ${catalog_name}""" sql """ CREATE CATALOG ${catalog_name} PROPERTIES ( @@ -40,34 +87,13 @@ suite("test_iceberg_show_create", "p0,external,doris,external_docker,external_do );""" sql """ switch ${catalog_name} """ - - String db1 = "test_db1" - String db2 = "test_db2" - String tb1 = "test_tb1" - - sql """ drop table if exists ${db1}.${tb1} """ sql """ drop database if exists ${db1} """ - sql """ drop database if exists ${db2} """ - sql """ create database ${db1} properties ('location'='s3a://warehouse/wh/${db1}') """ - sql """ create database ${db2} """ - - String result = "" - result = sql "show create database ${db1}" - logger.info("${result}") - assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db1}")) - - result = sql "show create database ${db2}" - logger.info("${result}") - assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db2}")) - - sql """ create table ${db1}.${tb1} (id int) """ - result = sql "show create table ${db1}.${tb1}" - logger.info("${result}") - assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db1}/${tb1}")) - - sql """ drop table ${db1}.${tb1} """ - sql """ drop database ${db1} """ - sql """ drop database ${db2} """ + test { + sql """ create database ${db1} properties ('location'='${warehouse}/other_location') """ + exception "Not supported: create database with properties for iceberg catalog type" + } + sql """ drop database if exists ${db1} """ + sql """drop catalog if exists ${catalog_name}""" }