database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[bugfix](iceberg)Restrictions on creating a database (#39641) (#40051)

Browse Source 
bp #39641

Co-authored-by: wuwenchi <wuwenchihdu@hotmail.com>
This commit is contained in:
Mingyu Chen
2024-08-28 21:45:03 +08:00
committed by GitHub
parent d1c3b857cf
commit a32a2d8bb4
6 changed files with 65 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java
View File

@ -58,7 +58,7 @@ public class IcebergGlueExternalCatalog extends IcebergExternalCatalog {
catalogProperties.get(S3Properties.Env.ENDPOINT));
catalogProperties.putIfAbsent(S3FileIOProperties.ENDPOINT, endpoint);
glueCatalog.initialize(icebergCatalogType, catalogProperties);
glueCatalog.initialize(getName(), catalogProperties);
catalog = glueCatalog;
}

2
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java
View File

@ -44,7 +44,7 @@ public class IcebergHMSExternalCatalog extends IcebergExternalCatalog {
Map<String, String> catalogProperties = catalogProperty.getProperties();
String metastoreUris = catalogProperty.getOrDefault(HMSProperties.HIVE_METASTORE_URIS, "");
catalogProperties.put(CatalogProperties.URI, metastoreUris);
hiveCatalog.initialize(icebergCatalogType, catalogProperties);
hiveCatalog.initialize(getName(), catalogProperties);
catalog = hiveCatalog;
}
}

2
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java
View File

@ -61,7 +61,7 @@ public class IcebergHadoopExternalCatalog extends IcebergExternalCatalog {
String warehouse = catalogProperty.getHadoopProperties().get(CatalogProperties.WAREHOUSE_LOCATION);
hadoopCatalog.setConf(conf);
catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
hadoopCatalog.initialize(icebergCatalogType, catalogProperties);
hadoopCatalog.initialize(getName(), catalogProperties);
catalog = hadoopCatalog;
}
}

5
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
View File

@ -106,6 +106,11 @@ public class IcebergMetadataOps implements ExternalMetadataOps {
ErrorReport.reportDdlException(ErrorCode.ERR_DB_CREATE_EXISTS, dbName);
}
}
String icebergCatalogType = dorisCatalog.getIcebergCatalogType();
if (!properties.isEmpty() && !IcebergExternalCatalog.ICEBERG_HMS.equals(icebergCatalogType)) {
throw new DdlException(
"Not supported: create database with properties for iceberg catalog type: " + icebergCatalogType);
}
nsCatalog.createNamespace(Namespace.of(dbName), properties);
dorisCatalog.onRefreshCache(true);
}

2
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java
View File

@ -47,7 +47,7 @@ public class IcebergRestExternalCatalog extends IcebergExternalCatalog {
Configuration conf = replaceS3Properties(getConfiguration());
catalog = CatalogUtil.buildIcebergCatalog(icebergCatalogType,
catalog = CatalogUtil.buildIcebergCatalog(getName(),
convertToRestCatalogProperties(),
conf);
}

86
regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy
View File

@ -22,11 +22,58 @@ suite("test_iceberg_show_create", "p0,external,doris,external_docker,external_do
return
}
String catalog_name = "test_iceberg_show_create"
String hivePrefix = "hive2";
String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort")
String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort")
String iceberg_catalog_name = "test_iceberg_write_partitions_iceberg_${hivePrefix}"
String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
String default_fs = "hdfs://${externalEnvIp}:${hdfs_port}"
String warehouse = "${default_fs}/warehouse"
sql """drop catalog if exists ${catalog_name}"""
sql """create catalog if not exists ${catalog_name} properties (
'type'='iceberg',
'iceberg.catalog.type'='hms',
'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
'fs.defaultFS' = '${default_fs}',
'warehouse' = '${warehouse}',
'use_meta_cache' = 'true'
);"""
sql """ switch ${catalog_name} """
String db1 = "test_db1"
String db2 = "test_db2"
String tb1 = "test_tb1"
sql """ drop table if exists ${db1}.${tb1} """
sql """ drop database if exists ${db1} """
sql """ drop database if exists ${db2} """
sql """ create database ${db1} properties ('location'='${warehouse}/other_location') """
sql """ create database ${db2} """
String result = ""
result = sql "show create database ${db1}"
logger.info("${result}")
assertTrue(result.toString().containsIgnoreCase("${warehouse}/other_location"))
result = sql "show create database ${db2}"
logger.info("${result}")
assertTrue(result.toString().containsIgnoreCase("${warehouse}/${db2}"))
sql """ create table ${db1}.${tb1} (id int) """
result = sql "show create table ${db1}.${tb1}"
logger.info("${result}")
assertTrue(result.toString().containsIgnoreCase("${warehouse}/other_location/${tb1}"))
sql """ drop table ${db1}.${tb1} """
sql """ drop database ${db1} """
sql """ drop database ${db2} """
String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port")
String minio_port = context.config.otherConfigs.get("iceberg_minio_port")
String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
String catalog_name = "test_iceberg_show_create"
sql """drop catalog if exists ${catalog_name}"""
sql """
CREATE CATALOG ${catalog_name} PROPERTIES (
@ -40,34 +87,13 @@ suite("test_iceberg_show_create", "p0,external,doris,external_docker,external_do
);"""
sql """ switch ${catalog_name} """
String db1 = "test_db1"
String db2 = "test_db2"
String tb1 = "test_tb1"
sql """ drop table if exists ${db1}.${tb1} """
sql """ drop database if exists ${db1} """
sql """ drop database if exists ${db2} """
sql """ create database ${db1} properties ('location'='s3a://warehouse/wh/${db1}') """
sql """ create database ${db2} """
String result = ""
result = sql "show create database ${db1}"
logger.info("${result}")
assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db1}"))
result = sql "show create database ${db2}"
logger.info("${result}")
assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db2}"))
sql """ create table ${db1}.${tb1} (id int) """
result = sql "show create table ${db1}.${tb1}"
logger.info("${result}")
assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db1}/${tb1}"))
sql """ drop table ${db1}.${tb1} """
sql """ drop database ${db1} """
sql """ drop database ${db2} """
test {
sql """ create database ${db1} properties ('location'='${warehouse}/other_location') """
exception "Not supported: create database with properties for iceberg catalog type"
}
sql """ drop database if exists ${db1} """
sql """drop catalog if exists ${catalog_name}"""
}