database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](external) process tbl/db exist when create/drop db/tbl (#33119)

Browse Source 
Issue Number: #31442
This commit is contained in:
slothever
2024-04-03 10:01:57 +08:00
committed by morningman
parent 7a05396cd1
commit f0ac21e231
8 changed files with 198 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
View File

@ -27,6 +27,8 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.JdbcResource;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.ExternalDatabase;
import org.apache.doris.datasource.jdbc.client.JdbcClient;
@ -40,7 +42,6 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -99,6 +100,14 @@ public class HiveMetadataOps implements ExternalMetadataOps {
String fullDbName = stmt.getFullDbName();
Map<String, String> properties = stmt.getProperties();
long dbId = Env.getCurrentEnv().getNextId();
if (databaseExist(fullDbName)) {
if (stmt.isSetIfNotExists()) {
LOG.info("create database[{}] which already exists", fullDbName);
return;
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_DB_CREATE_EXISTS, fullDbName);
}
}
try {
HiveDatabaseMetadata catalogDatabase = new HiveDatabaseMetadata();
catalogDatabase.setDbName(fullDbName);
@ -119,6 +128,14 @@ public class HiveMetadataOps implements ExternalMetadataOps {
@Override
public void dropDb(DropDbStmt stmt) throws DdlException {
String dbName = stmt.getDbName();
if (!databaseExist(dbName)) {
if (stmt.isSetIfExists()) {
LOG.info("drop database[{}] which does not exist", dbName);
return;
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_DB_DROP_EXISTS, dbName);
}
}
try {
client.dropDatabase(dbName);
catalog.onRefresh(true);
@ -135,6 +152,14 @@ public class HiveMetadataOps implements ExternalMetadataOps {
if (db == null) {
throw new UserException("Failed to get database: '" + dbName + "' in catalog: " + catalog.getName());
}
if (tableExist(dbName, tblName)) {
if (stmt.isSetIfNotExists()) {
LOG.info("create table[{}] which already exists", tblName);
return;
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tblName);
}
}
try {
Map<String, String> props = stmt.getProperties();
String fileFormat = props.getOrDefault(FILE_FORMAT_KEY, Config.hive_default_file_format);
@ -186,17 +211,6 @@ public class HiveMetadataOps implements ExternalMetadataOps {
}
}
private static List<FieldSchema> parsePartitionKeys(Map<String, String> props) {
List<FieldSchema> parsedKeys = new ArrayList<>();
String pkStr = props.getOrDefault("partition_keys", "");
if (pkStr.isEmpty()) {
return parsedKeys;
} else {
// TODO: parse string to partition keys list
return parsedKeys;
}
}
@Override
public void dropTable(DropTableStmt stmt) throws DdlException {
String dbName = stmt.getDbName();
@ -204,6 +218,14 @@ public class HiveMetadataOps implements ExternalMetadataOps {
if (db == null) {
throw new DdlException("Failed to get database: '" + dbName + "' in catalog: " + catalog.getName());
}
if (!tableExist(dbName, stmt.getTableName())) {
if (stmt.isSetIfExists()) {
LOG.info("drop table[{}] which does not exist", dbName);
return;
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_TABLE, stmt.getTableName(), dbName);
}
}
try {
client.dropTable(dbName, stmt.getTableName());
db.setUnInitialized(true);
@ -222,6 +244,11 @@ public class HiveMetadataOps implements ExternalMetadataOps {
return client.tableExists(dbName, tblName);
}
@Override
public boolean databaseExist(String dbName) {
return listDatabaseNames().contains(dbName);
}
public List<String> listDatabaseNames() {
return client.getAllDatabases();
}

12
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveUtil.java
View File

@ -29,6 +29,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
@ -255,4 +256,15 @@ public final class HiveUtil {
}
return Pair.of(hiveCols, hiveParts);
}
public static Database toHiveDatabase(HiveDatabaseMetadata hiveDb) {
Database database = new Database();
database.setName(hiveDb.getDbName());
if (StringUtils.isNotEmpty(hiveDb.getLocationUri())) {
database.setLocationUri(hiveDb.getLocationUri());
}
database.setParameters(hiveDb.getProperties());
database.setDescription(hiveDb.getComment());
return database;
}
}

14
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java
View File

@ -31,7 +31,6 @@ import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.ValidReaderWriteIdList;
import org.apache.hadoop.hive.common.ValidTxnList;
@ -137,7 +136,7 @@ public class ThriftHMSCachedClient implements HMSCachedClient {
if (db instanceof HiveDatabaseMetadata) {
HiveDatabaseMetadata hiveDb = (HiveDatabaseMetadata) db;
ugiDoAs(() -> {
client.client.createDatabase(toHiveDatabase(hiveDb));
client.client.createDatabase(HiveUtil.toHiveDatabase(hiveDb));
return null;
});
}
@ -150,17 +149,6 @@ public class ThriftHMSCachedClient implements HMSCachedClient {
}
}
private static Database toHiveDatabase(HiveDatabaseMetadata hiveDb) {
Database database = new Database();
database.setName(hiveDb.getDbName());
if (StringUtils.isNotEmpty(hiveDb.getLocationUri())) {
database.setLocationUri(hiveDb.getLocationUri());
}
database.setParameters(hiveDb.getProperties());
database.setDescription(hiveDb.getComment());
return database;
}
@Override
public void createTable(TableMetadata tbl, boolean ignoreIfExists) {
if (tableExists(tbl.getDbName(), tbl.getTableName())) {

38
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
View File

@ -25,6 +25,8 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.StructField;
import org.apache.doris.catalog.StructType;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.DorisTypeVisitor;
import org.apache.doris.datasource.ExternalCatalog;
@ -67,6 +69,10 @@ public class IcebergMetadataOps implements ExternalMetadataOps {
return catalog.tableExists(TableIdentifier.of(dbName, tblName));
}
public boolean databaseExist(String dbName) {
return nsCatalog.namespaceExists(Namespace.of(dbName));
}
public List<String> listDatabaseNames() {
return nsCatalog.listNamespaces().stream()
.map(e -> e.toString())
@ -84,6 +90,14 @@ public class IcebergMetadataOps implements ExternalMetadataOps {
SupportsNamespaces nsCatalog = (SupportsNamespaces) catalog;
String dbName = stmt.getFullDbName();
Map<String, String> properties = stmt.getProperties();
if (databaseExist(dbName)) {
if (stmt.isSetIfNotExists()) {
LOG.info("create database[{}] which already exists", dbName);
return;
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_DB_CREATE_EXISTS, dbName);
}
}
nsCatalog.createNamespace(Namespace.of(dbName), properties);
dorisCatalog.onRefresh(true);
}
@ -97,6 +111,14 @@ public class IcebergMetadataOps implements ExternalMetadataOps {
dorisCatalog.getIdToDb().remove(aLong);
dorisCatalog.getDbNameToId().remove(dbName);
}
if (!databaseExist(dbName)) {
if (stmt.isSetIfExists()) {
LOG.info("drop database[{}] which does not exist", dbName);
return;
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_DB_DROP_EXISTS, dbName);
}
}
nsCatalog.dropNamespace(Namespace.of(dbName));
dorisCatalog.onRefresh(true);
}
@ -109,6 +131,14 @@ public class IcebergMetadataOps implements ExternalMetadataOps {
throw new UserException("Failed to get database: '" + dbName + "' in catalog: " + dorisCatalog.getName());
}
String tableName = stmt.getTableName();
if (tableExist(dbName, tableName)) {
if (stmt.isSetIfNotExists()) {
LOG.info("create table[{}] which already exists", tableName);
return;
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName);
}
}
List<Column> columns = stmt.getColumns();
List<StructField> collect = columns.stream()
.map(col -> new StructField(col.getName(), col.getType(), col.getComment(), col.isAllowNull()))
@ -132,6 +162,14 @@ public class IcebergMetadataOps implements ExternalMetadataOps {
throw new DdlException("Failed to get database: '" + dbName + "' in catalog: " + dorisCatalog.getName());
}
String tableName = stmt.getTableName();
if (!tableExist(dbName, tableName)) {
if (stmt.isSetIfExists()) {
LOG.info("drop table[{}] which does not exist", dbName);
return;
} else {
ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_TABLE, tableName, dbName);
}
}
catalog.dropTable(TableIdentifier.of(dbName, tableName));
db.setUnInitialized(true);
}

2
fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
View File

@ -79,4 +79,6 @@ public interface ExternalMetadataOps {
* @return
*/
boolean tableExist(String dbName, String tblName);
boolean databaseExist(String dbName);
}

103
fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.analysis.CreateCatalogStmt;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DbName;
import org.apache.doris.analysis.DropDbStmt;
import org.apache.doris.analysis.HashDistributionDesc;
import org.apache.doris.analysis.SwitchStmt;
import org.apache.doris.catalog.Column;
@ -29,6 +30,9 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.Config;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.FeConstants;
import org.apache.doris.datasource.DatabaseMetadata;
import org.apache.doris.datasource.ExternalDatabase;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.TableMetadata;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.StatementContext;
@ -52,6 +56,8 @@ import org.apache.doris.utframe.TestWithFeService;
import mockit.Mock;
import mockit.MockUp;
import mockit.Mocked;
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
import org.junit.jupiter.api.Assertions;
@ -66,6 +72,7 @@ import java.util.Optional;
import java.util.Set;
public class HiveDDLAndDMLPlanTest extends TestWithFeService {
private static final String mockedCtlName = "hive";
private static final String mockedDbName = "mockedDb";
private final NereidsParser nereidsParser = new NereidsParser();
@ -74,6 +81,9 @@ public class HiveDDLAndDMLPlanTest extends TestWithFeService {
private List<FieldSchema> checkedHiveCols;
private final Set<String> createdDbs = new HashSet<>();
private final Set<String> createdTables = new HashSet<>();
@Override
protected void runBeforeAll() throws Exception {
connectContext.getSessionVariable().enableFallbackToOriginalPlanner = false;
@ -104,7 +114,8 @@ public class HiveDDLAndDMLPlanTest extends TestWithFeService {
createTable(createSourceInterPTable, true);
// create external catalog and switch it
CreateCatalogStmt hiveCatalog = createStmt("create catalog hive properties('type' = 'hms',"
CreateCatalogStmt hiveCatalog = createStmt("create catalog " + mockedCtlName
+ " properties('type' = 'hms',"
+ " 'hive.metastore.uris' = 'thrift://192.168.0.1:9083');");
Env.getCurrentEnv().getCatalogMgr().createCatalog(hiveCatalog);
switchHive();
@ -113,19 +124,37 @@ public class HiveDDLAndDMLPlanTest extends TestWithFeService {
Map<String, String> dbProps = new HashMap<>();
dbProps.put(HiveMetadataOps.LOCATION_URI_KEY, "file://loc/db");
new MockUp<ThriftHMSCachedClient>(ThriftHMSCachedClient.class) {
@Mock
public void createDatabase(DatabaseMetadata db) {
if (db instanceof HiveDatabaseMetadata) {
Database hiveDb = HiveUtil.toHiveDatabase((HiveDatabaseMetadata) db);
createdDbs.add(hiveDb.getName());
}
}
@Mock
public Database getDatabase(String dbName) {
if (createdDbs.contains(dbName)) {
return new Database(dbName, "", "", null);
}
return null;
}
@Mock
public boolean tableExists(String dbName, String tblName) {
return createdTables.contains(tblName);
}
@Mock
public List<String> getAllDatabases() {
return new ArrayList<String>() {
{
add(mockedDbName);
}
};
return new ArrayList<>(createdDbs);
}
@Mock
public void createTable(TableMetadata tbl, boolean ignoreIfExists) {
if (tbl instanceof HiveTableMetadata) {
Table table = HiveUtil.toHiveTable((HiveTableMetadata) tbl);
createdTables.add(table.getTableName());
if (checkedHiveCols == null) {
// if checkedHiveCols is null, skip column check
return;
@ -143,6 +172,8 @@ public class HiveDDLAndDMLPlanTest extends TestWithFeService {
};
CreateDbStmt createDbStmt = new CreateDbStmt(true, new DbName("hive", mockedDbName), dbProps);
Env.getCurrentEnv().createDb(createDbStmt);
// checkout ifNotExists
Env.getCurrentEnv().createDb(createDbStmt);
useDatabase(mockedDbName);
// un-partitioned table
@ -167,18 +198,29 @@ public class HiveDDLAndDMLPlanTest extends TestWithFeService {
createTable(createSourceExtTable, true);
HMSExternalCatalog hmsExternalCatalog = (HMSExternalCatalog) Env.getCurrentEnv().getCatalogMgr()
.getCatalog("hive");
.getCatalog(mockedCtlName);
new MockUp<HMSExternalCatalog>(HMSExternalCatalog.class) {
// mock after ThriftHMSCachedClient is mocked
@Mock
public ExternalDatabase<? extends ExternalTable> getDbNullable(String dbName) {
if (createdDbs.contains(dbName)) {
return new HMSExternalDatabase(hmsExternalCatalog, RandomUtils.nextLong(), dbName);
}
return null;
}
};
new MockUp<HMSExternalDatabase>(HMSExternalDatabase.class) {
// mock after ThriftHMSCachedClient is mocked
@Mock
HMSExternalTable getTableNullable(String tableName) {
return new HMSExternalTable(0, tableName, mockedDbName, hmsExternalCatalog);
if (createdTables.contains(tableName)) {
return new HMSExternalTable(0, tableName, mockedDbName, hmsExternalCatalog);
}
return null;
}
};
new MockUp<HMSExternalTable>(HMSExternalTable.class) {
@Mock
protected synchronized void makeSureInitialized() {
// mocked
}
// mock after ThriftHMSCachedClient is mocked
};
}
@ -195,7 +237,38 @@ public class HiveDDLAndDMLPlanTest extends TestWithFeService {
@Override
protected void runAfterAll() throws Exception {
switchHive();
dropDatabase(mockedDbName);
String createDbStmtStr = "DROP DATABASE IF EXISTS " + mockedDbName;
DropDbStmt createDbStmt = (DropDbStmt) parseAndAnalyzeStmt(createDbStmtStr);
Env.getCurrentEnv().dropDb(createDbStmt);
// check IF EXISTS
Env.getCurrentEnv().dropDb(createDbStmt);
}
@Test
public void testExistsDbOrTbl() throws Exception {
switchHive();
String db = "exists_db";
String createDbStmtStr = "CREATE DATABASE IF NOT EXISTS " + db;
createDatabaseWithSql(createDbStmtStr);
createDatabaseWithSql(createDbStmtStr);
useDatabase(db);
String createTableIfNotExists = "CREATE TABLE IF NOT EXISTS test_tbl(\n"
+ " `col1` BOOLEAN COMMENT 'col1',"
+ " `col2` INT COMMENT 'col2'"
+ ") ENGINE=hive\n"
+ "PROPERTIES (\n"
+ " 'location_uri'='hdfs://loc/db/tbl',\n"
+ " 'file_format'='orc')";
createTable(createTableIfNotExists, true);
createTable(createTableIfNotExists, true);
dropTableWithSql("DROP TABLE IF EXISTS test_tbl");
dropTableWithSql("DROP TABLE IF EXISTS test_tbl");
String dropDbStmtStr = "DROP DATABASE IF EXISTS " + db;
dropDatabaseWithSql(dropDbStmtStr);
dropDatabaseWithSql(dropDbStmtStr);
}
@Test
@ -220,7 +293,7 @@ public class HiveDDLAndDMLPlanTest extends TestWithFeService {
createTable(createUnPartTable, true);
dropTable("unpart_tbl", true);
String createPartTable = "CREATE TABLE `part_tbl`(\n"
String createPartTable = "CREATE TABLE IF NOT EXISTS `part_tbl`(\n"
+ " `col1` BOOLEAN COMMENT 'col1',\n"
+ " `col2` INT COMMENT 'col2',\n"
+ " `col3` BIGINT COMMENT 'col3',\n"
@ -235,6 +308,8 @@ public class HiveDDLAndDMLPlanTest extends TestWithFeService {
+ " 'location_uri'='hdfs://loc/db/tbl',\n"
+ " 'file_format'='parquet')";
createTable(createPartTable, true);
// check IF NOT EXISTS
createTable(createPartTable, true);
dropTable("part_tbl", true);
String createBucketedTableErr = "CREATE TABLE `err_buck_tbl`(\n"

7
fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java
View File

@ -142,6 +142,13 @@ public class HiveMetadataOpsTest {
@Test
public void testCreateAndDropAll() throws UserException {
new MockUp<HMSExternalDatabase>(HMSExternalDatabase.class) {
// create table if getTableNullable return null
@Mock
HMSExternalTable getTableNullable(String tableName) {
return null;
}
};
Map<String, String> dbProps = new HashMap<>();
dbProps.put(HiveMetadataOps.LOCATION_URI_KEY, "file://loc/db");
createDb("mockedDb", dbProps);

10
fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
View File

@ -605,6 +605,11 @@ public abstract class TestWithFeService {
Env.getCurrentEnv().dropDb(createDbStmt);
}
public void dropDatabaseWithSql(String dropDbSql) throws Exception {
DropDbStmt dropDbStmt = (DropDbStmt) parseAndAnalyzeStmt(dropDbSql);
Env.getCurrentEnv().dropDb(dropDbStmt);
}
public void useDatabase(String dbName) {
connectContext.setDatabase(dbName);
}
@ -653,6 +658,11 @@ public abstract class TestWithFeService {
Env.getCurrentEnv().dropTable(dropTableStmt);
}
public void dropTableWithSql(String dropTableSql) throws Exception {
DropTableStmt dropTableStmt = (DropTableStmt) parseAndAnalyzeStmt(dropTableSql);
Env.getCurrentEnv().dropTable(dropTableStmt);
}
public void recoverTable(String table) throws Exception {
RecoverTableStmt recoverTableStmt = (RecoverTableStmt) parseAndAnalyzeStmt(
"recover table " + table + ";", connectContext);