database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](catalog) fix potential catalog cache dead lock #34609 (#34614)

Browse Source 
bp #34609
This commit is contained in:
Mingyu Chen
2024-05-11 18:06:56 +08:00
committed by GitHub
parent e23a89f0da
commit 11360b27a2
4 changed files with 22 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
View File

@ -225,9 +225,7 @@ public abstract class ExternalCatalog
initLocalObjects();
if (!initialized) {
if (useMetaCache.get()) {
if (metaCache != null) {
metaCache.invalidateAll();
} else {
if (metaCache == null) {
metaCache = Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache(
name,
OptionalLong.of(86400L),
@ -344,7 +342,6 @@ public abstract class ExternalCatalog
dbId = dbNameToId.get(dbName);
tmpDbNameToId.put(dbName, dbId);
ExternalDatabase<? extends ExternalTable> db = idToDb.get(dbId);
db.setUnInitialized(invalidCacheInInit);
tmpIdToDb.put(dbId, db);
initCatalogLog.addRefreshDb(dbId);
} else {
@ -379,6 +376,15 @@ public abstract class ExternalCatalog
synchronized (this.propLock) {
this.convertedProperties = null;
}
if (useMetaCache.isPresent()) {
if (useMetaCache.get() && metaCache != null) {
metaCache.invalidateAll();
} else if (!useMetaCache.get()) {
for (ExternalDatabase<? extends ExternalTable> db : idToDb.values()) {
db.setUnInitialized(invalidCache);
}
}
}
this.invalidCacheInInit = invalidCache;
if (invalidCache) {
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalogCache(id);
@ -586,7 +592,6 @@ public abstract class ExternalCatalog
// Because replyInitCatalog can only be called when `use_meta_cache` is false.
// And if `use_meta_cache` is false, getDbForReplay() will not return null
Preconditions.checkNotNull(db.get());
db.get().setUnInitialized(invalidCacheInInit);
tmpDbNameToId.put(db.get().getFullName(), db.get().getId());
tmpIdToDb.put(db.get().getId(), db.get());
}

15
fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
View File

@ -118,6 +118,15 @@ public abstract class ExternalDatabase<T extends ExternalTable>
public void setUnInitialized(boolean invalidCache) {
this.initialized = false;
this.invalidCacheInInit = invalidCache;
if (extCatalog.getUseMetaCache().isPresent()) {
if (extCatalog.getUseMetaCache().get() && metaCache != null) {
metaCache.invalidateAll();
} else if (!extCatalog.getUseMetaCache().get()) {
for (T table : idToTbl.values()) {
table.unsetObjectCreated();
}
}
}
if (invalidCache) {
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDbCache(extCatalog.getId(), name);
}
@ -131,9 +140,7 @@ public abstract class ExternalDatabase<T extends ExternalTable>
extCatalog.makeSureInitialized();
if (!initialized) {
if (extCatalog.getUseMetaCache().get()) {
if (metaCache != null) {
metaCache.invalidateAll();
} else {
if (metaCache == null) {
metaCache = Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache(
name,
OptionalLong.of(86400L),
@ -175,7 +182,6 @@ public abstract class ExternalDatabase<T extends ExternalTable>
// So we need add a validation here to avoid table(s) not found, this is just a temporary solution
// because later we will remove all the logics about InitCatalogLog/InitDatabaseLog.
if (table.isPresent()) {
table.get().unsetObjectCreated();
tmpTableNameToId.put(table.get().getName(), table.get().getId());
tmpIdToTbl.put(table.get().getId(), table.get());
}
@ -206,7 +212,6 @@ public abstract class ExternalDatabase<T extends ExternalTable>
tblId = tableNameToId.get(tableName);
tmpTableNameToId.put(tableName, tblId);
T table = idToTbl.get(tblId);
table.unsetObjectCreated();
tmpIdToTbl.put(tblId, table);
initDatabaseLog.addRefreshTable(tblId);
} else {