database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](external catalog)Optimize the process of refreshing catalog for 2.1 (#39205) (#39186)

Browse Source 
## Proposed changes

bp: #39205

When the catalog attributes have not changed, refreshing the catalog
only requires processing the cache, without rebuilding the entire
catalog.
This commit is contained in:
wuwenchi
2024-08-17 17:02:06 +08:00
committed by GitHub
parent 273a62584c
commit e01d051acf
6 changed files with 19 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
View File

@ -76,7 +76,7 @@ public class RefreshManager {
private void refreshCatalogInternal(CatalogIf catalog, boolean invalidCache) {
String catalogName = catalog.getName();
if (!catalogName.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
((ExternalCatalog) catalog).onRefresh(invalidCache);
((ExternalCatalog) catalog).onRefreshCache(invalidCache);
LOG.info("refresh catalog {} with invalidCache {}", catalogName, invalidCache);
}
}

9
fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
View File

@ -388,6 +388,15 @@ public abstract class ExternalCatalog
synchronized (this.propLock) {
this.convertedProperties = null;
}
refreshOnlyCatalogCache(invalidCache);
}
public void onRefreshCache(boolean invalidCache) {
refreshOnlyCatalogCache(invalidCache);
}
private void refreshOnlyCatalogCache(boolean invalidCache) {
if (useMetaCache.isPresent()) {
if (useMetaCache.get() && metaCache != null) {
metaCache.invalidateAll();

4
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
View File

@ -126,7 +126,7 @@ public class HiveMetadataOps implements ExternalMetadataOps {
catalogDatabase.setProperties(properties);
catalogDatabase.setComment(properties.getOrDefault("comment", ""));
client.createDatabase(catalogDatabase);
catalog.onRefresh(true);
catalog.onRefreshCache(true);
} catch (Exception e) {
throw new RuntimeException(e.getMessage(), e);
}
@ -146,7 +146,7 @@ public class HiveMetadataOps implements ExternalMetadataOps {
}
try {
client.dropDatabase(dbName);
catalog.onRefresh(true);
catalog.onRefreshCache(true);
} catch (Exception e) {
throw new RuntimeException(e.getMessage(), e);
}

4
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
View File

@ -107,7 +107,7 @@ public class IcebergMetadataOps implements ExternalMetadataOps {
}
}
nsCatalog.createNamespace(Namespace.of(dbName), properties);
dorisCatalog.onRefresh(true);
dorisCatalog.onRefreshCache(true);
}
@Override
@ -123,7 +123,7 @@ public class IcebergMetadataOps implements ExternalMetadataOps {
}
SupportsNamespaces nsCatalog = (SupportsNamespaces) catalog;
nsCatalog.dropNamespace(Namespace.of(dbName));
dorisCatalog.onRefresh(true);
dorisCatalog.onRefreshCache(true);
}
@Override

5
fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
View File

@ -121,6 +121,11 @@ public class JdbcExternalCatalog extends ExternalCatalog {
}
}
@Override
public void onRefreshCache(boolean invalidCache) {
onRefresh(invalidCache);
}
@Override
public void onClose() {
super.onClose();