database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[refactor](catalog) set "use_meta_cache" default to true (#38244)(#38352)(#38619) (#38355)

Browse Source 
bp #38244 #38352 #38619

---------

Co-authored-by: Yulei-Yang <yulei.yang0699@gmail.com>
This commit is contained in:
Mingyu Chen
2024-08-02 14:13:38 +08:00
committed by GitHub
parent 9b07cd2069
commit da7b2cf578
10 changed files with 47 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
View File

@ -662,9 +662,9 @@ public class Util {
}
}
// Only used for external table's id generation
// And the table's id must >=0, see DescriptorTable.toThrift()
public static long genTableIdByName(String tblName) {
return Math.abs(sha256long(tblName));
// Only used for external db/table's id generation
// And the db/table's id must >=0, see DescriptorTable.toThrift()
public static long genIdByName(String... names) {
return Math.abs(sha256long(String.join(".", names)));
}
}

6
fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
View File

@ -665,7 +665,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
long tblId;
HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog;
if (hmsCatalog.getUseMetaCache().get()) {
tblId = Util.genTableIdByName(tableName);
tblId = Util.genIdByName(catalogName, dbName, tableName);
} else {
tblId = Env.getCurrentEnv().getExternalMetaIdMgr().getTblId(catalog.getId(), dbName, tableName);
}
@ -703,7 +703,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
((HMSExternalCatalog) catalog).unregisterDatabase(dbName);
}
public void registerExternalDatabase(String dbName, String catalogName, boolean ignoreIfExists)
public void registerExternalDatabaseFromEvent(String dbName, String catalogName, boolean ignoreIfExists)
throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
@ -723,7 +723,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog;
long dbId;
if (hmsCatalog.getUseMetaCache().get()) {
dbId = Util.genTableIdByName(dbName);
dbId = Util.genIdByName(catalogName, dbName);
} else {
dbId = Env.getCurrentEnv().getExternalMetaIdMgr().getDbId(catalog.getId(), dbName);
}

19
fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
View File

@ -99,8 +99,7 @@ public abstract class ExternalCatalog
public static final String DORIS_VERSION = "doris.version";
public static final String DORIS_VERSION_VALUE = Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH;
public static final String USE_META_CACHE = "use_meta_cache";
// Set default value to false to be compatible with older version meta data.
public static final boolean DEFAULT_USE_META_CACHE = false;
public static final boolean DEFAULT_USE_META_CACHE = true;
// Unique id of this catalog, will be assigned after catalog is loaded.
@SerializedName(value = "id")
@ -240,7 +239,7 @@ public abstract class ExternalCatalog
Config.max_hive_table_cache_num,
ignored -> getFilteredDatabaseNames(),
dbName -> Optional.ofNullable(
buildDbForInit(dbName, Util.genTableIdByName(dbName), logType)),
buildDbForInit(dbName, Util.genIdByName(name, dbName), logType)),
(key, value, cause) -> value.ifPresent(v -> v.setUnInitialized(invalidCacheInInit)));
}
setLastUpdateTime(System.currentTimeMillis());
@ -289,6 +288,12 @@ public abstract class ExternalCatalog
throw new DdlException("Invalid properties: " + CatalogMgr.METADATA_REFRESH_INTERVAL_SEC);
}
}
if (properties.getOrDefault(ExternalCatalog.USE_META_CACHE, "true").equals("false")) {
LOG.warn("force to set use_meta_cache to true for catalog: {} when creating", name);
getCatalogProperty().addProperty(ExternalCatalog.USE_META_CACHE, "true");
useMetaCache = Optional.of(true);
}
}
/**
@ -497,7 +502,9 @@ public abstract class ExternalCatalog
}
if (useMetaCache.get()) {
return metaCache.getMetaObj(realDbName).orElse(null);
// must use full qualified name to generate id.
// otherwise, if 2 catalogs have the same db name, the id will be the same.
return metaCache.getMetaObj(realDbName, Util.genIdByName(getQualifiedName(realDbName))).orElse(null);
} else {
if (dbNameToId.containsKey(realDbName)) {
return idToDb.get(dbNameToId.get(realDbName));
@ -854,4 +861,8 @@ public abstract class ExternalCatalog
throw e;
}
}
public String getQualifiedName(String dbName) {
return String.join(".", name, dbName);
}
}

11
fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
View File

@ -148,7 +148,8 @@ public abstract class ExternalDatabase<T extends ExternalTable>
Config.max_hive_table_cache_num,
ignored -> listTableNames(),
tableName -> Optional.ofNullable(
buildTableForInit(tableName, Util.genTableIdByName(tableName), extCatalog)),
buildTableForInit(tableName,
Util.genIdByName(extCatalog.getName(), name, tableName), extCatalog)),
(key, value, cause) -> value.ifPresent(ExternalTable::unsetObjectCreated));
}
setLastUpdateTime(System.currentTimeMillis());
@ -375,7 +376,9 @@ public abstract class ExternalDatabase<T extends ExternalTable>
public T getTableNullable(String tableName) {
makeSureInitialized();
if (extCatalog.getUseMetaCache().get()) {
return metaCache.getMetaObj(tableName).orElse(null);
// must use full qualified name to generate id.
// otherwise, if 2 databases have the same table name, the id will be the same.
return metaCache.getMetaObj(tableName, Util.genIdByName(getQualifiedName(tableName))).orElse(null);
} else {
if (!tableNameToId.containsKey(tableName)) {
return null;
@ -493,4 +496,8 @@ public abstract class ExternalDatabase<T extends ExternalTable>
setLastUpdateTime(System.currentTimeMillis());
return true;
}
public String getQualifiedName(String tblName) {
return String.join(".", extCatalog.getName(), name, tblName);
}
}

2
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java
View File

@ -88,7 +88,7 @@ public class AlterDatabaseEvent extends MetastoreEvent {
return;
}
Env.getCurrentEnv().getCatalogMgr().unregisterExternalDatabase(dbBefore.getName(), catalogName, true);
Env.getCurrentEnv().getCatalogMgr().registerExternalDatabase(dbAfter.getName(), catalogName, true);
Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(dbAfter.getName(), catalogName, true);
}

2
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
View File

@ -55,7 +55,7 @@ public class CreateDatabaseEvent extends MetastoreEvent {
protected void process() throws MetastoreNotificationException {
try {
infoLog("catalogName:[{}],dbName:[{}]", catalogName, dbName);
Env.getCurrentEnv().getCatalogMgr().registerExternalDatabase(dbName, catalogName, true);
Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(dbName, catalogName, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"), e);

7
fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
View File

@ -18,7 +18,6 @@
package org.apache.doris.datasource.metacache;
import org.apache.doris.common.CacheFactory;
import org.apache.doris.common.util.Util;
import com.github.benmanes.caffeine.cache.CacheLoader;
import com.github.benmanes.caffeine.cache.LoadingCache;
@ -75,12 +74,12 @@ public class MetaCache<T> {
return namesCache.get("");
}
public Optional<T> getMetaObj(String name) {
public Optional<T> getMetaObj(String name, long id) {
Optional<T> val = metaObjCache.getIfPresent(name);
if (val == null) {
synchronized (metaObjCache) {
val = metaObjCache.get(name);
idToName.put(Util.genTableIdByName(name), name);
idToName.put(id, name);
}
}
return val;
@ -88,7 +87,7 @@ public class MetaCache<T> {
public Optional<T> getMetaObjById(long id) {
String name = idToName.get(id);
return name == null ? Optional.empty() : getMetaObj(name);
return name == null ? Optional.empty() : getMetaObj(name, id);
}
public void updateCache(String objName, T obj) {

3
fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java
View File

@ -91,6 +91,9 @@ public class RefreshDbTest extends TestWithFeService {
}
long l3 = db1.getLastUpdateTime();
Assertions.assertTrue(l3 == l2);
// when use_meta_cache is true, the table will be recreated after refresh.
// so we need to get table again
table = db1.getTable("tbl11").get();
Assertions.assertFalse(table.isObjectCreated());
test1.getDbNullable("db1").getTables();
Assertions.assertFalse(table.isObjectCreated());

5
fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
View File

@ -144,9 +144,12 @@ public class RefreshCatalogTest extends TestWithFeService {
// not triggered init method
long l3 = test2.getLastUpdateTime();
Assertions.assertTrue(l3 == l2);
// when use_meta_cache is true, the table will be recreated after refresh.
// so we need to get table again
table = (TestExternalTable) test2.getDbNullable("db1").getTable("tbl11").get();
Assertions.assertFalse(table.isObjectCreated());
test2.getDbNullable("db1").getTables();
// Assertions.assertFalse(table.isObjectCreated());
Assertions.assertFalse(table.isObjectCreated());
try {
DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt);
} catch (Exception e) {

4
regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy
View File

@ -48,6 +48,10 @@ suite("test_iceberg_table_stats", "p0,external,doris,external_docker,external_do
Thread.sleep(2000)
retry++
}
if (cnt != act) {
def result2 = sql """select * from ${table_name} order by 1 limit 10;"""
print result2
}
assertEquals(act, cnt)
}