diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java index 79b5450877..050ab7774e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java @@ -662,9 +662,9 @@ public class Util { } } - // Only used for external table's id generation - // And the table's id must >=0, see DescriptorTable.toThrift() - public static long genTableIdByName(String tblName) { - return Math.abs(sha256long(tblName)); + // Only used for external db/table's id generation + // And the db/table's id must >=0, see DescriptorTable.toThrift() + public static long genIdByName(String... names) { + return Math.abs(sha256long(String.join(".", names))); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index b2c60f2cc1..a8aa34cf6c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -665,7 +665,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable { long tblId; HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; if (hmsCatalog.getUseMetaCache().get()) { - tblId = Util.genTableIdByName(tableName); + tblId = Util.genIdByName(catalogName, dbName, tableName); } else { tblId = Env.getCurrentEnv().getExternalMetaIdMgr().getTblId(catalog.getId(), dbName, tableName); } @@ -703,7 +703,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable { ((HMSExternalCatalog) catalog).unregisterDatabase(dbName); } - public void registerExternalDatabase(String dbName, String catalogName, boolean ignoreIfExists) + public void registerExternalDatabaseFromEvent(String dbName, String catalogName, boolean ignoreIfExists) throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); if (catalog == null) { @@ -723,7 +723,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable { HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; long dbId; if (hmsCatalog.getUseMetaCache().get()) { - dbId = Util.genTableIdByName(dbName); + dbId = Util.genIdByName(catalogName, dbName); } else { dbId = Env.getCurrentEnv().getExternalMetaIdMgr().getDbId(catalog.getId(), dbName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 35434a43cb..b6e94c3d39 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -99,8 +99,7 @@ public abstract class ExternalCatalog public static final String DORIS_VERSION = "doris.version"; public static final String DORIS_VERSION_VALUE = Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH; public static final String USE_META_CACHE = "use_meta_cache"; - // Set default value to false to be compatible with older version meta data. - public static final boolean DEFAULT_USE_META_CACHE = false; + public static final boolean DEFAULT_USE_META_CACHE = true; // Unique id of this catalog, will be assigned after catalog is loaded. @SerializedName(value = "id") @@ -240,7 +239,7 @@ public abstract class ExternalCatalog Config.max_hive_table_cache_num, ignored -> getFilteredDatabaseNames(), dbName -> Optional.ofNullable( - buildDbForInit(dbName, Util.genTableIdByName(dbName), logType)), + buildDbForInit(dbName, Util.genIdByName(name, dbName), logType)), (key, value, cause) -> value.ifPresent(v -> v.setUnInitialized(invalidCacheInInit))); } setLastUpdateTime(System.currentTimeMillis()); @@ -289,6 +288,12 @@ public abstract class ExternalCatalog throw new DdlException("Invalid properties: " + CatalogMgr.METADATA_REFRESH_INTERVAL_SEC); } } + + if (properties.getOrDefault(ExternalCatalog.USE_META_CACHE, "true").equals("false")) { + LOG.warn("force to set use_meta_cache to true for catalog: {} when creating", name); + getCatalogProperty().addProperty(ExternalCatalog.USE_META_CACHE, "true"); + useMetaCache = Optional.of(true); + } } /** @@ -497,7 +502,9 @@ public abstract class ExternalCatalog } if (useMetaCache.get()) { - return metaCache.getMetaObj(realDbName).orElse(null); + // must use full qualified name to generate id. + // otherwise, if 2 catalogs have the same db name, the id will be the same. + return metaCache.getMetaObj(realDbName, Util.genIdByName(getQualifiedName(realDbName))).orElse(null); } else { if (dbNameToId.containsKey(realDbName)) { return idToDb.get(dbNameToId.get(realDbName)); @@ -854,4 +861,8 @@ public abstract class ExternalCatalog throw e; } } + + public String getQualifiedName(String dbName) { + return String.join(".", name, dbName); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index f34be46d4b..2919633858 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -148,7 +148,8 @@ public abstract class ExternalDatabase Config.max_hive_table_cache_num, ignored -> listTableNames(), tableName -> Optional.ofNullable( - buildTableForInit(tableName, Util.genTableIdByName(tableName), extCatalog)), + buildTableForInit(tableName, + Util.genIdByName(extCatalog.getName(), name, tableName), extCatalog)), (key, value, cause) -> value.ifPresent(ExternalTable::unsetObjectCreated)); } setLastUpdateTime(System.currentTimeMillis()); @@ -375,7 +376,9 @@ public abstract class ExternalDatabase public T getTableNullable(String tableName) { makeSureInitialized(); if (extCatalog.getUseMetaCache().get()) { - return metaCache.getMetaObj(tableName).orElse(null); + // must use full qualified name to generate id. + // otherwise, if 2 databases have the same table name, the id will be the same. + return metaCache.getMetaObj(tableName, Util.genIdByName(getQualifiedName(tableName))).orElse(null); } else { if (!tableNameToId.containsKey(tableName)) { return null; @@ -493,4 +496,8 @@ public abstract class ExternalDatabase setLastUpdateTime(System.currentTimeMillis()); return true; } + + public String getQualifiedName(String tblName) { + return String.join(".", extCatalog.getName(), name, tblName); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java index c4529d5d48..ef828025e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java @@ -88,7 +88,7 @@ public class AlterDatabaseEvent extends MetastoreEvent { return; } Env.getCurrentEnv().getCatalogMgr().unregisterExternalDatabase(dbBefore.getName(), catalogName, true); - Env.getCurrentEnv().getCatalogMgr().registerExternalDatabase(dbAfter.getName(), catalogName, true); + Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(dbAfter.getName(), catalogName, true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java index 8829d2ae98..2dd4c5671b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java @@ -55,7 +55,7 @@ public class CreateDatabaseEvent extends MetastoreEvent { protected void process() throws MetastoreNotificationException { try { infoLog("catalogName:[{}],dbName:[{}]", catalogName, dbName); - Env.getCurrentEnv().getCatalogMgr().registerExternalDatabase(dbName, catalogName, true); + Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(dbName, catalogName, true); } catch (DdlException e) { throw new MetastoreNotificationException( debugString("Failed to process event"), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java index da8f068dfd..c251db3a5c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java @@ -18,7 +18,6 @@ package org.apache.doris.datasource.metacache; import org.apache.doris.common.CacheFactory; -import org.apache.doris.common.util.Util; import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.LoadingCache; @@ -75,12 +74,12 @@ public class MetaCache { return namesCache.get(""); } - public Optional getMetaObj(String name) { + public Optional getMetaObj(String name, long id) { Optional val = metaObjCache.getIfPresent(name); if (val == null) { synchronized (metaObjCache) { val = metaObjCache.get(name); - idToName.put(Util.genTableIdByName(name), name); + idToName.put(id, name); } } return val; @@ -88,7 +87,7 @@ public class MetaCache { public Optional getMetaObjById(long id) { String name = idToName.get(id); - return name == null ? Optional.empty() : getMetaObj(name); + return name == null ? Optional.empty() : getMetaObj(name, id); } public void updateCache(String objName, T obj) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java index fd920e9632..fd7a6b1a24 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java @@ -91,6 +91,9 @@ public class RefreshDbTest extends TestWithFeService { } long l3 = db1.getLastUpdateTime(); Assertions.assertTrue(l3 == l2); + // when use_meta_cache is true, the table will be recreated after refresh. + // so we need to get table again + table = db1.getTable("tbl11").get(); Assertions.assertFalse(table.isObjectCreated()); test1.getDbNullable("db1").getTables(); Assertions.assertFalse(table.isObjectCreated()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java index f64e6523eb..439385993f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java @@ -144,9 +144,12 @@ public class RefreshCatalogTest extends TestWithFeService { // not triggered init method long l3 = test2.getLastUpdateTime(); Assertions.assertTrue(l3 == l2); + // when use_meta_cache is true, the table will be recreated after refresh. + // so we need to get table again + table = (TestExternalTable) test2.getDbNullable("db1").getTable("tbl11").get(); Assertions.assertFalse(table.isObjectCreated()); test2.getDbNullable("db1").getTables(); - // Assertions.assertFalse(table.isObjectCreated()); + Assertions.assertFalse(table.isObjectCreated()); try { DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); } catch (Exception e) { diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy index b8eacf6d9e..15c23fa74d 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy @@ -48,6 +48,10 @@ suite("test_iceberg_table_stats", "p0,external,doris,external_docker,external_do Thread.sleep(2000) retry++ } + if (cnt != act) { + def result2 = sql """select * from ${table_name} order by 1 limit 10;""" + print result2 + } assertEquals(act, cnt) }