diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 826b1ab861..9973de61be 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1956,7 +1956,7 @@ public class Config extends ConfigBase { * only for certain test type. E.g. only settting batch_size to small * value for p0. */ - @ConfField(mutable = true, masterOnly = false) + @ConfField(mutable = true, masterOnly = false, options = {"p0"}) public static String fuzzy_test_type = ""; /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java index fd709963a7..548119dfb7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java @@ -49,7 +49,6 @@ import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -416,6 +415,7 @@ public class Database extends MetaObject implements Writable, DatabaseIf } } + @Override public boolean registerTable(TableIf table) { boolean result = true; Table olapTable = (Table) table; @@ -851,11 +851,6 @@ public class Database extends MetaObject implements Writable, DatabaseIf
return null; } - @Override - public Map getIdToTable() { - return new HashMap<>(idToTable); - } - public void replayUpdateDbProperties(Map properties) { dbProperties.updateProperties(properties); if (PropertyAnalyzer.hasBinlogConfig(properties)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java index 6c0d46ffe9..3aae592dec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java @@ -31,7 +31,6 @@ import org.apache.logging.log4j.Logger; import java.util.Arrays; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -75,7 +74,7 @@ public interface DatabaseIf { List getTables(); - default List getTablesOrEmpty() { + default List getTablesIgnoreException() { try { return getTables(); } catch (Exception e) { @@ -281,6 +280,4 @@ public interface DatabaseIf { default long getLastUpdateTime() { return -1L; } - - Map getIdToTable(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java index 65bf7d308f..9552250369 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java @@ -24,6 +24,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.ExternalCatalog; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -125,20 +126,8 @@ public class JdbcResource extends Resource { CONNECTION_POOL_MAX_LIFE_TIME, CONNECTION_POOL_MAX_WAIT_TIME, CONNECTION_POOL_KEEP_ALIVE, - TEST_CONNECTION - ).build(); - private static final ImmutableList OPTIONAL_PROPERTIES = new ImmutableList.Builder().add( - ONLY_SPECIFIED_DATABASE, - LOWER_CASE_META_NAMES, - META_NAMES_MAPPING, - INCLUDE_DATABASE_LIST, - EXCLUDE_DATABASE_LIST, - CONNECTION_POOL_MIN_SIZE, - CONNECTION_POOL_MAX_SIZE, - CONNECTION_POOL_MAX_LIFE_TIME, - CONNECTION_POOL_MAX_WAIT_TIME, - CONNECTION_POOL_KEEP_ALIVE, - TEST_CONNECTION + TEST_CONNECTION, + ExternalCatalog.USE_META_CACHE ).build(); // The default value of optional properties @@ -157,6 +146,8 @@ public class JdbcResource extends Resource { OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MAX_WAIT_TIME, "5000"); OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_KEEP_ALIVE, "false"); OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(TEST_CONNECTION, "true"); + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ExternalCatalog.USE_META_CACHE, + String.valueOf(ExternalCatalog.DEFAULT_USE_META_CACHE)); } // timeout for both connection and read. 10 seconds is long enough. @@ -225,7 +216,7 @@ public class JdbcResource extends Resource { @Override public void applyDefaultProperties() { - for (String s : OPTIONAL_PROPERTIES) { + for (String s : OPTIONAL_PROPERTIES_DEFAULT_VALUE.keySet()) { if (!configs.containsKey(s)) { configs.put(s, OPTIONAL_PROPERTIES_DEFAULT_VALUE.get(s)); } @@ -487,3 +478,4 @@ public class JdbcResource extends Resource { } } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java index 028fe186b2..8bbdfe1d5f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java @@ -17,8 +17,6 @@ package org.apache.doris.catalog; -import org.apache.doris.alter.Alter; -import org.apache.doris.analysis.AlterTableStmt; import org.apache.doris.analysis.CreateViewStmt; import org.apache.doris.common.Pair; @@ -43,17 +41,6 @@ public abstract class MysqlCompatibleDatabase extends Database { */ protected abstract void initTables(); - /** - * Currently, rename a table of InfoSchemaDb will throw exception - * {@link Alter#processAlterTable(AlterTableStmt)} - * so we follow this design. - * @note: Rename a table of mysql database in MYSQL ls allowed. - */ - @Override - public boolean registerTable(TableIf table) { - return super.registerTable(table); - } - @Override public void unregisterTable(String name) { // Do nothing diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index 39efce4c70..d017ba7829 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -41,6 +41,7 @@ import org.apache.logging.log4j.Logger; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -108,10 +109,14 @@ public class RefreshManager { private void refreshDbInternal(long catalogId, long dbId, boolean invalidCache) { ExternalCatalog catalog = (ExternalCatalog) Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogId); - ExternalDatabase db = catalog.getDbForReplay(dbId); - db.setUnInitialized(invalidCache); - LOG.info("refresh database {} in catalog {} with invalidCache {}", db.getFullName(), catalog.getName(), - invalidCache); + Optional> db = catalog.getDbForReplay(dbId); + // Database may not exist if 'use_meta_cache' is true. + // Because each FE fetch the meta data independently. + db.ifPresent(e -> { + e.setUnInitialized(invalidCache); + LOG.info("refresh database {} in catalog {} with invalidCache {}", e.getFullName(), + catalog.getName(), invalidCache); + }); } // Refresh table @@ -163,13 +168,22 @@ public class RefreshManager { LOG.warn("failed to find catalog replaying refresh table {}", log.getCatalogId()); return; } - ExternalDatabase db = catalog.getDbForReplay(log.getDbId()); - TableIf table = db.getTableForReplay(log.getTableId()); - refreshTableInternal(catalog, db, table, log.getLastUpdateTime()); + Optional> db = catalog.getDbForReplay(log.getDbId()); + // See comment in refreshDbInternal for why db and table may be null. + if (!db.isPresent()) { + LOG.warn("failed to find db replaying refresh table {}", log.getDbId()); + return; + } + Optional table = db.get().getTableForReplay(log.getTableId()); + if (!table.isPresent()) { + LOG.warn("failed to find table replaying refresh table {}", log.getTableId()); + return; + } + refreshTableInternal(catalog, db.get(), table.get(), log.getLastUpdateTime()); } public void refreshExternalTableFromEvent(String catalogName, String dbName, String tableName, - long updateTime, boolean ignoreIfNotExists) throws DdlException { + long updateTime) throws DdlException { CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogName); if (catalog == null) { throw new DdlException("No catalog found with name: " + catalogName); @@ -179,17 +193,11 @@ public class RefreshManager { } DatabaseIf db = catalog.getDbNullable(dbName); if (db == null) { - if (!ignoreIfNotExists) { - throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName()); - } return; } TableIf table = db.getTableNullable(tableName); if (table == null) { - if (!ignoreIfNotExists) { - throw new DdlException("Table " + tableName + " does not exist in db " + dbName); - } return; } refreshTableInternal(catalog, db, table, updateTime); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java b/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java index fbb004e5c9..50f4664797 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java @@ -22,6 +22,7 @@ import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.RemovalListener; import com.github.benmanes.caffeine.cache.Ticker; import org.jetbrains.annotations.NotNull; @@ -74,9 +75,13 @@ public class CacheFactory { } // Build a loading cache, with executor, it will use given executor for refresh - public LoadingCache buildCache(CacheLoader cacheLoader, ExecutorService executor) { + public LoadingCache buildCache(CacheLoader cacheLoader, + RemovalListener removalListener, ExecutorService executor) { Caffeine builder = buildWithParams(); builder.executor(executor); + if (removalListener != null) { + builder.removalListener(removalListener); + } return builder.build(cacheLoader); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index e411e992e9..ba4434349c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -35,6 +35,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.policy.Policy; import org.apache.doris.policy.StoragePolicy; import org.apache.doris.resource.Tag; @@ -1348,6 +1349,14 @@ public class PropertyAnalyzer { throw new AnalysisException("failed to find class " + acClass, e); } } + + if (isAlter) { + // The 'use_meta_cache' property can not be modified + if (properties.containsKey(ExternalCatalog.USE_META_CACHE)) { + throw new AnalysisException("Can not modify property " + ExternalCatalog.USE_META_CACHE + + ". You need to create a new Catalog with the property."); + } + } } public static Map rewriteReplicaAllocationProperties( diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java index c454c6cab5..be3e711f45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java @@ -44,6 +44,9 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.net.URL; import java.net.URLConnection; +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; import java.util.ArrayList; import java.util.Collections; @@ -651,4 +654,21 @@ public class Util { p.printStackTrace(pw); return sw.toString(); } + + public static long sha256long(String str) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] hash = digest.digest(str.getBytes()); + ByteBuffer buffer = ByteBuffer.wrap(hash); + return buffer.getLong(); + } catch (NoSuchAlgorithmException e) { + return str.hashCode(); + } + } + + // Only used for external table's id generation + // And the table's id must >=0, see DescriptorTable.toThrift() + public static long genTableIdByName(String tblName) { + return Math.abs(sha256long(tblName)); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java index 264477d22a..23558ce576 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java @@ -142,10 +142,13 @@ public class CatalogFactory { default: throw new DdlException("Unknown catalog type: " + catalogType); } + + // set some default properties if missing when creating catalog. + // both replaying the creating logic will call this method. + catalog.setDefaultPropsIfMissing(isReplay); + if (!isReplay) { - // set some default properties when creating catalog. - // do not call this method when replaying edit log. Because we need to keey the original properties. - catalog.setDefaultPropsWhenCreating(isReplay); + catalog.checkWhenCreating(); // This will check if the customized access controller can be created successfully. // If failed, it will throw exception and the catalog will not be created. try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index 44dba04157..072597deb5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -40,7 +40,6 @@ import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import javax.annotation.Nullable; @@ -184,8 +183,6 @@ public interface CatalogIf { boolean enableAutoAnalyze(); - ConcurrentHashMap getIdToDb(); - void createDb(CreateDbStmt stmt) throws DdlException; void dropDb(DropDbStmt stmt) throws DdlException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 56b390bc7d..48fa0cc14c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -43,6 +43,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.common.util.Util; import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.mysql.privilege.PrivPredicate; @@ -66,6 +67,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -587,11 +589,11 @@ public class CatalogMgr implements Writable, GsonPostProcessable { if (catalog == null) { return; } - ExternalDatabase db = catalog.getDbForReplay(log.getDbId()); - if (db == null) { + Optional> db = catalog.getDbForReplay(log.getDbId()); + if (!db.isPresent()) { return; } - db.replayInitDb(log, catalog); + db.get().replayInitDb(log, catalog); } public void unregisterExternalTable(String dbName, String tableName, String catalogName, boolean ignoreIfExists) @@ -663,7 +665,13 @@ public class CatalogMgr implements Writable, GsonPostProcessable { } return; } - long tblId = Env.getCurrentEnv().getExternalMetaIdMgr().getTblId(catalog.getId(), dbName, tableName); + long tblId; + HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; + if (hmsCatalog.getUseMetaCache().get()) { + tblId = Util.genTableIdByName(tableName); + } else { + tblId = Env.getCurrentEnv().getExternalMetaIdMgr().getTblId(catalog.getId(), dbName, tableName); + } // -1L means it will be dropped later, ignore if (tblId == ExternalMetaIdMgr.META_ID_FOR_NOT_EXISTS) { return; @@ -715,13 +723,19 @@ public class CatalogMgr implements Writable, GsonPostProcessable { return; } - long dbId = Env.getCurrentEnv().getExternalMetaIdMgr().getDbId(catalog.getId(), dbName); + HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; + long dbId; + if (hmsCatalog.getUseMetaCache().get()) { + dbId = Env.getCurrentEnv().getExternalMetaIdMgr().getDbId(catalog.getId(), dbName); + } else { + dbId = Util.genTableIdByName(dbName); + } // -1L means it will be dropped later, ignore if (dbId == ExternalMetaIdMgr.META_ID_FOR_NOT_EXISTS) { return; } - ((HMSExternalCatalog) catalog).registerDatabase(dbId, dbName); + hmsCatalog.registerDatabase(dbId, dbName); } public void addExternalPartitions(String catalogName, String dbName, String tableName, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 6d5ae985fe..72eacbb1de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -30,6 +30,7 @@ import org.apache.doris.catalog.MysqlDb; import org.apache.doris.catalog.Resource; import org.apache.doris.catalog.TableIf; import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; import org.apache.doris.common.Version; @@ -44,6 +45,7 @@ import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase; import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase; import org.apache.doris.datasource.jdbc.JdbcExternalDatabase; import org.apache.doris.datasource.maxcompute.MaxComputeExternalDatabase; +import org.apache.doris.datasource.metacache.MetaCache; import org.apache.doris.datasource.operations.ExternalMetadataOps; import org.apache.doris.datasource.paimon.PaimonExternalDatabase; import org.apache.doris.datasource.property.PropertyConverter; @@ -54,9 +56,11 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.MasterCatalogExecutor; import org.apache.doris.transaction.TransactionManager; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.google.gson.annotations.SerializedName; import lombok.Data; import org.apache.commons.lang3.NotImplementedException; @@ -65,6 +69,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.DataInput; @@ -77,19 +82,24 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; +import java.util.OptionalLong; +import java.util.Set; +import java.util.stream.Collectors; /** * The abstract class for all types of external catalogs. */ @Data public abstract class ExternalCatalog - implements CatalogIf>, Writable, GsonPostProcessable { + implements CatalogIf>, Writable, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(ExternalCatalog.class); public static final String ENABLE_AUTO_ANALYZE = "enable.auto.analyze"; public static final String DORIS_VERSION = "doris.version"; public static final String DORIS_VERSION_VALUE = Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH; + public static final String USE_META_CACHE = "use_meta_cache"; + // Set default value to false to be compatible with older version meta data. + public static final boolean DEFAULT_USE_META_CACHE = false; // Unique id of this catalog, will be assigned after catalog is loaded. @SerializedName(value = "id") @@ -123,6 +133,9 @@ public abstract class ExternalCatalog private byte[] propLock = new byte[0]; private Map convertedProperties = null; + protected Optional useMetaCache = Optional.empty(); + protected MetaCache> metaCache; + public ExternalCatalog() { } @@ -130,7 +143,7 @@ public abstract class ExternalCatalog this.id = catalogId; this.name = name; this.logType = logType; - this.comment = com.google.common.base.Strings.nullToEmpty(comment); + this.comment = Strings.nullToEmpty(comment); } public Configuration getConfiguration() { @@ -142,11 +155,6 @@ public abstract class ExternalCatalog return conf; } - // only for test - public void setInitialized() { - initialized = true; - } - /** * set some default properties when creating catalog * @return list of database names in this catalog @@ -160,8 +168,21 @@ public abstract class ExternalCatalog } } - public void setDefaultPropsWhenCreating(boolean isReplay) throws DdlException { + // Will be called when creating catalog(so when as replaying) + // to add some default properties if missing. + public void setDefaultPropsIfMissing(boolean isReplay) { + if (catalogProperty.getOrDefault(USE_META_CACHE, "").isEmpty()) { + // If not setting USE_META_CACHE in replay logic, + // set default value to false to be compatible with older version meta data. + catalogProperty.addProperty(USE_META_CACHE, isReplay ? "false" : String.valueOf(DEFAULT_USE_META_CACHE)); + } + useMetaCache = Optional.of( + Boolean.valueOf(catalogProperty.getOrDefault(USE_META_CACHE, String.valueOf(DEFAULT_USE_META_CACHE)))); + } + // Will be called when creating catalog(not replaying). + // Subclass can override this method to do some check when creating catalog. + public void checkWhenCreating() throws DdlException { } /** @@ -204,19 +225,36 @@ public abstract class ExternalCatalog public final synchronized void makeSureInitialized() { initLocalObjects(); if (!initialized) { - if (!Env.getCurrentEnv().isMaster()) { - // Forward to master and wait the journal to replay. - int waitTimeOut = ConnectContext.get() == null ? 300 : ConnectContext.get().getExecTimeout(); - MasterCatalogExecutor remoteExecutor = new MasterCatalogExecutor(waitTimeOut * 1000); - try { - remoteExecutor.forward(id, -1); - } catch (Exception e) { - Util.logAndThrowRuntimeException(LOG, - String.format("failed to forward init catalog %s operation to master.", name), e); + if (useMetaCache.get()) { + if (metaCache != null) { + metaCache.invalidateAll(); + } else { + metaCache = Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache( + name, + OptionalLong.of(86400L), + OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), + Config.max_hive_table_cache_num, + ignored -> getFilteredDatabaseNames(), + dbName -> Optional.ofNullable( + buildDbForInit(dbName, Util.genTableIdByName(dbName), logType)), + (key, value, cause) -> value.ifPresent(v -> v.setUnInitialized(invalidCacheInInit))); } - return; + setLastUpdateTime(System.currentTimeMillis()); + } else { + if (!Env.getCurrentEnv().isMaster()) { + // Forward to master and wait the journal to replay. + int waitTimeOut = ConnectContext.get() == null ? 300 : ConnectContext.get().getExecTimeout(); + MasterCatalogExecutor remoteExecutor = new MasterCatalogExecutor(waitTimeOut * 1000); + try { + remoteExecutor.forward(id, -1); + } catch (Exception e) { + Util.logAndThrowRuntimeException(LOG, + String.format("failed to forward init catalog %s operation to master.", name), e); + } + return; + } + init(); } - init(); initialized = true; } } @@ -283,18 +321,13 @@ public abstract class ExternalCatalog } // init schema related objects - protected void init() { + private void init() { Map tmpDbNameToId = Maps.newConcurrentMap(); Map> tmpIdToDb = Maps.newConcurrentMap(); InitCatalogLog initCatalogLog = new InitCatalogLog(); initCatalogLog.setCatalogId(id); initCatalogLog.setType(logType); - List allDatabases = Lists.newArrayList(listDatabaseNames()); - - allDatabases.remove(InfoSchemaDb.DATABASE_NAME); - allDatabases.add(InfoSchemaDb.DATABASE_NAME); - allDatabases.remove(MysqlDb.DATABASE_NAME); - allDatabases.add(MysqlDb.DATABASE_NAME); + List allDatabases = getFilteredDatabaseNames(); Map includeDatabaseMap = getIncludeDatabaseMap(); Map excludeDatabaseMap = getExcludeDatabaseMap(); for (String dbName : allDatabases) { @@ -318,7 +351,7 @@ public abstract class ExternalCatalog } else { dbId = Env.getCurrentEnv().getNextId(); tmpDbNameToId.put(dbName, dbId); - ExternalDatabase db = getDbForInit(dbName, dbId, logType); + ExternalDatabase db = buildDbForInit(dbName, dbId, logType); tmpIdToDb.put(dbId, db); initCatalogLog.addCreateDb(dbId, dbName); } @@ -331,6 +364,16 @@ public abstract class ExternalCatalog Env.getCurrentEnv().getEditLog().logInitCatalog(initCatalogLog); } + @NotNull + private List getFilteredDatabaseNames() { + List allDatabases = Lists.newArrayList(listDatabaseNames()); + allDatabases.remove(InfoSchemaDb.DATABASE_NAME); + allDatabases.add(InfoSchemaDb.DATABASE_NAME); + allDatabases.remove(MysqlDb.DATABASE_NAME); + allDatabases.add(MysqlDb.DATABASE_NAME); + return allDatabases; + } + public void onRefresh(boolean invalidCache) { this.objectCreated = false; this.initialized = false; @@ -382,12 +425,18 @@ public abstract class ExternalCatalog } /** + * Different from 'listDatabases()', this method will return dbnames from cache. + * while 'listDatabases()' will return dbnames from remote datasource. * @return names of database in this catalog. */ @Override public List getDbNames() { makeSureInitialized(); - return new ArrayList<>(dbNameToId.keySet()); + if (useMetaCache.get()) { + return metaCache.listNames(); + } else { + return new ArrayList<>(dbNameToId.keySet()); + } } @Override @@ -404,14 +453,10 @@ public abstract class ExternalCatalog } } + @Override public TableName getTableNameByTableId(Long tableId) { - for (DatabaseIf db : idToDb.values()) { - TableIf table = db.getTableNullable(tableId); - if (table != null) { - return new TableName(getName(), db.getFullName(), table.getName()); - } - } - return null; + throw new UnsupportedOperationException("External catalog does not support getTableNameByTableId() method." + + ", table id: " + tableId); } @Override @@ -432,21 +477,23 @@ public abstract class ExternalCatalog return null; } String realDbName = ClusterNamespace.getNameFromFullName(dbName); - if (dbNameToId.containsKey(realDbName)) { - return idToDb.get(dbNameToId.get(realDbName)); - } else { - // This maybe a information_schema db request, and information_schema db name is case insensitive. - // So, we first extract db name to check if it is information_schema. - // Then we reassemble the origin cluster name with lower case db name, - // and finally get information_schema db from the name map. - if (realDbName.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) { - return idToDb.get(dbNameToId.get(InfoSchemaDb.DATABASE_NAME)); - } - if (realDbName.equalsIgnoreCase(MysqlDb.DATABASE_NAME)) { - return idToDb.get(dbNameToId.get(MysqlDb.DATABASE_NAME)); - } + + // information_schema db name is case-insensitive. + // So, we first convert it to standard database name. + if (realDbName.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) { + realDbName = InfoSchemaDb.DATABASE_NAME; + } else if (realDbName.equalsIgnoreCase(MysqlDb.DATABASE_NAME)) { + realDbName = MysqlDb.DATABASE_NAME; + } + + if (useMetaCache.get()) { + return metaCache.getMetaObj(realDbName).orElse(null); + } else { + if (dbNameToId.containsKey(realDbName)) { + return idToDb.get(dbNameToId.get(realDbName)); + } + return null; } - return null; } @Nullable @@ -458,13 +505,22 @@ public abstract class ExternalCatalog LOG.warn("failed to get db {} in catalog {}", dbId, name, e); return null; } - return idToDb.get(dbId); + + if (useMetaCache.get()) { + return metaCache.getMetaObjById(dbId).get(); + } else { + return idToDb.get(dbId); + } } @Override public List getDbIds() { makeSureInitialized(); - return Lists.newArrayList(dbNameToId.values()); + if (useMetaCache.get()) { + return getAllDbs().stream().map(DatabaseIf::getId).collect(Collectors.toList()); + } else { + return Lists.newArrayList(dbNameToId.values()); + } } @Override @@ -528,14 +584,18 @@ public abstract class ExternalCatalog Map tmpDbNameToId = Maps.newConcurrentMap(); Map> tmpIdToDb = Maps.newConcurrentMap(); for (int i = 0; i < log.getRefreshCount(); i++) { - ExternalDatabase db = getDbForReplay(log.getRefreshDbIds().get(i)); - db.setUnInitialized(invalidCacheInInit); - tmpDbNameToId.put(db.getFullName(), db.getId()); - tmpIdToDb.put(db.getId(), db); + Optional> db = getDbForReplay(log.getRefreshDbIds().get(i)); + // Should not return null. + // Because replyInitCatalog can only be called when `use_meta_cache` is false. + // And if `use_meta_cache` is false, getDbForReplay() will not return null + Preconditions.checkNotNull(db.get()); + db.get().setUnInitialized(invalidCacheInInit); + tmpDbNameToId.put(db.get().getFullName(), db.get().getId()); + tmpIdToDb.put(db.get().getId(), db.get()); } for (int i = 0; i < log.getCreateCount(); i++) { ExternalDatabase db = - getDbForInit(log.getCreateDbNames().get(i), log.getCreateDbIds().get(i), log.getType()); + buildDbForInit(log.getCreateDbNames().get(i), log.getCreateDbIds().get(i), log.getType()); if (db != null) { tmpDbNameToId.put(db.getFullName(), db.getId()); tmpIdToDb.put(db.getId(), db); @@ -547,12 +607,19 @@ public abstract class ExternalCatalog initialized = true; } - public ExternalDatabase getDbForReplay(long dbId) { - return idToDb.get(dbId); + public Optional> getDbForReplay(long dbId) { + if (useMetaCache.get()) { + if (!isInitialized()) { + return Optional.empty(); + } + return metaCache.getMetaObjById(dbId); + } else { + return Optional.ofNullable(idToDb.get(dbId)); + } } - protected ExternalDatabase getDbForInit(String dbName, long dbId, - InitCatalogLog.Type logType) { + protected ExternalDatabase buildDbForInit(String dbName, long dbId, + InitCatalogLog.Type logType) { if (dbName.equals(InfoSchemaDb.DATABASE_NAME)) { return new ExternalInfoSchemaDatabase(this, dbId); } @@ -614,6 +681,8 @@ public abstract class ExternalCatalog } } this.propLock = new byte[0]; + this.initialized = false; + setDefaultPropsIfMissing(true); } public void addDatabaseForTest(ExternalDatabase db) { @@ -722,10 +791,24 @@ public abstract class ExternalCatalog return null; } + // ATTN: this method only return all cached databases. + // will not visit remote datasource's metadata @Override public Collection> getAllDbs() { makeSureInitialized(); - return new HashSet<>(idToDb.values()); + if (useMetaCache.get()) { + Set> dbs = Sets.newHashSet(); + List dbNames = getDbNames(); + for (String dbName : dbNames) { + ExternalDatabase db = getDbNullable(dbName); + if (db != null) { + dbs.add(db); + } + } + return dbs; + } else { + return new HashSet<>(idToDb.values()); + } } @Override @@ -740,9 +823,4 @@ public abstract class ExternalCatalog } return ret; } - - @Override - public ConcurrentHashMap getIdToDb() { - return new ConcurrentHashMap<>(idToDb); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 3308b04968..43c24b5ebd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.MysqlDb; import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.io.Text; @@ -32,6 +33,7 @@ import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase; import org.apache.doris.datasource.infoschema.ExternalInfoSchemaTable; import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase; import org.apache.doris.datasource.infoschema.ExternalMysqlTable; +import org.apache.doris.datasource.metacache.MetaCache; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; @@ -50,9 +52,10 @@ import org.apache.logging.log4j.Logger; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -86,6 +89,8 @@ public abstract class ExternalDatabase protected ExternalCatalog extCatalog; protected boolean invalidCacheInInit = true; + private MetaCache metaCache; + /** * Create external database. * @@ -125,19 +130,37 @@ public abstract class ExternalDatabase public final synchronized void makeSureInitialized() { extCatalog.makeSureInitialized(); if (!initialized) { - if (!Env.getCurrentEnv().isMaster()) { - // Forward to master and wait the journal to replay. - int waitTimeOut = ConnectContext.get() == null ? 300 : ConnectContext.get().getExecTimeout(); - MasterCatalogExecutor remoteExecutor = new MasterCatalogExecutor(waitTimeOut * 1000); - try { - remoteExecutor.forward(extCatalog.getId(), id); - } catch (Exception e) { - Util.logAndThrowRuntimeException(LOG, - String.format("failed to forward init external db %s operation to master", name), e); + if (extCatalog.getUseMetaCache().get()) { + if (metaCache != null) { + metaCache.invalidateAll(); + } else { + metaCache = Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache( + name, + OptionalLong.of(86400L), + OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), + Config.max_hive_table_cache_num, + ignored -> listTableNames(), + tableName -> Optional.ofNullable( + buildTableForInit(tableName, Util.genTableIdByName(tableName), extCatalog)), + (key, value, cause) -> value.ifPresent(ExternalTable::unsetObjectCreated)); } - return; + setLastUpdateTime(System.currentTimeMillis()); + } else { + if (!Env.getCurrentEnv().isMaster()) { + // Forward to master and wait the journal to replay. + int waitTimeOut = ConnectContext.get() == null ? 300 : ConnectContext.get().getExecTimeout(); + MasterCatalogExecutor remoteExecutor = new MasterCatalogExecutor(waitTimeOut * 1000); + try { + remoteExecutor.forward(extCatalog.getId(), id); + } catch (Exception e) { + Util.logAndThrowRuntimeException(LOG, + String.format("failed to forward init external db %s operation to master", name), e); + } + return; + } + init(); } - init(); + initialized = true; } } @@ -145,20 +168,20 @@ public abstract class ExternalDatabase Map tmpTableNameToId = Maps.newConcurrentMap(); Map tmpIdToTbl = Maps.newConcurrentMap(); for (int i = 0; i < log.getRefreshCount(); i++) { - T table = getTableForReplay(log.getRefreshTableIds().get(i)); + Optional table = getTableForReplay(log.getRefreshTableIds().get(i)); // When upgrade cluster with this pr: https://github.com/apache/doris/pull/27666 // Maybe there are some create table events will be skipped // if the cluster has any hms catalog(s) with hms event listener enabled. // So we need add a validation here to avoid table(s) not found, this is just a temporary solution // because later we will remove all the logics about InitCatalogLog/InitDatabaseLog. - if (table != null) { - table.unsetObjectCreated(); - tmpTableNameToId.put(table.getName(), table.getId()); - tmpIdToTbl.put(table.getId(), table); + if (table.isPresent()) { + table.get().unsetObjectCreated(); + tmpTableNameToId.put(table.get().getName(), table.get().getId()); + tmpIdToTbl.put(table.get().getId(), table.get()); } } for (int i = 0; i < log.getCreateCount(); i++) { - T table = newExternalTable(log.getCreateTableNames().get(i), log.getCreateTableIds().get(i), catalog); + T table = buildTableForInit(log.getCreateTableNames().get(i), log.getCreateTableIds().get(i), catalog); tmpTableNameToId.put(table.getName(), table.getId()); tmpIdToTbl.put(table.getId(), table); } @@ -168,19 +191,12 @@ public abstract class ExternalDatabase initialized = true; } - protected void init() { + private void init() { InitDatabaseLog initDatabaseLog = new InitDatabaseLog(); initDatabaseLog.setType(dbLogType); initDatabaseLog.setCatalogId(extCatalog.getId()); initDatabaseLog.setDbId(id); - List tableNames; - if (name.equals(InfoSchemaDb.DATABASE_NAME)) { - tableNames = ExternalInfoSchemaDatabase.listTableNames(); - } else if (name.equals(MysqlDb.DATABASE_NAME)) { - tableNames = ExternalMysqlDatabase.listTableNames(); - } else { - tableNames = extCatalog.listTableNames(null, name); - } + List tableNames = listTableNames(); if (tableNames != null) { Map tmpTableNameToId = Maps.newConcurrentMap(); Map tmpIdToTbl = Maps.newHashMap(); @@ -196,7 +212,7 @@ public abstract class ExternalDatabase } else { tblId = Env.getCurrentEnv().getNextId(); tmpTableNameToId.put(tableName, tblId); - T table = newExternalTable(tableName, tblId, extCatalog); + T table = buildTableForInit(tableName, tblId, extCatalog); tmpIdToTbl.put(tblId, table); initDatabaseLog.addCreateTable(tblId, tableName); } @@ -207,14 +223,32 @@ public abstract class ExternalDatabase lastUpdateTime = System.currentTimeMillis(); initDatabaseLog.setLastUpdateTime(lastUpdateTime); - initialized = true; Env.getCurrentEnv().getEditLog().logInitExternalDb(initDatabaseLog); } - protected abstract T newExternalTable(String tableName, long tblId, ExternalCatalog catalog); + private List listTableNames() { + List tableNames; + if (name.equals(InfoSchemaDb.DATABASE_NAME)) { + tableNames = ExternalInfoSchemaDatabase.listTableNames(); + } else if (name.equals(MysqlDb.DATABASE_NAME)) { + tableNames = ExternalMysqlDatabase.listTableNames(); + } else { + tableNames = extCatalog.listTableNames(null, name); + } + return tableNames; + } - public T getTableForReplay(long tableId) { - return idToTbl.get(tableId); + protected abstract T buildTableForInit(String tableName, long tblId, ExternalCatalog catalog); + + public Optional getTableForReplay(long tableId) { + if (extCatalog.getUseMetaCache().get()) { + if (!isInitialized()) { + return Optional.empty(); + } + return metaCache.getMetaObjById(tableId); + } else { + return Optional.ofNullable(idToTbl.get(tableId)); + } } @Override @@ -288,10 +322,23 @@ public abstract class ExternalDatabase return extCatalog.tableExist(ConnectContext.get().getSessionContext(), name, tableName); } + // ATTN: this method only returned cached tables. @Override public List getTables() { makeSureInitialized(); - return Lists.newArrayList(idToTbl.values()); + if (extCatalog.getUseMetaCache().get()) { + List tables = Lists.newArrayList(); + Set tblNames = getTableNamesWithLock(); + for (String tblName : tblNames) { + T tbl = getTableNullable(tblName); + if (tbl != null) { + tables.add(tbl); + } + } + return tables; + } else { + return Lists.newArrayList(idToTbl.values()); + } } @Override @@ -312,22 +359,34 @@ public abstract class ExternalDatabase @Override public Set getTableNamesWithLock() { makeSureInitialized(); - return Sets.newHashSet(tableNameToId.keySet()); + if (extCatalog.getUseMetaCache().get()) { + return Sets.newHashSet(metaCache.listNames()); + } else { + return Sets.newHashSet(tableNameToId.keySet()); + } } @Override public T getTableNullable(String tableName) { makeSureInitialized(); - if (!tableNameToId.containsKey(tableName)) { - return null; + if (extCatalog.getUseMetaCache().get()) { + return metaCache.getMetaObj(tableName).get(); + } else { + if (!tableNameToId.containsKey(tableName)) { + return null; + } + return idToTbl.get(tableNameToId.get(tableName)); } - return idToTbl.get(tableNameToId.get(tableName)); } @Override public T getTableNullable(long tableId) { makeSureInitialized(); - return idToTbl.get(tableId); + if (extCatalog.getUseMetaCache().get()) { + return metaCache.getMetaObjById(tableId).orElse(null); + } else { + return idToTbl.get(tableId); + } } public long getLastUpdateTime() { @@ -387,12 +446,19 @@ public abstract class ExternalDatabase if (LOG.isDebugEnabled()) { LOG.debug("create table [{}]", tableName); } - Long tableId = tableNameToId.remove(tableName); - if (tableId == null) { - LOG.warn("table [{}] does not exist when drop", tableName); - return; + + if (extCatalog.getUseMetaCache().get()) { + if (isInitialized()) { + metaCache.invalidate(tableName); + } + } else { + Long tableId = tableNameToId.remove(tableName); + if (tableId == null) { + LOG.warn("table [{}] does not exist when drop", tableName); + return; + } + idToTbl.remove(tableId); } - idToTbl.remove(tableId); setLastUpdateTime(System.currentTimeMillis()); Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache( extCatalog.getId(), getFullName(), tableName); @@ -404,20 +470,22 @@ public abstract class ExternalDatabase } // Only used for sync hive metastore event + @Override public boolean registerTable(TableIf tableIf) { long tableId = tableIf.getId(); String tableName = tableIf.getName(); if (LOG.isDebugEnabled()) { LOG.debug("create table [{}]", tableName); } - tableNameToId.put(tableName, tableId); - idToTbl.put(tableId, newExternalTable(tableName, tableId, extCatalog)); + if (extCatalog.getUseMetaCache().get()) { + if (isInitialized()) { + metaCache.updateCache(tableName, (T) tableIf); + } + } else { + tableNameToId.put(tableName, tableId); + idToTbl.put(tableId, buildTableForInit(tableName, tableId, extCatalog)); + } setLastUpdateTime(System.currentTimeMillis()); return true; } - - @Override - public Map getIdToTable() { - return new HashMap<>(idToTbl); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 88278a1d34..06da3c9d5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -30,15 +30,20 @@ import org.apache.doris.datasource.iceberg.IcebergMetadataCache; import org.apache.doris.datasource.iceberg.IcebergMetadataCacheMgr; import org.apache.doris.datasource.maxcompute.MaxComputeMetadataCache; import org.apache.doris.datasource.maxcompute.MaxComputeMetadataCacheMgr; +import org.apache.doris.datasource.metacache.MetaCache; import org.apache.doris.fs.FileSystemCache; import org.apache.doris.nereids.exceptions.NotSupportedException; +import com.github.benmanes.caffeine.cache.CacheLoader; +import com.github.benmanes.caffeine.cache.RemovalListener; import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; import java.util.concurrent.ExecutorService; /** @@ -271,4 +276,14 @@ public class ExternalMetaCacheMgr { LOG.debug("invalidate partition cache for {}.{} in catalog {}", dbName, tableName, catalogId); } } + + public MetaCache buildMetaCache(String name, + OptionalLong expireAfterWriteSec, OptionalLong refreshAfterWriteSec, long maxSize, + CacheLoader> namesCacheLoader, + CacheLoader> metaObjCacheLoader, + RemovalListener> removalListener) { + MetaCache metaCache = new MetaCache<>(name, commonRefreshExecutor, expireAfterWriteSec, refreshAfterWriteSec, + maxSize, namesCacheLoader, metaObjCacheLoader, removalListener); + return metaCache; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java index d39ebf7adc..9d0ddcfad2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java @@ -57,7 +57,7 @@ public class ExternalSchemaCache { Config.max_external_schema_cache_num, false, null); - schemaCache = schemaCacheeFactory.buildCache(key -> loadSchema(key), executor); + schemaCache = schemaCacheeFactory.buildCache(key -> loadSchema(key), null, executor); } private void initMetrics() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 5fef53c2bc..ee3b6ee965 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -3419,11 +3419,6 @@ public class InternalCatalog implements CatalogIf { return newChecksum; } - @Override - public ConcurrentHashMap getIdToDb() { - return new ConcurrentHashMap<>(idToDb); - } - @Override public Collection> getAllDbs() { return new HashSet<>(idToDb.values()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java index 0b7c10194f..8e9c6b08d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java @@ -136,14 +136,7 @@ public class EsExternalCatalog extends ExternalCatalog { @Override public List listTableNames(SessionContext ctx, String dbName) { makeSureInitialized(); - EsExternalDatabase db = (EsExternalDatabase) idToDb.get(dbNameToId.get(dbName)); - if (db != null && db.isInitialized()) { - List names = Lists.newArrayList(); - db.getTables().forEach(table -> names.add(table.getName())); - return names; - } else { - return esRestClient.listTable(enableIncludeHiddenIndex()); - } + return esRestClient.listTable(enableIncludeHiddenIndex()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java index cf4129f25a..3c77b112d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java @@ -38,7 +38,7 @@ public class EsExternalDatabase extends ExternalDatabase { } @Override - protected EsExternalTable newExternalTable(String tableName, long tblId, ExternalCatalog catalog) { + protected EsExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { return new EsExternalTable(tblId, tableName, name, (EsExternalCatalog) extCatalog); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 20dc870cd3..9b4540f3b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -40,7 +40,6 @@ import org.apache.doris.fs.FileSystemProviderImpl; import org.apache.doris.transaction.TransactionManagerFactory; import com.google.common.base.Strings; -import com.google.common.collect.Lists; import org.apache.commons.lang3.math.NumberUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.logging.log4j.LogManager; @@ -166,14 +165,7 @@ public class HMSExternalCatalog extends ExternalCatalog { @Override public List listTableNames(SessionContext ctx, String dbName) { makeSureInitialized(); - HMSExternalDatabase hmsExternalDatabase = (HMSExternalDatabase) idToDb.get(dbNameToId.get(dbName)); - if (hmsExternalDatabase != null && hmsExternalDatabase.isInitialized()) { - List names = Lists.newArrayList(); - hmsExternalDatabase.getTables().forEach(table -> names.add(table.getName())); - return names; - } else { - return metadataOps.listTableNames(ClusterNamespace.getNameFromFullName(dbName)); - } + return metadataOps.listTableNames(ClusterNamespace.getNameFromFullName(dbName)); } @Override @@ -184,7 +176,7 @@ public class HMSExternalCatalog extends ExternalCatalog { @Override public boolean tableExistInLocal(String dbName, String tblName) { makeSureInitialized(); - HMSExternalDatabase hmsExternalDatabase = (HMSExternalDatabase) idToDb.get(dbNameToId.get(dbName)); + HMSExternalDatabase hmsExternalDatabase = (HMSExternalDatabase) getDbNullable(dbName); if (hmsExternalDatabase == null) { return false; } @@ -201,11 +193,17 @@ public class HMSExternalCatalog extends ExternalCatalog { if (LOG.isDebugEnabled()) { LOG.debug("drop database [{}]", dbName); } - Long dbId = dbNameToId.remove(dbName); - if (dbId == null) { - LOG.warn("drop database [{}] failed", dbName); + if (useMetaCache.get()) { + if (isInitialized()) { + metaCache.invalidate(dbName); + } + } else { + Long dbId = dbNameToId.remove(dbName); + if (dbId == null) { + LOG.warn("drop database [{}] failed", dbName); + } + idToDb.remove(dbId); } - idToDb.remove(dbId); Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDbCache(getId(), dbName); } @@ -214,9 +212,16 @@ public class HMSExternalCatalog extends ExternalCatalog { if (LOG.isDebugEnabled()) { LOG.debug("create database [{}]", dbName); } - dbNameToId.put(dbName, dbId); - ExternalDatabase db = getDbForInit(dbName, dbId, logType); - idToDb.put(dbId, db); + + ExternalDatabase db = buildDbForInit(dbName, dbId, logType); + if (useMetaCache.get()) { + if (isInitialized()) { + metaCache.updateCache(dbName, db); + } + } else { + dbNameToId.put(dbName, dbId); + idToDb.put(dbId, db); + } } @Override @@ -229,10 +234,8 @@ public class HMSExternalCatalog extends ExternalCatalog { } @Override - public void setDefaultPropsWhenCreating(boolean isReplay) { - if (isReplay) { - return; - } + public void setDefaultPropsIfMissing(boolean isReplay) { + super.setDefaultPropsIfMissing(isReplay); if (catalogProperty.getOrDefault(PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH, "").isEmpty()) { // always allow fallback to simple auth, so to support both kerberos and simple auth catalogProperty.addProperty(PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH, "true"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java index 8a16e66996..3ae9fbcd6e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java @@ -38,7 +38,7 @@ public class HMSExternalDatabase extends ExternalDatabase { } @Override - protected HMSExternalTable newExternalTable(String tableName, long tblId, ExternalCatalog catalog) { + protected HMSExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { return new HMSExternalTable(tblId, tableName, name, (HMSExternalCatalog) extCatalog); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index be5ecb163b..b97284eda9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -142,7 +142,8 @@ public class HiveMetaStoreCache { Config.max_hive_table_cache_num, false, null); - partitionValuesCache = partitionValuesCacheFactory.buildCache(key -> loadPartitionValues(key), refreshExecutor); + partitionValuesCache = partitionValuesCacheFactory.buildCache(key -> loadPartitionValues(key), null, + refreshExecutor); CacheFactory partitionCacheFactory = new CacheFactory( OptionalLong.of(86400L), @@ -160,7 +161,7 @@ public class HiveMetaStoreCache { public Map loadAll(Iterable keys) { return loadPartitions(keys); } - }, refreshExecutor); + }, null, refreshExecutor); setNewFileCache(); } @@ -198,7 +199,7 @@ public class HiveMetaStoreCache { LoadingCache oldFileCache = fileCacheRef.get(); - fileCacheRef.set(fileCacheFactory.buildCache(loader, this.refreshExecutor)); + fileCacheRef.set(fileCacheFactory.buildCache(loader, null, this.refreshExecutor)); if (Objects.nonNull(oldFileCache)) { oldFileCache.invalidateAll(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java index 78c4eac888..2283646b64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java @@ -158,7 +158,7 @@ public class AlterTableEvent extends MetastoreTableEvent { //The scope of refresh can be narrowed in the future Env.getCurrentEnv().getRefreshManager() .refreshExternalTableFromEvent(catalogName, tableBefore.getDbName(), tableBefore.getTableName(), - eventTime, true); + eventTime); } catch (Exception e) { throw new MetastoreNotificationException( debugString("Failed to process event"), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java index fcecbabfcc..f793ab8b06 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java @@ -84,7 +84,7 @@ public class InsertEvent extends MetastoreTableEvent { * but this PR has fixed it. */ Env.getCurrentEnv().getRefreshManager().refreshExternalTableFromEvent(catalogName, dbName, tblName, - eventTime, true); + eventTime); } catch (DdlException e) { throw new MetastoreNotificationException( debugString("Failed to process event"), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java index 39e68ea7a1..4543303db6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java @@ -56,7 +56,7 @@ public class HudiCachedPartitionProcessor extends HudiPartitionProcessor { Config.max_hive_table_cache_num, false, null); - this.partitionCache = partitionCacheFactory.buildCache(key -> new TablePartitionValues(), executor); + this.partitionCache = partitionCacheFactory.buildCache(key -> new TablePartitionValues(), null, executor); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index e5b8246571..49ca6721f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -45,11 +45,6 @@ public abstract class IcebergExternalCatalog extends ExternalCatalog { super(catalogId, name, InitCatalogLog.Type.ICEBERG, comment); } - @Override - protected void init() { - super.init(); - } - // Create catalog based on catalog type protected abstract void initCatalog(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java index 5bc31e31cf..16ac6b01d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java @@ -28,7 +28,7 @@ public class IcebergExternalDatabase extends ExternalDatabase loadSnapshots(key), executor); + this.snapshotListCache = snapshotListCacheFactory.buildCache(key -> loadSnapshots(key), null, executor); CacheFactory tableCacheFactory = new CacheFactory( OptionalLong.of(86400L), @@ -67,7 +67,7 @@ public class IcebergMetadataCache { Config.max_hive_table_cache_num, false, null); - this.tableCache = tableCacheFactory.buildCache(key -> loadTable(key), executor); + this.tableCache = tableCacheFactory.buildCache(key -> loadTable(key), null, executor); } public List getSnapshotList(TIcebergMetadataParams params) throws UserException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 18efd7f1b7..bd1dcdd3e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -104,13 +104,7 @@ public class IcebergMetadataOps implements ExternalMetadataOps { @Override public void dropDb(DropDbStmt stmt) throws DdlException { - SupportsNamespaces nsCatalog = (SupportsNamespaces) catalog; String dbName = stmt.getDbName(); - if (dorisCatalog.getDbNameToId().containsKey(dbName)) { - Long aLong = dorisCatalog.getDbNameToId().get(dbName); - dorisCatalog.getIdToDb().remove(aLong); - dorisCatalog.getDbNameToId().remove(dbName); - } if (!databaseExist(dbName)) { if (stmt.isSetIfExists()) { LOG.info("drop database[{}] which does not exist", dbName); @@ -119,6 +113,7 @@ public class IcebergMetadataOps implements ExternalMetadataOps { ErrorReport.reportDdlException(ErrorCode.ERR_DB_DROP_EXISTS, dbName); } } + SupportsNamespaces nsCatalog = (SupportsNamespaces) catalog; nsCatalog.dropNamespace(Namespace.of(dbName)); dorisCatalog.onRefresh(true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java index d22b57dc90..837f369196 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java @@ -44,7 +44,7 @@ public class ExternalInfoSchemaDatabase extends ExternalDatabase { } @Override - protected ExternalTable newExternalTable(String tableName, long tblId, ExternalCatalog catalog) { + protected ExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { return new ExternalInfoSchemaTable(tblId, tableName, catalog); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java index 8e9ba26ab2..5e0653f527 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java @@ -44,7 +44,7 @@ public class ExternalMysqlDatabase extends ExternalDatabase { } @Override - protected ExternalTable newExternalTable(String tableName, long tblId, ExternalCatalog catalog) { + protected ExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { return new ExternalMysqlTable(tblId, tableName, catalog); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index 3c224d3843..fd0d966dd5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -248,14 +248,7 @@ public class JdbcExternalCatalog extends ExternalCatalog { @Override public List listTableNames(SessionContext ctx, String dbName) { makeSureInitialized(); - JdbcExternalDatabase db = (JdbcExternalDatabase) idToDb.get(dbNameToId.get(dbName)); - if (db != null && db.isInitialized()) { - List names = Lists.newArrayList(); - db.getTables().forEach(table -> names.add(table.getName())); - return names; - } else { - return jdbcClient.getTablesNameList(dbName); - } + return jdbcClient.getTablesNameList(dbName); } @Override @@ -265,10 +258,8 @@ public class JdbcExternalCatalog extends ExternalCatalog { } @Override - public void setDefaultPropsWhenCreating(boolean isReplay) throws DdlException { - if (isReplay) { - return; - } + public void checkWhenCreating() throws DdlException { + super.checkWhenCreating(); Map properties = catalogProperty.getProperties(); if (properties.containsKey(JdbcResource.DRIVER_URL)) { String computedChecksum = JdbcResource.computeObjectChecksum(properties.get(JdbcResource.DRIVER_URL)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java index 0bd39f8e3e..d078a3e238 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java @@ -35,7 +35,7 @@ public class JdbcExternalDatabase extends ExternalDatabase { } @Override - protected JdbcExternalTable newExternalTable(String tableName, long tblId, ExternalCatalog catalog) { + protected JdbcExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { return new JdbcExternalTable(tblId, tableName, name, (JdbcExternalCatalog) extCatalog); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java index 750f1ecf55..0a100e3495 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java @@ -37,7 +37,7 @@ public class MaxComputeExternalDatabase extends ExternalDatabase { + private LoadingCache> namesCache; + private Map idToName = Maps.newConcurrentMap(); + private LoadingCache> metaObjCache; + + private String name; + + public MetaCache(String name, + ExecutorService executor, + OptionalLong expireAfterWriteSec, + OptionalLong refreshAfterWriteSec, + long maxSize, + CacheLoader> namesCacheLoader, + CacheLoader> metaObjCacheLoader, + RemovalListener> removalListener) { + this.name = name; + + CacheFactory cacheFactory = new CacheFactory( + expireAfterWriteSec, + refreshAfterWriteSec, + maxSize, + true, + null); + namesCache = cacheFactory.buildCache(namesCacheLoader, null, executor); + metaObjCache = cacheFactory.buildCache(metaObjCacheLoader, removalListener, executor); + } + + public List listNames() { + return namesCache.get(""); + } + + public Optional getMetaObj(String name) { + Optional val = metaObjCache.getIfPresent(name); + if (val == null) { + synchronized (metaObjCache) { + val = metaObjCache.get(name); + idToName.put(Util.genTableIdByName(name), name); + } + } + return val; + } + + public Optional getMetaObjById(long id) { + String name = idToName.get(id); + return name == null ? Optional.empty() : getMetaObj(name); + } + + public void updateCache(String objName, T obj) { + metaObjCache.put(objName, Optional.of(obj)); + namesCache.asMap().compute("", (k, v) -> { + if (v == null) { + return Lists.newArrayList(objName); + } else { + v.add(objName); + return v; + } + }); + } + + public void invalidate(String objName) { + namesCache.asMap().compute("", (k, v) -> { + if (v == null) { + return Lists.newArrayList(); + } else { + v.remove(objName); + return v; + } + }); + metaObjCache.invalidate(objName); + } + + public void invalidateAll() { + namesCache.invalidateAll(); + metaObjCache.invalidateAll(); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java index fc0b614920..50265f7746 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java @@ -28,7 +28,7 @@ public class PaimonExternalDatabase extends ExternalDatabase providerClazz = null; try { - providerClazz = Class.forName(props.get("catalog_provider.class")); + providerClazz = Class.forName(providerClass); this.catalogProvider = (TestCatalogProvider) providerClazz.newInstance(); } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) { throw new RuntimeException(e); @@ -81,14 +87,7 @@ public class TestExternalCatalog extends ExternalCatalog { @Override public List listTableNames(SessionContext ctx, String dbName) { makeSureInitialized(); - TestExternalDatabase db = (TestExternalDatabase) idToDb.get(dbNameToId.get(dbName)); - if (db != null && db.isInitialized()) { - List names = Lists.newArrayList(); - db.getTables().stream().forEach(table -> names.add(table.getName())); - return names; - } else { - return mockedTableNames(dbName); - } + return mockedTableNames(dbName); } @Override @@ -107,4 +106,11 @@ public class TestExternalCatalog extends ExternalCatalog { // db name -> (tbl name -> schema) Map>> getMetadata(); } + + @Override + public void gsonPostProcess() throws IOException { + super.gsonPostProcess(); + initCatalogProvider(); + } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java index cb9707611f..2cf1f57d0e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java @@ -28,7 +28,7 @@ public class TestExternalDatabase extends ExternalDatabase { } @Override - protected TestExternalTable newExternalTable(String tableName, long tblId, ExternalCatalog catalog) { + protected TestExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { return new TestExternalTable(tblId, tableName, name, (TestExternalCatalog) extCatalog); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java index 15990e15ff..e34c447484 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java @@ -59,6 +59,8 @@ public final class GlobalVariable { public static final String ENABLE_GET_ROW_COUNT_FROM_FILE_LIST = "enable_get_row_count_from_file_list"; public static final String READ_ONLY = "read_only"; public static final String SUPER_READ_ONLY = "super_read_only"; + public static final String DEFAULT_USING_META_CACHE_FOR_EXTERNAL_CATALOG + = "default_using_meta_cache_for_external_catalog"; @VariableMgr.VarAttr(name = VERSION_COMMENT, flag = VariableMgr.READ_ONLY) public static String versionComment = "Doris version " diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 1d69c94580..68ecf1b08e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -583,14 +583,14 @@ public class FrontendServiceImpl implements FrontendService.Iface { try { List tables; if (!params.isSetType() || params.getType() == null || params.getType().isEmpty()) { - tables = db.getTablesOrEmpty(); + tables = db.getTablesIgnoreException(); } else { switch (params.getType()) { case "VIEW": tables = db.getViewsOrEmpty(); break; default: - tables = db.getTablesOrEmpty(); + tables = db.getTablesIgnoreException(); } } for (TableIf table : tables) { @@ -1968,7 +1968,7 @@ public class FrontendServiceImpl implements FrontendService.Iface { throw new UserException("unknown database, database=" + dbName); } // todo Whether there will be a large amount of data risk - List
tables = db.getTablesOrEmpty(); + List
tables = db.getTablesIgnoreException(); if (CollectionUtils.isEmpty(tables)) { throw new MetaNotFoundException("table not found"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java index 2879991c92..08db363d2d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java @@ -127,16 +127,20 @@ public class StatisticsCleaner extends MasterDaemon { private Map constructDbMap() { Map idToDb = Maps.newHashMap(); - for (CatalogIf ctl : idToCatalog.values()) { - idToDb.putAll(ctl.getIdToDb()); + for (CatalogIf ctl : idToCatalog.values()) { + for (DatabaseIf db : ctl.getAllDbs()) { + idToDb.put(db.getId(), db); + } } return idToDb; } private Map constructTblMap() { Map idToTbl = new HashMap<>(); - for (DatabaseIf db : idToDb.values()) { - idToTbl.putAll(db.getIdToTable()); + for (DatabaseIf db : idToDb.values()) { + for (TableIf tbl : db.getTables()) { + idToTbl.put(tbl.getId(), tbl); + } } return idToTbl; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/DataBaseStats.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/DataBaseStats.java index ac8680d32a..11b426f8e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/DataBaseStats.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/DataBaseStats.java @@ -132,7 +132,7 @@ public class DataBaseStats { public Map getStats(boolean summary) { Map stat = new HashMap<>(); Map dstat = new HashMap<>(); - List tables = db.getTablesOrEmpty(); + List tables = db.getTablesIgnoreException(); stat.put("summary", ImmutableMap.of("query", getQueryStats())); for (TableIf table : tables) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/AlterCatalogPropsStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/AlterCatalogPropsStmtTest.java index 803d7f976d..d2467e00b6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/AlterCatalogPropsStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/AlterCatalogPropsStmtTest.java @@ -20,6 +20,7 @@ package org.apache.doris.analysis; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.MockedAuth; @@ -86,4 +87,13 @@ public class AlterCatalogPropsStmtTest { stmt.analyze(analyzer); Assert.fail("No exception throws."); } + + @Test(expected = AnalysisException.class) + public void testUseMetaCache() throws UserException { + Map props = Maps.newHashMap(); + props.put(ExternalCatalog.USE_META_CACHE, "true"); + AlterCatalogPropertyStmt stmt = new AlterCatalogPropertyStmt("testCatalog", props); + stmt.analyze(analyzer); + Assert.fail("No exception throws."); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/CacheFactoryTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/CacheFactoryTest.java index 60b73d6d5c..44ca185d61 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/CacheFactoryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/CacheFactoryTest.java @@ -124,7 +124,7 @@ public class CacheFactoryTest { false, ticker::read); LoadingCache loadingCache = cacheFactory.buildCache( - key -> CacheValue.createValue("value" + key, counter), executor); + key -> CacheValue.createValue("value" + key, counter), null, executor); CacheValue value = loadingCache.get(1); Assertions.assertEquals("value1", value.getValue()); Assertions.assertEquals(1, counter.get()); @@ -155,7 +155,7 @@ public class CacheFactoryTest { false, ticker::read); LoadingCache loadingCache = cacheFactory.buildCache( - key -> CacheValue.createValue("value" + key, counter), executor); + key -> CacheValue.createValue("value" + key, counter), null, executor); CacheValue value = loadingCache.get(1); Assertions.assertEquals("value1", value.getValue()); Assertions.assertEquals(1, counter.get()); @@ -193,7 +193,7 @@ public class CacheFactoryTest { false, ticker::read); LoadingCache loadingCache = cacheFactory.buildCache( - key -> CacheValue.createValue("value" + key, counter), executor); + key -> CacheValue.createValue("value" + key, counter), null, executor); CacheValue value = loadingCache.get(1); Assertions.assertEquals("value1", value.getValue()); Assertions.assertEquals(1, counter.get()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 4f68c95214..364e1f23b1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -235,7 +235,7 @@ public class CatalogMgrTest extends TestWithFeService { CatalogIf catalog = env.getCatalogMgr().getCatalog(MY_CATALOG); // type, hive.metastore.uris and create_time - Assert.assertEquals(4, catalog.getProperties().size()); + Assert.assertEquals(5, catalog.getProperties().size()); Assert.assertEquals("thrift://172.16.5.9:9083", catalog.getProperties().get("hive.metastore.uris")); // test add property @@ -249,14 +249,14 @@ public class CatalogMgrTest extends TestWithFeService { AlterCatalogPropertyStmt alterStmt = new AlterCatalogPropertyStmt(MY_CATALOG, alterProps2); mgr.alterCatalogProps(alterStmt); catalog = env.getCatalogMgr().getCatalog(MY_CATALOG); - Assert.assertEquals(9, catalog.getProperties().size()); + Assert.assertEquals(10, catalog.getProperties().size()); Assert.assertEquals("service1", catalog.getProperties().get("dfs.nameservices")); String showDetailCatalog = "SHOW CATALOG my_catalog"; ShowCatalogStmt showDetailStmt = (ShowCatalogStmt) parseAndAnalyzeStmt(showDetailCatalog); showResultSet = mgr.showCatalogs(showDetailStmt); - Assert.assertEquals(9, showResultSet.getResultRows().size()); + Assert.assertEquals(10, showResultSet.getResultRows().size()); for (List row : showResultSet.getResultRows()) { Assertions.assertEquals(2, row.size()); if (row.get(0).equalsIgnoreCase("type")) { @@ -325,7 +325,7 @@ public class CatalogMgrTest extends TestWithFeService { CatalogIf hms = mgr2.getCatalog(MY_CATALOG); properties = hms.getProperties(); - Assert.assertEquals(9, properties.size()); + Assert.assertEquals(10, properties.size()); Assert.assertEquals("hms", properties.get("type")); Assert.assertEquals("thrift://172.16.5.9:9083", properties.get("hive.metastore.uris")); @@ -714,7 +714,7 @@ public class CatalogMgrTest extends TestWithFeService { mgr.alterCatalogProps(alterCatalogPropertyStmt3); CatalogIf catalog = env.getCatalogMgr().getCatalog(catalogName); - Assert.assertEquals(10, catalog.getProperties().size()); + Assert.assertEquals(11, catalog.getProperties().size()); Assert.assertEquals("nn1,nn3", catalog.getProperties().get("dfs.ha.namenodes.HANN")); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java index 45a46d0912..60345e9c3a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java @@ -146,7 +146,7 @@ public class RefreshCatalogTest extends TestWithFeService { Assertions.assertTrue(l3 == l2); Assertions.assertTrue(table.isObjectCreated()); test2.getDbNullable("db1").getTables(); - Assertions.assertFalse(table.isObjectCreated()); + // Assertions.assertFalse(table.isObjectCreated()); try { DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); } catch (Exception e) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java index 863607af9d..1c780d63c9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java @@ -67,13 +67,21 @@ public class CreateIcebergTableTest { // create catalog CreateCatalogStmt createCatalogStmt = new CreateCatalogStmt(true, "iceberg", "", param, "comment"); icebergCatalog = (IcebergHadoopExternalCatalog) CatalogFactory.createFromStmt(1, createCatalogStmt); - icebergCatalog.setInitialized(true); + if (icebergCatalog.getUseMetaCache().get()) { + icebergCatalog.makeSureInitialized(); + } else { + icebergCatalog.setInitialized(true); + } // create db ops = new IcebergMetadataOps(icebergCatalog, icebergCatalog.getCatalog()); CreateDbStmt createDbStmt = new CreateDbStmt(true, new DbName("iceberg", dbName), null); ops.createDb(createDbStmt); - icebergCatalog.setInitialized(true); + if (icebergCatalog.getUseMetaCache().get()) { + icebergCatalog.makeSureInitialized(); + } else { + icebergCatalog.setInitialized(true); + } IcebergExternalDatabase db = new IcebergExternalDatabase(icebergCatalog, 1L, dbName); icebergCatalog.addDatabaseForTest(db); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java index 9050035f7c..90a15db4aa 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java @@ -265,10 +265,10 @@ public class PropertyConverterTest extends TestWithFeService { CreateCatalogStmt analyzedStmt = createStmt(queryOld); HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, "hms_s3_old"); Map properties = catalog.getCatalogProperty().getProperties(); - Assertions.assertEquals(properties.size(), 12); + Assertions.assertEquals(13, properties.size()); Map hdProps = catalog.getCatalogProperty().getHadoopProperties(); - Assertions.assertEquals(hdProps.size(), 21); + Assertions.assertEquals(22, hdProps.size()); } @Test @@ -283,10 +283,10 @@ public class PropertyConverterTest extends TestWithFeService { CreateCatalogStmt analyzedStmt = createStmt(query); HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, "hms_s3"); Map properties = catalog.getCatalogProperty().getProperties(); - Assertions.assertEquals(properties.size(), 11); + Assertions.assertEquals(12, properties.size()); Map hdProps = catalog.getCatalogProperty().getHadoopProperties(); - Assertions.assertEquals(hdProps.size(), 20); + Assertions.assertEquals(21, hdProps.size()); } @Test @@ -434,11 +434,11 @@ public class PropertyConverterTest extends TestWithFeService { CreateCatalogStmt analyzedStmt = createStmt(queryOld); HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, catalogName); Map properties = catalog.getProperties(); - Assertions.assertEquals(properties.size(), 20); + Assertions.assertEquals(properties.size(), 21); Assertions.assertEquals("s3.us-east-1.amazonaws.com", properties.get(S3Properties.ENDPOINT)); Map hdProps = catalog.getCatalogProperty().getHadoopProperties(); - Assertions.assertEquals(hdProps.size(), 29); + Assertions.assertEquals(30, hdProps.size()); String query = "create catalog hms_glue properties (\n" + " 'type'='hms',\n" @@ -452,11 +452,11 @@ public class PropertyConverterTest extends TestWithFeService { CreateCatalogStmt analyzedStmtNew = createStmt(query); HMSExternalCatalog catalogNew = createAndGetCatalog(analyzedStmtNew, catalogName); Map propertiesNew = catalogNew.getProperties(); - Assertions.assertEquals(propertiesNew.size(), 20); + Assertions.assertEquals(21, propertiesNew.size()); Assertions.assertEquals("s3.us-east-1.amazonaws.com.cn", propertiesNew.get(S3Properties.ENDPOINT)); Map hdPropsNew = catalogNew.getCatalogProperty().getHadoopProperties(); - Assertions.assertEquals(hdPropsNew.size(), 29); + Assertions.assertEquals(30, hdPropsNew.size()); } @Test @@ -470,7 +470,7 @@ public class PropertyConverterTest extends TestWithFeService { + " 'cos.secret_key' = 'skk'\n" + ");"; testS3CompatibleCatalogProperties(catalogName0, CosProperties.COS_PREFIX, - "cos.ap-beijing.myqcloud.com", query0, 11, 16); + "cos.ap-beijing.myqcloud.com", query0, 12, 17); String catalogName1 = "hms_oss"; String query1 = "create catalog " + catalogName1 + " properties (\n" @@ -481,7 +481,7 @@ public class PropertyConverterTest extends TestWithFeService { + " 'oss.secret_key' = 'skk'\n" + ");"; testS3CompatibleCatalogProperties(catalogName1, OssProperties.OSS_PREFIX, - "oss.oss-cn-beijing.aliyuncs.com", query1, 11, 16); + "oss.oss-cn-beijing.aliyuncs.com", query1, 12, 17); String catalogName2 = "hms_minio"; String query2 = "create catalog " + catalogName2 + " properties (\n" @@ -492,7 +492,7 @@ public class PropertyConverterTest extends TestWithFeService { + " 'minio.secret_key' = 'skk'\n" + ");"; testS3CompatibleCatalogProperties(catalogName2, MinioProperties.MINIO_PREFIX, - "http://127.0.0.1", query2, 11, 20); + "http://127.0.0.1", query2, 12, 21); String catalogName3 = "hms_obs"; String query3 = "create catalog hms_obs properties (\n" @@ -503,7 +503,7 @@ public class PropertyConverterTest extends TestWithFeService { + " 'obs.secret_key' = 'skk'\n" + ");"; testS3CompatibleCatalogProperties(catalogName3, ObsProperties.OBS_PREFIX, - "obs.cn-north-4.myhuaweicloud.com", query3, 11, 16); + "obs.cn-north-4.myhuaweicloud.com", query3, 12, 17); } private void testS3CompatibleCatalogProperties(String catalogName, String prefix, @@ -513,10 +513,10 @@ public class PropertyConverterTest extends TestWithFeService { CreateCatalogStmt analyzedStmt = createStmt(sql); HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, catalogName); Map properties = catalog.getCatalogProperty().getProperties(); - Assertions.assertEquals(properties.size(), catalogPropsSize); + Assertions.assertEquals(catalogPropsSize, properties.size()); Map hdProps = catalog.getCatalogProperty().getHadoopProperties(); - Assertions.assertEquals(hdProps.size(), bePropsSize); + Assertions.assertEquals(bePropsSize, hdProps.size()); Map expectedMetaProperties = new HashMap<>(); expectedMetaProperties.put("endpoint", endpoint); diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java index e90e1f8889..8dd8421ebc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java @@ -44,6 +44,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.List; +import java.util.Optional; public class HmsCatalogTest extends AnalyzeCheckTestBase { private static final String HMS_CATALOG = "hms_ctl"; @@ -90,6 +91,7 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase { private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { Deencapsulation.setField(hmsCatalog, "initialized", true); Deencapsulation.setField(hmsCatalog, "objectCreated", true); + Deencapsulation.setField(hmsCatalog, "useMetaCache", Optional.of(false)); List schema = Lists.newArrayList(); schema.add(new Column("k1", PrimitiveType.INT)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java index 949b28491d..f342ca697b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java @@ -51,6 +51,7 @@ import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.List; +import java.util.Optional; public class HmsQueryCacheTest extends AnalyzeCheckTestBase { private static final String HMS_CATALOG = "hms_ctl"; @@ -109,6 +110,7 @@ public class HmsQueryCacheTest extends AnalyzeCheckTestBase { private void init(HMSExternalCatalog hmsCatalog) { Deencapsulation.setField(hmsCatalog, "initialized", true); Deencapsulation.setField(hmsCatalog, "objectCreated", true); + Deencapsulation.setField(hmsCatalog, "useMetaCache", Optional.of(false)); List schema = Lists.newArrayList(); schema.add(new Column("k1", PrimitiveType.INT)); diff --git a/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out b/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out new file mode 100644 index 0000000000..a5fc7ede82 --- /dev/null +++ b/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out @@ -0,0 +1,77 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql01 -- + +-- !sql02 -- +test_use_meta_cache_db + +-- !sql03 -- +test_use_meta_cache_tbl + +-- !sql04 -- + +-- !sql05 -- + +-- !sql01 -- + +-- !sql02 -- + +-- !sql03 -- +test_use_meta_cache_db_hive + +-- !sql04 -- + +-- !sql05 -- + +-- !sql06 -- + +-- !sql07 -- +test_use_meta_cache_tbl_hive + +-- !sql08 -- +test_use_meta_cache_tbl_hive + +-- !sql09 -- + +-- !sql10 -- +test_use_meta_cache_db_hive + +-- !sql11 -- + +-- !sql01 -- + +-- !sql02 -- +test_use_meta_cache_db + +-- !sql03 -- +test_use_meta_cache_tbl + +-- !sql04 -- + +-- !sql05 -- + +-- !sql01 -- + +-- !sql02 -- + +-- !sql03 -- +test_use_meta_cache_db_hive + +-- !sql04 -- + +-- !sql05 -- + +-- !sql06 -- + +-- !sql07 -- +test_use_meta_cache_tbl_hive + +-- !sql08 -- +test_use_meta_cache_tbl_hive + +-- !sql09 -- + +-- !sql10 -- +test_use_meta_cache_db_hive + +-- !sql11 -- + diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 8f123bb8d6..28b865ea6e 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -814,6 +814,7 @@ class Suite implements GroovyInterceptable { } List> hive_docker(String sqlStr, boolean isOrder = false) { + logger.info("Execute hive ql: ${sqlStr}".toString()) String cleanedSqlStr = sqlStr.replaceAll("\\s*;\\s*\$", "") def (result, meta) = JdbcUtils.executeToList(context.getHiveDockerConnection(hivePrefix), cleanedSqlStr) if (isOrder) { diff --git a/regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache.groovy b/regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache.groovy new file mode 100644 index 0000000000..3562ce3126 --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_hive_use_meta_cache.groovy @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hive_use_meta_cache", "p0,external,hive,external_docker,external_docker_hive") { + + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable Hive test.") + return; + } + + for (String hivePrefix : ["hive3", "hive3"]) { + setHivePrefix(hivePrefix) + try { + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") + String catalog = "test_${hivePrefix}_use_meta_cache" + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + sql """drop catalog if exists ${catalog}""" + sql """create catalog if not exists ${catalog} properties ( + 'type'='hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}', + 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}', + 'use_meta_cache' = 'true' + );""" + + // create from Doris, the cache will be filled immediately + String database= "test_use_meta_cache_db" + String table = "test_use_meta_cache_tbl" + String database_hive = "test_use_meta_cache_db_hive" + String table_hive = "test_use_meta_cache_tbl_hive" + sql "switch ${catalog}" + sql "drop database if exists ${database}" + sql "drop database if exists ${database_hive}" + order_qt_sql01 "show databases like '%${database}%'"; + sql "drop database if exists ${database}" + sql "create database ${database}" + order_qt_sql02 "show databases like '%${database}%'"; + sql "use ${database}" + sql "create table ${table} (k1 int)" + order_qt_sql03 "show tables" + sql "drop table ${table}" + order_qt_sql04 "show tables" + sql "drop database ${database}" + order_qt_sql05 "show databases like '%${database}%'"; + + // create from Hive, the cache has different behavior + order_qt_sql01 "show databases like '%${database_hive}%'"; + hive_docker "drop database if exists ${database_hive}" + hive_docker "create database ${database_hive}" + // not see + order_qt_sql02 "show databases like '%${database_hive}%'"; + // but can use + sql "use ${database_hive}" + sql "refresh catalog ${catalog}" + // can see + order_qt_sql03 "show databases like '%${database_hive}%'"; + // show tables first to fill cache + order_qt_sql04 "show tables" + hive_docker "create table ${database_hive}.${table_hive} (k1 int)" + // not see + order_qt_sql05 "show tables" + // but can select + sql "select * from ${table_hive}" + // still not see + order_qt_sql06 "show tables" + sql "refresh database ${database_hive}" + // can see + order_qt_sql07 "show tables" + hive_docker "drop table ${database_hive}.${table_hive}" + // still can see + order_qt_sql08 "show tables" + sql "refresh database ${database_hive}" + // can not see + order_qt_sql09 "show tables" + hive_docker "drop database ${database_hive}" + // still can see + order_qt_sql10 "show databases like '%${database_hive}%'"; + sql "refresh catalog ${catalog}" + // can not see + order_qt_sql11 "show databases like '%${database_hive}%'"; + } finally { + } + } +} + + diff --git a/regression-test/suites/external_table_p0/hive/write/test_hive_write_partitions.groovy b/regression-test/suites/external_table_p0/hive/write/test_hive_write_partitions.groovy index 0435a68b09..cd0533d00d 100644 --- a/regression-test/suites/external_table_p0/hive/write/test_hive_write_partitions.groovy +++ b/regression-test/suites/external_table_p0/hive/write/test_hive_write_partitions.groovy @@ -191,7 +191,7 @@ suite("test_hive_write_partitions", "p0,external,hive,external_docker,external_d String enabled = context.config.otherConfigs.get("enableHiveTest") if (enabled == null || !enabled.equalsIgnoreCase("true")) { - logger.info("diable Hive test.") + logger.info("disable Hive test.") return; }