diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 435722e06b..2f8186db72 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -225,9 +225,7 @@ public abstract class ExternalCatalog
         initLocalObjects();
         if (!initialized) {
             if (useMetaCache.get()) {
-                if (metaCache != null) {
-                    metaCache.invalidateAll();
-                } else {
+                if (metaCache == null) {
                     metaCache = Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache(
                             name,
                             OptionalLong.of(86400L),
@@ -344,7 +342,6 @@ public abstract class ExternalCatalog
                 dbId = dbNameToId.get(dbName);
                 tmpDbNameToId.put(dbName, dbId);
                 ExternalDatabase<? extends ExternalTable> db = idToDb.get(dbId);
-                db.setUnInitialized(invalidCacheInInit);
                 tmpIdToDb.put(dbId, db);
                 initCatalogLog.addRefreshDb(dbId);
             } else {
@@ -379,6 +376,15 @@ public abstract class ExternalCatalog
         synchronized (this.propLock) {
             this.convertedProperties = null;
         }
+        if (useMetaCache.isPresent()) {
+            if (useMetaCache.get() && metaCache != null) {
+                metaCache.invalidateAll();
+            } else if (!useMetaCache.get()) {
+                for (ExternalDatabase<? extends ExternalTable> db : idToDb.values()) {
+                    db.setUnInitialized(invalidCache);
+                }
+            }
+        }
         this.invalidCacheInInit = invalidCache;
         if (invalidCache) {
             Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalogCache(id);
@@ -586,7 +592,6 @@ public abstract class ExternalCatalog
             // Because replyInitCatalog can only be called when `use_meta_cache` is false.
             // And if `use_meta_cache` is false, getDbForReplay() will not return null
             Preconditions.checkNotNull(db.get());
-            db.get().setUnInitialized(invalidCacheInInit);
             tmpDbNameToId.put(db.get().getFullName(), db.get().getId());
             tmpIdToDb.put(db.get().getId(), db.get());
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
index 6ab3421abc..f34be46d4b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -118,6 +118,15 @@ public abstract class ExternalDatabase<T extends ExternalTable>
     public void setUnInitialized(boolean invalidCache) {
         this.initialized = false;
         this.invalidCacheInInit = invalidCache;
+        if (extCatalog.getUseMetaCache().isPresent()) {
+            if (extCatalog.getUseMetaCache().get() && metaCache != null) {
+                metaCache.invalidateAll();
+            } else if (!extCatalog.getUseMetaCache().get()) {
+                for (T table : idToTbl.values()) {
+                    table.unsetObjectCreated();
+                }
+            }
+        }
         if (invalidCache) {
             Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDbCache(extCatalog.getId(), name);
         }
@@ -131,9 +140,7 @@ public abstract class ExternalDatabase<T extends ExternalTable>
         extCatalog.makeSureInitialized();
         if (!initialized) {
             if (extCatalog.getUseMetaCache().get()) {
-                if (metaCache != null) {
-                    metaCache.invalidateAll();
-                } else {
+                if (metaCache == null) {
                     metaCache = Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache(
                             name,
                             OptionalLong.of(86400L),
@@ -175,7 +182,6 @@ public abstract class ExternalDatabase<T extends ExternalTable>
             // So we need add a validation here to avoid table(s) not found, this is just a temporary solution
             // because later we will remove all the logics about InitCatalogLog/InitDatabaseLog.
             if (table.isPresent()) {
-                table.get().unsetObjectCreated();
                 tmpTableNameToId.put(table.get().getName(), table.get().getId());
                 tmpIdToTbl.put(table.get().getId(), table.get());
             }
@@ -206,7 +212,6 @@ public abstract class ExternalDatabase<T extends ExternalTable>
                     tblId = tableNameToId.get(tableName);
                     tmpTableNameToId.put(tableName, tblId);
                     T table = idToTbl.get(tblId);
-                    table.unsetObjectCreated();
                     tmpIdToTbl.put(tblId, table);
                     initDatabaseLog.addRefreshTable(tblId);
                 } else {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java
index 1358e03200..fd920e9632 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java
@@ -91,7 +91,7 @@ public class RefreshDbTest extends TestWithFeService {
         }
         long l3 = db1.getLastUpdateTime();
         Assertions.assertTrue(l3 == l2);
-        Assertions.assertTrue(table.isObjectCreated());
+        Assertions.assertFalse(table.isObjectCreated());
         test1.getDbNullable("db1").getTables();
         Assertions.assertFalse(table.isObjectCreated());
         try {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
index 60345e9c3a..f64e6523eb 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
@@ -144,7 +144,7 @@ public class RefreshCatalogTest extends TestWithFeService {
         // not triggered init method
         long l3 = test2.getLastUpdateTime();
         Assertions.assertTrue(l3 == l2);
-        Assertions.assertTrue(table.isObjectCreated());
+        Assertions.assertFalse(table.isObjectCreated());
         test2.getDbNullable("db1").getTables();
         // Assertions.assertFalse(table.isObjectCreated());
         try {