diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java
index 22f0cad9d0..08a410b859 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java
@@ -17,7 +17,6 @@
 
 package org.apache.doris.statistics;
 
-import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.common.ThreadPoolManager;
 import org.apache.doris.qe.InternalQueryExecutionException;
@@ -64,7 +63,8 @@ public class ColumnStatisticsCacheLoader extends StatisticsCacheLoader<Optional<
         try {
             columnResults = StatisticsRepository.loadColStats(key.tableId, key.idxId, key.colName);
         } catch (InternalQueryExecutionException e) {
-            retryLoad(key);
+            LOG.info("Failed to load stats for table {} column {}. Reason:{}",
+                    key.tableId, key.colName, e.getMessage());
             return Optional.empty();
         }
         ColumnStatistic columnStatistics;
@@ -80,42 +80,4 @@ public class ColumnStatisticsCacheLoader extends StatisticsCacheLoader<Optional<
             return Optional.of(columnStatistics);
         }
     }
-
-    private void retryLoad(StatisticsCacheKey key) {
-        singleThreadPool.submit(new RetryTask(key, 1));
-    }
-
-    private static class RetryTask implements Runnable {
-        StatisticsCacheKey key;
-        int retryTimes;
-
-        public RetryTask(StatisticsCacheKey key, int retryTimes) {
-            this.key = key;
-            this.retryTimes = retryTimes;
-        }
-
-        @Override
-        public void run() {
-            List<ResultRow> columnResults = null;
-            try {
-                columnResults = StatisticsRepository.loadColStats(key.tableId, key.idxId, key.colName);
-            } catch (InternalQueryExecutionException e) {
-                if (this.retryTimes < StatisticConstants.LOAD_RETRY_TIMES) {
-                    retryTimes++;
-                    singleThreadPool.submit(this);
-                }
-                return;
-            }
-            ColumnStatistic columnStatistics;
-            try {
-                columnStatistics = StatisticsUtil.deserializeToColumnStatistics(columnResults);
-            } catch (Exception e) {
-                LOG.warn("Exception to deserialize column statistics", e);
-                return;
-            }
-            if (columnStatistics != null) {
-                Env.getCurrentEnv().getStatisticsCache().putCache(key, columnStatistics);
-            }
-        }
-    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
index d4b91b0736..fbec9a60fa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
@@ -39,9 +39,7 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import java.time.Duration;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -201,7 +199,6 @@ public class StatisticsCache {
         if (CollectionUtils.isEmpty(recentStatsUpdatedCols)) {
             return;
         }
-        Map<StatisticsCacheKey, ColumnStatistic> keyToColStats = new HashMap<>();
         for (ResultRow r : recentStatsUpdatedCols) {
             try {
                 StatsId statsId = new StatsId(r);
@@ -211,7 +208,6 @@ public class StatisticsCache {
                 final StatisticsCacheKey k =
                         new StatisticsCacheKey(tblId, idxId, colId);
                 final ColumnStatistic c = ColumnStatistic.fromResultRow(r);
-                keyToColStats.put(k, c);
                 putCache(k, c);
             } catch (Throwable t) {
                 LOG.warn("Error when preheating stats cache", t);