diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
index c90cc35c85..d065c5cd3e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
@@ -97,28 +97,38 @@ public class StructInfoMap {
      * refresh group expression map
      *
      * @param group the root group
+     *
+     * @return whether groupExpressionMap is updated
      */
-    public void refresh(Group group) {
+    public boolean refresh(Group group) {
         Set<Group> refreshedGroup = new HashSet<>();
+        int originSize = groupExpressionMap.size();
         for (GroupExpression groupExpression : group.getLogicalExpressions()) {
             List<Set<BitSet>> childrenTableMap = new ArrayList<>();
+            boolean needRefresh = false;
             if (groupExpression.children().isEmpty()) {
-                groupExpressionMap.put(constructLeaf(groupExpression), Pair.of(groupExpression, new ArrayList<>()));
+                BitSet leaf = constructLeaf(groupExpression);
+                groupExpressionMap.put(leaf, Pair.of(groupExpression, new ArrayList<>()));
                 continue;
             }
+
             for (Group child : groupExpression.children()) {
                 if (!refreshedGroup.contains(child)) {
                     StructInfoMap childStructInfoMap = child.getstructInfoMap();
-                    childStructInfoMap.refresh(child);
+                    needRefresh |= childStructInfoMap.refresh(child);
                 }
                 refreshedGroup.add(child);
                 childrenTableMap.add(child.getstructInfoMap().getTableMaps());
             }
-            Set<Pair<BitSet, List<BitSet>>> bitSetWithChildren = cartesianProduct(childrenTableMap);
-            for (Pair<BitSet, List<BitSet>> bitSetWithChild : bitSetWithChildren) {
-                groupExpressionMap.put(bitSetWithChild.first, Pair.of(groupExpression, bitSetWithChild.second));
+
+            if (needRefresh) {
+                Set<Pair<BitSet, List<BitSet>>> bitSetWithChildren = cartesianProduct(childrenTableMap);
+                for (Pair<BitSet, List<BitSet>> bitSetWithChild : bitSetWithChildren) {
+                    groupExpressionMap.put(bitSetWithChild.first, Pair.of(groupExpression, bitSetWithChild.second));
+                }
             }
         }
+        return originSize != groupExpressionMap.size();
     }
 
     private BitSet constructLeaf(GroupExpression groupExpression) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
index 08db05b506..c2d9d072ac 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
@@ -82,6 +82,59 @@ class StructInfoMapTest extends SqlTestBase {
         dropMvByNereids("drop materialized view mv1");
     }
 
+    @Test
+    void testLazyRefresh() throws Exception {
+        CascadesContext c1 = createCascadesContext(
+                "select T1.id from T1 inner join T2 "
+                        + "on T1.id = T2.id "
+                        + "inner join T3 on T1.id = T3.id",
+                connectContext
+        );
+        PlanChecker.from(c1)
+                .analyze()
+                .rewrite()
+                .optimize();
+        Group root = c1.getMemo().getRoot();
+        Set<BitSet> tableMaps = root.getstructInfoMap().getTableMaps();
+        Assertions.assertTrue(tableMaps.isEmpty());
+        boolean refreshed = root.getstructInfoMap().refresh(root);
+        Assertions.assertTrue(refreshed);
+        refreshed = root.getstructInfoMap().refresh(root);
+        Assertions.assertFalse(refreshed);
+        Assertions.assertEquals(1, tableMaps.size());
+        new MockUp<MTMVRelationManager>() {
+            @Mock
+            public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) {
+                return true;
+            }
+        };
+        connectContext.getSessionVariable().enableMaterializedViewRewrite = true;
+        createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n"
+                + "        DISTRIBUTED BY RANDOM BUCKETS 1\n"
+                + "        PROPERTIES ('replication_num' = '1') \n"
+                + "        as select T1.id from T1 inner join T2 "
+                + "on T1.id = T2.id;");
+        c1 = createCascadesContext(
+                "select T1.id from T1 inner join T2 "
+                        + "on T1.id = T2.id "
+                        + "inner join T3 on T1.id = T3.id",
+                connectContext
+        );
+        PlanChecker.from(c1)
+                .analyze()
+                .rewrite()
+                .optimize()
+                .printlnBestPlanTree();
+        root = c1.getMemo().getRoot();
+        refreshed = root.getstructInfoMap().refresh(root);
+        Assertions.assertTrue(refreshed);
+        refreshed = root.getstructInfoMap().refresh(root);
+        Assertions.assertFalse(refreshed);
+        tableMaps = root.getstructInfoMap().getTableMaps();
+        Assertions.assertEquals(2, tableMaps.size());
+        dropMvByNereids("drop materialized view mv1");
+    }
+
     @Test
     void testTableChild() throws Exception {
         CascadesContext c1 = createCascadesContext(