From 3ba42a7823d1d5ee4fbb13c5d5820f1b06748ae2 Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Fri, 26 Apr 2024 19:31:09 +0800 Subject: [PATCH] [improvement](mtmv) Optimize the nested materialized view performance (#34163) Record increase refersh version more accurately. The refreshVersion in the memo will increase when mv rewrite successfully. In query rewrite, if refresh version is different from the current struct info map in group, will refresh the group struct info or not. --- .../java/org/apache/doris/nereids/memo/Memo.java | 4 +++- .../apache/doris/nereids/memo/StructInfoMap.java | 14 ++++++++------ .../exploration/mv/MaterializedViewUtils.java | 3 +-- .../doris/nereids/memo/StructInfoMapTest.java | 14 +++++++------- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java index 8793cb5be5..1e6b285187 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java @@ -416,7 +416,9 @@ public class Memo { throw new IllegalStateException("Insert a plan into targetGroup but differ in logicalproperties"); } // TODO Support sync materialized view in the future - if (plan instanceof CatalogRelation && ((CatalogRelation) plan).getTable() instanceof MTMV) { + if (plan instanceof LogicalPlan && plan instanceof CatalogRelation + && ((CatalogRelation) plan).getTable() instanceof MTMV + && !plan.getGroupExpression().isPresent()) { refreshVersion.incrementAndGet(); } Optional groupExpr = plan.getGroupExpression(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java index efa2bef179..ae07c2043a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java @@ -59,7 +59,7 @@ public class StructInfoMap { return structInfo; } if (groupExpressionMap.isEmpty() || !groupExpressionMap.containsKey(tableMap)) { - refresh(group, memo.getRefreshVersion(), foldTableMap); + refresh(group, memo.getRefreshVersion()); group.getstructInfoMap().setRefreshVersion(memo.getRefreshVersion()); } if (groupExpressionMap.containsKey(tableMap)) { @@ -118,7 +118,10 @@ public class StructInfoMap { * @param group the root group * */ - public void refresh(Group group, long refreshVersion, BitSet targetBitSet) { + public void refresh(Group group, long memoVersion) { + if (memoVersion == group.getstructInfoMap().refreshVersion) { + return; + } Set refreshedGroup = new HashSet<>(); for (GroupExpression groupExpression : group.getLogicalExpressions()) { List> childrenTableMap = new LinkedList<>(); @@ -129,10 +132,9 @@ public class StructInfoMap { } for (Group child : groupExpression.children()) { StructInfoMap childStructInfoMap = child.getstructInfoMap(); - if (!refreshedGroup.contains(child.getGroupId().asInt()) - && refreshVersion != childStructInfoMap.getRefreshVersion()) { - childStructInfoMap.refresh(child, refreshVersion, targetBitSet); - childStructInfoMap.setRefreshVersion(refreshVersion); + if (!refreshedGroup.contains(child.getGroupId().asInt())) { + childStructInfoMap.refresh(child, memoVersion); + childStructInfoMap.setRefreshVersion(memoVersion); } refreshedGroup.add(child.getGroupId().asInt()); childrenTableMap.add(child.getstructInfoMap().getTableMaps()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 5f7dc419ea..73029b4ec1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -150,8 +150,7 @@ public class MaterializedViewUtils { StructInfoMap structInfoMap = ownerGroup.getstructInfoMap(); if (cascadesContext.getMemo().getRefreshVersion() != structInfoMap.getRefreshVersion() || structInfoMap.getTableMaps().isEmpty()) { - structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion(), - materializedViewTableSet); + structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion()); structInfoMap.setRefreshVersion(cascadesContext.getMemo().getRefreshVersion()); } Set queryTableSets = structInfoMap.getTableMaps(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java index 9192f86cf3..217cbf2f1a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java @@ -50,7 +50,7 @@ class StructInfoMapTest extends SqlTestBase { Group root = c1.getMemo().getRoot(); Set tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertTrue(tableMaps.isEmpty()); - root.getstructInfoMap().refresh(root, 1, new BitSet()); + root.getstructInfoMap().refresh(root, 1); Assertions.assertEquals(1, tableMaps.size()); new MockUp() { @Mock @@ -76,7 +76,7 @@ class StructInfoMapTest extends SqlTestBase { .optimize() .printlnBestPlanTree(); root = c1.getMemo().getRoot(); - root.getstructInfoMap().refresh(root, 1, new BitSet()); + root.getstructInfoMap().refresh(root, 1); tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertEquals(2, tableMaps.size()); dropMvByNereids("drop materialized view mv1"); @@ -97,8 +97,8 @@ class StructInfoMapTest extends SqlTestBase { Group root = c1.getMemo().getRoot(); Set tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertTrue(tableMaps.isEmpty()); - root.getstructInfoMap().refresh(root, 1, new BitSet()); - root.getstructInfoMap().refresh(root, 1, new BitSet()); + root.getstructInfoMap().refresh(root, 1); + root.getstructInfoMap().refresh(root, 1); Assertions.assertEquals(1, tableMaps.size()); new MockUp() { @Mock @@ -124,8 +124,8 @@ class StructInfoMapTest extends SqlTestBase { .optimize() .printlnBestPlanTree(); root = c1.getMemo().getRoot(); - root.getstructInfoMap().refresh(root, 1, new BitSet()); - root.getstructInfoMap().refresh(root, 1, new BitSet()); + root.getstructInfoMap().refresh(root, 1); + root.getstructInfoMap().refresh(root, 1); tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertEquals(2, tableMaps.size()); dropMvByNereids("drop materialized view mv1"); @@ -162,7 +162,7 @@ class StructInfoMapTest extends SqlTestBase { .rewrite() .optimize(); Group root = c1.getMemo().getRoot(); - root.getstructInfoMap().refresh(root, 1, new BitSet()); + root.getstructInfoMap().refresh(root, 1); StructInfoMap structInfoMap = root.getstructInfoMap(); Assertions.assertEquals(2, structInfoMap.getTableMaps().size()); BitSet mvMap = structInfoMap.getTableMaps().stream()