database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](mtmv) Add enable materialized view nest rewrite switch (#34197)

Browse Source 
* [opt](mtmv) Add enable materialized view nest rewrite switch

* fix ut

* fix ut2
This commit is contained in:
seawinde
2024-04-27 21:55:35 +08:00
committed by yiguolei
parent 818022cadf
commit 92dc8ed718
9 changed files with 71 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
View File

@ -36,6 +36,7 @@ import org.apache.doris.nereids.trees.plans.LeafPlan;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation;
import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
@ -72,7 +73,6 @@ public class Memo {
EventChannel.getDefaultChannel().addConsumers(new LogConsumer(GroupMergeEvent.class, EventChannel.LOG)));
private static long stateId = 0;
private final ConnectContext connectContext;
private final Set<Long> needRefreshTableIdSet = new HashSet<>();
private final AtomicLong refreshVersion = new AtomicLong(1);
private final IdGenerator<GroupId> groupIdGenerator = GroupId.createGenerator();
private final Map<GroupId, Group> groups = Maps.newLinkedHashMap();
@ -416,7 +416,9 @@ public class Memo {
throw new IllegalStateException("Insert a plan into targetGroup but differ in logicalproperties");
}
// TODO Support sync materialized view in the future
if (plan instanceof LogicalPlan && plan instanceof CatalogRelation
if (connectContext != null
&& connectContext.getSessionVariable().isEnableMaterializedViewNestRewrite()
&& plan instanceof LogicalCatalogRelation
&& ((CatalogRelation) plan).getTable() instanceof MTMV
&& !plan.getGroupExpression().isPresent()) {
refreshVersion.incrementAndGet();

21
fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
View File

@ -48,12 +48,10 @@ public class StructInfoMap {
* get struct info according to table map
*
* @param tableMap the original table map
* @param foldTableMap the fold table map
* @param group the group that the mv matched
* @return struct info or null if not found
*/
public @Nullable StructInfo getStructInfo(Memo memo, BitSet tableMap, BitSet foldTableMap,
Group group, Plan originPlan) {
public @Nullable StructInfo getStructInfo(Memo memo, BitSet tableMap, Group group, Plan originPlan) {
StructInfo structInfo = infoMap.get(tableMap);
if (structInfo != null) {
return structInfo;
@ -84,10 +82,6 @@ public class StructInfoMap {
return groupExpressionMap.get(tableMap);
}
public long getRefreshVersion() {
return refreshVersion;
}
public void setRefreshVersion(long refreshVersion) {
this.refreshVersion = refreshVersion;
}
@ -119,7 +113,8 @@ public class StructInfoMap {
*
*/
public void refresh(Group group, long memoVersion) {
if (memoVersion == group.getstructInfoMap().refreshVersion) {
StructInfoMap structInfoMap = group.getstructInfoMap();
if (!structInfoMap.getTableMaps().isEmpty() && memoVersion == structInfoMap.refreshVersion) {
return;
}
Set<Integer> refreshedGroup = new HashSet<>();
@ -152,8 +147,7 @@ public class StructInfoMap {
}
// if cumulative child table map is different from current
// or current group expression map is empty, should update the groupExpressionMap currently
Collection<Pair<BitSet, List<BitSet>>> bitSetWithChildren = cartesianProduct(childrenTableMap,
new BitSet());
Collection<Pair<BitSet, List<BitSet>>> bitSetWithChildren = cartesianProduct(childrenTableMap);
for (Pair<BitSet, List<BitSet>> bitSetWithChild : bitSetWithChildren) {
groupExpressionMap.putIfAbsent(bitSetWithChild.first,
Pair.of(groupExpression, bitSetWithChild.second));
@ -173,8 +167,7 @@ public class StructInfoMap {
return tableMap;
}
private Collection<Pair<BitSet, List<BitSet>>> cartesianProduct(List<Set<BitSet>> childrenTableMap,
BitSet targetBitSet) {
private Collection<Pair<BitSet, List<BitSet>>> cartesianProduct(List<Set<BitSet>> childrenTableMap) {
Set<List<BitSet>> cartesianLists = Sets.cartesianProduct(childrenTableMap);
List<Pair<BitSet, List<BitSet>>> resultPairSet = new LinkedList<>();
for (List<BitSet> bitSetList : cartesianLists) {
@ -182,10 +175,6 @@ public class StructInfoMap {
for (BitSet b : bitSetList) {
bitSet.or(b);
}
// filter the useless bitset which targetBitSet not contains, avoid exponential expansion
if (!targetBitSet.isEmpty() && !StructInfo.containsAll(targetBitSet, bitSet)) {
continue;
}
resultPairSet.add(Pair.of(bitSet, bitSetList));
}
return resultPairSet;

3
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View File

@ -107,7 +107,6 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
if (checkIfRewritten(queryPlan, context)) {
continue;
}
context.tryReGenerateMvScanPlan(cascadesContext);
// check mv plan is valid or not
if (!checkPattern(context.getStructInfo())) {
context.recordFailReason(context.getStructInfo(),
@ -321,6 +320,8 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
continue;
}
recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext);
// if rewrite successfully, try to regenerate mv scan because it maybe used again
materializationContext.tryReGenerateMvScanPlan(cascadesContext);
rewriteResults.add(rewrittenPlan);
}
return rewriteResults;

11
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
View File

@ -148,11 +148,10 @@ public class MaterializedViewUtils {
if (plan.getGroupExpression().isPresent()) {
Group ownerGroup = plan.getGroupExpression().get().getOwnerGroup();
StructInfoMap structInfoMap = ownerGroup.getstructInfoMap();
if (cascadesContext.getMemo().getRefreshVersion() != structInfoMap.getRefreshVersion()
|| structInfoMap.getTableMaps().isEmpty()) {
structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion());
structInfoMap.setRefreshVersion(cascadesContext.getMemo().getRefreshVersion());
}
// Refresh struct info in current level plan from top to bottom
structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion());
structInfoMap.setRefreshVersion(cascadesContext.getMemo().getRefreshVersion());
Set<BitSet> queryTableSets = structInfoMap.getTableMaps();
ImmutableList.Builder<StructInfo> structInfosBuilder = ImmutableList.builder();
if (!queryTableSets.isEmpty()) {
@ -163,7 +162,7 @@ public class MaterializedViewUtils {
continue;
}
StructInfo structInfo = structInfoMap.getStructInfo(cascadesContext.getMemo(),
queryTableSet, queryTableSet, ownerGroup, plan);
queryTableSet, ownerGroup, plan);
if (structInfo != null) {
structInfosBuilder.add(structInfo);
}

12
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -518,6 +518,9 @@ public class SessionVariable implements Serializable, Writable {
public static final String ENABLE_MATERIALIZED_VIEW_UNION_REWRITE
= "enable_materialized_view_union_rewrite";
public static final String ENABLE_MATERIALIZED_VIEW_NEST_REWRITE
= "enable_materialized_view_nest_rewrite";
public static final String CREATE_TABLE_PARTITION_MAX_NUM
= "create_table_partition_max_num";
@ -1630,6 +1633,11 @@ public class SessionVariable implements Serializable, Writable {
+ "respond to the query"})
public boolean enableMaterializedViewUnionRewrite = false;
@VariableMgr.VarAttr(name = ENABLE_MATERIALIZED_VIEW_NEST_REWRITE, needForward = true,
description = {"是否允许嵌套物化视图改写",
"Whether enable materialized view nest rewrite"})
public boolean enableMaterializedViewNestRewrite = false;
@VariableMgr.VarAttr(name = CREATE_TABLE_PARTITION_MAX_NUM, needForward = true,
description = {"建表时创建分区的最大数量",
"The maximum number of partitions created during table creation"})
@ -3656,6 +3664,10 @@ public class SessionVariable implements Serializable, Writable {
return enableMaterializedViewUnionRewrite;
}
public boolean isEnableMaterializedViewNestRewrite() {
return enableMaterializedViewNestRewrite;
}
public int getCreateTablePartitionMaxNum() {
return createTablePartitionMaxNum;
}