database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](mtmv) Optimize the nested materialized view rewrite performance (#34050)

Browse Source 
Optimize the nested materialized view rewrite performance when exists many join
This is brought by #33362
This commit is contained in:
seawinde
2024-04-24 18:56:36 +08:00
committed by yiguolei
parent 67b394f2b0
commit a720e03a02
3 changed files with 57 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
View File

@ -411,6 +411,9 @@ public class Memo {
Preconditions.checkState(groupExpressions.containsKey(groupExpr.get()));
return CopyInResult.of(false, groupExpr.get());
}
if (targetGroup != null) {
targetGroup.getstructInfoMap().setRefreshed(false);
}
List<Group> childrenGroups = Lists.newArrayList();
for (int i = 0; i < plan.children().size(); i++) {
// skip useless project.
@ -559,6 +562,7 @@ public class Memo {
if (source == root) {
root = destination;
}
destination.getstructInfoMap().setRefreshed(false);
groups.remove(source.getGroupId());
}

59
fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
View File

@ -29,10 +29,11 @@ import java.util.BitSet;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
/**
@ -41,6 +42,7 @@ import javax.annotation.Nullable;
public class StructInfoMap {
private final Map<BitSet, Pair<GroupExpression, List<BitSet>>> groupExpressionMap = new HashMap<>();
private final Map<BitSet, StructInfo> infoMap = new HashMap<>();
private boolean refreshed;
/**
* get struct info according to table map
@ -79,6 +81,14 @@ public class StructInfoMap {
return groupExpressionMap.get(tableMap);
}
public boolean isRefreshed() {
return refreshed;
}
public void setRefreshed(boolean refreshed) {
this.refreshed = refreshed;
}
private StructInfo constructStructInfo(GroupExpression groupExpression, List<BitSet> children,
BitSet tableMap, Plan originPlan) {
// this plan is not origin plan, should record origin plan in struct info
@ -111,7 +121,7 @@ public class StructInfoMap {
int originSize = groupExpressionMap.size();
for (GroupExpression groupExpression : group.getLogicalExpressions()) {
List<Set<BitSet>> childrenTableMap = new ArrayList<>();
boolean needRefresh = false;
boolean needRefresh = groupExpressionMap.isEmpty();
if (groupExpression.children().isEmpty()) {
BitSet leaf = constructLeaf(groupExpression);
groupExpressionMap.put(leaf, Pair.of(groupExpression, new ArrayList<>()));
@ -119,18 +129,33 @@ public class StructInfoMap {
}
for (Group child : groupExpression.children()) {
if (!refreshedGroup.contains(child)) {
if (!refreshedGroup.contains(child) && !child.getstructInfoMap().isRefreshed()) {
StructInfoMap childStructInfoMap = child.getstructInfoMap();
needRefresh |= childStructInfoMap.refresh(child);
childStructInfoMap.setRefreshed(true);
}
refreshedGroup.add(child);
childrenTableMap.add(child.getstructInfoMap().getTableMaps());
}
// if one same groupExpression have refreshed, continue
BitSet oneOfGroupExpressionTableSet = new BitSet();
for (Set<BitSet> groupExpressionBitSet : childrenTableMap) {
Iterator<BitSet> iterator = groupExpressionBitSet.iterator();
if (iterator.hasNext()) {
oneOfGroupExpressionTableSet.or(iterator.next());
}
}
if (groupExpressionMap.containsKey(oneOfGroupExpressionTableSet)) {
continue;
}
// if cumulative child table map is different from current
// or current group expression map is empty, should update the groupExpressionMap currently
Set<Pair<BitSet, List<BitSet>>> bitSetWithChildren = cartesianProduct(childrenTableMap);
for (Pair<BitSet, List<BitSet>> bitSetWithChild : bitSetWithChildren) {
groupExpressionMap.putIfAbsent(bitSetWithChild.first, Pair.of(groupExpression, bitSetWithChild.second));
Collection<Pair<BitSet, List<BitSet>>> bitSetWithChildren = cartesianProduct(childrenTableMap);
if (needRefresh) {
for (Pair<BitSet, List<BitSet>> bitSetWithChild : bitSetWithChildren) {
groupExpressionMap.putIfAbsent(bitSetWithChild.first,
Pair.of(groupExpression, bitSetWithChild.second));
}
}
}
return originSize != groupExpressionMap.size();
@ -147,17 +172,17 @@ public class StructInfoMap {
return tableMap;
}
private Set<Pair<BitSet, List<BitSet>>> cartesianProduct(List<Set<BitSet>> childrenTableMap) {
return Sets.cartesianProduct(childrenTableMap)
.stream()
.map(bitSetList -> {
BitSet bitSet = new BitSet();
for (BitSet b : bitSetList) {
bitSet.or(b);
}
return Pair.of(bitSet, bitSetList);
})
.collect(Collectors.toSet());
private Collection<Pair<BitSet, List<BitSet>>> cartesianProduct(List<Set<BitSet>> childrenTableMap) {
Set<List<BitSet>> cartesianLists = Sets.cartesianProduct(childrenTableMap);
List<Pair<BitSet, List<BitSet>>> resultPairSet = new LinkedList<>();
for (List<BitSet> bitSetList : cartesianLists) {
BitSet bitSet = new BitSet();
for (BitSet b : bitSetList) {
bitSet.or(b);
}
resultPairSet.add(Pair.of(bitSet, bitSetList));
}
return resultPairSet;
}
@Override

14
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
View File

@ -443,10 +443,18 @@ public class StructInfo {
}
}
/** Judge if source contains all target */
public static boolean containsAll(BitSet source, BitSet target) {
BitSet intersection = (BitSet) source.clone();
intersection.and(target);
return intersection.equals(target);
if (source.size() < target.size()) {
return false;
}
for (int i = target.nextSetBit(0); i >= 0; i = target.nextSetBit(i + 1)) {
boolean contains = source.get(i);
if (!contains) {
return false;
}
}
return true;
}
/**