database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](mtmv) generate bi-map between base table and materialized view partitions (#35131)

Browse Source
This commit is contained in:
zhangdong
2024-05-22 20:07:06 +08:00
committed by yiguolei
parent 9ba995317a
commit a52ee6e9b9
2 changed files with 100 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.PartitionKeyDesc;
import org.apache.doris.catalog.OlapTableFactory.MTMVParams;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.job.common.TaskStatus;
@ -316,6 +317,44 @@ public class MTMV extends OlapTable {
return result;
}
/**
* Calculate the partition and associated partition mapping relationship of the MTMV
* It is the result of real-time comparison calculation, so there may be some costs,
* so it should be called with caution.
* The reason for not directly calling `calculatePartitionMappings` and
* generating a reverse index is to directly generate two maps here,
* without the need to traverse them again
*
* @return mvPartitionName ==> relationPartitionNames and relationPartitionName ==> mvPartitionName
* @throws AnalysisException
*/
public Pair<Map<String, Set<String>>, Map<String, String>> calculateDoublyPartitionMappings()
throws AnalysisException {
if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.SELF_MANAGE) {
return Pair.of(Maps.newHashMap(), Maps.newHashMap());
}
long start = System.currentTimeMillis();
Map<String, Set<String>> mvToBase = Maps.newHashMap();
Map<String, String> baseToMv = Maps.newHashMap();
Map<PartitionKeyDesc, Set<String>> relatedPartitionDescs = MTMVPartitionUtil
.generateRelatedPartitionDescs(mvPartitionInfo, mvProperties);
Map<String, PartitionItem> mvPartitionItems = getAndCopyPartitionItems();
for (Entry<String, PartitionItem> entry : mvPartitionItems.entrySet()) {
Set<String> basePartitionNames = relatedPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(),
Sets.newHashSet());
String mvPartitionName = entry.getKey();
mvToBase.put(mvPartitionName, basePartitionNames);
for (String basePartitionName : basePartitionNames) {
baseToMv.put(basePartitionName, mvPartitionName);
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("calculateDoublyPartitionMappings use [{}] mills, mvName is [{}]",
System.currentTimeMillis() - start, name);
}
return Pair.of(mvToBase, baseToMv);
}
/**
* Calculate the partition and associated partition mapping relationship of the MTMV
* It is the result of real-time comparison calculation, so there may be some costs,