database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhance](mtmv) add baseTablesOneLevel for MTMVRelation and MTMVRelationManager (#34593)

Browse Source 
baseTables in MTMVRelation  stores all baseTables in the nested materialized view,
now adding baseTablesOneLevel to only store the baseTables at the current level.
This commit is contained in:
zhangdong
2024-05-10 10:27:45 +08:00
committed by yiguolei
parent 719e50f353
commit 958f1a2d2d
6 changed files with 154 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
View File

@ -117,14 +117,14 @@ public class MTMVPlanUtil {
}
public static MTMVRelation generateMTMVRelation(Plan plan) {
return new MTMVRelation(getBaseTables(plan), getBaseViews(plan));
return new MTMVRelation(getBaseTables(plan, true), getBaseTables(plan, false), getBaseViews(plan));
}
private static Set<BaseTableInfo> getBaseTables(Plan plan) {
private static Set<BaseTableInfo> getBaseTables(Plan plan, boolean expand) {
TableCollectorContext collectorContext =
new TableCollector.TableCollectorContext(
com.google.common.collect.Sets
.newHashSet(TableType.values()), true);
.newHashSet(TableType.values()), expand);
plan.accept(TableCollector.INSTANCE, collectorContext);
Set<TableIf> collectedTables = collectorContext.getCollectedTables();
return transferTableIfToInfo(collectedTables);

12
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java
View File

@ -30,9 +30,13 @@ public class MTMVRelation {
private Set<BaseTableInfo> baseTables;
@SerializedName("bv")
private Set<BaseTableInfo> baseViews;
@SerializedName("btol")
private Set<BaseTableInfo> baseTablesOneLevel;
public MTMVRelation(Set<BaseTableInfo> baseTables, Set<BaseTableInfo> baseViews) {
public MTMVRelation(Set<BaseTableInfo> baseTables, Set<BaseTableInfo> baseTablesOneLevel,
Set<BaseTableInfo> baseViews) {
this.baseTables = baseTables;
this.baseTablesOneLevel = baseTablesOneLevel;
this.baseViews = baseViews;
}
@ -40,6 +44,11 @@ public class MTMVRelation {
return baseTables;
}
public Set<BaseTableInfo> getBaseTablesOneLevel() {
// For compatibility, previously created MTMV may not have baseTablesOneLevel
return baseTablesOneLevel == null ? baseTables : baseTablesOneLevel;
}
public Set<BaseTableInfo> getBaseViews() {
return baseViews;
}
@ -48,6 +57,7 @@ public class MTMVRelation {
public String toInfoString() {
return "MTMVRelation{"
+ "baseTables=" + baseTables
+ ", baseTablesOneLevel=" + baseTablesOneLevel
+ ", baseViews=" + baseViews
+ '}';
}

34
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
View File

@ -53,12 +53,22 @@ import java.util.Set;
*/
public class MTMVRelationManager implements MTMVHookService {
private static final Logger LOG = LogManager.getLogger(MTMVRelationManager.class);
// when
// create mv1 as select * from table1;
// create mv2 as select * from mv1;
// `tableMTMVs` will have 3 pair: table1 ==> mv1,mv1==>mv2, table1 ==> mv2
// `tableMTMVsOneLevel` will have 2 pair: table1 ==> mv1,mv1==>mv2
private Map<BaseTableInfo, Set<BaseTableInfo>> tableMTMVs = Maps.newConcurrentMap();
private Map<BaseTableInfo, Set<BaseTableInfo>> tableMTMVsOneLevel = Maps.newConcurrentMap();
private Set<BaseTableInfo> getMtmvsByBaseTable(BaseTableInfo table) {
public Set<BaseTableInfo> getMtmvsByBaseTable(BaseTableInfo table) {
return tableMTMVs.getOrDefault(table, ImmutableSet.of());
}
public Set<BaseTableInfo> getMtmvsByBaseTableOneLevel(BaseTableInfo table) {
return tableMTMVsOneLevel.getOrDefault(table, ImmutableSet.of());
}
/**
* if At least one partition is available, return this mtmv
*
@ -104,7 +114,14 @@ public class MTMVRelationManager implements MTMVHookService {
return tableMTMVs.get(baseTableInfo);
}
private void refreshMTMVCache(MTMVRelation relation, BaseTableInfo mtmvInfo) {
private Set<BaseTableInfo> getOrCreateMTMVsOneLevel(BaseTableInfo baseTableInfo) {
if (!tableMTMVsOneLevel.containsKey(baseTableInfo)) {
tableMTMVsOneLevel.put(baseTableInfo, Sets.newConcurrentHashSet());
}
return tableMTMVsOneLevel.get(baseTableInfo);
}
public void refreshMTMVCache(MTMVRelation relation, BaseTableInfo mtmvInfo) {
LOG.info("refreshMTMVCache,relation: {}, mtmvInfo: {}", relation, mtmvInfo);
removeMTMV(mtmvInfo);
addMTMV(relation, mtmvInfo);
@ -116,6 +133,7 @@ public class MTMVRelationManager implements MTMVHookService {
}
addMTMVTables(relation.getBaseTables(), mtmvInfo);
addMTMVTables(relation.getBaseViews(), mtmvInfo);
addMTMVTablesOneLevel(relation.getBaseTablesOneLevel(), mtmvInfo);
}
private void addMTMVTables(Set<BaseTableInfo> baseTables, BaseTableInfo mtmvInfo) {
@ -127,10 +145,22 @@ public class MTMVRelationManager implements MTMVHookService {
}
}
private void addMTMVTablesOneLevel(Set<BaseTableInfo> baseTables, BaseTableInfo mtmvInfo) {
if (CollectionUtils.isEmpty(baseTables)) {
return;
}
for (BaseTableInfo baseTableInfo : baseTables) {
getOrCreateMTMVsOneLevel(baseTableInfo).add(mtmvInfo);
}
}
private void removeMTMV(BaseTableInfo mtmvInfo) {
for (Set<BaseTableInfo> sets : tableMTMVs.values()) {
sets.remove(mtmvInfo);
}
for (Set<BaseTableInfo> sets : tableMTMVsOneLevel.values()) {
sets.remove(mtmvInfo);
}
}
@Override