database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feat](Nereids): use table map to construct struct info (#32058)

Browse Source
This commit is contained in:
谢健
2024-03-12 15:52:36 +08:00
committed by yiguolei
parent 45824d959c
commit 2da57526a3
3 changed files with 108 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.nereids.memo;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
@ -37,11 +38,12 @@ import javax.annotation.Nullable;
* Representation for group in cascades optimizer.
*/
public class StructInfoMap {
private final Map<BitSet, GroupExpression> groupExpressionMap = new HashMap<>();
private final Map<BitSet, Pair<GroupExpression, List<BitSet>>> groupExpressionMap = new HashMap<>();
private final Map<BitSet, StructInfo> infoMap = new HashMap<>();
/**
* get struct info according to table map
*
* @param mvTableMap the original table map
* @param foldTableMap the fold table map
* @param group the group that the mv matched
@ -54,7 +56,10 @@ public class StructInfoMap {
refresh(group);
}
if (groupExpressionMap.containsKey(mvTableMap)) {
StructInfo structInfo = constructStructInfo(groupExpressionMap.get(mvTableMap));
Pair<GroupExpression, List<BitSet>> groupExpressionBitSetPair = getGroupExpressionWithChildren(
mvTableMap);
StructInfo structInfo = constructStructInfo(groupExpressionBitSetPair.first,
groupExpressionBitSetPair.second, mvTableMap);
infoMap.put(mvTableMap, structInfo);
}
}
@ -66,20 +71,39 @@ public class StructInfoMap {
return groupExpressionMap.keySet();
}
private StructInfo constructStructInfo(GroupExpression groupExpression) {
throw new RuntimeException("has not been implemented for" + groupExpression);
public Pair<GroupExpression, List<BitSet>> getGroupExpressionWithChildren(BitSet tableMap) {
return groupExpressionMap.get(tableMap);
}
private StructInfo constructStructInfo(GroupExpression groupExpression, List<BitSet> children, BitSet tableMap) {
Plan plan = constructPlan(groupExpression, children, tableMap);
return StructInfo.of(plan).get(0);
}
private Plan constructPlan(GroupExpression groupExpression, List<BitSet> children, BitSet tableMap) {
List<Plan> childrenPlan = new ArrayList<>();
for (int i = 0; i < children.size(); i++) {
StructInfoMap structInfoMap = groupExpression.child(i).getstructInfoMap();
BitSet childMap = children.get(i);
Pair<GroupExpression, List<BitSet>> groupExpressionBitSetPair
= structInfoMap.getGroupExpressionWithChildren(childMap);
childrenPlan.add(
constructPlan(groupExpressionBitSetPair.first, groupExpressionBitSetPair.second, childMap));
}
return groupExpression.getPlan().withChildren(childrenPlan);
}
/**
* refresh group expression map
*
* @param group the root group
*/
public void refresh(Group group) {
List<Set<BitSet>> childrenTableMap = new ArrayList<>();
Set<Group> refreshedGroup = new HashSet<>();
for (GroupExpression groupExpression : group.getLogicalExpressions()) {
List<Set<BitSet>> childrenTableMap = new ArrayList<>();
if (groupExpression.children().isEmpty()) {
groupExpressionMap.put(constructLeaf(groupExpression), groupExpression);
groupExpressionMap.put(constructLeaf(groupExpression), Pair.of(groupExpression, new ArrayList<>()));
continue;
}
for (Group child : groupExpression.children()) {
@ -90,9 +114,9 @@ public class StructInfoMap {
refreshedGroup.add(child);
childrenTableMap.add(child.getstructInfoMap().getTableMaps());
}
Set<BitSet> bitSets = cartesianProduct(childrenTableMap);
for (BitSet bitSet : bitSets) {
groupExpressionMap.put(bitSet, groupExpression);
Set<Pair<BitSet, List<BitSet>>> bitSetWithChildren = cartesianProduct(childrenTableMap);
for (Pair<BitSet, List<BitSet>> bitSetWithChild : bitSetWithChildren) {
groupExpressionMap.put(bitSetWithChild.first, Pair.of(groupExpression, bitSetWithChild.second));
}
}
}
@ -108,7 +132,7 @@ public class StructInfoMap {
return tableMap;
}
private Set<BitSet> cartesianProduct(List<Set<BitSet>> childrenTableMap) {
private Set<Pair<BitSet, List<BitSet>>> cartesianProduct(List<Set<BitSet>> childrenTableMap) {
return Sets.cartesianProduct(childrenTableMap)
.stream()
.map(bitSetList -> {
@ -116,7 +140,7 @@ public class StructInfoMap {
for (BitSet b : bitSetList) {
bitSet.or(b);
}
return bitSet;
return Pair.of(bitSet, bitSetList);
})
.collect(Collectors.toSet());
}

48
fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.nereids.memo;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.mtmv.MTMVRelationManager;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
import org.apache.doris.nereids.sqltest.SqlTestBase;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.qe.ConnectContext;
@ -31,6 +32,7 @@ import org.junit.jupiter.api.Test;
import java.util.BitSet;
import java.util.Set;
import java.util.stream.Collectors;
class StructInfoMapTest extends SqlTestBase {
@Test
@ -77,5 +79,51 @@ class StructInfoMapTest extends SqlTestBase {
root.getstructInfoMap().refresh(root);
tableMaps = root.getstructInfoMap().getTableMaps();
Assertions.assertEquals(2, tableMaps.size());
dropMvByNereids("drop materialized view mv1");
}
@Test
void testTableChild() throws Exception {
CascadesContext c1 = createCascadesContext(
"select T1.id from T1 inner join T2 "
+ "on T1.id = T2.id "
+ "inner join T3 on T1.id = T3.id",
connectContext
);
new MockUp<MTMVRelationManager>() {
@Mock
public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) {
return true;
}
};
connectContext.getSessionVariable().enableMaterializedViewRewrite = true;
createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n"
+ " DISTRIBUTED BY RANDOM BUCKETS 1\n"
+ " PROPERTIES ('replication_num' = '1') \n"
+ " as select T1.id from T1 inner join T2 "
+ "on T1.id = T2.id;");
c1 = createCascadesContext(
"select T1.id from T1 inner join T2 "
+ "on T1.id = T2.id "
+ "inner join T3 on T1.id = T3.id",
connectContext
);
PlanChecker.from(c1)
.analyze()
.rewrite()
.optimize();
Group root = c1.getMemo().getRoot();
root.getstructInfoMap().refresh(root);
StructInfoMap structInfoMap = root.getstructInfoMap();
Assertions.assertEquals(2, structInfoMap.getTableMaps().size());
BitSet mvMap = structInfoMap.getTableMaps().stream()
.filter(b -> b.cardinality() == 2)
.collect(Collectors.toList()).get(0);
StructInfo structInfo = structInfoMap.getStructInfo(mvMap, mvMap, root);
System.out.println(structInfo.getOriginalPlan().treeString());
BitSet bitSet = new BitSet();
structInfo.getRelations().forEach(r -> bitSet.set((int) r.getTable().getId()));
Assertions.assertEquals(bitSet, mvMap);
dropMvByNereids("drop materialized view mv1");
}
}

25
fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
View File

@ -69,6 +69,7 @@ import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand;
import org.apache.doris.nereids.trees.plans.commands.DropConstraintCommand;
import org.apache.doris.nereids.trees.plans.commands.DropMTMVCommand;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.util.MemoTestUtils;
import org.apache.doris.persist.CreateTableInfo;
@ -823,6 +824,30 @@ public abstract class TestWithFeService {
}
protected void dropMvByNereids(String sql) throws Exception {
new MockUp<EditLog>() {
@Mock
public void logCreateTable(CreateTableInfo info) {
System.out.println("skip log create table...");
}
@Mock
public void logCreateJob(AbstractJob job) {
System.out.println("skip log create job...");
}
};
NereidsParser nereidsParser = new NereidsParser();
LogicalPlan parsed = nereidsParser.parseSingle(sql);
StmtExecutor stmtExecutor = new StmtExecutor(connectContext, sql);
if (parsed instanceof DropMTMVCommand) {
((DropMTMVCommand) parsed).run(connectContext, stmtExecutor);
}
checkAlterJob();
// waiting table state to normal
Thread.sleep(1000);
}
private void updateReplicaPathHash() {
com.google.common.collect.Table<Long, Long, Replica> replicaMetaTable = Env.getCurrentInvertedIndex()
.getReplicaMetaTable();