database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feat](Nereids): add struct info map in group (#31800)

Browse Source
This commit is contained in:
谢健
2024-03-06 16:49:04 +08:00
committed by yiguolei
parent 2e4201e406
commit d0847a8953
7 changed files with 252 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
View File

@ -35,6 +35,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo;
import org.apache.doris.persist.AlterMTMV;
import org.apache.doris.qe.ConnectContext;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@ -71,8 +72,7 @@ public class MTMVRelationManager implements MTMVHookService {
for (BaseTableInfo tableInfo : mvInfos) {
try {
MTMV mtmv = (MTMV) MTMVUtil.getTable(tableInfo);
if (!CollectionUtils
.isEmpty(MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv, ctx, System.currentTimeMillis()))) {
if (isMVPartitionValid(mtmv, ctx)) {
res.add(mtmv);
}
} catch (AnalysisException e) {
@ -83,6 +83,12 @@ public class MTMVRelationManager implements MTMVHookService {
return res;
}
@VisibleForTesting
public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) {
return !CollectionUtils
.isEmpty(MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv, ctx, System.currentTimeMillis()));
}
private Set<BaseTableInfo> getMTMVInfos(List<BaseTableInfo> tableInfos) {
Set<BaseTableInfo> mvInfos = Sets.newHashSet();
for (BaseTableInfo tableInfo : tableInfos) {

7
fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
View File

@ -19,7 +19,6 @@ package org.apache.doris.nereids.memo;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.cost.Cost;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
@ -78,6 +77,8 @@ public class Group {
private List<StructInfo> structInfos = new ArrayList<>();
private StructInfoMap structInfoMap = new StructInfoMap();
/**
* Constructor for Group.
*
@ -418,8 +419,8 @@ public class Group {
return false;
}
public List<HyperGraph> getHyperGraphs() {
return new ArrayList<>();
public StructInfoMap getstructInfoMap() {
return structInfoMap;
}
public boolean isProjectGroup() {

123
fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java Normal file
View File

@ -0,0 +1,123 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.memo;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
/**
* Representation for group in cascades optimizer.
*/
public class StructInfoMap {
private final Map<BitSet, GroupExpression> groupExpressionMap = new HashMap<>();
private final Map<BitSet, StructInfo> infoMap = new HashMap<>();
/**
* get struct info according to table map
* @param mvTableMap the original table map
* @param foldTableMap the fold table map
* @param group the group that the mv matched
* @return struct info or null if not found
*/
public @Nullable StructInfo getStructInfo(BitSet mvTableMap, BitSet foldTableMap, Group group) {
if (!infoMap.containsKey(mvTableMap)) {
if ((groupExpressionMap.containsKey(foldTableMap) || groupExpressionMap.isEmpty())
&& !groupExpressionMap.containsKey(mvTableMap)) {
refresh(group);
}
if (groupExpressionMap.containsKey(mvTableMap)) {
StructInfo structInfo = constructStructInfo(groupExpressionMap.get(mvTableMap));
infoMap.put(mvTableMap, structInfo);
}
}
return infoMap.get(mvTableMap);
}
public Set<BitSet> getTableMaps() {
return groupExpressionMap.keySet();
}
private StructInfo constructStructInfo(GroupExpression groupExpression) {
throw new RuntimeException("has not been implemented for" + groupExpression);
}
/**
* refresh group expression map
* @param group the root group
*/
public void refresh(Group group) {
List<Set<BitSet>> childrenTableMap = new ArrayList<>();
Set<Group> refreshedGroup = new HashSet<>();
for (GroupExpression groupExpression : group.getLogicalExpressions()) {
if (groupExpression.children().isEmpty()) {
groupExpressionMap.put(constructLeaf(groupExpression), groupExpression);
continue;
}
for (Group child : groupExpression.children()) {
if (!refreshedGroup.contains(child)) {
StructInfoMap childStructInfoMap = child.getstructInfoMap();
childStructInfoMap.refresh(child);
}
refreshedGroup.add(child);
childrenTableMap.add(child.getstructInfoMap().getTableMaps());
}
Set<BitSet> bitSets = cartesianProduct(childrenTableMap);
for (BitSet bitSet : bitSets) {
groupExpressionMap.put(bitSet, groupExpression);
}
}
}
private BitSet constructLeaf(GroupExpression groupExpression) {
Plan plan = groupExpression.getPlan();
BitSet tableMap = new BitSet();
if (plan instanceof LogicalCatalogRelation) {
// TODO: Bitmap is not compatible with long, use tree map instead
tableMap.set((int) ((LogicalCatalogRelation) plan).getTable().getId());
}
// one row relation / CTE consumer
return tableMap;
}
private Set<BitSet> cartesianProduct(List<Set<BitSet>> childrenTableMap) {
return Sets.cartesianProduct(childrenTableMap)
.stream()
.map(bitSetList -> {
BitSet bitSet = new BitSet();
for (BitSet b : bitSetList) {
bitSet.or(b);
}
return bitSet;
})
.collect(Collectors.toSet());
}
}

7
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
View File

@ -32,6 +32,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.visitor.TableCollector;
import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@ -56,7 +57,11 @@ public class InitMaterializationContextHook implements PlannerHook {
initMaterializationContext(planner.getCascadesContext());
}
private void initMaterializationContext(CascadesContext cascadesContext) {
/**
* init materialization context
*/
@VisibleForTesting
public void initMaterializationContext(CascadesContext cascadesContext) {
if (!cascadesContext.getConnectContext().getSessionVariable().isEnableMaterializedViewRewrite()) {
return;
}