database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](Nereids): support comparing outer join in materialized view (#28237)

Browse Source
This commit is contained in:
谢健
2023-12-12 15:36:36 +08:00
committed by GitHub
parent d9fb77ad10
commit 92e04c1453
8 changed files with 211 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
View File

@ -27,6 +27,7 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.DPhyperNode;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
@ -42,6 +43,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.util.PlanUtils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
@ -52,6 +54,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
/**
* The graph is a join graph, whose node is the leaf plan and edge is a join operator.
@ -577,6 +580,99 @@ public class HyperGraph {
LongBitmap.getIterator(addedNodes).forEach(index -> nodes.get(index).attachEdge(edge));
}
/**
* compare hypergraph
*
* @param viewHG the compared hyper graph
* @return null represents not compatible, or return some expression which can
* be pull up from this hyper graph
*/
public @Nullable List<Expression> isLogicCompatible(HyperGraph viewHG, LogicalCompatibilityContext ctx) {
if (viewHG.filterEdges.size() != filterEdges.size() && viewHG.joinEdges.size() != joinEdges.size()) {
return null;
}
Map<Edge, Edge> queryToView = constructEdgeMap(viewHG, ctx.getQueryToViewEdgeExpressionMapping());
if (queryToView.size() != filterEdges.size() + joinEdges.size()) {
return null;
}
boolean allMatch = queryToView.entrySet().stream()
.allMatch(entry ->
compareEdgeWithNode(entry.getKey(), entry.getValue(), ctx.getQueryToViewNodeIDMapping()));
if (!allMatch) {
return null;
}
return ImmutableList.of();
}
private Map<Edge, Edge> constructEdgeMap(HyperGraph viewHG, Map<Expression, Expression> exprMap) {
Map<Expression, Edge> exprToEdge = constructExprMap(viewHG);
Map<Edge, Edge> queryToView = new HashMap<>();
joinEdges.stream()
.filter(e -> !e.getExpressions().isEmpty()
&& exprMap.containsKey(e.getExpression(0))
&& compareEdgeWithExpr(e, exprToEdge.get(exprMap.get(e.getExpression(0))), exprMap))
.forEach(e -> queryToView.put(e, exprToEdge.get(exprMap.get(e.getExpression(0)))));
filterEdges.stream()
.filter(e -> !e.getExpressions().isEmpty()
&& exprMap.containsKey(e.getExpression(0))
&& compareEdgeWithExpr(e, exprToEdge.get(exprMap.get(e.getExpression(0))), exprMap))
.forEach(e -> queryToView.put(e, exprToEdge.get(exprMap.get(e.getExpression(0)))));
return queryToView;
}
private boolean compareEdgeWithNode(Edge t, Edge o, Map<Integer, Integer> nodeMap) {
if (t instanceof FilterEdge && o instanceof FilterEdge) {
return false;
} else if (t instanceof JoinEdge && o instanceof JoinEdge) {
return compareJoinEdge((JoinEdge) t, (JoinEdge) o, nodeMap);
}
return false;
}
private boolean compareJoinEdge(JoinEdge t, JoinEdge o, Map<Integer, Integer> nodeMap) {
long tLeft = t.getLeftExtendedNodes();
long tRight = t.getRightExtendedNodes();
long oLeft = o.getLeftExtendedNodes();
long oRight = o.getRightExtendedNodes();
if (!t.getJoinType().equals(o.getJoinType())) {
if (!t.getJoinType().swap().equals(o.getJoinType())) {
return false;
}
oRight = o.getLeftExtendedNodes();
oLeft = o.getRightExtendedNodes();
}
return compareNodeMap(tLeft, oLeft, nodeMap) && compareNodeMap(tRight, oRight, nodeMap);
}
private boolean compareNodeMap(long bitmap1, long bitmap2, Map<Integer, Integer> nodeIDMap) {
long newBitmap1 = LongBitmap.newBitmap();
for (int i : LongBitmap.getIterator(bitmap1)) {
int mappedI = nodeIDMap.getOrDefault(i, 0);
newBitmap1 = LongBitmap.set(newBitmap1, mappedI);
}
return bitmap2 == newBitmap1;
}
private boolean compareEdgeWithExpr(Edge t, Edge o, Map<Expression, Expression> expressionMap) {
if (t.getExpressions().size() != o.getExpressions().size()) {
return false;
}
int size = t.getExpressions().size();
for (int i = 0; i < size; i++) {
if (!expressionMap.get(t.getExpression(i)).equals(o.getExpression(i))) {
return false;
}
}
return true;
}
private Map<Expression, Edge> constructExprMap(HyperGraph hyperGraph) {
Map<Expression, Edge> exprToEdge = new HashMap<>();
hyperGraph.joinEdges.forEach(edge -> edge.getExpressions().forEach(expr -> exprToEdge.put(expr, edge)));
hyperGraph.filterEdges.forEach(edge -> edge.getExpressions().forEach(expr -> exprToEdge.put(expr, edge)));
return exprToEdge;
}
/**
* For the given hyperGraph, make a textual representation in the form
* of a dotty graph. You can save this to a file and then use Graphviz

4
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
View File

@ -171,6 +171,10 @@ public abstract class Edge {
public abstract List<? extends Expression> getExpressions();
public Expression getExpression(int i) {
return getExpressions().get(i);
}
@Override
public String toString() {
return String.format("<%s - %s>", LongBitmap.toString(leftExtendedNodes), LongBitmap.toString(

4
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View File

@ -105,7 +105,7 @@ public abstract class AbstractMaterializedViewRule {
LogicalCompatibilityContext.from(queryToViewTableMapping, queryToViewSlotMapping,
queryStructInfo, viewStructInfo);
// todo outer join compatibility check
if (!StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, compatibilityContext)) {
if (StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, compatibilityContext) == null) {
continue;
}
SplitPredicate compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo,
@ -347,7 +347,7 @@ public abstract class AbstractMaterializedViewRule {
/**
* Extract struct info from plan, support to get struct info from logical plan or plan in group.
*/
protected List<StructInfo> extractStructInfo(Plan plan, CascadesContext cascadesContext) {
public static List<StructInfo> extractStructInfo(Plan plan, CascadesContext cascadesContext) {
if (plan.getGroupExpression().isPresent()
&& !plan.getGroupExpression().get().getOwnerGroup().getStructInfos().isEmpty()) {
return plan.getGroupExpression().get().getOwnerGroup().getStructInfos();

11
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
View File

@ -36,19 +36,26 @@ import java.util.Map;
* For outer join we should check the outer join compatibility between query and view
*/
public class LogicalCompatibilityContext {
private BiMap<StructInfoNode, StructInfoNode> queryToViewNodeMapping;
private BiMap<Expression, Expression> queryToViewEdgeExpressionMapping;
private final BiMap<StructInfoNode, StructInfoNode> queryToViewNodeMapping;
private final BiMap<Expression, Expression> queryToViewEdgeExpressionMapping;
private final BiMap<Integer, Integer> queryToViewNodeIDMapping;
public LogicalCompatibilityContext(BiMap<StructInfoNode, StructInfoNode> queryToViewNodeMapping,
BiMap<Expression, Expression> queryToViewEdgeExpressionMapping) {
this.queryToViewNodeMapping = queryToViewNodeMapping;
this.queryToViewEdgeExpressionMapping = queryToViewEdgeExpressionMapping;
this.queryToViewNodeIDMapping = HashBiMap.create();
queryToViewNodeMapping.forEach((k, v) -> queryToViewNodeIDMapping.put(k.getIndex(), v.getIndex()));
}
public BiMap<StructInfoNode, StructInfoNode> getQueryToViewNodeMapping() {
return queryToViewNodeMapping;
}
public BiMap<Integer, Integer> getQueryToViewNodeIDMapping() {
return queryToViewNodeIDMapping;
}
public BiMap<Expression, Expression> getQueryToViewEdgeExpressionMapping() {
return queryToViewEdgeExpressionMapping;
}

8
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
View File

@ -38,6 +38,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@ -237,10 +238,11 @@ public class StructInfo {
* For inner join should judge only the join tables,
* for other join type should also judge the join direction, it's input filter that can not be pulled up etc.
*/
public static boolean isGraphLogicalEquals(StructInfo queryStructInfo, StructInfo viewStructInfo,
public static @Nullable List<Expression> isGraphLogicalEquals(StructInfo queryStructInfo, StructInfo viewStructInfo,
LogicalCompatibilityContext compatibilityContext) {
// TODO: if not inner join, should check the join graph logical equivalence
return true;
// TODO: open it after supporting filter
// return queryStructInfo.hyperGraph.isLogicCompatible(viewStructInfo.hyperGraph, compatibilityContext);
return ImmutableList.of();
}
private static class RelationCollector extends DefaultPlanVisitor<Void, List<CatalogRelation>> {

1
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java
View File

@ -126,6 +126,7 @@ public class ExpressionLineageReplacer extends DefaultPlanVisitor<Expression, Ex
.map(each -> each.collectToList(NamedExpression.class::isInstance))
.flatMap(Collection::stream)
.map(NamedExpression.class::cast)
.distinct()
.collect(Collectors.toMap(NamedExpression::getExprId, expr -> expr));
}

90
fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java Normal file
View File

@ -0,0 +1,90 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.jobs.joinorder.hypergraph;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.RuleSet;
import org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewRule;
import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.sqltest.SqlTestBase;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.util.HyperGraphBuilder;
import org.apache.doris.nereids.util.PlanChecker;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
class CompareOuterJoinTest extends SqlTestBase {
@Test
void testStarGraphWithInnerJoin() {
// t2
// |
//t3-- t1 -- t4
// |
// t5
CascadesContext c1 = createCascadesContext(
"select * from T1, T2, T3, T4 where "
+ "T1.id = T2.id "
+ "and T1.id = T3.id "
+ "and T1.id = T4.id ",
connectContext
);
Plan p1 = PlanChecker.from(c1)
.analyze()
.rewrite()
.getPlan();
Plan p2 = PlanChecker.from(c1)
.analyze()
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0);
HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
Assertions.assertTrue(h1.isLogicCompatible(h2, constructContext(p1, p2)) != null);
}
@Test
void testRandomQuery() {
Plan p1 = new HyperGraphBuilder().randomBuildPlanWith(3, 3);
p1 = PlanChecker.from(connectContext, p1)
.analyze()
.rewrite()
.getPlan();
Plan p2 = PlanChecker.from(connectContext, p1)
.analyze()
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0);
HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
Assertions.assertTrue(h1.isLogicCompatible(h2, constructContext(p1, p2)) != null);
}
LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
StructInfo st1 = AbstractMaterializedViewRule.extractStructInfo(p1,
null).get(0);
StructInfo st2 = AbstractMaterializedViewRule.extractStructInfo(p2,
null).get(0);
RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0);
SlotMapping sm = SlotMapping.generate(rm);
return LogicalCompatibilityContext.from(rm, sm, st1, st2);
}
}

4
fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
View File

@ -564,6 +564,10 @@ public class PlanChecker {
return cascadesContext.getMemo().copyOut();
}
public List<Plan> getAllPlan() {
return cascadesContext.getMemo().copyOutAll();
}
private PhysicalPlan chooseBestPlan(Group rootGroup, PhysicalProperties physicalProperties) {
GroupExpression groupExpression = rootGroup.getLowestCostPlan(physicalProperties).orElseThrow(
() -> new AnalysisException("lowestCostPlans with physicalProperties("