[feature](Nereids): refresh view hypergraph after inferring join (#29469)

This commit is contained in:
谢健
2024-01-08 15:30:14 +08:00
committed by yiguolei
parent 0126b09c11
commit b59a8c9365
6 changed files with 399 additions and 103 deletions

View File

@ -0,0 +1,108 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.jobs.joinorder.hypergraph;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.FilterEdge;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin;
import org.apache.doris.nereids.trees.plans.JoinType;
import java.util.BitSet;
import java.util.List;
/**
* This is a conflict rule maker to
*/
public class ConflictRulesMaker {
private ConflictRulesMaker() {}
/**
* make conflict rules for filter edges
*/
public static void makeFilterConflictRules(
JoinEdge joinEdge, List<JoinEdge> joinEdges, List<FilterEdge> filterEdges) {
long leftSubNodes = joinEdge.getLeftSubNodes(joinEdges);
long rightSubNodes = joinEdge.getRightSubNodes(joinEdges);
filterEdges.forEach(e -> {
if (LongBitmap.isSubset(e.getReferenceNodes(), leftSubNodes)
&& !PushDownFilterThroughJoin.COULD_PUSH_THROUGH_LEFT.contains(joinEdge.getJoinType())) {
e.addLeftRejectEdge(joinEdge);
}
if (LongBitmap.isSubset(e.getReferenceNodes(), rightSubNodes)
&& !PushDownFilterThroughJoin.COULD_PUSH_THROUGH_RIGHT.contains(joinEdge.getJoinType())) {
e.addRightRejectEdge(joinEdge);
}
});
}
/**
* Make edge with CD-C algorithm in
* On the correct and complete enumeration of the core search
*/
public static void makeJoinConflictRules(JoinEdge edgeB, List<JoinEdge> joinEdges) {
BitSet leftSubTreeEdges = subTreeEdges(edgeB.getLeftChildEdges(), joinEdges);
BitSet rightSubTreeEdges = subTreeEdges(edgeB.getRightChildEdges(), joinEdges);
long leftRequired = edgeB.getLeftRequiredNodes();
long rightRequired = edgeB.getRightRequiredNodes();
for (int i = leftSubTreeEdges.nextSetBit(0); i >= 0; i = leftSubTreeEdges.nextSetBit(i + 1)) {
JoinEdge childA = joinEdges.get(i);
if (!JoinType.isAssoc(childA.getJoinType(), edgeB.getJoinType())) {
leftRequired = LongBitmap.newBitmapUnion(leftRequired, childA.getLeftSubNodes(joinEdges));
childA.addLeftRejectEdge(edgeB);
}
if (!JoinType.isLAssoc(childA.getJoinType(), edgeB.getJoinType())) {
leftRequired = LongBitmap.newBitmapUnion(leftRequired, childA.getRightSubNodes(joinEdges));
childA.addLeftRejectEdge(edgeB);
}
}
for (int i = rightSubTreeEdges.nextSetBit(0); i >= 0; i = rightSubTreeEdges.nextSetBit(i + 1)) {
JoinEdge childA = joinEdges.get(i);
if (!JoinType.isAssoc(edgeB.getJoinType(), childA.getJoinType())) {
rightRequired = LongBitmap.newBitmapUnion(rightRequired, childA.getRightSubNodes(joinEdges));
childA.addRightRejectEdge(edgeB);
}
if (!JoinType.isRAssoc(edgeB.getJoinType(), childA.getJoinType())) {
rightRequired = LongBitmap.newBitmapUnion(rightRequired, childA.getLeftSubNodes(joinEdges));
childA.addRightRejectEdge(edgeB);
}
}
edgeB.setLeftExtendedNodes(leftRequired);
edgeB.setRightExtendedNodes(rightRequired);
}
private static BitSet subTreeEdge(Edge edge, List<JoinEdge> joinEdges) {
long subTreeNodes = edge.getSubTreeNodes();
BitSet subEdges = new BitSet();
joinEdges.stream()
.filter(e -> LongBitmap.isSubset(subTreeNodes, e.getReferenceNodes()))
.forEach(e -> subEdges.set(e.getIndex()));
return subEdges;
}
private static BitSet subTreeEdges(BitSet edgeSet, List<JoinEdge> joinEdges) {
BitSet bitSet = new BitSet();
edgeSet.stream()
.mapToObj(i -> subTreeEdge(joinEdges.get(i), joinEdges))
.forEach(bitSet::or);
return bitSet;
}
}

View File

@ -28,7 +28,6 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.DPhyperNode;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@ -244,8 +243,9 @@ public class HyperGraph {
joinEdges.add(edge);
}
curJoinEdges.stream().forEach(i -> joinEdges.get(i).addCurJoinEdges(curJoinEdges));
curJoinEdges.stream().forEach(i -> makeJoinConflictRules(joinEdges.get(i)));
curJoinEdges.stream().forEach(i -> makeFilterConflictRules(joinEdges.get(i)));
curJoinEdges.stream().forEach(i -> ConflictRulesMaker.makeJoinConflictRules(joinEdges.get(i), joinEdges));
curJoinEdges.stream().forEach(i ->
ConflictRulesMaker.makeFilterConflictRules(joinEdges.get(i), joinEdges, filterEdges));
return curJoinEdges;
// In MySQL, each edge is reversed and store in edges again for reducing the branch miss
// We don't implement this trick now.
@ -260,73 +260,6 @@ public class HyperGraph {
return bitSet;
}
private void makeFilterConflictRules(JoinEdge joinEdge) {
long leftSubNodes = joinEdge.getLeftSubNodes(joinEdges);
long rightSubNodes = joinEdge.getRightSubNodes(joinEdges);
filterEdges.forEach(e -> {
if (LongBitmap.isSubset(e.getReferenceNodes(), leftSubNodes)
&& !PushDownFilterThroughJoin.COULD_PUSH_THROUGH_LEFT.contains(joinEdge.getJoinType())) {
e.addLeftRejectEdge(joinEdge);
}
if (LongBitmap.isSubset(e.getReferenceNodes(), rightSubNodes)
&& !PushDownFilterThroughJoin.COULD_PUSH_THROUGH_RIGHT.contains(joinEdge.getJoinType())) {
e.addRightRejectEdge(joinEdge);
}
});
}
// Make edge with CD-C algorithm in
// On the correct and complete enumeration of the core search
private void makeJoinConflictRules(JoinEdge edgeB) {
BitSet leftSubTreeEdges = subTreeEdges(edgeB.getLeftChildEdges());
BitSet rightSubTreeEdges = subTreeEdges(edgeB.getRightChildEdges());
long leftRequired = edgeB.getLeftRequiredNodes();
long rightRequired = edgeB.getRightRequiredNodes();
for (int i = leftSubTreeEdges.nextSetBit(0); i >= 0; i = leftSubTreeEdges.nextSetBit(i + 1)) {
JoinEdge childA = joinEdges.get(i);
if (!JoinType.isAssoc(childA.getJoinType(), edgeB.getJoinType())) {
leftRequired = LongBitmap.newBitmapUnion(leftRequired, childA.getLeftSubNodes(joinEdges));
childA.addLeftRejectEdge(edgeB);
}
if (!JoinType.isLAssoc(childA.getJoinType(), edgeB.getJoinType())) {
leftRequired = LongBitmap.newBitmapUnion(leftRequired, childA.getRightSubNodes(joinEdges));
childA.addLeftRejectEdge(edgeB);
}
}
for (int i = rightSubTreeEdges.nextSetBit(0); i >= 0; i = rightSubTreeEdges.nextSetBit(i + 1)) {
JoinEdge childA = joinEdges.get(i);
if (!JoinType.isAssoc(edgeB.getJoinType(), childA.getJoinType())) {
rightRequired = LongBitmap.newBitmapUnion(rightRequired, childA.getRightSubNodes(joinEdges));
childA.addRightRejectEdge(edgeB);
}
if (!JoinType.isRAssoc(edgeB.getJoinType(), childA.getJoinType())) {
rightRequired = LongBitmap.newBitmapUnion(rightRequired, childA.getLeftSubNodes(joinEdges));
childA.addRightRejectEdge(edgeB);
}
}
edgeB.setLeftExtendedNodes(leftRequired);
edgeB.setRightExtendedNodes(rightRequired);
}
private BitSet subTreeEdge(Edge edge) {
long subTreeNodes = edge.getSubTreeNodes();
BitSet subEdges = new BitSet();
joinEdges.stream()
.filter(e -> LongBitmap.isSubset(subTreeNodes, e.getReferenceNodes()))
.forEach(e -> subEdges.set(e.getIndex()));
return subEdges;
}
private BitSet subTreeEdges(BitSet edgeSet) {
BitSet bitSet = new BitSet();
edgeSet.stream()
.mapToObj(i -> subTreeEdge(joinEdges.get(i)))
.forEach(bitSet::or);
return bitSet;
}
// Try to calculate the ends of an expression.
// left = ref_nodes \cap left_tree , right = ref_nodes \cap right_tree
// if left = 0, recursively calculate it in left tree

View File

@ -47,4 +47,8 @@ public class FilterEdge extends Edge {
public List<? extends Expression> getExpressions() {
return filter.getExpressions();
}
public FilterEdge clear() {
return new FilterEdge(filter, getIndex(), getLeftChildEdges(), getSubTreeNodes(), getLeftRequiredNodes());
}
}

View File

@ -61,6 +61,11 @@ public class JoinEdge extends Edge {
return join.getJoinType();
}
public JoinEdge withJoinTypeAndCleanCR(JoinType joinType) {
return new JoinEdge(join.withJoinType(joinType), getIndex(), getLeftChildEdges(), getRightChildEdges(),
getSubTreeNodes(), getLeftRequiredNodes(), getRightRequiredNodes());
}
public LogicalJoin<? extends Plan, ? extends Plan> getJoin() {
return join;
}

View File

@ -18,6 +18,7 @@
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.ConflictRulesMaker;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge;
@ -40,7 +41,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@ -64,10 +64,12 @@ public class HyperGraphComparator {
// record inferred edges when comparing mv
private final HyperGraph queryHyperGraph;
private final HyperGraph viewHyperGraph;
private final Map<JoinEdge, Pair<JoinType, Set<Slot>>> inferredViewEdgeMap = new HashMap<>();
private final Map<Edge, List<? extends Expression>> pullUpQueryExprWithEdge = new HashMap<>();
private final Map<Edge, List<? extends Expression>> pullUpViewExprWithEdge = new HashMap<>();
private final LogicalCompatibilityContext logicalCompatibilityContext;
private final Map<JoinEdge, Pair<JoinType, Set<Slot>>> inferredViewEdgeWithCond = new HashMap<>();
private List<JoinEdge> viewJoinEdgesAfterInferring;
private List<FilterEdge> viewFilterEdgesAfterInferring;
public HyperGraphComparator(HyperGraph queryHyperGraph, HyperGraph viewHyperGraph,
LogicalCompatibilityContext logicalCompatibilityContext) {
@ -89,9 +91,18 @@ public class HyperGraphComparator {
private ComparisonResult isLogicCompatible() {
// 1 try to construct a map which can be mapped from edge to edge
Map<Edge, Edge> queryToView = constructMapWithNode();
Map<Edge, Edge> queryToView = constructQueryToViewMapWithExpr();
if (!makeViewJoinCompatible(queryToView)) {
return ComparisonResult.newInvalidResWithErrorMessage("Join types are not compatible\n");
}
refreshViewEdges();
// 2. compare them by expression and extract residual expr
// 2. compare them by expression and nodes. Note compare edges after inferring for nodes
boolean matchNodes = queryToView.entrySet().stream()
.allMatch(e -> compareEdgeWithNode(e.getKey(), e.getValue()));
if (!matchNodes) {
return ComparisonResult.newInvalidResWithErrorMessage("Join nodes are not compatible\n");
}
queryToView.forEach(this::compareEdgeWithExpr);
// 3. process residual edges
@ -122,12 +133,12 @@ public class HyperGraphComparator {
List<? extends Expression> rawFilter = e.getValue().stream()
.filter(expr -> !ExpressionUtils.isInferred(expr))
.collect(Collectors.toList());
if (!rawFilter.isEmpty() && !canPullUp(e.getKey())) {
if (!rawFilter.isEmpty() && !canPullUp(getViewEdgeAfterInferring(e.getKey()))) {
return ComparisonResult.newInvalidResWithErrorMessage(getErrorMessage() + "with error edge\n" + e);
}
builder.addViewExpressions(rawFilter);
}
for (Pair<JoinType, Set<Slot>> inferredCond : inferredViewEdgeMap.values()) {
for (Pair<JoinType, Set<Slot>> inferredCond : inferredViewEdgeWithCond.values()) {
builder.addViewNoNullableSlot(inferredCond.second);
}
return builder.build();
@ -145,7 +156,15 @@ public class HyperGraphComparator {
getViewJoinEdges(),
getQueryFilterEdges(),
getViewFilterEdges(),
inferredViewEdgeMap);
inferredViewEdgeWithCond);
}
private Edge getViewEdgeAfterInferring(Edge edge) {
if (edge instanceof JoinEdge) {
return viewJoinEdgesAfterInferring.get(edge.getIndex());
} else {
return viewFilterEdgesAfterInferring.get(edge.getIndex());
}
}
private boolean canPullUp(Edge edge) {
@ -154,10 +173,10 @@ public class HyperGraphComparator {
return false;
}
boolean pullFromLeft = edge.getLeftRejectEdge().stream()
.map(e -> inferredViewEdgeMap.getOrDefault(e, Pair.of(e.getJoinType(), null)))
.map(e -> inferredViewEdgeWithCond.getOrDefault(e, Pair.of(e.getJoinType(), null)))
.allMatch(e -> canPullFromLeft(edge, e.first));
boolean pullFromRight = edge.getRightRejectEdge().stream()
.map(e -> inferredViewEdgeMap.getOrDefault(e, Pair.of(e.getJoinType(), null)))
.map(e -> inferredViewEdgeWithCond.getOrDefault(e, Pair.of(e.getJoinType(), null)))
.allMatch(e -> canPullFromRight(edge, e.first));
return pullFromLeft && pullFromRight;
}
@ -198,6 +217,24 @@ public class HyperGraphComparator {
return ImmutableSet.copyOf(queryHyperGraph.getFilterEdges());
}
private List<Edge> getQueryEdges() {
return ImmutableList.<Edge>builder()
.addAll(getQueryJoinEdges())
.addAll(getQueryFilterEdges()).build();
}
private boolean makeViewJoinCompatible(Map<Edge, Edge> queryToView) {
for (Entry<Edge, Edge> entry : queryToView.entrySet()) {
if (entry.getKey() instanceof JoinEdge && entry.getValue() instanceof JoinEdge) {
boolean res = compareJoinEdgeOrInfer((JoinEdge) entry.getKey(), (JoinEdge) entry.getValue());
if (!res) {
return false;
}
}
}
return true;
}
private Set<FilterEdge> getViewFilterEdgeSet() {
return ImmutableSet.copyOf(viewHyperGraph.getFilterEdges());
}
@ -214,6 +251,12 @@ public class HyperGraphComparator {
return viewHyperGraph.getFilterEdges();
}
private List<Edge> getViewEdges() {
return ImmutableList.<Edge>builder()
.addAll(getViewJoinEdges())
.addAll(getViewFilterEdges()).build();
}
private Map<Expression, Expression> getQueryToViewExprMap() {
return logicalCompatibilityContext.getQueryToViewEdgeExpressionMapping();
}
@ -222,43 +265,74 @@ public class HyperGraphComparator {
return logicalCompatibilityContext.getQueryToViewNodeIDMapping();
}
private Map<Edge, Edge> constructMapWithNode() {
// TODO use hash map to reduce loop
Map<Edge, Edge> joinEdgeMap = getQueryJoinEdges().stream().map(qe -> {
Optional<JoinEdge> viewEdge = getViewJoinEdges().stream()
.filter(ve -> compareEdgeWithNode(qe, ve)).findFirst();
return Pair.of(qe, viewEdge);
}).filter(e -> e.second.isPresent()).collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second.get()));
Map<Edge, Edge> filterEdgeMap = getQueryFilterEdges().stream().map(qe -> {
Optional<FilterEdge> viewEdge = getViewFilterEdges().stream()
.filter(ve -> compareEdgeWithNode(qe, ve)).findFirst();
return Pair.of(qe, viewEdge);
}).filter(e -> e.second.isPresent()).collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second.get()));
return ImmutableMap.<Edge, Edge>builder().putAll(joinEdgeMap).putAll(filterEdgeMap).build();
private Map<Edge, Edge> constructQueryToViewMapWithExpr() {
Map<Expression, Edge> viewExprToEdge = getViewEdges().stream()
.flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e)))
.collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second));
Map<Expression, Edge> queryExprToEdge = getQueryEdges().stream()
.flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e)))
.collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second));
return queryExprToEdge.entrySet().stream()
.filter(entry -> viewExprToEdge.containsKey(getViewExprFromQueryExpr(entry.getKey())))
.map(entry -> Pair.of(entry.getValue(),
viewExprToEdge.get(getViewExprFromQueryExpr(entry.getKey()))))
.distinct()
.collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second));
}
private Expression getViewExprFromQueryExpr(Expression query) {
return logicalCompatibilityContext.getQueryToViewEdgeExpressionMapping().get(query);
}
private void refreshViewEdges() {
List<FilterEdge> newFilterEdges = getViewFilterEdges().stream()
.map(FilterEdge::clear)
.collect(ImmutableList.toImmutableList());
List<JoinEdge> newJoinEdges = new ArrayList<>();
for (JoinEdge joinEdge : getViewJoinEdges()) {
JoinType newJoinType = inferredViewEdgeWithCond
.getOrDefault(joinEdge, Pair.of(joinEdge.getJoinType(), null)).first;
JoinEdge newJoinEdge = joinEdge.withJoinTypeAndCleanCR(newJoinType);
newJoinEdges.add(newJoinEdge);
ConflictRulesMaker.makeJoinConflictRules(newJoinEdge, newJoinEdges);
ConflictRulesMaker.makeFilterConflictRules(newJoinEdge, newJoinEdges, newFilterEdges);
}
viewJoinEdgesAfterInferring = ImmutableList.copyOf(newJoinEdges);
viewFilterEdgesAfterInferring = ImmutableList.copyOf(newFilterEdges);
}
private boolean compareEdgeWithNode(Edge query, Edge view) {
if (query instanceof FilterEdge && view instanceof FilterEdge) {
return compareEdgeWithFilter((FilterEdge) query, (FilterEdge) view);
return compareFilterEdgeWithNode((FilterEdge) query, viewFilterEdgesAfterInferring.get(view.getIndex()));
} else if (query instanceof JoinEdge && view instanceof JoinEdge) {
return compareJoinEdge((JoinEdge) query, (JoinEdge) view);
return compareJoinEdgeWithNode((JoinEdge) query, viewJoinEdgesAfterInferring.get(view.getIndex()));
}
return false;
}
private boolean compareEdgeWithFilter(FilterEdge query, FilterEdge view) {
long qChild = query.getReferenceNodes();
long vChild = view.getReferenceNodes();
return rewriteQueryNodeMap(qChild) == vChild;
private boolean compareFilterEdgeWithNode(FilterEdge query, FilterEdge view) {
return rewriteQueryNodeMap(query.getReferenceNodes()) == view.getReferenceNodes();
}
private boolean compareJoinEdge(JoinEdge query, JoinEdge view) {
private boolean compareJoinEdgeWithNode(JoinEdge query, JoinEdge view) {
boolean res = false;
if (query.getJoinType().swap() == view.getJoinType()) {
res |= rewriteQueryNodeMap(query.getLeftExtendedNodes()) == view.getRightExtendedNodes()
&& rewriteQueryNodeMap(query.getRightExtendedNodes()) == view.getLeftExtendedNodes();
}
res |= rewriteQueryNodeMap(query.getLeftExtendedNodes()) == view.getLeftExtendedNodes()
&& rewriteQueryNodeMap(query.getRightExtendedNodes()) == view.getRightExtendedNodes();
return res;
}
private boolean compareJoinEdgeOrInfer(JoinEdge query, JoinEdge view) {
if (query.getJoinType().equals(view.getJoinType())
|| canInferredJoinTypeMap.containsKey(Pair.of(query.getJoinType(), view.getJoinType()))) {
if (tryInferEdge(query, view)) {
return true;
}
}
if (query.getJoinType().swap().equals(view.getJoinType())
@ -272,8 +346,8 @@ public class HyperGraphComparator {
}
private boolean tryInferEdge(JoinEdge query, JoinEdge view) {
if (rewriteQueryNodeMap(query.getLeftExtendedNodes()) != view.getLeftExtendedNodes()
|| rewriteQueryNodeMap(query.getRightExtendedNodes()) != view.getRightExtendedNodes()) {
if (rewriteQueryNodeMap(query.getLeftRequiredNodes()) != view.getLeftRequiredNodes()
|| rewriteQueryNodeMap(query.getRightRequiredNodes()) != view.getRightRequiredNodes()) {
return false;
}
if (!query.getJoinType().equals(view.getJoinType())) {
@ -286,7 +360,7 @@ public class HyperGraphComparator {
noNullableChild.first ? view.getJoin().left().getOutputSet() : ImmutableSet.of(),
noNullableChild.second ? view.getJoin().right().getOutputSet() : ImmutableSet.of()
);
inferredViewEdgeMap.put(view, Pair.of(query.getJoinType(), noNullableSlot));
inferredViewEdgeWithCond.put(view, Pair.of(query.getJoinType(), noNullableSlot));
}
return true;
}