From b59a8c93651b5a0f0e256cf6e8b28d4a1c72022c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=A2=E5=81=A5?= Date: Mon, 8 Jan 2024 15:30:14 +0800 Subject: [PATCH] [feature](Nereids): refresh view hypergraph after inferring join (#29469) --- .../hypergraph/ConflictRulesMaker.java | 108 +++++++++++ .../jobs/joinorder/hypergraph/HyperGraph.java | 73 +------- .../joinorder/hypergraph/edge/FilterEdge.java | 4 + .../joinorder/hypergraph/edge/JoinEdge.java | 5 + .../exploration/mv/HyperGraphComparator.java | 140 ++++++++++---- .../mv/HyperGraphComparatorTest.java | 172 ++++++++++++++++++ 6 files changed, 399 insertions(+), 103 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/ConflictRulesMaker.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/ConflictRulesMaker.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/ConflictRulesMaker.java new file mode 100644 index 0000000000..11db05843f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/ConflictRulesMaker.java @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.joinorder.hypergraph; + +import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.FilterEdge; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge; +import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin; +import org.apache.doris.nereids.trees.plans.JoinType; + +import java.util.BitSet; +import java.util.List; + +/** + * This is a conflict rule maker to + */ +public class ConflictRulesMaker { + private ConflictRulesMaker() {} + + /** + * make conflict rules for filter edges + */ + public static void makeFilterConflictRules( + JoinEdge joinEdge, List joinEdges, List filterEdges) { + long leftSubNodes = joinEdge.getLeftSubNodes(joinEdges); + long rightSubNodes = joinEdge.getRightSubNodes(joinEdges); + filterEdges.forEach(e -> { + if (LongBitmap.isSubset(e.getReferenceNodes(), leftSubNodes) + && !PushDownFilterThroughJoin.COULD_PUSH_THROUGH_LEFT.contains(joinEdge.getJoinType())) { + e.addLeftRejectEdge(joinEdge); + } + if (LongBitmap.isSubset(e.getReferenceNodes(), rightSubNodes) + && !PushDownFilterThroughJoin.COULD_PUSH_THROUGH_RIGHT.contains(joinEdge.getJoinType())) { + e.addRightRejectEdge(joinEdge); + } + }); + } + + /** + * Make edge with CD-C algorithm in + * On the correct and complete enumeration of the core search + */ + public static void makeJoinConflictRules(JoinEdge edgeB, List joinEdges) { + BitSet leftSubTreeEdges = subTreeEdges(edgeB.getLeftChildEdges(), joinEdges); + BitSet rightSubTreeEdges = subTreeEdges(edgeB.getRightChildEdges(), joinEdges); + long leftRequired = edgeB.getLeftRequiredNodes(); + long rightRequired = edgeB.getRightRequiredNodes(); + + for (int i = leftSubTreeEdges.nextSetBit(0); i >= 0; i = leftSubTreeEdges.nextSetBit(i + 1)) { + JoinEdge childA = joinEdges.get(i); + if (!JoinType.isAssoc(childA.getJoinType(), edgeB.getJoinType())) { + leftRequired = LongBitmap.newBitmapUnion(leftRequired, childA.getLeftSubNodes(joinEdges)); + childA.addLeftRejectEdge(edgeB); + } + if (!JoinType.isLAssoc(childA.getJoinType(), edgeB.getJoinType())) { + leftRequired = LongBitmap.newBitmapUnion(leftRequired, childA.getRightSubNodes(joinEdges)); + childA.addLeftRejectEdge(edgeB); + } + } + + for (int i = rightSubTreeEdges.nextSetBit(0); i >= 0; i = rightSubTreeEdges.nextSetBit(i + 1)) { + JoinEdge childA = joinEdges.get(i); + if (!JoinType.isAssoc(edgeB.getJoinType(), childA.getJoinType())) { + rightRequired = LongBitmap.newBitmapUnion(rightRequired, childA.getRightSubNodes(joinEdges)); + childA.addRightRejectEdge(edgeB); + } + if (!JoinType.isRAssoc(edgeB.getJoinType(), childA.getJoinType())) { + rightRequired = LongBitmap.newBitmapUnion(rightRequired, childA.getLeftSubNodes(joinEdges)); + childA.addRightRejectEdge(edgeB); + } + } + edgeB.setLeftExtendedNodes(leftRequired); + edgeB.setRightExtendedNodes(rightRequired); + } + + private static BitSet subTreeEdge(Edge edge, List joinEdges) { + long subTreeNodes = edge.getSubTreeNodes(); + BitSet subEdges = new BitSet(); + joinEdges.stream() + .filter(e -> LongBitmap.isSubset(subTreeNodes, e.getReferenceNodes())) + .forEach(e -> subEdges.set(e.getIndex())); + return subEdges; + } + + private static BitSet subTreeEdges(BitSet edgeSet, List joinEdges) { + BitSet bitSet = new BitSet(); + edgeSet.stream() + .mapToObj(i -> subTreeEdge(joinEdges.get(i), joinEdges)) + .forEach(bitSet::or); + return bitSet; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java index d78374d134..8a0bd8daaa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java @@ -28,7 +28,6 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.DPhyperNode; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -244,8 +243,9 @@ public class HyperGraph { joinEdges.add(edge); } curJoinEdges.stream().forEach(i -> joinEdges.get(i).addCurJoinEdges(curJoinEdges)); - curJoinEdges.stream().forEach(i -> makeJoinConflictRules(joinEdges.get(i))); - curJoinEdges.stream().forEach(i -> makeFilterConflictRules(joinEdges.get(i))); + curJoinEdges.stream().forEach(i -> ConflictRulesMaker.makeJoinConflictRules(joinEdges.get(i), joinEdges)); + curJoinEdges.stream().forEach(i -> + ConflictRulesMaker.makeFilterConflictRules(joinEdges.get(i), joinEdges, filterEdges)); return curJoinEdges; // In MySQL, each edge is reversed and store in edges again for reducing the branch miss // We don't implement this trick now. @@ -260,73 +260,6 @@ public class HyperGraph { return bitSet; } - private void makeFilterConflictRules(JoinEdge joinEdge) { - long leftSubNodes = joinEdge.getLeftSubNodes(joinEdges); - long rightSubNodes = joinEdge.getRightSubNodes(joinEdges); - filterEdges.forEach(e -> { - if (LongBitmap.isSubset(e.getReferenceNodes(), leftSubNodes) - && !PushDownFilterThroughJoin.COULD_PUSH_THROUGH_LEFT.contains(joinEdge.getJoinType())) { - e.addLeftRejectEdge(joinEdge); - } - if (LongBitmap.isSubset(e.getReferenceNodes(), rightSubNodes) - && !PushDownFilterThroughJoin.COULD_PUSH_THROUGH_RIGHT.contains(joinEdge.getJoinType())) { - e.addRightRejectEdge(joinEdge); - } - }); - } - - // Make edge with CD-C algorithm in - // On the correct and complete enumeration of the core search - private void makeJoinConflictRules(JoinEdge edgeB) { - BitSet leftSubTreeEdges = subTreeEdges(edgeB.getLeftChildEdges()); - BitSet rightSubTreeEdges = subTreeEdges(edgeB.getRightChildEdges()); - long leftRequired = edgeB.getLeftRequiredNodes(); - long rightRequired = edgeB.getRightRequiredNodes(); - - for (int i = leftSubTreeEdges.nextSetBit(0); i >= 0; i = leftSubTreeEdges.nextSetBit(i + 1)) { - JoinEdge childA = joinEdges.get(i); - if (!JoinType.isAssoc(childA.getJoinType(), edgeB.getJoinType())) { - leftRequired = LongBitmap.newBitmapUnion(leftRequired, childA.getLeftSubNodes(joinEdges)); - childA.addLeftRejectEdge(edgeB); - } - if (!JoinType.isLAssoc(childA.getJoinType(), edgeB.getJoinType())) { - leftRequired = LongBitmap.newBitmapUnion(leftRequired, childA.getRightSubNodes(joinEdges)); - childA.addLeftRejectEdge(edgeB); - } - } - - for (int i = rightSubTreeEdges.nextSetBit(0); i >= 0; i = rightSubTreeEdges.nextSetBit(i + 1)) { - JoinEdge childA = joinEdges.get(i); - if (!JoinType.isAssoc(edgeB.getJoinType(), childA.getJoinType())) { - rightRequired = LongBitmap.newBitmapUnion(rightRequired, childA.getRightSubNodes(joinEdges)); - childA.addRightRejectEdge(edgeB); - } - if (!JoinType.isRAssoc(edgeB.getJoinType(), childA.getJoinType())) { - rightRequired = LongBitmap.newBitmapUnion(rightRequired, childA.getLeftSubNodes(joinEdges)); - childA.addRightRejectEdge(edgeB); - } - } - edgeB.setLeftExtendedNodes(leftRequired); - edgeB.setRightExtendedNodes(rightRequired); - } - - private BitSet subTreeEdge(Edge edge) { - long subTreeNodes = edge.getSubTreeNodes(); - BitSet subEdges = new BitSet(); - joinEdges.stream() - .filter(e -> LongBitmap.isSubset(subTreeNodes, e.getReferenceNodes())) - .forEach(e -> subEdges.set(e.getIndex())); - return subEdges; - } - - private BitSet subTreeEdges(BitSet edgeSet) { - BitSet bitSet = new BitSet(); - edgeSet.stream() - .mapToObj(i -> subTreeEdge(joinEdges.get(i))) - .forEach(bitSet::or); - return bitSet; - } - // Try to calculate the ends of an expression. // left = ref_nodes \cap left_tree , right = ref_nodes \cap right_tree // if left = 0, recursively calculate it in left tree diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/FilterEdge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/FilterEdge.java index 57c6d9660d..52c94abe3a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/FilterEdge.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/FilterEdge.java @@ -47,4 +47,8 @@ public class FilterEdge extends Edge { public List getExpressions() { return filter.getExpressions(); } + + public FilterEdge clear() { + return new FilterEdge(filter, getIndex(), getLeftChildEdges(), getSubTreeNodes(), getLeftRequiredNodes()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/JoinEdge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/JoinEdge.java index 635bf91f36..c23be5f16e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/JoinEdge.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/JoinEdge.java @@ -61,6 +61,11 @@ public class JoinEdge extends Edge { return join.getJoinType(); } + public JoinEdge withJoinTypeAndCleanCR(JoinType joinType) { + return new JoinEdge(join.withJoinType(joinType), getIndex(), getLeftChildEdges(), getRightChildEdges(), + getSubTreeNodes(), getLeftRequiredNodes(), getRightRequiredNodes()); + } + public LogicalJoin getJoin() { return join; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java index b92e352f5c..7817475c7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.ConflictRulesMaker; import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap; import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge; @@ -40,7 +41,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -64,10 +64,12 @@ public class HyperGraphComparator { // record inferred edges when comparing mv private final HyperGraph queryHyperGraph; private final HyperGraph viewHyperGraph; - private final Map>> inferredViewEdgeMap = new HashMap<>(); private final Map> pullUpQueryExprWithEdge = new HashMap<>(); private final Map> pullUpViewExprWithEdge = new HashMap<>(); private final LogicalCompatibilityContext logicalCompatibilityContext; + private final Map>> inferredViewEdgeWithCond = new HashMap<>(); + private List viewJoinEdgesAfterInferring; + private List viewFilterEdgesAfterInferring; public HyperGraphComparator(HyperGraph queryHyperGraph, HyperGraph viewHyperGraph, LogicalCompatibilityContext logicalCompatibilityContext) { @@ -89,9 +91,18 @@ public class HyperGraphComparator { private ComparisonResult isLogicCompatible() { // 1 try to construct a map which can be mapped from edge to edge - Map queryToView = constructMapWithNode(); + Map queryToView = constructQueryToViewMapWithExpr(); + if (!makeViewJoinCompatible(queryToView)) { + return ComparisonResult.newInvalidResWithErrorMessage("Join types are not compatible\n"); + } + refreshViewEdges(); - // 2. compare them by expression and extract residual expr + // 2. compare them by expression and nodes. Note compare edges after inferring for nodes + boolean matchNodes = queryToView.entrySet().stream() + .allMatch(e -> compareEdgeWithNode(e.getKey(), e.getValue())); + if (!matchNodes) { + return ComparisonResult.newInvalidResWithErrorMessage("Join nodes are not compatible\n"); + } queryToView.forEach(this::compareEdgeWithExpr); // 3. process residual edges @@ -122,12 +133,12 @@ public class HyperGraphComparator { List rawFilter = e.getValue().stream() .filter(expr -> !ExpressionUtils.isInferred(expr)) .collect(Collectors.toList()); - if (!rawFilter.isEmpty() && !canPullUp(e.getKey())) { + if (!rawFilter.isEmpty() && !canPullUp(getViewEdgeAfterInferring(e.getKey()))) { return ComparisonResult.newInvalidResWithErrorMessage(getErrorMessage() + "with error edge\n" + e); } builder.addViewExpressions(rawFilter); } - for (Pair> inferredCond : inferredViewEdgeMap.values()) { + for (Pair> inferredCond : inferredViewEdgeWithCond.values()) { builder.addViewNoNullableSlot(inferredCond.second); } return builder.build(); @@ -145,7 +156,15 @@ public class HyperGraphComparator { getViewJoinEdges(), getQueryFilterEdges(), getViewFilterEdges(), - inferredViewEdgeMap); + inferredViewEdgeWithCond); + } + + private Edge getViewEdgeAfterInferring(Edge edge) { + if (edge instanceof JoinEdge) { + return viewJoinEdgesAfterInferring.get(edge.getIndex()); + } else { + return viewFilterEdgesAfterInferring.get(edge.getIndex()); + } } private boolean canPullUp(Edge edge) { @@ -154,10 +173,10 @@ public class HyperGraphComparator { return false; } boolean pullFromLeft = edge.getLeftRejectEdge().stream() - .map(e -> inferredViewEdgeMap.getOrDefault(e, Pair.of(e.getJoinType(), null))) + .map(e -> inferredViewEdgeWithCond.getOrDefault(e, Pair.of(e.getJoinType(), null))) .allMatch(e -> canPullFromLeft(edge, e.first)); boolean pullFromRight = edge.getRightRejectEdge().stream() - .map(e -> inferredViewEdgeMap.getOrDefault(e, Pair.of(e.getJoinType(), null))) + .map(e -> inferredViewEdgeWithCond.getOrDefault(e, Pair.of(e.getJoinType(), null))) .allMatch(e -> canPullFromRight(edge, e.first)); return pullFromLeft && pullFromRight; } @@ -198,6 +217,24 @@ public class HyperGraphComparator { return ImmutableSet.copyOf(queryHyperGraph.getFilterEdges()); } + private List getQueryEdges() { + return ImmutableList.builder() + .addAll(getQueryJoinEdges()) + .addAll(getQueryFilterEdges()).build(); + } + + private boolean makeViewJoinCompatible(Map queryToView) { + for (Entry entry : queryToView.entrySet()) { + if (entry.getKey() instanceof JoinEdge && entry.getValue() instanceof JoinEdge) { + boolean res = compareJoinEdgeOrInfer((JoinEdge) entry.getKey(), (JoinEdge) entry.getValue()); + if (!res) { + return false; + } + } + } + return true; + } + private Set getViewFilterEdgeSet() { return ImmutableSet.copyOf(viewHyperGraph.getFilterEdges()); } @@ -214,6 +251,12 @@ public class HyperGraphComparator { return viewHyperGraph.getFilterEdges(); } + private List getViewEdges() { + return ImmutableList.builder() + .addAll(getViewJoinEdges()) + .addAll(getViewFilterEdges()).build(); + } + private Map getQueryToViewExprMap() { return logicalCompatibilityContext.getQueryToViewEdgeExpressionMapping(); } @@ -222,43 +265,74 @@ public class HyperGraphComparator { return logicalCompatibilityContext.getQueryToViewNodeIDMapping(); } - private Map constructMapWithNode() { - // TODO use hash map to reduce loop - Map joinEdgeMap = getQueryJoinEdges().stream().map(qe -> { - Optional viewEdge = getViewJoinEdges().stream() - .filter(ve -> compareEdgeWithNode(qe, ve)).findFirst(); - return Pair.of(qe, viewEdge); - }).filter(e -> e.second.isPresent()).collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second.get())); - Map filterEdgeMap = getQueryFilterEdges().stream().map(qe -> { - Optional viewEdge = getViewFilterEdges().stream() - .filter(ve -> compareEdgeWithNode(qe, ve)).findFirst(); - return Pair.of(qe, viewEdge); - }).filter(e -> e.second.isPresent()).collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second.get())); - return ImmutableMap.builder().putAll(joinEdgeMap).putAll(filterEdgeMap).build(); + private Map constructQueryToViewMapWithExpr() { + Map viewExprToEdge = getViewEdges().stream() + .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e))) + .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second)); + Map queryExprToEdge = getQueryEdges().stream() + .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e))) + .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second)); + return queryExprToEdge.entrySet().stream() + .filter(entry -> viewExprToEdge.containsKey(getViewExprFromQueryExpr(entry.getKey()))) + .map(entry -> Pair.of(entry.getValue(), + viewExprToEdge.get(getViewExprFromQueryExpr(entry.getKey())))) + .distinct() + .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second)); + } + + private Expression getViewExprFromQueryExpr(Expression query) { + return logicalCompatibilityContext.getQueryToViewEdgeExpressionMapping().get(query); + } + + private void refreshViewEdges() { + List newFilterEdges = getViewFilterEdges().stream() + .map(FilterEdge::clear) + .collect(ImmutableList.toImmutableList()); + + List newJoinEdges = new ArrayList<>(); + for (JoinEdge joinEdge : getViewJoinEdges()) { + JoinType newJoinType = inferredViewEdgeWithCond + .getOrDefault(joinEdge, Pair.of(joinEdge.getJoinType(), null)).first; + JoinEdge newJoinEdge = joinEdge.withJoinTypeAndCleanCR(newJoinType); + newJoinEdges.add(newJoinEdge); + ConflictRulesMaker.makeJoinConflictRules(newJoinEdge, newJoinEdges); + ConflictRulesMaker.makeFilterConflictRules(newJoinEdge, newJoinEdges, newFilterEdges); + } + + viewJoinEdgesAfterInferring = ImmutableList.copyOf(newJoinEdges); + viewFilterEdgesAfterInferring = ImmutableList.copyOf(newFilterEdges); } private boolean compareEdgeWithNode(Edge query, Edge view) { if (query instanceof FilterEdge && view instanceof FilterEdge) { - return compareEdgeWithFilter((FilterEdge) query, (FilterEdge) view); + return compareFilterEdgeWithNode((FilterEdge) query, viewFilterEdgesAfterInferring.get(view.getIndex())); } else if (query instanceof JoinEdge && view instanceof JoinEdge) { - return compareJoinEdge((JoinEdge) query, (JoinEdge) view); + return compareJoinEdgeWithNode((JoinEdge) query, viewJoinEdgesAfterInferring.get(view.getIndex())); } return false; } - private boolean compareEdgeWithFilter(FilterEdge query, FilterEdge view) { - long qChild = query.getReferenceNodes(); - long vChild = view.getReferenceNodes(); - return rewriteQueryNodeMap(qChild) == vChild; + private boolean compareFilterEdgeWithNode(FilterEdge query, FilterEdge view) { + return rewriteQueryNodeMap(query.getReferenceNodes()) == view.getReferenceNodes(); } - private boolean compareJoinEdge(JoinEdge query, JoinEdge view) { + private boolean compareJoinEdgeWithNode(JoinEdge query, JoinEdge view) { + boolean res = false; + if (query.getJoinType().swap() == view.getJoinType()) { + res |= rewriteQueryNodeMap(query.getLeftExtendedNodes()) == view.getRightExtendedNodes() + && rewriteQueryNodeMap(query.getRightExtendedNodes()) == view.getLeftExtendedNodes(); + } + res |= rewriteQueryNodeMap(query.getLeftExtendedNodes()) == view.getLeftExtendedNodes() + && rewriteQueryNodeMap(query.getRightExtendedNodes()) == view.getRightExtendedNodes(); + return res; + } + + private boolean compareJoinEdgeOrInfer(JoinEdge query, JoinEdge view) { if (query.getJoinType().equals(view.getJoinType()) || canInferredJoinTypeMap.containsKey(Pair.of(query.getJoinType(), view.getJoinType()))) { if (tryInferEdge(query, view)) { return true; } - } if (query.getJoinType().swap().equals(view.getJoinType()) @@ -272,8 +346,8 @@ public class HyperGraphComparator { } private boolean tryInferEdge(JoinEdge query, JoinEdge view) { - if (rewriteQueryNodeMap(query.getLeftExtendedNodes()) != view.getLeftExtendedNodes() - || rewriteQueryNodeMap(query.getRightExtendedNodes()) != view.getRightExtendedNodes()) { + if (rewriteQueryNodeMap(query.getLeftRequiredNodes()) != view.getLeftRequiredNodes() + || rewriteQueryNodeMap(query.getRightRequiredNodes()) != view.getRightRequiredNodes()) { return false; } if (!query.getJoinType().equals(view.getJoinType())) { @@ -286,7 +360,7 @@ public class HyperGraphComparator { noNullableChild.first ? view.getJoin().left().getOutputSet() : ImmutableSet.of(), noNullableChild.second ? view.getJoin().right().getOutputSet() : ImmutableSet.of() ); - inferredViewEdgeMap.put(view, Pair.of(query.getJoinType(), noNullableSlot)); + inferredViewEdgeWithCond.put(view, Pair.of(query.getJoinType(), noNullableSlot)); } return true; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java new file mode 100644 index 0000000000..77b7fd6729 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; +import org.apache.doris.nereids.rules.RuleSet; +import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.sqltest.SqlTestBase; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.util.PlanChecker; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +class HyperGraphComparatorTest extends SqlTestBase { + @Test + void testInnerJoinAndLOJ() { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES"); + CascadesContext c1 = createCascadesContext( + "select * from T1 inner join T2 " + + "on T1.id = T2.id " + + "inner join T3 on T1.id = T3.id", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 left outer join T2 " + + "on T1.id = T2.id " + + "left outer join T3 on T1.id = T3.id", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0); + HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + Assertions.assertTrue(!res.isInvalid()); + Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); + } + + @Test + void testIJAndLojAssoc() { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES"); + CascadesContext c1 = createCascadesContext( + "select * from T1 inner join T3 " + + "on T1.id = T3.id " + + "inner join T2 on T1.id = T2.id", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 left outer join T2 " + + "on T1.id = T2.id " + + "left outer join T3 on T1.id = T3.id", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0); + HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + Assertions.assertTrue(!res.isInvalid()); + Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); + } + + @Test + void testIJAndLojAssocWithFilter() { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES"); + CascadesContext c1 = createCascadesContext( + "select * from T1 inner join T3 " + + "on T1.id = T3.id " + + "inner join T2 on T1.id = T2.id", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 left outer join " + + "(select * from T2 where T2.id = 1) T2 " + + "on T1.id = T2.id " + + "left outer join T3 on T1.id = T3.id", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0); + HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + Assertions.assertTrue(!res.isInvalid()); + Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); + } + + @Disabled + @Test + void testIJAndLojAssocWithJoinCond() { + connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES"); + CascadesContext c1 = createCascadesContext( + "select * from T1 inner join T3 " + + "on T1.id = T3.id " + + "inner join T2 on T1.id = T2.id", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 left outer join " + + "(" + + "select T1.* from T1 left outer join T3 " + + "on T1.id = T3.id and T1.score = T3.score " + + ") T2 " + + "on T1.id = T2.id ", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0); + HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + Assertions.assertTrue(!res.isInvalid()); + Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); + } + + LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { + StructInfo st1 = AbstractMaterializedViewRule.extractStructInfo(p1, + null).get(0); + StructInfo st2 = AbstractMaterializedViewRule.extractStructInfo(p2, + null).get(0); + RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); + SlotMapping sm = SlotMapping.generate(rm); + return LogicalCompatibilityContext.from(rm, sm, st1, st2); + } +}