From 1d984e0ebb1010f58b860c3089254053566d5b13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=A2=E5=81=A5?= Date: Mon, 25 Dec 2023 12:53:14 +0800 Subject: [PATCH] return residual expr of join (#28760) --- .../jobs/joinorder/hypergraph/HyperGraph.java | 158 +++++++++++------- .../jobs/joinorder/hypergraph/edge/Edge.java | 22 +++ .../joinorder/hypergraph/edge/FilterEdge.java | 15 -- .../mv/AbstractMaterializedViewRule.java | 6 +- .../exploration/mv/ComparisonResult.java | 94 +++++++++++ .../rules/exploration/mv/StructInfo.java | 2 +- .../hypergraph/CompareOuterJoinTest.java | 13 +- .../hypergraph/PullupExpressionTest.java | 151 +++++++++++++++++ .../exploration/mv/BuildStructInfoTest.java | 4 +- 9 files changed, 380 insertions(+), 85 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java index e534fc1fa8..7bd33c64b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.DPhyperNode; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult; import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext; import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin; import org.apache.doris.nereids.trees.expressions.Alias; @@ -44,18 +45,21 @@ import org.apache.doris.nereids.util.PlanUtils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import java.util.ArrayList; import java.util.BitSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; +import java.util.Map.Entry; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import javax.annotation.Nullable; /** * The graph is a join graph, whose node is the leaf plan and edge is a join operator. @@ -268,11 +272,11 @@ public class HyperGraph { filterEdges.forEach(e -> { if (LongBitmap.isSubset(e.getReferenceNodes(), leftSubNodes) && !PushDownFilterThroughJoin.COULD_PUSH_THROUGH_LEFT.contains(joinEdge.getJoinType())) { - e.addRejectJoin(joinEdge); + e.addRejectEdge(joinEdge); } if (LongBitmap.isSubset(e.getReferenceNodes(), rightSubNodes) && !PushDownFilterThroughJoin.COULD_PUSH_THROUGH_RIGHT.contains(joinEdge.getJoinType())) { - e.addRejectJoin(joinEdge); + e.addRejectEdge(joinEdge); } }); } @@ -289,9 +293,11 @@ public class HyperGraph { JoinEdge childA = joinEdges.get(i); if (!JoinType.isAssoc(childA.getJoinType(), edgeB.getJoinType())) { leftRequired = LongBitmap.newBitmapUnion(leftRequired, childA.getLeftSubNodes(joinEdges)); + childA.addRejectEdge(edgeB); } if (!JoinType.isLAssoc(childA.getJoinType(), edgeB.getJoinType())) { leftRequired = LongBitmap.newBitmapUnion(leftRequired, childA.getRightSubNodes(joinEdges)); + childA.addRejectEdge(edgeB); } } @@ -299,9 +305,11 @@ public class HyperGraph { JoinEdge childA = joinEdges.get(i); if (!JoinType.isAssoc(edgeB.getJoinType(), childA.getJoinType())) { rightRequired = LongBitmap.newBitmapUnion(rightRequired, childA.getRightSubNodes(joinEdges)); + childA.addRejectEdge(edgeB); } if (!JoinType.isRAssoc(edgeB.getJoinType(), childA.getJoinType())) { rightRequired = LongBitmap.newBitmapUnion(rightRequired, childA.getLeftSubNodes(joinEdges)); + childA.addRejectEdge(edgeB); } } edgeB.setLeftExtendedNodes(leftRequired); @@ -593,57 +601,75 @@ public class HyperGraph { * compare hypergraph * * @param viewHG the compared hyper graph - * @return null represents not compatible, or return some expression which can - * be pull up from this hyper graph + * @return Comparison result */ - public @Nullable List isLogicCompatible(HyperGraph viewHG, LogicalCompatibilityContext ctx) { - Map queryToView = constructEdgeMap(viewHG, ctx.getQueryToViewEdgeExpressionMapping()); + public ComparisonResult isLogicCompatible(HyperGraph viewHG, LogicalCompatibilityContext ctx) { + // 1 try to construct a map which can be mapped from edge to edge + Map queryToView = constructMapWithNode(viewHG, ctx.getQueryToViewNodeIDMapping()); - // All edge in view must have a mapped edge in query - if (queryToView.size() != viewHG.edgeSize()) { - return null; + // 2. compare them by expression and extract residual expr + ComparisonResult.Builder builder = new ComparisonResult.Builder(); + ComparisonResult edgeCompareRes = compareEdgesWithExpr(queryToView, ctx.getQueryToViewEdgeExpressionMapping()); + if (edgeCompareRes.isInvalid()) { + return ComparisonResult.INVALID; + } + builder.addComparisonResult(edgeCompareRes); + + // 3. pull join edge of view is no sense, so reject them + if (!queryToView.values().containsAll(viewHG.joinEdges)) { + return ComparisonResult.INVALID; } - boolean allMatch = queryToView.entrySet().stream() - .allMatch(entry -> - compareEdgeWithNode(entry.getKey(), entry.getValue(), ctx.getQueryToViewNodeIDMapping())); - if (!allMatch) { - return null; + // 4. process residual edges + List residualQueryJoin = + processOrphanEdges(Sets.difference(Sets.newHashSet(joinEdges), queryToView.keySet())); + if (residualQueryJoin == null) { + return ComparisonResult.INVALID; } + builder.addQueryExpressions(residualQueryJoin); - // join edges must be identical - boolean isJoinIdentical = joinEdges.stream() - .allMatch(queryToView::containsKey); - if (!isJoinIdentical) { - return null; + List residualQueryFilter = + processOrphanEdges(Sets.difference(Sets.newHashSet(filterEdges), queryToView.keySet())); + if (residualQueryFilter == null) { + return ComparisonResult.INVALID; } + builder.addQueryExpressions(residualQueryFilter); - // extract all top filters - List residualFilterEdges = filterEdges.stream() - .filter(e -> !queryToView.containsKey(e)) - .collect(ImmutableList.toImmutableList()); - if (residualFilterEdges.stream().anyMatch(e -> !e.isTopFilter())) { - return null; + List residualViewFilter = + processOrphanEdges( + Sets.difference(Sets.newHashSet(viewHG.filterEdges), Sets.newHashSet(queryToView.values()))); + if (residualViewFilter == null) { + return ComparisonResult.INVALID; } - return residualFilterEdges.stream() - .flatMap(e -> e.getExpressions().stream()) - .collect(ImmutableList.toImmutableList()); + builder.addViewExpressions(residualViewFilter); + + return builder.build(); } - private Map constructEdgeMap(HyperGraph viewHG, Map exprMap) { - Map exprToEdge = constructExprMap(viewHG); - Map queryToView = new HashMap<>(); - joinEdges.stream() - .filter(e -> !e.getExpressions().isEmpty() - && exprMap.containsKey(e.getExpression(0)) - && compareEdgeWithExpr(e, exprToEdge.get(exprMap.get(e.getExpression(0))), exprMap)) - .forEach(e -> queryToView.put(e, exprToEdge.get(exprMap.get(e.getExpression(0))))); - filterEdges.stream() - .filter(e -> !e.getExpressions().isEmpty() - && exprMap.containsKey(e.getExpression(0)) - && compareEdgeWithExpr(e, exprToEdge.get(exprMap.get(e.getExpression(0))), exprMap)) - .forEach(e -> queryToView.put(e, exprToEdge.get(exprMap.get(e.getExpression(0))))); - return queryToView; + private List processOrphanEdges(Set edges) { + List expressions = new ArrayList<>(); + for (Edge edge : edges) { + if (!edge.canPullUp()) { + return null; + } + expressions.addAll(edge.getExpressions()); + } + return expressions; + } + + private Map constructMapWithNode(HyperGraph viewHG, Map nodeMap) { + // TODO use hash map to reduce loop + Map joinEdgeMap = joinEdges.stream().map(qe -> { + Optional viewEdge = viewHG.joinEdges.stream() + .filter(ve -> compareEdgeWithNode(qe, ve, nodeMap)).findFirst(); + return Pair.of(qe, viewEdge); + }).filter(e -> e.second.isPresent()).collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second.get())); + Map filterEdgeMap = filterEdges.stream().map(qe -> { + Optional viewEdge = viewHG.filterEdges.stream() + .filter(ve -> compareEdgeWithNode(qe, ve, nodeMap)).findFirst(); + return Pair.of(qe, viewEdge); + }).filter(e -> e.second.isPresent()).collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second.get())); + return ImmutableMap.builder().putAll(joinEdgeMap).putAll(filterEdgeMap).build(); } private boolean compareEdgeWithNode(Edge t, Edge o, Map nodeMap) { @@ -686,24 +712,40 @@ public class HyperGraph { return bitmap2 == newBitmap1; } - private boolean compareEdgeWithExpr(Edge t, Edge o, Map expressionMap) { - if (t.getExpressions().size() != o.getExpressions().size()) { - return false; - } - int size = t.getExpressions().size(); - for (int i = 0; i < size; i++) { - if (!Objects.equals(expressionMap.get(t.getExpression(i)), o.getExpression(i))) { - return false; + private ComparisonResult compareEdgesWithExpr(Map queryToViewedgeMap, + Map queryToView) { + ComparisonResult.Builder builder = new ComparisonResult.Builder(); + for (Entry e : queryToViewedgeMap.entrySet()) { + ComparisonResult res = compareEdgeWithExpr(e.getKey(), e.getValue(), queryToView); + if (res.isInvalid()) { + return ComparisonResult.INVALID; } + builder.addComparisonResult(res); } - return true; + return builder.build(); } - private Map constructExprMap(HyperGraph hyperGraph) { - Map exprToEdge = new HashMap<>(); - hyperGraph.joinEdges.forEach(edge -> edge.getExpressions().forEach(expr -> exprToEdge.put(expr, edge))); - hyperGraph.filterEdges.forEach(edge -> edge.getExpressions().forEach(expr -> exprToEdge.put(expr, edge))); - return exprToEdge; + private ComparisonResult compareEdgeWithExpr(Edge query, Edge view, Map queryToView) { + Set queryExprSet = query.getExpressionSet(); + Set viewExprSet = view.getExpressionSet(); + + Set equalViewExpr = new HashSet<>(); + List residualQueryExpr = new ArrayList<>(); + for (Expression queryExpr : queryExprSet) { + if (queryToView.containsKey(queryExpr) && viewExprSet.contains(queryToView.get(queryExpr))) { + equalViewExpr.add(queryToView.get(queryExpr)); + } else { + residualQueryExpr.add(queryExpr); + } + } + List residualViewExpr = ImmutableList.copyOf(Sets.difference(viewExprSet, equalViewExpr)); + if (!residualViewExpr.isEmpty() && !view.canPullUp()) { + return ComparisonResult.INVALID; + } + if (!residualQueryExpr.isEmpty() && !query.canPullUp()) { + return ComparisonResult.INVALID; + } + return new ComparisonResult(residualQueryExpr, residualViewExpr); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java index 4b88ce9f60..c47300922b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java @@ -22,6 +22,8 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import com.google.common.collect.ImmutableSet; + import java.util.BitSet; import java.util.List; import java.util.Set; @@ -51,6 +53,8 @@ public abstract class Edge { // record all sub nodes behind in this operator. It's T function in paper private final long subTreeNodes; + private long rejectNodes = 0; + /** * Create simple edge. */ @@ -71,6 +75,10 @@ public abstract class Edge { return LongBitmap.getCardinality(leftExtendedNodes) == 1 && LongBitmap.getCardinality(rightExtendedNodes) == 1; } + public void addRejectEdge(Edge edge) { + rejectNodes = LongBitmap.newBitmapUnion(edge.getReferenceNodes(), rejectNodes); + } + public void addLeftExtendNode(long left) { this.leftExtendedNodes = LongBitmap.or(this.leftExtendedNodes, left); } @@ -171,6 +179,20 @@ public abstract class Edge { public abstract List getExpressions(); + public Set getExpressionSet() { + return ImmutableSet.copyOf(getExpressions()); + } + + public boolean canPullUp() { + // Only inner join and filter with none rejectNodes can be pull up + return rejectNodes == 0 + && !(this instanceof JoinEdge && !((JoinEdge) this).getJoinType().isInnerJoin()); + } + + public long getRejectNodes() { + return rejectNodes; + } + public Expression getExpression(int i) { return getExpressions().get(i); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/FilterEdge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/FilterEdge.java index ec03787102..57c6d9660d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/FilterEdge.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/FilterEdge.java @@ -22,7 +22,6 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import java.util.ArrayList; import java.util.BitSet; import java.util.List; import java.util.Set; @@ -32,25 +31,11 @@ import java.util.Set; */ public class FilterEdge extends Edge { private final LogicalFilter filter; - private final List rejectEdges; public FilterEdge(LogicalFilter filter, int index, BitSet childEdges, long subTreeNodes, long childRequireNodes) { super(index, childEdges, new BitSet(), subTreeNodes, childRequireNodes, 0L); this.filter = filter; - rejectEdges = new ArrayList<>(); - } - - public void addRejectJoin(JoinEdge joinEdge) { - rejectEdges.add(joinEdge.getIndex()); - } - - public List getRejectEdges() { - return rejectEdges; - } - - public boolean isTopFilter() { - return rejectEdges.isEmpty(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 90ebe567c4..c63e2d85af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -136,12 +136,14 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from(queryToViewTableMapping, queryToViewSlotMapping, queryStructInfo, viewStructInfo); - List pulledUpExpressions = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, + ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, compatibilityContext); - if (pulledUpExpressions == null) { + if (comparisonResult.isInvalid()) { logger.debug(currentClassName + " graph logical is not equals so continue"); continue; } + // TODO: Use set of list? And consider view expr + List pulledUpExpressions = ImmutableList.copyOf(comparisonResult.getQueryExpressions()); // set pulled up expression to queryStructInfo predicates and update related predicates if (!pulledUpExpressions.isEmpty()) { queryStructInfo.addPredicates(pulledUpExpressions); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java new file mode 100644 index 0000000000..cc8284f33e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv; + +import org.apache.doris.nereids.trees.expressions.Expression; + +import com.google.common.collect.ImmutableList; + +import java.util.Collection; +import java.util.List; + +/** + * comparison result of view and query + */ +public class ComparisonResult { + public static final ComparisonResult INVALID = new ComparisonResult(ImmutableList.of(), ImmutableList.of(), false); + public static final ComparisonResult EMPTY = new ComparisonResult(ImmutableList.of(), ImmutableList.of(), true); + private final boolean valid; + private final List viewExpressions; + private final List queryExpressions; + + public ComparisonResult(List queryExpressions, List viewExpressions) { + this(queryExpressions, viewExpressions, true); + } + + ComparisonResult(List queryExpressions, List viewExpressions, boolean valid) { + this.viewExpressions = ImmutableList.copyOf(viewExpressions); + this.queryExpressions = ImmutableList.copyOf(queryExpressions); + this.valid = valid; + } + + public List getViewExpressions() { + return viewExpressions; + } + + public List getQueryExpressions() { + return queryExpressions; + } + + public boolean isInvalid() { + return !valid; + } + + /** + * Builder + */ + public static class Builder { + ImmutableList.Builder queryBuilder = new ImmutableList.Builder<>(); + ImmutableList.Builder viewBuilder = new ImmutableList.Builder<>(); + boolean valid = true; + + /** + * add comparisonResult + */ + public Builder addComparisonResult(ComparisonResult comparisonResult) { + if (comparisonResult.isInvalid()) { + valid = false; + return this; + } + queryBuilder.addAll(comparisonResult.getQueryExpressions()); + viewBuilder.addAll(comparisonResult.getViewExpressions()); + return this; + } + + public Builder addQueryExpressions(Collection expressions) { + queryBuilder.addAll(expressions); + return this; + } + + public Builder addViewExpressions(Collection expressions) { + viewBuilder.addAll(expressions); + return this; + } + + public ComparisonResult build() { + return new ComparisonResult(queryBuilder.build(), viewBuilder.build(), valid); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 3c9814cdce..20da9ee12f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -263,7 +263,7 @@ public class StructInfo { * For inner join should judge only the join tables, * for other join type should also judge the join direction, it's input filter that can not be pulled up etc. */ - public static @Nullable List isGraphLogicalEquals(StructInfo queryStructInfo, StructInfo viewStructInfo, + public static ComparisonResult isGraphLogicalEquals(StructInfo queryStructInfo, StructInfo viewStructInfo, LogicalCompatibilityContext compatibilityContext) { return queryStructInfo.hyperGraph.isLogicCompatible(viewStructInfo.hyperGraph, compatibilityContext); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java index d0e5084a1c..cfc88b2aa3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java @@ -62,7 +62,7 @@ class CompareOuterJoinTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0); HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0); - Assertions.assertTrue(h1.isLogicCompatible(h2, constructContext(p1, p2)) != null); + Assertions.assertFalse(h1.isLogicCompatible(h2, constructContext(p1, p2)).isInvalid()); } @Test @@ -79,7 +79,7 @@ class CompareOuterJoinTest extends SqlTestBase { .getAllPlan().get(0); HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0); HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0); - Assertions.assertTrue(h1.isLogicCompatible(h2, constructContext(p1, p2)) != null); + Assertions.assertFalse(h1.isLogicCompatible(h2, constructContext(p1, p2)).isInvalid()); } @Test @@ -104,7 +104,7 @@ class CompareOuterJoinTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0); HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0); - List exprList = h1.isLogicCompatible(h2, constructContext(p1, p2)); + List exprList = h1.isLogicCompatible(h2, constructContext(p1, p2)).getQueryExpressions(); Assertions.assertEquals(1, exprList.size()); Assertions.assertEquals("(id = 0)", exprList.get(0).toSql()); } @@ -132,7 +132,7 @@ class CompareOuterJoinTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0); HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0); - List exprList = h1.isLogicCompatible(h2, constructContext(p1, p2)); + List exprList = h1.isLogicCompatible(h2, constructContext(p1, p2)).getQueryExpressions(); Assertions.assertEquals(0, exprList.size()); } @@ -159,7 +159,7 @@ class CompareOuterJoinTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0); HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0); - List exprList = h1.isLogicCompatible(h2, constructContext(p1, p2)); + List exprList = h1.isLogicCompatible(h2, constructContext(p1, p2)).getQueryExpressions(); Assertions.assertEquals(1, exprList.size()); Assertions.assertEquals("(id = 0)", exprList.get(0).toSql()); } @@ -187,8 +187,7 @@ class CompareOuterJoinTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0); HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0); - List exprList = h1.isLogicCompatible(h2, constructContext(p1, p2)); - Assertions.assertEquals(null, exprList); + Assertions.assertTrue(h1.isLogicCompatible(h2, constructContext(p1, p2)).isInvalid()); } LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java new file mode 100644 index 0000000000..d4818a844e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.joinorder.hypergraph; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.RuleSet; +import org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewRule; +import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult; +import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo; +import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.sqltest.SqlTestBase; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.util.PlanChecker; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class PullupExpressionTest extends SqlTestBase { + @Test + void testPullUpQueryFilter() { + CascadesContext c1 = createCascadesContext( + "select * from T1 join T2 on T1.id = T2.id where T1.id = 1", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 join T2 on T1.id = T2.id", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0); + HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0); + ComparisonResult res = h1.isLogicCompatible(h2, constructContext(p1, p2)); + Assertions.assertEquals(2, res.getQueryExpressions().size()); + Assertions.assertEquals("(id = 1)", res.getQueryExpressions().get(0).toSql()); + Assertions.assertEquals("(id = 1)", res.getQueryExpressions().get(1).toSql()); + } + + @Test + void testPullUpQueryJoinCondition() { + CascadesContext c1 = createCascadesContext( + "select * from T1 join T2 on T1.id = T2.id and T1.score = T2.score", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 join T2 on T1.id = T2.id", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0); + HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0); + ComparisonResult res = h1.isLogicCompatible(h2, constructContext(p1, p2)); + Assertions.assertEquals(1, res.getQueryExpressions().size()); + Assertions.assertEquals("(score = score)", res.getQueryExpressions().get(0).toSql()); + } + + @Test + void testPullUpViewFilter() { + CascadesContext c1 = createCascadesContext( + "select * from T1 join T2 on T1.id = T2.id", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 join T2 on T1.id = T2.id where T1.id = 1 and T2.id = 1", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0); + HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0); + ComparisonResult res = h1.isLogicCompatible(h2, constructContext(p1, p2)); + Assertions.assertEquals(2, res.getViewExpressions().size()); + Assertions.assertEquals("(id = 1)", res.getViewExpressions().get(0).toSql()); + Assertions.assertEquals("(id = 1)", res.getViewExpressions().get(1).toSql()); + } + + @Test + void testPullUpViewJoinCondition() { + CascadesContext c1 = createCascadesContext( + "select * from T1 join T2 on T1.id = T2.id ", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan().child(0); + CascadesContext c2 = createCascadesContext( + "select * from T1 join T2 on T1.id = T2.id and T1.score = T2.score", + connectContext + ); + Plan p2 = PlanChecker.from(c2) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0).child(0); + HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0); + HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0); + ComparisonResult res = h1.isLogicCompatible(h2, constructContext(p1, p2)); + Assertions.assertEquals(1, res.getViewExpressions().size()); + Assertions.assertEquals("(score = score)", res.getViewExpressions().get(0).toSql()); + } + + LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { + StructInfo st1 = AbstractMaterializedViewRule.extractStructInfo(p1, + null).get(0); + StructInfo st2 = AbstractMaterializedViewRule.extractStructInfo(p2, + null).get(0); + RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); + SlotMapping sm = SlotMapping.generate(rm); + return LogicalCompatibilityContext.from(rm, sm, st1, st2); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java index 6ac41a4981..f68365f670 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java @@ -79,7 +79,7 @@ class BuildStructInfoTest extends SqlTestBase { .when(j -> { HyperGraph structInfo = HyperGraph.toStructInfo(j).get(0); Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin()); - Assertions.assertEquals(0, (int) structInfo.getFilterEdge(0).getRejectEdges().get(0)); + Assertions.assertEquals(3L, structInfo.getFilterEdge(0).getRejectNodes()); return true; })); @@ -92,7 +92,7 @@ class BuildStructInfoTest extends SqlTestBase { .when(j -> { HyperGraph structInfo = HyperGraph.toStructInfo(j).get(0); Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin()); - Assertions.assertTrue(structInfo.getFilterEdge(0).getRejectEdges().isEmpty()); + Assertions.assertEquals(0, structInfo.getFilterEdge(0).getRejectNodes()); return true; })); }