diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java index 17c77c855e..9f04ae38cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.DPhyperNode; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext; import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; @@ -42,6 +43,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.util.PlanUtils; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; @@ -52,6 +54,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import javax.annotation.Nullable; /** * The graph is a join graph, whose node is the leaf plan and edge is a join operator. @@ -577,6 +580,99 @@ public class HyperGraph { LongBitmap.getIterator(addedNodes).forEach(index -> nodes.get(index).attachEdge(edge)); } + /** + * compare hypergraph + * + * @param viewHG the compared hyper graph + * @return null represents not compatible, or return some expression which can + * be pull up from this hyper graph + */ + public @Nullable List isLogicCompatible(HyperGraph viewHG, LogicalCompatibilityContext ctx) { + if (viewHG.filterEdges.size() != filterEdges.size() && viewHG.joinEdges.size() != joinEdges.size()) { + return null; + } + Map queryToView = constructEdgeMap(viewHG, ctx.getQueryToViewEdgeExpressionMapping()); + if (queryToView.size() != filterEdges.size() + joinEdges.size()) { + return null; + } + boolean allMatch = queryToView.entrySet().stream() + .allMatch(entry -> + compareEdgeWithNode(entry.getKey(), entry.getValue(), ctx.getQueryToViewNodeIDMapping())); + if (!allMatch) { + return null; + } + return ImmutableList.of(); + } + + private Map constructEdgeMap(HyperGraph viewHG, Map exprMap) { + Map exprToEdge = constructExprMap(viewHG); + Map queryToView = new HashMap<>(); + joinEdges.stream() + .filter(e -> !e.getExpressions().isEmpty() + && exprMap.containsKey(e.getExpression(0)) + && compareEdgeWithExpr(e, exprToEdge.get(exprMap.get(e.getExpression(0))), exprMap)) + .forEach(e -> queryToView.put(e, exprToEdge.get(exprMap.get(e.getExpression(0))))); + filterEdges.stream() + .filter(e -> !e.getExpressions().isEmpty() + && exprMap.containsKey(e.getExpression(0)) + && compareEdgeWithExpr(e, exprToEdge.get(exprMap.get(e.getExpression(0))), exprMap)) + .forEach(e -> queryToView.put(e, exprToEdge.get(exprMap.get(e.getExpression(0))))); + return queryToView; + } + + private boolean compareEdgeWithNode(Edge t, Edge o, Map nodeMap) { + if (t instanceof FilterEdge && o instanceof FilterEdge) { + return false; + } else if (t instanceof JoinEdge && o instanceof JoinEdge) { + return compareJoinEdge((JoinEdge) t, (JoinEdge) o, nodeMap); + } + return false; + } + + private boolean compareJoinEdge(JoinEdge t, JoinEdge o, Map nodeMap) { + long tLeft = t.getLeftExtendedNodes(); + long tRight = t.getRightExtendedNodes(); + long oLeft = o.getLeftExtendedNodes(); + long oRight = o.getRightExtendedNodes(); + if (!t.getJoinType().equals(o.getJoinType())) { + if (!t.getJoinType().swap().equals(o.getJoinType())) { + return false; + } + oRight = o.getLeftExtendedNodes(); + oLeft = o.getRightExtendedNodes(); + } + return compareNodeMap(tLeft, oLeft, nodeMap) && compareNodeMap(tRight, oRight, nodeMap); + } + + private boolean compareNodeMap(long bitmap1, long bitmap2, Map nodeIDMap) { + long newBitmap1 = LongBitmap.newBitmap(); + for (int i : LongBitmap.getIterator(bitmap1)) { + int mappedI = nodeIDMap.getOrDefault(i, 0); + newBitmap1 = LongBitmap.set(newBitmap1, mappedI); + } + return bitmap2 == newBitmap1; + } + + private boolean compareEdgeWithExpr(Edge t, Edge o, Map expressionMap) { + if (t.getExpressions().size() != o.getExpressions().size()) { + return false; + } + int size = t.getExpressions().size(); + for (int i = 0; i < size; i++) { + if (!expressionMap.get(t.getExpression(i)).equals(o.getExpression(i))) { + return false; + } + } + return true; + } + + private Map constructExprMap(HyperGraph hyperGraph) { + Map exprToEdge = new HashMap<>(); + hyperGraph.joinEdges.forEach(edge -> edge.getExpressions().forEach(expr -> exprToEdge.put(expr, edge))); + hyperGraph.filterEdges.forEach(edge -> edge.getExpressions().forEach(expr -> exprToEdge.put(expr, edge))); + return exprToEdge; + } + /** * For the given hyperGraph, make a textual representation in the form * of a dotty graph. You can save this to a file and then use Graphviz diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java index 6f0920e1f5..4b88ce9f60 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java @@ -171,6 +171,10 @@ public abstract class Edge { public abstract List getExpressions(); + public Expression getExpression(int i) { + return getExpressions().get(i); + } + @Override public String toString() { return String.format("<%s - %s>", LongBitmap.toString(leftExtendedNodes), LongBitmap.toString( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 09b2ea7761..c265b1de3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -105,7 +105,7 @@ public abstract class AbstractMaterializedViewRule { LogicalCompatibilityContext.from(queryToViewTableMapping, queryToViewSlotMapping, queryStructInfo, viewStructInfo); // todo outer join compatibility check - if (!StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, compatibilityContext)) { + if (StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, compatibilityContext) == null) { continue; } SplitPredicate compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo, @@ -347,7 +347,7 @@ public abstract class AbstractMaterializedViewRule { /** * Extract struct info from plan, support to get struct info from logical plan or plan in group. */ - protected List extractStructInfo(Plan plan, CascadesContext cascadesContext) { + public static List extractStructInfo(Plan plan, CascadesContext cascadesContext) { if (plan.getGroupExpression().isPresent() && !plan.getGroupExpression().get().getOwnerGroup().getStructInfos().isEmpty()) { return plan.getGroupExpression().get().getOwnerGroup().getStructInfos(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java index 3a7cd1d2bc..2632b7e4c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java @@ -36,19 +36,26 @@ import java.util.Map; * For outer join we should check the outer join compatibility between query and view */ public class LogicalCompatibilityContext { - private BiMap queryToViewNodeMapping; - private BiMap queryToViewEdgeExpressionMapping; + private final BiMap queryToViewNodeMapping; + private final BiMap queryToViewEdgeExpressionMapping; + private final BiMap queryToViewNodeIDMapping; public LogicalCompatibilityContext(BiMap queryToViewNodeMapping, BiMap queryToViewEdgeExpressionMapping) { this.queryToViewNodeMapping = queryToViewNodeMapping; this.queryToViewEdgeExpressionMapping = queryToViewEdgeExpressionMapping; + this.queryToViewNodeIDMapping = HashBiMap.create(); + queryToViewNodeMapping.forEach((k, v) -> queryToViewNodeIDMapping.put(k.getIndex(), v.getIndex())); } public BiMap getQueryToViewNodeMapping() { return queryToViewNodeMapping; } + public BiMap getQueryToViewNodeIDMapping() { + return queryToViewNodeIDMapping; + } + public BiMap getQueryToViewEdgeExpressionMapping() { return queryToViewEdgeExpressionMapping; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 3e23ae2e49..f889058fa2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -38,6 +38,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.nereids.util.ExpressionUtils; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -237,10 +238,11 @@ public class StructInfo { * For inner join should judge only the join tables, * for other join type should also judge the join direction, it's input filter that can not be pulled up etc. */ - public static boolean isGraphLogicalEquals(StructInfo queryStructInfo, StructInfo viewStructInfo, + public static @Nullable List isGraphLogicalEquals(StructInfo queryStructInfo, StructInfo viewStructInfo, LogicalCompatibilityContext compatibilityContext) { - // TODO: if not inner join, should check the join graph logical equivalence - return true; + // TODO: open it after supporting filter + // return queryStructInfo.hyperGraph.isLogicCompatible(viewStructInfo.hyperGraph, compatibilityContext); + return ImmutableList.of(); } private static class RelationCollector extends DefaultPlanVisitor> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java index 888b538d04..ca490b9ef1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java @@ -126,6 +126,7 @@ public class ExpressionLineageReplacer extends DefaultPlanVisitor each.collectToList(NamedExpression.class::isInstance)) .flatMap(Collection::stream) .map(NamedExpression.class::cast) + .distinct() .collect(Collectors.toMap(NamedExpression::getExprId, expr -> expr)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java new file mode 100644 index 0000000000..f8ba3ead2e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.joinorder.hypergraph; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.RuleSet; +import org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewRule; +import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo; +import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.sqltest.SqlTestBase; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.util.HyperGraphBuilder; +import org.apache.doris.nereids.util.PlanChecker; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class CompareOuterJoinTest extends SqlTestBase { + @Test + void testStarGraphWithInnerJoin() { + // t2 + // | + //t3-- t1 -- t4 + // | + // t5 + CascadesContext c1 = createCascadesContext( + "select * from T1, T2, T3, T4 where " + + "T1.id = T2.id " + + "and T1.id = T3.id " + + "and T1.id = T4.id ", + connectContext + ); + Plan p1 = PlanChecker.from(c1) + .analyze() + .rewrite() + .getPlan(); + Plan p2 = PlanChecker.from(c1) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0); + HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0); + HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0); + Assertions.assertTrue(h1.isLogicCompatible(h2, constructContext(p1, p2)) != null); + } + + @Test + void testRandomQuery() { + Plan p1 = new HyperGraphBuilder().randomBuildPlanWith(3, 3); + p1 = PlanChecker.from(connectContext, p1) + .analyze() + .rewrite() + .getPlan(); + Plan p2 = PlanChecker.from(connectContext, p1) + .analyze() + .rewrite() + .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) + .getAllPlan().get(0); + HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0); + HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0); + Assertions.assertTrue(h1.isLogicCompatible(h2, constructContext(p1, p2)) != null); + } + + LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { + StructInfo st1 = AbstractMaterializedViewRule.extractStructInfo(p1, + null).get(0); + StructInfo st2 = AbstractMaterializedViewRule.extractStructInfo(p2, + null).get(0); + RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); + SlotMapping sm = SlotMapping.generate(rm); + return LogicalCompatibilityContext.from(rm, sm, st1, st2); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index 058a2de2f0..afdc5a8769 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -564,6 +564,10 @@ public class PlanChecker { return cascadesContext.getMemo().copyOut(); } + public List getAllPlan() { + return cascadesContext.getMemo().copyOutAll(); + } + private PhysicalPlan chooseBestPlan(Group rootGroup, PhysicalProperties physicalProperties) { GroupExpression groupExpression = rootGroup.getLowestCostPlan(physicalProperties).orElseThrow( () -> new AnalysisException("lowestCostPlans with physicalProperties("