From 4d12d8885ef25c48dcc78b3e1909b48c649c2884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=A2=E5=81=A5?= Date: Tue, 17 Oct 2023 14:10:21 +0800 Subject: [PATCH] [feature](Nereids): graphSimplifier should compare edge1BeforeEdge2 and edge2BeforeEdge1 (#25416) --- .../jobs/joinorder/hypergraph/Edge.java | 8 +++--- .../joinorder/hypergraph/GraphSimplifier.java | 20 +++++++------ .../jobs/joinorder/hypergraph/HyperGraph.java | 17 +++++------ .../hypergraph/GraphSimplifierTest.java | 28 ++++++++++++++++++- .../doris/nereids/util/HyperGraphBuilder.java | 12 ++++---- 5 files changed, 58 insertions(+), 27 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Edge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Edge.java index 601aa001ae..8dc3f12bc4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Edge.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Edge.java @@ -53,13 +53,13 @@ public class Edge { private long referenceNodes = LongBitmap.newBitmap(); // record the left child edges and right child edges in origin plan tree - private BitSet leftChildEdges; - private BitSet rightChildEdges; + private final BitSet leftChildEdges; + private final BitSet rightChildEdges; // record the edges in the same operator - private BitSet curJoinEdges = new BitSet(); + private final BitSet curJoinEdges = new BitSet(); // record all sub nodes behind in this operator. It's T function in paper - private Long subTreeNodes; + private final long subTreeNodes; /** * Create simple edge. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java index 1beb4a4f06..a8ed7e81e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java @@ -418,14 +418,18 @@ public class GraphSimplifier { private SimplificationStep orderJoin(Pair edge1Before2, Pair edge2Before1, int edgeIndex1, int edgeIndex2) { - Edge edge = processMissedEdges(edgeIndex1, edgeIndex2, edge1Before2.second); - Cost cost1Before2 = calCost(edge, edge1Before2.first, - cacheStats.get(edge1Before2.second.getLeftExtendedNodes()), - cacheStats.get(edge1Before2.second.getRightExtendedNodes())); - edge = processMissedEdges(edgeIndex1, edgeIndex2, edge2Before1.second); - Cost cost2Before1 = calCost(edge, edge1Before2.first, + // TODO: Consider miss edges when construct join. + // considering + // a + // / \ + // b - c + // when constructing edge_ab before edge_bc. edge_ac should be added on top join + Cost cost1Before2 = calCost(edge1Before2.second, edge1Before2.first, cacheStats.get(edge1Before2.second.getLeftExtendedNodes()), cacheStats.get(edge1Before2.second.getRightExtendedNodes())); + Cost cost2Before1 = calCost(edge2Before1.second, edge2Before1.first, + cacheStats.get(edge2Before1.second.getLeftExtendedNodes()), + cacheStats.get(edge2Before1.second.getRightExtendedNodes())); double benefit = Double.MAX_VALUE; SimplificationStep step; // Choose the plan with smaller cost and make the simplification step to replace the old edge by it. @@ -505,10 +509,10 @@ public class GraphSimplifier { Edge edge2 = graph.getEdge(j); if (edge1.isSub(edge2)) { Preconditions.checkArgument(circleDetector.tryAddDirectedEdge(i, j), - String.format("Edge %s violates Edge %s", edge1, edge2)); + "Edge %s violates Edge %s", edge1, edge2); } else if (edge2.isSub(edge1)) { Preconditions.checkArgument(circleDetector.tryAddDirectedEdge(j, i), - String.format("Edge %s violates Edge %s", edge2, edge1)); + "Edge %s violates Edge %s", edge2, edge1); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java index 67f246cd80..ccb1aff31c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java @@ -227,19 +227,20 @@ public class HyperGraph { edgeB.setRightExtendedNodes(rightRequired); } - private BitSet subTreeEdges(Edge edge) { - BitSet bitSet = new BitSet(); - bitSet.or(subTreeEdges(edge.getLeftChildEdges())); - bitSet.or(subTreeEdges(edge.getRightChildEdges())); - bitSet.set(edge.getIndex()); - return bitSet; + private BitSet subTreeEdge(Edge edge) { + long subTreeNodes = edge.getSubTreeNodes(); + BitSet subEdges = new BitSet(); + edges.stream() + .filter(e -> LongBitmap.isSubset(subTreeNodes, e.getReferenceNodes())) + .forEach(e -> subEdges.set(e.getIndex())); + return subEdges; } private BitSet subTreeEdges(BitSet edgeSet) { BitSet bitSet = new BitSet(); edgeSet.stream() - .mapToObj(i -> subTreeEdges(edges.get(i))) - .forEach(b -> bitSet.or(b)); + .mapToObj(i -> subTreeEdge(edges.get(i))) + .forEach(bitSet::or); return bitSet; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifierTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifierTest.java index 3cf0b5f895..bd86c810b8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifierTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifierTest.java @@ -21,10 +21,12 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.receiver.Counter; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.util.HyperGraphBuilder; +import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -public class GraphSimplifierTest { +class GraphSimplifierTest { @Test void testStarQuery() { // t1 @@ -210,4 +212,28 @@ public class GraphSimplifierTest { Assertions.assertTrue(counter.getLimit() >= 0); } } + + @Disabled + @Test + void benchGraphSimplifier() { + int tableNum = 64; + int edgeNum = 64 * 63 / 2; + int limit = 1000; + + int times = 1; + double totalTime = 0; + for (int i = 0; i < times; i++) { + totalTime += benchGraphSimplifier(tableNum, edgeNum, limit); + } + System.out.println(totalTime / times); + } + + double benchGraphSimplifier(int tableNum, int edgeNum, int limit) { + HyperGraph hyperGraph = new HyperGraphBuilder(Sets.newHashSet(JoinType.INNER_JOIN)) + .randomBuildWith(tableNum, edgeNum); + double now = System.currentTimeMillis(); + GraphSimplifier graphSimplifier = new GraphSimplifier(hyperGraph); + graphSimplifier.simplifyGraph(limit); + return System.currentTimeMillis() - now; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/HyperGraphBuilder.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/HyperGraphBuilder.java index e33c28ae93..853dfd0789 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/HyperGraphBuilder.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/HyperGraphBuilder.java @@ -131,9 +131,9 @@ public class HyperGraphBuilder { private void randomBuildInit(int tableNum, int edgeNum) { Preconditions.checkArgument(edgeNum >= tableNum - 1, - String.format("We can't build a connected graph with %d tables %d edges", tableNum, edgeNum)); + "We can't build a connected graph with %s tables %s edges", tableNum, edgeNum); Preconditions.checkArgument(edgeNum <= tableNum * (tableNum - 1) / 2, - String.format("The edges are redundant with %d tables %d edges", tableNum, edgeNum)); + "The edges are redundant with %s tables %s edges", tableNum, edgeNum); int[] tableRowCounts = new int[tableNum]; for (int i = 1; i <= tableNum; i++) { @@ -203,9 +203,9 @@ public class HyperGraphBuilder { public HyperGraphBuilder addEdge(JoinType joinType, int node1, int node2) { Preconditions.checkArgument(node1 >= 0 && node1 < rowCounts.size(), - String.format("%d must in [%d, %d)", node1, 0, rowCounts.size())); + "%d must in [%s, %ds", node1, 0, rowCounts.size()); Preconditions.checkArgument(node2 >= 0 && node1 < rowCounts.size(), - String.format("%d must in [%d, %d)", node1, 0, rowCounts.size())); + "%d must in [%d, %d)", node1, 0, rowCounts.size()); BitSet leftBitmap = new BitSet(); leftBitmap.set(node1); @@ -218,8 +218,8 @@ public class HyperGraphBuilder { if (!fullKey.isPresent()) { Optional leftKey = findPlan(leftBitmap); Optional rightKey = findPlan(rightBitmap); - assert leftKey.isPresent() && rightKey.isPresent() : String.format("can not find plan %s-%s", leftBitmap, - rightBitmap); + Preconditions.checkArgument(leftKey.isPresent() && rightKey.isPresent(), + "can not find plan %s-%s", leftBitmap, rightBitmap); Plan leftPlan = plans.get(leftKey.get()); Plan rightPlan = plans.get(rightKey.get()); LogicalJoin join = new LogicalJoin<>(joinType, leftPlan, rightPlan);