From 36737fe9f41407c3941c20d07db7bc3771c560fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=A2=E5=81=A5?= Date: Wed, 30 Nov 2022 21:35:45 +0800 Subject: [PATCH] [feature](Nereids): Add cache to avoid repeatly calculation in DPhyp (#14585) --- .../jobs/joinorder/hypergraph/Edge.java | 15 +- .../joinorder/hypergraph/GraphSimplifier.java | 4 +- .../jobs/joinorder/hypergraph/HyperGraph.java | 6 - .../jobs/joinorder/hypergraph/Node.java | 55 +--- .../hypergraph/SubgraphEnumerator.java | 261 ++++++++++++++---- .../joinorder/hypergraph/bitmap/Bitmap.java | 13 +- .../hypergraph/receiver/AbstractReceiver.java | 3 +- .../hypergraph/receiver/Counter.java | 5 +- .../hypergraph/receiver/PlanReceiver.java | 19 +- .../hypergraph/GraphSimplifierTest.java | 34 ++- .../hypergraph/SubgraphEnumeratorTest.java | 8 +- 11 files changed, 290 insertions(+), 133 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Edge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Edge.java index 576e7910c6..8ea7e167fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Edge.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Edge.java @@ -36,6 +36,7 @@ public class Edge { // left and right may not overlap, and both must have at least one bit set. private BitSet left = Bitmap.newBitmap(); private BitSet right = Bitmap.newBitmap(); + private BitSet referenceNodes = Bitmap.newBitmap(); /** * Create simple edge. @@ -56,21 +57,25 @@ public class Edge { public void addLeftNode(BitSet left) { Bitmap.or(this.left, left); + Bitmap.or(referenceNodes, left); } public void addLeftNodes(BitSet... bitSets) { for (BitSet bitSet : bitSets) { Bitmap.or(this.left, bitSet); + Bitmap.or(referenceNodes, bitSet); } } public void addRightNode(BitSet right) { Bitmap.or(this.right, right); + Bitmap.or(referenceNodes, right); } public void addRightNodes(BitSet... bitSets) { for (BitSet bitSet : bitSets) { Bitmap.or(this.right, bitSet); + Bitmap.or(referenceNodes, bitSet); } } @@ -79,6 +84,7 @@ public class Edge { } public void setLeft(BitSet left) { + referenceNodes.clear(); this.left = left; } @@ -87,18 +93,21 @@ public class Edge { } public void setRight(BitSet right) { + referenceNodes.clear(); this.right = right; } public boolean isSub(Edge edge) { // When this join reference nodes is a subset of other join, then this join must appear before that join - BitSet bitSet = getReferenceNodes(); BitSet otherBitset = edge.getReferenceNodes(); - return Bitmap.isSubset(bitSet, otherBitset); + return Bitmap.isSubset(getReferenceNodes(), otherBitset); } public BitSet getReferenceNodes() { - return Bitmap.newBitmapUnion(this.left, this.right); + if (referenceNodes.cardinality() == 0) { + referenceNodes = Bitmap.newBitmapUnion(left, right); + } + return referenceNodes; } public Edge reverse(int index) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java index 88c35d479c..83dc41c747 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java @@ -74,7 +74,7 @@ public class GraphSimplifier { simplifications.add(bestSimplification); } for (Node node : graph.getNodes()) { - cachePlan.put(node.getBitSet(), node.getPlan()); + cachePlan.put(node.getNodeMap(), node.getPlan()); } circleDetector = new CircleDetector(edgeSize); } @@ -208,12 +208,12 @@ public class GraphSimplifier { SimplificationStep bestStep = bestSimplification.getStep(); while (bestSimplification.bestNeighbor == -1 || !circleDetector.tryAddDirectedEdge(bestStep.beforeIndex, bestStep.afterIndex)) { + processNeighbors(bestStep.afterIndex, 0, edgeSize); if (priorityQueue.isEmpty()) { return null; } bestSimplification = priorityQueue.poll(); bestSimplification.isInQueue = false; - processNeighbors(bestStep.afterIndex, 0, edgeSize); bestStep = bestSimplification.getStep(); } return bestStep; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java index 3155bfec9a..004824cf47 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java @@ -61,12 +61,6 @@ public class HyperGraph { return nodes.get(index); } - public void splitEdgesForNodes() { - for (Node node : nodes) { - node.splitEdges(); - } - } - public void addNode(Group group) { Preconditions.checkArgument(!group.isJoinGroup()); // TODO: replace plan with group expression or others diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Node.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Node.java index 8eb4bb843e..ed7fd964c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Node.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Node.java @@ -79,7 +79,7 @@ public class Node { return index; } - public BitSet getBitSet() { + public BitSet getNodeMap() { return Bitmap.newBitmap(index); } @@ -87,34 +87,6 @@ public class Node { return group.getLogicalExpression().getPlan(); } - // public void setPlan(Plan plan) { - // this.plan = plan; - // } - - public List getComplexEdges() { - return complexEdges; - } - - public void setComplexEdges(List complexEdges) { - this.complexEdges = complexEdges; - } - - public List getSimpleEdges() { - return simpleEdges; - } - - public void setSimpleEdges(List simpleEdges) { - this.simpleEdges = simpleEdges; - } - - public BitSet getSimpleNeighborhood() { - return simpleNeighborhood; - } - - public void setSimpleNeighborhood(BitSet simpleNeighborhood) { - this.simpleNeighborhood = simpleNeighborhood; - } - /** * Attach all edge in this node if the edge references this node * @@ -133,31 +105,6 @@ public class Node { edges.remove(edge); } - /** - * This function split edge into complex edges and simple edges - * We do it after constructing HyperGraph because the edge may be modified - * by graph simplifier. - */ - public void splitEdges() { - simpleEdges.clear(); - Bitmap.clear(simpleNeighborhood); - complexEdges.clear(); - Bitmap.clear(complexNeighborhood); - for (Edge edge : edges) { - if (edge.isSimple()) { - simpleEdges.add(edge); - Bitmap.or(simpleNeighborhood, edge.getLeft()); - Bitmap.or(simpleNeighborhood, edge.getRight()); - } else { - complexEdges.add(edge); - Bitmap.or(complexNeighborhood, edge.getLeft()); - Bitmap.or(complexNeighborhood, edge.getRight()); - } - } - Bitmap.unset(simpleNeighborhood, index); - Bitmap.unset(complexNeighborhood, index); - } - public String getName() { return getPlan().getType().name() + index; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/SubgraphEnumerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/SubgraphEnumerator.java index 03be7bf34c..f196d262c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/SubgraphEnumerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/SubgraphEnumerator.java @@ -21,7 +21,11 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.Bitmap; import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.SubsetIterator; import org.apache.doris.nereids.jobs.joinorder.hypergraph.receiver.AbstractReceiver; +import com.google.common.base.Preconditions; + +import java.util.ArrayList; import java.util.BitSet; +import java.util.HashMap; import java.util.List; /** @@ -34,6 +38,9 @@ public class SubgraphEnumerator { AbstractReceiver receiver; //The enumerated hyperGraph HyperGraph hyperGraph; + EdgeCalculator edgeCalculator; + NeighborhoodCalculator neighborhoodCalculator; + // These caches are used to avoid repetitive computation public SubgraphEnumerator(AbstractReceiver receiver, HyperGraph hyperGraph) { this.receiver = receiver; @@ -50,12 +57,20 @@ public class SubgraphEnumerator { List nodes = hyperGraph.getNodes(); // Init all nodes in Receiver for (Node node : nodes) { - receiver.addGroup(node.getBitSet(), node.getGroup()); + receiver.addGroup(node.getNodeMap(), node.getGroup()); } - hyperGraph.splitEdgesForNodes(); int size = nodes.size(); - // We skip the last element due to it can't generate valid csg-cmp pair + // Init edgeCalculator + edgeCalculator = new EdgeCalculator(hyperGraph.getEdges()); + for (Node node : nodes) { + edgeCalculator.initSubgraph(node.getNodeMap()); + } + + // Init neighborhoodCalculator + neighborhoodCalculator = new NeighborhoodCalculator(); + + // We skip the last element because it can't generate valid csg-cmp pair BitSet forbiddenNodes = Bitmap.newBitmapBetween(0, size - 1); for (int i = size - 2; i >= 0; i--) { BitSet csg = Bitmap.newBitmap(i); @@ -70,11 +85,12 @@ public class SubgraphEnumerator { // The general purpose of EnumerateCsgRec is to extend a given set csg, which // induces a connected subgraph of G to a larger set with the same property. private boolean enumerateCsgRec(BitSet csg, BitSet forbiddenNodes) { - BitSet neighborhood = calcNeighborhood(csg, forbiddenNodes); + BitSet neighborhood = neighborhoodCalculator.calcNeighborhood(csg, forbiddenNodes, edgeCalculator); SubsetIterator subsetIterator = Bitmap.getSubsetIterator(neighborhood); for (BitSet subset : subsetIterator) { BitSet newCsg = Bitmap.newBitmapUnion(csg, subset); if (receiver.contain(newCsg)) { + edgeCalculator.unionEdges(csg, subset); if (!emitCsg(newCsg)) { return false; } @@ -92,22 +108,20 @@ public class SubgraphEnumerator { } private boolean enumerateCmpRec(BitSet csg, BitSet cmp, BitSet forbiddenNodes) { - BitSet neighborhood = calcNeighborhood(cmp, forbiddenNodes); + BitSet neighborhood = neighborhoodCalculator.calcNeighborhood(cmp, forbiddenNodes, edgeCalculator); SubsetIterator subsetIterator = new SubsetIterator(neighborhood); for (BitSet subset : subsetIterator) { BitSet newCmp = Bitmap.newBitmapUnion(cmp, subset); // We need to check whether Cmp is connected and then try to find hyper edge if (receiver.contain(newCmp)) { - // We check all edges for finding an edge. That is inefficient. - // MySQL use full neighborhood for it. Or a hashMap may be useful - for (Edge edge : hyperGraph.getEdges()) { - if (Bitmap.isSubset(edge.getLeft(), csg) && Bitmap.isSubset(edge.getRight(), newCmp) || ( - Bitmap.isSubset(edge.getLeft(), newCmp) && Bitmap.isSubset(edge.getRight(), csg))) { - if (!receiver.emitCsgCmp(csg, newCmp, edge)) { - return false; - } - break; - } + edgeCalculator.unionEdges(cmp, subset); + // We check all edges for finding an edge. + List edges = edgeCalculator.connectCsgCmp(csg, newCmp); + if (edges.isEmpty()) { + continue; + } + if (!receiver.emitCsgCmp(csg, newCmp, edges)) { + return false; } } } @@ -128,17 +142,18 @@ public class SubgraphEnumerator { private boolean emitCsg(BitSet csg) { BitSet forbiddenNodes = Bitmap.newBitmapBetween(0, Bitmap.nextSetBit(csg, 0)); Bitmap.or(forbiddenNodes, csg); - BitSet neighborhoods = calcNeighborhood(csg, Bitmap.newBitmap(forbiddenNodes)); + BitSet neighborhoods = neighborhoodCalculator.calcNeighborhood(csg, Bitmap.newBitmap(forbiddenNodes), + edgeCalculator); for (int nodeIndex : Bitmap.getReverseIterator(neighborhoods)) { BitSet cmp = Bitmap.newBitmap(nodeIndex); // whether there is an edge between csg and cmp - Node cmpNode = hyperGraph.getNode(nodeIndex); - Edge edge = cmpNode.tryGetEdgeWith(csg); - if (edge != null) { - if (!receiver.emitCsgCmp(csg, cmp, edge)) { - return false; - } + List edges = edgeCalculator.connectCsgCmp(csg, cmp); + if (edges.isEmpty()) { + continue; + } + if (!receiver.emitCsgCmp(csg, cmp, edges)) { + return false; } // In order to avoid enumerate repeated cmp, e.g., @@ -160,33 +175,185 @@ public class SubgraphEnumerator { return true; } - // This function is used to calculate neighborhoods of given subgraph. - // Though a direct way is to add all nodes u that satisfies: - // \in E && v \in subgraph && v \intersect X = empty - // We don't used it because they can cause some repeated subgraph when - // expand csg and cmp. In fact, we just need a seed node that can be expanded - // to all subgraph. That is any one node of hyper nodes. In fact, the neighborhoods - // is the minimum set that we choose one node from above v. + class NeighborhoodCalculator { + // This function is used to calculate neighborhoods of given subgraph. + // Though a direct way is to add all nodes u that satisfies: + // \in E && v \in subgraph && v \intersect X = empty + // We don't used it because they can cause some repeated subgraph when + // expand csg and cmp. In fact, we just need a seed node that can be expanded + // to all subgraph. That is any one node of hyper nodes. In fact, the neighborhoods + // is the minimum set that we choose one node from above v. + public BitSet calcNeighborhood(BitSet subgraph, BitSet forbiddenNodes, EdgeCalculator edgeCalculator) { + BitSet neighborhoods = Bitmap.newBitmap(); + edgeCalculator.foundSimpleEdgesContain(subgraph) + .forEach(edge -> neighborhoods.or(edge.getReferenceNodes())); + Bitmap.or(forbiddenNodes, subgraph); + Bitmap.andNot(neighborhoods, forbiddenNodes); + Bitmap.or(forbiddenNodes, neighborhoods); - // Note there are many tricks implemented in MySQL, such as neighbor cache, complex edges - // We hope implement them after a benchmark. - private BitSet calcNeighborhood(BitSet subGraph, BitSet forbiddenNodes) { - BitSet neighborhoods = Bitmap.newBitmap(); - Bitmap.getIterator(subGraph) - .forEach(nodeIndex -> Bitmap.or(neighborhoods, hyperGraph.getNode(nodeIndex).getSimpleNeighborhood())); - Bitmap.andNot(neighborhoods, forbiddenNodes); - Bitmap.or(forbiddenNodes, subGraph); - Bitmap.or(forbiddenNodes, neighborhoods); - - for (Edge edge : hyperGraph.getEdges()) { - BitSet left = edge.getLeft(); - BitSet right = edge.getRight(); - if (Bitmap.isSubset(left, subGraph) && !Bitmap.isOverlap(right, forbiddenNodes)) { - Bitmap.set(neighborhoods, right.nextSetBit(0)); - } else if (Bitmap.isSubset(right, subGraph) && !Bitmap.isOverlap(left, forbiddenNodes)) { - Bitmap.set(neighborhoods, left.nextSetBit(0)); + for (Edge edge : edgeCalculator.foundComplexEdgesContain(subgraph)) { + BitSet left = edge.getLeft(); + BitSet right = edge.getRight(); + if (Bitmap.isSubset(left, subgraph) && !Bitmap.isOverlap(right, forbiddenNodes)) { + Bitmap.set(neighborhoods, right.nextSetBit(0)); + } else if (Bitmap.isSubset(right, subgraph) && !Bitmap.isOverlap(left, forbiddenNodes)) { + Bitmap.set(neighborhoods, left.nextSetBit(0)); + } } + return neighborhoods; + } + } + + class EdgeCalculator { + final List edges; + // It cached all edges that contained by this subgraph, Note we always + // use bitset store edge map because the number of edges can be very large + // We split these into simple edges (only one node on each side) and complex edges (others) + // because we can often quickly discard all simple edges by testing the set of interesting nodes + // against the “simple_neighborhood” bitmap. These data will be calculated before enumerate. + + HashMap containSimpleEdges = new HashMap<>(); + HashMap containComplexEdges = new HashMap<>(); + // It cached all edges that overlap by this subgraph. All this edges must be + // complex edges + HashMap overlapEdges = new HashMap<>(); + + EdgeCalculator(List edges) { + this.edges = edges; + } + + public void initSubgraph(BitSet subgraph) { + BitSet simpleContains = new BitSet(); + BitSet complexContains = new BitSet(); + BitSet overlaps = new BitSet(); + for (Edge edge : edges) { + if (isContainEdge(subgraph, edge)) { + if (edge.isSimple()) { + simpleContains.set(edge.getIndex()); + } else { + complexContains.set(edge.getIndex()); + } + } else if (isOverlapEdge(subgraph, edge)) { + overlaps.set(edge.getIndex()); + } + } + if (containSimpleEdges.containsKey(subgraph)) { + complexContains.or(containComplexEdges.get(subgraph)); + simpleContains.or(containSimpleEdges.get(subgraph)); + } + if (overlapEdges.containsKey(subgraph)) { + overlaps.or(overlapEdges.get(subgraph)); + } + overlapEdges.put(subgraph, overlaps); + containSimpleEdges.put(subgraph, simpleContains); + containComplexEdges.put(subgraph, complexContains); + } + + public void unionEdges(BitSet bitSet1, BitSet bitSet2) { + // When union two sub graphs, we only need to check overlap edges. + // However, if all reference nodes are contained by the subgraph, + // we should remove it. + if (!containSimpleEdges.containsKey(bitSet1)) { + initSubgraph(bitSet1); + } + if (!containSimpleEdges.containsKey(bitSet2)) { + initSubgraph(bitSet2); + } + BitSet subgraph = Bitmap.newBitmapUnion(bitSet1, bitSet2); + if (containSimpleEdges.containsKey(subgraph)) { + return; + } + BitSet simpleContains = Bitmap.newBitmapUnion(containSimpleEdges.get(bitSet1), + containSimpleEdges.get(bitSet2)); + BitSet complexContains = Bitmap.newBitmapUnion(containComplexEdges.get(bitSet1), + containComplexEdges.get(bitSet2)); + BitSet overlaps = Bitmap.newBitmapUnion(overlapEdges.get(bitSet1), + overlapEdges.get(bitSet2)); + for (int index : overlaps.stream().toArray()) { + Edge edge = edges.get(index); + if (isContainEdge(subgraph, edge)) { + overlaps.set(index, false); + if (edge.isSimple()) { + simpleContains.set(index); + } else { + complexContains.set(index); + } + } + } + simpleContains = removeInvalidEdges(subgraph, simpleContains); + complexContains = removeInvalidEdges(subgraph, complexContains); + containSimpleEdges.put(subgraph, simpleContains); + containComplexEdges.put(subgraph, complexContains); + overlapEdges.put(subgraph, overlaps); + } + + public List connectCsgCmp(BitSet bitSet1, BitSet bitSet2) { + Preconditions.checkArgument( + containSimpleEdges.containsKey(bitSet1) && containSimpleEdges.containsKey(bitSet2)); + List foundEdges = new ArrayList<>(); + BitSet edgeMap = Bitmap.newBitmapIntersect(containSimpleEdges.get(bitSet1), + containSimpleEdges.get(bitSet2)); + Bitmap.or(edgeMap, Bitmap.newBitmapIntersect(containComplexEdges.get(bitSet1), + containComplexEdges.get(bitSet2))); + edgeMap.stream().forEach(index -> foundEdges.add(edges.get(index))); + return foundEdges; + } + + public List foundEdgesContain(BitSet subgraph) { + Preconditions.checkArgument(containSimpleEdges.containsKey(subgraph)); + BitSet edgeMap = containSimpleEdges.get(subgraph); + edgeMap.or(containComplexEdges.get(subgraph)); + List foundEdges = new ArrayList<>(); + edgeMap.stream().forEach(index -> foundEdges.add(edges.get(index))); + return foundEdges; + } + + public List foundSimpleEdgesContain(BitSet subgraph) { + List foundEdges = new ArrayList<>(); + if (!containSimpleEdges.containsKey(subgraph)) { + return foundEdges; + } + BitSet edgeMap = containSimpleEdges.get(subgraph); + edgeMap.stream().forEach(index -> foundEdges.add(edges.get(index))); + return foundEdges; + } + + public List foundComplexEdgesContain(BitSet subgraph) { + List foundEdges = new ArrayList<>(); + if (!containComplexEdges.containsKey(subgraph)) { + return foundEdges; + } + BitSet edgeMap = containComplexEdges.get(subgraph); + edgeMap.stream().forEach(index -> foundEdges.add(edges.get(index))); + return foundEdges; + } + + public int getEdgeSizeContain(BitSet subgraph) { + Preconditions.checkArgument(containSimpleEdges.containsKey(subgraph)); + return containSimpleEdges.get(subgraph).cardinality() + containSimpleEdges.get(subgraph).cardinality(); + } + + private boolean isContainEdge(BitSet subgraph, Edge edge) { + int containLeft = Bitmap.isSubset(edge.getLeft(), subgraph) ? 0 : 1; + int containRight = Bitmap.isSubset(edge.getRight(), subgraph) ? 0 : 1; + return containLeft + containRight == 1; + } + + private boolean isOverlapEdge(BitSet subgraph, Edge edge) { + int overlapLeft = Bitmap.isOverlap(edge.getLeft(), subgraph) ? 0 : 1; + int overlapRight = Bitmap.isOverlap(edge.getRight(), subgraph) ? 0 : 1; + return overlapLeft + overlapRight == 1; + } + + private BitSet removeInvalidEdges(BitSet subgraph, BitSet edgeMap) { + for (int index : edgeMap.stream().toArray()) { + Edge edge = edges.get(index); + if (!isOverlapEdge(subgraph, edge)) { + edgeMap.set(index, false); + } + } + return edgeMap; } - return neighborhoods; } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/bitmap/Bitmap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/bitmap/Bitmap.java index 619a9edf62..1942e29368 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/bitmap/Bitmap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/bitmap/Bitmap.java @@ -26,8 +26,8 @@ public class Bitmap { public static boolean isSubset(BitSet bitSet1, BitSet bitSet2) { BitSet bitSet = new BitSet(); bitSet.or(bitSet1); - bitSet.or(bitSet2); - return bitSet.equals(bitSet2); + bitSet.andNot(bitSet2); + return bitSet.cardinality() == 0; } public static BitSet newBitmap(int... values) { @@ -64,6 +64,14 @@ public class Bitmap { return u; } + //return bitset1 ∩ bitset2 + public static BitSet newBitmapIntersect(BitSet bitSet1, BitSet bitSet2) { + BitSet intersect = newBitmap(); + intersect.or(bitSet1); + intersect.and(bitSet2); + return intersect; + } + public static BitSet newBitmapBetween(int start, int end) { BitSet bitSet = new BitSet(); bitSet.set(start, end); @@ -121,5 +129,6 @@ public class Bitmap { public static SubsetIterator getSubsetIterator(BitSet bitSet) { return new SubsetIterator(bitSet); } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/AbstractReceiver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/AbstractReceiver.java index 13c85595d1..cfe6dbeb23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/AbstractReceiver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/AbstractReceiver.java @@ -21,12 +21,13 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.Edge; import org.apache.doris.nereids.memo.Group; import java.util.BitSet; +import java.util.List; /** * A interface of receiver */ public interface AbstractReceiver { - public boolean emitCsgCmp(BitSet csg, BitSet cmp, Edge edge); + public boolean emitCsgCmp(BitSet csg, BitSet cmp, List edges); public void addGroup(BitSet bitSet, Group group); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/Counter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/Counter.java index b3bba9a2e8..4a3969c12a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/Counter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/Counter.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import java.util.BitSet; import java.util.HashMap; +import java.util.List; /** * The Receiver is used for cached the plan that has been emitted and build the new plan @@ -47,10 +48,10 @@ public class Counter implements AbstractReceiver { * * @param left the bitmap of left child tree * @param right the bitmap of the right child tree - * @param edge the join operator + * @param edges the join operator * @return the left and the right can be connected by the edge */ - public boolean emitCsgCmp(BitSet left, BitSet right, Edge edge) { + public boolean emitCsgCmp(BitSet left, BitSet right, List edges) { Preconditions.checkArgument(counter.containsKey(left)); Preconditions.checkArgument(counter.containsKey(right)); emitCount += 1; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java index be708902a9..0658edcc1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java @@ -23,14 +23,17 @@ import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.stats.StatsCalculator; +import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import java.util.ArrayList; import java.util.BitSet; import java.util.HashMap; +import java.util.List; /** * The Receiver is used for cached the plan that has been emitted and build the new plan @@ -54,11 +57,11 @@ public class PlanReceiver implements AbstractReceiver { * * @param left the bitmap of left child tree * @param right the bitmap of the right child tree - * @param edge the join operator + * @param edges the join conditions that can be added in this operator * @return the left and the right can be connected by the edge */ @Override - public boolean emitCsgCmp(BitSet left, BitSet right, Edge edge) { + public boolean emitCsgCmp(BitSet left, BitSet right, List edges) { Preconditions.checkArgument(planTable.containsKey(left)); Preconditions.checkArgument(planTable.containsKey(right)); emitCount += 1; @@ -66,8 +69,8 @@ public class PlanReceiver implements AbstractReceiver { return false; } BitSet fullKey = Bitmap.newBitmapUnion(left, right); - Group group1 = constructGroup(left, right, edge); - Group group2 = constructGroup(right, left, edge); + Group group1 = constructGroup(left, right, edges); + Group group2 = constructGroup(right, left, edges); Group winnerGroup; if (group1.getLogicalExpression().getCostByProperties(PhysicalProperties.ANY) < group2.getLogicalExpression() .getCostByProperties(PhysicalProperties.ANY)) { @@ -113,7 +116,7 @@ public class PlanReceiver implements AbstractReceiver { return plan.getGroupExpression().get().getCostByProperties(PhysicalProperties.ANY); } - private Group constructGroup(BitSet left, BitSet right, Edge edge) { + private Group constructGroup(BitSet left, BitSet right, List edges) { Preconditions.checkArgument(planTable.containsKey(left)); Preconditions.checkArgument(planTable.containsKey(right)); Group leftGroup = planTable.get(left); @@ -122,7 +125,11 @@ public class PlanReceiver implements AbstractReceiver { Plan rightPlan = rightGroup.getLogicalExpression().getPlan(); double cost = getSimpleCost(leftPlan) + getSimpleCost(rightPlan); - LogicalJoin newJoin = new LogicalJoin(edge.getJoin().getJoinType(), edge.getJoin().getExpressions(), + List conditions = new ArrayList<>(); + for (Edge edge : edges) { + conditions.addAll(edge.getJoin().getExpressions()); + } + LogicalJoin newJoin = new LogicalJoin(edges.get(0).getJoin().getJoinType(), conditions, leftGroup.getLogicalExpression().getPlan(), rightGroup.getLogicalExpression().getPlan()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifierTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifierTest.java index 606780ba2c..817d0d36ba 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifierTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifierTest.java @@ -164,10 +164,36 @@ public class GraphSimplifierTest { } @Disabled + @Test + void testComplexQuery() { + HyperGraph hyperGraph = new HyperGraphBuilder() + .init(6, 2, 1, 3, 5, 4) + .addEdge(JoinType.INNER_JOIN, 3, 4) + .addEdge(JoinType.INNER_JOIN, 3, 5) + .addEdge(JoinType.INNER_JOIN, 2, 3) + .addEdge(JoinType.INNER_JOIN, 2, 5) + .addEdge(JoinType.INNER_JOIN, 2, 4) + .addEdge(JoinType.INNER_JOIN, 1, 5) + .addEdge(JoinType.INNER_JOIN, 1, 4) + .addEdge(JoinType.INNER_JOIN, 0, 2) + .build(); + GraphSimplifier graphSimplifier = new GraphSimplifier(hyperGraph); + graphSimplifier.initFirstStep(); + while (graphSimplifier.applySimplificationStep()) { + } + Counter counter = new Counter(); + SubgraphEnumerator subgraphEnumerator = new SubgraphEnumerator(counter, hyperGraph); + subgraphEnumerator.enumerate(); + for (int count : counter.getAllCount().values()) { + Assertions.assertTrue(count < 1000); + } + Assertions.assertTrue(graphSimplifier.isTotalOrder()); + } + @Test void testRandomQuery() { - for (int i = 0; i < 100; i++) { - HyperGraph hyperGraph = new HyperGraphBuilder().randomBuildWith(20, 40); + for (int i = 0; i < 10; i++) { + HyperGraph hyperGraph = new HyperGraphBuilder().randomBuildWith(6, 6); GraphSimplifier graphSimplifier = new GraphSimplifier(hyperGraph); graphSimplifier.initFirstStep(); while (graphSimplifier.applySimplificationStep()) { @@ -175,9 +201,6 @@ public class GraphSimplifierTest { Counter counter = new Counter(); SubgraphEnumerator subgraphEnumerator = new SubgraphEnumerator(counter, hyperGraph); subgraphEnumerator.enumerate(); - for (int count : counter.getAllCount().values()) { - Assertions.assertTrue(count < 1000); - } Assertions.assertTrue(graphSimplifier.isTotalOrder()); } } @@ -195,6 +218,5 @@ public class GraphSimplifierTest { subgraphEnumerator.enumerate(); Assertions.assertTrue(counter.getLimit() >= 0); } - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/SubgraphEnumeratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/SubgraphEnumeratorTest.java index 1b4e1fb306..4dabec273b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/SubgraphEnumeratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/SubgraphEnumeratorTest.java @@ -35,9 +35,9 @@ public class SubgraphEnumeratorTest { void testStarQuery() { // t2 // | - //t3-- t1 -- t4 + //t3-- t0 -- t4 // | - // t5 + // t1 HyperGraph hyperGraph = new HyperGraphBuilder() .init(10, 20, 30, 40, 50) .addEdge(JoinType.INNER_JOIN, 0, 1) @@ -96,8 +96,8 @@ public class SubgraphEnumeratorTest { @Test void testTime() { - int tableNum = 10; - int edgeNum = 40; + int tableNum = 20; + int edgeNum = 21; HyperGraph hyperGraph = new HyperGraphBuilder().randomBuildWith(tableNum, edgeNum); Counter counter = new Counter();