database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](Nereids): Add cache to avoid repeatly calculation in DPhyp (#14585)

Browse Source
This commit is contained in:
谢健
2022-11-30 21:35:45 +08:00
committed by GitHub
parent 9bbbcf031c
commit 36737fe9f4
11 changed files with 290 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Edge.java
View File

@ -36,6 +36,7 @@ public class Edge {
// left and right may not overlap, and both must have at least one bit set.
private BitSet left = Bitmap.newBitmap();
private BitSet right = Bitmap.newBitmap();
private BitSet referenceNodes = Bitmap.newBitmap();
/**
* Create simple edge.
@ -56,21 +57,25 @@ public class Edge {
public void addLeftNode(BitSet left) {
Bitmap.or(this.left, left);
Bitmap.or(referenceNodes, left);
}
public void addLeftNodes(BitSet... bitSets) {
for (BitSet bitSet : bitSets) {
Bitmap.or(this.left, bitSet);
Bitmap.or(referenceNodes, bitSet);
}
}
public void addRightNode(BitSet right) {
Bitmap.or(this.right, right);
Bitmap.or(referenceNodes, right);
}
public void addRightNodes(BitSet... bitSets) {
for (BitSet bitSet : bitSets) {
Bitmap.or(this.right, bitSet);
Bitmap.or(referenceNodes, bitSet);
}
}
@ -79,6 +84,7 @@ public class Edge {
}
public void setLeft(BitSet left) {
referenceNodes.clear();
this.left = left;
}
@ -87,18 +93,21 @@ public class Edge {
}
public void setRight(BitSet right) {
referenceNodes.clear();
this.right = right;
}
public boolean isSub(Edge edge) {
// When this join reference nodes is a subset of other join, then this join must appear before that join
BitSet bitSet = getReferenceNodes();
BitSet otherBitset = edge.getReferenceNodes();
return Bitmap.isSubset(bitSet, otherBitset);
return Bitmap.isSubset(getReferenceNodes(), otherBitset);
}
public BitSet getReferenceNodes() {
return Bitmap.newBitmapUnion(this.left, this.right);
if (referenceNodes.cardinality() == 0) {
referenceNodes = Bitmap.newBitmapUnion(left, right);
}
return referenceNodes;
}
public Edge reverse(int index) {

4
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java
View File

@ -74,7 +74,7 @@ public class GraphSimplifier {
simplifications.add(bestSimplification);
}
for (Node node : graph.getNodes()) {
cachePlan.put(node.getBitSet(), node.getPlan());
cachePlan.put(node.getNodeMap(), node.getPlan());
}
circleDetector = new CircleDetector(edgeSize);
}
@ -208,12 +208,12 @@ public class GraphSimplifier {
SimplificationStep bestStep = bestSimplification.getStep();
while (bestSimplification.bestNeighbor == -1 || !circleDetector.tryAddDirectedEdge(bestStep.beforeIndex,
bestStep.afterIndex)) {
processNeighbors(bestStep.afterIndex, 0, edgeSize);
if (priorityQueue.isEmpty()) {
return null;
}
bestSimplification = priorityQueue.poll();
bestSimplification.isInQueue = false;
processNeighbors(bestStep.afterIndex, 0, edgeSize);
bestStep = bestSimplification.getStep();
}
return bestStep;

6
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
View File

@ -61,12 +61,6 @@ public class HyperGraph {
return nodes.get(index);
}
public void splitEdgesForNodes() {
for (Node node : nodes) {
node.splitEdges();
}
}
public void addNode(Group group) {
Preconditions.checkArgument(!group.isJoinGroup());
// TODO: replace plan with group expression or others

55
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Node.java
View File

@ -79,7 +79,7 @@ public class Node {
return index;
}
public BitSet getBitSet() {
public BitSet getNodeMap() {
return Bitmap.newBitmap(index);
}
@ -87,34 +87,6 @@ public class Node {
return group.getLogicalExpression().getPlan();
}
// public void setPlan(Plan plan) {
// this.plan = plan;
// }
public List<Edge> getComplexEdges() {
return complexEdges;
}
public void setComplexEdges(List<Edge> complexEdges) {
this.complexEdges = complexEdges;
}
public List<Edge> getSimpleEdges() {
return simpleEdges;
}
public void setSimpleEdges(List<Edge> simpleEdges) {
this.simpleEdges = simpleEdges;
}
public BitSet getSimpleNeighborhood() {
return simpleNeighborhood;
}
public void setSimpleNeighborhood(BitSet simpleNeighborhood) {
this.simpleNeighborhood = simpleNeighborhood;
}
/**
* Attach all edge in this node if the edge references this node
*
@ -133,31 +105,6 @@ public class Node {
edges.remove(edge);
}
/**
* This function split edge into complex edges and simple edges
* We do it after constructing HyperGraph because the edge may be modified
* by graph simplifier.
*/
public void splitEdges() {
simpleEdges.clear();
Bitmap.clear(simpleNeighborhood);
complexEdges.clear();
Bitmap.clear(complexNeighborhood);
for (Edge edge : edges) {
if (edge.isSimple()) {
simpleEdges.add(edge);
Bitmap.or(simpleNeighborhood, edge.getLeft());
Bitmap.or(simpleNeighborhood, edge.getRight());
} else {
complexEdges.add(edge);
Bitmap.or(complexNeighborhood, edge.getLeft());
Bitmap.or(complexNeighborhood, edge.getRight());
}
}
Bitmap.unset(simpleNeighborhood, index);
Bitmap.unset(complexNeighborhood, index);
}
public String getName() {
return getPlan().getType().name() + index;
}

261
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/SubgraphEnumerator.java
View File

@ -21,7 +21,11 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.Bitmap;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.SubsetIterator;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.receiver.AbstractReceiver;
import com.google.common.base.Preconditions;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.HashMap;
import java.util.List;
/**
@ -34,6 +38,9 @@ public class SubgraphEnumerator {
AbstractReceiver receiver;
//The enumerated hyperGraph
HyperGraph hyperGraph;
EdgeCalculator edgeCalculator;
NeighborhoodCalculator neighborhoodCalculator;
// These caches are used to avoid repetitive computation
public SubgraphEnumerator(AbstractReceiver receiver, HyperGraph hyperGraph) {
this.receiver = receiver;
@ -50,12 +57,20 @@ public class SubgraphEnumerator {
List<Node> nodes = hyperGraph.getNodes();
// Init all nodes in Receiver
for (Node node : nodes) {
receiver.addGroup(node.getBitSet(), node.getGroup());
receiver.addGroup(node.getNodeMap(), node.getGroup());
}
hyperGraph.splitEdgesForNodes();
int size = nodes.size();
// We skip the last element due to it can't generate valid csg-cmp pair
// Init edgeCalculator
edgeCalculator = new EdgeCalculator(hyperGraph.getEdges());
for (Node node : nodes) {
edgeCalculator.initSubgraph(node.getNodeMap());
}
// Init neighborhoodCalculator
neighborhoodCalculator = new NeighborhoodCalculator();
// We skip the last element because it can't generate valid csg-cmp pair
BitSet forbiddenNodes = Bitmap.newBitmapBetween(0, size - 1);
for (int i = size - 2; i >= 0; i--) {
BitSet csg = Bitmap.newBitmap(i);
@ -70,11 +85,12 @@ public class SubgraphEnumerator {
// The general purpose of EnumerateCsgRec is to extend a given set csg, which
// induces a connected subgraph of G to a larger set with the same property.
private boolean enumerateCsgRec(BitSet csg, BitSet forbiddenNodes) {
BitSet neighborhood = calcNeighborhood(csg, forbiddenNodes);
BitSet neighborhood = neighborhoodCalculator.calcNeighborhood(csg, forbiddenNodes, edgeCalculator);
SubsetIterator subsetIterator = Bitmap.getSubsetIterator(neighborhood);
for (BitSet subset : subsetIterator) {
BitSet newCsg = Bitmap.newBitmapUnion(csg, subset);
if (receiver.contain(newCsg)) {
edgeCalculator.unionEdges(csg, subset);
if (!emitCsg(newCsg)) {
return false;
}
@ -92,22 +108,20 @@ public class SubgraphEnumerator {
}
private boolean enumerateCmpRec(BitSet csg, BitSet cmp, BitSet forbiddenNodes) {
BitSet neighborhood = calcNeighborhood(cmp, forbiddenNodes);
BitSet neighborhood = neighborhoodCalculator.calcNeighborhood(cmp, forbiddenNodes, edgeCalculator);
SubsetIterator subsetIterator = new SubsetIterator(neighborhood);
for (BitSet subset : subsetIterator) {
BitSet newCmp = Bitmap.newBitmapUnion(cmp, subset);
// We need to check whether Cmp is connected and then try to find hyper edge
if (receiver.contain(newCmp)) {
// We check all edges for finding an edge. That is inefficient.
// MySQL use full neighborhood for it. Or a hashMap may be useful
for (Edge edge : hyperGraph.getEdges()) {
if (Bitmap.isSubset(edge.getLeft(), csg) && Bitmap.isSubset(edge.getRight(), newCmp) || (
Bitmap.isSubset(edge.getLeft(), newCmp) && Bitmap.isSubset(edge.getRight(), csg))) {
if (!receiver.emitCsgCmp(csg, newCmp, edge)) {
return false;
}
break;
}
edgeCalculator.unionEdges(cmp, subset);
// We check all edges for finding an edge.
List<Edge> edges = edgeCalculator.connectCsgCmp(csg, newCmp);
if (edges.isEmpty()) {
continue;
}
if (!receiver.emitCsgCmp(csg, newCmp, edges)) {
return false;
}
}
}
@ -128,17 +142,18 @@ public class SubgraphEnumerator {
private boolean emitCsg(BitSet csg) {
BitSet forbiddenNodes = Bitmap.newBitmapBetween(0, Bitmap.nextSetBit(csg, 0));
Bitmap.or(forbiddenNodes, csg);
BitSet neighborhoods = calcNeighborhood(csg, Bitmap.newBitmap(forbiddenNodes));
BitSet neighborhoods = neighborhoodCalculator.calcNeighborhood(csg, Bitmap.newBitmap(forbiddenNodes),
edgeCalculator);
for (int nodeIndex : Bitmap.getReverseIterator(neighborhoods)) {
BitSet cmp = Bitmap.newBitmap(nodeIndex);
// whether there is an edge between csg and cmp
Node cmpNode = hyperGraph.getNode(nodeIndex);
Edge edge = cmpNode.tryGetEdgeWith(csg);
if (edge != null) {
if (!receiver.emitCsgCmp(csg, cmp, edge)) {
return false;
}
List<Edge> edges = edgeCalculator.connectCsgCmp(csg, cmp);
if (edges.isEmpty()) {
continue;
}
if (!receiver.emitCsgCmp(csg, cmp, edges)) {
return false;
}
// In order to avoid enumerate repeated cmp, e.g.,
@ -160,33 +175,185 @@ public class SubgraphEnumerator {
return true;
}
// This function is used to calculate neighborhoods of given subgraph.
// Though a direct way is to add all nodes u that satisfies:
// <u, v> \in E && v \in subgraph && v \intersect X = empty
// We don't used it because they can cause some repeated subgraph when
// expand csg and cmp. In fact, we just need a seed node that can be expanded
// to all subgraph. That is any one node of hyper nodes. In fact, the neighborhoods
// is the minimum set that we choose one node from above v.
class NeighborhoodCalculator {
// This function is used to calculate neighborhoods of given subgraph.
// Though a direct way is to add all nodes u that satisfies:
// <u, v> \in E && v \in subgraph && v \intersect X = empty
// We don't used it because they can cause some repeated subgraph when
// expand csg and cmp. In fact, we just need a seed node that can be expanded
// to all subgraph. That is any one node of hyper nodes. In fact, the neighborhoods
// is the minimum set that we choose one node from above v.
public BitSet calcNeighborhood(BitSet subgraph, BitSet forbiddenNodes, EdgeCalculator edgeCalculator) {
BitSet neighborhoods = Bitmap.newBitmap();
edgeCalculator.foundSimpleEdgesContain(subgraph)
.forEach(edge -> neighborhoods.or(edge.getReferenceNodes()));
Bitmap.or(forbiddenNodes, subgraph);
Bitmap.andNot(neighborhoods, forbiddenNodes);
Bitmap.or(forbiddenNodes, neighborhoods);
// Note there are many tricks implemented in MySQL, such as neighbor cache, complex edges
// We hope implement them after a benchmark.
private BitSet calcNeighborhood(BitSet subGraph, BitSet forbiddenNodes) {
BitSet neighborhoods = Bitmap.newBitmap();
Bitmap.getIterator(subGraph)
.forEach(nodeIndex -> Bitmap.or(neighborhoods, hyperGraph.getNode(nodeIndex).getSimpleNeighborhood()));
Bitmap.andNot(neighborhoods, forbiddenNodes);
Bitmap.or(forbiddenNodes, subGraph);
Bitmap.or(forbiddenNodes, neighborhoods);
for (Edge edge : hyperGraph.getEdges()) {
BitSet left = edge.getLeft();
BitSet right = edge.getRight();
if (Bitmap.isSubset(left, subGraph) && !Bitmap.isOverlap(right, forbiddenNodes)) {
Bitmap.set(neighborhoods, right.nextSetBit(0));
} else if (Bitmap.isSubset(right, subGraph) && !Bitmap.isOverlap(left, forbiddenNodes)) {
Bitmap.set(neighborhoods, left.nextSetBit(0));
for (Edge edge : edgeCalculator.foundComplexEdgesContain(subgraph)) {
BitSet left = edge.getLeft();
BitSet right = edge.getRight();
if (Bitmap.isSubset(left, subgraph) && !Bitmap.isOverlap(right, forbiddenNodes)) {
Bitmap.set(neighborhoods, right.nextSetBit(0));
} else if (Bitmap.isSubset(right, subgraph) && !Bitmap.isOverlap(left, forbiddenNodes)) {
Bitmap.set(neighborhoods, left.nextSetBit(0));
}
}
return neighborhoods;
}
}
class EdgeCalculator {
final List<Edge> edges;
// It cached all edges that contained by this subgraph, Note we always
// use bitset store edge map because the number of edges can be very large
// We split these into simple edges (only one node on each side) and complex edges (others)
// because we can often quickly discard all simple edges by testing the set of interesting nodes
// against the “simple_neighborhood” bitmap. These data will be calculated before enumerate.
HashMap<BitSet, BitSet> containSimpleEdges = new HashMap<>();
HashMap<BitSet, BitSet> containComplexEdges = new HashMap<>();
// It cached all edges that overlap by this subgraph. All this edges must be
// complex edges
HashMap<BitSet, BitSet> overlapEdges = new HashMap<>();
EdgeCalculator(List<Edge> edges) {
this.edges = edges;
}
public void initSubgraph(BitSet subgraph) {
BitSet simpleContains = new BitSet();
BitSet complexContains = new BitSet();
BitSet overlaps = new BitSet();
for (Edge edge : edges) {
if (isContainEdge(subgraph, edge)) {
if (edge.isSimple()) {
simpleContains.set(edge.getIndex());
} else {
complexContains.set(edge.getIndex());
}
} else if (isOverlapEdge(subgraph, edge)) {
overlaps.set(edge.getIndex());
}
}
if (containSimpleEdges.containsKey(subgraph)) {
complexContains.or(containComplexEdges.get(subgraph));
simpleContains.or(containSimpleEdges.get(subgraph));
}
if (overlapEdges.containsKey(subgraph)) {
overlaps.or(overlapEdges.get(subgraph));
}
overlapEdges.put(subgraph, overlaps);
containSimpleEdges.put(subgraph, simpleContains);
containComplexEdges.put(subgraph, complexContains);
}
public void unionEdges(BitSet bitSet1, BitSet bitSet2) {
// When union two sub graphs, we only need to check overlap edges.
// However, if all reference nodes are contained by the subgraph,
// we should remove it.
if (!containSimpleEdges.containsKey(bitSet1)) {
initSubgraph(bitSet1);
}
if (!containSimpleEdges.containsKey(bitSet2)) {
initSubgraph(bitSet2);
}
BitSet subgraph = Bitmap.newBitmapUnion(bitSet1, bitSet2);
if (containSimpleEdges.containsKey(subgraph)) {
return;
}
BitSet simpleContains = Bitmap.newBitmapUnion(containSimpleEdges.get(bitSet1),
containSimpleEdges.get(bitSet2));
BitSet complexContains = Bitmap.newBitmapUnion(containComplexEdges.get(bitSet1),
containComplexEdges.get(bitSet2));
BitSet overlaps = Bitmap.newBitmapUnion(overlapEdges.get(bitSet1),
overlapEdges.get(bitSet2));
for (int index : overlaps.stream().toArray()) {
Edge edge = edges.get(index);
if (isContainEdge(subgraph, edge)) {
overlaps.set(index, false);
if (edge.isSimple()) {
simpleContains.set(index);
} else {
complexContains.set(index);
}
}
}
simpleContains = removeInvalidEdges(subgraph, simpleContains);
complexContains = removeInvalidEdges(subgraph, complexContains);
containSimpleEdges.put(subgraph, simpleContains);
containComplexEdges.put(subgraph, complexContains);
overlapEdges.put(subgraph, overlaps);
}
public List<Edge> connectCsgCmp(BitSet bitSet1, BitSet bitSet2) {
Preconditions.checkArgument(
containSimpleEdges.containsKey(bitSet1) && containSimpleEdges.containsKey(bitSet2));
List<Edge> foundEdges = new ArrayList<>();
BitSet edgeMap = Bitmap.newBitmapIntersect(containSimpleEdges.get(bitSet1),
containSimpleEdges.get(bitSet2));
Bitmap.or(edgeMap, Bitmap.newBitmapIntersect(containComplexEdges.get(bitSet1),
containComplexEdges.get(bitSet2)));
edgeMap.stream().forEach(index -> foundEdges.add(edges.get(index)));
return foundEdges;
}
public List<Edge> foundEdgesContain(BitSet subgraph) {
Preconditions.checkArgument(containSimpleEdges.containsKey(subgraph));
BitSet edgeMap = containSimpleEdges.get(subgraph);
edgeMap.or(containComplexEdges.get(subgraph));
List<Edge> foundEdges = new ArrayList<>();
edgeMap.stream().forEach(index -> foundEdges.add(edges.get(index)));
return foundEdges;
}
public List<Edge> foundSimpleEdgesContain(BitSet subgraph) {
List<Edge> foundEdges = new ArrayList<>();
if (!containSimpleEdges.containsKey(subgraph)) {
return foundEdges;
}
BitSet edgeMap = containSimpleEdges.get(subgraph);
edgeMap.stream().forEach(index -> foundEdges.add(edges.get(index)));
return foundEdges;
}
public List<Edge> foundComplexEdgesContain(BitSet subgraph) {
List<Edge> foundEdges = new ArrayList<>();
if (!containComplexEdges.containsKey(subgraph)) {
return foundEdges;
}
BitSet edgeMap = containComplexEdges.get(subgraph);
edgeMap.stream().forEach(index -> foundEdges.add(edges.get(index)));
return foundEdges;
}
public int getEdgeSizeContain(BitSet subgraph) {
Preconditions.checkArgument(containSimpleEdges.containsKey(subgraph));
return containSimpleEdges.get(subgraph).cardinality() + containSimpleEdges.get(subgraph).cardinality();
}
private boolean isContainEdge(BitSet subgraph, Edge edge) {
int containLeft = Bitmap.isSubset(edge.getLeft(), subgraph) ? 0 : 1;
int containRight = Bitmap.isSubset(edge.getRight(), subgraph) ? 0 : 1;
return containLeft + containRight == 1;
}
private boolean isOverlapEdge(BitSet subgraph, Edge edge) {
int overlapLeft = Bitmap.isOverlap(edge.getLeft(), subgraph) ? 0 : 1;
int overlapRight = Bitmap.isOverlap(edge.getRight(), subgraph) ? 0 : 1;
return overlapLeft + overlapRight == 1;
}
private BitSet removeInvalidEdges(BitSet subgraph, BitSet edgeMap) {
for (int index : edgeMap.stream().toArray()) {
Edge edge = edges.get(index);
if (!isOverlapEdge(subgraph, edge)) {
edgeMap.set(index, false);
}
}
return edgeMap;
}
return neighborhoods;
}
}

13
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/bitmap/Bitmap.java
View File

@ -26,8 +26,8 @@ public class Bitmap {
public static boolean isSubset(BitSet bitSet1, BitSet bitSet2) {
BitSet bitSet = new BitSet();
bitSet.or(bitSet1);
bitSet.or(bitSet2);
return bitSet.equals(bitSet2);
bitSet.andNot(bitSet2);
return bitSet.cardinality() == 0;
}
public static BitSet newBitmap(int... values) {
@ -64,6 +64,14 @@ public class Bitmap {
return u;
}
//return bitset1 ∩ bitset2
public static BitSet newBitmapIntersect(BitSet bitSet1, BitSet bitSet2) {
BitSet intersect = newBitmap();
intersect.or(bitSet1);
intersect.and(bitSet2);
return intersect;
}
public static BitSet newBitmapBetween(int start, int end) {
BitSet bitSet = new BitSet();
bitSet.set(start, end);
@ -121,5 +129,6 @@ public class Bitmap {
public static SubsetIterator getSubsetIterator(BitSet bitSet) {
return new SubsetIterator(bitSet);
}
}

3
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/AbstractReceiver.java
View File

@ -21,12 +21,13 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.Edge;
import org.apache.doris.nereids.memo.Group;
import java.util.BitSet;
import java.util.List;
/**
* A interface of receiver
*/
public interface AbstractReceiver {
public boolean emitCsgCmp(BitSet csg, BitSet cmp, Edge edge);
public boolean emitCsgCmp(BitSet csg, BitSet cmp, List<Edge> edges);
public void addGroup(BitSet bitSet, Group group);

5
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/Counter.java
View File

@ -24,6 +24,7 @@ import com.google.common.base.Preconditions;
import java.util.BitSet;
import java.util.HashMap;
import java.util.List;
/**
* The Receiver is used for cached the plan that has been emitted and build the new plan
@ -47,10 +48,10 @@ public class Counter implements AbstractReceiver {
*
* @param left the bitmap of left child tree
* @param right the bitmap of the right child tree
* @param edge the join operator
* @param edges the join operator
* @return the left and the right can be connected by the edge
*/
public boolean emitCsgCmp(BitSet left, BitSet right, Edge edge) {
public boolean emitCsgCmp(BitSet left, BitSet right, List<Edge> edges) {
Preconditions.checkArgument(counter.containsKey(left));
Preconditions.checkArgument(counter.containsKey(right));
emitCount += 1;

19
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
View File

@ -23,14 +23,17 @@ import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.stats.StatsCalculator;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.HashMap;
import java.util.List;
/**
* The Receiver is used for cached the plan that has been emitted and build the new plan
@ -54,11 +57,11 @@ public class PlanReceiver implements AbstractReceiver {
*
* @param left the bitmap of left child tree
* @param right the bitmap of the right child tree
* @param edge the join operator
* @param edges the join conditions that can be added in this operator
* @return the left and the right can be connected by the edge
*/
@Override
public boolean emitCsgCmp(BitSet left, BitSet right, Edge edge) {
public boolean emitCsgCmp(BitSet left, BitSet right, List<Edge> edges) {
Preconditions.checkArgument(planTable.containsKey(left));
Preconditions.checkArgument(planTable.containsKey(right));
emitCount += 1;
@ -66,8 +69,8 @@ public class PlanReceiver implements AbstractReceiver {
return false;
}
BitSet fullKey = Bitmap.newBitmapUnion(left, right);
Group group1 = constructGroup(left, right, edge);
Group group2 = constructGroup(right, left, edge);
Group group1 = constructGroup(left, right, edges);
Group group2 = constructGroup(right, left, edges);
Group winnerGroup;
if (group1.getLogicalExpression().getCostByProperties(PhysicalProperties.ANY) < group2.getLogicalExpression()
.getCostByProperties(PhysicalProperties.ANY)) {
@ -113,7 +116,7 @@ public class PlanReceiver implements AbstractReceiver {
return plan.getGroupExpression().get().getCostByProperties(PhysicalProperties.ANY);
}
private Group constructGroup(BitSet left, BitSet right, Edge edge) {
private Group constructGroup(BitSet left, BitSet right, List<Edge> edges) {
Preconditions.checkArgument(planTable.containsKey(left));
Preconditions.checkArgument(planTable.containsKey(right));
Group leftGroup = planTable.get(left);
@ -122,7 +125,11 @@ public class PlanReceiver implements AbstractReceiver {
Plan rightPlan = rightGroup.getLogicalExpression().getPlan();
double cost = getSimpleCost(leftPlan) + getSimpleCost(rightPlan);
LogicalJoin newJoin = new LogicalJoin(edge.getJoin().getJoinType(), edge.getJoin().getExpressions(),
List<Expression> conditions = new ArrayList<>();
for (Edge edge : edges) {
conditions.addAll(edge.getJoin().getExpressions());
}
LogicalJoin newJoin = new LogicalJoin(edges.get(0).getJoin().getJoinType(), conditions,
leftGroup.getLogicalExpression().getPlan(),
rightGroup.getLogicalExpression().getPlan());

34
fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifierTest.java
View File

@ -164,10 +164,36 @@ public class GraphSimplifierTest {
}
@Disabled
@Test
void testComplexQuery() {
HyperGraph hyperGraph = new HyperGraphBuilder()
.init(6, 2, 1, 3, 5, 4)
.addEdge(JoinType.INNER_JOIN, 3, 4)
.addEdge(JoinType.INNER_JOIN, 3, 5)
.addEdge(JoinType.INNER_JOIN, 2, 3)
.addEdge(JoinType.INNER_JOIN, 2, 5)
.addEdge(JoinType.INNER_JOIN, 2, 4)
.addEdge(JoinType.INNER_JOIN, 1, 5)
.addEdge(JoinType.INNER_JOIN, 1, 4)
.addEdge(JoinType.INNER_JOIN, 0, 2)
.build();
GraphSimplifier graphSimplifier = new GraphSimplifier(hyperGraph);
graphSimplifier.initFirstStep();
while (graphSimplifier.applySimplificationStep()) {
}
Counter counter = new Counter();
SubgraphEnumerator subgraphEnumerator = new SubgraphEnumerator(counter, hyperGraph);
subgraphEnumerator.enumerate();
for (int count : counter.getAllCount().values()) {
Assertions.assertTrue(count < 1000);
}
Assertions.assertTrue(graphSimplifier.isTotalOrder());
}
@Test
void testRandomQuery() {
for (int i = 0; i < 100; i++) {
HyperGraph hyperGraph = new HyperGraphBuilder().randomBuildWith(20, 40);
for (int i = 0; i < 10; i++) {
HyperGraph hyperGraph = new HyperGraphBuilder().randomBuildWith(6, 6);
GraphSimplifier graphSimplifier = new GraphSimplifier(hyperGraph);
graphSimplifier.initFirstStep();
while (graphSimplifier.applySimplificationStep()) {
@ -175,9 +201,6 @@ public class GraphSimplifierTest {
Counter counter = new Counter();
SubgraphEnumerator subgraphEnumerator = new SubgraphEnumerator(counter, hyperGraph);
subgraphEnumerator.enumerate();
for (int count : counter.getAllCount().values()) {
Assertions.assertTrue(count < 1000);
}
Assertions.assertTrue(graphSimplifier.isTotalOrder());
}
}
@ -195,6 +218,5 @@ public class GraphSimplifierTest {
subgraphEnumerator.enumerate();
Assertions.assertTrue(counter.getLimit() >= 0);
}
}
}

8
fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/SubgraphEnumeratorTest.java
View File

@ -35,9 +35,9 @@ public class SubgraphEnumeratorTest {
void testStarQuery() {
// t2
// |
//t3-- t1 -- t4
//t3-- t0 -- t4
// |
// t5
// t1
HyperGraph hyperGraph = new HyperGraphBuilder()
.init(10, 20, 30, 40, 50)
.addEdge(JoinType.INNER_JOIN, 0, 1)
@ -96,8 +96,8 @@ public class SubgraphEnumeratorTest {
@Test
void testTime() {
int tableNum = 10;
int edgeNum = 40;
int tableNum = 20;
int edgeNum = 21;
HyperGraph hyperGraph = new HyperGraphBuilder().randomBuildWith(tableNum, edgeNum);
Counter counter = new Counter();