database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](Nereids): add builder for hyper graph (#30061)

Browse Source
This commit is contained in:
谢健
2024-01-23 15:31:53 +08:00
committed by yiguolei
parent 8c6e5202d4
commit f4a10c3fbc
12 changed files with 404 additions and 442 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/JoinOrderJob.java
View File

@ -73,11 +73,12 @@ public class JoinOrderJob extends Job {
}
private Group optimizeJoin(Group group) {
HyperGraph hyperGraph = HyperGraph.toDPhyperGraph(group);
for (AbstractNode node : hyperGraph.getNodes()) {
HyperGraph.Builder builder = HyperGraph.builderForDPhyper(group);
for (AbstractNode node : builder.getNodes()) {
DPhyperNode dPhyperNode = (DPhyperNode) node;
hyperGraph.updateNode(node.getIndex(), optimizePlan(dPhyperNode.getGroup()));
builder.updateNode(node.getIndex(), optimizePlan(dPhyperNode.getGroup()));
}
HyperGraph hyperGraph = builder.build();
// TODO: Right now, we just hardcode the limit with 10000, maybe we need a better way to set it
int limit = 1000;
PlanReceiver planReceiver = new PlanReceiver(this.context, limit, hyperGraph,

727
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
View File

@ -42,6 +42,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.util.PlanUtils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
@ -58,21 +60,25 @@ import java.util.stream.Collectors;
* It's used for join ordering
*/
public class HyperGraph {
private final List<JoinEdge> joinEdges = new ArrayList<>();
private final List<FilterEdge> filterEdges = new ArrayList<>();
private final List<AbstractNode> nodes = new ArrayList<>();
private final HashMap<Slot, Long> slotToNodeMap = new HashMap<>();
// record all edges that can be placed on the subgraph
private final Map<Long, BitSet> treeEdgesCache = new HashMap<>();
private final List<JoinEdge> joinEdges;
private final List<FilterEdge> filterEdges;
private final List<AbstractNode> nodes;
private final Set<Slot> finalOutputs;
// Record the complex project expression for some subgraph
// e.g. project (a + b)
// |-- join(t1.a = t2.b)
private final HashMap<Long, List<NamedExpression>> complexProject = new HashMap<>();
private final Map<Long, List<NamedExpression>> complexProject;
HyperGraph(Set<Slot> finalOutputs) {
HyperGraph(Set<Slot> finalOutputs, List<JoinEdge> joinEdges, List<AbstractNode> nodes, List<FilterEdge> filterEdges,
Map<Long, List<NamedExpression>> complexProject) {
this.finalOutputs = ImmutableSet.copyOf(finalOutputs);
this.joinEdges = ImmutableList.copyOf(joinEdges);
this.nodes = ImmutableList.copyOf(nodes);
this.complexProject = ImmutableMap.copyOf(complexProject);
this.filterEdges = ImmutableList.copyOf(filterEdges);
}
public List<JoinEdge> getJoinEdges() {
@ -103,191 +109,10 @@ public class HyperGraph {
return nodes.get(index);
}
/**
* Store the relation between Alias Slot and Original Slot and its expression
* e.g.,
* a = b
* |--- project((c + d) as b)
* <p>
* a = b
* |--- project((c + 1) as b)
*
* @param alias The alias Expression in project Operator
*/
public boolean addAlias(Alias alias, long subTreeNodes) {
Slot aliasSlot = alias.toSlot();
if (slotToNodeMap.containsKey(aliasSlot)) {
return true;
}
long bitmap = LongBitmap.newBitmap();
for (Slot slot : alias.getInputSlots()) {
bitmap = LongBitmap.or(bitmap, slotToNodeMap.get(slot));
}
// The case hit when there are some constant aliases such as:
// select * from t1 join (
// select *, 1 as b1 from t2)
// on t1.b = b1
// just reference them all for this slot
if (bitmap == 0) {
bitmap = subTreeNodes;
}
Preconditions.checkArgument(bitmap > 0, "slot must belong to some table");
slotToNodeMap.put(aliasSlot, bitmap);
if (!complexProject.containsKey(bitmap)) {
complexProject.put(bitmap, new ArrayList<>());
}
alias = (Alias) PlanUtils.mergeProjections(complexProject.get(bitmap), Lists.newArrayList(alias)).get(0);
complexProject.get(bitmap).add(alias);
return true;
}
/**
* add end node to HyperGraph
*
* @param group The group that is the end node in graph
* @return return the node index
*/
private int addDPHyperNode(Group group) {
for (Slot slot : group.getLogicalExpression().getPlan().getOutput()) {
Preconditions.checkArgument(!slotToNodeMap.containsKey(slot));
slotToNodeMap.put(slot, LongBitmap.newBitmap(nodes.size()));
}
nodes.add(new DPhyperNode(nodes.size(), group));
return nodes.size() - 1;
}
/**
* add end node to HyperGraph
*
* @param plan The plan that is the end node in graph
* @return return the node index
*/
private int addStructInfoNode(Plan plan) {
for (Slot slot : plan.getOutput()) {
Preconditions.checkArgument(!slotToNodeMap.containsKey(slot));
slotToNodeMap.put(slot, LongBitmap.newBitmap(nodes.size()));
}
nodes.add(new StructInfoNode(nodes.size(), plan));
return nodes.size() - 1;
}
private int addStructInfoNode(List<HyperGraph> childGraphs) {
for (Slot slot : childGraphs.get(0).finalOutputs) {
Preconditions.checkArgument(!slotToNodeMap.containsKey(slot));
slotToNodeMap.put(slot, LongBitmap.newBitmap(nodes.size()));
}
nodes.add(new StructInfoNode(nodes.size(), childGraphs));
return nodes.size() - 1;
}
public void updateNode(int idx, Group group) {
Preconditions.checkArgument(nodes.get(idx) instanceof DPhyperNode);
nodes.set(idx, ((DPhyperNode) nodes.get(idx)).withGroup(group));
}
public HashMap<Long, List<NamedExpression>> getComplexProject() {
public Map<Long, List<NamedExpression>> getComplexProject() {
return complexProject;
}
private void addEdgeOfInfo(JoinEdge edge) {
long nodeMap = calNodeMap(edge.getInputSlots());
Preconditions.checkArgument(LongBitmap.getCardinality(nodeMap) > 1,
"edge must have more than one ends");
long left = LongBitmap.newBitmap(LongBitmap.nextSetBit(nodeMap, 0));
long right = LongBitmap.newBitmapDiff(nodeMap, left);
this.joinEdges.add(new JoinEdge(edge.getJoin(), joinEdges.size(),
null, null, 0, left, right));
}
/**
* try to add edge for join group
*
* @param join The join plan
*/
private BitSet addJoin(LogicalJoin<?, ?> join,
Pair<BitSet, Long> leftEdgeNodes, Pair<BitSet, Long> rightEdgeNodes) {
HashMap<Pair<Long, Long>, Pair<List<Expression>, List<Expression>>> conjuncts = new HashMap<>();
for (Expression expression : join.getHashJoinConjuncts()) {
// TODO: avoid calling calculateEnds if calNodeMap's results are same
Pair<Long, Long> ends = calculateEnds(calNodeMap(expression.getInputSlots()), leftEdgeNodes,
rightEdgeNodes);
if (!conjuncts.containsKey(ends)) {
conjuncts.put(ends, Pair.of(new ArrayList<>(), new ArrayList<>()));
}
conjuncts.get(ends).first.add(expression);
}
for (Expression expression : join.getOtherJoinConjuncts()) {
Pair<Long, Long> ends = calculateEnds(calNodeMap(expression.getInputSlots()), leftEdgeNodes,
rightEdgeNodes);
if (!conjuncts.containsKey(ends)) {
conjuncts.put(ends, Pair.of(new ArrayList<>(), new ArrayList<>()));
}
conjuncts.get(ends).second.add(expression);
}
BitSet curJoinEdges = new BitSet();
for (Map.Entry<Pair<Long, Long>, Pair<List<Expression>, List<Expression>>> entry : conjuncts
.entrySet()) {
LogicalJoin<?, ?> singleJoin = new LogicalJoin<>(join.getJoinType(), entry.getValue().first,
entry.getValue().second,
new DistributeHint(DistributeType.NONE), join.getMarkJoinSlotReference(),
Lists.newArrayList(join.left(), join.right()));
Pair<Long, Long> ends = entry.getKey();
JoinEdge edge = new JoinEdge(singleJoin, joinEdges.size(), leftEdgeNodes.first, rightEdgeNodes.first,
LongBitmap.newBitmapUnion(leftEdgeNodes.second, rightEdgeNodes.second), ends.first, ends.second);
for (int nodeIndex : LongBitmap.getIterator(edge.getReferenceNodes())) {
nodes.get(nodeIndex).attachEdge(edge);
}
curJoinEdges.set(edge.getIndex());
joinEdges.add(edge);
}
curJoinEdges.stream().forEach(i -> joinEdges.get(i).addCurJoinEdges(curJoinEdges));
curJoinEdges.stream().forEach(i -> ConflictRulesMaker.makeJoinConflictRules(joinEdges.get(i), joinEdges));
curJoinEdges.stream().forEach(i ->
ConflictRulesMaker.makeFilterConflictRules(joinEdges.get(i), joinEdges, filterEdges));
return curJoinEdges;
// In MySQL, each edge is reversed and store in edges again for reducing the branch miss
// We don't implement this trick now.
}
private BitSet addFilter(LogicalFilter<?> filter, Pair<BitSet, Long> childEdgeNodes) {
FilterEdge edge = new FilterEdge(filter, filterEdges.size(), childEdgeNodes.first, childEdgeNodes.second,
childEdgeNodes.second);
filterEdges.add(edge);
BitSet bitSet = new BitSet();
bitSet.set(edge.getIndex());
return bitSet;
}
// Try to calculate the ends of an expression.
// left = ref_nodes \cap left_tree , right = ref_nodes \cap right_tree
// if left = 0, recursively calculate it in left tree
private Pair<Long, Long> calculateEnds(long allNodes, Pair<BitSet, Long> leftEdgeNodes,
Pair<BitSet, Long> rightEdgeNodes) {
long left = LongBitmap.newBitmapIntersect(allNodes, leftEdgeNodes.second);
long right = LongBitmap.newBitmapIntersect(allNodes, rightEdgeNodes.second);
if (left == 0) {
Preconditions.checkArgument(leftEdgeNodes.first.cardinality() > 0,
"the number of the table which expression reference is less 2");
Pair<BitSet, Long> llEdgesNodes = joinEdges.get(leftEdgeNodes.first.nextSetBit(0)).getLeftEdgeNodes(
joinEdges);
Pair<BitSet, Long> lrEdgesNodes = joinEdges.get(leftEdgeNodes.first.nextSetBit(0)).getRightEdgeNodes(
joinEdges);
return calculateEnds(allNodes, llEdgesNodes, lrEdgesNodes);
}
if (right == 0) {
Preconditions.checkArgument(rightEdgeNodes.first.cardinality() > 0,
"the number of the table which expression reference is less 2");
Pair<BitSet, Long> rlEdgesNodes = joinEdges.get(rightEdgeNodes.first.nextSetBit(0)).getLeftEdgeNodes(
joinEdges);
Pair<BitSet, Long> rrEdgesNodes = joinEdges.get(rightEdgeNodes.first.nextSetBit(0)).getRightEdgeNodes(
joinEdges);
return calculateEnds(allNodes, rlEdgesNodes, rrEdgesNodes);
}
return Pair.of(left, right);
}
public BitSet getEdgesInOperator(long left, long right) {
BitSet operatorEdgesMap = new BitSet();
operatorEdgesMap.or(getEdgesInTree(LongBitmap.or(left, right)));
@ -312,182 +137,6 @@ public class HyperGraph {
return treeEdgesCache.get(treeNodesMap);
}
private long calNodeMap(Set<Slot> slots) {
Preconditions.checkArgument(slots.size() != 0);
long bitmap = LongBitmap.newBitmap();
for (Slot slot : slots) {
Preconditions.checkArgument(slotToNodeMap.containsKey(slot));
bitmap = LongBitmap.or(bitmap, slotToNodeMap.get(slot));
}
return bitmap;
}
public static List<HyperGraph> toStructInfo(Plan plan) {
HyperGraph hyperGraph = new HyperGraph(plan.getOutputSet());
hyperGraph.buildStructInfo(plan);
return hyperGraph.flatChildren();
}
private List<HyperGraph> flatChildren() {
if (nodes.stream().noneMatch(n -> ((StructInfoNode) n).needToFlat())) {
return Lists.newArrayList(this);
}
List<HyperGraph> res = new ArrayList<>();
res.add(new HyperGraph(finalOutputs));
for (AbstractNode node : nodes) {
res = flatChild((StructInfoNode) node, res);
}
for (JoinEdge edge : joinEdges) {
res.forEach(g -> g.addEdgeOfInfo(edge));
}
return res;
}
private List<HyperGraph> flatChild(StructInfoNode infoNode, List<HyperGraph> hyperGraphs) {
if (!infoNode.needToFlat()) {
hyperGraphs.forEach(g -> g.addStructInfoNode(infoNode.getPlan()));
return hyperGraphs;
}
return hyperGraphs.stream().flatMap(g ->
infoNode.getGraphs().stream().map(subGraph -> {
HyperGraph hyperGraph = new HyperGraph(g.finalOutputs);
hyperGraph.addStructInfo(g);
hyperGraph.addStructInfo(subGraph);
return hyperGraph;
})
).collect(Collectors.toList());
}
public static HyperGraph toDPhyperGraph(Group group) {
HyperGraph hyperGraph = new HyperGraph(group.getLogicalProperties().getOutputSet());
hyperGraph.buildDPhyperGraph(group.getLogicalExpressions().get(0));
return hyperGraph;
}
// Build Graph for DPhyper
private Pair<BitSet, Long> buildDPhyperGraph(GroupExpression groupExpression) {
// process Project
if (isValidProject(groupExpression.getPlan())) {
LogicalProject<?> project = (LogicalProject<?>) groupExpression.getPlan();
Pair<BitSet, Long> res = this.buildDPhyperGraph(groupExpression.child(0).getLogicalExpressions().get(0));
for (NamedExpression expr : project.getProjects()) {
if (expr instanceof Alias) {
this.addAlias((Alias) expr, res.second);
}
}
return res;
}
// process Join
if (isValidJoin(groupExpression.getPlan())) {
LogicalJoin<?, ?> join = (LogicalJoin<?, ?>) groupExpression.getPlan();
Pair<BitSet, Long> left = this.buildDPhyperGraph(groupExpression.child(0).getLogicalExpressions().get(0));
Pair<BitSet, Long> right = this.buildDPhyperGraph(groupExpression.child(1).getLogicalExpressions().get(0));
return Pair.of(this.addJoin(join, left, right),
LongBitmap.or(left.second, right.second));
}
// process Other Node
int idx = this.addDPHyperNode(groupExpression.getOwnerGroup());
return Pair.of(new BitSet(), LongBitmap.newBitmap(idx));
}
private void addStructInfo(HyperGraph other) {
int offset = this.getNodes().size();
other.getNodes().forEach(n -> this.addStructInfoNode(n.getPlan()));
other.getComplexProject().forEach((t, projectList) ->
projectList.forEach(e -> this.addAlias((Alias) e, t << offset)));
other.getJoinEdges().forEach(this::addEdgeOfInfo);
}
// Build Graph for matching mv, return join edge set and nodes in this plan
private Pair<BitSet, Long> buildStructInfo(Plan plan) {
if (plan instanceof GroupPlan) {
Group group = ((GroupPlan) plan).getGroup();
List<HyperGraph> childGraphs = ((GroupPlan) plan).getGroup().getHyperGraphs();
if (childGraphs.size() != 0) {
int idx = addStructInfoNode(childGraphs);
return Pair.of(new BitSet(), LongBitmap.newBitmap(idx));
}
GroupExpression groupExpression = group.getLogicalExpressions().get(0);
return buildStructInfo(groupExpression.getPlan()
.withChildren(
groupExpression.children().stream().map(GroupPlan::new).collect(Collectors.toList())));
}
// process Project
if (isValidProject(plan)) {
LogicalProject<?> project = (LogicalProject<?>) plan;
Pair<BitSet, Long> res = this.buildStructInfo(plan.child(0));
for (NamedExpression expr : project.getProjects()) {
if (expr instanceof Alias) {
this.addAlias((Alias) expr, res.second);
}
}
return res;
}
// process Join
if (isValidJoinForStructInfo(plan)) {
LogicalJoin<?, ?> join = (LogicalJoin<?, ?>) plan;
Pair<BitSet, Long> left = this.buildStructInfo(plan.child(0));
Pair<BitSet, Long> right = this.buildStructInfo(plan.child(1));
return Pair.of(this.addJoin(join, left, right),
LongBitmap.or(left.second, right.second));
}
if (isValidFilter(plan)) {
LogicalFilter<?> filter = (LogicalFilter<?>) plan;
Pair<BitSet, Long> child = this.buildStructInfo(filter.child());
this.addFilter(filter, child);
return Pair.of(new BitSet(), child.second);
}
// process Other Node
int idx = this.addStructInfoNode(plan);
return Pair.of(new BitSet(), LongBitmap.newBitmap(idx));
}
/**
* inner join group without mark slot
*/
public static boolean isValidJoin(Plan plan) {
if (!(plan instanceof LogicalJoin)) {
return false;
}
LogicalJoin<?, ?> join = (LogicalJoin<?, ?>) plan;
return join.getJoinType() == JoinType.INNER_JOIN
&& !join.isMarkJoin()
&& !join.getExpressions().isEmpty();
}
/**
* inner join group without mark slot
*/
public static boolean isValidJoinForStructInfo(Plan plan) {
if (!(plan instanceof LogicalJoin)) {
return false;
}
LogicalJoin<?, ?> join = (LogicalJoin<?, ?>) plan;
return !join.isMarkJoin()
&& !join.getExpressions().isEmpty();
}
public static boolean isValidFilter(Plan plan) {
return plan instanceof LogicalFilter;
}
/**
* the project with alias and slot
*/
public static boolean isValidProject(Plan plan) {
if (!(plan instanceof LogicalProject)) {
return false;
}
return ((LogicalProject<? extends Plan>) plan).getProjects().stream()
.allMatch(e -> e instanceof Slot || e instanceof Alias);
}
/**
* Graph simplifier need to update the edge for join ordering
*
@ -520,8 +169,45 @@ public class HyperGraph {
LongBitmap.getIterator(addedNodes).forEach(index -> nodes.get(index).attachEdge(edge));
}
public int edgeSize() {
return joinEdges.size() + filterEdges.size();
/**
* inner join group without mark slot
*/
public static boolean isValidJoinForStructInfo(Plan plan) {
if (!(plan instanceof LogicalJoin)) {
return false;
}
LogicalJoin<?, ?> join = (LogicalJoin<?, ?>) plan;
return !join.isMarkJoin()
&& !join.getExpressions().isEmpty();
}
public static boolean isValidFilter(Plan plan) {
return plan instanceof LogicalFilter;
}
/**
* the project with alias and slot
*/
public static boolean isValidProject(Plan plan) {
if (!(plan instanceof LogicalProject)) {
return false;
}
return ((LogicalProject<? extends Plan>) plan).getProjects().stream()
.allMatch(e -> e instanceof Slot || e instanceof Alias);
}
/**
* inner join group without mark slot
*/
public static boolean isValidJoin(Plan plan) {
if (!(plan instanceof LogicalJoin)) {
return false;
}
LogicalJoin<?, ?> join = (LogicalJoin<?, ?>) plan;
return join.getJoinType() == JoinType.INNER_JOIN
&& !join.isMarkJoin()
&& !join.getExpressions().isEmpty();
}
/**
@ -535,7 +221,7 @@ public class HyperGraph {
*/
public String toDottyHyperGraph() {
StringBuilder builder = new StringBuilder();
builder.append(String.format("digraph G { # %d edges\n", joinEdges.size()));
builder.append(String.format("digraph G { # %d edges%n", joinEdges.size()));
List<String> graphvisNodes = new ArrayList<>();
for (AbstractNode node : nodes) {
String nodeName = node.getName();
@ -547,7 +233,7 @@ public class HyperGraph {
double rowCount = (node instanceof DPhyperNode)
? ((DPhyperNode) node).getRowCount()
: -1;
builder.append(String.format(" %s [label=\"%s \n rowCount=%.2f\"];\n",
builder.append(String.format(" %s [label=\"%s %n rowCount=%.2f\"];%n",
nodeID, nodeName, rowCount));
graphvisNodes.add(nodeName);
}
@ -563,11 +249,11 @@ public class HyperGraph {
int leftIndex = LongBitmap.lowestOneIndex(edge.getLeftExtendedNodes());
int rightIndex = LongBitmap.lowestOneIndex(edge.getRightExtendedNodes());
builder.append(String.format("%s -> %s [label=\"%s\"%s]\n", graphvisNodes.get(leftIndex),
builder.append(String.format("%s -> %s [label=\"%s\"%s]%n", graphvisNodes.get(leftIndex),
graphvisNodes.get(rightIndex), label, arrowHead));
} else {
// Hyper edge is considered as a tiny virtual node
builder.append(String.format("e%d [shape=circle, width=.001, label=\"\"]\n", i));
builder.append(String.format("e%d [shape=circle, width=.001, label=\"\"]%n", i));
String leftLabel = "";
String rightLabel = "";
@ -577,21 +263,312 @@ public class HyperGraph {
leftLabel = label;
}
int finalI = i;
String finalLeftLabel = leftLabel;
for (int nodeIndex : LongBitmap.getIterator(edge.getLeftExtendedNodes())) {
builder.append(String.format("%s -> e%d [arrowhead=none, label=\"%s\"]\n",
graphvisNodes.get(nodeIndex), finalI, finalLeftLabel));
builder.append(String.format("%s -> e%d [arrowhead=none, label=\"%s\"]%n",
graphvisNodes.get(nodeIndex), i, finalLeftLabel));
}
String finalRightLabel = rightLabel;
for (int nodeIndex : LongBitmap.getIterator(edge.getRightExtendedNodes())) {
builder.append(String.format("%s -> e%d [arrowhead=none, label=\"%s\"]\n",
graphvisNodes.get(nodeIndex), finalI, finalRightLabel));
builder.append(String.format("%s -> e%d [arrowhead=none, label=\"%s\"]%n",
graphvisNodes.get(nodeIndex), i, finalRightLabel));
}
}
}
builder.append("}\n");
return builder.toString();
}
public static HyperGraph.Builder builderForDPhyper(Group group) {
return new HyperGraph.Builder().buildHyperGraphForDPhyper(group);
}
public static HyperGraph.Builder builderForMv(Plan plan) {
return new HyperGraph.Builder().buildHyperGraphForMv(plan);
}
/**
* Builder of HyperGraph
*/
public static class Builder {
private final List<JoinEdge> joinEdges = new ArrayList<>();
private final List<FilterEdge> filterEdges = new ArrayList<>();
private final List<AbstractNode> nodes = new ArrayList<>();
// These hyperGraphs should be replaced nodes when building all
private final Map<Long, List<HyperGraph>> replacedHyperGraphs = new HashMap<>();
private final HashMap<Slot, Long> slotToNodeMap = new HashMap<>();
private final Map<Long, List<NamedExpression>> complexProject = new HashMap<>();
private Set<Slot> finalOutputs;
public List<AbstractNode> getNodes() {
return nodes;
}
private HyperGraph.Builder buildHyperGraphForDPhyper(Group group) {
finalOutputs = group.getLogicalProperties().getOutputSet();
this.buildForDPhyper(group.getLogicalExpression());
return this;
}
private HyperGraph.Builder buildHyperGraphForMv(Plan plan) {
finalOutputs = plan.getOutputSet();
this.buildForMv(plan);
return this;
}
public HyperGraph build() {
return new HyperGraph(finalOutputs, joinEdges, nodes, filterEdges, complexProject);
}
public List<HyperGraph> buildAll() {
return ImmutableList.of(build());
}
public void updateNode(int idx, Group group) {
Preconditions.checkArgument(nodes.get(idx) instanceof DPhyperNode);
nodes.set(idx, ((DPhyperNode) nodes.get(idx)).withGroup(group));
}
// Build Graph for DPhyper
private Pair<BitSet, Long> buildForDPhyper(GroupExpression groupExpression) {
// process Project
if (isValidProject(groupExpression.getPlan())) {
LogicalProject<?> project = (LogicalProject<?>) groupExpression.getPlan();
Pair<BitSet, Long> res = this.buildForDPhyper(groupExpression.child(0).getLogicalExpressions().get(0));
for (NamedExpression expr : project.getProjects()) {
if (expr instanceof Alias) {
this.addAlias((Alias) expr, res.second);
}
}
return res;
}
// process Join
if (isValidJoin(groupExpression.getPlan())) {
LogicalJoin<?, ?> join = (LogicalJoin<?, ?>) groupExpression.getPlan();
Pair<BitSet, Long> left =
this.buildForDPhyper(groupExpression.child(0).getLogicalExpressions().get(0));
Pair<BitSet, Long> right =
this.buildForDPhyper(groupExpression.child(1).getLogicalExpressions().get(0));
return Pair.of(this.addJoin(join, left, right),
LongBitmap.or(left.second, right.second));
}
// process Other Node
int idx = this.addDPHyperNode(groupExpression.getOwnerGroup());
return Pair.of(new BitSet(), LongBitmap.newBitmap(idx));
}
// Build Graph for matching mv, return join edge set and nodes in this plan
private Pair<BitSet, Long> buildForMv(Plan plan) {
if (plan instanceof GroupPlan) {
Group group = ((GroupPlan) plan).getGroup();
GroupExpression groupExpression = group.getLogicalExpressions().get(0);
return buildForMv(groupExpression.getPlan()
.withChildren(
groupExpression.children().stream().map(GroupPlan::new).collect(Collectors.toList())));
}
// process Project
if (isValidProject(plan)) {
LogicalProject<?> project = (LogicalProject<?>) plan;
Pair<BitSet, Long> res = this.buildForMv(plan.child(0));
for (NamedExpression expr : project.getProjects()) {
if (expr instanceof Alias) {
this.addAlias((Alias) expr, res.second);
}
}
return res;
}
// process Join
if (isValidJoinForStructInfo(plan)) {
LogicalJoin<?, ?> join = (LogicalJoin<?, ?>) plan;
Pair<BitSet, Long> left = this.buildForMv(plan.child(0));
Pair<BitSet, Long> right = this.buildForMv(plan.child(1));
return Pair.of(this.addJoin(join, left, right),
LongBitmap.or(left.second, right.second));
}
if (isValidFilter(plan)) {
LogicalFilter<?> filter = (LogicalFilter<?>) plan;
Pair<BitSet, Long> child = this.buildForMv(filter.child());
this.addFilter(filter, child);
return Pair.of(new BitSet(), child.second);
}
// process Other Node
int idx = this.addStructInfoNode(plan);
return Pair.of(new BitSet(), LongBitmap.newBitmap(idx));
}
/**
* Store the relation between Alias Slot and Original Slot and its expression
* e.g.,
* a = b
* |--- project((c + d) as b)
* <p>
* a = b
* |--- project((c + 1) as b)
*
* @param alias The alias Expression in project Operator
*/
public boolean addAlias(Alias alias, long subTreeNodes) {
Slot aliasSlot = alias.toSlot();
if (slotToNodeMap.containsKey(aliasSlot)) {
return true;
}
long bitmap = LongBitmap.newBitmap();
for (Slot slot : alias.getInputSlots()) {
bitmap = LongBitmap.or(bitmap, slotToNodeMap.get(slot));
}
// The case hit when there are some constant aliases such as:
// select * from t1 join (
// select *, 1 as b1 from t2)
// on t1.b = b1
// just reference them all for this slot
if (bitmap == 0) {
bitmap = subTreeNodes;
}
Preconditions.checkArgument(bitmap > 0, "slot must belong to some table");
slotToNodeMap.put(aliasSlot, bitmap);
if (!complexProject.containsKey(bitmap)) {
complexProject.put(bitmap, new ArrayList<>());
}
alias = (Alias) PlanUtils.mergeProjections(complexProject.get(bitmap), Lists.newArrayList(alias)).get(0);
complexProject.get(bitmap).add(alias);
return true;
}
/**
* add end node to HyperGraph
*
* @param group The group that is the end node in graph
* @return return the node index
*/
private int addDPHyperNode(Group group) {
for (Slot slot : group.getLogicalExpression().getPlan().getOutput()) {
Preconditions.checkArgument(!slotToNodeMap.containsKey(slot));
slotToNodeMap.put(slot, LongBitmap.newBitmap(nodes.size()));
}
nodes.add(new DPhyperNode(nodes.size(), group));
return nodes.size() - 1;
}
/**
* add end node to HyperGraph
*
* @param plan The plan that is the end node in graph
* @return return the node index
*/
private int addStructInfoNode(Plan plan) {
for (Slot slot : plan.getOutput()) {
Preconditions.checkArgument(!slotToNodeMap.containsKey(slot));
slotToNodeMap.put(slot, LongBitmap.newBitmap(nodes.size()));
}
nodes.add(new StructInfoNode(nodes.size(), plan));
return nodes.size() - 1;
}
private long calNodeMap(Set<Slot> slots) {
Preconditions.checkArgument(slots.size() != 0);
long bitmap = LongBitmap.newBitmap();
for (Slot slot : slots) {
Preconditions.checkArgument(slotToNodeMap.containsKey(slot));
bitmap = LongBitmap.or(bitmap, slotToNodeMap.get(slot));
}
return bitmap;
}
/**
* try to add edge for join group
*
* @param join The join plan
*/
private BitSet addJoin(LogicalJoin<?, ?> join,
Pair<BitSet, Long> leftEdgeNodes, Pair<BitSet, Long> rightEdgeNodes) {
HashMap<Pair<Long, Long>, Pair<List<Expression>, List<Expression>>> conjuncts = new HashMap<>();
for (Expression expression : join.getHashJoinConjuncts()) {
// TODO: avoid calling calculateEnds if calNodeMap's results are same
Pair<Long, Long> ends = calculateEnds(calNodeMap(expression.getInputSlots()), leftEdgeNodes,
rightEdgeNodes);
if (!conjuncts.containsKey(ends)) {
conjuncts.put(ends, Pair.of(new ArrayList<>(), new ArrayList<>()));
}
conjuncts.get(ends).first.add(expression);
}
for (Expression expression : join.getOtherJoinConjuncts()) {
Pair<Long, Long> ends = calculateEnds(calNodeMap(expression.getInputSlots()), leftEdgeNodes,
rightEdgeNodes);
if (!conjuncts.containsKey(ends)) {
conjuncts.put(ends, Pair.of(new ArrayList<>(), new ArrayList<>()));
}
conjuncts.get(ends).second.add(expression);
}
BitSet curJoinEdges = new BitSet();
for (Map.Entry<Pair<Long, Long>, Pair<List<Expression>, List<Expression>>> entry : conjuncts
.entrySet()) {
LogicalJoin<?, ?> singleJoin = new LogicalJoin<>(join.getJoinType(), entry.getValue().first,
entry.getValue().second,
new DistributeHint(DistributeType.NONE), join.getMarkJoinSlotReference(),
Lists.newArrayList(join.left(), join.right()));
Pair<Long, Long> ends = entry.getKey();
JoinEdge edge = new JoinEdge(singleJoin, joinEdges.size(), leftEdgeNodes.first, rightEdgeNodes.first,
LongBitmap.newBitmapUnion(leftEdgeNodes.second, rightEdgeNodes.second),
ends.first, ends.second);
for (int nodeIndex : LongBitmap.getIterator(edge.getReferenceNodes())) {
nodes.get(nodeIndex).attachEdge(edge);
}
curJoinEdges.set(edge.getIndex());
joinEdges.add(edge);
}
curJoinEdges.stream().forEach(i -> joinEdges.get(i).addCurJoinEdges(curJoinEdges));
curJoinEdges.stream().forEach(i -> ConflictRulesMaker.makeJoinConflictRules(joinEdges.get(i), joinEdges));
curJoinEdges.stream().forEach(i ->
ConflictRulesMaker.makeFilterConflictRules(joinEdges.get(i), joinEdges, filterEdges));
return curJoinEdges;
// In MySQL, each edge is reversed and store in edges again for reducing the branch miss
// We don't implement this trick now.
}
private BitSet addFilter(LogicalFilter<?> filter, Pair<BitSet, Long> childEdgeNodes) {
FilterEdge edge = new FilterEdge(filter, filterEdges.size(), childEdgeNodes.first, childEdgeNodes.second,
childEdgeNodes.second);
filterEdges.add(edge);
BitSet bitSet = new BitSet();
bitSet.set(edge.getIndex());
return bitSet;
}
// Try to calculate the ends of an expression.
// left = ref_nodes \cap left_tree , right = ref_nodes \cap right_tree
// if left = 0, recursively calculate it in left tree
private Pair<Long, Long> calculateEnds(long allNodes, Pair<BitSet, Long> leftEdgeNodes,
Pair<BitSet, Long> rightEdgeNodes) {
long left = LongBitmap.newBitmapIntersect(allNodes, leftEdgeNodes.second);
long right = LongBitmap.newBitmapIntersect(allNodes, rightEdgeNodes.second);
if (left == 0) {
Preconditions.checkArgument(leftEdgeNodes.first.cardinality() > 0,
"the number of the table which expression reference is less 2");
Pair<BitSet, Long> llEdgesNodes = joinEdges.get(leftEdgeNodes.first.nextSetBit(0)).getLeftEdgeNodes(
joinEdges);
Pair<BitSet, Long> lrEdgesNodes = joinEdges.get(leftEdgeNodes.first.nextSetBit(0)).getRightEdgeNodes(
joinEdges);
return calculateEnds(allNodes, llEdgesNodes, lrEdgesNodes);
}
if (right == 0) {
Preconditions.checkArgument(rightEdgeNodes.first.cardinality() > 0,
"the number of the table which expression reference is less 2");
Pair<BitSet, Long> rlEdgesNodes = joinEdges.get(rightEdgeNodes.first.nextSetBit(0)).getLeftEdgeNodes(
joinEdges);
Pair<BitSet, Long> rrEdgesNodes = joinEdges.get(rightEdgeNodes.first.nextSetBit(0)).getRightEdgeNodes(
joinEdges);
return calculateEnds(allNodes, rlEdgesNodes, rrEdgesNodes);
}
return Pair.of(left, right);
}
}
}

16
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/StructInfoNode.java
View File

@ -17,7 +17,6 @@
package org.apache.doris.nereids.jobs.joinorder.hypergraph.node;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.GroupPlan;
@ -44,8 +43,6 @@ import javax.annotation.Nullable;
* HyperGraph Node.
*/
public class StructInfoNode extends AbstractNode {
private List<HyperGraph> graphs = new ArrayList<>();
private final List<Set<Expression>> expressions;
private final Set<CatalogRelation> relationSet;
@ -59,11 +56,6 @@ public class StructInfoNode extends AbstractNode {
this(index, plan, new ArrayList<>());
}
public StructInfoNode(int index, List<HyperGraph> graphs) {
this(index, graphs.get(0).getNode(0).getPlan(), new ArrayList<>());
this.graphs = graphs;
}
private @Nullable List<Set<Expression>> collectExpressions(Plan plan) {
if (plan instanceof LeafPlan) {
return ImmutableList.of();
@ -122,14 +114,6 @@ public class StructInfoNode extends AbstractNode {
return plan.withChildren(children);
}
public boolean needToFlat() {
return !graphs.isEmpty();
}
public List<HyperGraph> getGraphs() {
return graphs;
}
@Override
public String toString() {
return Utils.toSqlString("StructInfoNode[" + this.getName() + "]",

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
View File

@ -225,7 +225,7 @@ public class StructInfo {
// if single table without join, the bottom is
originalPlan.accept(PLAN_SPLITTER, planSplitContext);
List<HyperGraph> structInfos = HyperGraph.toStructInfo(planSplitContext.getBottomPlan());
List<HyperGraph> structInfos = HyperGraph.builderForMv(planSplitContext.getBottomPlan()).buildAll();
return structInfos.stream()
.map(hyperGraph -> StructInfo.of(originalPlan, planSplitContext.getTopPlan(),
planSplitContext.getBottomPlan(), hyperGraph))