database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](nereides) support sort translator (#10678)

Browse Source 
Physical sort:
     * 1. Build sortInfo
     *    There are two types of slotRef:
     *    one is generated by the previous node, collectively called old.
     *    the other is newly generated by the sort node, collectively called new.
     *    Filling of sortInfo related data structures,
     *    a. ordering use newSlotRef.
     *    b. sortTupleSlotExprs use oldSlotRef.
     * 2. Create sortNode
     * 3. Create mergeFragment

TODO:
1.Currently, columns that do not exist in select but exist in order by cannot be parsed.
eg: select key from table order by value;

2.For the combination of Literal and slotRefrance in select, there is a problem with parsing, 
eg: select key ,(10-value) from table;
This commit is contained in:
zhengshiJ
2022-07-08 19:22:48 +08:00
committed by GitHub
parent 2b1d8ac28a
commit e6da00bb26
7 changed files with 98 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java
View File

@ -141,6 +141,10 @@ public class SortInfo {
this.sortTupleSlotExprs = sortTupleSlotExprs;
}
public void setSortTupleDesc(TupleDescriptor tupleDesc) {
sortTupleDesc = tupleDesc;
}
public TupleDescriptor getSortTupleDescriptor() {
return sortTupleDesc;
}

6
fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
View File

@ -25,7 +25,6 @@ import org.apache.doris.analysis.StatementBase;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Id;
import org.apache.doris.common.UserException;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator;
@ -39,7 +38,6 @@ import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.memo.Memo;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
@ -104,8 +102,8 @@ public class NereidsPlanner extends Planner {
outputCandidates.put(slotDescriptor.getId().asInt(), slotRef);
}
}
physicalPlan.getOutput().stream().map(Slot::getExprId)
.map(Id::asInt).forEach(i -> outputExprs.add(outputCandidates.get(i)));
physicalPlan.getOutput().stream()
.forEach(i -> outputExprs.add(planTranslatorContext.findExpr(i)));
root.setOutputExprs(outputExprs);
root.getPlanRoot().convertToVectoriezd();

56
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View File

@ -204,6 +204,26 @@ public class PhysicalPlanTranslator extends PlanOperatorVisitor<PlanFragment, Pl
return planFragment;
}
/**
* Physical sort:
* 1. Build sortInfo
* There are two types of slotRef:
* one is generated by the previous node, collectively called old.
* the other is newly generated by the sort node, collectively called new.
* Filling of sortInfo related data structures,
* a. ordering use newSlotRef.
* b. sortTupleSlotExprs use oldSlotRef.
* 2. Create sortNode
* 3. Create mergeFragment
* TODO: When the slotRef of sort is currently generated,
* it will be based on the expression in select and orderBy expression in to ensure the uniqueness of slotRef.
* But eg:
* select a+1 from table order by a+1;
* the expressions of the two are inconsistent.
* The former will perform an additional Alisa.
* Currently we cannot test whether this will have any effect.
* After a+1 can be parsed , reprocessing.
*/
@Override
public PlanFragment visitPhysicalSort(PhysicalUnaryPlan<PhysicalHeapSort, Plan> sort,
PlanTranslatorContext context) {
@ -211,24 +231,35 @@ public class PhysicalPlanTranslator extends PlanOperatorVisitor<PlanFragment, Pl
PhysicalHeapSort physicalHeapSort = sort.getOperator();
long limit = physicalHeapSort.getLimit();
// TODO: need to discuss how to process field: SortNode::resolvedTupleExprs
List<Expr> execOrderingExprList = Lists.newArrayList();
List<Expr> oldOrderingExprList = Lists.newArrayList();
List<Boolean> ascOrderList = Lists.newArrayList();
List<Boolean> nullsFirstParamList = Lists.newArrayList();
List<OrderKey> orderKeyList = physicalHeapSort.getOrderKeys();
// 1.Get previous slotRef
orderKeyList.forEach(k -> {
execOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context));
oldOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context));
ascOrderList.add(k.isAsc());
nullsFirstParamList.add(k.isNullFirst());
});
List<Expr> sortTupleOutputList = new ArrayList<>();
List<Slot> outputList = sort.getOutput();
TupleDescriptor tupleDesc = generateTupleDesc(outputList, context, null);
SortInfo sortInfo = new SortInfo(execOrderingExprList, ascOrderList, nullsFirstParamList, tupleDesc);
outputList.forEach(k -> {
sortTupleOutputList.add(ExpressionTranslator.translate(k, context));
});
// 2. Generate new Tuple
TupleDescriptor tupleDesc = generateTupleDesc(outputList, orderKeyList, context, null);
// 3. Get current slotRef
List<Expr> newOrderingExprList = Lists.newArrayList();
orderKeyList.forEach(k -> {
newOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context));
});
// 4. fill in SortInfo members
SortInfo sortInfo = new SortInfo(newOrderingExprList, ascOrderList, nullsFirstParamList, tupleDesc);
PlanNode childNode = childFragment.getPlanRoot();
// TODO: notice topN
SortNode sortNode = new SortNode(context.nextNodeId(), childNode, sortInfo, true,
physicalHeapSort.hasLimit(), physicalHeapSort.getOffset());
sortNode.finalizeForNereids(tupleDesc, sortTupleOutputList, oldOrderingExprList);
childFragment.addPlanRoot(sortNode);
if (!childFragment.isPartitioned()) {
return childFragment;
@ -362,6 +393,19 @@ public class PhysicalPlanTranslator extends PlanOperatorVisitor<PlanFragment, Pl
return tupleDescriptor;
}
private TupleDescriptor generateTupleDesc(List<Slot> slotList, List<OrderKey> orderKeyList,
PlanTranslatorContext context, Table table) {
TupleDescriptor tupleDescriptor = context.generateTupleDesc();
tupleDescriptor.setTable(table);
for (Slot slot : slotList) {
context.createSlotDesc(tupleDescriptor, (SlotReference) slot);
}
for (OrderKey orderKey : orderKeyList) {
context.createSlotDesc(tupleDescriptor, orderKey.getExpr());
}
return tupleDescriptor;
}
private PlanFragment createParentFragment(PlanFragment childFragment, DataPartition parentPartition,
PlanTranslatorContext ctx) {
ExchangeNode exchangeNode = new ExchangeNode(ctx.nextNodeId(), childFragment.getPlanRoot(), false);

13
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
View File

@ -106,7 +106,7 @@ public class PlanTranslatorContext {
* Create SlotDesc and add it to the mappings from expression to the stales epxr
*/
public SlotDescriptor createSlotDesc(TupleDescriptor tupleDesc, SlotReference slotReference) {
SlotDescriptor slotDescriptor = this.addSlotDesc(tupleDesc, slotReference.getExprId().asInt());
SlotDescriptor slotDescriptor = this.addSlotDesc(tupleDesc);
Column column = slotReference.getColumn();
// Only the SlotDesc that in the tuple generated for scan node would have corresponding column.
if (column != null) {
@ -118,6 +118,17 @@ public class PlanTranslatorContext {
return slotDescriptor;
}
/**
* Create slotDesc with Expression.
*/
public void createSlotDesc(TupleDescriptor tupleDesc, Expression expression) {
if (!expressionToExecExpr.containsKey(expression)) {
SlotDescriptor slotDescriptor = this.addSlotDesc(tupleDesc);
slotDescriptor.setType(expression.getDataType().toCatalogDataType());
this.addSlotRefMapping(expression, new SlotRef(slotDescriptor));
}
}
public TupleDescriptor getTupleDesc(TupleId tupleId) {
return descTable.getTupleDesc(tupleId);
}

7
fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
View File

@ -999,4 +999,11 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
output.append("\n");
}
}
/**
* Supplement the information be needed for nodes generated by the new optimizer.
*/
public void finalizeForNereids() {
}
}

23
fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java
View File

@ -22,11 +22,13 @@ package org.apache.doris.planner;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.ExprId;
import org.apache.doris.analysis.ExprSubstitutionMap;
import org.apache.doris.analysis.SlotDescriptor;
import org.apache.doris.analysis.SlotId;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.SortInfo;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.common.UserException;
import org.apache.doris.statistics.StatisticalType;
@ -44,6 +46,7 @@ import com.google.common.collect.Lists;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@ -264,4 +267,24 @@ public class SortNode extends PlanNode {
Expr.getIds(resolvedTupleExprs, null, result);
return new HashSet<>(result);
}
/**
* Supplement the information needed by be for the sort node.
*/
public void finalizeForNereids(TupleDescriptor tupleDescriptor,
List<Expr> outputList, List<Expr> orderingExpr) {
List<Expr> sortTupleSlotExprs = new ArrayList<>();
sortTupleSlotExprs.addAll(outputList);
sortTupleSlotExprs.addAll(orderingExpr);
List<Expr> afterDeduplication = new ArrayList<>();
Set<ExprId> exprIds = new HashSet<>();
for (int i = 0; i < sortTupleSlotExprs.size(); i++) {
Expr expr = sortTupleSlotExprs.get(i);
if (!exprIds.contains(expr.getId())) {
afterDeduplication.add(expr);
}
}
info.setSortTupleDesc(tupleDescriptor);
info.setSortTupleSlotExprs(afterDeduplication);
}
}

1
fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
View File

@ -978,7 +978,6 @@ public class StmtExecutor implements ProfileWriter {
return;
}
MysqlChannel channel = context.getMysqlChannel();
boolean isOutfileQuery = queryStmt.hasOutFileClause();