From e6da00bb261ef6a04207336e318e64b2558193c3 Mon Sep 17 00:00:00 2001 From: zhengshiJ <32082872+zhengshiJ@users.noreply.github.com> Date: Fri, 8 Jul 2022 19:22:48 +0800 Subject: [PATCH] [feature](nereides) support sort translator (#10678) Physical sort: * 1. Build sortInfo * There are two types of slotRef: * one is generated by the previous node, collectively called old. * the other is newly generated by the sort node, collectively called new. * Filling of sortInfo related data structures, * a. ordering use newSlotRef. * b. sortTupleSlotExprs use oldSlotRef. * 2. Create sortNode * 3. Create mergeFragment TODO: 1.Currently, columns that do not exist in select but exist in order by cannot be parsed. eg: select key from table order by value; 2.For the combination of Literal and slotRefrance in select, there is a problem with parsing, eg: select key ,(10-value) from table; --- .../org/apache/doris/analysis/SortInfo.java | 4 ++ .../apache/doris/nereids/NereidsPlanner.java | 6 +- .../translator/PhysicalPlanTranslator.java | 56 +++++++++++++++++-- .../translator/PlanTranslatorContext.java | 13 ++++- .../org/apache/doris/planner/PlanNode.java | 7 +++ .../org/apache/doris/planner/SortNode.java | 23 ++++++++ .../org/apache/doris/qe/StmtExecutor.java | 1 - 7 files changed, 98 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java index 05763e8d07..63394db4cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java @@ -141,6 +141,10 @@ public class SortInfo { this.sortTupleSlotExprs = sortTupleSlotExprs; } + public void setSortTupleDesc(TupleDescriptor tupleDesc) { + sortTupleDesc = tupleDesc; + } + public TupleDescriptor getSortTupleDescriptor() { return sortTupleDesc; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index bc7aed0bf1..23fafa518d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -25,7 +25,6 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Id; import org.apache.doris.common.UserException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator; @@ -39,7 +38,6 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; @@ -104,8 +102,8 @@ public class NereidsPlanner extends Planner { outputCandidates.put(slotDescriptor.getId().asInt(), slotRef); } } - physicalPlan.getOutput().stream().map(Slot::getExprId) - .map(Id::asInt).forEach(i -> outputExprs.add(outputCandidates.get(i))); + physicalPlan.getOutput().stream() + .forEach(i -> outputExprs.add(planTranslatorContext.findExpr(i))); root.setOutputExprs(outputExprs); root.getPlanRoot().convertToVectoriezd(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 511ac61fb4..0dd6b9e859 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -204,6 +204,26 @@ public class PhysicalPlanTranslator extends PlanOperatorVisitor sort, PlanTranslatorContext context) { @@ -211,24 +231,35 @@ public class PhysicalPlanTranslator extends PlanOperatorVisitor execOrderingExprList = Lists.newArrayList(); + List oldOrderingExprList = Lists.newArrayList(); List ascOrderList = Lists.newArrayList(); List nullsFirstParamList = Lists.newArrayList(); List orderKeyList = physicalHeapSort.getOrderKeys(); + // 1.Get previous slotRef orderKeyList.forEach(k -> { - execOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context)); + oldOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context)); ascOrderList.add(k.isAsc()); nullsFirstParamList.add(k.isNullFirst()); }); - + List sortTupleOutputList = new ArrayList<>(); List outputList = sort.getOutput(); - TupleDescriptor tupleDesc = generateTupleDesc(outputList, context, null); - SortInfo sortInfo = new SortInfo(execOrderingExprList, ascOrderList, nullsFirstParamList, tupleDesc); - + outputList.forEach(k -> { + sortTupleOutputList.add(ExpressionTranslator.translate(k, context)); + }); + // 2. Generate new Tuple + TupleDescriptor tupleDesc = generateTupleDesc(outputList, orderKeyList, context, null); + // 3. Get current slotRef + List newOrderingExprList = Lists.newArrayList(); + orderKeyList.forEach(k -> { + newOrderingExprList.add(ExpressionTranslator.translate(k.getExpr(), context)); + }); + // 4. fill in SortInfo members + SortInfo sortInfo = new SortInfo(newOrderingExprList, ascOrderList, nullsFirstParamList, tupleDesc); PlanNode childNode = childFragment.getPlanRoot(); // TODO: notice topN SortNode sortNode = new SortNode(context.nextNodeId(), childNode, sortInfo, true, physicalHeapSort.hasLimit(), physicalHeapSort.getOffset()); + sortNode.finalizeForNereids(tupleDesc, sortTupleOutputList, oldOrderingExprList); childFragment.addPlanRoot(sortNode); if (!childFragment.isPartitioned()) { return childFragment; @@ -362,6 +393,19 @@ public class PhysicalPlanTranslator extends PlanOperatorVisitor slotList, List orderKeyList, + PlanTranslatorContext context, Table table) { + TupleDescriptor tupleDescriptor = context.generateTupleDesc(); + tupleDescriptor.setTable(table); + for (Slot slot : slotList) { + context.createSlotDesc(tupleDescriptor, (SlotReference) slot); + } + for (OrderKey orderKey : orderKeyList) { + context.createSlotDesc(tupleDescriptor, orderKey.getExpr()); + } + return tupleDescriptor; + } + private PlanFragment createParentFragment(PlanFragment childFragment, DataPartition parentPartition, PlanTranslatorContext ctx) { ExchangeNode exchangeNode = new ExchangeNode(ctx.nextNodeId(), childFragment.getPlanRoot(), false); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java index afde344669..cc3337bbd3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java @@ -106,7 +106,7 @@ public class PlanTranslatorContext { * Create SlotDesc and add it to the mappings from expression to the stales epxr */ public SlotDescriptor createSlotDesc(TupleDescriptor tupleDesc, SlotReference slotReference) { - SlotDescriptor slotDescriptor = this.addSlotDesc(tupleDesc, slotReference.getExprId().asInt()); + SlotDescriptor slotDescriptor = this.addSlotDesc(tupleDesc); Column column = slotReference.getColumn(); // Only the SlotDesc that in the tuple generated for scan node would have corresponding column. if (column != null) { @@ -118,6 +118,17 @@ public class PlanTranslatorContext { return slotDescriptor; } + /** + * Create slotDesc with Expression. + */ + public void createSlotDesc(TupleDescriptor tupleDesc, Expression expression) { + if (!expressionToExecExpr.containsKey(expression)) { + SlotDescriptor slotDescriptor = this.addSlotDesc(tupleDesc); + slotDescriptor.setType(expression.getDataType().toCatalogDataType()); + this.addSlotRefMapping(expression, new SlotRef(slotDescriptor)); + } + } + public TupleDescriptor getTupleDesc(TupleId tupleId) { return descTable.getTupleDesc(tupleId); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index c2dc140d90..074a52888f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -999,4 +999,11 @@ public abstract class PlanNode extends TreeNode implements PlanStats { output.append("\n"); } } + + /** + * Supplement the information be needed for nodes generated by the new optimizer. + */ + public void finalizeForNereids() { + + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java index db08f4eb48..232e703555 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java @@ -22,11 +22,13 @@ package org.apache.doris.planner; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.ExprId; import org.apache.doris.analysis.ExprSubstitutionMap; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.SortInfo; +import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.UserException; import org.apache.doris.statistics.StatisticalType; @@ -44,6 +46,7 @@ import com.google.common.collect.Lists; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -264,4 +267,24 @@ public class SortNode extends PlanNode { Expr.getIds(resolvedTupleExprs, null, result); return new HashSet<>(result); } + + /** + * Supplement the information needed by be for the sort node. + */ + public void finalizeForNereids(TupleDescriptor tupleDescriptor, + List outputList, List orderingExpr) { + List sortTupleSlotExprs = new ArrayList<>(); + sortTupleSlotExprs.addAll(outputList); + sortTupleSlotExprs.addAll(orderingExpr); + List afterDeduplication = new ArrayList<>(); + Set exprIds = new HashSet<>(); + for (int i = 0; i < sortTupleSlotExprs.size(); i++) { + Expr expr = sortTupleSlotExprs.get(i); + if (!exprIds.contains(expr.getId())) { + afterDeduplication.add(expr); + } + } + info.setSortTupleDesc(tupleDescriptor); + info.setSortTupleSlotExprs(afterDeduplication); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 34321cee03..6478551678 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -978,7 +978,6 @@ public class StmtExecutor implements ProfileWriter { return; } - MysqlChannel channel = context.getMysqlChannel(); boolean isOutfileQuery = queryStmt.hasOutFileClause();