database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](nereids) set proper sort info to scan node to enable TopN-opt (#21148)

Browse Source
This commit is contained in:
minghong
2023-06-26 19:54:37 +08:00
committed by GitHub
parent c19e35116b
commit 095550271b
2 changed files with 81 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fe/fe-core/src/main/java/org/apache/doris/analysis/SortInfo.java
View File

@ -139,6 +139,13 @@ public class SortInfo {
return materializedOrderingExprs;
}
public void addMaterializedOrderingExpr(Expr expr) {
if (materializedOrderingExprs == null) {
materializedOrderingExprs = Lists.newArrayList();
}
materializedOrderingExprs.add(expr);
}
public List<Expr> getSortTupleSlotExprs() {
return sortTupleSlotExprs;
}

75
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View File

@ -160,6 +160,7 @@ import org.apache.doris.planner.external.HiveScanNode;
import org.apache.doris.planner.external.hudi.HudiScanNode;
import org.apache.doris.planner.external.iceberg.IcebergScanNode;
import org.apache.doris.planner.external.paimon.PaimonScanNode;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.tablefunction.TableValuedFunctionIf;
import org.apache.doris.thrift.TColumn;
import org.apache.doris.thrift.TFetchOption;
@ -1069,7 +1070,25 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
PlanNode child = sortNode.getChild(0);
Preconditions.checkArgument(child instanceof OlapScanNode,
"topN opt expect OlapScanNode, but we get " + child);
((OlapScanNode) child).setUseTopnOpt(true);
OlapScanNode scanNode = ((OlapScanNode) child);
scanNode.setUseTopnOpt(true);
}
// push sort to scan opt
if (sortNode.getChild(0) instanceof OlapScanNode) {
OlapScanNode scanNode = ((OlapScanNode) sortNode.getChild(0));
if (checkPushSort(sortNode, scanNode.getOlapTable())) {
SortInfo sortInfo = sortNode.getSortInfo();
scanNode.setSortInfo(sortInfo);
scanNode.getSortInfo().setSortTupleSlotExprs(sortNode.getResolvedTupleExprs());
for (Expr expr : sortInfo.getOrderingExprs()) {
scanNode.getSortInfo().addMaterializedOrderingExpr(expr);
}
if (sortNode.getOffset() > 0) {
scanNode.setSortLimit(sortNode.getLimit() + sortNode.getOffset());
} else {
scanNode.setSortLimit(sortNode.getLimit());
}
}
}
addPlanRoot(currentFragment, sortNode, topN);
} else {
@ -1094,6 +1113,60 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
return currentFragment;
}
/**
* topN opt: using storage data ordering to accelerate topn operation.
* refer pr: optimize topn query if order by columns is prefix of sort keys of table (#10694)
*/
public boolean checkPushSort(SortNode sortNode, OlapTable olapTable) {
// Ensure limit is less then threshold
if (sortNode.getLimit() <= 0
|| sortNode.getLimit() > ConnectContext.get().getSessionVariable().topnOptLimitThreshold) {
return false;
}
// Ensure all isAscOrder is same, ande length != 0.
// Can't be zorder.
if (sortNode.getSortInfo().getIsAscOrder().stream().distinct().count() != 1
|| olapTable.isZOrderSort()) {
return false;
}
// Tablet's order by key only can be the front part of schema.
// Like: schema: a.b.c.d.e.f.g order by key: a.b.c (no a,b,d)
// Do **prefix match** to check if order by key can be pushed down.
// olap order by key: a.b.c.d
// sort key: (a) (a,b) (a,b,c) (a,b,c,d) is ok
// (a,c) (a,c,d), (a,c,b) (a,c,f) (a,b,c,d,e)is NOT ok
List<Expr> sortExprs = sortNode.getSortInfo().getOrderingExprs();
List<Boolean> nullsFirsts = sortNode.getSortInfo().getNullsFirst();
List<Boolean> isAscOrders = sortNode.getSortInfo().getIsAscOrder();
if (sortExprs.size() > olapTable.getDataSortInfo().getColNum()) {
return false;
}
for (int i = 0; i < sortExprs.size(); i++) {
// table key.
Column tableKey = olapTable.getFullSchema().get(i);
// sort slot.
Expr sortExpr = sortExprs.get(i);
if (sortExpr instanceof SlotRef) {
SlotRef slotRef = (SlotRef) sortExpr;
if (tableKey.equals(slotRef.getColumn())) {
// ORDER BY DESC NULLS FIRST can not be optimized to only read file tail,
// since NULLS is at file head but data is at tail
if (tableKey.isAllowNull() && nullsFirsts.get(i) && !isAscOrders.get(i)) {
return false;
}
} else {
return false;
}
} else {
return false;
}
}
return true;
}
private SortNode translateSortNode(AbstractPhysicalSort<? extends Plan> sort, PlanNode childNode,
PlanTranslatorContext context) {
List<Expr> oldOrderingExprList = Lists.newArrayList();