database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](nereids)print nereids node id in explain (#29238)

Browse Source 
* print nereids id in explain
This commit is contained in:
minghong
2024-01-09 16:15:59 +08:00
committed by yiguolei
parent e4707154fa
commit da182a8b6f
6 changed files with 68 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View File

@ -283,7 +283,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
}
}
ExchangeNode exchangeNode = new ExchangeNode(distribute.translatePlanNodeId(), inputFragment.getPlanRoot());
ExchangeNode exchangeNode = new ExchangeNode(context.nextPlanNodeId(), inputFragment.getPlanRoot());
updateLegacyPlanIdToPhysicalPlan(exchangeNode, distribute);
List<ExprId> validOutputIds = distribute.getOutputExprIds();
if (distribute.child() instanceof PhysicalHashAggregate) {
@ -463,13 +463,13 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
if (table instanceof HMSExternalTable) {
switch (((HMSExternalTable) table).getDlaType()) {
case HUDI:
scanNode = new HudiScanNode(fileScan.translatePlanNodeId(), tupleDescriptor, false);
scanNode = new HudiScanNode(context.nextPlanNodeId(), tupleDescriptor, false);
break;
case ICEBERG:
scanNode = new IcebergScanNode(fileScan.translatePlanNodeId(), tupleDescriptor, false);
scanNode = new IcebergScanNode(context.nextPlanNodeId(), tupleDescriptor, false);
break;
case HIVE:
scanNode = new HiveScanNode(fileScan.translatePlanNodeId(), tupleDescriptor, false);
scanNode = new HiveScanNode(context.nextPlanNodeId(), tupleDescriptor, false);
HiveScanNode hiveScanNode = (HiveScanNode) scanNode;
hiveScanNode.setSelectedPartitions(fileScan.getSelectedPartitions());
if (fileScan.getTableSample().isPresent()) {
@ -481,15 +481,15 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
throw new RuntimeException("do not support DLA type " + ((HMSExternalTable) table).getDlaType());
}
} else if (table instanceof IcebergExternalTable) {
scanNode = new IcebergScanNode(fileScan.translatePlanNodeId(), tupleDescriptor, false);
scanNode = new IcebergScanNode(context.nextPlanNodeId(), tupleDescriptor, false);
} else if (table instanceof PaimonExternalTable) {
scanNode = new PaimonScanNode(fileScan.translatePlanNodeId(), tupleDescriptor, false);
scanNode = new PaimonScanNode(context.nextPlanNodeId(), tupleDescriptor, false);
} else if (table instanceof MaxComputeExternalTable) {
scanNode = new MaxComputeScanNode(fileScan.translatePlanNodeId(), tupleDescriptor, false);
scanNode = new MaxComputeScanNode(context.nextPlanNodeId(), tupleDescriptor, false);
} else {
throw new RuntimeException("do not support table type " + table.getType());
}
scanNode.setNereidsId(fileScan.getId());
scanNode.addConjuncts(translateToLegacyConjuncts(fileScan.getConjuncts()));
scanNode.setPushDownAggNoGrouping(context.getRelationPushAggOp(fileScan.getRelationId()));
@ -528,8 +528,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
ArrayList<TupleId> tupleIds = new ArrayList<>();
tupleIds.add(tupleDescriptor.getId());
EmptySetNode emptySetNode = new EmptySetNode(emptyRelation.translatePlanNodeId(), tupleIds);
EmptySetNode emptySetNode = new EmptySetNode(context.nextPlanNodeId(), tupleIds);
emptySetNode.setNereidsId(emptyRelation.getId());
PlanFragment planFragment = createPlanFragment(emptySetNode,
DataPartition.UNPARTITIONED, emptyRelation);
context.addPlanFragment(planFragment);
@ -542,7 +542,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
List<Slot> slots = esScan.getOutput();
ExternalTable table = esScan.getTable();
TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table, context);
EsScanNode esScanNode = new EsScanNode(esScan.translatePlanNodeId(), tupleDescriptor, true);
EsScanNode esScanNode = new EsScanNode(context.nextPlanNodeId(), tupleDescriptor, true);
esScanNode.setNereidsId(esScan.getId());
esScanNode.addConjuncts(translateToLegacyConjuncts(esScan.getConjuncts()));
Utils.execWithUncheckedException(esScanNode::init);
context.addScanNode(esScanNode);
@ -564,8 +565,9 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
List<Slot> slots = jdbcScan.getOutput();
TableIf table = jdbcScan.getTable();
TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table, context);
JdbcScanNode jdbcScanNode = new JdbcScanNode(jdbcScan.translatePlanNodeId(), tupleDescriptor,
JdbcScanNode jdbcScanNode = new JdbcScanNode(context.nextPlanNodeId(), tupleDescriptor,
table instanceof JdbcExternalTable);
jdbcScanNode.setNereidsId(jdbcScan.getId());
jdbcScanNode.addConjuncts(translateToLegacyConjuncts(jdbcScan.getConjuncts()));
Utils.execWithUncheckedException(jdbcScanNode::init);
context.addScanNode(jdbcScanNode);
@ -587,8 +589,9 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
List<Slot> slots = odbcScan.getOutput();
TableIf table = odbcScan.getTable();
TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table, context);
OdbcScanNode odbcScanNode = new OdbcScanNode(odbcScan.translatePlanNodeId(), tupleDescriptor,
OdbcScanNode odbcScanNode = new OdbcScanNode(context.nextPlanNodeId(), tupleDescriptor,
(OdbcTable) table);
odbcScanNode.setNereidsId(odbcScan.getId());
odbcScanNode.addConjuncts(translateToLegacyConjuncts(odbcScan.getConjuncts()));
Utils.execWithUncheckedException(odbcScanNode::init);
context.addScanNode(odbcScanNode);
@ -615,7 +618,9 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
if (olapScan.getSelectedIndexId() != olapScan.getTable().getBaseIndexId()) {
generateTupleDesc(olapScan.getBaseOutputs(), olapTable, context);
}
OlapScanNode olapScanNode = new OlapScanNode(olapScan.translatePlanNodeId(), tupleDescriptor, "OlapScanNode");
OlapScanNode olapScanNode = new OlapScanNode(context.nextPlanNodeId(), tupleDescriptor, "OlapScanNode");
olapScanNode.setNereidsId(olapScan.getId());
// TODO: move all node set cardinality into one place
if (olapScan.getStats() != null) {
// NOTICE: we should not set stats row count
@ -724,7 +729,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
slotDescriptor.setIsNullable(slots.get(i).nullable());
}
UnionNode unionNode = new UnionNode(oneRowRelation.translatePlanNodeId(), oneRowTuple.getId());
UnionNode unionNode = new UnionNode(context.nextPlanNodeId(), oneRowTuple.getId());
unionNode.setNereidsId(oneRowRelation.getId());
unionNode.setCardinality(1L);
unionNode.addConstExprList(legacyExprs);
unionNode.finalizeForNereids(oneRowTuple.getSlots(), new ArrayList<>());
@ -746,10 +752,11 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
SchemaScanNode scanNode = null;
if (BackendPartitionedSchemaScanNode.isBackendPartitionedSchemaTable(
table.getName())) {
scanNode = new BackendPartitionedSchemaScanNode(schemaScan.translatePlanNodeId(), tupleDescriptor);
scanNode = new BackendPartitionedSchemaScanNode(context.nextPlanNodeId(), tupleDescriptor);
} else {
scanNode = new SchemaScanNode(schemaScan.translatePlanNodeId(), tupleDescriptor);
scanNode = new SchemaScanNode(context.nextPlanNodeId(), tupleDescriptor);
}
scanNode.setNereidsId(schemaScan.getId());
SchemaScanNode finalScanNode = scanNode;
context.getRuntimeTranslator().ifPresent(
runtimeFilterGenerator -> runtimeFilterGenerator.getContext().getTargetListByScan(schemaScan)
@ -771,7 +778,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
TupleDescriptor tupleDescriptor = generateTupleDesc(slots, tvfRelation.getFunction().getTable(), context);
TableValuedFunctionIf catalogFunction = tvfRelation.getFunction().getCatalogFunction();
ScanNode scanNode = catalogFunction.getScanNode(tvfRelation.translatePlanNodeId(), tupleDescriptor);
ScanNode scanNode = catalogFunction.getScanNode(context.nextPlanNodeId(), tupleDescriptor);
scanNode.setNereidsId(tvfRelation.getId());
Utils.execWithUncheckedException(scanNode::init);
context.getRuntimeTranslator().ifPresent(
runtimeFilterGenerator -> runtimeFilterGenerator.getContext().getTargetListByScan(tvfRelation)
@ -850,11 +858,12 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
boolean isPartial = aggregate.getAggregateParam().aggMode.productAggregateBuffer;
AggregateInfo aggInfo = AggregateInfo.create(execGroupingExpressions, execAggregateFunctions,
aggFunOutputIds, isPartial, outputTupleDesc, outputTupleDesc, aggregate.getAggPhase().toExec());
AggregationNode aggregationNode = new AggregationNode(aggregate.translatePlanNodeId(),
AggregationNode aggregationNode = new AggregationNode(context.nextPlanNodeId(),
inputPlanFragment.getPlanRoot(), aggInfo);
aggregationNode.setDistributeExprLists(distributeExprLists);
aggregationNode.setNereidsId(aggregate.getId());
if (!aggregate.getAggMode().isFinalPhase) {
aggregationNode.unsetNeedsFinalize();
}
@ -949,10 +958,11 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
PlanFragment currentFragment = assertNumRows.child().accept(this, context);
List<List<Expr>> distributeExprLists = getDistributeExprs(assertNumRows.child());
// create assertNode
AssertNumRowsNode assertNumRowsNode = new AssertNumRowsNode(assertNumRows.translatePlanNodeId(),
AssertNumRowsNode assertNumRowsNode = new AssertNumRowsNode(context.nextPlanNodeId(),
currentFragment.getPlanRoot(),
ExpressionTranslator.translateAssert(assertNumRows.getAssertNumRowsElement()));
assertNumRowsNode.setDistributeExprLists(distributeExprLists);
assertNumRowsNode.setNereidsId(assertNumRows.getId());
addPlanRoot(currentFragment, assertNumRowsNode, assertNumRows);
return currentFragment;
}
@ -1054,7 +1064,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
PlanNode planNode = inputFragment.getPlanRoot();
if (planNode instanceof ExchangeNode || planNode instanceof SortNode || planNode instanceof UnionNode) {
// the three nodes don't support conjuncts, need create a SelectNode to filter data
SelectNode selectNode = new SelectNode(filter.translatePlanNodeId(), planNode);
SelectNode selectNode = new SelectNode(context.nextPlanNodeId(), planNode);
selectNode.setNereidsId(filter.getId());
addConjunctsToPlanNode(filter, selectNode, context);
addPlanRoot(inputFragment, selectNode, filter);
} else {
@ -1088,8 +1099,9 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
.flatMap(List::stream)
.map(SlotDescriptor::getId)
.collect(Collectors.toList());
TableFunctionNode tableFunctionNode = new TableFunctionNode(generate.translatePlanNodeId(),
TableFunctionNode tableFunctionNode = new TableFunctionNode(context.nextPlanNodeId(),
currentFragment.getPlanRoot(), tupleDescriptor.getId(), functionCalls, outputSlotIds);
tableFunctionNode.setNereidsId(generate.getId());
addPlanRoot(currentFragment, tableFunctionNode, generate);
return currentFragment;
}
@ -1167,9 +1179,10 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
.map(e -> ExpressionTranslator.translate(e, context))
.collect(Collectors.toList());
HashJoinNode hashJoinNode = new HashJoinNode(hashJoin.translatePlanNodeId(), leftPlanRoot,
HashJoinNode hashJoinNode = new HashJoinNode(context.nextPlanNodeId(), leftPlanRoot,
rightPlanRoot, JoinType.toJoinOperator(joinType), execEqConjuncts, Lists.newArrayList(),
null, null, null, hashJoin.isMarkJoin());
hashJoinNode.setNereidsId(hashJoin.getId());
hashJoinNode.setDistributeExprLists(distributeExprLists);
PlanFragment currentFragment = connectJoinNode(hashJoinNode, leftFragment, rightFragment, context, hashJoin);
@ -1415,9 +1428,11 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
.collect(Collectors.toList());
JoinType joinType = nestedLoopJoin.getJoinType();
NestedLoopJoinNode nestedLoopJoinNode = new NestedLoopJoinNode(nestedLoopJoin.translatePlanNodeId(),
NestedLoopJoinNode nestedLoopJoinNode = new NestedLoopJoinNode(context.nextPlanNodeId(),
leftFragmentPlanRoot, rightFragmentPlanRoot, tupleIds, JoinType.toJoinOperator(joinType),
null, null, null, nestedLoopJoin.isMarkJoin());
nestedLoopJoinNode.setNereidsId(nestedLoopJoin.getId());
nestedLoopJoinNode.setDistributeExprLists(distributeExprLists);
if (nestedLoopJoin.getStats() != null) {
nestedLoopJoinNode.setCardinality((long) nestedLoopJoin.getStats().getRowCount());
@ -1752,14 +1767,15 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
SetOperationNode setOperationNode;
// create setOperationNode
if (setOperation instanceof PhysicalUnion) {
setOperationNode = new UnionNode(setOperation.translatePlanNodeId(), setTuple.getId());
setOperationNode = new UnionNode(context.nextPlanNodeId(), setTuple.getId());
} else if (setOperation instanceof PhysicalExcept) {
setOperationNode = new ExceptNode(setOperation.translatePlanNodeId(), setTuple.getId());
setOperationNode = new ExceptNode(context.nextPlanNodeId(), setTuple.getId());
} else if (setOperation instanceof PhysicalIntersect) {
setOperationNode = new IntersectNode(setOperation.translatePlanNodeId(), setTuple.getId());
setOperationNode = new IntersectNode(context.nextPlanNodeId(), setTuple.getId());
} else {
throw new RuntimeException("not support set operation type " + setOperation);
}
setOperationNode.setNereidsId(setOperation.getId());
setOperation.getRegularChildrenOutputs().stream()
.map(o -> o.stream()
@ -1983,9 +1999,10 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
.flatMap(Set::stream)
.collect(ImmutableSet.toImmutableSet());
RepeatNode repeatNode = new RepeatNode(repeat.translatePlanNodeId(),
RepeatNode repeatNode = new RepeatNode(context.nextPlanNodeId(),
inputPlanFragment.getPlanRoot(), groupingInfo, repeatSlotIdList,
allSlotId, repeat.computeVirtualSlotValues(sortedVirtualSlots));
repeatNode.setNereidsId(repeat.getId());
repeatNode.setDistributeExprLists(distributeExprLists);
addPlanRoot(inputPlanFragment, repeatNode, repeat);
updateLegacyPlanIdToPhysicalPlan(inputPlanFragment.getPlanRoot(), repeat);
@ -2060,7 +2077,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
// 4. generate AnalyticEvalNode
AnalyticEvalNode analyticEvalNode = new AnalyticEvalNode(
physicalWindow.translatePlanNodeId(),
context.nextPlanNodeId(),
inputPlanFragment.getPlanRoot(),
analyticFnCalls,
partitionExprs,
@ -2072,6 +2089,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
orderElementsIsNullableMatched,
bufferedTupleDesc
);
analyticEvalNode.setNereidsId(physicalWindow.getId());
analyticEvalNode.setDistributeExprLists(distributeExprLists);
PlanNode root = inputPlanFragment.getPlanRoot();
if (root instanceof SortNode) {
@ -2113,9 +2131,10 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
nullsFirstParams.add(k.isNullFirst());
});
SortInfo sortInfo = new SortInfo(orderingExprs, ascOrders, nullsFirstParams, sortTuple);
PartitionSortNode partitionSortNode = new PartitionSortNode(partitionTopN.translatePlanNodeId(), childNode,
PartitionSortNode partitionSortNode = new PartitionSortNode(context.nextPlanNodeId(), childNode,
partitionTopN.getFunction(), partitionExprs, sortInfo, partitionTopN.hasGlobalLimit(),
partitionTopN.getPartitionLimit(), partitionTopN.getPhase());
partitionSortNode.setNereidsId(partitionTopN.getId());
if (partitionTopN.getStats() != null) {
partitionSortNode.setCardinality((long) partitionTopN.getStats().getRowCount());
}
@ -2136,7 +2155,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
nullsFirstParams.add(k.isNullFirst());
});
SortInfo sortInfo = new SortInfo(orderingExprs, ascOrders, nullsFirstParams, sortTuple);
SortNode sortNode = new SortNode(sort.translatePlanNodeId(), childNode, sortInfo, sort instanceof PhysicalTopN);
SortNode sortNode = new SortNode(context.nextPlanNodeId(), childNode, sortInfo, sort instanceof PhysicalTopN);
sortNode.setNereidsId(sort.getId());
if (sort.getStats() != null) {
sortNode.setCardinality((long) sort.getStats().getRowCount());
}

9
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java
View File

@ -30,7 +30,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.util.MutableState;
import org.apache.doris.nereids.util.MutableState.EmptyMutableState;
import org.apache.doris.nereids.util.TreeStringUtils;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.statistics.Statistics;
import com.google.common.base.Preconditions;
@ -189,11 +188,7 @@ public abstract class AbstractPlan extends AbstractTreeNode<Plan> implements Pla
this.mutableState = this.mutableState.set(key, state);
}
/**
* used for PhysicalPlanTranslator only
* @return PlanNodeId
*/
public PlanNodeId translatePlanNodeId() {
return id.toPlanNodeId();
public int getId() {
return id.asInt();
}
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/ObjectId.java
View File

@ -20,7 +20,6 @@ package org.apache.doris.nereids.trees.plans;
import org.apache.doris.common.Id;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.planner.PlanNodeId;
/**
* relation id
@ -47,8 +46,4 @@ public class ObjectId extends Id<ObjectId> {
public String toString() {
return "ObjectId#" + id;
}
public PlanNodeId toPlanNodeId() {
return new PlanNodeId(id);
}
}

7
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java
View File

@ -120,10 +120,9 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca
@Override
public String toString() {
return Utils.toSqlString("PhysicalOlapScan[" + id.asInt() + "]" + getGroupIdWithPrefix(),
"table", table.getName(),
"stats", statistics,
"fr", getMutableState(AbstractPlan.FRAGMENT_ID)
return Utils.toSqlString("PhysicalOlapScan[" + relationId.asInt() + "]" + getGroupIdWithPrefix(),
"qualified", Utils.qualifiedName(qualifier, table.getName()),
"stats", statistics, "fr", getMutableState(AbstractPlan.FRAGMENT_ID)
);
}

12
fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
View File

@ -153,6 +153,8 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
protected List<Expr> projectList;
protected int nereidsId = -1;
private List<List<Expr>> distributeExprLists = new ArrayList<>();
protected PlanNode(PlanNodeId id, ArrayList<TupleId> tupleIds, String planNodeName,
@ -518,7 +520,11 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
// Print the current node
// The plan node header line will be prefixed by rootPrefix and the remaining details
// will be prefixed by detailPrefix.
expBuilder.append(rootPrefix + id.asInt() + ":" + planNodeName + "\n");
expBuilder.append(rootPrefix + id.asInt() + ":" + planNodeName);
if (nereidsId != -1) {
expBuilder.append("(" + nereidsId + ")");
}
expBuilder.append("\n");
expBuilder.append(getNodeExplainString(detailPrefix, detailLevel));
if (limit != -1) {
expBuilder.append(detailPrefix + "limit: " + limit + "\n");
@ -1205,4 +1211,8 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
public boolean pushDownAggNoGroupingCheckCol(FunctionCallExpr aggExpr, Column col) {
return false;
}
public void setNereidsId(int nereidsId) {
this.nereidsId = nereidsId;
}
}