database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

(fix)[multi-catalog][nereids] Reset ExternalFileScanNode required slots after Nereids planner do projection. #16549

Browse Source 
The new Nereids planner do column projection after creating scan node. For ExternalFileScanNode, this may cause the columns in required_slots mismatch with the slots after projection. This pr is to reset the required_slots after projection.
This commit is contained in:
Jibing-Li
2023-02-10 11:28:01 +08:00
committed by GitHub
parent 1b3902baa2
commit e9cd1d64ed
3 changed files with 46 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View File

@ -38,6 +38,8 @@ import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.external.ExternalTable;
import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.Util;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.properties.DistributionSpecAny;
import org.apache.doris.nereids.properties.DistributionSpecGather;
@ -128,8 +130,8 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.Collection;
@ -153,7 +155,7 @@ import java.util.stream.Stream;
* </STRONG>
*/
public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, PlanTranslatorContext> {
private static final Logger LOG = LoggerFactory.getLogger(PhysicalPlanTranslator.class);
private static final Logger LOG = LogManager.getLogger(PhysicalPlanTranslator.class);
/**
* Translate Nereids Physical Plan tree to Stale Planner PlanFragment tree.
@ -1098,7 +1100,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
inputPlanNode.setProjectList(execExprList);
inputPlanNode.setOutputTupleDesc(tupleDescriptor);
if (inputPlanNode instanceof OlapScanNode) {
if (inputPlanNode instanceof ScanNode) {
updateChildSlotsMaterialization(inputPlanNode, requiredSlotIdList, context);
return inputFragment;
}
@ -1126,6 +1128,14 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
context.getDescTable()
.getTupleDesc(execPlan.getTupleIds().get(0)).getSlots().get(0).setIsMaterialized(true);
}
if (execPlan instanceof ScanNode) {
try {
((ScanNode) execPlan).updateRequiredSlots();
} catch (UserException e) {
Util.logAndThrowRuntimeException(LOG,
"User Exception while reset external file scan node contexts.", e);
}
}
}
@Override

9
fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
View File

@ -125,6 +125,15 @@ public abstract class ScanNode extends PlanNode {
*/
public abstract List<TScanRangeLocations> getScanRangeLocations(long maxScanRangeLength);
/**
* Update required_slots in scan node contexts. This is called after Nereids planner do the projection.
* In the projection process, some slots may be removed. So call this to update the slots info.
* Currently, it is only used by ExternalFileScanNode, add the interface here to keep the Nereids code clean.
*/
public void updateRequiredSlots() throws UserException {
return;
}
// TODO(ML): move it into PrunerOptimizer
public void computeColumnFilter() {
for (Column column : desc.getTable().getBaseSchema()) {

23
fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java vendored
View File

@ -241,6 +241,29 @@ public class ExternalFileScanNode extends ExternalScanNode {
}
}
/**
* Reset required_slots in contexts. This is called after Nereids planner do the projection.
* In the projection process, some slots may be removed. So call this to update the slots info.
*/
@Override
public void updateRequiredSlots() throws UserException {
for (int i = 0; i < contexts.size(); i++) {
ParamCreateContext context = contexts.get(i);
FileScanProviderIf scanProvider = scanProviders.get(i);
context.params.unsetRequiredSlots();
for (SlotDescriptor slot : desc.getSlots()) {
if (!slot.isMaterialized()) {
continue;
}
TFileScanSlotInfo slotInfo = new TFileScanSlotInfo();
slotInfo.setSlotId(slot.getId().asInt());
slotInfo.setIsFileSlot(!scanProvider.getPathPartitionKeys().contains(slot.getColumn().getName()));
context.params.addToRequiredSlots(slotInfo);
}
}
}
private void initHMSExternalTable(HMSExternalTable hmsTable) throws UserException {
Preconditions.checkNotNull(hmsTable);