database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[pipelineX](file scan) Improve parallel tasks if ignore data distribution (#31328)

Browse Source
This commit is contained in:
Gabriel
2024-02-24 11:34:52 +08:00
committed by yiguolei
parent aee49adf1e
commit 8001f73e52
2 changed files with 15 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
View File

@ -503,6 +503,16 @@ public abstract class FileQueryScanNode extends FileScanNode {
return i;
}
@Override
public int getNumInstances() {
if (ConnectContext.get() != null
&& ConnectContext.get().getSessionVariable().getEnablePipelineXEngine()
&& ConnectContext.get().getSessionVariable().isIgnoreStorageDataDistribution()) {
return ConnectContext.get().getSessionVariable().getParallelExecInstanceNum();
}
return scanRangeLocations.size();
}
protected abstract TFileType getLocationType() throws UserException;
protected abstract TFileType getLocationType(String location) throws UserException;

5
fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
View File

@ -143,4 +143,9 @@ public class TVFScanNode extends FileQueryScanNode {
}
return splits;
}
@Override
public int getNumInstances() {
return scanRangeLocations.size();
}
}