From 8001f73e521b05b48bf2e535cf1284d59944c899 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 24 Feb 2024 11:34:52 +0800 Subject: [PATCH] [pipelineX](file scan) Improve parallel tasks if ignore data distribution (#31328) --- .../org/apache/doris/datasource/FileQueryScanNode.java | 10 ++++++++++ .../doris/datasource/tvf/source/TVFScanNode.java | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index 6c60692ea2..7afb04831c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -503,6 +503,16 @@ public abstract class FileQueryScanNode extends FileScanNode { return i; } + @Override + public int getNumInstances() { + if (ConnectContext.get() != null + && ConnectContext.get().getSessionVariable().getEnablePipelineXEngine() + && ConnectContext.get().getSessionVariable().isIgnoreStorageDataDistribution()) { + return ConnectContext.get().getSessionVariable().getParallelExecInstanceNum(); + } + return scanRangeLocations.size(); + } + protected abstract TFileType getLocationType() throws UserException; protected abstract TFileType getLocationType(String location) throws UserException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java index 83aa49f5ab..dcd248a978 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java @@ -143,4 +143,9 @@ public class TVFScanNode extends FileQueryScanNode { } return splits; } + + @Override + public int getNumInstances() { + return scanRangeLocations.size(); + } }