database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](catalog) set timeout for split fetch (#39346) (#39624)

Browse Source 
bp #39346
This commit is contained in:
Mingyu Chen
2024-08-20 21:59:55 +08:00
committed by GitHub
parent 0e21dba817
commit a3fd13fee6
4 changed files with 18 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
fe/fe-core/src/main/java/org/apache/doris/datasource/SplitSource.java
View File

@ -73,7 +73,7 @@ public class SplitSource {
}
List<TScanRangeLocations> scanRanges = Lists.newArrayListWithExpectedSize(maxBatchSize);
long startTime = System.currentTimeMillis();
while (scanRanges.size() < maxBatchSize) {
while (scanRanges.size() < maxBatchSize && System.currentTimeMillis() - startTime < maxWaitTime) {
BlockingQueue<Collection<TScanRangeLocations>> splits = splitAssignment.getAssignedSplits(backend);
if (splits == null) {
isLastBatch.set(true);
@ -92,10 +92,15 @@ public class SplitSource {
break;
}
} catch (InterruptedException e) {
throw new UserException("Failed to get next batch of splits", e);
throw new UserException(e.getMessage(), e);
}
}
}
if (scanRanges.isEmpty() && !isLastBatch.get()) {
// This is timeout
throw new UserException("Timeout. Max wait time(ms): " + maxWaitTime);
}
return scanRanges;
}
}

6
fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
View File

@ -976,10 +976,14 @@ public class FrontendServiceImpl implements FrontendService.Iface {
try {
List<TScanRangeLocations> locations = splitSource.getNextBatch(request.getMaxNumSplits());
result.setSplits(locations);
result.status = new TStatus(TStatusCode.OK);
return result;
} catch (Exception e) {
throw new TException("Failed to get split source " + request.getSplitSourceId(), e);
LOG.warn("failed to fetch split batch with source id {}", request.getSplitSourceId(), e);
result.status = new TStatus(TStatusCode.INTERNAL_ERROR);
result.status.addToErrorMsgs(e.getMessage());
}
return result;
}
@Override