database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](scan) read scan ranges in the order of partitions (#33515) (#33657)

Browse Source 
backport: #33515
This commit is contained in:
Ashin Gau
2024-04-15 16:20:12 +08:00
committed by yiguolei
parent d0394b7f89
commit 2cd4012541
4 changed files with 79 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
fe/fe-core/src/test/java/org/apache/doris/planner/FederationBackendPolicyTest.java
View File

@ -288,6 +288,21 @@ public class FederationBackendPolicyTest {
}
public static void sortSplits(List<Split> splits) {
splits.sort((split1, split2) -> {
int pathComparison = split1.getPathString().compareTo(split2.getPathString());
if (pathComparison != 0) {
return pathComparison;
}
int startComparison = Long.compare(split1.getStart(), split2.getStart());
if (startComparison != 0) {
return startComparison;
}
return Long.compare(split1.getLength(), split2.getLength());
});
}
@Test
public void testGenerateRandomly() throws UserException {
SystemInfoService service = new SystemInfoService();
@ -367,7 +382,7 @@ public class FederationBackendPolicyTest {
List<Split> totalSplits = new ArrayList<>();
totalSplits.addAll(remoteSplits);
totalSplits.addAll(localSplits);
Collections.shuffle(totalSplits);
sortSplits(totalSplits);
Multimap<Backend, Split> assignment = policy.computeScanRangeAssignment(totalSplits);
if (i == 0) {
result = ArrayListMultimap.create(assignment);
@ -489,7 +504,7 @@ public class FederationBackendPolicyTest {
List<Split> totalSplits = new ArrayList<>();
totalSplits.addAll(remoteSplits);
totalSplits.addAll(localSplits);
Collections.shuffle(totalSplits);
sortSplits(totalSplits);
Multimap<Backend, Split> assignment = policy.computeScanRangeAssignment(totalSplits);
if (i == 0) {
result = ArrayListMultimap.create(assignment);