database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Enhance](ComputeNode) change logic of BeSelectionPolicy.getCandidateBackends (#16737)

Browse Source 
The previous logic is how many cn can be returned at most. Instead,
if the number of cn is less than expectBeNum, need to use mix to fill in,
until the number of cn equals with expectBeNum or mix nodes are also used up
This commit is contained in:
zhangdong
2023-02-16 10:31:24 +08:00
committed by GitHub
parent bd3ea233f8
commit 118ce9cb16
3 changed files with 12 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/planner/external/BackendPolicy.java vendored
View File

@ -58,7 +58,7 @@ public class BackendPolicy {
.needLoadAvailable()
.addTags(tags)
.preferComputeNode()
.assignCandidateNum(Config.backend_num_for_federation)
.assignExpectBeNum(Config.backend_num_for_federation)
.build();
backends.addAll(policy.getCandidateBackends(Env.getCurrentSystemInfo().getIdToBackend().values()));
if (backends.isEmpty()) {

16
fe/fe-core/src/main/java/org/apache/doris/system/BeSelectionPolicy.java
View File

@ -47,7 +47,7 @@ public class BeSelectionPolicy {
public boolean allowOnSameHost = false;
public boolean preferComputeNode = false;
public int candidateNum = Integer.MAX_VALUE;
public int expectBeNum = 0;
private BeSelectionPolicy() {
@ -105,8 +105,8 @@ public class BeSelectionPolicy {
return this;
}
public Builder assignCandidateNum(int candidateNum) {
policy.candidateNum = candidateNum;
public Builder assignExpectBeNum(int expectBeNum) {
policy.expectBeNum = expectBeNum;
return this;
}
@ -141,25 +141,21 @@ public class BeSelectionPolicy {
public List<Backend> getCandidateBackends(ImmutableCollection<Backend> backends) {
List<Backend> filterBackends = backends.stream().filter(this::isMatch).collect(Collectors.toList());
Collections.shuffle(filterBackends);
List<Backend> candidates = new ArrayList<>();
if (preferComputeNode) {
int num = 0;
// pick compute node first
for (Backend backend : filterBackends) {
if (backend.isComputeNode()) {
if (num >= candidateNum) {
break;
}
candidates.add(backend);
num++;
}
}
// fill with some mix node.
if (num < candidateNum) {
if (num < expectBeNum) {
for (Backend backend : filterBackends) {
if (backend.isMixNode()) {
if (num >= candidateNum) {
if (num >= expectBeNum) {
break;
}
candidates.add(backend);
@ -170,7 +166,7 @@ public class BeSelectionPolicy {
} else {
candidates.addAll(filterBackends);
}
Collections.shuffle(candidates);
return candidates;
}

10
fe/fe-core/src/test/java/org/apache/doris/system/SystemInfoServiceTest.java
View File

@ -239,11 +239,11 @@ public class SystemInfoServiceTest {
Assert.assertEquals(1, infoService.selectBackendIdsByPolicy(policy02, 1).size());
BeSelectionPolicy policy03 = new BeSelectionPolicy.Builder().addTags(Sets.newHashSet(taga))
.setStorageMedium(TStorageMedium.HDD).preferComputeNode().assignCandidateNum(0).build();
Assert.assertEquals(0, infoService.selectBackendIdsByPolicy(policy03, 1).size());
.setStorageMedium(TStorageMedium.HDD).preferComputeNode().assignExpectBeNum(0).build();
Assert.assertEquals(1, infoService.selectBackendIdsByPolicy(policy03, 1).size());
BeSelectionPolicy policy04 = new BeSelectionPolicy.Builder().addTags(Sets.newHashSet(taga))
.setStorageMedium(TStorageMedium.HDD).preferComputeNode().assignCandidateNum(1).build();
.setStorageMedium(TStorageMedium.HDD).preferComputeNode().assignExpectBeNum(1).build();
Assert.assertEquals(1, infoService.selectBackendIdsByPolicy(policy04, 1).size());
// one compute node and two mix node
@ -264,11 +264,11 @@ public class SystemInfoServiceTest {
Assert.assertEquals(0, infoService.selectBackendIdsByPolicy(policy05, 3).size());
BeSelectionPolicy policy06 = new BeSelectionPolicy.Builder().addTags(Sets.newHashSet(taga))
.setStorageMedium(TStorageMedium.HDD).preferComputeNode().assignCandidateNum(2).build();
.setStorageMedium(TStorageMedium.HDD).preferComputeNode().assignExpectBeNum(2).build();
Assert.assertEquals(2, infoService.selectBackendIdsByPolicy(policy06, 2).size());
BeSelectionPolicy policy07 = new BeSelectionPolicy.Builder().addTags(Sets.newHashSet(taga))
.setStorageMedium(TStorageMedium.HDD).preferComputeNode().assignCandidateNum(3).build();
.setStorageMedium(TStorageMedium.HDD).preferComputeNode().assignExpectBeNum(3).build();
Assert.assertEquals(3, infoService.selectBackendIdsByPolicy(policy07, 3).size());
}