database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Pick "[enhance](Cooldown) Use config to control whether use cooldown replica for scanning first" (#38322)

Browse Source 
## Proposed changes

<!--Describe your changes.-->

Same as master #37492
This commit is contained in:
AlexYue
2024-07-25 12:17:38 +08:00
committed by GitHub
parent 70cde39fe0
commit e396f853a0
2 changed files with 25 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fe/fe-common/src/main/java/org/apache/doris/common/Config.java
View File

@ -2735,6 +2735,9 @@ public class Config extends ConfigBase {
"Stream_Load When importing, the maximum length of label is limited"})
public static int label_regex_length = 128;
@ConfField(mutable = true)
public static boolean enable_cooldown_replica_affinity = true;
//==========================================================================
// end of cloud config
//==========================================================================

42
fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
View File

@ -836,27 +836,29 @@ public class OlapScanNode extends ScanNode {
}
}
final long coolDownReplicaId = tablet.getCooldownReplicaId();
// we prefer to query using cooldown replica to make sure the cache is fully utilized
// for example: consider there are 3BEs(A,B,C) and each has one replica for tablet X. and X
// is now under cooldown
// first time we choose BE A, and A will download data into cache while the other two's cache is empty
// second time we choose BE B, this time B will be cached, C is still empty
// third time we choose BE C, after this time all replica is cached
// but it means we will do 3 S3 IO to get the data which will bring 3 slow query
if (-1L != coolDownReplicaId) {
final Optional<Replica> replicaOptional = replicas.stream()
.filter(r -> r.getId() == coolDownReplicaId).findAny();
replicaOptional.ifPresent(
r -> {
Backend backend = Env.getCurrentSystemInfo()
.getBackend(r.getBackendId());
if (backend != null && backend.isAlive()) {
replicas.clear();
replicas.add(r);
if (Config.enable_cooldown_replica_affinity) {
final long coolDownReplicaId = tablet.getCooldownReplicaId();
// we prefer to query using cooldown replica to make sure the cache is fully utilized
// for example: consider there are 3BEs(A,B,C) and each has one replica for tablet X. and X
// is now under cooldown
// first time we choose BE A, and A will download data into cache while the other two's cache is empty
// second time we choose BE B, this time B will be cached, C is still empty
// third time we choose BE C, after this time all replica is cached
// but it means we will do 3 S3 IO to get the data which will bring 3 slow query
if (-1L != coolDownReplicaId) {
final Optional<Replica> replicaOptional = replicas.stream()
.filter(r -> r.getId() == coolDownReplicaId).findAny();
replicaOptional.ifPresent(
r -> {
Backend backend = Env.getCurrentSystemInfo()
.getBackend(r.getBackendId());
if (backend != null && backend.isAlive()) {
replicas.clear();
replicas.add(r);
}
}
}
);
);
}
}
boolean tabletIsNull = true;
boolean collectedStat = false;