database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[cherrypick] Fix profile eject (#38095)

Browse Source 
pick https://github.com/apache/doris/pull/38030
This commit is contained in:
zhiqiang
2024-07-18 23:38:56 +08:00
committed by GitHub
parent 4b31e52b24
commit 426493a7f1
2 changed files with 24 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
View File

@ -153,22 +153,27 @@ public class ProfileManager {
LOG.debug("Add execution profile {} to profile manager",
DebugUtil.printId(executionProfile.getQueryId()));
}
// Check if there are some query profiles that not finish collecting, should
// remove them to release memory.
// This branch has two purposes:
// 1. discard profile collecting if its collection not finished in 5 seconds after query finished.
// 2. prevent execution profile from leakage. If we have too many execution profiles in memory,
// we will remove execution profiles of query that has finished in 5 seconds ago.
if (queryIdToExecutionProfiles.size() > 2 * Config.max_query_profile_num) {
List<ExecutionProfile> finishOrExpireExecutionProfiles = Lists.newArrayList();
for (ExecutionProfile tmpProfile : queryIdToExecutionProfiles.values()) {
if (System.currentTimeMillis() - tmpProfile.getQueryFinishTime()
> Config.profile_async_collect_expire_time_secs * 1000) {
boolean queryFinishedLongEnough = tmpProfile.getQueryFinishTime() > 0
&& System.currentTimeMillis() - tmpProfile.getQueryFinishTime()
> Config.profile_async_collect_expire_time_secs * 1000;
if (queryFinishedLongEnough) {
finishOrExpireExecutionProfiles.add(tmpProfile);
}
}
StringBuilder stringBuilder = new StringBuilder();
for (ExecutionProfile tmp : finishOrExpireExecutionProfiles) {
stringBuilder.append(DebugUtil.printId(tmp.getQueryId())).append(",");
queryIdToExecutionProfiles.remove(tmp.getQueryId());
if (LOG.isDebugEnabled()) {
LOG.debug("Remove expired execution profile {}", DebugUtil.printId(tmp.getQueryId()));
}
}
LOG.warn("Remove expired execution profiles {}", stringBuilder.toString());
}
} finally {
writeLock.unlock();
@ -203,10 +208,19 @@ public class ProfileManager {
ProfileElement profileElementRemoved = queryIdToProfileMap.remove(queryIdDeque.getFirst());
// If the Profile object is removed from manager, then related execution profile is also useless.
if (profileElementRemoved != null) {
StringBuilder sb = new StringBuilder();
for (ExecutionProfile executionProfile : profileElementRemoved.profile.getExecutionProfiles()) {
sb.append(DebugUtil.printId(executionProfile.getQueryId())).append(",");
this.queryIdToExecutionProfiles.remove(executionProfile.getQueryId());
}
LOG.warn("Remove expired profile {}, execution profiles {},"
+ " queryIdDeque size {}, profile count {},"
+ " execution profile count {} max_query_profile_num {}",
profileElementRemoved.profile.getSummaryProfile().getProfileId(),
sb.toString(), queryIdDeque.size(), queryIdToProfileMap.size(),
queryIdToExecutionProfiles.size(), Config.max_query_profile_num);
}
queryIdDeque.removeFirst();
}
queryIdDeque.addLast(key);

5
fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java
View File

@ -99,10 +99,11 @@ public final class QeProcessorImpl implements QeProcessor {
if (result != null) {
throw new UserException("queryId " + queryId + " already exists");
}
// Should add the execution profile to profile manager, BE will report the profile to FE and FE
// will update it in ProfileManager
ProfileManager.getInstance().addExecutionProfile(info.getCoord().getExecutionProfile());
if (info.coord.getQueryOptions().enable_profile) {
ProfileManager.getInstance().addExecutionProfile(info.getCoord().getExecutionProfile());
}
}
@Override