database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](profile) Insert into add LoadChannel runtime profile (#18908)

Browse Source 
TabletSink and LoadChannel in BE are M: N relationship,
Every once in a while LoadChannel will randomly return its own runtime profile to a TabletSink, so usually all LoadChannel runtime profiles are saved on each TabletSink, and the timeliness of the same LoadChannel profile saved on different TabletSinks is different, and each TabletSink will periodically send fe reports all the LoadChannel profiles saved by itself, and ensures to update the latest LoadChannel profile according to the timestamp.
This commit is contained in:
Xinyi Zou
2023-04-24 09:41:57 +08:00
committed by GitHub
parent d2f50ce3f5
commit 8e4710079d
29 changed files with 265 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreeBuilder.java
View File

@ -132,7 +132,7 @@ public class ProfileTreeBuilder {
public void build() throws UserException {
reset();
checkProfile();
analyzeAndBuildFragmentTrees();
analyzeAndBuild();
assembleFragmentTrees();
}
@ -151,8 +151,33 @@ public class ProfileTreeBuilder {
}
}
private void analyzeAndBuildFragmentTrees() throws UserException {
private void analyzeAndBuild() throws UserException {
List<Pair<RuntimeProfile, Boolean>> childrenFragment = profileRoot.getChildList();
for (Pair<RuntimeProfile, Boolean> pair : childrenFragment) {
String name = pair.first.getName();
if (name.equals("Fragments")) {
analyzeAndBuildFragmentTrees(pair.first);
} else if (name.equals("LoadChannels")) {
analyzeAndBuildLoadChannels(pair.first);
} else {
throw new UserException("Invalid execution profile name: " + name);
}
}
}
private void analyzeAndBuildLoadChannels(RuntimeProfile loadChannelsProfile) throws UserException {
List<Pair<RuntimeProfile, Boolean>> childrenFragment = loadChannelsProfile.getChildList();
for (Pair<RuntimeProfile, Boolean> pair : childrenFragment) {
analyzeAndBuildLoadChannel(pair.first);
}
}
private void analyzeAndBuildLoadChannel(RuntimeProfile loadChannelsProfil) throws UserException {
// TODO, `show load profile` add load channel profile, or add `show load channel profile`.
}
private void analyzeAndBuildFragmentTrees(RuntimeProfile fragmentsProfile) throws UserException {
List<Pair<RuntimeProfile, Boolean>> childrenFragment = fragmentsProfile.getChildList();
for (Pair<RuntimeProfile, Boolean> pair : childrenFragment) {
analyzeAndBuildFragmentTree(pair.first);
}
@ -191,7 +216,7 @@ public class ProfileTreeBuilder {
fragmentTreeRoot = instanceTreeRoot;
}
// 2. Build tree for each single instance
// 3. Build tree for each single instance
int i = 0;
Map<String, ProfileTreeNode> instanceTrees = Maps.newHashMap();
for (Pair<RuntimeProfile, Boolean> pair : fragmentChildren) {

7
fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java
View File

@ -64,6 +64,8 @@ public class RuntimeProfile {
private String name;
private Long timestamp = -1L;
public RuntimeProfile(String name) {
this();
this.name = name;
@ -137,6 +139,11 @@ public class RuntimeProfile {
// preorder traversal, idx should be modified in the traversal process
private void update(List<TRuntimeProfileNode> nodes, Reference<Integer> idx) {
TRuntimeProfileNode node = nodes.get(idx.getRef());
// Make sure to update the latest LoadChannel profile according to the timestamp.
if (node.timestamp != -1 && node.timestamp < timestamp) {
return;
}
Preconditions.checkState(timestamp == -1 || node.timestamp != -1);
// update this level's counters
if (node.counters != null) {
for (TCounter tcounter : node.counters) {

19
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
View File

@ -191,7 +191,9 @@ public class Coordinator {
private RuntimeProfile queryProfile;
private RuntimeProfile fragmentsProfile;
private List<RuntimeProfile> fragmentProfile;
private RuntimeProfile loadChannelProfile;
private ProfileWriter profileWriter;
@ -526,12 +528,17 @@ public class Coordinator {
int fragmentSize = fragments.size();
queryProfile = new RuntimeProfile("Execution Profile " + DebugUtil.printId(queryId));
fragmentsProfile = new RuntimeProfile("Fragments");
queryProfile.addChild(fragmentsProfile);
fragmentProfile = new ArrayList<RuntimeProfile>();
for (int i = 0; i < fragmentSize; i++) {
fragmentProfile.add(new RuntimeProfile("Fragment " + i));
queryProfile.addChild(fragmentProfile.get(i));
fragmentsProfile.addChild(fragmentProfile.get(i));
}
loadChannelProfile = new RuntimeProfile("LoadChannels");
queryProfile.addChild(loadChannelProfile);
this.idToBackend = Env.getCurrentSystemInfo().getIdToBackend();
if (LOG.isDebugEnabled()) {
LOG.debug("idToBackend size={}", idToBackend.size());
@ -729,7 +736,7 @@ public class Coordinator {
for (TExecPlanFragmentParams tParam : tParams) {
BackendExecState execState =
new BackendExecState(fragment.getFragmentId(), instanceId++,
profileFragmentId, tParam, this.addressToBackendID);
profileFragmentId, tParam, this.addressToBackendID, loadChannelProfile);
// Each tParam will set the total number of Fragments that need to be executed on the same BE,
// and the BE will determine whether all Fragments have been executed based on this information.
// Notice. load fragment has a small probability that FragmentNumOnHost is 0, for unknown reasons.
@ -2562,6 +2569,7 @@ public class Coordinator {
boolean hasCanceled;
int profileFragmentId;
RuntimeProfile profile;
RuntimeProfile loadChannelProfile;
TNetworkAddress brpcAddress;
TNetworkAddress address;
Backend backend;
@ -2569,7 +2577,8 @@ public class Coordinator {
TUniqueId instanceId;
public BackendExecState(PlanFragmentId fragmentId, int instanceId, int profileFragmentId,
TExecPlanFragmentParams rpcParams, Map<TNetworkAddress, Long> addressToBackendID) {
TExecPlanFragmentParams rpcParams, Map<TNetworkAddress, Long> addressToBackendID,
RuntimeProfile loadChannelProfile) {
this.profileFragmentId = profileFragmentId;
this.fragmentId = fragmentId;
this.rpcParams = rpcParams;
@ -2582,6 +2591,7 @@ public class Coordinator {
this.brpcAddress = new TNetworkAddress(backend.getIp(), backend.getBrpcPort());
String name = "Instance " + DebugUtil.printId(fi.instanceId) + " (host=" + address + ")";
this.loadChannelProfile = loadChannelProfile;
this.profile = new RuntimeProfile(name);
this.hasCanceled = false;
this.lastMissingHeartbeatTime = backend.getLastMissingHeartbeatTime();
@ -2611,6 +2621,9 @@ public class Coordinator {
if (params.isSetProfile()) {
profile.update(params.profile);
}
if (params.isSetLoadChannelProfile()) {
loadChannelProfile.update(params.loadChannelProfile);
}
this.done = params.done;
if (statsErrorEstimator != null) {
statsErrorEstimator.updateExactReturnedRows(params);