database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Enhancement](profile) use queryId of originStatement as the queryId. (#15898)

Browse Source 
* [Enhancement](profile) use queryId of originStatement as the queryId.

Co-authored-by: wangxiangyu@360shuke.com <wangxiangyu@360shuke.com>
This commit is contained in:
wxy
2023-01-15 18:26:14 +08:00
committed by GitHub
parent 0d61ca7cdd
commit 067b33bca7
7 changed files with 33 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java
View File

@ -34,6 +34,7 @@ public class ShowQueryProfileStmt extends ShowStmt {
// This should be same as ProfileManager.PROFILE_HEADERS
public static final ShowResultSetMetaData META_DATA_QUERY_IDS =
ShowResultSetMetaData.builder()
.addColumn(new Column("JobId", ScalarType.createVarchar(128)))
.addColumn(new Column("QueryId", ScalarType.createVarchar(128)))
.addColumn(new Column("User", ScalarType.createVarchar(128)))
.addColumn(new Column("DefaultDb", ScalarType.createVarchar(128)))

21
fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
View File

@ -58,6 +58,8 @@ public class ProfileManager {
private static volatile ProfileManager INSTANCE = null;
// private static final int ARRAY_SIZE = 100;
// private static final int TOTAL_LEN = 1000 * ARRAY_SIZE ;
// just use for load profile and export profile
public static final String JOB_ID = "Job ID";
public static final String QUERY_ID = "Query ID";
public static final String START_TIME = "Start Time";
public static final String END_TIME = "End Time";
@ -84,7 +86,7 @@ public class ProfileManager {
}
public static final ArrayList<String> PROFILE_HEADERS = new ArrayList(
Arrays.asList(QUERY_ID, USER, DEFAULT_DB, SQL_STATEMENT, QUERY_TYPE,
Arrays.asList(JOB_ID, QUERY_ID, USER, DEFAULT_DB, SQL_STATEMENT, QUERY_TYPE,
START_TIME, END_TIME, TOTAL_TIME, QUERY_STATE, TRACE_ID));
private class ProfileElement {
@ -148,25 +150,26 @@ public class ProfileManager {
}
ProfileElement element = createElement(profile);
String queryId = element.infoStrings.get(ProfileManager.QUERY_ID);
String key = isQueryProfile(profile) ? element.infoStrings.get(ProfileManager.QUERY_ID)
: element.infoStrings.get(ProfileManager.JOB_ID);
// check when push in, which can ensure every element in the list has QUERY_ID column,
// so there is no need to check when remove element from list.
if (Strings.isNullOrEmpty(queryId)) {
if (Strings.isNullOrEmpty(key)) {
LOG.warn("the key or value of Map is null, "
+ "may be forget to insert 'QUERY_ID' column into infoStrings");
+ "may be forget to insert 'QUERY_ID' or 'JOB_ID' column into infoStrings");
}
// a profile may be updated multiple times in queryIdToProfileMap,
// and only needs to be inserted into the queryIdDeque for the first time.
queryIdToProfileMap.put(queryId, element);
queryIdToProfileMap.put(key, element);
writeLock.lock();
try {
if (!queryIdDeque.contains(queryId)) {
if (!queryIdDeque.contains(key)) {
if (queryIdDeque.size() >= Config.max_query_profile_num) {
queryIdToProfileMap.remove(queryIdDeque.getFirst());
queryIdDeque.removeFirst();
}
queryIdDeque.addLast(queryId);
queryIdDeque.addLast(key);
}
} finally {
writeLock.unlock();
@ -333,4 +336,8 @@ public class ProfileManager {
readLock.unlock();
}
}
public boolean isQueryProfile(RuntimeProfile profile) {
return "Query".equals(profile.getName());
}
}

9
fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
View File

@ -51,6 +51,7 @@ import org.apache.doris.common.Status;
import org.apache.doris.common.UserException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.planner.DataPartition;
@ -116,6 +117,7 @@ public class ExportJob implements Writable {
}
private long id;
private String queryId;
private String label;
private long dbId;
private long tableId;
@ -175,6 +177,7 @@ public class ExportJob implements Writable {
public ExportJob() {
this.id = -1;
this.queryId = "";
this.dbId = -1;
this.tableId = -1;
this.state = JobState.PENDING;
@ -201,11 +204,11 @@ public class ExportJob implements Writable {
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName);
Preconditions.checkNotNull(stmt.getBrokerDesc());
this.brokerDesc = stmt.getBrokerDesc();
this.columnSeparator = stmt.getColumnSeparator();
this.lineDelimiter = stmt.getLineDelimiter();
this.properties = stmt.getProperties();
this.label = this.properties.get(ExportStmt.LABEL);
this.queryId = ConnectContext.get() != null ? DebugUtil.printId(ConnectContext.get().queryId()) : "N/A";
String path = stmt.getPath();
Preconditions.checkArgument(!Strings.isNullOrEmpty(path));
@ -729,6 +732,10 @@ public class ExportJob implements Writable {
return label;
}
public String getQueryId() {
return queryId;
}
@Override
public String toString() {
return "ExportJob [jobId=" + id

3
fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
View File

@ -314,7 +314,8 @@ public class BrokerLoadJob extends BulkLoadJob {
}
RuntimeProfile summaryProfile = new RuntimeProfile("Summary");
summaryProfile.addInfoString(ProfileManager.QUERY_ID, String.valueOf(id));
summaryProfile.addInfoString(ProfileManager.JOB_ID, String.valueOf(this.id));
summaryProfile.addInfoString(ProfileManager.QUERY_ID, this.queryId);
summaryProfile.addInfoString(ProfileManager.START_TIME, TimeUtils.longToTimeString(createTimestamp));
summaryProfile.addInfoString(ProfileManager.END_TIME, TimeUtils.longToTimeString(finishTimestamp));
summaryProfile.addInfoString(ProfileManager.TOTAL_TIME,

5
fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
View File

@ -31,6 +31,7 @@ import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.LogBuilder;
import org.apache.doris.common.util.LogKey;
import org.apache.doris.common.util.SqlParserUtils;
@ -74,6 +75,8 @@ public abstract class BulkLoadJob extends LoadJob {
// input params
protected BrokerDesc brokerDesc;
// queryId of OriginStatement
protected String queryId;
// this param is used to persist the expr of columns
// the origin stmt is persisted instead of columns expr
// the expr of columns will be reanalyze when the log is replayed
@ -101,9 +104,11 @@ public abstract class BulkLoadJob extends LoadJob {
this.userInfo = userInfo;
if (ConnectContext.get() != null) {
this.queryId = DebugUtil.printId(ConnectContext.get().queryId());
SessionVariable var = ConnectContext.get().getSessionVariable();
sessionVariables.put(SessionVariable.SQL_MODE, Long.toString(var.getSqlMode()));
} else {
this.queryId = "N/A";
sessionVariables.put(SessionVariable.SQL_MODE, String.valueOf(SqlModeHelper.MODE_DEFAULT));
}
}

1
fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
View File

@ -302,6 +302,7 @@ public class StmtExecutor implements ProfileWriter {
private Map<String, String> getSummaryInfo() {
Map<String, String> infos = Maps.newLinkedHashMap();
infos.put(ProfileManager.JOB_ID, "N/A");
infos.put(ProfileManager.QUERY_ID, DebugUtil.printId(context.queryId()));
infos.put(ProfileManager.QUERY_TYPE, queryType);
infos.put(ProfileManager.DORIS_VERSION, Version.DORIS_BUILD_VERSION);

3
fe/fe-core/src/main/java/org/apache/doris/task/ExportExportingTask.java
View File

@ -252,7 +252,8 @@ public class ExportExportingTask extends MasterTask {
private void initProfile() {
profile = new RuntimeProfile("ExportJob");
RuntimeProfile summaryProfile = new RuntimeProfile("Summary");
summaryProfile.addInfoString(ProfileManager.QUERY_ID, String.valueOf(job.getId()));
summaryProfile.addInfoString(ProfileManager.JOB_ID, String.valueOf(job.getId()));
summaryProfile.addInfoString(ProfileManager.QUERY_ID, job.getQueryId());
summaryProfile.addInfoString(ProfileManager.START_TIME, TimeUtils.longToTimeString(job.getStartTimeMs()));
long currentTimestamp = System.currentTimeMillis();