database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Feature](profile)Support active_queries TVF (#29999)

Browse Source
This commit is contained in:
wangbo
2024-01-16 21:24:30 +08:00
committed by yiguolei
parent d658a44cef
commit 2dcdf07dd4
27 changed files with 348 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.catalog;
import org.apache.doris.nereids.trees.expressions.functions.table.ActiveQueries;
import org.apache.doris.nereids.trees.expressions.functions.table.Backends;
import org.apache.doris.nereids.trees.expressions.functions.table.Catalogs;
import org.apache.doris.nereids.trees.expressions.functions.table.Frontends;
@ -29,7 +30,6 @@ import org.apache.doris.nereids.trees.expressions.functions.table.Jobs;
import org.apache.doris.nereids.trees.expressions.functions.table.Local;
import org.apache.doris.nereids.trees.expressions.functions.table.MvInfos;
import org.apache.doris.nereids.trees.expressions.functions.table.Numbers;
import org.apache.doris.nereids.trees.expressions.functions.table.Queries;
import org.apache.doris.nereids.trees.expressions.functions.table.S3;
import org.apache.doris.nereids.trees.expressions.functions.table.Tasks;
import org.apache.doris.nereids.trees.expressions.functions.table.WorkloadGroups;
@ -54,7 +54,7 @@ public class BuiltinTableValuedFunctions implements FunctionHelper {
tableValued(Hdfs.class, "hdfs"),
tableValued(HttpStream.class, "http_stream"),
tableValued(Numbers.class, "numbers"),
tableValued(Queries.class, "queries"),
tableValued(ActiveQueries.class, "active_queries"),
tableValued(S3.class, "s3"),
tableValued(MvInfos.class, "mv_infos"),
tableValued(Jobs.class, "jobs"),

63
fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/QueryDetailAction.java
View File

@ -1,63 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.httpv2.rest;
import org.apache.doris.httpv2.entity.ResponseEntityBuilder;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.QueryDetail;
import org.apache.doris.qe.QueryDetailQueue;
import com.google.common.base.Strings;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
import java.util.Map;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
// This class is used to get current query_id of connection_id.
// Every connection holds at most one query at every point.
// Some we can get query_id firstly, and get query by query_id.
@RestController
public class QueryDetailAction extends RestBaseController {
private static final Logger LOG = LogManager.getLogger(QueryDetailAction.class);
@RequestMapping(path = "/api/query_detail", method = RequestMethod.GET)
protected Object query_detail(HttpServletRequest request, HttpServletResponse response) {
executeCheckPassword(request, response);
checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
String eventTimeStr = request.getParameter("event_time");
if (Strings.isNullOrEmpty(eventTimeStr)) {
return ResponseEntityBuilder.badRequest("Missing event_time");
}
long eventTime = Long.valueOf(eventTimeStr.trim());
List<QueryDetail> queryDetails = QueryDetailQueue.getQueryDetails(eventTime);
Map<String, List<QueryDetail>> result = Maps.newHashMap();
result.put("query_details", queryDetails);
return ResponseEntityBuilder.ok(result);
}
}

10
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Queries.java → fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/ActiveQueries.java
View File

@ -22,7 +22,7 @@ import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Properties;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.coercion.AnyDataType;
import org.apache.doris.tablefunction.QueriesTableValuedFunction;
import org.apache.doris.tablefunction.ActiveQueriesTableValuedFunction;
import org.apache.doris.tablefunction.TableValuedFunctionIf;
import java.util.Map;
@ -30,10 +30,10 @@ import java.util.Map;
/**
* queries tvf
*/
public class Queries extends TableValuedFunction {
public class ActiveQueries extends TableValuedFunction {
public Queries(Properties properties) {
super("queries", properties);
public ActiveQueries(Properties properties) {
super("active_queries", properties);
}
@Override
@ -45,7 +45,7 @@ public class Queries extends TableValuedFunction {
protected TableValuedFunctionIf toCatalogFunction() {
try {
Map<String, String> arguments = getTVFProperties().getMap();
return new QueriesTableValuedFunction(arguments);
return new ActiveQueriesTableValuedFunction(arguments);
} catch (Throwable t) {
throw new AnalysisException("Can not build FrontendsTableValuedFunction by "
+ this + ": " + t.getMessage(), t);

4
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.expressions.visitor;
import org.apache.doris.nereids.trees.expressions.functions.table.ActiveQueries;
import org.apache.doris.nereids.trees.expressions.functions.table.Backends;
import org.apache.doris.nereids.trees.expressions.functions.table.Catalogs;
import org.apache.doris.nereids.trees.expressions.functions.table.Frontends;
@ -29,7 +30,6 @@ import org.apache.doris.nereids.trees.expressions.functions.table.Jobs;
import org.apache.doris.nereids.trees.expressions.functions.table.Local;
import org.apache.doris.nereids.trees.expressions.functions.table.MvInfos;
import org.apache.doris.nereids.trees.expressions.functions.table.Numbers;
import org.apache.doris.nereids.trees.expressions.functions.table.Queries;
import org.apache.doris.nereids.trees.expressions.functions.table.S3;
import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction;
import org.apache.doris.nereids.trees.expressions.functions.table.Tasks;
@ -91,7 +91,7 @@ public interface TableValuedFunctionVisitor<R, C> {
return visitTableValuedFunction(numbers, context);
}
default R visitQueries(Queries queries, C context) {
default R visitQueries(ActiveQueries queries, C context) {
return visitTableValuedFunction(queries, context);
}

12
fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
View File

@ -89,18 +89,6 @@ public class AuditLogHelper {
}
}
auditEventBuilder.setIsQuery(true);
if (ctx.getQueryDetail() != null) {
ctx.getQueryDetail().setEventTime(endTime);
ctx.getQueryDetail().setEndTime(endTime);
ctx.getQueryDetail().setLatency(elapseMs);
if (ctx.isKilled()) {
ctx.getQueryDetail().setState(QueryDetail.QueryMemState.CANCELLED);
} else {
ctx.getQueryDetail().setState(QueryDetail.QueryMemState.FINISHED);
}
QueryDetailQueue.addOrUpdateQueryDetail(ctx.getQueryDetail());
ctx.setQueryDetail(null);
}
} else {
auditEventBuilder.setIsQuery(false);
}

3
fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessor.java
View File

@ -18,6 +18,7 @@
package org.apache.doris.qe;
import org.apache.doris.common.UserException;
import org.apache.doris.qe.QeProcessorImpl.QueryInfo;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TReportExecStatusParams;
import org.apache.doris.thrift.TReportExecStatusResult;
@ -45,4 +46,6 @@ public interface QeProcessor {
Coordinator getCoordinator(TUniqueId queryId);
List<Coordinator> getAllCoordinators();
Map<String, QueryInfo> getQueryInfoMap();
}

13
fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java
View File

@ -40,6 +40,7 @@ import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicInteger;
@ -246,6 +247,18 @@ public final class QeProcessorImpl implements QeProcessor {
return "";
}
public Map<String, QueryInfo> getQueryInfoMap() {
Map<String, QueryInfo> retQueryInfoMap = Maps.newHashMap();
Set<TUniqueId> queryIdSet = coordinatorMap.keySet();
for (TUniqueId qid : queryIdSet) {
QueryInfo queryInfo = coordinatorMap.get(qid);
if (queryInfo != null) {
retQueryInfoMap.put(DebugUtil.printId(qid), queryInfo);
}
}
return retQueryInfoMap;
}
public static final class QueryInfo {
private final ConnectContext connectContext;
private final Coordinator coord;

9
fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
View File

@ -1447,15 +1447,6 @@ public class StmtExecutor {
Queriable queryStmt = (Queriable) parsedStmt;
QueryDetail queryDetail = new QueryDetail(context.getStartTime(),
DebugUtil.printId(context.queryId()),
context.getStartTime(), -1, -1,
QueryDetail.QueryMemState.RUNNING,
context.getDatabase(),
originStmt.originStmt);
context.setQueryDetail(queryDetail);
QueryDetailQueue.addOrUpdateQueryDetail(queryDetail);
if (queryStmt.isExplain()) {
String explainString = planner.getExplainString(queryStmt.getExplainOptions());
handleExplainStmt(explainString, false);

4
fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadRuntimeStatusMgr.java
View File

@ -167,6 +167,10 @@ public class WorkloadRuntimeStatusMgr {
return retQueryMap;
}
public Map<Long, Map<String, TQueryStatistics>> getBeQueryStatsMap() {
return beToQueryStatsMap;
}
private void mergeQueryStatistics(TQueryStatistics dst, TQueryStatistics src) {
dst.scan_rows += src.scan_rows;
dst.scan_bytes += src.scan_bytes;

29
fe/fe-core/src/main/java/org/apache/doris/tablefunction/QueriesTableValuedFunction.java → fe/fe-core/src/main/java/org/apache/doris/tablefunction/ActiveQueriesTableValuedFunction.java
View File

@ -31,19 +31,24 @@ import com.google.common.collect.ImmutableMap;
import java.util.List;
import java.util.Map;
public class QueriesTableValuedFunction extends MetadataTableValuedFunction {
public static final String NAME = "queries";
public class ActiveQueriesTableValuedFunction extends MetadataTableValuedFunction {
public static final String NAME = "active_queries";
private static final ImmutableList<Column> SCHEMA = ImmutableList.of(
new Column("QueryId", ScalarType.createStringType()),
new Column("StartTime", PrimitiveType.BIGINT),
new Column("EndTime", PrimitiveType.BIGINT),
new Column("EventTime", PrimitiveType.BIGINT),
new Column("Latency", PrimitiveType.BIGINT),
new Column("State", ScalarType.createStringType()),
new Column("Database", ScalarType.createStringType()),
new Column("Sql", ScalarType.createStringType()),
new Column("FrontendInstance", ScalarType.createStringType()));
new Column("BeHost", ScalarType.createStringType()),
new Column("BePort", PrimitiveType.BIGINT),
new Column("QueryId", ScalarType.createStringType()),
new Column("StartTime", ScalarType.createStringType()),
new Column("QueryTimeMs", PrimitiveType.BIGINT),
new Column("WorkloadGroupId", PrimitiveType.BIGINT),
new Column("QueryCpuTimeMs", PrimitiveType.BIGINT),
new Column("ScanRows", PrimitiveType.BIGINT),
new Column("ScanBytes", PrimitiveType.BIGINT),
new Column("BePeakMemoryBytes", PrimitiveType.BIGINT),
new Column("CurrentUsedMemoryBytes", PrimitiveType.BIGINT),
new Column("Database", ScalarType.createStringType()),
new Column("FrontendInstance", ScalarType.createStringType()),
new Column("Sql", ScalarType.createStringType()));
private static final ImmutableMap<String, Integer> COLUMN_TO_INDEX;
@ -59,7 +64,7 @@ public class QueriesTableValuedFunction extends MetadataTableValuedFunction {
return COLUMN_TO_INDEX.get(columnName.toLowerCase());
}
public QueriesTableValuedFunction(Map<String, String> params) throws AnalysisException {
public ActiveQueriesTableValuedFunction(Map<String, String> params) throws AnalysisException {
if (params.size() != 0) {
throw new AnalysisException("Queries table-valued-function does not support any params");
}

96
fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
View File

@ -37,8 +37,8 @@ import org.apache.doris.mtmv.MTMVUtil;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.planner.external.iceberg.IcebergMetadataCache;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.QueryDetail;
import org.apache.doris.qe.QueryDetailQueue;
import org.apache.doris.qe.QeProcessorImpl;
import org.apache.doris.qe.QeProcessorImpl.QueryInfo;
import org.apache.doris.system.Backend;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.FrontendService;
@ -54,6 +54,7 @@ import org.apache.doris.thrift.TMetadataTableRequestParams;
import org.apache.doris.thrift.TMetadataType;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TQueriesMetadataParams;
import org.apache.doris.thrift.TQueryStatistics;
import org.apache.doris.thrift.TRow;
import org.apache.doris.thrift.TStatus;
import org.apache.doris.thrift.TStatusCode;
@ -70,11 +71,14 @@ import org.apache.logging.log4j.Logger;
import org.apache.thrift.TException;
import org.jetbrains.annotations.NotNull;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
public class MetadataGenerator {
@ -414,6 +418,49 @@ public class MetadataGenerator {
return result;
}
private static TRow makeQueryStatisticsTRow(SimpleDateFormat sdf, String queryId, Backend be,
String selfNode, QueryInfo queryInfo, TQueryStatistics qs) {
TRow trow = new TRow();
if (be != null) {
trow.addToColumnValue(new TCell().setStringVal(be.getHost()));
trow.addToColumnValue(new TCell().setLongVal(be.getBePort()));
} else {
trow.addToColumnValue(new TCell().setStringVal("invalid host"));
trow.addToColumnValue(new TCell().setLongVal(-1));
}
trow.addToColumnValue(new TCell().setStringVal(queryId));
String strDate = sdf.format(new Date(queryInfo.getStartExecTime()));
trow.addToColumnValue(new TCell().setStringVal(strDate));
trow.addToColumnValue(new TCell().setLongVal(System.currentTimeMillis() - queryInfo.getStartExecTime()));
if (qs != null) {
trow.addToColumnValue(new TCell().setLongVal(qs.workload_group_id));
trow.addToColumnValue(new TCell().setLongVal(qs.cpu_ms));
trow.addToColumnValue(new TCell().setLongVal(qs.scan_rows));
trow.addToColumnValue(new TCell().setLongVal(qs.scan_bytes));
trow.addToColumnValue(new TCell().setLongVal(qs.max_peak_memory_bytes));
trow.addToColumnValue(new TCell().setLongVal(qs.current_used_memory_bytes));
} else {
trow.addToColumnValue(new TCell().setLongVal(0L));
trow.addToColumnValue(new TCell().setLongVal(0L));
trow.addToColumnValue(new TCell().setLongVal(0L));
trow.addToColumnValue(new TCell().setLongVal(0L));
trow.addToColumnValue(new TCell().setLongVal(0L));
trow.addToColumnValue(new TCell().setLongVal(0L));
}
if (queryInfo.getConnectContext() != null) {
trow.addToColumnValue(new TCell().setStringVal(queryInfo.getConnectContext().getDatabase()));
} else {
trow.addToColumnValue(new TCell().setStringVal(""));
}
trow.addToColumnValue(new TCell().setStringVal(selfNode));
trow.addToColumnValue(new TCell().setStringVal(queryInfo.getSql()));
return trow;
}
private static TFetchSchemaTableDataResult queriesMetadataResult(TMetadataTableRequestParams params,
TFetchSchemaTableDataRequest parentRequest) {
if (!params.isSetQueriesMetadataParams()) {
@ -429,24 +476,37 @@ public class MetadataGenerator {
}
selfNode = NetUtils.getHostnameByIp(selfNode);
// get query
Map<Long, Map<String, TQueryStatistics>> beQsMap = Env.getCurrentEnv().getWorkloadRuntimeStatusMgr()
.getBeQueryStatsMap();
Set<Long> beIdSet = beQsMap.keySet();
List<TRow> dataBatch = Lists.newArrayList();
List<QueryDetail> queries = QueryDetailQueue.getQueryDetails(0L);
for (QueryDetail query : queries) {
TRow trow = new TRow();
trow.addToColumnValue(new TCell().setStringVal(query.getQueryId()));
trow.addToColumnValue(new TCell().setLongVal(query.getStartTime()));
trow.addToColumnValue(new TCell().setLongVal(query.getEndTime()));
trow.addToColumnValue(new TCell().setLongVal(query.getEventTime()));
if (query.getState() == QueryDetail.QueryMemState.RUNNING) {
trow.addToColumnValue(new TCell().setLongVal(System.currentTimeMillis() - query.getStartTime()));
} else {
trow.addToColumnValue(new TCell().setLongVal(query.getLatency()));
Map<String, QueryInfo> queryInfoMap = QeProcessorImpl.INSTANCE.getQueryInfoMap();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
for (Long beId : beIdSet) {
Map<String, TQueryStatistics> qsMap = beQsMap.get(beId);
if (qsMap == null) {
continue;
}
Set<String> queryIdSet = qsMap.keySet();
for (String queryId : queryIdSet) {
QueryInfo queryInfo = queryInfoMap.get(queryId);
if (queryInfo == null) {
continue;
}
//todo(wb) add connect context for insert select
if (queryInfo.getConnectContext() != null && !Env.getCurrentEnv().getAccessManager()
.checkDbPriv(queryInfo.getConnectContext(), queryInfo.getConnectContext().getDatabase(),
PrivPredicate.SELECT)) {
continue;
}
TQueryStatistics qs = qsMap.get(queryId);
Backend be = Env.getCurrentEnv().getClusterInfo().getBackend(beId);
TRow tRow = makeQueryStatisticsTRow(sdf, queryId, be, selfNode, queryInfo, qs);
dataBatch.add(tRow);
}
trow.addToColumnValue(new TCell().setStringVal(query.getState().toString()));
trow.addToColumnValue(new TCell().setStringVal(query.getDatabase()));
trow.addToColumnValue(new TCell().setStringVal(query.getSql()));
trow.addToColumnValue(new TCell().setStringVal(selfNode));
dataBatch.add(trow);
}
/* Get the query results from other FE also */

2
fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java
View File

@ -50,7 +50,7 @@ public abstract class MetadataTableValuedFunction extends TableValuedFunctionIf
case TASKS:
return TasksTableValuedFunction.getColumnIndexFromColumnName(columnName, params);
case QUERIES:
return QueriesTableValuedFunction.getColumnIndexFromColumnName(columnName);
return ActiveQueriesTableValuedFunction.getColumnIndexFromColumnName(columnName);
case WORKLOAD_SCHED_POLICY:
return WorkloadSchedPolicyTableValuedFunction.getColumnIndexFromColumnName(columnName);
default:

4
fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
View File

@ -74,8 +74,8 @@ public abstract class TableValuedFunctionIf {
return new TasksTableValuedFunction(params);
case GroupCommitTableValuedFunction.NAME:
return new GroupCommitTableValuedFunction(params);
case QueriesTableValuedFunction.NAME:
return new QueriesTableValuedFunction(params);
case ActiveQueriesTableValuedFunction.NAME:
return new ActiveQueriesTableValuedFunction(params);
case WorkloadSchedPolicyTableValuedFunction.NAME:
return new WorkloadSchedPolicyTableValuedFunction(params);
default: