database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](stats) Don't save colToPartitions anymore to save mem (#27879)

Browse Source
This commit is contained in:
AKIRA
2023-12-01 19:54:30 +08:00
committed by GitHub
parent 19281e3590
commit 1451a835b7
4 changed files with 24 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
View File

@ -97,10 +97,8 @@ public class AnalysisInfo implements Writable {
public final long tblId;
// TODO: Map here is wired, List is enough
@SerializedName("colToPartitions")
public final Map<String, Set<String>> colToPartitions;
@SerializedName("partitionNames")
public final Set<String> partitionNames;
@SerializedName("colName")

12
fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java
View File

@ -178,6 +178,18 @@ public class AnalysisJob {
public void deregisterJob() {
analysisManager.removeJob(jobInfo.jobId);
for (BaseAnalysisTask task : queryingTask) {
task.info.colToPartitions.clear();
if (task.info.partitionNames != null) {
task.info.partitionNames.clear();
}
}
for (BaseAnalysisTask task : queryFinished) {
task.info.colToPartitions.clear();
if (task.info.partitionNames != null) {
task.info.partitionNames.clear();
}
}
}
protected void syncLoadStats() {

11
fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
View File

@ -621,7 +621,10 @@ public class AnalysisManager implements Writable {
tableStats.update(jobInfo, tbl);
logCreateTableStats(tableStats);
}
jobInfo.colToPartitions.clear();
if (jobInfo.partitionNames != null) {
jobInfo.partitionNames.clear();
}
}
public List<AnalysisInfo> showAnalysisJob(ShowAnalyzeStmt stmt) {
@ -801,6 +804,9 @@ public class AnalysisManager implements Writable {
while (analysisJobInfoMap.size() >= Config.analyze_record_limit) {
analysisJobInfoMap.remove(analysisJobInfoMap.pollFirstEntry().getKey());
}
if (jobInfo.message != null && jobInfo.message.length() >= StatisticConstants.MSG_LEN_UPPER_BOUND) {
jobInfo.message = jobInfo.message.substring(0, StatisticConstants.MSG_LEN_UPPER_BOUND);
}
this.analysisJobInfoMap.put(jobInfo.jobId, jobInfo);
}
@ -808,6 +814,9 @@ public class AnalysisManager implements Writable {
while (analysisTaskInfoMap.size() >= Config.analyze_record_limit) {
analysisTaskInfoMap.remove(analysisTaskInfoMap.pollFirstEntry().getKey());
}
if (taskInfo.message != null && taskInfo.message.length() >= StatisticConstants.MSG_LEN_UPPER_BOUND) {
taskInfo.message = taskInfo.message.substring(0, StatisticConstants.MSG_LEN_UPPER_BOUND);
}
this.analysisTaskInfoMap.put(taskInfo.taskId, taskInfo);
}

2
fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
View File

@ -100,6 +100,8 @@ public class StatisticConstants {
public static final int AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD = 70;
public static final int MSG_LEN_UPPER_BOUND = 1024;
static {
SYSTEM_DBS.add(SystemInfoService.DEFAULT_CLUSTER
+ ClusterNamespace.CLUSTER_DELIMITER + FeConstants.INTERNAL_DB_NAME);