database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feat](stats) enable set auto analyze time by set global session variable (#24026)

Browse Source
This commit is contained in:
AKIRA
2023-09-13 11:59:25 +09:00
committed by GitHub
parent 026cefbfbc
commit f205473426
2 changed files with 66 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -49,6 +49,8 @@ import java.io.DataOutput;
import java.io.IOException;
import java.io.Serializable;
import java.lang.reflect.Field;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@ -397,6 +399,10 @@ public class SessionVariable implements Serializable, Writable {
public static final String INVERTED_INDEX_CONJUNCTION_OPT_THRESHOLD = "inverted_index_conjunction_opt_threshold";
public static final String FULL_AUTO_ANALYZE_START_TIME = "full_auto_analyze_start_time";
public static final String FULL_AUTO_ANALYZE_END_TIME = "full_auto_analyze_end_time";
public static final List<String> DEBUG_VARIABLES = ImmutableList.of(
SKIP_DELETE_PREDICATE,
SKIP_DELETE_BITMAP,
@ -1159,6 +1165,18 @@ public class SessionVariable implements Serializable, Writable {
+ " use a skiplist to optimize the intersection."})
public int invertedIndexConjunctionOptThreshold = 1000;
@VariableMgr.VarAttr(name = FULL_AUTO_ANALYZE_START_TIME, needForward = true, checker = "checkAnalyzeTimeFormat",
description = {"该参数定义自动ANALYZE例程的开始时间",
"This parameter defines the start time for the automatic ANALYZE routine."},
flag = VariableMgr.GLOBAL)
public String fullAutoAnalyzeStartTime = "";
@VariableMgr.VarAttr(name = FULL_AUTO_ANALYZE_END_TIME, needForward = true, checker = "checkAnalyzeTimeFormat",
description = {"该参数定义自动ANALYZE例程的结束时间",
"This parameter defines the end time for the automatic ANALYZE routine."},
flag = VariableMgr.GLOBAL)
public String fullAutoAnalyzeEndTime = "";
// If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables,
// not the default value set in the code.
public void initFuzzyModeVariables() {
@ -2578,5 +2596,15 @@ public class SessionVariable implements Serializable, Writable {
}
return connectContext.getSessionVariable().enableAggState;
}
public void checkAnalyzeTimeFormat(String time) {
try {
DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("HH:mm:ss");
timeFormatter.parse(time);
} catch (DateTimeParseException e) {
LOG.warn("Parse analyze start/end time format fail", e);
throw new UnsupportedOperationException("Expect format: HH:mm:ss");
}
}
}

40
fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
View File

@ -17,19 +17,25 @@
package org.apache.doris.statistics;
import org.apache.doris.analysis.SetType;
import org.apache.doris.analysis.TableName;
import org.apache.doris.analysis.VariableExpr;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.View;
import org.apache.doris.common.Config;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.VariableMgr;
import org.apache.doris.statistics.AnalysisInfo.JobType;
import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.collect.Maps;
import org.apache.commons.lang3.StringUtils;
import org.apache.hudi.common.util.VisibleForTesting;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -170,9 +176,12 @@ public class StatisticsAutoCollector extends StatisticsCollector {
private boolean checkAnalyzeTime(LocalTime now) {
try {
Pair<LocalTime, LocalTime> range = findRangeFromGlobalSessionVar();
DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("HH:mm:ss");
LocalTime start = LocalTime.parse(Config.full_auto_analyze_start_time, timeFormatter);
LocalTime end = LocalTime.parse(Config.full_auto_analyze_end_time, timeFormatter);
LocalTime start = range == null
? LocalTime.parse(Config.full_auto_analyze_start_time, timeFormatter) : range.first;
LocalTime end = range == null
? LocalTime.parse(Config.full_auto_analyze_end_time, timeFormatter) : range.second;
if (start.isAfter(end) && (now.isAfter(start) || now.isBefore(end))) {
return true;
@ -184,4 +193,31 @@ public class StatisticsAutoCollector extends StatisticsCollector {
return true;
}
}
private Pair<LocalTime, LocalTime> findRangeFromGlobalSessionVar() {
try {
String startTime =
findRangeFromGlobalSessionVar(SessionVariable.FULL_AUTO_ANALYZE_START_TIME)
.fullAutoAnalyzeStartTime;
if (StringUtils.isEmpty(startTime)) {
return null;
}
String endTime = findRangeFromGlobalSessionVar(SessionVariable.FULL_AUTO_ANALYZE_END_TIME)
.fullAutoAnalyzeEndTime;
if (StringUtils.isEmpty(startTime)) {
return null;
}
DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("HH:mm:ss");
return Pair.of(LocalTime.parse(startTime, timeFormatter), LocalTime.parse(endTime, timeFormatter));
} catch (Exception e) {
return null;
}
}
private SessionVariable findRangeFromGlobalSessionVar(String varName) throws Exception {
SessionVariable sessionVariable = VariableMgr.newSessionVariable();
VariableExpr variableExpr = new VariableExpr(varName, SetType.GLOBAL);
VariableMgr.getValue(sessionVariable, variableExpr);
return sessionVariable;
}
}