database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](variable) add annotations for variables (#22292)

Browse Source
This commit is contained in:
Gabriel
2023-08-08 22:16:42 +08:00
committed by GitHub
parent 124c1b16cf
commit 7bfcee6e71
8 changed files with 72 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
fe/fe-common/src/main/java/org/apache/doris/common/Config.java
View File

@ -17,8 +17,6 @@
package org.apache.doris.common;
import org.apache.doris.common.ExperimentalUtil.ExperimentalType;
public class Config extends ConfigBase {
@ConfField(description = {"用户自定义配置文件的路径，用于存放 fe_custom.conf。该文件中的配置会覆盖 fe.conf 中的配置",
@ -293,7 +291,7 @@ public class Config extends ConfigBase {
public static long max_bdbje_clock_delta_ms = 5000; // 5s
@ConfField(description = {"是否启用所有 http 接口的认证",
"Whether to enable all http interface authentication"}, expType = ExperimentalType.EXPERIMENTAL)
"Whether to enable all http interface authentication"}, varType = VariableAnnotation.EXPERIMENTAL)
public static boolean enable_all_http_auth = false;
@ConfField(description = {"FE http 端口，目前所有 FE 的 http 端口必须相同",
@ -323,7 +321,7 @@ public class Config extends ConfigBase {
@ConfField(description = {"是否启用 https，如果启用，http 端口将不可用",
"Whether to enable https, if enabled, http port will not be available"},
expType = ExperimentalType.EXPERIMENTAL)
varType = VariableAnnotation.EXPERIMENTAL)
public static boolean enable_https = false;
@ConfField(description = {"Jetty 的 acceptor 线程数。Jetty的线程架构模型很简单，分为三个线程池：acceptor、selector 和 worker。"
@ -572,7 +570,7 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true, description = {
"是否启用 stream load 和 broker load 的单副本写入。",
"Whether to enable to write single replica for stream load and broker load."},
expType = ExperimentalType.EXPERIMENTAL)
varType = VariableAnnotation.EXPERIMENTAL)
public static boolean enable_single_replica_load = false;
@ConfField(mutable = true, masterOnly = true, description = {
@ -1524,7 +1522,7 @@ public class Config extends ConfigBase {
public static int scheduler_job_task_max_num = 10;
// enable_workload_group should be immutable and temporarily set to mutable during the development test phase
@ConfField(mutable = true, expType = ExperimentalType.EXPERIMENTAL)
@ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL)
public static boolean enable_workload_group = false;
@ConfField(mutable = true)
@ -1603,7 +1601,7 @@ public class Config extends ConfigBase {
/*
* mtmv is still under dev, remove this config when it is graduate.
*/
@ConfField(mutable = true, masterOnly = true, expType = ExperimentalType.EXPERIMENTAL)
@ConfField(mutable = true, masterOnly = true, varType = VariableAnnotation.EXPERIMENTAL)
public static boolean enable_mtmv = false;
/* Max running task num at the same time, otherwise the submitted task will still be keep in pending poll*/
@ -1773,7 +1771,7 @@ public class Config extends ConfigBase {
* When enable_fqdn_mode is true, the name of the pod where be is located will remain unchanged
* after reconstruction, while the ip can be changed.
*/
@ConfField(mutable = false, expType = ExperimentalType.EXPERIMENTAL)
@ConfField(mutable = false, varType = VariableAnnotation.EXPERIMENTAL)
public static boolean enable_fqdn_mode = false;
/**
@ -1796,7 +1794,7 @@ public class Config extends ConfigBase {
* If set to true, doris will try to parse the ddl of a hive view and try to execute the query
* otherwise it will throw an AnalysisException.
*/
@ConfField(mutable = true, expType = ExperimentalType.EXPERIMENTAL)
@ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL)
public static boolean enable_query_hive_views = false;
/**
@ -1826,7 +1824,7 @@ public class Config extends ConfigBase {
/**
* If set to ture, doris will establish an encrypted channel based on the SSL protocol with mysql.
*/
@ConfField(mutable = false, masterOnly = false, expType = ExperimentalType.EXPERIMENTAL)
@ConfField(mutable = false, masterOnly = false, varType = VariableAnnotation.EXPERIMENTAL)
public static boolean enable_ssl = false;
/**
@ -2008,7 +2006,7 @@ public class Config extends ConfigBase {
+ " including the specific reason why they are unqueryable, will be printed out."})
public static boolean show_details_for_unaccessible_tablet = false;
@ConfField(mutable = false, masterOnly = false, expType = ExperimentalType.EXPERIMENTAL, description = {
@ConfField(mutable = false, masterOnly = false, varType = VariableAnnotation.EXPERIMENTAL, description = {
"是否启用binlog特性",
"Whether to enable binlog feature"})
public static boolean enable_feature_binlog = false;

33
fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java
View File

@ -17,8 +17,6 @@
package org.apache.doris.common;
import org.apache.doris.common.ExperimentalUtil.ExperimentalType;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@ -54,7 +52,7 @@ public class ConfigBase {
String comment() default "";
ExperimentalType expType() default ExperimentalType.NONE;
VariableAnnotation varType() default VariableAnnotation.NONE;
Class<? extends ConfHandler> callback() default DefaultConfHandler.class;
@ -103,10 +101,7 @@ public class ConfigBase {
continue;
}
confFields.put(field.getName(), field);
if (confField.expType() == ExperimentalType.EXPERIMENTAL
|| confField.expType() == ExperimentalType.EXPERIMENTAL_ONLINE) {
confFields.put(ExperimentalUtil.EXPERIMENTAL_PREFIX + field.getName(), field);
}
confFields.put(confField.varType().getPrefix() + field.getName(), field);
}
initConf(confFile);
@ -120,10 +115,7 @@ public class ConfigBase {
continue;
}
ldapConfFields.put(field.getName(), field);
if (confField.expType() == ExperimentalType.EXPERIMENTAL
|| confField.expType() == ExperimentalType.EXPERIMENTAL_ONLINE) {
ldapConfFields.put(ExperimentalUtil.EXPERIMENTAL_PREFIX + field.getName(), field);
}
ldapConfFields.put(confField.varType().getPrefix() + field.getName(), field);
}
initConf(ldapConfFile);
}
@ -223,8 +215,7 @@ public class ConfigBase {
// ensure that field has property string
String confKey = f.getName();
String confVal = props.getProperty(confKey,
props.getProperty(ExperimentalUtil.EXPERIMENTAL_PREFIX + confKey));
String confVal = props.getProperty(confKey, props.getProperty(anno.varType().getPrefix() + confKey));
if (Strings.isNullOrEmpty(confVal)) {
continue;
}
@ -341,10 +332,8 @@ public class ConfigBase {
/**
* Get display name of experimental configs.
* For an experimental config, the given "configsToFilter" contains both config w/o "experimental_" prefix.
* We need to return the right display name for these configs, by following rules:
* 1. If this config is EXPERIMENTAL, only return the config with "experimental_" prefix.
* 2. If this config is not EXPERIMENTAL, only return the config without "experimental_" prefix.
* For an experimental/deprecated config, the given "configsToFilter" contains both config w/o
* "experimental_/deprecated_" prefix.
*
* @param configsToFilter
* @param allConfigs
@ -353,12 +342,8 @@ public class ConfigBase {
for (Map.Entry<String, Field> e : configsToFilter.entrySet()) {
Field f = e.getValue();
ConfField confField = f.getAnnotation(ConfField.class);
boolean isExperimental = e.getKey().startsWith(ExperimentalUtil.EXPERIMENTAL_PREFIX);
if (isExperimental && confField.expType() != ExperimentalType.EXPERIMENTAL) {
continue;
}
if (!isExperimental && confField.expType() == ExperimentalType.EXPERIMENTAL) {
if (!e.getKey().startsWith(confField.varType().getPrefix())) {
continue;
}
allConfigs.put(e.getKey(), f);
@ -426,14 +411,14 @@ public class ConfigBase {
}
}
public static int getConfigNumByExperimentalType(ExperimentalType type) {
public static int getConfigNumByVariableAnnotation(VariableAnnotation type) {
int num = 0;
for (Field field : Config.class.getFields()) {
ConfField confField = field.getAnnotation(ConfField.class);
if (confField == null) {
continue;
}
if (confField.expType() == type) {
if (confField.varType() == type) {
++num;
}
}

31
fe/fe-common/src/main/java/org/apache/doris/common/ExperimentalUtil.java → fe/fe-common/src/main/java/org/apache/doris/common/VariableAnnotation.java
View File

@ -17,20 +17,25 @@
package org.apache.doris.common;
// Currently, this is for FE config and session variable.
public class ExperimentalUtil {
public static final String EXPERIMENTAL_PREFIX = "experimental_";
public enum VariableAnnotation {
NONE(""),
// A deprecated item and it will be deleted in future
DEPRECATED("deprecated_"),
// An experimental item, it will be shown with `experimental_` prefix
// And user can set it with or without `experimental_` prefix.
EXPERIMENTAL("experimental_"),
// A previous experimental item but now it is GA.
// it will be shown without `experimental_` prefix.
// But user can set it with or without `experimental_` prefix, for compatibility.
EXPERIMENTAL_ONLINE("");
private String prefix = "experimental_";
public enum ExperimentalType {
// Not an experimental item
NONE,
// An experimental item, it will be shown with `experimental_` prefix
// And user can set it with or without `experimental_` prefix.
EXPERIMENTAL,
// A previous experimental item but now it is GA.
// it will be shown without `experimental_` prefix.
// But user can set it with or without `experimental_` prefix, for compatibility.
EXPERIMENTAL_ONLINE
VariableAnnotation(String prefix) {
this.prefix = prefix;
}
public String getPrefix() {
return prefix;
}
}

12
fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java
View File

@ -19,8 +19,6 @@ package org.apache.doris.common.util;
import org.apache.doris.common.Config;
import org.apache.doris.common.ConfigBase.ConfField;
import org.apache.doris.common.ExperimentalUtil;
import org.apache.doris.common.ExperimentalUtil.ExperimentalType;
import org.apache.doris.qe.GlobalVariable;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.VariableMgr;
@ -162,10 +160,7 @@ public class DocGenerator {
if (confField == null) {
return null;
}
String configName = field.getName();
if (confField.expType() == ExperimentalType.EXPERIMENTAL) {
configName = ExperimentalUtil.EXPERIMENTAL_PREFIX + configName;
}
String configName = confField.varType().getPrefix() + field.getName();
sb.append("### `").append(configName).append("`\n\n");
sb.append(confField.description()[lang.idx]).append("\n\n");
sb.append(TYPE[lang.idx]).append("`").append(field.getType().getSimpleName()).append("`\n\n");
@ -243,10 +238,7 @@ public class DocGenerator {
return null;
}
StringBuilder sb = new StringBuilder();
String varName = varAttr.name();
if (varAttr.expType() == ExperimentalType.EXPERIMENTAL) {
varName = ExperimentalUtil.EXPERIMENTAL_PREFIX + varName;
}
String varName = varAttr.varType().getPrefix() + varAttr.name();
sb.append("### `").append(varName).append("`\n\n");
sb.append(varAttr.description()[lang.idx]).append("\n\n");
sb.append(TYPE[lang.idx]).append("`").append(field.getType().getSimpleName()).append("`\n\n");

22
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -22,7 +22,7 @@ import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExperimentalUtil.ExperimentalType;
import org.apache.doris.common.VariableAnnotation;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.TimeUtils;
@ -549,7 +549,7 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = ENABLE_COLOCATE_SCAN)
public boolean enableColocateScan = false;
@VariableMgr.VarAttr(name = ENABLE_BUCKET_SHUFFLE_JOIN, expType = ExperimentalType.EXPERIMENTAL_ONLINE)
@VariableMgr.VarAttr(name = ENABLE_BUCKET_SHUFFLE_JOIN, varType = VariableAnnotation.EXPERIMENTAL_ONLINE)
public boolean enableBucketShuffleJoin = true;
@VariableMgr.VarAttr(name = PREFER_JOIN_METHOD)
@ -636,13 +636,13 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = ENABLE_STRICT_CONSISTENCY_DML, needForward = true)
public boolean enableStrictConsistencyDml = false;
@VariableMgr.VarAttr(name = ENABLE_VECTORIZED_ENGINE, expType = ExperimentalType.EXPERIMENTAL_ONLINE)
@VariableMgr.VarAttr(name = ENABLE_VECTORIZED_ENGINE, varType = VariableAnnotation.EXPERIMENTAL_ONLINE)
public boolean enableVectorizedEngine = true;
@VariableMgr.VarAttr(name = ENABLE_PIPELINE_ENGINE, fuzzy = true, expType = ExperimentalType.EXPERIMENTAL)
@VariableMgr.VarAttr(name = ENABLE_PIPELINE_ENGINE, fuzzy = true, varType = VariableAnnotation.EXPERIMENTAL)
private boolean enablePipelineEngine = true;
@VariableMgr.VarAttr(name = ENABLE_AGG_STATE, fuzzy = false, expType = ExperimentalType.EXPERIMENTAL)
@VariableMgr.VarAttr(name = ENABLE_AGG_STATE, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL)
public boolean enableAggState = false;
@VariableMgr.VarAttr(name = ENABLE_PARALLEL_OUTFILE)
@ -795,7 +795,7 @@ public class SessionVariable implements Serializable, Writable {
* would be coming soon.
*/
@VariableMgr.VarAttr(name = ENABLE_NEREIDS_PLANNER, needForward = true,
fuzzy = true, expType = ExperimentalType.EXPERIMENTAL)
fuzzy = true, varType = VariableAnnotation.EXPERIMENTAL)
private boolean enableNereidsPlanner = true;
@VariableMgr.VarAttr(name = DISABLE_NEREIDS_RULES, needForward = true)
@ -840,7 +840,7 @@ public class SessionVariable implements Serializable, Writable {
public String sessionContext = "";
@VariableMgr.VarAttr(name = ENABLE_SINGLE_REPLICA_INSERT,
needForward = true, expType = ExperimentalType.EXPERIMENTAL)
needForward = true, varType = VariableAnnotation.EXPERIMENTAL)
public boolean enableSingleReplicaInsert = false;
@VariableMgr.VarAttr(name = ENABLE_FUNCTION_PUSHDOWN, fuzzy = true)
@ -852,7 +852,7 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = ENABLE_COMMON_EXPR_PUSHDOWN, fuzzy = true)
public boolean enableCommonExprPushdown = true;
@VariableMgr.VarAttr(name = ENABLE_LOCAL_EXCHANGE, fuzzy = true)
@VariableMgr.VarAttr(name = ENABLE_LOCAL_EXCHANGE, fuzzy = true, varType = VariableAnnotation.DEPRECATED)
public boolean enableLocalExchange = true;
/**
@ -2403,8 +2403,8 @@ public class SessionVariable implements Serializable, Writable {
VariableMgr.setVar(this, new SetVar(SessionVariable.ENABLE_NEREIDS_PLANNER, new StringLiteral("false")));
}
// return number of variables by given experimental type
public int getVariableNumByExperimentalType(ExperimentalType type) {
// return number of variables by given variable annotation
public int getVariableNumByVariableAnnotation(VariableAnnotation type) {
int num = 0;
Field[] fields = SessionVariable.class.getDeclaredFields();
for (Field f : fields) {
@ -2412,7 +2412,7 @@ public class SessionVariable implements Serializable, Writable {
if (varAttr == null) {
continue;
}
if (varAttr.expType() == type) {
if (varAttr.varType() == type) {
++num;
}
}

25
fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
View File

@ -33,9 +33,8 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.ExperimentalUtil;
import org.apache.doris.common.ExperimentalUtil.ExperimentalType;
import org.apache.doris.common.PatternMatcher;
import org.apache.doris.common.VariableAnnotation;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.persist.GlobalVarPersistInfo;
@ -307,16 +306,22 @@ public class VariableMgr {
private static VarContext setVarPreCheck(SetVar setVar) throws DdlException {
String varName = setVar.getVariable();
boolean hasExpPrefix = false;
if (varName.startsWith(ExperimentalUtil.EXPERIMENTAL_PREFIX)) {
varName = varName.substring(ExperimentalUtil.EXPERIMENTAL_PREFIX.length());
if (varName.startsWith(VariableAnnotation.EXPERIMENTAL.getPrefix())) {
varName = varName.substring(VariableAnnotation.EXPERIMENTAL.getPrefix().length());
hasExpPrefix = true;
}
if (varName.startsWith(VariableAnnotation.DEPRECATED.getPrefix())) {
varName = varName.substring(VariableAnnotation.DEPRECATED.getPrefix().length());
hasExpPrefix = true;
}
VarContext ctx = ctxByVarName.get(varName);
if (ctx == null) {
ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_SYSTEM_VARIABLE, setVar.getVariable());
}
// for non experimental variables, can not set it with "experimental_" prefix
if (hasExpPrefix && ctx.getField().getAnnotation(VarAttr.class).expType() == ExperimentalType.NONE) {
// for non-matched prefix, report an error
VariableAnnotation varType = ctx.getField().getAnnotation(VarAttr.class).varType();
if (hasExpPrefix && (!setVar.getVariable().startsWith(varType.getPrefix())
|| varType == VariableAnnotation.NONE)) {
ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_SYSTEM_VARIABLE, setVar.getVariable());
}
return ctx;
@ -678,11 +683,7 @@ public class VariableMgr {
for (Map.Entry<String, VarContext> entry : ctxByVarName.entrySet()) {
VarContext varContext = entry.getValue();
VarAttr varAttr = varContext.getField().getAnnotation(VarAttr.class);
if (varAttr.expType() == ExperimentalType.EXPERIMENTAL) {
result.put(ExperimentalUtil.EXPERIMENTAL_PREFIX + entry.getKey(), varContext);
} else {
result.put(entry.getKey(), varContext);
}
result.put(varAttr.varType().getPrefix() + entry.getKey(), varContext);
}
return ImmutableMap.copyOf(result);
}
@ -762,7 +763,7 @@ public class VariableMgr {
// Set to true if this variable is fuzzy
boolean fuzzy() default false;
ExperimentalType expType() default ExperimentalType.NONE;
VariableAnnotation varType() default VariableAnnotation.NONE;
// description for this config item.
// There should be 2 elements in the array.

10
fe/fe-core/src/test/java/org/apache/doris/analysis/AdminSetConfigStmtTest.java
View File

@ -23,9 +23,9 @@ import org.apache.doris.common.CaseSensibility;
import org.apache.doris.common.Config;
import org.apache.doris.common.ConfigBase;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExperimentalUtil.ExperimentalType;
import org.apache.doris.common.PatternMatcher;
import org.apache.doris.common.PatternMatcherWrapper;
import org.apache.doris.common.VariableAnnotation;
import org.apache.doris.utframe.TestWithFeService;
import org.junit.Assert;
@ -77,11 +77,17 @@ public class AdminSetConfigStmtTest extends TestWithFeService {
Assert.assertNotEquals(enableMtmv, Config.enable_mtmv);
// 3. show config
int num = ConfigBase.getConfigNumByExperimentalType(ExperimentalType.EXPERIMENTAL);
int num = ConfigBase.getConfigNumByVariableAnnotation(VariableAnnotation.EXPERIMENTAL);
PatternMatcher matcher = PatternMatcherWrapper.createMysqlPattern("%experimental%",
CaseSensibility.CONFIG.getCaseSensibility());
List<List<String>> results = ConfigBase.getConfigInfo(matcher);
Assert.assertEquals(num, results.size());
num = ConfigBase.getConfigNumByVariableAnnotation(VariableAnnotation.DEPRECATED);
matcher = PatternMatcherWrapper.createMysqlPattern("%deprecated%",
CaseSensibility.CONFIG.getCaseSensibility());
results = ConfigBase.getConfigInfo(matcher);
Assert.assertEquals(num, results.size());
}
}

4
fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
View File

@ -23,10 +23,10 @@ import org.apache.doris.analysis.ShowVariablesStmt;
import org.apache.doris.common.CaseSensibility;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.ExperimentalUtil.ExperimentalType;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.PatternMatcher;
import org.apache.doris.common.PatternMatcherWrapper;
import org.apache.doris.common.VariableAnnotation;
import org.apache.doris.common.util.ProfileManager;
import org.apache.doris.common.util.RuntimeProfile;
import org.apache.doris.load.ExportJob;
@ -132,7 +132,7 @@ public class SessionVariablesTest extends TestWithFeService {
matcher = PatternMatcherWrapper.createMysqlPattern(showStmt.getPattern(),
CaseSensibility.VARIABLES.getCaseSensibility());
}
int num = sessionVar.getVariableNumByExperimentalType(ExperimentalType.EXPERIMENTAL);
int num = sessionVar.getVariableNumByVariableAnnotation(VariableAnnotation.EXPERIMENTAL);
List<List<String>> result = VariableMgr.dump(showStmt.getType(), sessionVar, matcher);
Assert.assertEquals(num, result.size());
}