database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](config) support "experimental" prefix for FE config (#18699)

Browse Source 
For each release of Doris, there are some experimental features.
These feature may not stable or qualified enough, and user need to use it by setting config or session variables,
eg, set enable_mtmv = true, otherwise, these feature is disable by default.

We should explicitly tell user which features are experimental, so that user will notice that and decide whether to
use it.

Changes
In this PR, I support the experimental_ prefix for FE config and session variables.

Session Variable

Given enable_nereids_planner as an example.

The Nereids planner is an experimental feature in Doris, so there is an EXPERIMENTAL annotation for it:

@VariableMgr.VarAttr(..., expType = ExperimentalType.EXPERIMENTAL)
private boolean enableNereidsPlanner = false;
And for compatibility, user can set it by:

set enable_nereids_planner = true;
set experimental_enable_nereids_planner = true;
And for show variables, it will only show experimental_enable_nereids_planner entry.

And you can also see all experimental session variables by:

show variables like "%experimental%"
Config

Same as session variable, give enable_mtmv as an example.

@ConfField(..., expType = ExperimentalType.EXPERIMENTAL)
public static boolean enable_mtmv = false;
User can set it in fe.conf or ADMIN SET FRONTEND CONFIG stmt with both names:

enable_mtmv
experimental_enable_mtmv
And user can see all experimental FE configs by:

ADMIN SHOW FRONTEND CONFIG LIKE "%experimental%";
TODO
Support this feature for BE config

Only add experimental for:

enable_pipeline_engine
enable_nereids_planner
enable_single_replica_insert
and FE config:

enable_mtmv
enabel_ssl
enable_fqdn_mode
Should modify other config and session vars
This commit is contained in:
Mingyu Chen
2023-04-16 18:32:10 +08:00
committed by GitHub
parent 652e7fe4ab
commit 1cbbc60822
10 changed files with 299 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.analysis.SetVar;
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExperimentalUtil.ExperimentalType;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.TimeUtils;
@ -459,7 +460,7 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = ENABLE_COLOCATE_SCAN)
public boolean enableColocateScan = false;
@VariableMgr.VarAttr(name = ENABLE_BUCKET_SHUFFLE_JOIN)
@VariableMgr.VarAttr(name = ENABLE_BUCKET_SHUFFLE_JOIN, expType = ExperimentalType.EXPERIMENTAL_ONLINE)
public boolean enableBucketShuffleJoin = true;
@VariableMgr.VarAttr(name = PREFER_JOIN_METHOD)
@ -529,7 +530,7 @@ public class SessionVariable implements Serializable, Writable {
public boolean extractWideRangeExpr = true;
@VariableMgr.VarAttr(name = ENABLE_PIPELINE_ENGINE, fuzzy = true)
@VariableMgr.VarAttr(name = ENABLE_PIPELINE_ENGINE, fuzzy = true, expType = ExperimentalType.EXPERIMENTAL)
public boolean enablePipelineEngine = false;
@VariableMgr.VarAttr(name = ENABLE_PARALLEL_OUTFILE)
@ -620,7 +621,7 @@ public class SessionVariable implements Serializable, Writable {
* the new optimizer is fully developed. I hope that day
* would be coming soon.
*/
@VariableMgr.VarAttr(name = ENABLE_NEREIDS_PLANNER, needForward = true)
@VariableMgr.VarAttr(name = ENABLE_NEREIDS_PLANNER, needForward = true, expType = ExperimentalType.EXPERIMENTAL)
private boolean enableNereidsPlanner = false;
@VariableMgr.VarAttr(name = DISABLE_NEREIDS_RULES, needForward = true)
@ -661,7 +662,8 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = SESSION_CONTEXT, needForward = true)
public String sessionContext = "";
@VariableMgr.VarAttr(name = ENABLE_SINGLE_REPLICA_INSERT, needForward = true)
@VariableMgr.VarAttr(name = ENABLE_SINGLE_REPLICA_INSERT,
needForward = true, expType = ExperimentalType.EXPERIMENTAL)
public boolean enableSingleReplicaInsert = false;
@VariableMgr.VarAttr(name = ENABLE_FUNCTION_PUSHDOWN)
@ -743,7 +745,6 @@ public class SessionVariable implements Serializable, Writable {
public boolean enableUnicodeNameSupport = false;
@VariableMgr.VarAttr(name = REPEAT_MAX_NUM, needForward = true)
public int repeatMaxNum = 10000;
@VariableMgr.VarAttr(name = GROUP_CONCAT_MAX_LEN)
@ -1922,7 +1923,22 @@ public class SessionVariable implements Serializable, Writable {
return;
}
setIsSingleSetVar(true);
VariableMgr.setVar(this,
new SetVar(SessionVariable.ENABLE_NEREIDS_PLANNER, new StringLiteral("false")));
VariableMgr.setVar(this, new SetVar(SessionVariable.ENABLE_NEREIDS_PLANNER, new StringLiteral("false")));
}
// return number of variables by given experimental type
public int getVariableNumByExperimentalType(ExperimentalType type) {
int num = 0;
Field[] fields = SessionVariable.class.getDeclaredFields();
for (Field f : fields) {
VarAttr varAttr = f.getAnnotation(VarAttr.class);
if (varAttr == null) {
continue;
}
if (varAttr.expType() == type) {
++num;
}
}
return num;
}
}

45
fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
View File

@ -26,6 +26,8 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.ExperimentalUtil;
import org.apache.doris.common.ExperimentalUtil.ExperimentalType;
import org.apache.doris.common.PatternMatcher;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.persist.GlobalVarPersistInfo;
@ -34,6 +36,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.commons.lang.SerializationUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -108,6 +111,10 @@ public class VariableMgr {
// Map variable name to variable context which have enough information to change variable value.
// This map contains info of all session and global variables.
private static ImmutableMap<String, VarContext> ctxByVarName;
// Built from ctxByVarName.
// If a session variable "foo" is an experimental variable,
// its display name is "experimental_foo"
private static ImmutableMap<String, VarContext> ctxByDisplayVarName;
// This variable is equivalent to the default value of session variables.
// Whenever a new session is established, the value in this object is copied to the session-level variable.
@ -131,6 +138,7 @@ public class VariableMgr {
defaultSessionVariable = new SessionVariable();
ImmutableSortedMap.Builder<String, VarContext> builder = getStringVarContextBuilder(defaultSessionVariable);
ctxByVarName = builder.build();
ctxByDisplayVarName = getDisplaySessionVars();
}
public static SessionVariable getDefaultSessionVariable() {
@ -243,10 +251,20 @@ public class VariableMgr {
// setVar: variable information that needs to be set
public static void setVar(SessionVariable sessionVariable, SetVar setVar)
throws DdlException {
VarContext ctx = ctxByVarName.get(setVar.getVariable());
String varName = setVar.getVariable();
boolean hasExpPrefix = false;
if (varName.startsWith(ExperimentalUtil.EXPERIMENTAL_PREFIX)) {
varName = varName.substring(ExperimentalUtil.EXPERIMENTAL_PREFIX.length());
hasExpPrefix = true;
}
VarContext ctx = ctxByVarName.get(varName);
if (ctx == null) {
ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_SYSTEM_VARIABLE, setVar.getVariable());
}
// for non experimental variables, can not set it with "experimental_" prefix
if (hasExpPrefix && ctx.getField().getAnnotation(VarAttr.class).expType() == ExperimentalType.NONE) {
ErrorReport.reportDdlException(ErrorCode.ERR_UNKNOWN_SYSTEM_VARIABLE, setVar.getVariable());
}
// Check variable attribute and setVar
checkUpdate(setVar, ctx.getFlag());
@ -518,13 +536,34 @@ public class VariableMgr {
return "";
}
/**
* return the VarContext map with display var name.
* For example, if a session variable "foo" is an experimental variable,
* its display name is "experimental_foo"
*
* @return
*/
private static ImmutableMap<String, VarContext> getDisplaySessionVars() {
Map<String, VarContext> result = Maps.newHashMap();
for (Map.Entry<String, VarContext> entry : ctxByVarName.entrySet()) {
VarContext varContext = entry.getValue();
VarAttr varAttr = varContext.getField().getAnnotation(VarAttr.class);
if (varAttr.expType() == ExperimentalType.EXPERIMENTAL) {
result.put(ExperimentalUtil.EXPERIMENTAL_PREFIX + entry.getKey(), varContext);
} else {
result.put(entry.getKey(), varContext);
}
}
return ImmutableMap.copyOf(result);
}
// Dump all fields. Used for `show variables`
public static List<List<String>> dump(SetType type, SessionVariable sessionVar, PatternMatcher matcher) {
List<List<String>> rows = Lists.newArrayList();
// Hold the read lock when session dump, because this option need to access global variable.
rlock.lock();
try {
for (Map.Entry<String, VarContext> entry : ctxByVarName.entrySet()) {
for (Map.Entry<String, VarContext> entry : ctxByDisplayVarName.entrySet()) {
// Filter variable not match to the regex.
if (matcher != null && !matcher.match(entry.getKey())) {
continue;
@ -589,6 +628,8 @@ public class VariableMgr {
// Set to true if this variable is fuzzy
boolean fuzzy() default false;
ExperimentalType expType() default ExperimentalType.NONE;
}
private static class VarContext {