database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](config) support "experimental" prefix for FE config (#18699)

Browse Source 
For each release of Doris, there are some experimental features.
These feature may not stable or qualified enough, and user need to use it by setting config or session variables,
eg, set enable_mtmv = true, otherwise, these feature is disable by default.

We should explicitly tell user which features are experimental, so that user will notice that and decide whether to
use it.

Changes
In this PR, I support the experimental_ prefix for FE config and session variables.

Session Variable

Given enable_nereids_planner as an example.

The Nereids planner is an experimental feature in Doris, so there is an EXPERIMENTAL annotation for it:

@VariableMgr.VarAttr(..., expType = ExperimentalType.EXPERIMENTAL)
private boolean enableNereidsPlanner = false;
And for compatibility, user can set it by:

set enable_nereids_planner = true;
set experimental_enable_nereids_planner = true;
And for show variables, it will only show experimental_enable_nereids_planner entry.

And you can also see all experimental session variables by:

show variables like "%experimental%"
Config

Same as session variable, give enable_mtmv as an example.

@ConfField(..., expType = ExperimentalType.EXPERIMENTAL)
public static boolean enable_mtmv = false;
User can set it in fe.conf or ADMIN SET FRONTEND CONFIG stmt with both names:

enable_mtmv
experimental_enable_mtmv
And user can see all experimental FE configs by:

ADMIN SHOW FRONTEND CONFIG LIKE "%experimental%";
TODO
Support this feature for BE config

Only add experimental for:

enable_pipeline_engine
enable_nereids_planner
enable_single_replica_insert
and FE config:

enable_mtmv
enabel_ssl
enable_fqdn_mode
Should modify other config and session vars
This commit is contained in:
Mingyu Chen
2023-04-16 18:32:10 +08:00
committed by GitHub
parent 652e7fe4ab
commit 1cbbc60822
10 changed files with 299 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-common/src/main/java/org/apache/doris/common/Config.java
View File

@ -17,6 +17,8 @@
package org.apache.doris.common;
import org.apache.doris.common.ExperimentalUtil.ExperimentalType;
public class Config extends ConfigBase {
/**
@ -1827,7 +1829,7 @@ public class Config extends ConfigBase {
/*
* mtmv is still under dev, remove this config when it is graduate.
*/
@ConfField(mutable = true, masterOnly = true)
@ConfField(mutable = true, masterOnly = true, expType = ExperimentalType.EXPERIMENTAL)
public static boolean enable_mtmv = false;
/* Max running task num at the same time, otherwise the submitted task will still be keep in pending poll*/
@ -1999,7 +2001,7 @@ public class Config extends ConfigBase {
* When enable_fqdn_mode is true, the name of the pod where be is located will remain unchanged
* after reconstruction, while the ip can be changed.
*/
@ConfField(mutable = false, masterOnly = true)
@ConfField(mutable = false, masterOnly = true, expType = ExperimentalType.EXPERIMENTAL)
public static boolean enable_fqdn_mode = false;
/**
@ -2036,7 +2038,7 @@ public class Config extends ConfigBase {
/**
* If set to ture, doris will establish an encrypted channel based on the SSL protocol with mysql.
*/
@ConfField(mutable = false, masterOnly = false)
@ConfField(mutable = false, masterOnly = false, expType = ExperimentalType.EXPERIMENTAL)
public static boolean enable_ssl = true;
/**

82
fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java
View File

@ -17,6 +17,8 @@
package org.apache.doris.common;
import org.apache.doris.common.ExperimentalUtil.ExperimentalType;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@ -46,10 +48,14 @@ public class ConfigBase {
@Retention(RetentionPolicy.RUNTIME)
public @interface ConfField {
String value() default "";
boolean mutable() default false;
boolean masterOnly() default false;
String comment() default "";
ExperimentalType expType() default ExperimentalType.NONE;
Class<? extends ConfHandler> callback() default DefaultConfHandler.class;
}
@ -87,7 +93,11 @@ public class ConfigBase {
if (confField == null) {
continue;
}
confFields.put(confField.value().equals("") ? field.getName() : confField.value(), field);
confFields.put(field.getName(), field);
if (confField.expType() == ExperimentalType.EXPERIMENTAL
|| confField.expType() == ExperimentalType.EXPERIMENTAL_ONLINE) {
confFields.put(ExperimentalUtil.EXPERIMENTAL_PREFIX + field.getName(), field);
}
}
initConf(confFile);
@ -100,7 +110,11 @@ public class ConfigBase {
if (confField == null) {
continue;
}
ldapConfFields.put(confField.value().equals("") ? field.getName() : confField.value(), field);
ldapConfFields.put(field.getName(), field);
if (confField.expType() == ExperimentalType.EXPERIMENTAL
|| confField.expType() == ExperimentalType.EXPERIMENTAL_ONLINE) {
ldapConfFields.put(ExperimentalUtil.EXPERIMENTAL_PREFIX + field.getName(), field);
}
}
initConf(ldapConfFile);
}
@ -129,7 +143,7 @@ public class ConfigBase {
for (Field f : fields) {
ConfField anno = f.getAnnotation(ConfField.class);
if (anno != null) {
map.put(anno.value().isEmpty() ? f.getName() : anno.value(), getConfValue(f));
map.put(f.getName(), getConfValue(f));
}
}
return map;
@ -199,8 +213,9 @@ public class ConfigBase {
}
// ensure that field has property string
String confKey = anno.value().equals("") ? f.getName() : anno.value();
String confVal = props.getProperty(confKey);
String confKey = f.getName();
String confVal = props.getProperty(confKey,
props.getProperty(ExperimentalUtil.EXPERIMENTAL_PREFIX + confKey));
if (Strings.isNullOrEmpty(confVal)) {
continue;
}
@ -214,7 +229,7 @@ public class ConfigBase {
}
}
public static void setConfigField(Field f, String confVal) throws Exception {
private static void setConfigField(Field f, String confVal) throws Exception {
confVal = confVal.trim();
String[] sa = confVal.split(",");
@ -320,22 +335,49 @@ public class ConfigBase {
LOG.info("set config {} to {}", key, value);
}
/**
* Get display name of experimental configs.
* For an experimental config, the given "configsToFilter" contains both config w/o "experimental_" prefix.
* We need to return the right display name for these configs, by following rules:
* 1. If this config is EXPERIMENTAL, only return the config with "experimental_" prefix.
* 2. If this config is not EXPERIMENTAL, only return the config without "experimental_" prefix.
*
* @param configsToFilter
* @param allConfigs
*/
private static void getDisplayConfigInfo(Map<String, Field> configsToFilter, Map<String, Field> allConfigs) {
for (Map.Entry<String, Field> e : configsToFilter.entrySet()) {
Field f = e.getValue();
ConfField confField = f.getAnnotation(ConfField.class);
boolean isExperimental = e.getKey().startsWith(ExperimentalUtil.EXPERIMENTAL_PREFIX);
if (isExperimental && confField.expType() != ExperimentalType.EXPERIMENTAL) {
continue;
}
if (!isExperimental && confField.expType() == ExperimentalType.EXPERIMENTAL) {
continue;
}
allConfigs.put(e.getKey(), f);
}
}
public static synchronized List<List<String>> getConfigInfo(PatternMatcher matcher) {
Map<String, Field> allConfFields = Maps.newHashMap();
allConfFields.putAll(confFields);
allConfFields.putAll(ldapConfFields);
getDisplayConfigInfo(confFields, allConfFields);
getDisplayConfigInfo(ldapConfFields, allConfFields);
return allConfFields.entrySet().stream().sorted(Map.Entry.comparingByKey()).flatMap(e -> {
String confKey = e.getKey();
Field f = e.getValue();
ConfField anno = f.getAnnotation(ConfField.class);
ConfField confField = f.getAnnotation(ConfField.class);
if (matcher == null || matcher.match(confKey)) {
List<String> config = Lists.newArrayList();
config.add(confKey);
config.add(getConfValue(f));
config.add(f.getType().getSimpleName());
config.add(String.valueOf(anno.mutable()));
config.add(String.valueOf(anno.masterOnly()));
config.add(anno.comment());
config.add(String.valueOf(confField.mutable()));
config.add(String.valueOf(confField.masterOnly()));
config.add(confField.comment());
return Stream.of(config);
} else {
return Stream.empty();
@ -379,4 +421,18 @@ public class ConfigBase {
+ "will overwrite the configurations in fe.conf");
}
}
public static int getConfigNumByExperimentalType(ExperimentalType type) {
int num = 0;
for (Field field : Config.class.getFields()) {
ConfField confField = field.getAnnotation(ConfField.class);
if (confField == null) {
continue;
}
if (confField.expType() == type) {
++num;
}
}
return num;
}
}

36
fe/fe-common/src/main/java/org/apache/doris/common/ExperimentalUtil.java Normal file
View File

@ -0,0 +1,36 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.common;
// Currently, this is for FE config and session variable.
public class ExperimentalUtil {
public static final String EXPERIMENTAL_PREFIX = "experimental_";
public enum ExperimentalType {
// Not an experimental item
NONE,
// An experimental item, it will be shown with `experimental_` prefix
// And user can set it with or without `experimental_` prefix.
EXPERIMENTAL,
// A previous experimental item but now it is GA.
// it will be shown without `experimental_` prefix.
// But user can set it with or without `experimental_` prefix, for compatibility.
EXPERIMENTAL_ONLINE
}
}