database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](regression-test) support exclude suite/group/directory (#9096)

Browse Source 
regression testing framework support skip some suite/group/directory
This commit is contained in:
924060929
2022-05-05 09:50:07 +08:00
committed by GitHub
parent 23d673a178
commit 4ca3eb7fe2
7 changed files with 162 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
docs/zh-CN/developer-guide/regression-testing.md
View File

@ -109,6 +109,13 @@ testSuites = ""
// 默认会加载的用例目录, 可以通过run-regression-test.sh --run -d来动态指定和覆盖
testDirectories = ""
// 排除这些组的用例，可通过run-regression-test.sh --run -xg来动态指定和覆盖
excludeGroups = ""
// 排除这些suite，可通过run-regression-test.sh --run -xs来动态指定和覆盖
excludeSuites = ""
// 排除这些目录，可通过run-regression-test.sh --run -xd来动态指定和覆盖
excludeDirectories = ""
// 其他自定义配置
customConf1 = "test_custom_conf_value"
```
@ -517,6 +524,15 @@ thread, lazyCheck, events, connect, selectUnionAll
# 测试demo目录下的sql_action
./run-regression-test.sh --run -d demo -s sql_action
# 测试demo目录下用例，排除sql_action用例
./run-regression-test.sh --run -d demo -xs sql_action
# 排除demo目录的用例
./run-regression-test.sh --run -xd demo
# 排除demo group的用例
./run-regression-test.sh --run -xg demo
# 自定义配置
./run-regression-test.sh --run -conf a=b

7
regression-test/conf/regression-conf.groovy
View File

@ -41,6 +41,13 @@ testSuites = ""
// empty directories will test all directories
testDirectories = ""
// this groups will not be executed
excludeGroups = ""
// this suites will not be executed
excludeSuites = ""
// this directories will not be executed
excludeDirectories = ""
customConf1 = "test_custom_conf_value"
// for test csv with header

63
regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
View File

@ -47,8 +47,11 @@ class Config {
public String dataPath
public String testGroups
public String excludeGroups
public String testSuites
public String excludeSuites
public String testDirectories
public String excludeDirectories
public boolean generateOutputFile
public boolean forceGenerateOutputFile
public boolean randomOrder
@ -58,6 +61,11 @@ class Config {
public Set<String> suiteWildcard = new HashSet<>()
public Set<String> groups = new HashSet<>()
public Set<String> directories = new HashSet<>()
public Set<String> excludeSuiteWildcard = new HashSet<>()
public Set<String> excludeGroupSet = new HashSet<>()
public Set<String> excludeDirectorySet = new HashSet<>()
public InetSocketAddress feHttpInetSocketAddress
public Integer parallel
public Integer suiteParallel
@ -69,7 +77,8 @@ class Config {
Config(String defaultDb, String jdbcUrl, String jdbcUser, String jdbcPassword,
String feHttpAddress, String feHttpUser, String feHttpPassword,
String suitePath, String dataPath, String testGroups, String testSuites, String testDirectories) {
String suitePath, String dataPath, String testGroups, String excludeGroups,
String testSuites, String excludeSuites, String testDirectories, String excludeDirectories) {
this.defaultDb = defaultDb
this.jdbcUrl = jdbcUrl
this.jdbcUser = jdbcUser
@ -80,8 +89,11 @@ class Config {
this.suitePath = suitePath
this.dataPath = dataPath
this.testGroups = testGroups
this.excludeGroups = excludeGroups
this.testSuites = testSuites
this.excludeSuites = excludeSuites
this.testDirectories = testDirectories
this.excludeDirectories = excludeDirectories
}
static Config fromCommandLine(CommandLine cmd) {
@ -103,8 +115,8 @@ class Config {
config.dataPath = FileUtils.getCanonicalPath(cmd.getOptionValue(dataOpt, config.dataPath))
config.suiteWildcard = cmd.getOptionValue(suiteOpt, config.testSuites)
.split(",")
.collect({g -> g.trim()})
.findAll({g -> g != null && g.length() > 0})
.collect({s -> s.trim()})
.findAll({s -> s != null && s.length() > 0})
.toSet()
config.groups = cmd.getOptionValue(groupsOpt, config.testGroups)
.split(",")
@ -116,6 +128,21 @@ class Config {
.collect({d -> d.trim()})
.findAll({d -> d != null && d.length() > 0})
.toSet()
config.excludeSuiteWildcard = cmd.getOptionValue(excludeSuiteOpt, config.excludeSuites)
.split(",")
.collect({s -> s.trim()})
.findAll({s -> s != null && s.length() > 0})
.toSet()
config.excludeGroupSet = cmd.getOptionValue(excludeGroupsOpt, config.excludeGroups)
.split(",")
.collect({g -> g.trim()})
.findAll({g -> g != null && g.length() > 0})
.toSet()
config.excludeDirectorySet = cmd.getOptionValue(excludeDirectoriesOpt, config.excludeDirectories)
.split(",")
.collect({d -> d.trim()})
.findAll({d -> d != null && d.length() > 0})
.toSet()
config.feHttpAddress = cmd.getOptionValue(feHttpAddressOpt, config.feHttpAddress)
try {
@ -162,8 +189,11 @@ class Config {
configToString(obj.suitePath),
configToString(obj.dataPath),
configToString(obj.testGroups),
configToString(obj.excludeGroups),
configToString(obj.testSuites),
configToString(obj.testDirectories)
configToString(obj.excludeSuites),
configToString(obj.testDirectories),
configToString(obj.excludeDirectories)
)
def declareFileNames = config.getClass()
@ -230,16 +260,31 @@ class Config {
log.info("Set testGroups to '${config.testGroups}' because not specify.".toString())
}
if (config.excludeGroups == null) {
config.excludeGroups = ""
log.info("Set excludeGroups to empty because not specify.".toString())
}
if (config.testDirectories == null) {
config.testDirectories = ""
log.info("Set testDirectories to empty because not specify.".toString())
}
if (config.excludeDirectories == null) {
config.excludeDirectories = ""
log.info("Set excludeDirectories to empty because not specify.".toString())
}
if (config.testSuites == null) {
config.testSuites = ""
log.info("Set testSuites to empty because not specify.".toString())
}
if (config.excludeSuites == null) {
config.excludeSuites = ""
log.info("Set excludeSuites to empty because not specify.".toString())
}
if (config.parallel == null) {
config.parallel = 1
log.info("Set parallel to 1 because not specify.".toString())
@ -289,7 +334,7 @@ class Config {
Predicate<String> getDirectoryFilter() {
return (Predicate<String>) { String directoryName ->
if (directories.isEmpty()) {
if (directories.isEmpty() && excludeDirectorySet.isEmpty()) {
return true
}
@ -302,7 +347,13 @@ class Config {
parentPath = currentPath + File.separator
}
return allLevelPaths.any {directories.contains(it) }
if (!directories.isEmpty() && !allLevelPaths.any({directories.contains(it) })) {
return false
}
if (!excludeDirectorySet.isEmpty() && allLevelPaths.any({ excludeDirectorySet.contains(it) })) {
return false
}
return true
}
}

33
regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
View File

@ -38,8 +38,11 @@ class ConfigOptions {
static Option pathOpt
static Option dataOpt
static Option suiteOpt
static Option excludeSuiteOpt
static Option groupsOpt
static Option excludeGroupsOpt
static Option directoriesOpt
static Option excludeDirectoriesOpt
static Option confOpt
static Option genOutOpt
static Option forceGenOutOpt
@ -123,6 +126,15 @@ class ConfigOptions {
.longOpt("suite")
.desc("the suite name wildcard to be test")
.build()
excludeSuiteOpt = Option.builder("xs")
.argName("excludeSuiteName")
.required(false)
.hasArg(true)
.optionalArg(true)
.type(String.class)
.longOpt("excludeSuite")
.desc("the suite name wildcard will not be tested")
.build()
groupsOpt = Option.builder("g")
.argName("groups")
.required(false)
@ -132,6 +144,15 @@ class ConfigOptions {
.longOpt("groups")
.desc("the suite group to be test")
.build()
excludeGroupsOpt = Option.builder("xg")
.argName("excludeGroupNames")
.required(false)
.hasArg(true)
.optionalArg(true)
.type(String.class)
.longOpt("excludeGroups")
.desc("the suite group will not be tested")
.build()
directoriesOpt = Option.builder("d")
.argName("directories")
.required(false)
@ -141,6 +162,15 @@ class ConfigOptions {
.longOpt("directories")
.desc("only the use cases in these directories can be executed")
.build()
excludeDirectoriesOpt = Option.builder("xd")
.argName("excludeDirectoryNames")
.required(false)
.hasArg(true)
.optionalArg(true)
.type(String.class)
.longOpt("excludeDirectories")
.desc("the use cases in these directories will not be tested")
.build()
feHttpAddressOpt = Option.builder("ha")
.argName("address")
.required(false)
@ -239,8 +269,11 @@ class ConfigOptions {
.addOption(dataOpt)
.addOption(confOpt)
.addOption(suiteOpt)
.addOption(excludeSuiteOpt)
.addOption(groupsOpt)
.addOption(excludeGroupsOpt)
.addOption(directoriesOpt)
.addOption(excludeDirectoriesOpt)
.addOption(feHttpAddressOpt)
.addOption(feHttpUserOpt)
.addOption(feHttpPasswordOpt)

45
regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy
View File

@ -173,18 +173,39 @@ class RegressionTest {
return recorder
}
static boolean canRun(Config config, String suiteName, String group) {
Set<String> suiteGroups = group.split(',').collect { g -> g.trim() }.toSet()
if (config.suiteWildcard.size() == 0 ||
(suiteName != null && (config.suiteWildcard.any {
suiteWildcard -> Wildcard.match(suiteName, suiteWildcard)
}))) {
if (config.groups == null || config.groups.isEmpty()
|| !config.groups.intersect(suiteGroups).isEmpty()) {
return true
}
static boolean filterSuites(Config config, String suiteName) {
if (config.suiteWildcard.isEmpty() && config.excludeSuiteWildcard.isEmpty()) {
return true
}
return false
if (!config.suiteWildcard.isEmpty() && !config.suiteWildcard.any {
suiteWildcard -> Wildcard.match(suiteName, suiteWildcard)
}) {
return false
}
if (!config.excludeSuiteWildcard.isEmpty() && config.excludeSuiteWildcard.any {
excludeSuiteWildcard -> Wildcard.match(suiteName, excludeSuiteWildcard)
}) {
return false
}
return true
}
static boolean filterGroups(Config config, String group) {
if (config.groups.isEmpty() && config.excludeGroupSet.isEmpty()) {
return true
}
Set<String> suiteGroups = group.split(',').collect { g -> g.trim() }.toSet()
if (!config.groups.isEmpty() && config.groups.intersect(suiteGroups).isEmpty()) {
return false
}
if (!config.excludeGroupSet.isEmpty() && !config.excludeGroupSet.intersect(suiteGroups).isEmpty()) {
return false
}
return true
}
static boolean canRun(Config config, String suiteName, String group) {
return filterGroups(config, group) && filterSuites(config, suiteName)
}
static List<EventListener> getEventListeners(Config config, Recorder recorder) {
@ -226,7 +247,7 @@ class RegressionTest {
String successList = recorder.successList.collect { info ->
"${info.file.absolutePath}: group=${info.group}, name=${info.suiteName}"
}.join('\n')
log.info("SuccessList suites:\n${successList}".toString())
log.info("Success suites:\n${successList}".toString())
}
// print failure list

23
regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptSource.groovy
View File

@ -59,26 +59,27 @@ class SqlFileSource implements ScriptSource {
return SuiteScript.getDefaultGroups(suiteRoot, file)
}
List<String> getSqls(String sql) {
try {
return SqlUtils.splitAndGetNonEmptySql(sql)
} catch (Throwable t) {
log.warn("Try to execute whole file text as one sql, because can not split sql:\n${sql}", t)
return [sql]
}
}
@Override
SuiteScript toScript(ScriptContext scriptContext, GroovyShell shell) {
String suiteName = file.name.substring(0, file.name.lastIndexOf("."))
String groupName = getGroup()
boolean order = suiteName.endsWith("_order")
String tag = suiteName
String sql = file.text
List<String> sqls
try {
sqls = SqlUtils.splitAndGetNonEmptySql(sql)
} catch (Throwable t) {
sqls = [sql]
log.warn("Try to execute whole file text as one sql, because can not split sql:\n${sql}", t)
}
SuiteScript script = new SuiteScript() {
@Override
Object run() {
suite(suiteName, groupName) {
String tag = suiteName
boolean order = suiteName.endsWith("_order")
List<String> sqls = getSqls(file.text)
for (int i = 0; i < sqls.size(); ++i) {
String singleSql = sqls.get(i)
String tagName = (i == 0) ? tag : "${tag}_${i + 1}"

24
run-regression-test.sh
View File

@ -16,26 +16,6 @@
# specific language governing permissions and limitations
# under the License.
#####################################################################
# This script is used to run regression test of Doris Backend
# Usage: $0 <shell_options> <framework_options>
# Optional shell_options:
# --clean clean output of regression test
# --teamcity print teamcity service messages
# --run run regression test. build framework if necessary
#
# Optional framework_options
# -h print all other_options
# -s xxx suite name
# -g xxx group name
# -c xxx jdbc url
# -u xxx jdbc user
# -genOut generate .out file
# -forceGenOut delete and generate .out file
#
# log to ${DORIS_HOME}/output/regression/log
#####################################################################
set -eo pipefail
#set -x
@ -56,7 +36,11 @@ Usage: $0 <shell_options> <framework_options>
Optional framework_options:
-s run a specified suite
-g run a specified group
-d run a specified directory
-h **print all framework options usage**
-xs exclude the specified suite
-xg exclude the specified group
-xd exclude the specified directory
-genOut generate .out file if not exist
-forceGenOut delete and generate .out file if not exist
-parallel run tests using specified threads