database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[regression](load)Open broker load regression test (#13163)

Browse Source
This commit is contained in:
Jibing-Li
2022-10-10 18:49:44 +08:00
committed by GitHub
parent f007e0aed0
commit 7c0695c793
2 changed files with 29 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
regression-test/conf/regression-conf.groovy
View File

@ -62,6 +62,6 @@ hdfsPasswd = ""
brokerName = "broker_name"
// broker load test config
enableBrokerLoad=false
enableBrokerLoad=true
ak=""
sk=""

54
regression-test/suites/load_p0/broker_load/test_broker_load.groovy
View File

@ -39,27 +39,27 @@ suite("test_broker_load", "p0") {
"parquet_s3_case7", // col5 will be ignored, load normally
"parquet_s3_case8" // first column in table is not specified, will load default value for it.
]
def paths = ["s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/path/*/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
"s3://doris-community-test-1308700295/load/data/part*",
def paths = ["s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/path/*/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
"s3://doris-build-hk-1308700295/regression/load/data/part*",
]
def columns_list = ["""p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",
"""p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",
@ -148,7 +148,7 @@ suite("test_broker_load", "p0") {
"",
"type:LOAD_RUN_FAIL; msg:errCode = 2, detailMessage = failed to find default value expr for slot: x1",
"",
"type:LOAD_RUN_FAIL; msg:errCode = 2, detailMessage = failed to init reader for file s3://doris-community-test-1308700295/load/data/part-00000-cb9099f7-a053-4f9a-80af-c659cfa947cc-c000.snappy.parquet, err: No columns found in file",
"type:LOAD_RUN_FAIL; msg:errCode = 2, detailMessage = failed to init reader for file s3://doris-build-hk-1308700295/regression/load/data/part-00000-cb9099f7-a053-4f9a-80af-c659cfa947cc-c000.snappy.parquet, err: No columns found in file",
"type:LOAD_RUN_FAIL; msg:errCode = 2, detailMessage = failed to find default value expr for slot: x1",
"",
"",
@ -181,7 +181,7 @@ suite("test_broker_load", "p0") {
"""
}
def set_be_config = { ->
def set_be_config = { flag->
String[][] backends = sql """ show backends; """
assertTrue(backends.size() > 0)
for (String[] backend in backends) {
@ -191,9 +191,9 @@ suite("test_broker_load", "p0") {
setConfigCommand.append(":")
setConfigCommand.append(backend[5])
setConfigCommand.append("/api/update_config?")
String command1 = setConfigCommand.toString() + "enable_new_load_scan_node=true"
String command1 = setConfigCommand.toString() + "enable_new_load_scan_node=$flag"
logger.info(command1)
String command2 = setConfigCommand.toString() + "enable_new_file_scanner=true"
String command2 = setConfigCommand.toString() + "enable_new_file_scanner=$flag"
logger.info(command2)
def process1 = command1.execute()
int code = process1.waitFor()
@ -207,7 +207,7 @@ suite("test_broker_load", "p0") {
if (enabled != null && enabled.equalsIgnoreCase("true")) {
def uuids = []
sql """ADMIN SET FRONTEND CONFIG ("enable_new_load_scan_node" = "true");"""
set_be_config.call()
set_be_config.call('true')
try {
i = 0
for (String table in tables) {
@ -251,6 +251,8 @@ suite("test_broker_load", "p0") {
order_qt_parquet_s3_case8 """ select count(*) from parquet_s3_case8 where p_partkey=1"""
} finally {
sql """ADMIN SET FRONTEND CONFIG ("enable_new_load_scan_node" = "false");"""
set_be_config.call('false')
for (String table in tables) {
sql new File("""${context.file.parent}/ddl/${table}_drop.sql""").text
}