database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[regression](load)split dataset to cover more situation (#12311)

Browse Source
This commit is contained in:
Dongyang Li
2022-09-05 19:25:01 +08:00
committed by GitHub
parent e175a7ed63
commit a47eb55d7c
3 changed files with 105 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
regression-test/suites/ssb_sf1/load.groovy
View File

@ -57,7 +57,41 @@ suite("load") {
set 'columns', columns[i]
// relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
// also, you can stream load a http stream, e.g. http://xxx/some.csv
file """${context.sf1DataPath}/ssb/sf1/${tableName}.tbl.gz"""
file """${context.sf1DataPath}/ssb/sf1/${tableName}.tbl.split00.gz"""
time 10000 // limit inflight 10s
// stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows
// if declared a check callback, the default check condition will ignore.
// So you must check all condition
check { result, exception, startTime, endTime ->
if (exception != null) {
throw exception
}
log.info("Stream load result: ${result}".toString())
def json = parseJson(result)
assertEquals("success", json.Status.toLowerCase())
assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
}
}
streamLoad {
// a default db 'regression_test' is specified in
// ${DORIS_HOME}/conf/regression-conf.groovy
table tableName
// default label is UUID:
// set 'label' UUID.randomUUID().toString()
// default column_separator is specify in doris fe config, usually is '\t'.
// this line change to ','
set 'column_separator', '|'
set 'compress_type', 'GZ'
set 'columns', columns[i]
// relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
// also, you can stream load a http stream, e.g. http://xxx/some.csv
file """${context.sf1DataPath}/ssb/sf1/${tableName}.tbl.split01.gz"""
time 10000 // limit inflight 10s

36
regression-test/suites/tpch_sf1_p1/load.groovy
View File

@ -44,7 +44,41 @@ suite("load") {
// relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
// also, you can stream load a http stream, e.g. http://xxx/some.csv
file """${context.sf1DataPath}/tpch/sf1/${tableName}.csv.gz"""
file """${context.sf1DataPath}/tpch/sf1/${tableName}.csv.split00.gz"""
time 10000 // limit inflight 10s
// stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows
// if declared a check callback, the default check condition will ignore.
// So you must check all condition
check { result, exception, startTime, endTime ->
if (exception != null) {
throw exception
}
log.info("Stream load result: ${result}".toString())
def json = parseJson(result)
assertEquals("success", json.Status.toLowerCase())
assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
}
}
streamLoad {
// a default db 'regression_test' is specified in
// ${DORIS_HOME}/conf/regression-conf.groovy
table tableName
// default label is UUID:
// set 'label' UUID.randomUUID().toString()
// default column_separator is specify in doris fe config, usually is '\t'.
// this line change to ','
set 'column_separator', '|'
set 'compress_type', 'GZ'
// relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
// also, you can stream load a http stream, e.g. http://xxx/some.csv
file """${context.sf1DataPath}/tpch/sf1/${tableName}.csv.split01.gz"""
time 10000 // limit inflight 10s

36
regression-test/suites/tpch_sf1_p1/tpch_sf1/load.groovy
View File

@ -53,7 +53,41 @@ suite("load") {
// relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
// also, you can stream load a http stream, e.g. http://xxx/some.csv
file """${context.sf1DataPath}/tpch/sf1/${tableName}.csv.gz"""
file """${context.sf1DataPath}/tpch/sf1/${tableName}.csv.split00.gz"""
time 10000 // limit inflight 10s
// stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows
// if declared a check callback, the default check condition will ignore.
// So you must check all condition
check { result, exception, startTime, endTime ->
if (exception != null) {
throw exception
}
log.info("Stream load result: ${result}".toString())
def json = parseJson(result)
assertEquals("success", json.Status.toLowerCase())
assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
}
}
streamLoad {
// a default db 'regression_test' is specified in
// ${DORIS_HOME}/conf/regression-conf.groovy
table tableName
// default label is UUID:
// set 'label' UUID.randomUUID().toString()
// default column_separator is specify in doris fe config, usually is '\t'.
// this line change to ','
set 'column_separator', '|'
set 'compress_type', 'GZ'
// relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
// also, you can stream load a http stream, e.g. http://xxx/some.csv
file """${context.sf1DataPath}/tpch/sf1/${tableName}.csv.split01.gz"""
time 10000 // limit inflight 10s