[test](load) add more s3 load regression test cases (#24906)
This commit is contained in:
Siyang Tang
@ -1,4 +1,286 @@
|
||||
-- This file is automatically generated. You should know what you did if you want to edit this
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
40
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
36
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
40
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
36
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
40
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
36
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
40
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
36
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
40
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
36
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
60
|
||||
|
||||
-- !select --
|
||||
34
|
||||
|
||||
-- !select --
|
||||
34
|
||||
|
||||
-- !select --
|
||||
34
|
||||
|
||||
-- !select --
|
||||
54
|
||||
|
||||
-- !select --
|
||||
17
|
||||
|
||||
-- !select --
|
||||
17
|
||||
|
||||
-- !select --
|
||||
80
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
72
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
100
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
90
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
120
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
108
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
140
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
126
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
160
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
144
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
180
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
162
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
200
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
@ -6,7 +288,7 @@
|
||||
180
|
||||
|
||||
-- !select --
|
||||
200
|
||||
18
|
||||
|
||||
-- !select --
|
||||
18
|
||||
@ -15,8 +297,98 @@
|
||||
48
|
||||
|
||||
-- !select --
|
||||
18
|
||||
48
|
||||
|
||||
-- !select --
|
||||
220
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
198
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
240
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
216
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
260
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
280
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
47
|
||||
|
||||
-- !select --
|
||||
47
|
||||
|
||||
-- !select --
|
||||
48
|
||||
|
||||
-- !select --
|
||||
216
|
||||
|
||||
-- !select --
|
||||
280
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
47
|
||||
|
||||
-- !select --
|
||||
18
|
||||
|
||||
-- !select --
|
||||
47
|
||||
|
||||
|
||||
@ -191,3 +191,90 @@
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
162
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
141
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
182
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
159
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
202
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
177
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
222
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
20
|
||||
|
||||
-- !select --
|
||||
195
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
-- !select --
|
||||
16
|
||||
|
||||
|
||||
@ -61,6 +61,7 @@ PARTITION BY RANGE(k01)
|
||||
)
|
||||
DISTRIBUTED BY HASH(k00) BUCKETS 32
|
||||
PROPERTIES (
|
||||
"function_column.sequence_col" = "k12",
|
||||
"bloom_filter_columns"="k05",
|
||||
"replication_num" = "1",
|
||||
"enable_unique_key_merge_on_write" = "true"
|
||||
|
||||
@ -41,6 +41,7 @@ suite("test_s3_load", "load_p0") {
|
||||
]
|
||||
|
||||
def uniqTables = [
|
||||
"mow_tbl_basic",
|
||||
"uniq_tbl_basic"
|
||||
]
|
||||
|
||||
@ -371,6 +372,80 @@ suite("test_s3_load", "load_p0") {
|
||||
"", "", "", "", "ORDER BY k01"))
|
||||
}
|
||||
|
||||
/*========================================================== json ==========================================================*/
|
||||
|
||||
for (String table : basicTables) {
|
||||
attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.json",
|
||||
"${table}", "", "", "FORMAT AS \"json\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
|
||||
"", "", "", "", "PROPERTIES(\"strip_outer_array\" = \"true\", \"fuzzy_parse\" = \"true\")"))
|
||||
}
|
||||
|
||||
attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.json",
|
||||
"agg_tbl_basic", "", "", "FORMAT AS \"json\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
|
||||
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "PROPERTIES(\"strip_outer_array\" = \"true\", \"fuzzy_parse\" = \"true\")"))
|
||||
|
||||
for (String table : arrayTables) {
|
||||
attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.json",
|
||||
"${table}", "", "", "FORMAT AS \"json\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
|
||||
"", "", "", "", "PROPERTIES(\"strip_outer_array\" = \"true\", \"fuzzy_parse\" = \"true\")"))
|
||||
}
|
||||
|
||||
for (String table : basicTables) {
|
||||
attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json",
|
||||
"${table}", "", "", "FORMAT AS \"JSON\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
|
||||
"", "", "", "", "PROPERTIES(\"read_json_by_line\" = \"true\")"))
|
||||
}
|
||||
|
||||
attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json",
|
||||
"agg_tbl_basic", "", "", "FORMAT AS \"JSON\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
|
||||
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "PROPERTIES(\"read_json_by_line\" = \"true\")"))
|
||||
|
||||
for (String table : arrayTables) {
|
||||
attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_by_line.json",
|
||||
"${table}", "", "", "FORMAT AS \"JSON\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
|
||||
"", "", "", "", "PROPERTIES(\"read_json_by_line\" = \"true\")"))
|
||||
}
|
||||
|
||||
for (String table : basicTables) {
|
||||
attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.parq",
|
||||
"${table}", "", "", "FORMAT AS \"parquet\"", "(K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18)",
|
||||
"", "", "", "", ""))
|
||||
}
|
||||
|
||||
attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.parq",
|
||||
"agg_tbl_basic", "", "", "FORMAT AS \"PARQUET\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
|
||||
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", ""))
|
||||
|
||||
// for (String table : arrayTables) {
|
||||
// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.parq",
|
||||
// "${table}", "", "", "FORMAT AS \"parquet\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
|
||||
// "", "", "", "", ""))
|
||||
// }
|
||||
|
||||
for (String table : basicTables) {
|
||||
attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.orc",
|
||||
"${table}", "", "", "FORMAT AS \"orc\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
|
||||
"", "", "", "", ""))
|
||||
}
|
||||
|
||||
attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.orc",
|
||||
"agg_tbl_basic", "", "", "FORMAT AS \"ORC\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
|
||||
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", ""))
|
||||
|
||||
// for (String table : arrayTables) {
|
||||
// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.parq",
|
||||
// "${table}", "", "", "FORMAT AS \"parquet\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
|
||||
// "", "", "", "", ""))
|
||||
// }
|
||||
|
||||
for(String table : uniqTables) {
|
||||
def attributes = new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_delete.csv",
|
||||
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,__DEL__)",
|
||||
"", "", "", "", "DELETE ON __DEL__=true")
|
||||
attributes.dataDesc.mergeType = "MERGE"
|
||||
attributesList.add(attributes)
|
||||
}
|
||||
|
||||
def ak = getS3AK()
|
||||
def sk = getS3SK()
|
||||
|
||||
@ -387,6 +462,7 @@ suite("test_s3_load", "load_p0") {
|
||||
|
||||
def sql_str = """
|
||||
LOAD LABEL $label (
|
||||
$attributes.dataDesc.mergeType
|
||||
DATA INFILE("$attributes.dataDesc.path")
|
||||
INTO TABLE $attributes.dataDesc.tableName
|
||||
$attributes.dataDesc.columnTermClause
|
||||
@ -411,10 +487,7 @@ suite("test_s3_load", "load_p0") {
|
||||
logger.info("submit sql: ${sql_str}");
|
||||
sql """${sql_str}"""
|
||||
logger.info("Submit load with lable: $label, table: $attributes.dataDesc.tableName, path: $attributes.dataDesc.path")
|
||||
++i
|
||||
}
|
||||
|
||||
for (LoadAttributes attributes : attributesList) {
|
||||
def max_try_milli_secs = 600000
|
||||
while (max_try_milli_secs > 0) {
|
||||
String[][] result = sql """ show load where label="$attributes.label" order by createtime desc limit 1; """
|
||||
@ -439,14 +512,44 @@ suite("test_s3_load", "load_p0") {
|
||||
assertTrue(false, "load Timeout: $attributes.label")
|
||||
}
|
||||
}
|
||||
qt_select """ select count(*) from $attributes.dataDesc.tableName """
|
||||
++i
|
||||
}
|
||||
|
||||
// for (LoadAttributes attributes : attributesList) {
|
||||
// def max_try_milli_secs = 600000
|
||||
// while (max_try_milli_secs > 0) {
|
||||
// String[][] result = sql """ show load where label="$attributes.label" order by createtime desc limit 1; """
|
||||
// if (result[0][2].equals("FINISHED")) {
|
||||
// if (attributes.isExceptFailed) {
|
||||
// assertTrue(false, "load should be failed but was success: $result")
|
||||
// }
|
||||
// logger.info("Load FINISHED " + attributes.label + ": $result")
|
||||
// break
|
||||
// }
|
||||
// if (result[0][2].equals("CANCELLED")) {
|
||||
// if (attributes.isExceptFailed) {
|
||||
// logger.info("Load FINISHED " + attributes.label)
|
||||
// break
|
||||
// }
|
||||
// assertTrue(false, "load failed: $result")
|
||||
// break
|
||||
// }
|
||||
// Thread.sleep(1000)
|
||||
// max_try_milli_secs -= 1000
|
||||
// if (max_try_milli_secs <= 0) {
|
||||
// assertTrue(false, "load Timeout: $attributes.label")
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
for(String tbl : tables) {
|
||||
qt_select """ select count(*) from ${tbl} """
|
||||
}
|
||||
}
|
||||
|
||||
class DataDesc {
|
||||
public String mergeType = ""
|
||||
public String path
|
||||
public String tableName
|
||||
public String lineTermClause
|
||||
|
||||
@ -41,7 +41,8 @@ suite("test_s3_tvf", "load_p0") {
|
||||
]
|
||||
|
||||
def uniqTable = [
|
||||
"uniq_tbl_basic_tvf"
|
||||
"uniq_tbl_basic_tvf",
|
||||
"mow_tbl_basic_tvf"
|
||||
]
|
||||
|
||||
def attributeList = [
|
||||
@ -273,6 +274,98 @@ suite("test_s3_tvf", "load_p0") {
|
||||
.addProperty("column_separator", "|"))
|
||||
}
|
||||
|
||||
for(String table : basicTables) {
|
||||
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18", "", "")
|
||||
.addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.parq")
|
||||
.addProperty("format", "parquet")
|
||||
.addProperty("column_separator", "|"))
|
||||
}
|
||||
|
||||
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18, to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21",
|
||||
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
|
||||
.addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.parq")
|
||||
.addProperty("format", "parquet")
|
||||
.addProperty("column_separator", "|"))
|
||||
|
||||
for(String table : arrayTables) {
|
||||
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "", "")
|
||||
.addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.parq")
|
||||
.addProperty("format", "parquet")
|
||||
.addProperty("column_separator", "|"))
|
||||
}
|
||||
|
||||
for(String table : basicTables) {
|
||||
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18", "", "")
|
||||
.addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.orc")
|
||||
.addProperty("format", "orc")
|
||||
.addProperty("column_separator", "|"))
|
||||
}
|
||||
|
||||
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18, to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21",
|
||||
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
|
||||
.addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.orc")
|
||||
.addProperty("format", "orc")
|
||||
.addProperty("column_separator", "|"))
|
||||
|
||||
for(String table : arrayTables) {
|
||||
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "", "")
|
||||
.addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.orc")
|
||||
.addProperty("format", "orc")
|
||||
.addProperty("column_separator", "|"))
|
||||
}
|
||||
|
||||
for(String table : basicTables) {
|
||||
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18", "", "")
|
||||
.addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
|
||||
.addProperty("format", "json")
|
||||
.addProperty("read_json_by_line", "false")
|
||||
.addProperty("strip_outer_array", "true")
|
||||
.addProperty("column_separator", "|"))
|
||||
}
|
||||
|
||||
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18, to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21",
|
||||
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
|
||||
.addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
|
||||
.addProperty("format", "json")
|
||||
.addProperty("read_json_by_line", "false")
|
||||
.addProperty("strip_outer_array", "true")
|
||||
.addProperty("column_separator", "|"))
|
||||
|
||||
for(String table : arrayTables) {
|
||||
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "", "")
|
||||
.addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.json")
|
||||
.addProperty("format", "json")
|
||||
.addProperty("read_json_by_line", "false")
|
||||
.addProperty("strip_outer_array", "true")
|
||||
.addProperty("column_separator", "|"))
|
||||
}
|
||||
|
||||
for(String table : basicTables) {
|
||||
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18", "", "")
|
||||
.addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
|
||||
.addProperty("format", "json")
|
||||
.addProperty("read_json_by_line", "true")
|
||||
.addProperty("strip_outer_array", "false")
|
||||
.addProperty("column_separator", "|"))
|
||||
}
|
||||
|
||||
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18, to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21",
|
||||
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
|
||||
.addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
|
||||
.addProperty("format", "json")
|
||||
.addProperty("read_json_by_line", "true")
|
||||
.addProperty("strip_outer_array", "false")
|
||||
.addProperty("column_separator", "|"))
|
||||
|
||||
for(String table : arrayTables) {
|
||||
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "", "")
|
||||
.addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_by_line.json")
|
||||
.addProperty("format", "json")
|
||||
.addProperty("read_json_by_line", "true")
|
||||
.addProperty("strip_outer_array", "false")
|
||||
.addProperty("column_separator", "|"))
|
||||
}
|
||||
|
||||
def ak = getS3AK()
|
||||
def sk = getS3SK()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user