database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](regression-test) Make test_decimal256_outfile_csv export the data to S3 rather than local file system (#36592)(#42211) (#42270)

Browse Source 
bp  (#36592)(#42211)

---------

Co-authored-by: TengJianPing <18241664+jacktengg@users.noreply.github.com>
Co-authored-by: Tiewei Fang <43782773+BePPPower@users.noreply.github.com>
This commit is contained in:
Rayner Chen
2024-10-22 23:57:30 +08:00
committed by GitHub
parent 157d67e7ca
commit bf90d918ca
2 changed files with 55 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
regression-test/data/datatype_p0/decimalv3/test_decimal256_outfile_csv.out
View File

@ -20,3 +20,24 @@
4999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000
9999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000
-- !select_tvf1 --
\N \N \N
-9999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000
-4999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000
-99999999999999999999999999999.999999999 1.0000000000 1.00000000000
0.000000000 0.0000000000 0.00000000000
1.000000000 999999999999999999999999999999999999999999999999999999999999999999.9999999999 99999999999999999999999999999999999999999999999999999999999999999.99999999999
1.000000000 999999999999999999999999999999999999999999999999999999999999999999.9999999999 99999999999999999999999999999999999999999999999999999999999999999.99999999999
2.000000000 499999999999999999999999999999999999999999999999999999999999999999.9999999999 49999999999999999999999999999999999999999999999999999999999999999.99999999999
3.000000000 333333333333333333333333333333333333333333333333333333333333333333.3333333333 33333333333333333333333333333333333333333333333333333333333333333.33333333333
3.000000000 333333333333333333333333333333333333333333333333333333333333333333.3333333333 33333333333333333333333333333333333333333333333333333333333333333.33333333333
4.000000000 -999999999999999999999999999999999999999999999999999999999999999999.9999999999 99999999999999999999999999999999999999999999999999999999999999999.99999999999
4.000000000 -999999999999999999999999999999999999999999999999999999999999999999.9999999999 99999999999999999999999999999999999999999999999999999999999999999.99999999999
5.000000000 -333333333333333333333333333333333333333333333333333333333333333333.3333333333 33333333333333333333333333333333333333333333333333333333333333333.33333333333
5.000000000 -333333333333333333333333333333333333333333333333333333333333333333.3333333333 33333333333333333333333333333333333333333333333333333333333333333.33333333333
6.000000000 \N 99999999999999999999999999999999999999999999999999999999999999999.99999999999
7.000000000 \N 99999999999999999999999999999999999999999999999999999999999999999.99999999999
99999999999999999999999999999.999999999 1.0000000000 1.00000000000
4999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000
9999999999999999999999999999999999999999999999999999999999999999999.999999999 1.0000000000 1.00000000000

88
regression-test/suites/datatype_p0/decimalv3/test_decimal256_outfile_csv.groovy
View File

@ -22,36 +22,15 @@ import java.nio.file.Files
import java.nio.file.Paths
suite("test_decimal256_outfile_csv") {
StringBuilder strBuilder = new StringBuilder()
strBuilder.append("curl --location-trusted -u " + context.config.jdbcUser + ":" + context.config.jdbcPassword)
strBuilder.append(" http://" + context.config.feHttpAddress + "/rest/v1/config/fe")
String command = strBuilder.toString()
def process = command.toString().execute()
def code = process.waitFor()
def err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
def out = process.getText()
logger.info("Request FE Config: code=" + code + ", out=" + out + ", err=" + err)
assertEquals(code, 0)
def response = parseJson(out.trim())
assertEquals(response.code, 0)
assertEquals(response.msg, "success")
def configJson = response.data.rows
boolean enableOutfileToLocal = false
for (Object conf: configJson) {
assert conf instanceof Map
if (((Map<String, String>) conf).get("Name").toLowerCase() == "enable_outfile_to_local") {
enableOutfileToLocal = ((Map<String, String>) conf).get("Value").toLowerCase() == "true"
}
}
if (!enableOutfileToLocal) {
logger.warn("Please set enable_outfile_to_local to true to run test_outfile")
return
}
sql "set enable_nereids_planner = true;"
sql "set enable_decimal256 = true;"
String ak = getS3AK()
String sk = getS3SK()
String s3_endpoint = getS3Endpoint()
String region = getS3Region()
String bucket = context.config.otherConfigs.get("s3BucketName");
sql "DROP TABLE IF EXISTS `test_decimal256_outfile_csv`"
sql """
CREATE TABLE IF NOT EXISTS `test_decimal256_outfile_csv` (
@ -105,36 +84,37 @@ suite("test_decimal256_outfile_csv") {
SELECT * FROM test_decimal256_outfile_csv t order by 1,2,3;
"""
def uuid = UUID.randomUUID().toString()
def outFilePath = """/tmp/test_decimal256_outfile_csv_${uuid}"""
List<List<Object>> backends = sql """ show backends """
assertTrue(backends.size() > 0)
if (backends.size() > 1) {
outFilePath = "/tmp"
def outFilePath = "${bucket}/outfile/csv/test_decimal256_outfile_csv/exp_"
def outfile_to_S3 = { export_table_name, foramt ->
// select ... into outfile ...
def res = sql """
SELECT * FROM ${export_table_name} t ORDER BY k1
INTO OUTFILE "s3://${outFilePath}"
FORMAT AS ${foramt}
PROPERTIES (
"s3.endpoint" = "${s3_endpoint}",
"s3.region" = "${region}",
"s3.secret_key"="${sk}",
"s3.access_key" = "${ak}"
);
"""
return res[0][3]
}
try {
logger.info("outfile: " + outFilePath)
// check outfile
File path = new File(outFilePath)
if (!path.exists()) {
assert path.mkdirs()
} else {
throw new IllegalStateException("""${outFilePath} already exists! """)
}
sql """
SELECT * FROM test_decimal256_outfile_csv t order by 1,2,3 INTO OUTFILE "file://${outFilePath}/" properties("column_separator" = ",");
"""
File[] files = path.listFiles()
assert files.length == 1
List<String> outLines = Files.readAllLines(Paths.get(files[0].getAbsolutePath()), StandardCharsets.UTF_8);
assert outLines.size() == 19
def outfile_url = outfile_to_S3("test_decimal256_outfile_csv", "csv")
qt_select_tvf1 """ SELECT * FROM S3 (
"uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.csv",
"ACCESS_KEY"= "${ak}",
"SECRET_KEY" = "${sk}",
"format" = "csv",
"region" = "${region}"
);
"""
} finally {
File path = new File(outFilePath)
if (path.exists()) {
for (File f: path.listFiles()) {
f.delete();
}
path.delete();
}
}
}