database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Enhancement](Load) stream tvf support json (#23752)

Browse Source 
stream tvf support json

[{"id":1, "name":"ftw", "age":18}]
[{"id":2, "name":"xxx", "age":17}]
[{"id":3, "name":"yyy", "age":19}]
example:

curl -v --location-trusted -u root: -H "sql: insert into test.t1(c1, c2) select id, name from http_stream(\"format\" = \"json\", \"strip_outer_array\" = \"true\", \"read_json_by_line\" = \"true\")" -T /root/json_file.json http://127.0.0.1:8030/api/_http_stream
This commit is contained in:
zzzzzzzs
2023-09-02 01:09:06 +08:00
committed by GitHub
parent 75e2bc8a25
commit 6630f92878
4 changed files with 301 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
View File

@ -227,13 +227,13 @@ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctio
}
// TODO Support is needed in the future
if (getTFileType() == TFileType.FILE_STREAM && (formatString.equals("json")
|| formatString.equals("csv_with_names")
if (getTFileType() == TFileType.FILE_STREAM && (formatString.equals("csv_with_names")
|| formatString.equals("csv_with_names_and_types")
|| formatString.equals("parquet")
|| formatString.equals("avro")
|| formatString.equals("orc"))) {
throw new AnalysisException("current http_stream does not yet support json, parquet and orc");
throw new AnalysisException("current http_stream does not yet support csv_with_names, "
+ "csv_with_names_and_types, parquet, avro and orc");
}
columnSeparator = validParams.getOrDefault(COLUMN_SEPARATOR, DEFAULT_COLUMN_SEPARATOR);
lineDelimiter = validParams.getOrDefault(LINE_DELIMITER, DEFAULT_LINE_DELIMITER);
@ -524,6 +524,7 @@ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctio
// set TFileRangeDesc
TFileRangeDesc fileRangeDesc = new TFileRangeDesc();
fileRangeDesc.setLoadId(ctx.queryId());
fileRangeDesc.setFileType(getTFileType());
fileRangeDesc.setCompressType(Util.getOrInferCompressType(compressionType, firstFile.getPath()));
fileRangeDesc.setPath(firstFile.getPath());