database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](Table-Valued Function) fix be core when user sepcified empty column_separator using hdfs tvf (#24369)

Browse Source
This commit is contained in:
Tiewei Fang
2023-09-14 23:19:48 +08:00
committed by GitHub
parent 5ba1f62da8
commit c5ef6cfea2
3 changed files with 33 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
View File

@ -18,6 +18,7 @@
package org.apache.doris.tablefunction;
import org.apache.doris.analysis.BrokerDesc;
import org.apache.doris.analysis.Separator;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Column;
@ -239,7 +240,17 @@ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctio
}
columnSeparator = validParams.getOrDefault(COLUMN_SEPARATOR, DEFAULT_COLUMN_SEPARATOR);
if (Strings.isNullOrEmpty(columnSeparator)) {
throw new AnalysisException("column_separator can not be empty.");
}
columnSeparator = Separator.convertSeparator(columnSeparator);
lineDelimiter = validParams.getOrDefault(LINE_DELIMITER, DEFAULT_LINE_DELIMITER);
if (Strings.isNullOrEmpty(lineDelimiter)) {
throw new AnalysisException("line_delimiter can not be empty.");
}
lineDelimiter = Separator.convertSeparator(lineDelimiter);
jsonRoot = validParams.getOrDefault(JSON_ROOT, "");
jsonPaths = validParams.getOrDefault(JSON_PATHS, "");
readJsonByLine = Boolean.valueOf(validParams.get(READ_JSON_BY_LINE)).booleanValue();
@ -248,6 +259,7 @@ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctio
fuzzyParse = Boolean.valueOf(validParams.get(FUZZY_PARSE)).booleanValue();
trimDoubleQuotes = Boolean.valueOf(validParams.get(TRIM_DOUBLE_QUOTES)).booleanValue();
skipLines = Integer.valueOf(validParams.getOrDefault(SKIP_LINES, "0")).intValue();
try {
compressionType = Util.getFileCompressType(validParams.getOrDefault(COMPRESS_TYPE, "UNKNOWN"));
} catch (IllegalArgumentException e) {

5
fe/fe-core/src/test/java/org/apache/doris/analysis/S3TvfLoadStmtTest.java
View File

@ -129,15 +129,12 @@ public class S3TvfLoadStmtTest {
@Injectable
Table targetTable;
@Injectable
DataDescription dataDescription;
@Test
public void testColumnMappings() throws Exception {
// c1/c2/c3 in both file and table, and c5 is only in table
final List<ImportColumnDesc> columnsDescList = getColumnsDescList(
"c1,c2,c3,c1=upper(c1), tmp_c4=c1 + 1, c5 = tmp_c4+1");
// DataDescription dataDescription = buildDataDesc(colNames, null, null, null);
DataDescription dataDescription = buildDataDesc(colNames, null, null, null);
new Expectations() {
{
dataDescription.getParsedColumnExprList();