database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature-wip](new-scan) support Json reader (#13546)

Browse Source 
Issue Number: close #12574
This pr adds `NewJsonReader` which implements GenericReader interface to support read json format file.

TODO:
1. modify `_scann_eof` later.
2. Rename `NewJsonReader` to `JsonReader` when `JsonReader` is deleted.
This commit is contained in:
Tiewei Fang
2022-10-26 12:52:21 +08:00
committed by GitHub
parent 44c9163b3c
commit c418bbd2d1
31 changed files with 2820 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java
View File

@ -121,7 +121,10 @@ public class DataDescription {
private String jsonPaths = "";
private String jsonRoot = "";
private boolean fuzzyParse = false;
private boolean readJsonByLine = false;
// the default must be true.
// So that for broker load, this is always true,
// and for stream load, it will set on demand.
private boolean readJsonByLine = true;
private boolean numAsString = false;
private String sequenceCol;
@ -616,6 +619,10 @@ public class DataDescription {
return !Strings.isNullOrEmpty(srcTableName);
}
public boolean isReadJsonByLine() {
return readJsonByLine;
}
/*
* Analyze parsedExprMap and columnToHadoopFunction from columns, columns from path and columnMappingList
* Example:

2
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View File

@ -4031,7 +4031,7 @@ public class Env {
}
}
public void renameColumn(Database db, OlapTable table, String colName,
private void renameColumn(Database db, OlapTable table, String colName,
String newColName, boolean isReplay) throws DdlException {
if (table.getState() != OlapTableState.NORMAL) {
throw new DdlException("Table[" + table.getName() + "] is under " + table.getState());

6
fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
View File

@ -256,9 +256,9 @@ public class BrokerFileGroup implements Writable {
jsonPaths = dataDescription.getJsonPaths();
jsonRoot = dataDescription.getJsonRoot();
fuzzyParse = dataDescription.isFuzzyParse();
// For broker load, we only support reading json format data line by line,
// so we set readJsonByLine to true here.
readJsonByLine = true;
// ATTN: for broker load, we only support reading json format data line by line,
// so if this is set to false, it must be stream load.
readJsonByLine = dataDescription.isReadJsonByLine();
numAsString = dataDescription.isNumAsString();
}
}

5
fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
View File

@ -42,7 +42,6 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.Util;
import org.apache.doris.load.BrokerFileGroup;
import org.apache.doris.load.LoadErrorHub;
import org.apache.doris.load.loadv2.LoadTask;
@ -174,10 +173,6 @@ public class StreamLoadPlanner {
// create scan node
if (Config.enable_new_load_scan_node && Config.enable_vectorized_load) {
ExternalFileScanNode fileScanNode = new ExternalFileScanNode(new PlanNodeId(0), scanTupleDesc);
if (!Util.isCsvFormat(taskInfo.getFormatType())) {
throw new AnalysisException(
"New stream load scan load not support non-csv type now: " + taskInfo.getFormatType());
}
// 1. create file group
DataDescription dataDescription = new DataDescription(destTable.getName(), taskInfo);
dataDescription.analyzeWithoutCheckPriv(db.getFullName());