database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Bug] Fix the bug of Largetint and Decimal json load failed. (#4983)

Browse Source 
Use param of json load "num_as_string" to use flag kParseNumbersAsStringsFlag to parse json data.
This commit is contained in:
HappenLee
2020-12-06 08:49:30 +08:00
committed by GitHub
parent b1b99ae884
commit b954dfd82d
18 changed files with 374 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRoutineLoadStmt.java
View File

@ -51,6 +51,7 @@ public class AlterRoutineLoadStmt extends DdlStmt {
.add(CreateRoutineLoadStmt.JSONPATHS)
.add(CreateRoutineLoadStmt.JSONROOT)
.add(CreateRoutineLoadStmt.STRIP_OUTER_ARRAY)
.add(CreateRoutineLoadStmt.NUM_AS_STRING)
.add(LoadStmt.STRICT_MODE)
.add(LoadStmt.TIMEZONE)
.build();
@ -181,6 +182,12 @@ public class AlterRoutineLoadStmt extends DdlStmt {
analyzedJobProperties.put(jobProperties.get(CreateRoutineLoadStmt.STRIP_OUTER_ARRAY),
String.valueOf(stripOuterArray));
}
if (jobProperties.containsKey(CreateRoutineLoadStmt.NUM_AS_STRING)) {
boolean numAsString = Boolean.valueOf(jobProperties.get(CreateRoutineLoadStmt.NUM_AS_STRING));
analyzedJobProperties.put(jobProperties.get(CreateRoutineLoadStmt.NUM_AS_STRING),
String.valueOf(numAsString));
}
}
private void checkDataSourceProperties() throws AnalysisException {

8
fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java
View File

@ -101,6 +101,7 @@ public class CreateRoutineLoadStmt extends DdlStmt {
public static final String STRIP_OUTER_ARRAY = "strip_outer_array";
public static final String JSONPATHS = "jsonpaths";
public static final String JSONROOT = "json_root";
public static final String NUM_AS_STRING = "num_as_string";
// kafka type properties
public static final String KAFKA_BROKER_LIST_PROPERTY = "kafka_broker_list";
@ -122,6 +123,7 @@ public class CreateRoutineLoadStmt extends DdlStmt {
.add(FORMAT)
.add(JSONPATHS)
.add(STRIP_OUTER_ARRAY)
.add(NUM_AS_STRING)
.add(JSONROOT)
.add(LoadStmt.STRICT_MODE)
.add(LoadStmt.TIMEZONE)
@ -165,6 +167,7 @@ public class CreateRoutineLoadStmt extends DdlStmt {
private String jsonPaths = "";
private String jsonRoot = ""; // MUST be a jsonpath string
private boolean stripOuterArray = false;
private boolean numAsString = false;
// kafka related properties
private String kafkaBrokerList;
@ -255,6 +258,10 @@ public class CreateRoutineLoadStmt extends DdlStmt {
return stripOuterArray;
}
public boolean isNumAsString() {
return numAsString;
}
public String getJsonPaths() {
return jsonPaths;
}
@ -431,6 +438,7 @@ public class CreateRoutineLoadStmt extends DdlStmt {
jsonPaths = jobProperties.get(JSONPATHS);
jsonRoot = jobProperties.get(JSONROOT);
stripOuterArray = Boolean.valueOf(jobProperties.getOrDefault(STRIP_OUTER_ARRAY, "false"));
numAsString = Boolean.valueOf(jobProperties.getOrDefault(NUM_AS_STRING, "false"));
} else {
throw new UserException("Format type is invalid. format=`" + format + "`");
}

2
fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/UploadAction.java
View File

@ -270,6 +270,7 @@ public class UploadAction extends RestBaseController {
public String jsonPaths;
public String stripOuterArray;
public String jsonRoot;
public String numAsString;
public LoadContext(HttpServletRequest request, String db, String tbl, String user, String passwd, TmpFileMgr.TmpFile file) {
@ -299,6 +300,7 @@ public class UploadAction extends RestBaseController {
this.format = request.getHeader("format");
this.jsonPaths = request.getHeader("jsonpaths");
this.stripOuterArray = request.getHeader("strip_outer_array");
this.numAsString = request.getHeader("num_as_string");
this.jsonRoot = request.getHeader("json_root");
}
}

12
fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
View File

@ -193,6 +193,7 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
*/
private static final String PROPS_FORMAT = "format";
private static final String PROPS_STRIP_OUTER_ARRAY = "strip_outer_array";
private static final String PROPS_NUM_AS_STRING = "num_as_string";
private static final String PROPS_JSONPATHS = "jsonpaths";
private static final String PROPS_JSONROOT = "json_root";
@ -302,6 +303,7 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
if (Strings.isNullOrEmpty(stmt.getFormat()) || stmt.getFormat().equals("csv")) {
jobProperties.put(PROPS_FORMAT, "csv");
jobProperties.put(PROPS_STRIP_OUTER_ARRAY, "false");
jobProperties.put(PROPS_NUM_AS_STRING, "false");
jobProperties.put(PROPS_JSONPATHS, "");
jobProperties.put(PROPS_JSONROOT, "");
} else if (stmt.getFormat().equals("json")) {
@ -321,6 +323,12 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
} else {
jobProperties.put(PROPS_STRIP_OUTER_ARRAY, "false");
}
if (stmt.isNumAsString()) {
jobProperties.put(PROPS_NUM_AS_STRING, "true");
} else {
jobProperties.put(PROPS_NUM_AS_STRING, "false");
}
} else {
throw new UserException("Invalid format type.");
}
@ -546,6 +554,10 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
return Boolean.valueOf(jobProperties.get(PROPS_STRIP_OUTER_ARRAY));
}
public boolean isNumAsString() {
return Boolean.valueOf(jobProperties.get(PROPS_NUM_AS_STRING));
}
@Override
public String getPath() {
return null;

1
fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java
View File

@ -93,6 +93,7 @@ public class StreamLoadScanNode extends LoadScanNode {
rangeDesc.setJsonRoot(taskInfo.getJsonRoot());
}
rangeDesc.setStripOuterArray(taskInfo.isStripOuterArray());
rangeDesc.setNumAsString(taskInfo.isNumAsString());
}
rangeDesc.splittable = false;
switch (taskInfo.getFileType()) {

1
fe/fe-core/src/main/java/org/apache/doris/task/LoadTaskInfo.java
View File

@ -42,6 +42,7 @@ public interface LoadTaskInfo {
public String getJsonPaths();
public String getJsonRoot();
public boolean isStripOuterArray();
public boolean isNumAsString();
public String getPath();
public List<ImportColumnDesc> getColumnExprDescs();
public boolean isStrictMode();

12
fe/fe-core/src/main/java/org/apache/doris/task/StreamLoadTask.java
View File

@ -57,6 +57,7 @@ public class StreamLoadTask implements LoadTaskInfo {
private TFileType fileType;
private TFileFormatType formatType;
private boolean stripOuterArray;
private boolean numAsString;
private String jsonPaths;
private String jsonRoot;
@ -83,6 +84,7 @@ public class StreamLoadTask implements LoadTaskInfo {
this.jsonPaths = "";
this.jsonRoot = "";
this.stripOuterArray = false;
this.numAsString = false;
}
public TUniqueId getId() {
@ -141,10 +143,19 @@ public class StreamLoadTask implements LoadTaskInfo {
return stripOuterArray;
}
@Override
public boolean isNumAsString() {
return numAsString;
}
public void setStripOuterArray(boolean stripOuterArray) {
this.stripOuterArray = stripOuterArray;
}
public void setNumAsString(boolean numAsString) {
this.numAsString = numAsString;
}
public String getJsonPaths() {
return jsonPaths;
}
@ -227,6 +238,7 @@ public class StreamLoadTask implements LoadTaskInfo {
jsonRoot = request.getJsonRoot();
}
stripOuterArray = request.isStripOuterArray();
numAsString = request.isNumAsString();
}
if (request.isSetMergeType()) {
try {