database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](routine load) fix enclose and escape can not set in routine load job (#38402) (#38825)

Browse Source 
pick (#38402)
This commit is contained in:
hui lai
2024-08-04 22:17:12 +08:00
committed by GitHub
parent 6035edad0b
commit 79b07d0b8a
7 changed files with 235 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRoutineLoadStmt.java
View File

@ -68,6 +68,8 @@ public class AlterRoutineLoadStmt extends DdlStmt {
.add(LoadStmt.STRICT_MODE)
.add(LoadStmt.TIMEZONE)
.add(CreateRoutineLoadStmt.WORKLOAD_GROUP)
.add(LoadStmt.KEY_ENCLOSE)
.add(LoadStmt.KEY_ESCAPE)
.build();
private final LabelName labelName;
@ -250,6 +252,12 @@ public class AlterRoutineLoadStmt extends DdlStmt {
.getWorkloadGroup(ConnectContext.get().getCurrentUserIdentity(), workloadGroup);
analyzedJobProperties.put(CreateRoutineLoadStmt.WORKLOAD_GROUP, String.valueOf(wgId));
}
if (jobProperties.containsKey(LoadStmt.KEY_ENCLOSE)) {
analyzedJobProperties.put(LoadStmt.KEY_ENCLOSE, jobProperties.get(LoadStmt.KEY_ENCLOSE));
}
if (jobProperties.containsKey(LoadStmt.KEY_ESCAPE)) {
analyzedJobProperties.put(LoadStmt.KEY_ESCAPE, jobProperties.get(LoadStmt.KEY_ESCAPE));
}
}
private void checkDataSourceProperties() throws UserException {

32
fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java
View File

@ -141,6 +141,8 @@ public class CreateRoutineLoadStmt extends DdlStmt {
.add(LOAD_TO_SINGLE_TABLET)
.add(PARTIAL_COLUMNS)
.add(WORKLOAD_GROUP)
.add(LoadStmt.KEY_ENCLOSE)
.add(LoadStmt.KEY_ESCAPE)
.build();
private final LabelName labelName;
@ -178,9 +180,9 @@ public class CreateRoutineLoadStmt extends DdlStmt {
private boolean numAsString = false;
private boolean fuzzyParse = false;
private String enclose;
private byte enclose;
private String escape;
private byte escape;
private long workloadGroupId = -1;
@ -311,11 +313,11 @@ public class CreateRoutineLoadStmt extends DdlStmt {
return jsonPaths;
}
public String getEnclose() {
public byte getEnclose() {
return enclose;
}
public String getEscape() {
public byte getEscape() {
return escape;
}
@ -507,14 +509,24 @@ public class CreateRoutineLoadStmt extends DdlStmt {
loadToSingleTablet = Util.getBooleanPropertyOrDefault(jobProperties.get(LoadStmt.LOAD_TO_SINGLE_TABLET),
RoutineLoadJob.DEFAULT_LOAD_TO_SINGLE_TABLET,
LoadStmt.LOAD_TO_SINGLE_TABLET + " should be a boolean");
enclose = jobProperties.get(LoadStmt.KEY_ENCLOSE);
if (enclose != null && enclose.length() != 1) {
throw new AnalysisException("enclose must be single-char");
String encloseStr = jobProperties.get(LoadStmt.KEY_ENCLOSE);
if (encloseStr != null) {
if (encloseStr.length() != 1) {
throw new AnalysisException("enclose must be single-char");
} else {
enclose = encloseStr.getBytes()[0];
}
}
escape = jobProperties.get(LoadStmt.KEY_ESCAPE);
if (escape != null && escape.length() != 1) {
throw new AnalysisException("escape must be single-char");
String escapeStr = jobProperties.get(LoadStmt.KEY_ESCAPE);
if (escapeStr != null) {
if (escapeStr.length() != 1) {
throw new AnalysisException("enclose must be single-char");
} else {
escape = escapeStr.getBytes()[0];
}
}
String inputWorkloadGroupStr = jobProperties.get(WORKLOAD_GROUP);
if (!StringUtils.isEmpty(inputWorkloadGroupStr)) {
this.workloadGroupId = Env.getCurrentEnv().getWorkloadGroupMgr()

10
fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
View File

@ -390,11 +390,13 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
} else {
jobProperties.put(PROPS_FUZZY_PARSE, "false");
}
if (stmt.getEnclose() != null) {
jobProperties.put(LoadStmt.KEY_ENCLOSE, stmt.getEnclose());
if (String.valueOf(stmt.getEnclose()) != null) {
this.enclose = stmt.getEnclose();
jobProperties.put(LoadStmt.KEY_ENCLOSE, String.valueOf(stmt.getEnclose()));
}
if (stmt.getEscape() != null) {
jobProperties.put(LoadStmt.KEY_ESCAPE, stmt.getEscape());
if (String.valueOf(stmt.getEscape()) != null) {
this.escape = stmt.getEscape();
jobProperties.put(LoadStmt.KEY_ESCAPE, String.valueOf(stmt.getEscape()));
}
if (stmt.getWorkloadGroupId() > 0) {
jobProperties.put(WORKLOAD_GROUP, String.valueOf(stmt.getWorkloadGroupId()));