database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Alter Job] Cancel the alter job after a task failed for 3 times (#2447)

Browse Source 
To avoid waiting timeout when it is a invalid alter job.
This commit is contained in:
WingC
2019-12-18 19:17:34 +08:00
committed by Mingyu Chen
parent 63ea05f9c7
commit e1ff744a99
4 changed files with 24 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fe/src/main/java/org/apache/doris/alter/RollupJobV2.java
View File

@ -373,6 +373,12 @@ public class RollupJobV2 extends AlterJobV2 {
if (!rollupBatchTask.isFinished()) {
LOG.info("rollup tasks not finished. job: {}", jobId);
List<AgentTask> tasks = rollupBatchTask.getUnfinishedTasks(2000);
for (AgentTask task : tasks) {
if (task.getFailedTimes() >= 3) {
throw new AlterCancelException("rollup task failed after try three times: " + task.getErrorMsg());
}
}
return;
}

6
fe/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
View File

@ -406,6 +406,12 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
if (!schemaChangeBatchTask.isFinished()) {
LOG.info("schema change tasks not finished. job: {}", jobId);
List<AgentTask> tasks = schemaChangeBatchTask.getUnfinishedTasks(2000);
for (AgentTask task : tasks) {
if (task.getFailedTimes() >= 3) {
throw new AlterCancelException("schema change task failed after try three times: " + task.getErrorMsg());
}
}
return;
}

3
fe/src/main/java/org/apache/doris/master/MasterImpl.java
View File

@ -130,6 +130,9 @@ public class MasterImpl {
} else {
if (taskStatus.getStatus_code() != TStatusCode.OK) {
task.failed();
String errMsg = "task type: " + taskType + ", status_code: " + taskStatus.getStatus_code().toString() +
", backendId: " + backend + ", signature: " + signature;
task.setErrorMsg(errMsg);
// We start to let FE perceive the task's error msg
if (taskType != TTaskType.MAKE_SNAPSHOT && taskType != TTaskType.UPLOAD
&& taskType != TTaskType.DOWNLOAD && taskType != TTaskType.MOVE

9
fe/src/main/java/org/apache/doris/task/AgentTask.java
View File

@ -34,6 +34,7 @@ public abstract class AgentTask {
protected TResourceInfo resourceInfo;
protected int failedTimes;
protected String errorMsg;
// some of process may use this member to check if the task is finished.
// some of are not.
// so whether the task is finished depends on caller's logic, not the value of this member.
@ -105,6 +106,14 @@ public abstract class AgentTask {
return this.failedTimes;
}
public void setErrorMsg(String errorMsg) {
this.errorMsg = errorMsg;
}
public String getErrorMsg() {
return errorMsg;
}
public void setFinished(boolean isFinished) {
this.isFinished = isFinished;
}