From 19c36dcc86615bb2b54d268054d10a85ba25ae9a Mon Sep 17 00:00:00 2001 From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com> Date: Wed, 22 Nov 2023 21:38:08 +0800 Subject: [PATCH] [Fix](statistics)Fix auto job start time incorrect bug (#27402) Before, the auto analyze job start time was the job creation time, not the start to execute time, which is inaccurate. This pr is to change the start time to the first task start to execute time. --- .../org/apache/doris/qe/ShowExecutor.java | 2 +- .../apache/doris/statistics/AnalysisInfo.java | 7 +++ .../doris/statistics/AnalysisManager.java | 9 +++- .../doris/statistics/AnalysisManagerTest.java | 51 +++++++++++++++++++ 4 files changed, 67 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 8f92115695..65440ac00d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -2662,7 +2662,7 @@ public class ShowExecutor { } row.add(analysisInfo.scheduleType.toString()); LocalDateTime startTime = - LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.createTime), + LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.startTime), java.time.ZoneId.systemDefault()); LocalDateTime endTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.endTime), diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java index ec0980d16b..0606350b8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java @@ -185,6 +185,9 @@ public class AnalysisInfo implements Writable { @SerializedName("createTime") public final long createTime = System.currentTimeMillis(); + @SerializedName("startTime") + public long startTime; + @SerializedName("endTime") public long endTime; /** @@ -330,6 +333,10 @@ public class AnalysisInfo implements Writable { return analysisInfo; } + public void markStartTime(long startTime) { + this.startTime = startTime; + } + public void markFinished() { state = AnalysisState.FINISHED; endTime = System.currentTimeMillis(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index dbb5dd668a..c6deaf0268 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -158,6 +158,7 @@ public class AnalysisManager implements Writable { // Set the job state to RUNNING when its first task becomes RUNNING. if (info.state.equals(AnalysisState.RUNNING) && job.state.equals(AnalysisState.PENDING)) { job.state = AnalysisState.RUNNING; + job.markStartTime(System.currentTimeMillis()); replayCreateAnalysisJob(job); } boolean allFinished = true; @@ -200,6 +201,13 @@ public class AnalysisManager implements Writable { if (job == null) { return null; } + synchronized (job) { + // Set the job state to RUNNING when its first task becomes RUNNING. + if (info.state.equals(AnalysisState.RUNNING) && job.state.equals(AnalysisState.PENDING)) { + job.state = AnalysisState.RUNNING; + job.markStartTime(System.currentTimeMillis()); + } + } int failedCount = 0; StringJoiner reason = new StringJoiner(", "); Map taskMap = analysisJobIdToTaskMap.get(info.jobId); @@ -1002,7 +1010,6 @@ public class AnalysisManager implements Writable { } public void registerSysJob(AnalysisInfo jobInfo, Map taskInfos) { - jobInfo.state = AnalysisState.RUNNING; systemJobInfoMap.put(jobInfo.jobId, jobInfo); analysisJobIdToTaskMap.put(jobInfo.jobId, taskInfos); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java index 345d8ee2cf..35f02b8811 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java @@ -322,6 +322,57 @@ public class AnalysisManagerTest { Assertions.assertTrue(analysisManager.systemJobInfoMap.isEmpty()); } + @Test + public void testSystemJobStartTime() { + new MockUp() { + + @Mock + protected void init(AnalysisInfo info) { + + } + }; + + new MockUp() { + @Mock + public void updateTableStats(AnalysisInfo jobInfo) { + } + + @Mock + protected void logAutoJob(AnalysisInfo autoJob) { + + } + }; + + AnalysisManager analysisManager = new AnalysisManager(); + AnalysisInfo job = new AnalysisInfoBuilder() + .setJobId(0) + .setColName("col1, col2").build(); + analysisManager.systemJobInfoMap.put(job.jobId, job); + AnalysisInfo task1 = new AnalysisInfoBuilder() + .setJobId(0) + .setTaskId(1) + .setState(AnalysisState.PENDING) + .setColName("col1").build(); + AnalysisInfo task2 = new AnalysisInfoBuilder() + .setJobId(0) + .setTaskId(1) + .setState(AnalysisState.PENDING) + .setColName("col2").build(); + OlapAnalysisTask ot1 = new OlapAnalysisTask(task1); + OlapAnalysisTask ot2 = new OlapAnalysisTask(task2); + Map taskMap = new HashMap<>(); + taskMap.put(ot1.info.taskId, ot1); + taskMap.put(ot2.info.taskId, ot2); + analysisManager.analysisJobIdToTaskMap.put(job.jobId, taskMap); + + job.state = AnalysisState.PENDING; + long l = System.currentTimeMillis(); + analysisManager.systemJobInfoMap.put(job.jobId, job); + analysisManager.systemJobStatusUpdater.apply(new TaskStatusWrapper(task1, + AnalysisState.RUNNING, "", 0)); + Assertions.assertTrue(job.startTime >= l); + } + @Test public void testReAnalyze() { new MockUp() {