database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Submit routine load task immediately (#682)

Browse Source 
1. Use submit_routine_load_task instead of agentTaskQueue
2. Remove thrift dependency in StreamLoadPlanner and StreamLoadScanNode
This commit is contained in:
EmmyMiao87
2019-03-04 11:25:58 +08:00
committed by ZHAO Chun
parent 9618d20a72
commit 152606fbd6
18 changed files with 456 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
fe/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java
View File

@ -62,6 +62,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.UUID;
public class KafkaRoutineLoadJobTest {
@ -125,7 +126,7 @@ public class KafkaRoutineLoadJobTest {
};
KafkaRoutineLoadJob kafkaRoutineLoadJob =
new KafkaRoutineLoadJob("1", "kafka_routine_load_job", 1L,
new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L,
1L, routineLoadDesc, 3, 0,
"", "", new KafkaProgress());
Deencapsulation.setField(kafkaRoutineLoadJob, "consumer", kafkaConsumer);
@ -149,7 +150,7 @@ public class KafkaRoutineLoadJobTest {
};
KafkaRoutineLoadJob kafkaRoutineLoadJob =
new KafkaRoutineLoadJob("1", "kafka_routine_load_job", 1L,
new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L,
1L, routineLoadDesc, 3, 0,
"", "", null);
@ -200,7 +201,7 @@ public class KafkaRoutineLoadJobTest {
};
RoutineLoadJob routineLoadJob =
new KafkaRoutineLoadJob("1", "kafka_routine_load_job", 1L,
new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L,
1L, routineLoadDesc, 3, 0,
"", "", null);
new Expectations() {
@ -214,7 +215,7 @@ public class KafkaRoutineLoadJobTest {
};
List<RoutineLoadTaskInfo> routineLoadTaskInfoList = new ArrayList<>();
KafkaTaskInfo kafkaTaskInfo = new KafkaTaskInfo("1", "1");
KafkaTaskInfo kafkaTaskInfo = new KafkaTaskInfo(new UUID(1, 1), 1L);
kafkaTaskInfo.addKafkaPartition(100);
kafkaTaskInfo.setLoadStartTimeMs(System.currentTimeMillis() - DEFAULT_TASK_TIMEOUT_SECONDS * 60 * 1000);
routineLoadTaskInfoList.add(kafkaTaskInfo);
@ -231,7 +232,7 @@ public class KafkaRoutineLoadJobTest {
new Expectations() {
{
routineLoadManager.getJob("1");
routineLoadManager.getJob(1L);
result = routineLoadJob;
}
};

2
fe/src/test/java/org/apache/doris/load/routineload/RoutineLoadSchedulerTest.java
View File

@ -74,7 +74,7 @@ public class RoutineLoadSchedulerTest {
};
RoutineLoadJob routineLoadJob =
new KafkaRoutineLoadJob("1", "kafka_routine_load_job", 1L,
new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L,
1L, routineLoadDesc ,3, 0,
"", "", new KafkaProgress());
routineLoadJob.setState(RoutineLoadJob.JobState.NEED_SCHEDULE);

7
fe/src/test/java/org/apache/doris/load/routineload/RoutineLoadTaskSchedulerTest.java
View File

@ -39,6 +39,7 @@ import org.junit.Test;
import java.util.Map;
import java.util.Queue;
import java.util.UUID;
import mockit.Deencapsulation;
import mockit.Expectations;
@ -64,7 +65,7 @@ public class RoutineLoadTaskSchedulerTest {
long beId = 100L;
Queue<RoutineLoadTaskInfo> routineLoadTaskInfoQueue = Queues.newLinkedBlockingQueue();
KafkaTaskInfo routineLoadTaskInfo1 = new KafkaTaskInfo("1", "1");
KafkaTaskInfo routineLoadTaskInfo1 = new KafkaTaskInfo(new UUID(1, 1), 1l);
routineLoadTaskInfo1.addKafkaPartition(1);
routineLoadTaskInfo1.addKafkaPartition(2);
routineLoadTaskInfoQueue.add(routineLoadTaskInfo1);
@ -116,7 +117,7 @@ public class RoutineLoadTaskSchedulerTest {
result = beId;
routineLoadManager.getJobByTaskId(anyString);
result = kafkaRoutineLoadJob1;
routineLoadManager.getJob(anyString);
routineLoadManager.getJob(anyLong);
result = kafkaRoutineLoadJob1;
}
};
@ -127,7 +128,7 @@ public class RoutineLoadTaskSchedulerTest {
//
// new Expectations() {
// {
// routineLoadTaskInfo1.createStreamLoadTask(anyLong);
// routineLoadTaskInfo1.createRoutineLoadTask(anyLong);
// result = kafkaRoutineLoadTask;
// }
// };

4
fe/src/test/java/org/apache/doris/planner/StreamLoadPlannerTest.java
View File

@ -23,6 +23,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.UserException;
import org.apache.doris.task.StreamLoadTask;
import org.apache.doris.thrift.TStreamLoadPutRequest;
import org.apache.doris.thrift.TUniqueId;
@ -70,7 +71,8 @@ public class StreamLoadPlannerTest {
TStreamLoadPutRequest request = new TStreamLoadPutRequest();
request.setTxnId(1);
request.setLoadId(new TUniqueId(2, 3));
StreamLoadPlanner planner = new StreamLoadPlanner(db, destTable, request);
StreamLoadPlanner planner = new StreamLoadPlanner(db, destTable,
StreamLoadTask.fromTStreamLoadPutRequest(request));
planner.plan();
}
}

40
fe/src/test/java/org/apache/doris/planner/StreamLoadScanNodeTest.java
View File

@ -35,6 +35,7 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.task.StreamLoadTask;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TFileType;
import org.apache.doris.thrift.TPlanNode;
@ -140,7 +141,8 @@ public class StreamLoadScanNodeTest {
}
TStreamLoadPutRequest request = getBaseRequest();
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable, request);
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable,
StreamLoadTask.fromTStreamLoadPutRequest(request));
new Expectations() {{
dstTable.getBaseSchema(); result = columns;
}};
@ -174,7 +176,8 @@ public class StreamLoadScanNodeTest {
TStreamLoadPutRequest request = getBaseRequest();
request.setColumns("k1, k2, v1");
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable, request);
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable,
StreamLoadTask.fromTStreamLoadPutRequest(request));
scanNode.init(analyzer);
scanNode.finalize(analyzer);
@ -203,7 +206,8 @@ public class StreamLoadScanNodeTest {
TStreamLoadPutRequest request = getBaseRequest();
request.setColumns("k1 k2 v1");
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable, request);
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable,
StreamLoadTask.fromTStreamLoadPutRequest(request));
scanNode.init(analyzer);
scanNode.finalize(analyzer);
@ -249,7 +253,8 @@ public class StreamLoadScanNodeTest {
TStreamLoadPutRequest request = getBaseRequest();
request.setFileType(TFileType.FILE_LOCAL);
request.setColumns("k1,k2,v1, v2=k2");
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable, request);
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable,
StreamLoadTask.fromTStreamLoadPutRequest(request));
scanNode.init(analyzer);
scanNode.finalize(analyzer);
@ -297,7 +302,8 @@ public class StreamLoadScanNodeTest {
TStreamLoadPutRequest request = getBaseRequest();
request.setFileType(TFileType.FILE_LOCAL);
request.setColumns("k1,k2, v1=hll_hash(k2)");
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable, request);
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable,
StreamLoadTask.fromTStreamLoadPutRequest(request));
scanNode.init(analyzer);
scanNode.finalize(analyzer);
@ -345,7 +351,8 @@ public class StreamLoadScanNodeTest {
TStreamLoadPutRequest request = getBaseRequest();
request.setFileType(TFileType.FILE_LOCAL);
request.setColumns("k1,k2, v1=hll_hash1(k2)");
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable, request);
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable,
StreamLoadTask.fromTStreamLoadPutRequest(request));
scanNode.init(analyzer);
scanNode.finalize(analyzer);
@ -375,7 +382,8 @@ public class StreamLoadScanNodeTest {
TStreamLoadPutRequest request = getBaseRequest();
request.setFileType(TFileType.FILE_LOCAL);
request.setColumns("k1,k2, v1=k2");
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable, request);
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable,
StreamLoadTask.fromTStreamLoadPutRequest(request));
scanNode.init(analyzer);
scanNode.finalize(analyzer);
@ -405,7 +413,8 @@ public class StreamLoadScanNodeTest {
TStreamLoadPutRequest request = getBaseRequest();
request.setFileType(TFileType.FILE_BROKER);
request.setColumns("k1,k2,v1, v2=k2");
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable, request);
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable,
StreamLoadTask.fromTStreamLoadPutRequest(request));
scanNode.init(analyzer);
scanNode.finalize(analyzer);
@ -434,7 +443,8 @@ public class StreamLoadScanNodeTest {
TStreamLoadPutRequest request = getBaseRequest();
request.setColumns("k1,k2,v1, v2=k3");
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable, request);
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable,
StreamLoadTask.fromTStreamLoadPutRequest(request));
scanNode.init(analyzer);
scanNode.finalize(analyzer);
@ -480,7 +490,8 @@ public class StreamLoadScanNodeTest {
TStreamLoadPutRequest request = getBaseRequest();
request.setColumns("k1,k2,v1, v2=k1");
request.setWhere("k1 = 1");
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable, request);
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable,
StreamLoadTask.fromTStreamLoadPutRequest(request));
scanNode.init(analyzer);
scanNode.finalize(analyzer);
@ -526,7 +537,8 @@ public class StreamLoadScanNodeTest {
TStreamLoadPutRequest request = getBaseRequest();
request.setColumns("k1,k2,v1, v2=k2");
request.setWhere("k1 1");
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable, request);
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable,
StreamLoadTask.fromTStreamLoadPutRequest(request));
scanNode.init(analyzer);
scanNode.finalize(analyzer);
@ -556,7 +568,8 @@ public class StreamLoadScanNodeTest {
TStreamLoadPutRequest request = getBaseRequest();
request.setColumns("k1,k2,v1, v2=k1");
request.setWhere("k5 = 1");
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable, request);
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable,
StreamLoadTask.fromTStreamLoadPutRequest(request));
scanNode.init(analyzer);
scanNode.finalize(analyzer);
@ -586,7 +599,8 @@ public class StreamLoadScanNodeTest {
TStreamLoadPutRequest request = getBaseRequest();
request.setColumns("k1,k2,v1, v2=k1");
request.setWhere("k1 + v2");
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable, request);
StreamLoadScanNode scanNode = new StreamLoadScanNode(new PlanNodeId(1), dstDesc, dstTable,
StreamLoadTask.fromTStreamLoadPutRequest(request));
scanNode.init(analyzer);
scanNode.finalize(analyzer);