database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Pipeline](sink) support olap table sink operator (#14872)

Browse Source 
* support olap table sink operator

* update config
This commit is contained in:
Pxl
2022-12-07 15:29:56 +08:00
committed by GitHub
parent cdbbf1e4ee
commit 48a9166aa4
16 changed files with 150 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fe/fe-core/src/main/java/org/apache/doris/common/Config.java
View File

@ -1683,6 +1683,9 @@ public class Config extends ConfigBase {
@ConfField
public static boolean enable_vectorized_load = true;
@ConfField
public static boolean enable_pipeline_load = false;
@ConfField(mutable = false, masterOnly = true)
public static int backend_rpc_timeout_ms = 60000; // 1 min

1
fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
View File

@ -137,6 +137,7 @@ public class LoadLoadingTask extends LoadTask {
curCoordinator.setQueryType(TQueryType.LOAD);
curCoordinator.setExecMemoryLimit(execMemLimit);
curCoordinator.setExecVecEngine(Config.enable_vectorized_load);
curCoordinator.setExecPipEngine(Config.enable_pipeline_load);
/*
* For broker load job, user only need to set mem limit by 'exec_mem_limit' property.
* And the variable 'load_mem_limit' does not make any effect.

2
fe/fe-core/src/main/java/org/apache/doris/load/update/UpdateStmtExecutor.java
View File

@ -24,6 +24,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.DuplicatedRequestException;
import org.apache.doris.common.ErrorCode;
@ -142,6 +143,7 @@ public class UpdateStmtExecutor {
updatePlanner.getFragments(), updatePlanner.getScanNodes(), TimeUtils.DEFAULT_TIME_ZONE, false);
coordinator.setQueryType(TQueryType.LOAD);
coordinator.setExecVecEngine(VectorizedUtil.isVectorized());
coordinator.setExecPipEngine(Config.enable_pipeline_load);
QeProcessorImpl.INSTANCE.registerQuery(queryId, coordinator);
analyzer.getContext().getExecutor().setCoord(coordinator);

1
fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
View File

@ -250,6 +250,7 @@ public class StreamLoadPlanner {
// for stream load, we use exec_mem_limit to limit the memory usage of load channel.
queryOptions.setLoadMemLimit(taskInfo.getMemLimit());
queryOptions.setEnableVectorizedEngine(Config.enable_vectorized_load);
queryOptions.setEnablePipelineEngine(Config.enable_pipeline_load);
queryOptions.setBeExecVersion(Config.be_exec_version);
params.setQueryOptions(queryOptions);

5
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
View File

@ -319,6 +319,7 @@ public class Coordinator {
private void initQueryOptions(ConnectContext context) {
this.queryOptions = context.getSessionVariable().toThrift();
this.queryOptions.setEnableVectorizedEngine(VectorizedUtil.isVectorized());
this.queryOptions.setEnablePipelineEngine(Config.enable_pipeline_load);
this.queryOptions.setBeExecVersion(Config.be_exec_version);
}
@ -342,6 +343,10 @@ public class Coordinator {
this.queryOptions.setEnableVectorizedEngine(vec);
}
public void setExecPipEngine(boolean vec) {
this.queryOptions.setEnablePipelineEngine(vec);
}
public Status getExecStatus() {
return queryStatus;
}