database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[refactor](group commit) remove future block (#27720)

Browse Source 
Co-authored-by: huanghaibin <284824253@qq.com>
This commit is contained in:
meiyi
2023-12-11 08:41:51 +08:00
committed by GitHub
parent 320ddf4987
commit 1e5ff40e17
40 changed files with 168 additions and 686 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
View File

@ -50,14 +50,12 @@ import org.apache.doris.planner.DataPartition;
import org.apache.doris.planner.DataSink;
import org.apache.doris.planner.ExportSink;
import org.apache.doris.planner.GroupCommitBlockSink;
import org.apache.doris.planner.GroupCommitOlapTableSink;
import org.apache.doris.planner.GroupCommitPlanner;
import org.apache.doris.planner.OlapTableSink;
import org.apache.doris.planner.external.jdbc.JdbcTableSink;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rewrite.ExprRewriter;
import org.apache.doris.service.FrontendOptions;
import org.apache.doris.tablefunction.GroupCommitTableValuedFunction;
import org.apache.doris.thrift.TQueryOptions;
import org.apache.doris.thrift.TUniqueId;
import org.apache.doris.transaction.TransactionState;
@ -153,8 +151,6 @@ public class NativeInsertStmt extends InsertStmt {
private TUniqueId loadId = null;
private ByteString execPlanFragmentParamsBytes = null;
private long tableId = -1;
// true if be generates an insert from group commit tvf stmt and executes to load data
public boolean isGroupCommitTvf = false;
public boolean isGroupCommitStreamLoadSql = false;
private GroupCommitPlanner groupCommitPlanner;
@ -970,12 +966,8 @@ public class NativeInsertStmt extends InsertStmt {
return dataSink;
}
if (targetTable instanceof OlapTable) {
checkInnerGroupCommit();
OlapTableSink sink;
if (isGroupCommitTvf) {
sink = new GroupCommitOlapTableSink((OlapTable) targetTable, olapTuple,
targetPartitionIds, analyzer.getContext().getSessionVariable().isEnableSingleReplicaInsert());
} else if (isGroupCommitStreamLoadSql) {
if (isGroupCommitStreamLoadSql) {
sink = new GroupCommitBlockSink((OlapTable) targetTable, olapTuple,
targetPartitionIds, analyzer.getContext().getSessionVariable().isEnableSingleReplicaInsert());
} else {
@ -1019,17 +1011,6 @@ public class NativeInsertStmt extends InsertStmt {
return dataSink;
}
private void checkInnerGroupCommit() {
List<TableRef> tableRefs = new ArrayList<>();
queryStmt.collectTableRefs(tableRefs);
if (tableRefs.size() == 1 && tableRefs.get(0) instanceof TableValuedFunctionRef) {
TableValuedFunctionRef tvfRef = (TableValuedFunctionRef) tableRefs.get(0);
if (tvfRef.getTableFunction() instanceof GroupCommitTableValuedFunction) {
isGroupCommitTvf = true;
}
}
}
public void complete() throws UserException {
if (!isExplain() && targetTable instanceof OlapTable) {
((OlapTableSink) dataSink).complete(analyzer);

36
fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitOlapTableSink.java
View File

@ -1,36 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.planner;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.thrift.TDataSinkType;
import java.util.List;
public class GroupCommitOlapTableSink extends OlapTableSink {
public GroupCommitOlapTableSink(OlapTable dstTable, TupleDescriptor tupleDescriptor, List<Long> partitionIds,
boolean singleReplicaLoad) {
super(dstTable, tupleDescriptor, partitionIds, singleReplicaLoad);
}
protected TDataSinkType getDataSinkType() {
return TDataSinkType.GROUP_COMMIT_OLAP_TABLE_SINK;
}
}

3
fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
View File

@ -2022,9 +2022,6 @@ public class FrontendServiceImpl implements FrontendService.Iface {
// The txn_id here is obtained from the NativeInsertStmt
result.getParams().setTxnConf(new TTxnParams().setTxnId(txn_id));
result.getParams().setImportLabel(parsedStmt.getLabel());
if (parsedStmt.isGroupCommitTvf) {
result.getParams().params.setGroupCommit(true);
}
result.setDbId(parsedStmt.getTargetTable().getDatabase().getId());
result.setTableId(parsedStmt.getTargetTable().getId());
result.setBaseSchemaVersion(((OlapTable) parsedStmt.getTargetTable()).getBaseSchemaVersion());