Change label of broker load txn (#1134)

* Change label of broker load txn

1. put broker load label into txn label
2. fix the bug of `label is already used`
3. fix partition error of new broker load

* Fix count error in mini load and broker load

There are three params (num_rows_load_total, num_rows_load_filtered, num_rows_load_unselected) which are used to count dpp.norm.ALL and dpp.abnorm.ALL.
num_rows_load_total is the number rows of source file.
num_rows_load_unselected is the not satisfied (where conjuncts) rows of num_rows_load_total
num_rows_load_filtered is the rows (quality not good enough) of (num_rows_load_total-num_rows_load_unselected)

be/src/exec/broker_scan_node.cpp

@@ -407,7 +407,6 @@ void BrokerScanNode::scanner_worker(int start_idx, int length) {
     }
 
     // Update stats
-    // _runtime_state->update_num_rows_load_success(counter.num_rows_returned);
     _runtime_state->update_num_rows_load_total(counter.num_rows_total);
     _runtime_state->update_num_rows_load_filtered(counter.num_rows_filtered);
     _runtime_state->update_num_rows_load_unselected(counter.num_rows_unselected);

be/src/exec/csv_scan_node.cpp

@@ -305,12 +305,9 @@ Status CsvScanNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eos
         }
         // split & check line & fill default value
         bool is_success = split_check_fill(line, state);
-        if (is_success) {
-            ++_normal_row_number;
-            state->set_normal_row_number(state->get_normal_row_number() + 1);
-        } else {
-            ++_error_row_number;
-            state->set_error_row_number(state->get_error_row_number() + 1);
+        ++_num_rows_load_total;
+        if (!is_success) {
+            ++_num_rows_load_filtered;
             continue;
         }
 
@@ -328,8 +325,10 @@ Status CsvScanNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eos
             _tuple = reinterpret_cast<Tuple*>(new_tuple);
         }
     }
-    VLOG_ROW << "normal_row_number: " << state->get_normal_row_number()
-            << "; error_row_number: " << state->get_error_row_number() << std::endl;
+    state->update_num_rows_load_total(_num_rows_load_total);
+    state->update_num_rows_load_filtered(_num_rows_load_filtered);
+    VLOG_ROW << "normal_row_number: " << state->num_rows_load_success()
+            << "; error_row_number: " << state->num_rows_load_filtered() << std::endl;
 
     row_batch->tuple_data_pool()->acquire_data(_tuple_pool.get(), false);
 
@@ -352,7 +351,7 @@ Status CsvScanNode::close(RuntimeState* state) {
 
     RETURN_IF_ERROR(ExecNode::close(state));
 
-    if (state->get_normal_row_number() == 0) {
+    if (state->num_rows_load_success() == 0) {
         std::stringstream error_msg;
         error_msg << "Read zero normal line file. ";
         state->append_error_msg_to_file("", error_msg.str(), true);
@@ -361,11 +360,11 @@ Status CsvScanNode::close(RuntimeState* state) {
     }
 
     // only write summary line if there are error lines
-    if (_error_row_number > 0) {
+    if (_num_rows_load_filtered > 0) {
         // Summary normal line and error line number info
         std::stringstream summary_msg;
-        summary_msg << "error line: " << _error_row_number
-            << "; normal line: " << _normal_row_number;
+        summary_msg << "error line: " << _num_rows_load_filtered
+            << "; normal line: " << state->num_rows_load_success();
         state->append_error_msg_to_file("", summary_msg.str(), true);
     }
 

be/src/exec/csv_scan_node.h

@@ -126,8 +126,8 @@ private:
     // Current RuntimeState
     RuntimeState* _runtime_state;
 
-    int64_t _error_row_number = 0L;
-    int64_t _normal_row_number = 0L;
+    int64_t _num_rows_load_total = 0L;
+    int64_t _num_rows_load_filtered = 0L;
 
     RuntimeProfile::Counter* _split_check_timer;
     RuntimeProfile::Counter* _split_line_timer;

be/src/exec/es_http_scan_node.cpp

@@ -424,9 +424,6 @@ void EsHttpScanNode::scanner_worker(int start_idx, int length, std::promise<Stat
             << status.get_error_msg();
     }
 
-    // Update stats
-    _runtime_state->update_num_rows_load_success(counter.num_rows_returned);
-    _runtime_state->update_num_rows_load_filtered(counter.num_rows_filtered);
 
     // scanner is going to finish 
     {

be/src/exec/olap_rewrite_node.cpp

@@ -247,7 +247,6 @@ bool OlapRewriteNode::copy_rows(RuntimeState* state, RowBatch* output_batch) {
     }
     if (num_rows_invalid > 0) {
         state->update_num_rows_load_filtered(num_rows_invalid);
-        state->update_num_rows_load_success(-1 * num_rows_invalid);
     }
 
     if (VLOG_ROW_IS_ON) {

be/src/exec/olap_table_sink.cpp

@@ -603,7 +603,6 @@ Status OlapTableSink::close(RuntimeState* state, Status close_status) {
         COUNTER_SET(_convert_batch_timer, _convert_batch_ns);
         COUNTER_SET(_validate_data_timer, _validate_data_ns);
         state->update_num_rows_load_filtered(_number_filtered_rows);
-        state->update_num_rows_load_success(_number_output_rows);
     } else {
         for (auto channel : _channels) {
             channel->cancel();

be/src/runtime/data_spliter.cpp

@@ -171,7 +171,6 @@ Status DataSpliter::process_partition(
             error_log << "there is no corresponding partition for this key: ";
             ctx->print_value(row, &error_log);
             state->update_num_rows_load_filtered(1);
-            state->update_num_rows_load_success(-1);
             return Status(error_log.str(), true);
         }
         *info = _partition_infos[*part_index];

be/src/runtime/etl_job_mgr.cpp

@@ -166,12 +166,12 @@ void EtlJobMgr::finalize_job(PlanFragmentExecutor* executor) {
             result.file_map[to_http_path(it)] = file_size;
         }
         // set statistics
-        result.process_normal_rows = state->get_normal_row_number();
-        result.process_abnormal_rows = state->get_error_row_number();
+        result.process_normal_rows = state->num_rows_load_success();
+        result.process_abnormal_rows = state->num_rows_load_filtered();
     } else {
         // get debug path
-        result.process_normal_rows = state->get_normal_row_number();
-        result.process_abnormal_rows = state->get_error_row_number();
+        result.process_normal_rows = state->num_rows_load_success();
+        result.process_abnormal_rows = state->num_rows_load_filtered();
     }
 
     result.debug_path = state->get_error_log_file_path();

be/src/runtime/fragment_mgr.cpp

@@ -266,7 +266,7 @@ void FragmentExecState::coordinator_callback(
             params.delta_urls.push_back(to_http_path(it));
         }
     }
-    if (runtime_state->num_rows_load_success() > 0 ||
+    if (runtime_state->num_rows_load_total() > 0 ||
             runtime_state->num_rows_load_filtered() > 0) {
         params.__isset.load_counters = true;
         // TODO(zc)

be/src/runtime/runtime_state.cpp

@@ -60,7 +60,6 @@ RuntimeState::RuntimeState(
             _per_fragment_instance_idx(0),
             _root_node_id(-1),
             _num_rows_load_total(0),
-            _num_rows_load_success(0),
             _num_rows_load_filtered(0),
             _num_rows_load_unselected(0),
             _num_print_error_rows(0),
@@ -88,7 +87,6 @@ RuntimeState::RuntimeState(
             _per_fragment_instance_idx(0),
             _root_node_id(-1),
             _num_rows_load_total(0),
-            _num_rows_load_success(0),
             _num_rows_load_filtered(0),
             _num_rows_load_unselected(0),
             _num_print_error_rows(0),

be/src/runtime/runtime_state.h

@@ -415,10 +415,6 @@ public:
         return _num_rows_load_total.load();
     }
 
-    int64_t num_rows_load_success() {
-        return _num_rows_load_success.load();
-    }
-
     int64_t num_rows_load_filtered() {
         return _num_rows_load_filtered.load();
     }
@@ -427,12 +423,12 @@ public:
         return _num_rows_load_unselected.load();
     }
 
-    void update_num_rows_load_total(int64_t num_rows) {
-        _num_rows_load_total.fetch_add(num_rows);
+    int64_t num_rows_load_success() {
+        return num_rows_load_total() - num_rows_load_filtered() - num_rows_load_unselected();
     }
 
-    void update_num_rows_load_success(int64_t num_rows) {
-        _num_rows_load_success.fetch_add(num_rows);
+    void update_num_rows_load_total(int64_t num_rows) {
+        _num_rows_load_total.fetch_add(num_rows);
     }
 
     void update_num_rows_load_filtered(int64_t num_rows) {
@@ -601,7 +597,6 @@ private:
     // put here to collect files??
     std::vector<std::string> _output_files;
     std::atomic<int64_t> _num_rows_load_total;  // total rows read from source
-    std::atomic<int64_t> _num_rows_load_success;    // rows loaded success
     std::atomic<int64_t> _num_rows_load_filtered;   // unqualified rows
     std::atomic<int64_t> _num_rows_load_unselected; // rows filtered by predicates
     std::atomic<int64_t> _num_print_error_rows;

docs/help/Contents/Data Manipulation/broker_load.md

@@ -342,42 +342,5 @@
         )
         WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password");
 
-    8. 导入的数据同一列做sum max然后生成两列指标
-
-       假设导入文件 file 有两列, 我们称为 k1 和 v1， my_table 表有 4 列，分别是 k1, sumv1, maxv1, minv1，建表语句如下：
-
-       CREATE TABLE my_db.my_table
-       (
-         k1 INT,
-         sumv1 INT SUM,
-         maxv1 INT MAX,
-         minv1 INT MIN
-       )
-         ENGINE=olap
-         AGGREGATE KEY(k1)
-         DISTRIBUTED BY HASH (k1) BUCKETS 32
-         PARTITION BY RANGE (k1)
-         (
-         PARTITION p1 VALUES LESS THAN ("2014-01-01"),
-         PARTITION p2 VALUES LESS THAN ("2014-06-01")
-       )
-
-       导入语句可以写成：
-
-       LOAD LABEL my_db.mylabel
-       (
-         DATA INFILE("hdfs://hdfs_host:hdfs_port/user/palo/data/input/file")
-         INTO TABLE `my_table`
-         PARTITION (p1, p2)
-         COLUMNS TERMINATED BY ","
-         (k1, v1)
-         SET (
-           sumv1 = v1,
-           maxv1 = v1,
-           minv1 = v1
-         )
-       )
-       WITH BROKER hdfs ("username"="hdfs_user", "password"="hdfs_password");
- 
 ## keyword
     BROKER LOAD

fe/src/main/java/org/apache/doris/analysis/LoadStmt.java

@@ -81,7 +81,7 @@ public class LoadStmt extends DdlStmt {
     private final Map<String, String> properties;
     private String user;
 
-    private static String version = "v1";
+    private String version = "v1";
 
     // properties set
     private final static ImmutableSet<String> PROPERTIES_SET = new ImmutableSet.Builder<String>()
@@ -184,11 +184,20 @@ public class LoadStmt extends DdlStmt {
             if (!versionProperty.equalsIgnoreCase(LoadManager.VERSION)) {
                 throw new DdlException(VERSION + " must be " + LoadManager.VERSION);
             }
-            version = LoadManager.VERSION;
         }
 
     }
 
+    private void analyzeVersion() {
+        if (properties == null) {
+            return;
+        }
+        final String versionProperty = properties.get(VERSION);
+        if (versionProperty != null) {
+            version = LoadManager.VERSION;
+        }
+    }
+
     @Override
     public void analyze(Analyzer analyzer) throws UserException {
         super.analyze(analyzer);
@@ -209,6 +218,7 @@ public class LoadStmt extends DdlStmt {
             throw new AnalysisException(e.getMessage());
         }
 
+        analyzeVersion();
         user = ConnectContext.get().getQualifiedUser();
     }
 

fe/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java

@@ -254,7 +254,6 @@ public class BrokerLoadJob extends LoadJob {
         }
         db.writeLock();
         try {
-
             Catalog.getCurrentGlobalTransactionMgr().commitTransaction(
                     dbId, transactionId, commitInfos);
         } catch (UserException e) {

fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java

@@ -30,7 +30,6 @@ import org.apache.doris.common.DdlException;
 import org.apache.doris.common.LabelAlreadyUsedException;
 import org.apache.doris.common.MetaNotFoundException;
 import org.apache.doris.common.UserException;
-import org.apache.doris.common.util.DebugUtil;
 import org.apache.doris.common.util.LogBuilder;
 import org.apache.doris.common.util.LogKey;
 import org.apache.doris.common.util.TimeUtils;
@@ -161,7 +160,7 @@ public abstract class LoadJob extends AbstractTxnStateChangeCallback implements
         return finishTimestamp;
     }
 
-    protected boolean isFinished() {
+    public boolean isFinished() {
         return state == JobState.FINISHED || state == JobState.CANCELLED;
     }
 
@@ -222,7 +221,7 @@ public abstract class LoadJob extends AbstractTxnStateChangeCallback implements
         // register txn state listener
         Catalog.getCurrentGlobalTransactionMgr().getCallbackFactory().addCallback(this);
         transactionId = Catalog.getCurrentGlobalTransactionMgr()
-                .beginTransaction(dbId, String.valueOf(id), -1, "FE: " + FrontendOptions.getLocalHostAddress(),
+                .beginTransaction(dbId, label, -1, "FE: " + FrontendOptions.getLocalHostAddress(),
                                   TransactionState.LoadJobSourceType.FRONTEND, id,
                                   timeoutSecond - 1);
     }
@@ -318,14 +317,10 @@ public abstract class LoadJob extends AbstractTxnStateChangeCallback implements
 
     protected void executeCancel(FailMsg failMsg) {
         LOG.warn(new LogBuilder(LogKey.LOAD_JOB, id)
+                         .add("transaction_id", transactionId)
                          .add("error_msg", "Failed to execute load with error " + failMsg.getMsg())
                          .build());
 
-        // reset txn id
-        if (transactionId != -1) {
-            transactionId = -1;
-        }
-
         // clean the loadingStatus
         loadingStatus.setState(TEtlState.CANCELLED);
 
@@ -336,17 +331,32 @@ public abstract class LoadJob extends AbstractTxnStateChangeCallback implements
         // set failMsg and state
         this.failMsg = failMsg;
         finishTimestamp = System.currentTimeMillis();
-        state = JobState.CANCELLED;
 
         // remove callback
         Catalog.getCurrentGlobalTransactionMgr().getCallbackFactory().removeCallback(id);
+        // abort txn
+        try {
+            LOG.debug(new LogBuilder(LogKey.LOAD_JOB, id)
+                              .add("transaction_id", transactionId)
+                              .add("msg", "begin to abort txn")
+                              .build());
+            Catalog.getCurrentGlobalTransactionMgr().abortTransaction(transactionId, failMsg.getMsg());
+        } catch (UserException e) {
+            LOG.warn(new LogBuilder(LogKey.LOAD_JOB, id)
+                             .add("transaction_id", transactionId)
+                             .add("error_msg", "failed to abort txn when job is cancelled, txn will be aborted later")
+                             .build());
+        }
+
+        // change state
+        state = JobState.CANCELLED;
     }
 
     private void executeFinish() {
         progress = 100;
         finishTimestamp = System.currentTimeMillis();
-        state = JobState.FINISHED;
         Catalog.getCurrentGlobalTransactionMgr().getCallbackFactory().removeCallback(id);
+        state = JobState.FINISHED;
 
         MetricRepo.COUNTER_LOAD_FINISHED.increase(1L);
     }
@@ -438,7 +448,7 @@ public abstract class LoadJob extends AbstractTxnStateChangeCallback implements
     public void beforeCommitted(TransactionState txnState) throws TransactionException {
         writeLock();
         try {
-            if (transactionId == -1) {
+            if (isFinished()) {
                 throw new TransactionException("txn could not be committed when job has been cancelled");
             }
             isCommitting = true;
@@ -478,7 +488,7 @@ public abstract class LoadJob extends AbstractTxnStateChangeCallback implements
         }
         writeLock();
         try {
-            if (transactionId == -1) {
+            if (isFinished()) {
                 return;
             }
             // cancel load job

fe/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java

@@ -26,11 +26,11 @@ import org.apache.doris.catalog.OlapTable;
 import org.apache.doris.common.LoadException;
 import org.apache.doris.common.Status;
 import org.apache.doris.common.UserException;
+import org.apache.doris.common.util.DebugUtil;
 import org.apache.doris.common.util.LogBuilder;
 import org.apache.doris.common.util.LogKey;
 import org.apache.doris.load.BrokerFileGroup;
 import org.apache.doris.load.FailMsg;
-import org.apache.doris.load.Load;
 import org.apache.doris.qe.Coordinator;
 import org.apache.doris.qe.QeProcessorImpl;
 import org.apache.doris.thrift.TBrokerFileStatus;
@@ -122,6 +122,13 @@ public class LoadLoadingTask extends LoadTask {
             throw new LoadException("failed to execute plan when the left time is less then 0");
         }
 
+        if (LOG.isDebugEnabled()) {
+            LOG.debug(new LogBuilder(LogKey.LOAD_JOB, callback.getCallbackId())
+                              .add("task_id", signature)
+                              .add("query_id", DebugUtil.printId(curCoordinator.getQueryId()))
+                              .add("msg", "begin to execute plan")
+                              .build());
+        }
         curCoordinator.exec();
         if (curCoordinator.join(waitSecond)) {
             Status status = curCoordinator.getExecStatus();

fe/src/main/java/org/apache/doris/load/loadv2/LoadManager.java

@@ -213,7 +213,7 @@ public class LoadManager {
             Map<String, List<LoadJob>> labelToLoadJobs = dbIdToLabelToLoadJobs.get(dbId);
             if (labelToLoadJobs.containsKey(label)) {
                 List<LoadJob> labelLoadJobs = labelToLoadJobs.get(label);
-                if (labelLoadJobs.stream().filter(entity -> !entity.isFinished()).count() != 0) {
+                if (labelLoadJobs.stream().filter(entity -> entity.getState() != JobState.CANCELLED).count() != 0) {
                     LOG.warn("Failed to add load job when label {} has been used.", label);
                     throw new LabelAlreadyUsedException(label);
                 }

fe/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java

@@ -159,7 +159,7 @@ public class LoadingTaskPlanner {
         if (Strings.isNullOrEmpty(result)) {
             return null;
         }
-        result = result.substring(0, result.length() - 2);
+        result = result.substring(0, result.length() - 1);
         return result;
     }
 }