hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Do not report ERROR when sample rate equals 100

Browse Source
This commit is contained in:
ZenoWang
2024-02-06 14:49:31 +00:00
committed by ob-robot
parent 1f1d5c08ae
commit c8ef409bf3
3710 changed files with 486984 additions and 3083329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/sql/engine/cmd/ob_load_data_impl.h
View File

@ -26,12 +26,11 @@
#include "lib/container/ob_bit_set.h"
#include "lib/utility/ob_utility.h"
#include "sql/resolver/cmd/ob_load_data_stmt.h"
#include "sql/printer/ob_raw_expr_printer.h"
#include "sql/resolver/expr/ob_raw_expr_printer.h"
#include "sql/optimizer/ob_table_location.h"
#include "sql/engine/cmd/ob_load_data_rpc.h"
#include "sql/engine/ob_des_exec_context.h"
#include "sql/engine/cmd/ob_load_data_parser.h"
#include "sql/engine/cmd/ob_load_data_file_reader.h"
#include "common/storage/ob_io_device.h"
namespace oceanbase
@ -385,8 +384,7 @@ struct ObCSVFormats {
class ObLoadFileDataTrimer
{
public:
ObLoadFileDataTrimer() : incomplate_data_(NULL), incomplate_data_len_(0),
incomplate_data_buf_len_(ObLoadFileBuffer::MAX_BUFFER_SIZE), lines_cnt_(0) {}
ObLoadFileDataTrimer() : incomplate_data_(NULL), incomplate_data_len_(0), lines_cnt_(0) {}
int init(common::ObIAllocator &allocator, const ObCSVFormats &formats);
//for debug
ObString get_incomplate_data_string() {
@ -398,12 +396,10 @@ public:
bool has_incomplate_data() const { return incomplate_data_len_ > 0; }
int64_t get_lines_count() const { return lines_cnt_; }
void commit_line_cnt(int64_t line_cnt) { lines_cnt_ += line_cnt; }
int expand_buf(common::ObIAllocator &allocator);
private:
ObCSVFormats formats_;
ObCSVFormats formats_;//TODO [load data] change to ObInverseParser(formats)
char *incomplate_data_;
int64_t incomplate_data_len_;
int64_t incomplate_data_buf_len_;
int64_t lines_cnt_;
};
@ -495,9 +491,6 @@ struct ObShuffleTaskHandle {
common::ObBitSet<> &main_string_values,
uint64_t tenant_id);
~ObShuffleTaskHandle();
int expand_buf(const int64_t max_size);
ObArenaAllocator allocator;
ObDesExecContext exec_ctx;
ObLoadFileBuffer *data_buffer;
@ -512,7 +505,6 @@ struct ObShuffleTaskHandle {
common::ObBitSet<> &string_values;
ObShuffleResult result;
ObSEArray<ObParserErrRec, 16> err_records;
common::ObMemAttr attr;
TO_STRING_KV("task_id", result.task_id_);
};
@ -685,7 +677,7 @@ public:
InsertTask,
};
struct ToolBox {
ToolBox() : file_reader(nullptr), job_status(nullptr), expr_buffer(nullptr), temp_handle(nullptr) {}
ToolBox() : device_handle_(NULL), fd_(), expr_buffer(nullptr) {}
int init(ObExecContext &ctx, ObLoadDataStmt &load_stmt);
int build_calc_partid_expr(ObExecContext &ctx,
ObLoadDataStmt &load_stmt,
@ -693,13 +685,14 @@ public:
int release_resources();
//modules
ObFileReader * file_reader;
ObFileReader file_reader;
ObIODevice* device_handle_;
ObIOFd fd_;
ObFileAppender file_appender;
ObFileReadCursor read_cursor;
ObLoadFileDataTrimer data_trimer;
ObInsertValueGenerator generator;
ObDataFragMgr data_frag_mgr;
ObFileReadParam file_read_param;
//running control
ObParallelTaskController shuffle_task_controller;
@ -718,7 +711,6 @@ public:
int64_t num_of_table_column;
int64_t parallel;
int64_t batch_row_count;
int64_t batch_buffer_size;
int64_t data_frag_mem_usage_limit; //limit = data_frag_mem_usage_limit * MAX_BUFFER_SIZE
int64_t file_size;
int64_t ignore_rows;
@ -745,7 +737,6 @@ public:
common::ObSEArray<int64_t, 1> file_buf_row_num;
int64_t last_session_check_ts;
common::ObSEArray<ObLoadTableColumnDesc, 16> insert_infos;
ObShuffleTaskHandle *temp_handle;
//debug values
int64_t insert_dispatch_rows;
@ -768,7 +759,7 @@ public:
int execute(ObExecContext &ctx, ObLoadDataStmt &load_stmt);
int shuffle_task_gen_and_dispatch(ObExecContext &ctx, ToolBox &box);
int next_file_buffer(ObExecContext &ctx, ToolBox &box, ObShuffleTaskHandle *handle, int64_t limit = INT64_MAX);
int next_file_buffer(ToolBox &box, ObLoadFileBuffer &data_buffer, int64_t limit = INT64_MAX);
int handle_returned_shuffle_task(ToolBox &box, ObShuffleTaskHandle &handle);
int wait_shuffle_task_return(ToolBox &box);