diff --git a/src/observer/table_load/ob_table_load_bucket.cpp b/src/observer/table_load/ob_table_load_bucket.cpp index dfc291cc3c..4fbe4318b7 100644 --- a/src/observer/table_load/ob_table_load_bucket.cpp +++ b/src/observer/table_load/ob_table_load_bucket.cpp @@ -41,6 +41,7 @@ int ObTableLoadBucket::init(const ObAddr &leader_addr) { int ObTableLoadBucket::add_row(const ObTabletID &tablet_id, const ObTableLoadObjRow &obj_row, int64_t batch_size, + int64_t row_size, bool &flag) { OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, bucket_add_row_time_us); @@ -51,9 +52,9 @@ int ObTableLoadBucket::add_row(const ObTabletID &tablet_id, flag = false; if (OB_FAIL(row_array_.push_back(tablet_obj_row))) { LOG_WARN("fail to add row", KR(ret)); - } - if (OB_SUCC(ret)) { - flag = (row_array_.count() >= batch_size); + } else { + row_size_ += tablet_obj_row.get_serialize_size(); + flag = (row_array_.count() >= batch_size || row_size_ >= row_size); } return ret; } diff --git a/src/observer/table_load/ob_table_load_bucket.h b/src/observer/table_load/ob_table_load_bucket.h index 9a9526ca85..23e2627e30 100644 --- a/src/observer/table_load/ob_table_load_bucket.h +++ b/src/observer/table_load/ob_table_load_bucket.h @@ -28,21 +28,24 @@ namespace observer class ObTableLoadBucket { public: - ObTableLoadBucket() : is_inited_(false), sequence_no_(0) {} + ObTableLoadBucket() : is_inited_(false), row_size_(0), sequence_no_(0) {} int add_row(const common::ObTabletID &tablet_id, const table::ObTableLoadObjRow &obj_row, int64_t batch_size, + int64_t row_size, bool &flag); void reset() { is_inited_ = false; leader_addr_.reset(); row_array_.reset(); + row_size_ = 0; } void clear_data() { row_array_.reset(); + row_size_ = 0; } bool is_inited() const { @@ -57,6 +60,7 @@ public: bool is_inited_; common::ObAddr leader_addr_; table::ObTableLoadTabletObjRowArray row_array_; + int64_t row_size_; uint64_t sequence_no_; }; diff --git a/src/observer/table_load/ob_table_load_client_task.cpp b/src/observer/table_load/ob_table_load_client_task.cpp index 3bb4284f15..522e64886f 100644 --- a/src/observer/table_load/ob_table_load_client_task.cpp +++ b/src/observer/table_load/ob_table_load_client_task.cpp @@ -291,7 +291,7 @@ public: load_param.table_id_ = table_schema->get_table_id(); load_param.parallel_ = task_param.get_parallel(); load_param.session_count_ = task_param.get_parallel(); - load_param.batch_size_ = 100; + load_param.batch_size_ = ObTableLoadParam::DEFAULT_BATCH_SIZE; load_param.max_error_row_count_ = task_param.get_max_error_row_count(); load_param.column_count_ = column_ids.count(); load_param.need_sort_ = true; diff --git a/src/observer/table_load/ob_table_load_data_row_handler.cpp b/src/observer/table_load/ob_table_load_data_row_handler.cpp index 0bf14ab1a6..ce83f6a0fe 100644 --- a/src/observer/table_load/ob_table_load_data_row_handler.cpp +++ b/src/observer/table_load/ob_table_load_data_row_handler.cpp @@ -58,7 +58,7 @@ int ObTableLoadDataRowHandler::init( } int ObTableLoadDataRowHandler::handle_insert_row(const ObTabletID tablet_id, - const blocksstable::ObDatumRow &row) + const ObDatumRow &row) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -74,8 +74,7 @@ int ObTableLoadDataRowHandler::handle_insert_row(const ObTabletID tablet_id, iter->second->project_->projector(tablet_id, row, false, index_tablet_id, index_row))) { LOG_WARN("fail to projector", KR(ret), K(tablet_id), K(row)); } else { - index_row.row_flag_.set_flag(blocksstable::DF_INSERT, - blocksstable::ObDmlRowFlagType::DF_TYPE_INSERT_DELETE); + index_row.row_flag_.set_flag(DF_INSERT); if (OB_FAIL(iter->second->get_index_table_builder(builder))) { LOG_WARN("get builder failed", KR(ret)); } else if (OB_FAIL(builder->append_row(index_tablet_id, 0, index_row))) { @@ -90,6 +89,47 @@ int ObTableLoadDataRowHandler::handle_insert_row(const ObTabletID tablet_id, return ret; } +int ObTableLoadDataRowHandler::handle_insert_batch(const ObTabletID &tablet_id, const ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadDataRowHandler not init", KR(ret), KP(this)); + } else if (0 == datum_rows.row_count_) { + // do nothing + } else { + FOREACH_X(iter, *index_store_table_ctx_map_, OB_SUCC(ret)) + { + ObTableLoadStoreTableCtx *store_table_ctx = iter->second; + ObIDirectLoadPartitionTableBuilder *builder = nullptr; + ObTabletID index_tablet_id; + ObDatumRow index_datum_row; + index_datum_row.row_flag_.set_flag(DF_INSERT); + if (OB_FAIL(store_table_ctx->get_index_table_builder(builder))) { + LOG_WARN("fail to get index table builder", KR(ret)); + } else if (OB_FAIL(store_table_ctx->project_->get_index_tablet_id(tablet_id, index_tablet_id))) { + LOG_WARN("fail to get index tablet id", KR(ret)); + } else if (OB_FAIL(store_table_ctx->project_->init_datum_row(index_datum_row))) { + LOG_WARN("fail to projector", KR(ret), K(tablet_id)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < datum_rows.row_count_; ++i) { + if (OB_FAIL(store_table_ctx->project_->projector(datum_rows, + i, + false/*has_multi_version_cols*/, + index_datum_row))) { + LOG_WARN("fail to projector", KR(ret), K(tablet_id)); + } else if (OB_FAIL(builder->append_row(index_tablet_id, 0, index_datum_row))) { + LOG_WARN("add row failed", KR(ret), K(index_datum_row)); + } + } + } + if (OB_SUCC(ret)) { + ATOMIC_AAF(&result_info_->rows_affected_, datum_rows.row_count_); + } + } + return ret; +} + int ObTableLoadDataRowHandler::handle_insert_row_with_multi_version(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) { int ret = OB_SUCCESS; @@ -106,8 +146,7 @@ int ObTableLoadDataRowHandler::handle_insert_row_with_multi_version(const ObTabl iter->second->project_->projector(tablet_id, row, true, index_tablet_id, index_row))) { LOG_WARN("fail to projector", KR(ret), K(tablet_id), K(row)); } else { - index_row.row_flag_.set_flag(blocksstable::DF_INSERT, - blocksstable::ObDmlRowFlagType::DF_TYPE_INSERT_DELETE); + index_row.row_flag_.set_flag(blocksstable::DF_INSERT); if (OB_FAIL(iter->second->get_index_table_builder(builder))) { LOG_WARN("get builder failed", KR(ret)); } else if (OB_FAIL(builder->append_row(index_tablet_id, 0, index_row))) { @@ -122,6 +161,47 @@ int ObTableLoadDataRowHandler::handle_insert_row_with_multi_version(const ObTabl return ret; } +int ObTableLoadDataRowHandler::handle_insert_batch_with_multi_version(const ObTabletID &tablet_id, const ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadDataRowHandler not init", KR(ret), KP(this)); + } else if (0 == datum_rows.row_count_) { + // do nothing + } else { + FOREACH_X(iter, *index_store_table_ctx_map_, OB_SUCC(ret)) + { + ObTableLoadStoreTableCtx *store_table_ctx = iter->second; + ObIDirectLoadPartitionTableBuilder *builder = nullptr; + ObTabletID index_tablet_id; + ObDatumRow index_datum_row; + index_datum_row.row_flag_.set_flag(DF_INSERT); + if (OB_FAIL(store_table_ctx->get_index_table_builder(builder))) { + LOG_WARN("fail to get index table builder", KR(ret)); + } else if (OB_FAIL(store_table_ctx->project_->get_index_tablet_id(tablet_id, index_tablet_id))) { + LOG_WARN("fail to get index tablet id", KR(ret)); + } else if (OB_FAIL(store_table_ctx->project_->init_datum_row(index_datum_row))) { + LOG_WARN("fail to projector", KR(ret), K(tablet_id)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < datum_rows.row_count_; ++i) { + if (OB_FAIL(store_table_ctx->project_->projector(datum_rows, + i, + true/*has_multi_version_cols*/, + index_datum_row))) { + LOG_WARN("fail to projector", KR(ret), K(tablet_id)); + } else if (OB_FAIL(builder->append_row(index_tablet_id, 0, index_datum_row))) { + LOG_WARN("add row failed", KR(ret), K(index_datum_row)); + } + } + } + if (OB_SUCC(ret)) { + ATOMIC_AAF(&result_info_->rows_affected_, datum_rows.row_count_); + } + } + return ret; +} + int ObTableLoadDataRowHandler::handle_update_row(const blocksstable::ObDatumRow &row) { UNUSED(row); @@ -207,8 +287,7 @@ int ObTableLoadDataRowHandler::handle_update_row(const ObTabletID tablet_id, } } if (OB_SUCC(ret)) { - index_row_new.row_flag_.set_flag(blocksstable::DF_INSERT, - blocksstable::ObDmlRowFlagType::DF_TYPE_INSERT_DELETE); + index_row_new.row_flag_.set_flag(ObDmlFlag::DF_INSERT); if (OB_FAIL(builder->append_row(index_tablet_id, 0, index_row_new))) { LOG_WARN("add row failed", KR(ret), K(index_row_new)); } diff --git a/src/observer/table_load/ob_table_load_data_row_handler.h b/src/observer/table_load/ob_table_load_data_row_handler.h index b719452872..0077659a23 100644 --- a/src/observer/table_load/ob_table_load_data_row_handler.h +++ b/src/observer/table_load/ob_table_load_data_row_handler.h @@ -12,6 +12,7 @@ #pragma once +#include "storage/blocksstable/ob_batch_datum_rows.h" #include "storage/blocksstable/ob_datum_row.h" #include "sql/engine/cmd/ob_load_data_utils.h" #include "storage/direct_load/ob_direct_load_dml_row_handler.h" @@ -49,11 +50,13 @@ public: ObTableLoadErrorRowHandler *error_row_handler, hash::ObHashMap *index_store_table_ctx_map); int handle_insert_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) override; + int handle_insert_batch(const ObTabletID &tablet_id, const blocksstable::ObBatchDatumRows &datum_rows) override; int handle_delete_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) { return OB_ERR_UNEXPECTED; } int handle_insert_row_with_multi_version(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) override; + int handle_insert_batch_with_multi_version(const ObTabletID &tablet_id, const blocksstable::ObBatchDatumRows &datum_rows) override; int handle_update_row(const blocksstable::ObDatumRow &row); int handle_update_row(common::ObArray &rows, const ObDirectLoadExternalRow *&row); diff --git a/src/observer/table_load/ob_table_load_empty_insert_tablet_ctx_manager.cpp b/src/observer/table_load/ob_table_load_empty_insert_tablet_ctx_manager.cpp index 0d65d7a3e8..ea0f617b3b 100644 --- a/src/observer/table_load/ob_table_load_empty_insert_tablet_ctx_manager.cpp +++ b/src/observer/table_load/ob_table_load_empty_insert_tablet_ctx_manager.cpp @@ -148,7 +148,7 @@ int ObTableLoadEmptyInsertTabletCtxManager::execute( insert_table_param.reserved_parallel_ = 0; insert_table_param.rowkey_column_count_ = table_load_schema.rowkey_column_count_; insert_table_param.column_count_ = table_load_schema.store_column_count_; - insert_table_param.lob_column_count_ = table_load_schema.lob_column_idxs_.count(); + insert_table_param.lob_inrow_threshold_ = table_load_schema.lob_inrow_threshold_; insert_table_param.is_partitioned_table_ = table_load_schema.is_partitioned_table_; insert_table_param.is_heap_table_ = table_load_schema.is_heap_table_; insert_table_param.is_column_store_ = table_load_schema.is_column_store_; @@ -157,7 +157,10 @@ int ObTableLoadEmptyInsertTabletCtxManager::execute( insert_table_param.datum_utils_ = &(table_load_schema.datum_utils_); insert_table_param.col_descs_ = &(table_load_schema.column_descs_); insert_table_param.cmp_funcs_ = &(table_load_schema.cmp_funcs_); + insert_table_param.lob_column_idxs_ = &(table_load_schema.lob_column_idxs_); insert_table_param.online_sample_percent_ = 1.0; + insert_table_param.is_no_logging_ = ddl_param.is_no_logging_; + insert_table_param.max_batch_size_ = 256; if (OB_FAIL(ret)) { // do nothing } else if (OB_FAIL(tmp_insert_table_ctx.init(insert_table_param, diff --git a/src/observer/table_load/ob_table_load_index_row_handler.h b/src/observer/table_load/ob_table_load_index_row_handler.h index cb2962e8fe..a51b847f80 100644 --- a/src/observer/table_load/ob_table_load_index_row_handler.h +++ b/src/observer/table_load/ob_table_load_index_row_handler.h @@ -27,11 +27,19 @@ public: ObTableLoadIndexRowHandler(); virtual ~ObTableLoadIndexRowHandler(); int handle_insert_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row); + int handle_insert_batch(const ObTabletID &tablet_id, const blocksstable::ObBatchDatumRows &datum_rows) override + { + return OB_ERR_UNEXPECTED; + } int handle_delete_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row); int handle_insert_row_with_multi_version(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) { return OB_ERR_UNEXPECTED; } + int handle_insert_batch_with_multi_version(const ObTabletID &tablet_id, const blocksstable::ObBatchDatumRows &datum_rows) + { + return OB_ERR_UNEXPECTED; + } int handle_update_row(const blocksstable::ObDatumRow &row) override { return OB_ERR_UNEXPECTED; }; int handle_update_row(common::ObArray &rows, const ObDirectLoadExternalRow *&row) override diff --git a/src/observer/table_load/ob_table_load_index_table_projector.cpp b/src/observer/table_load/ob_table_load_index_table_projector.cpp index 303387024e..1d9fccdc6e 100644 --- a/src/observer/table_load/ob_table_load_index_table_projector.cpp +++ b/src/observer/table_load/ob_table_load_index_table_projector.cpp @@ -10,15 +10,20 @@ * See the Mulan PubL v2 for more details. */ #define USING_LOG_PREFIX STORAGE + #include "observer/table_load/ob_table_load_index_table_projector.h" #include "lib/oblog/ob_log_module.h" #include "share/rc/ob_tenant_base.h" #include "storage/ob_i_store.h" +#include "storage/direct_load/ob_direct_load_vector_utils.h" namespace oceanbase { namespace observer { +using namespace blocksstable; +using namespace storage; + ObTableLoadIndexTableProjector::~ObTableLoadIndexTableProjector() { row_projector_.reset(); @@ -61,6 +66,18 @@ int ObTableLoadIndexTableProjector::get_index_tablet_id_and_part_id_by_data_tabl return ret; } +int ObTableLoadIndexTableProjector::get_index_tablet_id(const ObTabletID &data_tablet_id, ObTabletID &index_tablet_id) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadIndexTableProjector not init", KR(ret)); + } else if (OB_FAIL(tablet_projector_.get_refactored(data_tablet_id, index_tablet_id))) { + LOG_WARN("fail to get index tablet id", KR(ret), K(data_tablet_id)); + } + return ret; +} + int ObTableLoadIndexTableProjector::build_projector( const share::schema::ObTableSchema *data_table_schema, const share::schema::ObTableSchema *index_table_schema) @@ -192,5 +209,50 @@ int ObTableLoadIndexTableProjector::projector(const ObTabletID &data_tablet_id, return ret; } +int ObTableLoadIndexTableProjector::init_datum_row(ObDatumRow &datum_row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadIndexTableProjector not init", KR(ret)); + } else if (OB_FAIL(datum_row.init(column_num_))) { + LOG_WARN("fail to init index datum row", KR(ret)); + } + return ret; +} + +int ObTableLoadIndexTableProjector::projector(const ObBatchDatumRows &data_datum_rows, + const int64_t row_idx, + const bool has_multi_version_cols, + ObDatumRow &index_datum_row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadIndexTableProjector not init", KR(ret)); + } else if (OB_UNLIKELY(index_datum_row.count_ != column_num_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(index_datum_row.count_), K(column_num_)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < row_projector_.size(); i++) { + int64_t col_idx = row_projector_.at(i); + if (has_multi_version_cols && col_idx >= main_table_rowkey_column_num_) { + col_idx += ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); + } + if (OB_UNLIKELY(col_idx >= data_datum_rows.get_column_count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected col idx", KR(ret), K(i), K(row_projector_), K(col_idx), + K(data_datum_rows.get_column_count())); + } else { + ObIVector *vector = data_datum_rows.vectors_.at(col_idx); + if (OB_FAIL(ObDirectLoadVectorUtils::to_datum(vector, row_idx, index_datum_row.storage_datums_[i]))) { + LOG_WARN("fail to datum", KR(ret)); + } + } + } + } + return ret; +} + } // namespace observer } // namespace oceanbase \ No newline at end of file diff --git a/src/observer/table_load/ob_table_load_index_table_projector.h b/src/observer/table_load/ob_table_load_index_table_projector.h index 3d26c2e2b2..6c68651a63 100644 --- a/src/observer/table_load/ob_table_load_index_table_projector.h +++ b/src/observer/table_load/ob_table_load_index_table_projector.h @@ -16,6 +16,7 @@ #include "lib/hash/ob_hashmap.h" #include "ob_tablet_id.h" #include "share/schema/ob_table_schema.h" +#include "storage/blocksstable/ob_batch_datum_rows.h" #include "storage/blocksstable/ob_datum_row.h" namespace oceanbase @@ -31,10 +32,16 @@ public: } ~ObTableLoadIndexTableProjector(); int init(const share::schema::ObTableSchema *data_table_schema, const share::schema::ObTableSchema *index_table_schema); + int init_datum_row(blocksstable::ObDatumRow &datum_row); int projector(const ObTabletID &data_tablet_id, const blocksstable::ObDatumRow &origin_datum_row, const bool &have_multiversion_col, ObTabletID &index_tablet_id, blocksstable::ObDatumRow &out_datum_row); + int projector(const blocksstable::ObBatchDatumRows &data_datum_rows, // contain multi version cols + const int64_t row_idx, + const bool has_multi_version_cols, + blocksstable::ObDatumRow &index_datum_row); int get_index_tablet_id_and_part_id_by_data_tablet_id(const ObTabletID &data_tablet_id, ObTabletID &index_tablet_id, ObObjectID &part_id); + int get_index_tablet_id(const ObTabletID &data_tablet_id, ObTabletID &index_tablet_id); private: int build_projector(const share::schema::ObTableSchema *data_table_schema, const share::schema::ObTableSchema *index_table_schema); int build_tablet_projector(const share::schema::ObTableSchema *data_table_schema, const share::schema::ObTableSchema *index_table_schema); diff --git a/src/observer/table_load/ob_table_load_merger.cpp b/src/observer/table_load/ob_table_load_merger.cpp index a785533df7..0d694ddb93 100644 --- a/src/observer/table_load/ob_table_load_merger.cpp +++ b/src/observer/table_load/ob_table_load_merger.cpp @@ -22,7 +22,6 @@ #include "observer/table_load/ob_table_load_task.h" #include "observer/table_load/ob_table_load_task_scheduler.h" #include "observer/table_load/ob_table_load_trans_store.h" -#include "storage/direct_load/ob_direct_load_fast_heap_table.h" #include "storage/direct_load/ob_direct_load_multi_map.h" #include "storage/direct_load/ob_direct_load_range_splitter.h" #include "observer/table_load/ob_table_load_store_table_ctx.h" diff --git a/src/observer/table_load/ob_table_load_pre_sort_writer.cpp b/src/observer/table_load/ob_table_load_pre_sort_writer.cpp index 43bdc3473a..8d74e47657 100644 --- a/src/observer/table_load/ob_table_load_pre_sort_writer.cpp +++ b/src/observer/table_load/ob_table_load_pre_sort_writer.cpp @@ -16,10 +16,8 @@ #include "observer/table_load/ob_table_load_pre_sorter.h" #include "observer/table_load/ob_table_load_trans_store.h" #include "observer/table_load/ob_table_load_error_row_handler.h" - #include "share/table/ob_table_load_row.h" - #include "storage/blocksstable/ob_datum_row.h" - #include "storage/direct_load/ob_direct_load_mem_chunk.h" - #include "storage/direct_load/ob_direct_load_table_data_desc.h" +#include "storage/direct_load/ob_direct_load_table_data_desc.h" +#include "storage/direct_load/ob_direct_load_vector_utils.h" namespace oceanbase { @@ -110,16 +108,88 @@ int ObTableLoadPreSortWriter::write(int32_t session_id, return ret; } -int ObTableLoadPreSortWriter::px_write(const ObTabletID &tablet_id, - const blocksstable::ObDatumRow &row) +int ObTableLoadPreSortWriter::px_write(ObIVector *tablet_id_vector, + const ObIArray &vectors, + const ObBatchRows &batch_rows, + int64_t &affected_rows) { int ret = OB_SUCCESS; - ObTableLoadSequenceNo seq_no(0); + static const ObTableLoadSequenceNo seq_no(0); if (IS_NOT_INIT) { ret = OB_NOT_INIT; - LOG_WARN("ObTableLoadTransStoreWriter is not init", KR(ret)); - } else if (OB_FAIL(append_row(tablet_id, row, seq_no))) { - LOG_WARN("fail to append row", KR(ret)); + LOG_WARN("ObTableLoadPreSortWriter not init", KR(ret)); + } else if (OB_UNLIKELY(nullptr == tablet_id_vector || + vectors.count() != table_data_desc_->column_count_ || + (!batch_rows.all_rows_active_ && nullptr == batch_rows.skip_) || + batch_rows.size_ <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(tablet_id_vector), K(vectors.count()), K(batch_rows)); + } else if (!datum_row_.is_valid() && datum_row_.init(table_data_desc_->column_count_)) { + LOG_WARN("fail to init datum row", KR(ret)); + } else { + affected_rows = 0; + RowType const_row; + if (OB_UNLIKELY(-1 != chunk_node_id_ || nullptr != chunk_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected chunk not close", KR(ret), K(chunk_node_id_), KP(chunk_)); + } else if (OB_FAIL(chunks_manager_->get_chunk(chunk_node_id_, chunk_))) { + LOG_WARN("fail to get chunk", KR(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < batch_rows.size_; ++i) { + if (!batch_rows.all_rows_active_ && batch_rows.skip_->at(i)) { + continue; + } else { + ++affected_rows; + external_row_.tablet_id_ = ObDirectLoadVectorUtils::get_tablet_id(tablet_id_vector, i); + if (OB_FAIL(ObDirectLoadVectorUtils::to_datums(vectors, + i, + datum_row_.storage_datums_, + datum_row_.count_))) { + LOG_WARN("fail to transfer vectors to datums", KR(ret), K(i)); + } else if (OB_FAIL(external_row_.external_row_.from_datums(datum_row_.storage_datums_, + datum_row_.count_, + table_data_desc_->rowkey_column_num_, + seq_no, + false))) { + LOG_WARN("fail to cast row from datum", KR(ret)); + } else { + const_row = external_row_; + } + while (OB_SUCC(ret)) { + if (OB_FAIL(chunk_->add_item(const_row))) { + if (OB_UNLIKELY(OB_BUF_NOT_ENOUGH != ret)) { + LOG_WARN("fail to add item", KR(ret)); + } else { + ret = OB_SUCCESS; + if (OB_FAIL(chunks_manager_->close_chunk(chunk_node_id_))) { + LOG_WARN("fail to close chunk", KR(ret)); + } else { + chunk_node_id_ = -1; + chunk_ = nullptr; + if (OB_FAIL(chunks_manager_->get_chunk(chunk_node_id_, chunk_))) { + LOG_WARN("fail to get chunk", KR(ret)); + } + } + } + } else { + break; + } + } + } + } + if (OB_SUCC(ret)) { + ATOMIC_AAF(&pre_sorter_->ctx_->job_stat_->store_.processed_rows_, affected_rows); + } + if (OB_LIKELY(-1 != chunk_node_id_ && nullptr != chunk_)) { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(chunks_manager_->push_chunk(chunk_node_id_))) { + LOG_WARN("fail to push chunk", KR(tmp_ret), K(chunk_node_id_)); + ret = COVER_SUCC(tmp_ret); + } else { + chunk_ = nullptr; + chunk_node_id_ = -1; + } + } } return ret; } diff --git a/src/observer/table_load/ob_table_load_pre_sort_writer.h b/src/observer/table_load/ob_table_load_pre_sort_writer.h index df15993138..bd56f4658a 100644 --- a/src/observer/table_load/ob_table_load_pre_sort_writer.h +++ b/src/observer/table_load/ob_table_load_pre_sort_writer.h @@ -10,9 +10,11 @@ * See the Mulan PubL v2 for more details. */ - #ifndef _OB_TABLE_PRE_SORT_WRITER_ - #define _OB_TABLE_PRE_SORT_WRITER_ +#ifndef _OB_TABLE_PRE_SORT_WRITER_ +#define _OB_TABLE_PRE_SORT_WRITER_ +#include "share/table/ob_table_load_row_array.h" +#include "storage/blocksstable/ob_datum_row.h" #include "storage/direct_load/ob_direct_load_mem_define.h" #include "storage/direct_load/ob_direct_load_external_multi_partition_row.h" @@ -39,7 +41,10 @@ public: ObTableLoadErrorRowHandler *error_row_handler, ObDirectLoadTableDataDesc *table_data_desc); int write(int32_t session_id, const table::ObTableLoadTabletObjRowArray &row_array); - int px_write(const ObTabletID &tablet_id, const blocksstable::ObDatumRow &row); + int px_write(common::ObIVector *tablet_id_vector, + const ObIArray &vectors, + const sql::ObBatchRows &batch_rows, + int64_t &affected_rows); int close(); private: int append_row(const ObTabletID &tablet_id, @@ -53,6 +58,7 @@ private: ObTableLoadMemChunkManager *chunks_manager_; int64_t chunk_node_id_; ChunkType *chunk_; + blocksstable::ObDatumRow datum_row_; ObDirectLoadExternalMultiPartitionRow external_row_; bool is_inited_; }; diff --git a/src/observer/table_load/ob_table_load_pre_sorter.cpp b/src/observer/table_load/ob_table_load_pre_sorter.cpp index 7b5f771078..35e99563cd 100644 --- a/src/observer/table_load/ob_table_load_pre_sorter.cpp +++ b/src/observer/table_load/ob_table_load_pre_sorter.cpp @@ -402,7 +402,6 @@ int ObTableLoadPreSorter::build_merge_param(ObDirectLoadMergeParam& merge_param) merge_param.lob_column_idxs_ = &(ctx_->schema_.lob_column_idxs_); merge_param.table_data_desc_ = store_ctx_->data_store_table_ctx_->table_data_desc_; merge_param.datum_utils_ = &(ctx_->schema_.datum_utils_); - merge_param.lob_column_idxs_ = &(ctx_->schema_.lob_column_idxs_); merge_param.col_descs_ = &(ctx_->schema_.column_descs_); merge_param.lob_id_table_data_desc_ = store_ctx_->data_store_table_ctx_->lob_id_table_data_desc_; merge_param.lob_meta_datum_utils_ = &(ctx_->schema_.lob_meta_datum_utils_); diff --git a/src/observer/table_load/ob_table_load_schema.cpp b/src/observer/table_load/ob_table_load_schema.cpp index 27d6670f5d..c9b5130bb9 100644 --- a/src/observer/table_load/ob_table_load_schema.cpp +++ b/src/observer/table_load/ob_table_load_schema.cpp @@ -16,6 +16,7 @@ #include "observer/table_load/ob_table_load_utils.h" #include "share/rc/ob_tenant_base.h" #include "share/schema/ob_multi_version_schema_service.h" +#include "storage/direct_load/ob_direct_load_vector_utils.h" #include "storage/lob/ob_lob_meta.h" namespace oceanbase @@ -380,6 +381,24 @@ int ObTableLoadSchema::check_has_lob_column(const ObTableSchema *table_schema, b return ret; } +int ObTableLoadSchema::check_has_null_column(const ObTableSchema *table_schema, bool &bret) +{ + int ret = OB_SUCCESS; + bret = false; + for (ObTableSchema::const_column_iterator iter = table_schema->column_begin(); + OB_SUCC(ret) && iter != table_schema->column_end(); ++iter) { + ObColumnSchemaV2 *column_schema = *iter; + if (OB_ISNULL(column_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("invalid column schema", K(column_schema)); + } else if (column_schema->get_meta_type().is_null()) { + bret = true; + break; + } + } + return ret; +} + int ObTableLoadSchema::check_has_non_local_index(share::schema::ObSchemaGetterGuard &schema_guard, const share::schema::ObTableSchema *table_schema, bool &bret) @@ -584,7 +603,9 @@ ObTableLoadSchema::ObTableLoadSchema() part_level_(PARTITION_LEVEL_ZERO), schema_version_(0), lob_meta_table_id_(OB_INVALID_ID), + lob_inrow_threshold_(-1), index_table_count_(0), + non_partitioned_tablet_id_vector_(nullptr), is_inited_(false) { allocator_.set_tenant_id(MTL_ID()); @@ -613,6 +634,7 @@ void ObTableLoadSchema::reset() part_level_ = PARTITION_LEVEL_ZERO; schema_version_ = 0; lob_meta_table_id_ = OB_INVALID_ID; + lob_inrow_threshold_ = -1; index_table_count_ = 0; lob_column_idxs_.reset(); column_descs_.reset(); @@ -621,6 +643,7 @@ void ObTableLoadSchema::reset() lob_meta_column_descs_.reset(); lob_meta_datum_utils_.reset(); cmp_funcs_.reset(); + non_partitioned_tablet_id_vector_ = nullptr; allocator_.reset(); is_inited_ = false; } @@ -663,6 +686,7 @@ int ObTableLoadSchema::init_table_schema(const ObTableSchema *table_schema) if (table_schema->has_lob_aux_table()) { lob_meta_table_id_ = table_schema->get_aux_lob_meta_tid(); } + lob_inrow_threshold_ = table_schema->get_lob_inrow_threshold(); index_table_count_ = table_schema->get_simple_index_infos().count(); if (OB_FAIL(ObTableLoadUtils::deep_copy(table_schema->get_table_name_str(), table_name_, allocator_))) { @@ -675,6 +699,8 @@ int ObTableLoadSchema::init_table_schema(const ObTableSchema *table_schema) LOG_WARN("fail to prepare column descs", KR(ret)); } else if (OB_FAIL(table_schema->get_multi_version_column_descs(multi_version_column_descs_))) { LOG_WARN("fail to get multi version column descs", KR(ret)); + } else if (OB_FAIL(update_decimal_int_precision(table_schema, column_descs_))){ + LOG_WARN("update decimal int precision failed", K(ret)); } else if (OB_FAIL(update_decimal_int_precision(table_schema, multi_version_column_descs_))){ LOG_WARN("update decimal int precision failed", K(ret)); } else if (OB_FAIL(datum_utils_.init(multi_version_column_descs_, rowkey_column_count_, @@ -688,13 +714,6 @@ int ObTableLoadSchema::init_table_schema(const ObTableSchema *table_schema) LOG_WARN("fail to init cmp funcs", KR(ret)); } if (OB_SUCC(ret)) { - ObArray tablet_ids; - ObArray part_ids; - tablet_ids.set_tenant_id(MTL_ID()); - part_ids.set_tenant_id(MTL_ID()); - if (OB_FAIL(table_schema->get_all_tablet_and_object_ids(tablet_ids, part_ids))) { - LOG_WARN("fail to get all tablet ids", KR(ret)); - } for (ObTableSchema::const_column_iterator iter = table_schema->column_begin(); OB_SUCC(ret) && iter != table_schema->column_end(); ++iter) { ObColumnSchemaV2 *column_schema = *iter; @@ -710,6 +729,13 @@ int ObTableLoadSchema::init_table_schema(const ObTableSchema *table_schema) } }//end for } + if (OB_SUCC(ret) && !is_partitioned_table_) { + const ObTabletID &tablet_id = table_schema->get_tablet_id(); + if (OB_FAIL(ObDirectLoadVectorUtils::make_const_tablet_id_vector( + tablet_id, allocator_, non_partitioned_tablet_id_vector_))) { + LOG_WARN("fail to make const tablet id vector", KR(ret), K(tablet_id)); + } + } } return ret; } diff --git a/src/observer/table_load/ob_table_load_schema.h b/src/observer/table_load/ob_table_load_schema.h index cfaa27ff3c..b16451d883 100644 --- a/src/observer/table_load/ob_table_load_schema.h +++ b/src/observer/table_load/ob_table_load_schema.h @@ -75,6 +75,7 @@ public: static int check_has_unused_column(const share::schema::ObTableSchema *table_schema, bool &bret); static int check_has_roaringbitmap_column(const share::schema::ObTableSchema *table_schema, bool &bret); static int check_has_lob_column(const share::schema::ObTableSchema *table_schema, bool &bret); + static int check_has_null_column(const share::schema::ObTableSchema *table_schema, bool &bret); static int check_has_non_local_index(share::schema::ObSchemaGetterGuard &schema_guard, const share::schema::ObTableSchema *table_schema, bool &bret); @@ -121,6 +122,7 @@ public: share::schema::ObPartitionLevel part_level_; int64_t schema_version_; uint64_t lob_meta_table_id_; + int64_t lob_inrow_threshold_; int64_t index_table_count_; common::ObArray lob_column_idxs_; // if it is a heap table, it contains hidden primary key column @@ -131,6 +133,7 @@ public: common::ObArray lob_meta_column_descs_; blocksstable::ObStorageDatumUtils lob_meta_datum_utils_; blocksstable::ObStoreCmpFuncs cmp_funcs_; // for sql statistics + common::ObIVector *non_partitioned_tablet_id_vector_; bool is_inited_; }; diff --git a/src/observer/table_load/ob_table_load_service.cpp b/src/observer/table_load/ob_table_load_service.cpp index 3846a0a835..4f759b79f3 100644 --- a/src/observer/table_load/ob_table_load_service.cpp +++ b/src/observer/table_load/ob_table_load_service.cpp @@ -475,6 +475,7 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu bool has_invisible_column = false; bool has_unused_column = false; bool has_roaringbitmap_column = false; + bool has_null_column = false; bool has_non_normal_local_index = false; // check if it is a user table const char *tmp_prefix = ObDirectLoadMode::is_insert_overwrite(load_mode) ? InsertOverwritePrefix : EmptyPrefix; @@ -551,6 +552,14 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu LOG_WARN("direct-load does not support table has roaringbitmap column", KR(ret)); FORWARD_USER_ERROR_MSG(ret, "%sdirect-load does not support table has roaringbitmap column", tmp_prefix); } + // check has null column + else if (OB_FAIL(ObTableLoadSchema::check_has_null_column(table_schema, has_null_column))) { + LOG_WARN("fail to check has null column", KR(ret)); + } else if (has_null_column) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("direct-load does not support table has null column", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "%sdirect-load does not support table has null column", tmp_prefix); + } // check if table has mlog else if (table_schema->required_by_mview_refresh() && !table_schema->mv_container_table()) { ret = OB_NOT_SUPPORTED; diff --git a/src/observer/table_load/ob_table_load_stat.h b/src/observer/table_load/ob_table_load_stat.h index 39110e2885..99e435e6e4 100644 --- a/src/observer/table_load/ob_table_load_stat.h +++ b/src/observer/table_load/ob_table_load_stat.h @@ -14,6 +14,7 @@ #include "lib/coro/co_var.h" #include "lib/alloc/alloc_assist.h" +#include "lib/time/ob_time_utility.h" #include "lib/utility/ob_print_utils.h" namespace oceanbase diff --git a/src/observer/table_load/ob_table_load_store_table_ctx.cpp b/src/observer/table_load/ob_table_load_store_table_ctx.cpp index c01b47895d..7aaabd9847 100644 --- a/src/observer/table_load/ob_table_load_store_table_ctx.cpp +++ b/src/observer/table_load/ob_table_load_store_table_ctx.cpp @@ -21,11 +21,14 @@ #include "storage/direct_load/ob_direct_load_mem_context.h" #include "storage/direct_load/ob_direct_load_i_table.h" #include "storage/direct_load/ob_direct_load_dml_row_handler.h" +#include "storage/direct_load/ob_direct_load_sstable_scan_merge.h" namespace oceanbase { namespace observer { +using namespace table; + ObTableLoadStoreTableCtx::ObTableLoadStoreTableCtx() : allocator_("TLD_STCtx"), table_id_(OB_INVALID_ID), @@ -281,7 +284,7 @@ int ObTableLoadStoreTableCtx::init_insert_table_ctx() insert_table_param.reserved_parallel_ = is_fast_heap_table_ ? store_ctx_->ctx_->param_.session_count_ : 0; insert_table_param.rowkey_column_count_ = schema_->rowkey_column_count_; insert_table_param.column_count_ = schema_->store_column_count_; - insert_table_param.lob_column_count_ = schema_->lob_column_idxs_.count(); + insert_table_param.lob_inrow_threshold_ = schema_->lob_inrow_threshold_; insert_table_param.is_partitioned_table_ = schema_->is_partitioned_table_; insert_table_param.is_heap_table_ = schema_->is_heap_table_; insert_table_param.is_column_store_ = schema_->is_column_store_; @@ -291,8 +294,10 @@ int ObTableLoadStoreTableCtx::init_insert_table_ctx() insert_table_param.datum_utils_ = &(schema_->datum_utils_); insert_table_param.col_descs_ = &(schema_->column_descs_); insert_table_param.cmp_funcs_ = &(schema_->cmp_funcs_); + insert_table_param.lob_column_idxs_ = &(schema_->lob_column_idxs_); insert_table_param.online_sample_percent_ = store_ctx_->ctx_->param_.online_sample_percent_; insert_table_param.is_no_logging_ = store_ctx_->ctx_->ddl_param_.is_no_logging_; + insert_table_param.max_batch_size_ = store_ctx_->ctx_->param_.batch_size_; if (OB_ISNULL(insert_table_ctx_ = OB_NEWx(ObDirectLoadInsertTableContext, (&allocator_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; diff --git a/src/observer/table_load/ob_table_load_store_trans_px_writer.cpp b/src/observer/table_load/ob_table_load_store_trans_px_writer.cpp index 4ade7a5df8..578e941140 100644 --- a/src/observer/table_load/ob_table_load_store_trans_px_writer.cpp +++ b/src/observer/table_load/ob_table_load_store_trans_px_writer.cpp @@ -13,11 +13,13 @@ #define USING_LOG_PREFIX SERVER #include "observer/table_load/ob_table_load_store_trans_px_writer.h" +#include "observer/table_load/ob_table_load_pre_sort_writer.h" #include "observer/table_load/ob_table_load_store_ctx.h" +#include "observer/table_load/ob_table_load_store_table_ctx.h" #include "observer/table_load/ob_table_load_store_trans.h" #include "observer/table_load/ob_table_load_table_ctx.h" #include "observer/table_load/ob_table_load_trans_store.h" -#include "observer/table_load/ob_table_load_store_table_ctx.h" +#include "storage/direct_load/ob_direct_load_vector_utils.h" namespace oceanbase { @@ -25,18 +27,29 @@ namespace observer { using namespace common; using namespace share::schema; +using namespace sql; using namespace table; ObTableLoadStoreTransPXWriter::ObTableLoadStoreTransPXWriter() - : store_ctx_(nullptr), + : allocator_("TLD_PXWriter"), + store_ctx_(nullptr), trans_(nullptr), writer_(nullptr), - column_count_(0), - row_count_(0), is_heap_table_(false), + column_count_(0), + store_column_count_(0), + non_partitioned_tablet_id_vector_(nullptr), + tablet_id_set_(), + pre_sort_writer_(nullptr), + batch_ctx_(nullptr), + row_count_(0), + last_check_status_cycle_(0), + is_vectorized_(false), + use_rich_format_(false), can_write_(false), is_inited_(false) { + allocator_.set_tenant_id(MTL_ID()); } ObTableLoadStoreTransPXWriter::~ObTableLoadStoreTransPXWriter() @@ -48,9 +61,6 @@ void ObTableLoadStoreTransPXWriter::reset() { is_inited_ = false; can_write_ = false; - is_heap_table_ = false; - column_count_ = 0; - row_count_ = 0; if (nullptr != store_ctx_) { if (nullptr != trans_) { if (nullptr != writer_) { @@ -63,6 +73,28 @@ void ObTableLoadStoreTransPXWriter::reset() ATOMIC_AAF(&store_ctx_->px_writer_count_, -1); store_ctx_ = nullptr; } + is_heap_table_ = false; + column_count_ = 0; + store_column_count_ = 0; + non_partitioned_tablet_id_vector_ = nullptr; + if (tablet_id_set_.created()) { + tablet_id_set_.destroy(); + } + if (nullptr != pre_sort_writer_) { + pre_sort_writer_->~ObTableLoadPreSortWriter(); + allocator_.free(pre_sort_writer_); + pre_sort_writer_ = nullptr; + } + if (nullptr != batch_ctx_) { + batch_ctx_->~BatchCtx(); + allocator_.free(batch_ctx_); + batch_ctx_ = nullptr; + } + row_count_ = 0; + last_check_status_cycle_ = 0; + is_vectorized_ = false; + use_rich_format_ = false; + allocator_.reset(); } int ObTableLoadStoreTransPXWriter::init(ObTableLoadStoreCtx *store_ctx, ObTableLoadStoreTrans *trans, @@ -82,117 +114,331 @@ int ObTableLoadStoreTransPXWriter::init(ObTableLoadStoreCtx *store_ctx, trans_->inc_ref_count(); writer_->inc_ref_count(); ATOMIC_AAF(&store_ctx_->px_writer_count_, 1); - if (store_ctx_->enable_pre_sort_ - && OB_FAIL(pre_sort_writer_.init(store_ctx_->pre_sorter_, writer_, - store_ctx_->error_row_handler_, - &(store_ctx_->data_store_table_ctx_->table_data_desc_)))) { - LOG_WARN("fail to init pre sort wirter", KR(ret)); + + is_heap_table_ = store_ctx_->ctx_->schema_.is_heap_table_; + column_count_ = store_ctx_->ctx_->schema_.store_column_count_; + store_column_count_ = (is_heap_table_ ? column_count_ - 1 : column_count_); + if (!store_ctx_->ctx_->schema_.is_partitioned_table_) { + non_partitioned_tablet_id_vector_ = + store_ctx_->ctx_->schema_.non_partitioned_tablet_id_vector_; } - if (OB_SUCC(check_status())) { + if (OB_FAIL(init_tablet_id_set())) { + LOG_WARN("fail to init tablet id set", KR(ret)); + } else if (store_ctx_->enable_pre_sort_) { + if (OB_ISNULL(pre_sort_writer_ = OB_NEWx(ObTableLoadPreSortWriter, &allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObTableLoadPreSortWriter", KR(ret)); + } else if (OB_FAIL(pre_sort_writer_->init(store_ctx_->pre_sorter_, + writer_, + store_ctx_->error_row_handler_, + &(store_ctx_->data_store_table_ctx_->table_data_desc_)))) { + LOG_WARN("fail to init pre sort wirter", KR(ret)); + } + } + if (OB_SUCC(ret)) { is_inited_ = true; } } return ret; } -int ObTableLoadStoreTransPXWriter::prepare_write(const ObTabletID &tablet_id, - const ObIArray &column_ids) +int ObTableLoadStoreTransPXWriter::prepare_write(const ObIArray &column_ids, + const bool is_vectorized, + const bool use_rich_format) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObTableLoadStoreTransPXWriter not init", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!tablet_id.is_valid() || column_ids.empty())) { + } else if (OB_UNLIKELY(column_ids.count() != column_count_)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(tablet_id), K(column_ids)); + LOG_WARN("invalid args", KR(ret), K(column_ids), K(column_ids.count()), K(column_count_)); } else if (OB_UNLIKELY(can_write_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("already can write", KR(ret), KPC(this)); } else { - if (OB_FAIL(check_tablet(tablet_id))) { - LOG_WARN("fail to check tablet", KR(ret)); - } else if (OB_FAIL(check_columns(column_ids))) { + if (OB_FAIL(check_columns(column_ids))) { LOG_WARN("fail to check columns", KR(ret)); + } else if (OB_FAIL(init_batch_ctx(is_vectorized, use_rich_format))) { + LOG_WARN("fail to init batch ctx", KR(ret), K(is_vectorized), K(use_rich_format)); } else { - tablet_id_ = tablet_id; - column_count_ = column_ids.count(); - is_heap_table_ = store_ctx_->ctx_->schema_.is_heap_table_; + is_vectorized_ = is_vectorized; + use_rich_format_ = use_rich_format; can_write_ = true; } } return ret; } -int ObTableLoadStoreTransPXWriter::check_tablet(const ObTabletID &tablet_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!tablet_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(tablet_id)); - } else if (OB_ISNULL(store_ctx_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected store ctx is null", KR(ret), KPC(this)); - } else { - bool tablet_found = false; - for (int64_t i = 0; i < store_ctx_->data_store_table_ctx_->ls_partition_ids_.count(); ++i) { - const ObTableLoadLSIdAndPartitionId &ls_part_id = store_ctx_->data_store_table_ctx_->ls_partition_ids_.at(i); - if (ls_part_id.part_tablet_id_.tablet_id_ == tablet_id) { - tablet_found = true; - break; - } - } - if (OB_UNLIKELY(!tablet_found)) { - ret = OB_TABLET_NOT_EXIST; - LOG_WARN("tablet id not found", KR(ret), K(tablet_id), K(store_ctx_->data_store_table_ctx_->ls_partition_ids_)); - } - } - return ret; -} - int ObTableLoadStoreTransPXWriter::check_columns(const ObIArray &column_ids) { int ret = OB_SUCCESS; - ObTableLoadTableCtx *table_ctx = nullptr; if (OB_UNLIKELY(column_ids.empty())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(column_ids)); - } else if (OB_ISNULL(store_ctx_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected store ctx is null", KR(ret), KPC(this)); - } else if (OB_ISNULL(table_ctx = store_ctx_->ctx_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected table ctx is null", KR(ret), KPC(store_ctx_)); } else { - if (OB_UNLIKELY(table_ctx->schema_.column_descs_.count() != column_ids.count())) { + const ObIArray &col_descs = store_ctx_->ctx_->schema_.column_descs_; + if (OB_UNLIKELY(col_descs.count() != column_ids.count())) { ret = OB_SCHEMA_NOT_UPTODATE; - LOG_WARN("column count not match", KR(ret), K(table_ctx->schema_.column_descs_), - K(column_ids)); + LOG_WARN("column count not match", KR(ret), K(col_descs), K(column_ids)); } for (int64_t i = 0; OB_SUCC(ret) && i < column_ids.count(); ++i) { - const ObColDesc &col_desc = table_ctx->schema_.column_descs_.at(i); + const ObColDesc &col_desc = col_descs.at(i); const uint64_t column_id = column_ids.at(i); if (OB_UNLIKELY(col_desc.col_id_ != column_id)) { ret = OB_SCHEMA_NOT_UPTODATE; - LOG_WARN("column id not match", KR(ret), K(i), K(col_desc), K(column_id), - K(table_ctx->schema_.column_descs_), K(column_ids)); + LOG_WARN("column id not match", KR(ret), K(i), K(col_desc), K(column_id), K(col_descs), + K(column_ids)); } } } return ret; } +int ObTableLoadStoreTransPXWriter::init_batch_ctx(const bool is_vectorized, + const bool use_rich_format) +{ + int ret = OB_SUCCESS; + const ObIArray &col_descs = store_ctx_->ctx_->schema_.column_descs_; + const int64_t max_batch_size = store_ctx_->ctx_->param_.batch_size_; + if (OB_UNLIKELY(nullptr != batch_ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected batch ctx not null", KR(ret)); + } else if (OB_ISNULL(batch_ctx_ = OB_NEWx(BatchCtx, &allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new BatchCtx", KR(ret)); + } else { + batch_ctx_->max_batch_size_ = max_batch_size; + batch_ctx_->batch_vectors_.set_block_allocator(ModulePageAllocator(allocator_)); + batch_ctx_->const_vectors_.set_block_allocator(ModulePageAllocator(allocator_)); + batch_ctx_->append_vectors_.set_block_allocator(ModulePageAllocator(allocator_)); + batch_ctx_->heap_vectors_.set_block_allocator(ModulePageAllocator(allocator_)); + // non-vectorized + if (!is_vectorized) { + batch_ctx_->row_flag_.uncontain_hidden_pk_ = is_heap_table_; + if (OB_FAIL(ObDirectLoadVectorUtils::new_vector(VEC_FIXED, + ObDirectLoadVectorUtils::tablet_id_value_tc, + allocator_, + batch_ctx_->tablet_id_batch_vector_))) { + LOG_WARN("fail to new tablet id fixed vector", KR(ret)); + } else if (OB_FAIL(ObDirectLoadVectorUtils::prepare_vector(batch_ctx_->tablet_id_batch_vector_, + max_batch_size, + allocator_))) { + LOG_WARN("fail to prepare tablet id fixed vector", KR(ret), K(max_batch_size)); + } else if (OB_FAIL(batch_ctx_->batch_buffer_.init(col_descs, max_batch_size))) { + LOG_WARN("fail to init batch buffer", KR(ret), K(col_descs), K(max_batch_size)); + } + } + // vectorized + else { + if (!use_rich_format) { // vectorized 1.0 + // 向量化1.0需要把ObDatumVector转成ObVector + if (OB_FAIL(ObDirectLoadVectorUtils::new_vector(VEC_UNIFORM, + ObDirectLoadVectorUtils::tablet_id_value_tc, + allocator_, + batch_ctx_->tablet_id_batch_vector_))) { + LOG_WARN("fail to new tablet id uniform vector", KR(ret)); + } else if (OB_FAIL(ObDirectLoadVectorUtils::new_vector(VEC_UNIFORM_CONST, + ObDirectLoadVectorUtils::tablet_id_value_tc, + allocator_, + batch_ctx_->tablet_id_const_vector_))) { + LOG_WARN("fail to new tablet id uniform const vector", KR(ret)); + } + } else { // vectorized 2.0 + // 向量化2.0拿到的就是ObVector, do nothing + } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(batch_ctx_->col_fixed_ = to_bit_vector( + allocator_.alloc(ObBitVector::memory_size(column_count_))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObBitVector", KR(ret), K(column_count_)); + } else if (OB_FAIL(batch_ctx_->batch_vectors_.prepare_allocate(column_count_))) { + LOG_WARN("fail to prepare allocate", KR(ret), K(column_count_)); + } else if (OB_FAIL(batch_ctx_->const_vectors_.prepare_allocate(column_count_))) { + LOG_WARN("fail to prepare allocate", KR(ret), K(column_count_)); + } else if (OB_FAIL(batch_ctx_->append_vectors_.prepare_allocate(column_count_))) { + LOG_WARN("fail to prepare allocate", KR(ret), K(column_count_)); + } else { + batch_ctx_->col_fixed_->reset(column_count_); + } + for (int64_t i = 0; OB_SUCC(ret) && i < column_count_; ++i) { + const ObColDesc &col_desc = col_descs.at(i); + VecValueTypeClass value_tc = get_vec_value_tc(col_desc.col_type_.get_type(), + col_desc.col_type_.get_scale(), + col_desc.col_type_.get_stored_precision()); + const bool is_fixed = is_fixed_length_vec(value_tc); + ObIVector *batch_vector = nullptr; + ObIVector *const_vector = nullptr; + if (is_fixed) { + batch_ctx_->col_fixed_->set(i); + } + if (!use_rich_format) { // vectorized 1.0 + // 向量化1.0传入的是ObDatumVector, 需要转成ObVector, 都需要构造vector + // * fixed col不用浅拷贝, 不用分配vector成员内存 + // * unfixed col要浅拷贝, 需要分配vector成员内存 + if (OB_FAIL(ObDirectLoadVectorUtils::new_vector(VEC_UNIFORM, + value_tc, + allocator_, + batch_vector))) { + LOG_WARN("fail to new uniform vector", KR(ret), K(i), K(col_desc), K(value_tc)); + } else if (OB_FAIL(ObDirectLoadVectorUtils::new_vector(VEC_UNIFORM_CONST, + value_tc, + allocator_, + const_vector))) { + LOG_WARN("fail to new uniform const vector", KR(ret), K(i), K(col_desc), K(value_tc)); + } else if (is_fixed) { + } else if (OB_FAIL(ObDirectLoadVectorUtils::prepare_vector(batch_vector, + max_batch_size, + allocator_))) { + LOG_WARN("fail to prepare uniform vector", KR(ret), K(i), K(col_desc), K(max_batch_size)); + } else if (OB_FAIL(ObDirectLoadVectorUtils::prepare_vector(const_vector, + max_batch_size, + allocator_))) { + LOG_WARN("fail to prepare uniform const vector", KR(ret), K(i), K(col_desc), K(max_batch_size)); + } + } else { // vectorized 2.0 + // 向量化2.0传入的是ObVector + // * fixed col直接用传入的ObVector, 不用构造vector + // * unfixed col需要浅拷贝, 需要构造vector并分配vector成员内存 + if (is_fixed) { + } else if (OB_FAIL(ObDirectLoadVectorUtils::new_vector(VEC_DISCRETE, + value_tc, + allocator_, + batch_vector))) { + LOG_WARN("fail to new discrete vector", KR(ret), K(i), K(col_desc), K(value_tc)); + } else if (OB_FAIL(ObDirectLoadVectorUtils::prepare_vector(batch_vector, + max_batch_size, + allocator_))) { + LOG_WARN("fail to prepare discrete vector", KR(ret), K(i), K(col_desc), K(max_batch_size)); + } else if (OB_FAIL(ObDirectLoadVectorUtils::new_vector(VEC_UNIFORM_CONST, + value_tc, + allocator_, + const_vector))) { + LOG_WARN("fail to new uniform const vector", KR(ret), K(i), K(col_desc), K(value_tc)); + } else if (OB_FAIL(ObDirectLoadVectorUtils::prepare_vector(const_vector, + max_batch_size, + allocator_))) { + LOG_WARN("fail to prepare uniform const vector", KR(ret), K(i), K(col_desc), K(max_batch_size)); + } + } + if (OB_SUCC(ret)) { + batch_ctx_->batch_vectors_.at(i) = batch_vector; + batch_ctx_->const_vectors_.at(i) = const_vector; + } + } + } + if (OB_SUCC(ret)) { + if (is_heap_table_ && + OB_FAIL(batch_ctx_->heap_vectors_.prepare_allocate(store_column_count_))) { + LOG_WARN("fail to prepare allocate", KR(ret), K(store_column_count_)); + } + } + } + return ret; +} + +int ObTableLoadStoreTransPXWriter::init_tablet_id_set() +{ + int ret = OB_SUCCESS; + const int64_t bucket_num = 1024; + ObMemAttr attr(MTL_ID(), "TLD_TabletIdSet"); + if (OB_FAIL(tablet_id_set_.create(bucket_num, attr))) { + LOG_WARN("fail to create hashset", KR(ret)); + } else { + const ObIArray &ls_partition_ids = + store_ctx_->data_store_table_ctx_->ls_partition_ids_; + for (int64_t i = 0; OB_SUCC(ret) && i < ls_partition_ids.count(); ++i) { + const ObTabletID &tablet_id = ls_partition_ids.at(i).part_tablet_id_.tablet_id_; + if (OB_FAIL(tablet_id_set_.set_refactored(tablet_id))) { + LOG_WARN("fail to set hashset", KR(ret), K(i), K(tablet_id), K(ls_partition_ids)); + if (OB_LIKELY(OB_HASH_EXIST == ret)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected duplicate tablet id", KR(ret), K(i), K(tablet_id), + K(ls_partition_ids)); + } + } + } + } + return ret; +} + +int ObTableLoadStoreTransPXWriter::check_tablet(const ObTabletID &tablet_id) +{ + int ret = OB_SUCCESS; + ret = tablet_id_set_.exist_refactored(tablet_id); + if (OB_LIKELY(OB_HASH_EXIST == ret)) { + ret = OB_SUCCESS; + } else if (OB_LIKELY(OB_HASH_NOT_EXIST == ret)) { + ret = OB_TABLET_NOT_EXIST; + LOG_WARN("tablet id not exist", KR(ret), K(tablet_id), + K(store_ctx_->data_store_table_ctx_->ls_partition_ids_)); + } else { + LOG_WARN("fail to check exist tablet id", KR(ret)); + } + return ret; +} + +int ObTableLoadStoreTransPXWriter::check_tablet(ObIVector *vector, const ObBatchRows &batch_rows) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < batch_rows.size_; ++i) { + if (!batch_rows.all_rows_active_ && batch_rows.skip_->at(i)) { + continue; + } else { + const ObTabletID tablet_id = ObDirectLoadVectorUtils::get_tablet_id(vector, i); + if (OB_FAIL(check_tablet(tablet_id))) { + LOG_WARN("fail to check tablet", KR(ret), K(i), K(tablet_id)); + } + } + } + return ret; +} + +int ObTableLoadStoreTransPXWriter::check_tablet(const ObDatumVector &datum_vector, + const ObBatchRows &batch_rows) +{ + int ret = OB_SUCCESS; + if (datum_vector.is_batch()) { + for (int64_t i = 0; OB_SUCC(ret) && i < batch_rows.size_; ++i) { + if (!batch_rows.all_rows_active_ && batch_rows.skip_->at(i)) { + continue; + } else { + const ObDatum &datum = datum_vector.datums_[i]; + const ObTabletID tablet_id(datum.get_int()); + if (OB_FAIL(check_tablet(tablet_id))) { + LOG_WARN("fail to check tablet", KR(ret), K(i), K(datum), K(tablet_id)); + } + } + } + } else { + const ObDatum &datum = datum_vector.datums_[0]; + const ObTabletID tablet_id(datum.get_int()); + if (OB_FAIL(check_tablet(tablet_id))) { + LOG_WARN("fail to check tablet", KR(ret), K(datum), K(tablet_id)); + } + } + return ret; +} + int ObTableLoadStoreTransPXWriter::check_status() { int ret = OB_SUCCESS; - if (OB_FAIL(store_ctx_->check_status(ObTableLoadStatusType::LOADING))) { - LOG_WARN("fail to check status", KR(ret)); - } else if (OB_FAIL(trans_->check_trans_status(ObTableLoadTransStatusType::RUNNING))) { - LOG_WARN("fail to check trans status", KR(ret)); + const int64_t cycle = row_count_ / CHECK_STATUS_CYCLE; + if (cycle > last_check_status_cycle_) { + last_check_status_cycle_ = cycle; + if (OB_FAIL(store_ctx_->check_status(ObTableLoadStatusType::LOADING))) { + LOG_WARN("fail to check status", KR(ret)); + } else if (OB_FAIL(trans_->check_trans_status(ObTableLoadTransStatusType::RUNNING))) { + LOG_WARN("fail to check trans status", KR(ret)); + } } return ret; } -int ObTableLoadStoreTransPXWriter::write(const blocksstable::ObDatumRow &row) +int ObTableLoadStoreTransPXWriter::write_vector(ObIVector *tablet_id_vector, + const ObIArray &vectors, + const ObBatchRows &batch_rows, + int64_t &affected_rows) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -201,43 +447,215 @@ int ObTableLoadStoreTransPXWriter::write(const blocksstable::ObDatumRow &row) } else if (OB_UNLIKELY(!can_write_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("can not write", KR(ret), KPC(this)); - } else if (OB_UNLIKELY(!row.is_valid() || row.count_ != column_count_)) { + } else if (OB_UNLIKELY(!is_vectorized_ || !use_rich_format_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected write vector", KR(ret), K(is_vectorized_), K(use_rich_format_)); + } else if (OB_UNLIKELY(vectors.count() != column_count_ || nullptr == batch_rows.skip_ || + batch_rows.size_ <= 0 || batch_rows.size_ > batch_ctx_->max_batch_size_ || + batch_rows.end_)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(row), K(column_count_)); + LOG_WARN("invalid args", KR(ret), K(column_count_), K(batch_ctx_->max_batch_size_), + K(vectors.count()), K(batch_rows)); + } else if (nullptr != tablet_id_vector && OB_FAIL(check_tablet(tablet_id_vector, batch_rows))) { + LOG_WARN("fail to check tablet", KR(ret), KP(tablet_id_vector)); } else { - ObDatumRow new_row; - if (is_heap_table_) { - new_row.storage_datums_ = row.storage_datums_ + 1; - new_row.count_ = row.count_ - 1; - } else { - new_row.storage_datums_ = row.storage_datums_; - new_row.count_ = row.count_; + if (nullptr == tablet_id_vector) { + tablet_id_vector = non_partitioned_tablet_id_vector_; } - if (store_ctx_->enable_pre_sort_) { - if (OB_FAIL(pre_sort_writer_.px_write(tablet_id_, new_row))) { - LOG_WARN("fail to px write", KR(ret), K(row), K(new_row)); - } - } else { - if (OB_FAIL(writer_->px_write(tablet_id_, new_row))) { - LOG_WARN("fail to px write", KR(ret), K(row), K(new_row)); - } - } - if (OB_SUCC(ret)) { - row_count_++; - if (row_count_ % CHECK_STATUS_CYCLE == 0) { - if (OB_FAIL(check_status())) { - LOG_WARN("fail to check status", KR(ret)); + batch_ctx_->brs_.skip_ = batch_rows.skip_; + batch_ctx_->brs_.size_ = batch_rows.size_; + batch_ctx_->brs_.all_rows_active_ = + (batch_rows.all_rows_active_ || 0 == batch_rows.skip_->accumulate_bit_cnt(batch_rows.size_)); + for (int64_t i = 0; OB_SUCC(ret) && i < column_count_; ++i) { + if (batch_ctx_->col_fixed_->at(i)) { + batch_ctx_->append_vectors_.at(i) = vectors.at(i); + } else { // 浅拷贝 + ObIVector *vector = VEC_UNIFORM_CONST != vectors.at(i)->get_format() + ? batch_ctx_->batch_vectors_.at(i) + : batch_ctx_->const_vectors_.at(i); + if (OB_FAIL(ObDirectLoadVectorUtils::shallow_copy_vector(vectors.at(i), vector, + batch_rows.size_))) { + LOG_WARN("fail to shallow copy vector", KR(ret), K(i)); + } else { + batch_ctx_->append_vectors_.at(i) = vector; } } } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(flush_batch(tablet_id_vector, batch_ctx_->append_vectors_, batch_ctx_->brs_, affected_rows))) { + LOG_WARN("fail to flush batch", KR(ret)); + } + } + return ret; +} + +int ObTableLoadStoreTransPXWriter::write_batch(const ObDatumVector &tablet_id_datum_vector, + const ObIArray &datum_vectors, + const ObBatchRows &batch_rows, + int64_t &affected_rows) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadStoreTransPXWriter not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(!can_write_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("can not write", KR(ret), KPC(this)); + } else if (OB_UNLIKELY(!is_vectorized_ || use_rich_format_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected write batch", KR(ret), K(is_vectorized_), K(use_rich_format_)); + } else if (OB_UNLIKELY(datum_vectors.count() != column_count_ || nullptr == batch_rows.skip_ || + batch_rows.size_ <= 0 || batch_rows.size_ > batch_ctx_->max_batch_size_ || + batch_rows.end_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(column_count_), K(batch_ctx_->max_batch_size_), + K(datum_vectors), K(batch_rows)); + } else if (nullptr != tablet_id_datum_vector.datums_ && + OB_FAIL(check_tablet(tablet_id_datum_vector, batch_rows))) { + LOG_WARN("fail to check tablet", KR(ret), K(tablet_id_datum_vector)); + } else { + ObIVector *tablet_id_vector = nullptr; + if (nullptr != tablet_id_datum_vector.datums_) { + tablet_id_vector = tablet_id_datum_vector.is_batch() ? batch_ctx_->tablet_id_batch_vector_ + : batch_ctx_->tablet_id_const_vector_; + static_cast(tablet_id_vector)->set_datums(tablet_id_datum_vector.datums_); + } else { + tablet_id_vector = non_partitioned_tablet_id_vector_; + } + batch_ctx_->brs_.skip_ = batch_rows.skip_; + batch_ctx_->brs_.size_ = batch_rows.size_; + batch_ctx_->brs_.all_rows_active_ = + (batch_rows.all_rows_active_ || 0 == batch_rows.skip_->accumulate_bit_cnt(batch_rows.size_)); + for (int64_t i = 0; i < column_count_; ++i) { + const ObDatumVector &datum_vector = datum_vectors.at(i); + ObIVector *vector = datum_vector.is_batch() ? batch_ctx_->batch_vectors_.at(i) + : batch_ctx_->const_vectors_.at(i); + if (batch_ctx_->col_fixed_->at(i)) { + static_cast(vector)->set_datums(datum_vector.datums_); + } else { // 浅拷贝 + MEMCPY(static_cast(vector)->get_datums(), + datum_vector.datums_, + sizeof(ObDatum) * (datum_vector.is_batch() ? batch_rows.size_ : 1)); + } + batch_ctx_->append_vectors_.at(i) = vector; + } + if (OB_FAIL(flush_batch(tablet_id_vector, batch_ctx_->append_vectors_, batch_ctx_->brs_, affected_rows))) { + LOG_WARN("fail to flush batch", KR(ret)); + } + } + return ret; +} + +int ObTableLoadStoreTransPXWriter::write_row(const ObTabletID &tablet_id, const ObDatumRow &row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadStoreTransPXWriter not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(!can_write_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("can not write", KR(ret), KPC(this)); + } else if (OB_UNLIKELY(is_vectorized_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected write row", KR(ret), K(is_vectorized_), K(use_rich_format_)); + } else if (OB_UNLIKELY(!tablet_id.is_valid() || !row.is_valid() || row.count_ != column_count_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(tablet_id), K(row), K(column_count_)); + } else if (OB_FAIL(check_tablet(tablet_id))) { + LOG_WARN("fail to check tablet", KR(ret), K(tablet_id)); + } else { + bool is_full = false; + const int64_t batch_idx = batch_ctx_->batch_buffer_.get_row_count(); + if (OB_FAIL(ObDirectLoadVectorUtils::set_tablet_id(batch_ctx_->tablet_id_batch_vector_, + batch_idx, + tablet_id))) { + LOG_WARN("fail to set tablet id", KR(ret)); + } else if (OB_FAIL(batch_ctx_->batch_buffer_.append_row(is_heap_table_ ? row.storage_datums_ + 1 : row.storage_datums_, + is_heap_table_ ? row.count_ - 1 : row.count_, + batch_ctx_->row_flag_, + is_full))) { + LOG_WARN("fail to append row", KR(ret)); + } else if (is_full) { + if (OB_FAIL(flush_buffer())) { + LOG_WARN("fail to flush buffer", KR(ret)); + } + } } return ret; } -int ObTableLoadStoreTransPXWriter::finish_write() +int ObTableLoadStoreTransPXWriter::flush_buffer() { int ret = OB_SUCCESS; - if (store_ctx_->enable_pre_sort_ && OB_FAIL(pre_sort_writer_.close())) { + int64_t affected_rows = 0; + batch_ctx_->brs_.size_ = batch_ctx_->batch_buffer_.get_row_count(); + batch_ctx_->brs_.all_rows_active_ = true; + if (OB_FAIL(flush_batch(batch_ctx_->tablet_id_batch_vector_, + batch_ctx_->batch_buffer_.get_vectors(), + batch_ctx_->brs_, + affected_rows))) { + LOG_WARN("fail to flush batch", KR(ret)); + } else if (OB_UNLIKELY(affected_rows != batch_ctx_->brs_.size_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected row count", KR(ret), K(affected_rows), K(batch_ctx_->brs_.size_)); + } else { + batch_ctx_->batch_buffer_.reuse(); + } + return ret; +} + +int ObTableLoadStoreTransPXWriter::flush_batch(ObIVector *tablet_id_vector, + const ObIArray &vectors, + const ObBatchRows &batch_rows, + int64_t &affected_rows) +{ + int ret = OB_SUCCESS; + affected_rows = 0; + if (OB_UNLIKELY(nullptr == tablet_id_vector || vectors.count() != column_count_ || + (!batch_rows.all_rows_active_ && nullptr == batch_rows.skip_) || + batch_rows.size_ <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(column_count_), KP(tablet_id_vector), K(vectors.count()), + K(batch_rows)); + } else { + const ObIArray *append_vectors = nullptr; + if (is_heap_table_) { + for (int64_t i = 0; i < store_column_count_; ++i) { + batch_ctx_->heap_vectors_.at(i) = vectors.at(i + 1); + } + append_vectors = &batch_ctx_->heap_vectors_; + } else { + append_vectors = &vectors; + } + if (store_ctx_->enable_pre_sort_) { + if (OB_FAIL(pre_sort_writer_->px_write(tablet_id_vector, *append_vectors, batch_rows, affected_rows))) { + LOG_WARN("fail to px write", KR(ret)); + } + } else { + if (OB_FAIL(writer_->px_write(tablet_id_vector, *append_vectors, batch_rows, affected_rows))) { + LOG_WARN("fail to px write", KR(ret)); + } + } + if (OB_SUCC(ret)) { + row_count_ += affected_rows; + if (OB_FAIL(check_status())) { + LOG_WARN("fail to check status", KR(ret)); + } + } + } + return ret; +} + +int ObTableLoadStoreTransPXWriter::close() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadStoreTransPXWriter not init", KR(ret), KP(this)); + } else if (nullptr != batch_ctx_ && !batch_ctx_->batch_buffer_.empty() && + OB_FAIL(flush_buffer())) { + LOG_WARN("fail to flush buffer", KR(ret)); + } else if (nullptr != pre_sort_writer_ && OB_FAIL(pre_sort_writer_->close())) { LOG_WARN("fail to push chunk", KR(ret)); } return ret; diff --git a/src/observer/table_load/ob_table_load_store_trans_px_writer.h b/src/observer/table_load/ob_table_load_store_trans_px_writer.h index fd06d33e22..a6fde00601 100644 --- a/src/observer/table_load/ob_table_load_store_trans_px_writer.h +++ b/src/observer/table_load/ob_table_load_store_trans_px_writer.h @@ -13,19 +13,18 @@ #pragma once #include "common/ob_tablet_id.h" -#include "observer/table_load/ob_table_load_pre_sort_writer.h" +#include "lib/hash/ob_hashset.h" +#include "storage/direct_load/ob_direct_load_batch_row_buffer.h" +#include "storage/direct_load/ob_direct_load_row_iterator.h" namespace oceanbase { -namespace common -{ -class ObNewRow; -} // namespace common namespace observer { class ObTableLoadStoreCtx; class ObTableLoadStoreTrans; class ObTableLoadTransStoreWriter; +class ObTableLoadPreSortWriter; class ObTableLoadStoreTransPXWriter { @@ -37,36 +36,128 @@ public: int init(ObTableLoadStoreCtx *store_ctx, ObTableLoadStoreTrans *trans, ObTableLoadTransStoreWriter *writer); - int prepare_write(const common::ObTabletID &tablet_id, - const common::ObIArray &column_ids); - int write(const blocksstable::ObDatumRow &row); - int finish_write(); + int prepare_write(const common::ObIArray &column_ids, + const bool is_vectorized, + const bool use_rich_format); + // vectorized 2.0 + int write_vector(common::ObIVector *tablet_id_vector, + const ObIArray &vectors, + const sql::ObBatchRows &batch_rows, + int64_t &affected_rows); + // vectorized 1.0 + int write_batch(const common::ObDatumVector &tablet_id_datum_vector, + const common::ObIArray &datum_vectors, + const sql::ObBatchRows &batch_rows, + int64_t &affected_rows); + // non-vectorized + int write_row(const common::ObTabletID &tablet_id, + const blocksstable::ObDatumRow &row); + int close(); + bool is_inited() const { return is_inited_; } TO_STRING_KV(KP_(store_ctx), KP_(trans), KP_(writer), - K_(tablet_id), - K_(column_count), - K_(row_count), K_(is_heap_table), + K_(column_count), + K_(store_column_count), + KP_(non_partitioned_tablet_id_vector), + KP_(pre_sort_writer), + KP_(batch_ctx), + K_(row_count), + K_(last_check_status_cycle), + K_(is_vectorized), + K_(use_rich_format), K_(can_write), K_(is_inited)); private: const static int64_t CHECK_STATUS_CYCLE = 10000; - int check_tablet(const common::ObTabletID &tablet_id); int check_columns(const common::ObIArray &column_ids); + int init_batch_ctx(const bool is_vectorized, const bool use_rich_format); + int init_tablet_id_set(); + int check_tablet(const ObTabletID &tablet_id); + int check_tablet(common::ObIVector *vector, const sql::ObBatchRows &batch_rows); + int check_tablet(const common::ObDatumVector &datum_vector, const sql::ObBatchRows &batch_rows); int check_status(); + int flush_buffer(); + int flush_batch(common::ObIVector *tablet_id_vector, + const ObIArray &vectors, + const sql::ObBatchRows &batch_rows, + int64_t &affected_rows); private: + typedef common::hash::ObHashSet TabletIDSet; + + // 旁路导入在以下场景, 可能会对数据进行修改, 导致vector中数据指针地址被修改 + // * lob列 + // * ObDASUtils::reshape_vector_value + // 后续SQL对数据进行投影时会访问非法内存 + // 因此, 我们需要对vector做一次浅拷贝, 来保证SQL的vector中数据指针地址不发生变化 + // + // 目前测试现象: + // * SQL会重置数据指针地址 (没有得到所有场景都会重置的保证) + // * px吐出来的数据是可以直接写存储的格式, 也就是说已经做过了ObDASUtils::reshape_vector_value里的处理 + // 这个应该是SQL层保证的, 这样就可以优化掉DDL路径中的reshape操作 + // + // 采取保守方案, 保留浅拷贝和reshape + struct BatchCtx + { + public: + BatchCtx() + : max_batch_size_(0), + tablet_id_batch_vector_(nullptr), + tablet_id_const_vector_(nullptr), + col_fixed_(nullptr) + { + } + ~BatchCtx() {} + public: + int64_t max_batch_size_; + // non-vectorized : VEC_FIXED + // vectorized 1.0 : VEC_UNIFORM + // vectorized 2.0 : nullptr + ObIVector *tablet_id_batch_vector_; + // non-vectorized : nullptr + // vectorized 1.0 : VEC_UNIFORM_CONST + // vectorized 2.0 : nullptr + ObIVector *tablet_id_const_vector_; + // for non-vectorized + storage::ObDirectLoadRowFlag row_flag_; + storage::ObDirectLoadBatchRowBuffer batch_buffer_; + // for vectorized + // shallow copy unfixed cols + sql::ObBitVector *col_fixed_; + // vectorized 1.0 : VEC_UNIFORM + // vectorized 2.0 : nullptr(fixed), VEC_DISCRETE(unfixed) + ObArray batch_vectors_; + // vectorized 1.0 : VEC_UNIFORM_CONST + // vectorized 2.0 : nullptr(fixed), VEC_UNIFORM_CONST(unfixed) + ObArray const_vectors_; + ObArray append_vectors_; + // for write + ObArray heap_vectors_; + sql::ObBatchRows brs_; + }; + +private: + ObArenaAllocator allocator_; ObTableLoadStoreCtx *store_ctx_; ObTableLoadStoreTrans *trans_; ObTableLoadTransStoreWriter *writer_; - ObTableLoadPreSortWriter pre_sort_writer_; - ObTabletID tablet_id_; - int64_t column_count_; - int64_t row_count_; bool is_heap_table_; + // number of columns received from px + int64_t column_count_; + // number of columns write to direct load + int64_t store_column_count_; + ObIVector *non_partitioned_tablet_id_vector_; + TabletIDSet tablet_id_set_; + ObTableLoadPreSortWriter *pre_sort_writer_; + BatchCtx *batch_ctx_; + int64_t row_count_; + int64_t last_check_status_cycle_; + bool is_vectorized_; + bool use_rich_format_; bool can_write_; bool is_inited_; }; diff --git a/src/observer/table_load/ob_table_load_struct.h b/src/observer/table_load/ob_table_load_struct.h index 9c6f7a746f..6434c22bd0 100644 --- a/src/observer/table_load/ob_table_load_struct.h +++ b/src/observer/table_load/ob_table_load_struct.h @@ -95,6 +95,8 @@ enum class ObTableLoadExeMode { struct ObTableLoadParam { public: + static const int64_t MAX_BATCH_SIZE = 65536; + static const int64_t DEFAULT_BATCH_SIZE = 256; ObTableLoadParam() : tenant_id_(common::OB_INVALID_ID), table_id_(common::OB_INVALID_ID), @@ -138,7 +140,7 @@ public: common::OB_INVALID_ID != table_id_ && parallel_ > 0 && session_count_ > 0 && - batch_size_ > 0 && + batch_size_ > 0 && batch_size_ <= MAX_BATCH_SIZE && column_count_ > 0 && sql::ObLoadDupActionType::LOAD_INVALID_MODE != dup_action_ && storage::ObDirectLoadMethod::is_type_valid(method_) && diff --git a/src/observer/table_load/ob_table_load_task_scheduler.cpp b/src/observer/table_load/ob_table_load_task_scheduler.cpp index a8b270fed3..534d0e876a 100644 --- a/src/observer/table_load/ob_table_load_task_scheduler.cpp +++ b/src/observer/table_load/ob_table_load_task_scheduler.cpp @@ -20,7 +20,6 @@ #include "observer/table_load/ob_table_load_task.h" #include "share/ob_share_util.h" #include "share/rc/ob_tenant_base.h" -#include "storage/direct_load/ob_direct_load_table_builder_allocator.h" #include "share/resource_manager/ob_resource_manager.h" namespace oceanbase @@ -271,9 +270,6 @@ void ObTableLoadTaskThreadPoolScheduler::run(uint64_t thread_idx) state_ = STATE_STOPPING; } - // clear thread local variables - get_table_builder_allocator()->reset(); - LOG_INFO("table load task thread stopped", KP(this), "pid", get_tid_cache(), K(thread_idx)); } diff --git a/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp b/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp index 0f689e1589..f450c8dcc7 100644 --- a/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp +++ b/src/observer/table_load/ob_table_load_trans_bucket_writer.cpp @@ -361,6 +361,7 @@ int ObTableLoadTransBucketWriter::write_for_non_partitioned(SessionContext &sess } else if (OB_FAIL(load_bucket->add_row(session_ctx.partition_id_.tablet_id_, row, param_.batch_size_, + WRITE_ROW_SIZE, need_write))) { LOG_WARN("fail to add row", KR(ret)); } else if (need_write && OB_FAIL(write_load_bucket(session_ctx, load_bucket))) { @@ -433,6 +434,7 @@ int ObTableLoadTransBucketWriter::write_for_partitioned(SessionContext &session_ } else if (OB_FAIL(load_bucket->add_row(partition_id.tablet_id_, row, param_.batch_size_, + WRITE_ROW_SIZE, need_write))) { LOG_WARN("fail to add row", KR(ret)); } else if (need_write && OB_FAIL(write_load_bucket(session_ctx, load_bucket))) { diff --git a/src/observer/table_load/ob_table_load_trans_bucket_writer.h b/src/observer/table_load/ob_table_load_trans_bucket_writer.h index 7700878b11..269d9c1537 100644 --- a/src/observer/table_load/ob_table_load_trans_bucket_writer.h +++ b/src/observer/table_load/ob_table_load_trans_bucket_writer.h @@ -69,6 +69,7 @@ private: ObTableLoadBucket *&load_bucket); int write_load_bucket(SessionContext &session_ctx, ObTableLoadBucket *load_bucket); private: + static const int64_t WRITE_ROW_SIZE = 2LL * 1024 * 1024; ObTableLoadTransCtx *const trans_ctx_; ObTableLoadCoordinatorCtx *const coordinator_ctx_; const ObTableLoadParam ¶m_; diff --git a/src/observer/table_load/ob_table_load_trans_store.cpp b/src/observer/table_load/ob_table_load_trans_store.cpp index 5833fcbbeb..bec8807af6 100644 --- a/src/observer/table_load/ob_table_load_trans_store.cpp +++ b/src/observer/table_load/ob_table_load_trans_store.cpp @@ -15,20 +15,24 @@ #include "observer/table_load/ob_table_load_trans_store.h" #include "observer/omt/ob_tenant_timezone_mgr.h" #include "observer/table_load/ob_table_load_autoinc_nextval.h" -#include "observer/table_load/ob_table_load_error_row_handler.h" #include "observer/table_load/ob_table_load_data_row_handler.h" -#include "storage/direct_load/ob_direct_load_dml_row_handler.h" +#include "observer/table_load/ob_table_load_error_row_handler.h" #include "observer/table_load/ob_table_load_stat.h" #include "observer/table_load/ob_table_load_store_ctx.h" +#include "observer/table_load/ob_table_load_store_table_ctx.h" #include "observer/table_load/ob_table_load_table_ctx.h" #include "observer/table_load/ob_table_load_trans_ctx.h" #include "observer/table_load/ob_table_load_utils.h" -#include "observer/table_load/ob_table_load_store_table_ctx.h" -#include "sql/engine/cmd/ob_load_data_utils.h" -#include "sql/resolver/expr/ob_raw_expr_util.h" -#include "sql/ob_sql_utils.h" #include "share/ob_autoincrement_service.h" #include "share/sequence/ob_sequence_cache.h" +#include "sql/engine/cmd/ob_load_data_utils.h" +#include "sql/ob_sql_utils.h" +#include "sql/resolver/expr/ob_raw_expr_util.h" +#include "storage/direct_load/ob_direct_load_dml_row_handler.h" +#include "storage/direct_load/ob_direct_load_external_multi_partition_table.h" +#include "storage/direct_load/ob_direct_load_insert_table_row_writer.h" +#include "storage/direct_load/ob_direct_load_multiple_sstable_builder.h" +#include "storage/direct_load/ob_direct_load_vector_utils.h" namespace oceanbase { @@ -100,13 +104,561 @@ void ObTableLoadTransStore::reset() * ObTableLoadTransStoreWriter */ + /** + * StoreWriter + */ + +ObTableLoadTransStoreWriter::StoreWriter::StoreWriter() + : store_ctx_(nullptr), + trans_store_(nullptr), + session_id_(0), + allocator_("TLD_SW"), + is_single_part_(false), + is_closed_(false), + is_inited_(false) +{ + allocator_.set_tenant_id(MTL_ID()); + table_builders_.set_attr(ObMemAttr(MTL_ID(), "TLD_SW")); +} + +ObTableLoadTransStoreWriter::StoreWriter::~StoreWriter() +{ + reset(); +} + +void ObTableLoadTransStoreWriter::StoreWriter::reset() +{ + store_ctx_ = nullptr; + trans_store_ = nullptr; + session_id_ = 0; + table_builder_map_.destroy(); + for (int64_t i = 0; i < table_builders_.count(); ++i) { + ObIDirectLoadPartitionTableBuilder *table_builder = table_builders_.at(i); + table_builder->~ObIDirectLoadPartitionTableBuilder(); + allocator_.free(table_builder); + } + table_builders_.reset(); + allocator_.reset(); + is_single_part_ = false; + is_closed_ = false; + is_inited_ = false; +} + +int ObTableLoadTransStoreWriter::StoreWriter::init(ObTableLoadStoreCtx *store_ctx, + ObTableLoadTransStore *trans_store, + int32_t session_id) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("StoreWriter init twice", KR(ret), KP(this)); + } else if (OB_UNLIKELY(nullptr == store_ctx || nullptr == trans_store || session_id <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(store_ctx), KP(trans_store), K(session_id)); + } else { + if (OB_FAIL(table_builder_map_.create(64, "TLD_SW_Map", "TLD_SW_Map", MTL_ID()))) { + LOG_WARN("fail to create hashmap", KR(ret)); + } else if (OB_FAIL(datum_row_.init( + store_ctx->data_store_table_ctx_->table_data_desc_.column_count_))) { + LOG_WARN("fail to init datum row", KR(ret)); + } else { + store_ctx_ = store_ctx; + trans_store_ = trans_store; + session_id_ = session_id; + is_single_part_ = (store_ctx_->data_store_table_ctx_->is_multiple_mode_ || + 1 == store_ctx->data_store_table_ctx_->ls_partition_ids_.count()); + is_inited_ = true; + } + } + return ret; +} + +int ObTableLoadTransStoreWriter::StoreWriter::inner_append_row( + const ObTabletID &tablet_id, + const ObTableLoadSequenceNo &seq_no, + const ObDatumRow &datum_row) +{ + int ret = OB_SUCCESS; + ObIDirectLoadPartitionTableBuilder *table_builder = nullptr; + if (OB_FAIL(get_table_builder(tablet_id, table_builder))) { + LOG_WARN("fail to get table builder", KR(ret), K(tablet_id)); + } else if (OB_FAIL(table_builder->append_row(tablet_id, seq_no, datum_row))) { + LOG_WARN("fail to append row", KR(ret), K(datum_row)); + } + if (OB_FAIL(ret)) { + ObTableLoadErrorRowHandler *error_row_handler = store_ctx_->error_row_handler_; + ObDirectLoadDMLRowHandler *data_row_handler = store_ctx_->data_store_table_ctx_->row_handler_; + if (OB_LIKELY(OB_ERR_PRIMARY_KEY_DUPLICATE == ret)) { + if (OB_FAIL(data_row_handler->handle_update_row(datum_row))) { + LOG_WARN("fail to handle update row", KR(ret), K(datum_row)); + } + } else if (OB_LIKELY(OB_ROWKEY_ORDER_ERROR == ret)) { + if (OB_FAIL(error_row_handler->handle_error_row(ret))) { + LOG_WARN("fail to handle error row", KR(ret), K(tablet_id), K(datum_row)); + } + } + } + return ret; +} + +int ObTableLoadTransStoreWriter::StoreWriter::append_row(const ObTabletID &tablet_id, + const ObTableLoadSequenceNo &seq_no, + const ObDatumRow &datum_row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("StoreWriter not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(is_closed_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected writer is closed", KR(ret)); + } else if (OB_FAIL(inner_append_row(tablet_id, seq_no, datum_row))) { + LOG_WARN("fail to append row", KR(ret)); + } + return ret; +} + +int ObTableLoadTransStoreWriter::StoreWriter::append_batch( + ObIVector *tablet_id_vector, + const ObIArray &vectors, + const ObBatchRows &batch_rows, + int64_t &affected_rows) +{ + int ret = OB_SUCCESS; + affected_rows = 0; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("StoreWriter not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(is_closed_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected writer is closed", KR(ret)); + } else if (OB_UNLIKELY(nullptr == tablet_id_vector || + vectors.count() != datum_row_.get_column_count() || + (!batch_rows.all_rows_active_ && nullptr == batch_rows.skip_) || + batch_rows.size_ <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(tablet_id_vector), K(vectors), K(batch_rows)); + } else { + // TODO(suzhi.yt) 这一期只有px_write会走append_batch, 这里先写死seq_no + static const ObTableLoadSequenceNo seq_no(0); + for (int64_t i = 0; OB_SUCC(ret) && i < batch_rows.size_; ++i) { + if (!batch_rows.all_rows_active_ && batch_rows.skip_->at(i)) { + continue; + } else { + const ObTabletID tablet_id = ObDirectLoadVectorUtils::get_tablet_id(tablet_id_vector, i); + if (OB_FAIL(ObDirectLoadVectorUtils::to_datums(vectors, + i, + datum_row_.storage_datums_, + datum_row_.count_))) { + LOG_WARN("fail to transfer vectors to datums", KR(ret), K(i)); + } else if (OB_FAIL(inner_append_row(tablet_id, seq_no, datum_row_))) { + LOG_WARN("fail to append row", KR(ret)); + } else { + ++affected_rows; + } + } + } + } + return ret; +} + +int ObTableLoadTransStoreWriter::StoreWriter::close() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("StoreWriter not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(is_closed_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected writer is closed", KR(ret)); + } else { + ObTableLoadTransStore::SessionStore *session_store = + trans_store_->session_store_array_.at(session_id_ - 1); + for (int64_t i = 0; OB_SUCC(ret) && i < table_builders_.count(); ++i) { + ObIDirectLoadPartitionTableBuilder *table_builder = table_builders_.at(i); + if (OB_FAIL(table_builder->close())) { + LOG_WARN("fail to close table store", KR(ret)); + } else if (OB_FAIL(table_builder->get_tables(session_store->partition_table_array_, + session_store->allocator_))) { + LOG_WARN("fail to get tables", KR(ret)); + } + } + if (OB_SUCC(ret)) { + is_closed_ = true; + } + } + return ret; +} + +int ObTableLoadTransStoreWriter::StoreWriter::new_table_builder( + const ObTabletID &tablet_id, + ObIDirectLoadPartitionTableBuilder *&table_builder) +{ + int ret = OB_SUCCESS; + table_builder = nullptr; + if (store_ctx_->data_store_table_ctx_->is_multiple_mode_) { + // 排序路径 + ObDirectLoadExternalMultiPartitionTableBuildParam param; + param.table_data_desc_ = store_ctx_->data_store_table_ctx_->table_data_desc_; + param.datum_utils_ = &(store_ctx_->data_store_table_ctx_->schema_->datum_utils_); + param.file_mgr_ = store_ctx_->tmp_file_mgr_; + param.extra_buf_ = reinterpret_cast(1); // unuse, delete in future + param.extra_buf_size_ = param.table_data_desc_.extra_buf_size_; + ObDirectLoadExternalMultiPartitionTableBuilder *external_mp_table_builder = nullptr; + if (OB_ISNULL(table_builder = external_mp_table_builder = + OB_NEWx(ObDirectLoadExternalMultiPartitionTableBuilder, &allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObDirectLoadExternalMultiPartitionTableBuilder", KR(ret)); + } else if (OB_FAIL(external_mp_table_builder->init(param))) { + LOG_WARN("fail to init external multi partition table builder", KR(ret)); + } + } else { + // 有主键表不排序路径 + abort_unless(!store_ctx_->data_store_table_ctx_->table_data_desc_.is_heap_table_); + ObDirectLoadMultipleSSTableBuildParam param; + param.tablet_id_ = tablet_id; + param.table_data_desc_ = store_ctx_->data_store_table_ctx_->table_data_desc_; + param.datum_utils_ = &(store_ctx_->data_store_table_ctx_->schema_->datum_utils_); + param.file_mgr_ = store_ctx_->tmp_file_mgr_; + param.extra_buf_ = reinterpret_cast(1); // unuse, delete in future + param.extra_buf_size_ = param.table_data_desc_.extra_buf_size_; + ObDirectLoadMultipleSSTableBuilder *sstable_builder = nullptr; + if (OB_ISNULL(table_builder = sstable_builder = + OB_NEWx(ObDirectLoadMultipleSSTableBuilder, &allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObDirectLoadMultipleSSTableBuilder", KR(ret)); + } else if (OB_FAIL(sstable_builder->init(param))) { + LOG_WARN("fail to init sstable builder", KR(ret)); + } + } + if (OB_FAIL(ret)) { + if (nullptr != table_builder) { + table_builder->~ObIDirectLoadPartitionTableBuilder(); + allocator_.free(table_builder); + table_builder = nullptr; + } + } + return ret; +} + +int ObTableLoadTransStoreWriter::StoreWriter::get_table_builder( + const ObTabletID &tablet_id, + ObIDirectLoadPartitionTableBuilder *&table_builder) +{ + int ret = OB_SUCCESS; + table_builder = nullptr; + if (OB_UNLIKELY(!tablet_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(tablet_id)); + } else { + if (is_single_part_) { + if (!table_builders_.empty()) { + table_builder = table_builders_.at(0); + } + } else { + if (OB_FAIL(table_builder_map_.get_refactored(tablet_id, table_builder))) { + if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) { + LOG_WARN("fail to get refactored", KR(ret), K(tablet_id)); + } else { + ret = OB_SUCCESS; + } + } + } + if (OB_SUCC(ret) && nullptr == table_builder) { + // new table builder + if (OB_FAIL(new_table_builder(tablet_id, table_builder))) { + LOG_WARN("fail to new table builder", KR(ret), K(tablet_id)); + } else if (OB_FAIL(table_builder_map_.set_refactored(tablet_id, table_builder))) { + LOG_WARN("fail to set refactored", KR(ret), K(tablet_id)); + } else if (OB_FAIL(table_builders_.push_back(table_builder))) { + LOG_WARN("fail to push back", KR(ret)); + } + if (OB_FAIL(ret)) { + if (nullptr != table_builder) { + table_builder->~ObIDirectLoadPartitionTableBuilder(); + allocator_.free(table_builder); + table_builder = nullptr; + } + } + } + } + return ret; +} + + /** + * DirectWriter + */ + +ObTableLoadTransStoreWriter::DirectWriter::DirectWriter() + : store_ctx_(nullptr), + allocator_("TLD_DW"), + lob_allocator_("TLD_LobAlloc"), + max_batch_size_(0), + is_single_part_(false), + is_closed_(false), + is_inited_(false) +{ + allocator_.set_tenant_id(MTL_ID()); + lob_allocator_.set_tenant_id(MTL_ID()); + batch_writers_.set_attr(ObMemAttr(MTL_ID(), "TLD_DW")); +} + +ObTableLoadTransStoreWriter::DirectWriter::~DirectWriter() +{ + reset(); +} + +void ObTableLoadTransStoreWriter::DirectWriter::reset() +{ + store_ctx_ = nullptr; + batch_writer_map_.destroy(); + for (int64_t i = 0; i < batch_writers_.count(); ++i) { + ObDirectLoadInsertTableBatchRowDirectWriter *batch_writer = batch_writers_.at(i); + batch_writer->~ObDirectLoadInsertTableBatchRowDirectWriter(); + allocator_.free(batch_writer); + } + batch_writers_.reset(); + allocator_.reset(); + max_batch_size_ = 0; + is_single_part_ = false; + is_closed_ = false; + is_inited_ = false; +} + +int ObTableLoadTransStoreWriter::DirectWriter::init(ObTableLoadStoreCtx *store_ctx) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("DirectWriter init twice", KR(ret), KP(this)); + } else if (OB_UNLIKELY(nullptr == store_ctx)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(store_ctx)); + } else { + if (OB_FAIL(batch_writer_map_.create(64, "TLD_DW_Map", "TLD_DW_Map", MTL_ID()))) { + LOG_WARN("fail to create hashmap", KR(ret)); + } else { + store_ctx_ = store_ctx; + max_batch_size_ = store_ctx->ctx_->param_.batch_size_; + is_single_part_ = (1 == store_ctx->data_store_table_ctx_->ls_partition_ids_.count()); + is_inited_ = true; + } + } + return ret; +} + +int ObTableLoadTransStoreWriter::DirectWriter::append_row(const ObTabletID &tablet_id, + const ObTableLoadSequenceNo &seq_no, + const ObDatumRow &datum_row) +{ + UNUSED(seq_no); + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("DirectWriter not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(is_closed_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected writer is closed", KR(ret)); + } else { + ObDirectLoadInsertTableBatchRowDirectWriter *batch_writer = nullptr; + if (OB_FAIL(get_batch_writer(tablet_id, batch_writer))) { + LOG_WARN("fail to get batch writer", KR(ret), K(tablet_id)); + } else if (OB_FAIL(batch_writer->append_row(datum_row))) { + LOG_WARN("fail to append row", KR(ret)); + } + } + return ret; +} + +int ObTableLoadTransStoreWriter::DirectWriter::append_batch( + ObIVector *tablet_id_vector, + const ObIArray &vectors, + const ObBatchRows &batch_rows, + int64_t &affected_rows) +{ + int ret = OB_SUCCESS; + affected_rows = 0; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("DirectWriter not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(is_closed_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected writer is closed", KR(ret)); + } else if (OB_UNLIKELY(nullptr == tablet_id_vector || + (!batch_rows.all_rows_active_ && nullptr == batch_rows.skip_) || + batch_rows.size_ <= 0 || batch_rows.size_ > max_batch_size_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(tablet_id_vector), K(vectors), K(batch_rows), + K(max_batch_size_)); + } else { + const bool all_tablet_id_is_same = ObDirectLoadVectorUtils::check_all_tablet_id_is_same(tablet_id_vector, batch_rows.size_); + ObDirectLoadInsertTableBatchRowDirectWriter *batch_writer = nullptr; + // direct path + if (batch_rows.all_rows_active_ && all_tablet_id_is_same && batch_rows.size_ >= max_batch_size_ / 2) { + const ObTabletID tablet_id = ObDirectLoadVectorUtils::get_tablet_id(tablet_id_vector, 0); + if (OB_FAIL(get_batch_writer(tablet_id, batch_writer))) { + LOG_WARN("fail to get batch writer", KR(ret), K(tablet_id)); + } else if (OB_FAIL(batch_writer->append_batch(vectors, batch_rows.size_))) { + LOG_WARN("fail to append batch", KR(ret)); + } else { + affected_rows = batch_rows.size_; + } + } + // buffer path + else if (all_tablet_id_is_same) { + const ObTabletID tablet_id = ObDirectLoadVectorUtils::get_tablet_id(tablet_id_vector, 0); + if (OB_FAIL(get_batch_writer(tablet_id, batch_writer))) { + LOG_WARN("fail to get batch writer", KR(ret), K(tablet_id)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < batch_rows.size_; ++i) { + if (!batch_rows.all_rows_active_ && batch_rows.skip_->at(i)) { + continue; + } else if (OB_FAIL(batch_writer->append_row(vectors, i))) { + LOG_WARN("fail to append row", KR(ret), K(i)); + } else { + ++affected_rows; + } + } + } else if (batch_rows.all_rows_active_) { + for (int64_t i = 0; OB_SUCC(ret) && i < batch_rows.size_; ++i) { + const ObTabletID tablet_id = ObDirectLoadVectorUtils::get_tablet_id(tablet_id_vector, i); + if (OB_FAIL(get_batch_writer(tablet_id, batch_writer))) { + LOG_WARN("fail to get batch writer", KR(ret), K(tablet_id)); + } else if (OB_FAIL(batch_writer->append_row(vectors, i))) { + LOG_WARN("fail to append row", KR(ret), K(i)); + } + } + if (OB_SUCC(ret)) { + affected_rows = batch_rows.size_; + } + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < batch_rows.size_; ++i) { + if (!batch_rows.all_rows_active_ && batch_rows.skip_->at(i)) { + continue; + } else { + const ObTabletID tablet_id = ObDirectLoadVectorUtils::get_tablet_id(tablet_id_vector, i); + if (OB_FAIL(get_batch_writer(tablet_id, batch_writer))) { + LOG_WARN("fail to get batch writer", KR(ret), K(tablet_id)); + } else if (OB_FAIL(batch_writer->append_row(vectors, i))) { + LOG_WARN("fail to append row", KR(ret), K(i)); + } else { + ++affected_rows; + } + } + } + } + } + return ret; +} + +int ObTableLoadTransStoreWriter::DirectWriter::close() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("StoreWriter not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(is_closed_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected writer is closed", KR(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < batch_writers_.count(); ++i) { + ObDirectLoadInsertTableBatchRowDirectWriter *batch_writer = batch_writers_.at(i); + if (OB_FAIL(batch_writer->close())) { + LOG_WARN("fail to close", KR(ret)); + } + } + if (OB_SUCC(ret)) { + is_closed_ = true; + } + } + return ret; +} + +int ObTableLoadTransStoreWriter::DirectWriter::new_batch_writer( + const ObTabletID &tablet_id, + ObDirectLoadInsertTableBatchRowDirectWriter *&batch_writer) +{ + int ret = OB_SUCCESS; + batch_writer = nullptr; + ObDirectLoadInsertTabletContext *insert_tablet_ctx = nullptr; + ObDirectLoadInsertTableRowInfo row_info; + if (OB_FAIL(store_ctx_->data_store_table_ctx_->insert_table_ctx_->get_tablet_context( + tablet_id, insert_tablet_ctx))) { + LOG_WARN("fail to get tablet context ", KR(ret), K(tablet_id)); + } else if (OB_FAIL(insert_tablet_ctx->get_row_info(row_info))) { + LOG_WARN("fail to get row info", KR(ret)); + } else if (OB_ISNULL(batch_writer = + OB_NEWx(ObDirectLoadInsertTableBatchRowDirectWriter, &allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObDirectLoadInsertTableBatchRowDirectWriter", KR(ret)); + } else if (OB_FAIL(batch_writer->init(insert_tablet_ctx, + row_info, + store_ctx_->data_store_table_ctx_->row_handler_, + &lob_allocator_))) { + LOG_WARN("fail to init direct batch writer", KR(ret)); + } + if (OB_FAIL(ret)) { + if (nullptr != batch_writer) { + batch_writer->~ObDirectLoadInsertTableBatchRowDirectWriter(); + allocator_.free(batch_writer); + batch_writer = nullptr; + } + } + return ret; +} + +int ObTableLoadTransStoreWriter::DirectWriter::get_batch_writer( + const ObTabletID &tablet_id, + ObDirectLoadInsertTableBatchRowDirectWriter *&batch_writer) +{ + int ret = OB_SUCCESS; + batch_writer = nullptr; + if (OB_UNLIKELY(!tablet_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(tablet_id)); + } else { + if (is_single_part_) { + if (!batch_writers_.empty()) { + batch_writer = batch_writers_.at(0); + } + } else { + if (OB_FAIL(batch_writer_map_.get_refactored(tablet_id, batch_writer))) { + if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) { + LOG_WARN("fail to get refactored", KR(ret), K(tablet_id)); + } else { + ret = OB_SUCCESS; + } + } + } + if (OB_SUCC(ret) && nullptr == batch_writer) { + // new batch writer + if (OB_FAIL(new_batch_writer(tablet_id, batch_writer))) { + LOG_WARN("fail to new batch writer", KR(ret), K(tablet_id)); + } else if (OB_FAIL(batch_writer_map_.set_refactored(tablet_id, batch_writer))) { + LOG_WARN("fail to set refactored", KR(ret), K(tablet_id)); + } else if (OB_FAIL(batch_writers_.push_back(batch_writer))) { + LOG_WARN("fail to push back", KR(ret)); + } + if (OB_FAIL(ret)) { + if (nullptr != batch_writer) { + batch_writer->~ObDirectLoadInsertTableBatchRowDirectWriter(); + allocator_.free(batch_writer); + batch_writer = nullptr; + } + } + } + } + return ret; +} + + /** + * SessionContext + */ ObTableLoadTransStoreWriter::SessionContext::SessionContext(int32_t session_id, uint64_t tenant_id, ObDataTypeCastParams cast_params) : session_id_(session_id), cast_allocator_("TLD_TS_Caster"), cast_params_(cast_params), - last_receive_sequence_no_(0), - extra_buf_(nullptr), - extra_buf_size_(0) + last_receive_sequence_no_(0) { cast_allocator_.set_tenant_id(MTL_ID()); } @@ -114,6 +666,10 @@ ObTableLoadTransStoreWriter::SessionContext::SessionContext(int32_t session_id, ObTableLoadTransStoreWriter::SessionContext::~SessionContext() { datum_row_.reset(); + if (nullptr != writer_) { + writer_->~IWriter(); + writer_ = nullptr; + } } ObTableLoadTransStoreWriter::ObTableLoadTransStoreWriter(ObTableLoadTransStore *trans_store) @@ -136,13 +692,9 @@ ObTableLoadTransStoreWriter::ObTableLoadTransStoreWriter(ObTableLoadTransStore * ObTableLoadTransStoreWriter::~ObTableLoadTransStoreWriter() { if (nullptr != session_ctx_array_) { - int32_t session_count = param_.px_mode_? 1 : param_.write_session_count_; + int32_t session_count = param_.px_mode_ ? 1 : param_.write_session_count_; for (int64_t i = 0; i < session_count; ++i) { SessionContext *session_ctx = session_ctx_array_ + i; - if (OB_NOT_NULL(session_ctx->extra_buf_)) { - allocator_.free(session_ctx->extra_buf_); - session_ctx->extra_buf_ = nullptr; - } session_ctx->~SessionContext(); } allocator_.free(session_ctx_array_); @@ -153,7 +705,7 @@ ObTableLoadTransStoreWriter::~ObTableLoadTransStoreWriter() int ObTableLoadTransStoreWriter::init() { int ret = OB_SUCCESS; - int32_t session_count = param_.px_mode_? 1 : param_.write_session_count_; + int32_t session_count = param_.px_mode_ ? 1 : param_.write_session_count_; if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObTableLoadTransStoreWriter init twice", KR(ret), KP(this)); @@ -203,7 +755,7 @@ int ObTableLoadTransStoreWriter::init_session_ctx_array() { int ret = OB_SUCCESS; void *buf = nullptr; - int32_t session_count = param_.px_mode_? 1 : param_.write_session_count_; + int32_t session_count = param_.px_mode_ ? 1 : param_.write_session_count_; ObDataTypeCastParams cast_params(trans_ctx_->ctx_->session_info_->get_timezone_info()); if (OB_ISNULL(buf = allocator_.alloc(sizeof(SessionContext) * session_count))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -213,41 +765,32 @@ int ObTableLoadTransStoreWriter::init_session_ctx_array() } else { session_ctx_array_ = static_cast(buf); for (int64_t i = 0; i < session_count; ++i) { - new (session_ctx_array_ + i) - SessionContext(i + 1, param_.tenant_id_, cast_params); + new (session_ctx_array_ + i) SessionContext(i + 1, param_.tenant_id_, cast_params); } } - ObDirectLoadTableStoreParam param; - param.table_data_desc_ = *table_data_desc_; - param.datum_utils_ = &(store_ctx_->data_store_table_ctx_->schema_->datum_utils_); - param.file_mgr_ = store_ctx_->tmp_file_mgr_; - param.is_multiple_mode_ = store_ctx_->data_store_table_ctx_->is_multiple_mode_; - param.is_fast_heap_table_ = store_ctx_->data_store_table_ctx_->is_fast_heap_table_; - param.insert_table_ctx_ = store_ctx_->data_store_table_ctx_->insert_table_ctx_; - param.dml_row_handler_ = store_ctx_->data_store_table_ctx_->row_handler_; for (int64_t i = 0; OB_SUCC(ret) && i < session_count; ++i) { SessionContext *session_ctx = session_ctx_array_ + i; - if (param_.px_mode_) { - session_ctx->extra_buf_size_ = table_data_desc_->extra_buf_size_; - if (OB_ISNULL(session_ctx->extra_buf_ = - static_cast(allocator_.alloc(session_ctx->extra_buf_size_)))) { + // init writer_ + if (store_ctx_->data_store_table_ctx_->is_fast_heap_table_) { + DirectWriter *direct_writer = nullptr; + if (OB_ISNULL(session_ctx->writer_ = direct_writer = OB_NEWx(DirectWriter, &allocator_))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc memory", KR(ret)); - } else { - param.extra_buf_ = session_ctx->extra_buf_; - param.extra_buf_size_ = session_ctx->extra_buf_size_; + LOG_WARN("fail to new DirectWriter", KR(ret)); + } else if (OB_FAIL(direct_writer->init(store_ctx_))) { + LOG_WARN("fail to init direct writer", KR(ret)); } } else { - param.extra_buf_ = store_ctx_->session_ctx_array_[i].extra_buf_; - param.extra_buf_size_ = store_ctx_->session_ctx_array_[i].extra_buf_size_; - } - if (OB_SUCC(ret)) { - // init table_store_ - if (OB_FAIL(session_ctx->table_store_.init(param))) { - LOG_WARN("fail to init table store", KR(ret)); + StoreWriter *store_writer = nullptr; + if (OB_ISNULL(session_ctx->writer_ = store_writer = OB_NEWx(StoreWriter, &allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new StoreWriter", KR(ret)); + } else if (OB_FAIL(store_writer->init(store_ctx_, trans_store_, session_ctx->session_id_))) { + LOG_WARN("fail to init store writer", KR(ret)); } - // init datum_row_ - else if (OB_FAIL(session_ctx->datum_row_.init(table_data_desc_->column_count_))) { + } + // init datum_row_ + if (OB_SUCC(ret)) { + if (OB_FAIL(session_ctx->datum_row_.init(table_data_desc_->column_count_))) { LOG_WARN("fail to init datum row", KR(ret)); } else { session_ctx->datum_row_.row_flag_.set_flag(ObDmlFlag::DF_INSERT); @@ -262,7 +805,7 @@ int ObTableLoadTransStoreWriter::advance_sequence_no(int32_t session_id, uint64_ ObTableLoadMutexGuard &guard) { int ret = OB_SUCCESS; - int32_t session_count = param_.px_mode_? 1 : param_.write_session_count_; + int32_t session_count = param_.px_mode_ ? 1 : param_.write_session_count_; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObTableLoadTransStoreWriter not init", KR(ret)); @@ -311,7 +854,9 @@ int ObTableLoadTransStoreWriter::write(int32_t session_id, } else { ret = OB_SUCCESS; } - } else if (OB_FAIL(write_row_to_table_store(session_ctx.table_store_, row.tablet_id_, row.obj_row_.seq_no_, session_ctx.datum_row_))) { + } else if (OB_FAIL(session_ctx.writer_->append_row(row.tablet_id_, + row.obj_row_.seq_no_, + session_ctx.datum_row_))) { LOG_WARN("fail to write row", KR(ret), K(session_id), K(row.tablet_id_), K(i)); } } @@ -334,13 +879,10 @@ int ObTableLoadTransStoreWriter::px_write(const ObTabletID &tablet_id, const blo ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(tablet_id), K(row), KPC(table_data_desc_)); } else { - ObTableLoadSequenceNo seq_no(0); // pdml导入的行目前不存在主键冲突,先都用一个默认的seq_no + static const ObTableLoadSequenceNo seq_no(0); // pdml导入的行目前不存在主键冲突,先都用一个默认的seq_no SessionContext &session_ctx = session_ctx_array_[0]; - if (OB_FAIL(write_row_to_table_store(session_ctx.table_store_, - tablet_id, - seq_no, - row))) { - LOG_WARN("fail to write row", KR(ret), K(tablet_id), K(row)); + if (OB_FAIL(session_ctx.writer_->append_row(tablet_id, seq_no, row))) { + LOG_WARN("fail to append row", KR(ret), K(tablet_id), K(row)); } else { ATOMIC_AAF(&trans_ctx_->ctx_->job_stat_->store_.processed_rows_, 1); } @@ -348,10 +890,30 @@ int ObTableLoadTransStoreWriter::px_write(const ObTabletID &tablet_id, const blo return ret; } +int ObTableLoadTransStoreWriter::px_write(ObIVector *tablet_id_vector, + const ObIArray &vectors, + const ObBatchRows &batch_rows, + int64_t &affected_rows) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadTransStoreWriter not init", KR(ret)); + } else { + SessionContext &session_ctx = session_ctx_array_[0]; + if (OB_FAIL(session_ctx.writer_->append_batch(tablet_id_vector, vectors, batch_rows, affected_rows))) { + LOG_WARN("fail to append batch", KR(ret)); + } else { + ATOMIC_AAF(&trans_ctx_->ctx_->job_stat_->store_.processed_rows_, affected_rows); + } + } + return ret; +} + int ObTableLoadTransStoreWriter::flush(int32_t session_id) { int ret = OB_SUCCESS; - int32_t session_count = param_.px_mode_? 1 : param_.write_session_count_; + int32_t session_count = param_.px_mode_ ? 1 : param_.write_session_count_; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObTableLoadTransStoreWriter not init", KR(ret)); @@ -360,15 +922,8 @@ int ObTableLoadTransStoreWriter::flush(int32_t session_id) LOG_WARN("invalid args", KR(ret), K(session_id)); } else { SessionContext &session_ctx = session_ctx_array_[session_id - 1]; - ObTableLoadTransStore::SessionStore *session_store = - trans_store_->session_store_array_.at(session_id - 1); - if (OB_FAIL(session_ctx.table_store_.close())) { - LOG_WARN("fail to close table store", KR(ret), K(session_id)); - } else if (OB_FAIL(session_ctx.table_store_.get_tables(session_store->partition_table_array_, - session_store->allocator_))) { - LOG_WARN("fail to get tables", KR(ret)); - } else { - session_ctx.table_store_.clean_up(); + if (OB_FAIL(session_ctx.writer_->close())) { + LOG_WARN("fail to close writer", KR(ret), K(session_id)); } } return ret; @@ -377,7 +932,7 @@ int ObTableLoadTransStoreWriter::flush(int32_t session_id) int ObTableLoadTransStoreWriter::clean_up(int32_t session_id) { int ret = OB_SUCCESS; - int32_t session_count = param_.px_mode_? 1 : param_.write_session_count_; + int32_t session_count = param_.px_mode_ ? 1 : param_.write_session_count_; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObTableLoadTransStoreWriter not init", KR(ret)); @@ -386,7 +941,7 @@ int ObTableLoadTransStoreWriter::clean_up(int32_t session_id) LOG_WARN("invalid args", KR(ret), K(session_id)); } else { SessionContext &session_ctx = session_ctx_array_[session_id - 1]; - session_ctx.table_store_.clean_up(); + session_ctx.writer_->reset(); } return ret; } @@ -535,31 +1090,5 @@ int ObTableLoadTransStoreWriter::handle_identity_column(const ObColumnSchemaV2 * return ret; } -int ObTableLoadTransStoreWriter::write_row_to_table_store(ObDirectLoadTableStore &table_store, - const ObTabletID &tablet_id, - const ObTableLoadSequenceNo &seq_no, - const ObDatumRow &datum_row) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(table_store.append_row(tablet_id, seq_no, datum_row))) { - LOG_WARN("fail to append row", KR(ret), K(datum_row)); - } - if (OB_FAIL(ret)) { - ObTableLoadErrorRowHandler *error_row_handler = - trans_ctx_->ctx_->store_ctx_->error_row_handler_; - ObDirectLoadDMLRowHandler *data_row_handler = trans_ctx_->ctx_->store_ctx_->data_store_table_ctx_->row_handler_; - if (OB_LIKELY(OB_ERR_PRIMARY_KEY_DUPLICATE == ret)) { - if (OB_FAIL(data_row_handler->handle_update_row(datum_row))) { - LOG_WARN("fail to handle update row", KR(ret), K(datum_row)); - } - } else if (OB_LIKELY(OB_ROWKEY_ORDER_ERROR == ret)) { - if (OB_FAIL(error_row_handler->handle_error_row(ret))) { - LOG_WARN("fail to handle error row", KR(ret), K(tablet_id), K(datum_row)); - } - } - } - return ret; -} - } // namespace observer } // namespace oceanbase diff --git a/src/observer/table_load/ob_table_load_trans_store.h b/src/observer/table_load/ob_table_load_trans_store.h index 2f6343f0c3..8f2bcd54ef 100644 --- a/src/observer/table_load/ob_table_load_trans_store.h +++ b/src/observer/table_load/ob_table_load_trans_store.h @@ -22,13 +22,15 @@ #include "share/schema/ob_column_schema.h" #include "share/table/ob_table_load_row_array.h" #include "storage/blocksstable/ob_datum_row.h" -#include "storage/direct_load/ob_direct_load_table_store.h" namespace oceanbase { namespace storage { class ObDirectLoadTableDataDesc; +class ObIDirectLoadPartitionTable; +class ObIDirectLoadPartitionTableBuilder; +class ObDirectLoadInsertTableBatchRowDirectWriter; } // namespace storage namespace observer { @@ -75,8 +77,11 @@ public: // 只在对应工作线程中调用, 串行执行 int write(int32_t session_id, const table::ObTableLoadTabletObjRowArray &row_array); int px_write(const ObTabletID &tablet_id, const blocksstable::ObDatumRow &row); - int cast_row(int32_t session_id, - const ObNewRow &new_row, + int px_write(common::ObIVector *tablet_id_vector, + const ObIArray &vectors, + const sql::ObBatchRows &batch_rows, + int64_t &affected_rows); + int cast_row(int32_t session_id, const ObNewRow &new_row, const blocksstable::ObDatumRow *&datum_row); int flush(int32_t session_id); int clean_up(int32_t session_id); @@ -104,10 +109,98 @@ private: const common::ObObj &obj, common::ObObj &out_obj, common::ObArenaAllocator &cast_allocator); - int write_row_to_table_store(storage::ObDirectLoadTableStore &table_store, - const common::ObTabletID &tablet_id, - const table::ObTableLoadSequenceNo &seq_no, - const blocksstable::ObDatumRow &datum_row); + +private: + class IWriter + { + public: + IWriter() = default; + virtual ~IWriter() = default; + virtual void reset() = 0; + virtual int append_row(const common::ObTabletID &tablet_id, + const table::ObTableLoadSequenceNo &seq_no, + const blocksstable::ObDatumRow &datum_row) = 0; + virtual int append_batch(common::ObIVector *tablet_id_vector, + const ObIArray &vectors, + const sql::ObBatchRows &batch_rows, + int64_t &affected_rows) = 0; + virtual int close() = 0; + }; + + class StoreWriter : public IWriter + { + public: + StoreWriter(); + virtual ~StoreWriter(); + void reset() override; + int init(ObTableLoadStoreCtx *store_ctx, + ObTableLoadTransStore *trans_store, + int32_t session_id); + int append_row(const common::ObTabletID &tablet_id, + const table::ObTableLoadSequenceNo &seq_no, + const blocksstable::ObDatumRow &datum_row) override; + int append_batch(common::ObIVector *tablet_id_vector, + const ObIArray &vectors, + const sql::ObBatchRows &batch_rows, + int64_t &affected_rows) override; + int close() override; + private: + int new_table_builder(const common::ObTabletID &tablet_id, + ObIDirectLoadPartitionTableBuilder *&table_builder); + int get_table_builder(const common::ObTabletID &tablet_id, + ObIDirectLoadPartitionTableBuilder *&table_builder); + int inner_append_row(const common::ObTabletID &tablet_id, + const table::ObTableLoadSequenceNo &seq_no, + const blocksstable::ObDatumRow &datum_row); + private: + typedef common::hash::ObHashMap + TableBuilderMap; + ObTableLoadStoreCtx *store_ctx_; + ObTableLoadTransStore *trans_store_; + int32_t session_id_; + ObArenaAllocator allocator_; + TableBuilderMap table_builder_map_; + ObSEArray table_builders_; + blocksstable::ObDatumRow datum_row_; + bool is_single_part_; + bool is_closed_; + bool is_inited_; + }; + + class DirectWriter : public IWriter + { + public: + DirectWriter(); + virtual ~DirectWriter(); + void reset() override; + int init(ObTableLoadStoreCtx *store_ctx); + int append_row(const common::ObTabletID &tablet_id, + const table::ObTableLoadSequenceNo &seq_no, + const blocksstable::ObDatumRow &datum_row) override; + int append_batch(common::ObIVector *tablet_id_vector, + const ObIArray &vectors, + const sql::ObBatchRows &batch_rows, + int64_t &affected_rows) override; + int close() override; + private: + int new_batch_writer(const common::ObTabletID &tablet_id, + ObDirectLoadInsertTableBatchRowDirectWriter *&batch_writer); + int get_batch_writer(const common::ObTabletID &tablet_id, + ObDirectLoadInsertTableBatchRowDirectWriter *&batch_writer); + private: + typedef common::hash::ObHashMap + BatchWriterMap; + ObTableLoadStoreCtx *store_ctx_; + ObArenaAllocator allocator_; + ObArenaAllocator lob_allocator_; + BatchWriterMap batch_writer_map_; + ObSEArray batch_writers_; + int64_t max_batch_size_; + bool is_single_part_; + bool is_closed_; + bool is_inited_; + }; + private: ObTableLoadTransStore *const trans_store_; ObTableLoadTransCtx *const trans_ctx_; @@ -129,10 +222,8 @@ private: blocksstable::ObDatumRow datum_row_; common::ObArenaAllocator cast_allocator_; ObDataTypeCastParams cast_params_; - storage::ObDirectLoadTableStore table_store_; + IWriter *writer_; uint64_t last_receive_sequence_no_; - char *extra_buf_; - int64_t extra_buf_size_; }; SessionContext *session_ctx_array_; int64_t lob_inrow_threshold_; // for incremental direct load diff --git a/src/share/datum/ob_datum.h b/src/share/datum/ob_datum.h index a1f90f43b7..f177c7292a 100644 --- a/src/share/datum/ob_datum.h +++ b/src/share/datum/ob_datum.h @@ -462,6 +462,7 @@ struct ObDatumVector { ObDatum *at(const int64_t i) const { return datums_ + (mask_ & i); } void set_batch(const bool is) { mask_ = is ? UINT64_MAX : 0; } + bool is_batch() const { return UINT64_MAX == mask_; } TO_STRING_KV(KP(datums_), K(mask_)); ObDatum *datums_ = nullptr; diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt index 307eb3d75a..4f0d161e46 100644 --- a/src/sql/CMakeLists.txt +++ b/src/sql/CMakeLists.txt @@ -884,6 +884,10 @@ ob_set_subtarget(ob_sql engine_pdml engine/pdml/static/ob_px_sstable_insert_op.cpp ) +ob_set_subtarget(ob_sql engine_direct_load + engine/direct_load/ob_table_direct_insert_op.cpp +) + ob_set_subtarget(ob_sql engine_px engine/px/ob_dfo.cpp engine/px/ob_dfo_mgr.cpp diff --git a/src/sql/code_generator/ob_static_engine_cg.cpp b/src/sql/code_generator/ob_static_engine_cg.cpp index 4c765d5440..1e442aed51 100644 --- a/src/sql/code_generator/ob_static_engine_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_cg.cpp @@ -178,6 +178,7 @@ #include "sql/optimizer/ob_log_values_table_access.h" #include "sql/engine/basic/ob_values_table_access_op.h" #include "sql/engine/cmd/ob_table_direct_insert_service.h" +#include "sql/engine/direct_load/ob_table_direct_insert_op.h" namespace oceanbase { @@ -7280,31 +7281,6 @@ int ObStaticEngineCG::generate_spec(ObLogInsert &op, spec.is_pdml_index_maintain_ = op.is_index_maintenance(); spec.table_location_uncertain_ = op.is_table_location_uncertain(); // row-movement target table spec.is_pdml_update_split_ = op.is_pdml_update_split(); - ObDirectLoadOptimizerCtx &direct_load_optimizer_ctx = op.get_plan()->get_optimizer_context().get_direct_load_optimizer_ctx(); - spec.plan_->set_append_table_id(op.get_append_table_id()); - spec.plan_->set_enable_append(direct_load_optimizer_ctx.use_direct_load()); - spec.plan_->set_enable_inc_direct_load(ObDirectLoadMethod::is_incremental(direct_load_optimizer_ctx.load_method_)); - spec.plan_->set_enable_replace(direct_load_optimizer_ctx.insert_mode_ == ObDirectLoadInsertMode::INC_REPLACE); - spec.plan_->set_online_sample_percent(op.get_plan()->get_optimizer_context() - .get_exec_ctx()->get_table_direct_insert_ctx() - .get_online_sample_percent()); - spec.plan_->set_direct_load_need_sort(direct_load_optimizer_ctx.need_sort_); - // check is insert overwrite - bool is_insert_overwrite = false; - ObExecContext *exec_ctx = NULL; - ObPhysicalPlanCtx *plan_ctx = NULL; - if (OB_FAIL(check_is_insert_overwrite_stmt(log_plan, is_insert_overwrite))) { - LOG_WARN("check is insert overwrite failed", K(ret)); - } else if (OB_FALSE_IT(spec.plan_->set_is_insert_overwrite(is_insert_overwrite))) { - } else if (OB_ISNULL(exec_ctx = log_plan->get_optimizer_context().get_exec_ctx())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexcepted null exec ctx", KR(ret), KP(exec_ctx)); - } else if (OB_ISNULL(plan_ctx = exec_ctx->get_physical_plan_ctx())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null plan ctx", KR(ret), KP(plan_ctx)); - } else { - plan_ctx->set_is_direct_insert_plan(direct_load_optimizer_ctx.use_direct_load()); - } int64_t partition_expr_idx = OB_INVALID_INDEX; if (OB_FAIL(ret)) { // do nothing @@ -9672,7 +9648,10 @@ int ObStaticEngineCG::get_phy_op_type(ObLogicalOperator &log_op, } else if (op.get_insert_up()) { type = PHY_INSERT_ON_DUP; } else if (op.is_pdml()) { - if (op.get_plan()->get_optimizer_context().get_session_info()->get_ddl_info().is_ddl()) { + ObDirectLoadOptimizerCtx &direct_load_optimizer_ctx = op.get_plan()->get_optimizer_context().get_direct_load_optimizer_ctx(); + if (direct_load_optimizer_ctx.use_direct_load()) { + type = PHY_TABLE_DIRECT_INSERT; + } else if (op.get_plan()->get_optimizer_context().get_session_info()->get_ddl_info().is_ddl()) { type = PHY_PX_MULTI_PART_SSTABLE_INSERT; } else { type = PHY_PX_MULTI_PART_INSERT; @@ -10495,5 +10474,68 @@ int ObStaticEngineCG::generate_spec(ObLogExpand &op, ObExpandVecSpec &spec, cons return ret; } +int ObStaticEngineCG::generate_spec(ObLogInsert &op, + ObTableDirectInsertSpec &spec, + const bool in_root_job) +{ + int ret = OB_SUCCESS; + UNUSED(in_root_job); + const ObLogPlan *log_plan = op.get_plan(); + if (OB_UNLIKELY(op.get_index_dml_infos().count() != 1) || + OB_ISNULL(op.get_index_dml_infos().at(0)) || + OB_ISNULL(log_plan)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("index dml info is invalid", K(ret), K(op.get_index_dml_infos().count()), + KP(op.get_index_dml_infos().at(0)), KP(log_plan)); + } else { + const IndexDMLInfo &index_dml_info = *op.get_index_dml_infos().at(0); + phy_plan_->set_use_pdml(true); + spec.is_returning_ = op.pdml_is_returning(); + spec.is_pdml_index_maintain_ = op.is_index_maintenance(); + spec.table_location_uncertain_ = op.is_table_location_uncertain(); // row-movement target table + spec.is_pdml_update_split_ = op.is_pdml_update_split(); + ObDirectLoadOptimizerCtx &direct_load_optimizer_ctx = op.get_plan()->get_optimizer_context().get_direct_load_optimizer_ctx(); + spec.plan_->set_append_table_id(op.get_append_table_id()); + spec.plan_->set_enable_append(direct_load_optimizer_ctx.use_direct_load()); + spec.plan_->set_enable_inc_direct_load(ObDirectLoadMethod::is_incremental(direct_load_optimizer_ctx.load_method_)); + spec.plan_->set_enable_replace(direct_load_optimizer_ctx.insert_mode_ == ObDirectLoadInsertMode::INC_REPLACE); + spec.plan_->set_online_sample_percent(op.get_plan()->get_optimizer_context() + .get_exec_ctx()->get_table_direct_insert_ctx() + .get_online_sample_percent()); + spec.plan_->set_direct_load_need_sort(direct_load_optimizer_ctx.need_sort_); + // check is insert overwrite + bool is_insert_overwrite = false; + ObExecContext *exec_ctx = NULL; + ObPhysicalPlanCtx *plan_ctx = NULL; + if (OB_FAIL(check_is_insert_overwrite_stmt(log_plan, is_insert_overwrite))) { + LOG_WARN("check is insert overwrite failed", K(ret)); + } else if (OB_FALSE_IT(spec.plan_->set_is_insert_overwrite(is_insert_overwrite))) { + } else if (OB_ISNULL(exec_ctx = log_plan->get_optimizer_context().get_exec_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexcepted null exec ctx", KR(ret), KP(exec_ctx)); + } else if (OB_ISNULL(plan_ctx = exec_ctx->get_physical_plan_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null plan ctx", KR(ret), KP(plan_ctx)); + } else { + plan_ctx->set_is_direct_insert_plan(direct_load_optimizer_ctx.use_direct_load()); + } + int64_t partition_expr_idx = OB_INVALID_INDEX; + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(get_pdml_partition_id_column_idx(spec.get_child(0)->output_, partition_expr_idx))) { + LOG_WARN("failed to get partition id column idx", K(ret)); + } else { + spec.row_desc_.set_part_id_index(partition_expr_idx); + } + LOG_TRACE("pdml static cg information", K(ret), K(partition_expr_idx), K(index_dml_info)); + // 处理pdml-insert中的insert_row_exprs + OZ(dml_cg_service_.generate_insert_ctdef(op, index_dml_info, spec.ins_ctdef_)); + // table columns exprs in dml need to set IS_COLUMNLIZED flag + OZ(mark_expr_self_produced(index_dml_info.column_exprs_)); + OZ(mark_expr_self_produced(index_dml_info.column_convert_exprs_)); + } + return ret; +} + } // end namespace sql } // end namespace oceanbase diff --git a/src/sql/code_generator/ob_static_engine_cg.h b/src/sql/code_generator/ob_static_engine_cg.h index e8dbb35807..008dea18bf 100644 --- a/src/sql/code_generator/ob_static_engine_cg.h +++ b/src/sql/code_generator/ob_static_engine_cg.h @@ -150,6 +150,8 @@ class HashRollupRTInfo; class ObMergeGroupByVecSpec; class ObNestedLoopJoinVecSpec; +class ObTableDirectInsertSpec; + typedef common::ObList DASTableIdList; typedef common::ObSEArray, 1, common::ModulePageAllocator, true> RowParamMap; @@ -474,6 +476,9 @@ private: int generate_spec(ObLogExpand &op, ObExpandVecSpec &spec, const bool in_root_job); template int generate_merge_distinct_spec(ObLogDistinct &op, MergeDistinctSpecType &spec, const bool in_root_job); + + // direct load + int generate_spec(ObLogInsert &op, ObTableDirectInsertSpec &spec, const bool in_root_job); private: int disable_use_rich_format(const ObLogicalOperator &op, ObOpSpec &spec); int add_update_set(ObSubPlanFilterSpec &spec); diff --git a/src/sql/das/ob_das_insert_op.cpp b/src/sql/das/ob_das_insert_op.cpp index 5ca902be07..01d15ba0b2 100644 --- a/src/sql/das/ob_das_insert_op.cpp +++ b/src/sql/das/ob_das_insert_op.cpp @@ -52,9 +52,6 @@ int ObDASIndexDMLAdaptor::write_rows(cons { int ret = OB_SUCCESS; ObAccessService *as = MTL(ObAccessService *); - dml_param_.direct_insert_task_id_ = rtdef.direct_insert_task_id_; - dml_param_.ddl_task_id_ = rtdef.ddl_task_id_; - if (ctdef.table_param_.get_data_table().is_mlog_table() && !ctdef.is_access_mlog_as_master_table_) { ObDASMLogDMLIterator mlog_iter(ls_id, tablet_id, dml_param_, &iter, DAS_OP_TABLE_INSERT); diff --git a/src/sql/das/ob_das_utils.cpp b/src/sql/das/ob_das_utils.cpp index 5f084c930b..ce461af6dd 100644 --- a/src/sql/das/ob_das_utils.cpp +++ b/src/sql/das/ob_das_utils.cpp @@ -347,6 +347,414 @@ int ObDASUtils::reshape_datum_value(const ObObjMeta &col_type, return ret; } +static bool fast_check_vector_is_all_null(ObIVector *vector, const int64_t batch_size) +{ + bool is_all_null = false; + VectorFormat format = vector->get_format(); + switch (format) { + case VEC_FIXED: + case VEC_DISCRETE: + case VEC_CONTINUOUS: + is_all_null = static_cast(vector)->get_nulls()->is_all_true(batch_size); + break; + default: + break; + } + return is_all_null; +} + +static int new_discrete_vector(VecValueTypeClass value_tc, + const int64_t max_batch_size, + ObIAllocator &allocator, + ObDiscreteBase *&result_vec) +{ + int ret = OB_SUCCESS; + result_vec = nullptr; + ObIVector *vector = nullptr; + switch (value_tc) { +#define DISCRETE_VECTOR_INIT_SWITCH(value_tc) \ + case value_tc: { \ + using VecType = RTVectorType; \ + static_assert(sizeof(VecType) <= ObIVector::MAX_VECTOR_STRUCT_SIZE, \ + "vector size exceeds MAX_VECTOR_STRUCT_SIZE"); \ + vector = OB_NEWx(VecType, &allocator, nullptr, nullptr, nullptr); \ + break; \ + } + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_NUMBER); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_EXTEND); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_STRING); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_ENUM_SET_INNER); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_RAW); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_ROWID); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_LOB); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_JSON); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_GEO); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_UDT); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_COLLECTION); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_ROARINGBITMAP); +#undef DISCRETE_VECTOR_INIT_SWITCH + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected discrete vector value type class", KR(ret), K(value_tc)); + break; + } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(vector)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc vecttor", KR(ret)); + } else { + ObDiscreteBase *discrete_vec = static_cast(vector); + const int64_t nulls_size = ObBitVector::memory_size(max_batch_size); + const int64_t lens_size = sizeof(int32_t) * max_batch_size; + const int64_t ptrs_size = sizeof(char *) * max_batch_size; + ObBitVector *nulls = nullptr; + int32_t *lens = nullptr; + char **ptrs = nullptr; + if (OB_ISNULL(nulls = to_bit_vector(allocator.alloc(nulls_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", KR(ret), K(nulls_size)); + } else if (OB_ISNULL(lens = static_cast(allocator.alloc(lens_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", KR(ret), K(lens_size)); + } else if (OB_ISNULL(ptrs = static_cast(allocator.alloc(ptrs_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", KR(ret), K(ptrs_size)); + } else { + nulls->reset(max_batch_size); + discrete_vec->set_nulls(nulls); + discrete_vec->set_lens(lens); + discrete_vec->set_ptrs(ptrs); + result_vec = discrete_vec; + } + } + return ret; +} + +int ObDASUtils::reshape_vector_value(const ObObjMeta &col_type, + const ObAccuracy &col_accuracy, + ObIAllocator &allocator, + ObIVector *&vector, + const int64_t size) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(vector) || size <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(vector), K(size)); + } else if (fast_check_vector_is_all_null(vector, size)) { + // do nothing + } else if (col_type.is_binary()) { + const int32_t binary_len = col_accuracy.get_length(); + const char pad_char = '\0'; + const VectorFormat format = vector->get_format(); + switch (format) { + case VEC_CONTINUOUS: + { + ObContinuousBase *continuous_vec = static_cast(vector); + ObDiscreteBase *discrete_vec = nullptr; + char *data = continuous_vec->get_data(); + uint32_t *offsets = continuous_vec->get_offsets(); + char **ptrs = nullptr; + ObLength *lens = nullptr; + bool has_value_change = false; + VecValueTypeClass value_tc = get_vec_value_tc(col_type.get_type(), + col_type.get_scale(), + col_type.get_stored_precision()); + if (OB_FAIL(new_discrete_vector(value_tc, size, allocator, discrete_vec))) { + LOG_WARN("fail to new discrete vector", KR(ret)); + } else { + ptrs = discrete_vec->get_ptrs(); + lens = discrete_vec->get_lens(); + } + for (int64_t i = 0; OB_SUCC(ret) && i < size; ++i) { + if (continuous_vec->is_null(i)) { + discrete_vec->set_null(i); + } else { + const ObLength len = offsets[i + 1] - offsets[i]; + char *str = data + offsets[i]; + if (len < binary_len) { + char *dest_str = nullptr; + if (OB_ISNULL(dest_str = (char *)(allocator.alloc(binary_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem to binary", K(ret), K(binary_len)); + } else { + MEMCPY(dest_str, str, len); + MEMSET(dest_str + len, pad_char, binary_len - len); + ptrs[i] = dest_str; + lens[i] = binary_len; + has_value_change = true; + } + } else { + ptrs[i] = str; + lens[i] = binary_len; + } + } + } + if (OB_SUCC(ret) && has_value_change) { + vector = discrete_vec; + } + break; + } + case VEC_DISCRETE: + { + ObDiscreteBase *discrete_vec = static_cast(vector); + char **ptrs = discrete_vec->get_ptrs(); + ObLength *lens =discrete_vec->get_lens(); + for (int64_t i = 0; OB_SUCC(ret) && i < size; ++i) { + if (!discrete_vec->is_null(i) && lens[i] < binary_len) { + char *str = ptrs[i]; + ObLength len = lens[i]; + char *dest_str = nullptr; + if (OB_ISNULL(dest_str = (char *)(allocator.alloc(binary_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem to binary", K(ret), K(binary_len)); + } else { + MEMCPY(dest_str, str, len); + MEMSET(dest_str + len, pad_char, binary_len - len); + ptrs[i] = dest_str; + lens[i] = binary_len; + } + } + } + break; + } + case VEC_UNIFORM: + { + ObUniformBase *uniform_vec = static_cast(vector); + ObDatum *datums = uniform_vec->get_datums(); + for (int64_t i = 0; OB_SUCC(ret) && i < size; ++i) { + ObDatum &datum = datums[i]; + if (!datum.is_null() && datum.len_ < binary_len) { + const char *str = datum.ptr_; + ObLength len = datum.len_; + char *dest_str = nullptr; + if (OB_ISNULL(dest_str = (char *)(allocator.alloc(binary_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem to binary", K(ret), K(binary_len)); + } else { + MEMCPY(dest_str, str, len); + MEMSET(dest_str + len, pad_char, binary_len - len); + datum.ptr_ = dest_str; + datum.len_ = binary_len; + } + } + } + break; + } + case VEC_UNIFORM_CONST: + { + ObUniformBase *uniform_vec = static_cast(vector); + ObDatum &datum = uniform_vec->get_datums()[0]; + if (!datum.is_null() && datum.len_ < binary_len) { + const char *str = datum.ptr_; + ObLength len = datum.len_; + char *dest_str = nullptr; + if (OB_ISNULL(dest_str = (char *)(allocator.alloc(binary_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem to binary", K(ret), K(binary_len)); + } else { + MEMCPY(dest_str, str, len); + MEMSET(dest_str + len, pad_char, binary_len - len); + datum.ptr_ = dest_str; + datum.len_ = binary_len; + } + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected binary vector format", KR(ret), K(format), K(col_type)); + break; + } + } else if (col_type.is_fixed_len_char_type()) { + const ObString space_pattern = ObCharsetUtils::get_const_str(col_type.get_collation_type(), ' '); + const VectorFormat format = vector->get_format(); + switch (format) { + case VEC_CONTINUOUS: + { + ObContinuousBase *continuous_vec = static_cast(vector); + ObDiscreteBase *discrete_vec = nullptr; + char *data = continuous_vec->get_data(); + uint32_t *offsets = continuous_vec->get_offsets(); + char **ptrs = nullptr; + ObLength *lens = nullptr; + bool has_value_change = false; + VecValueTypeClass value_tc = get_vec_value_tc(col_type.get_type(), + col_type.get_scale(), + col_type.get_stored_precision()); + if (OB_FAIL(new_discrete_vector(value_tc, size, allocator, discrete_vec))) { + LOG_WARN("fail to new discrete vector", KR(ret)); + } else { + ptrs = discrete_vec->get_ptrs(); + lens = discrete_vec->get_lens(); + } + for (int64_t i = 0; OB_SUCC(ret) && i < size; ++i) { + if (continuous_vec->is_null(i)) { + discrete_vec->set_null(i); + } else { + const ObLength length = offsets[i + 1] - offsets[i]; + if (lib::is_oracle_mode() && 0 == length) { + // Oracle compatibility mode: '' as null + LOG_DEBUG("reshape empty string to null", K(i)); + continuous_vec->set_null(i); + discrete_vec->set_null(i); + } else { + ObLength len = length; + char *str = data + offsets[i]; + for (; len >= space_pattern.length(); len -= space_pattern.length()) { + if (0 != MEMCMP(str + len - space_pattern.length(), space_pattern.ptr(), space_pattern.length())) { + break; + } + } + ptrs[i] = str; + lens[i] = len; + if (len != length) { + has_value_change = true; + } + } + } + } + if (OB_SUCC(ret) && has_value_change) { + vector = discrete_vec; + } + break; + } + case VEC_DISCRETE: + { + ObDiscreteBase *discrete_vec = static_cast(vector); + char **ptrs = discrete_vec->get_ptrs(); + ObLength *lens =discrete_vec->get_lens(); + for (int64_t i = 0; OB_SUCC(ret) && i < size; ++i) { + if (!discrete_vec->is_null(i)) { + ObLength len = lens[i]; + if (lib::is_oracle_mode() && 0 == len) { + // Oracle compatibility mode: '' as null + LOG_DEBUG("reshape empty string to null", K(i)); + discrete_vec->set_null(i); + } else { + const char *str = ptrs[i]; + for (; len >= space_pattern.length(); len -= space_pattern.length()) { + if (0 != MEMCMP(str + len - space_pattern.length(), space_pattern.ptr(), space_pattern.length())) { + break; + } + } + lens[i] = len; + } + } + } + break; + } + case VEC_UNIFORM: + { + ObUniformBase *uniform_vec = static_cast(vector); + ObDatum *datums = uniform_vec->get_datums(); + for (int64_t i = 0; OB_SUCC(ret) && i < size; ++i) { + ObDatum &datum = datums[i]; + if (!datum.is_null()) { + ObLength len = datum.len_; + if (lib::is_oracle_mode() && 0 == len) { + // Oracle compatibility mode: '' as null + LOG_DEBUG("reshape empty string to null", K(i)); + datum.set_null(); + } else { + const char *str = datum.ptr_; + for (; len >= space_pattern.length(); len -= space_pattern.length()) { + if (0 != MEMCMP(str + len - space_pattern.length(), space_pattern.ptr(), space_pattern.length())) { + break; + } + } + datum.len_ = len; + } + } + } + break; + } + case VEC_UNIFORM_CONST: + { + ObUniformBase *uniform_vec = static_cast(vector); + ObDatum &datum = uniform_vec->get_datums()[0]; + if (!datum.is_null()) { + ObLength len = datum.len_; + if (lib::is_oracle_mode() && 0 == len) { + // Oracle compatibility mode: '' as null + LOG_DEBUG("reshape empty string to null"); + datum.set_null(); + } else { + const char *str = datum.ptr_; + for (; len >= space_pattern.length(); len -= space_pattern.length()) { + if (0 != MEMCMP(str + len - space_pattern.length(), space_pattern.ptr(), space_pattern.length())) { + break; + } + } + datum.len_ = len; + } + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected fixed len char vector format", KR(ret), K(format), K(col_type)); + break; + } + } else if (lib::is_oracle_mode() && col_type.is_character_type()) { + // Oracle compatibility mode: '' as null + const VectorFormat format = vector->get_format(); + switch (format) { + case VEC_CONTINUOUS: + { + ObContinuousBase *continuous_vec = static_cast(vector); + uint32_t *offsets = continuous_vec->get_offsets(); + for (int64_t i = 0; i < size; ++i) { + if (!continuous_vec->is_null(i) && offsets[i + 1] == offsets[i]) { + LOG_DEBUG("reshape empty string to null", K(i)); + continuous_vec->set_null(i); + } + } + break; + } + case VEC_DISCRETE: + { + ObDiscreteBase *discrete_vec = static_cast(vector); + ObLength *lens =discrete_vec->get_lens(); + for (int64_t i = 0; i < size; ++i) { + if (!discrete_vec->is_null(i) && 0 == lens[i]) { + LOG_DEBUG("reshape empty string to null", K(i)); + discrete_vec->set_null(i); + } + } + break; + } + case VEC_UNIFORM: + { + ObUniformBase *uniform_vec = static_cast(vector); + ObDatum *datums = uniform_vec->get_datums(); + for (int64_t i = 0; i < size; ++i) { + ObDatum &datum = datums[i]; + if (!datum.is_null() && 0 == datum.len_) { + LOG_DEBUG("reshape empty string to null", K(i)); + datum.set_null(); + } + } + break; + } + case VEC_UNIFORM_CONST: + { + ObUniformBase *uniform_vec = static_cast(vector); + ObDatum &datum = uniform_vec->get_datums()[0]; + if (!datum.is_null() && 0 == datum.len_) { + LOG_DEBUG("reshape empty string to null"); + datum.set_null(); + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected character vector format", KR(ret), K(format), K(col_type)); + break; + } + } + return ret; +} + int ObDASUtils::wait_das_retry(int64_t retry_cnt) { int ret = OB_SUCCESS; diff --git a/src/sql/das/ob_das_utils.h b/src/sql/das/ob_das_utils.h index 067149b242..b9e05ea4a3 100644 --- a/src/sql/das/ob_das_utils.h +++ b/src/sql/das/ob_das_utils.h @@ -63,6 +63,11 @@ public: const bool enable_oracle_empty_char_reshape_to_null, ObIAllocator &allocator, blocksstable::ObStorageDatum &datum_value); + static int reshape_vector_value(const ObObjMeta &col_type, + const ObAccuracy &col_accuracy, + ObIAllocator &allocator, + common::ObIVector *&vector, + const int64_t size); static int padding_fixed_string_value(int64_t max_len, ObIAllocator &alloc, ObObj &value); static int wait_das_retry(int64_t retry_cnt); static int find_child_das_def(const ObDASBaseCtDef *root_ctdef, diff --git a/src/sql/engine/basic/ob_temp_column_store.cpp b/src/sql/engine/basic/ob_temp_column_store.cpp index a4947aa3a7..6dcd6e3e14 100644 --- a/src/sql/engine/basic/ob_temp_column_store.cpp +++ b/src/sql/engine/basic/ob_temp_column_store.cpp @@ -20,6 +20,7 @@ #include "share/vector/ob_discrete_vector.h" #include "share/ob_define.h" #include "sql/engine/expr/ob_array_expr_utils.h" +#include "storage/ddl/ob_direct_load_struct.h" namespace oceanbase { @@ -28,6 +29,111 @@ using namespace common; namespace sql { +int ObTempColumnStore::ColumnBlock::calc_vector_size(const ObIVector *vec, + const uint16_t *selector, + const ObLength length, + const int64_t size, + int64_t &batch_mem_size) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == vec)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(vec)); + } else { + const VectorFormat format = vec->get_format(); + switch (format) { + case VEC_FIXED: + batch_mem_size += calc_size(static_cast(vec), selector, size); + break; + case VEC_DISCRETE: + batch_mem_size += calc_size(static_cast(vec), selector, size); + break; + case VEC_CONTINUOUS: + batch_mem_size += calc_size(static_cast(vec), selector, size); + break; + case VEC_UNIFORM: + batch_mem_size += calc_size(static_cast(vec), selector, size, length); + break; + case VEC_UNIFORM_CONST: + batch_mem_size += calc_size(static_cast(vec), selector, size, length); + break; + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", K(ret), K(format)); + break; + } + } + return ret; +} + +int ObTempColumnStore::ColumnBlock::vector_to_buf(const ObIVector *vec, + const uint16_t *selector, + const ObLength length, + const int64_t size, + char *head, + int64_t &pos) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == vec || nullptr == head)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(vec), KP(head)); + } else { + const VectorFormat format = vec->get_format(); + switch (format) { + case VEC_FIXED: + ret = to_buf(static_cast(vec), selector, size, head, pos); + break; + case VEC_DISCRETE: + ret = to_buf(static_cast(vec), selector, size, head, pos); + break; + case VEC_CONTINUOUS: + ret = to_buf(static_cast(vec), selector, size, head, pos); + break; + case VEC_UNIFORM: + ret = to_buf(static_cast(vec), selector, size, length, head, pos); + break; + case VEC_UNIFORM_CONST: + ret = to_buf(static_cast(vec), selector, size, length, head, pos); + break; + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", KR(ret), K(format)); + break; + } + } + return ret; +} + +int ObTempColumnStore::ColumnBlock::vector_from_buf(char *buf, + int64_t &pos, + const int64_t size, + ObIVector *vec) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == buf || nullptr == vec)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(buf), KP(vec)); + } else { + const VectorFormat format = vec->get_format(); + switch (format) { + case VEC_FIXED: + ret = from_buf(buf, pos, size, static_cast(vec)); + break; + case VEC_CONTINUOUS: + ret = from_buf(buf, pos, size, static_cast(vec)); + break; + case VEC_UNIFORM: + static_cast(vec)->set_all_null(size); + break; + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", KR(ret), K(format)); + break; + } + } + return ret; +} + int ObTempColumnStore::ColumnBlock::calc_nested_size(ObExpr &expr, ObEvalCtx &ctx, const uint16_t *selector, const ObArray &lengths, const int64_t size, int64_t &batch_mem_size) @@ -40,27 +146,9 @@ int ObTempColumnStore::ColumnBlock::calc_nested_size(ObExpr &expr, ObEvalCtx &ct } for (uint32_t i = 0; i < expr.attrs_cnt_ && OB_SUCC(ret); ++i) { ObIVector *vec = expr.attrs_[i]->get_vector(ctx); - const VectorFormat format = vec->get_format(); - switch (format) { - case VEC_FIXED: - batch_mem_size += calc_size(static_cast(vec), selector, size); - break; - case VEC_DISCRETE: - batch_mem_size += calc_size(static_cast(vec), selector, size); - break; - case VEC_CONTINUOUS: - batch_mem_size += calc_size(static_cast(vec), selector, size); - break; - case VEC_UNIFORM: - batch_mem_size += calc_size(static_cast(vec), selector, size, UNFIXED_LENGTH); - break; - case VEC_UNIFORM_CONST: - batch_mem_size += calc_size(static_cast(vec), selector, size, UNFIXED_LENGTH); - break; - default: - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected vector format", K(ret), K(size)); - } + if (OB_FAIL(calc_vector_size(vec, selector, UNFIXED_LENGTH, size, batch_mem_size))) { + LOG_WARN("fail to calc vector size", KR(ret)); + } } return ret; } @@ -87,26 +175,8 @@ int ObTempColumnStore::ColumnBlock::add_nested_batch(ObExpr &expr, ObEvalCtx &ct int ret = OB_SUCCESS; for (uint32_t i = 0; i < expr.attrs_cnt_ && OB_SUCC(ret); ++i) { ObIVector *vec = expr.attrs_[i]->get_vector(ctx); - const VectorFormat format = vec->get_format(); - switch (format) { - case VEC_FIXED: - ret = to_buf(static_cast(vec), selector, size, head, pos); - break; - case VEC_DISCRETE: - ret = to_buf(static_cast(vec), selector, size, head, pos); - break; - case VEC_CONTINUOUS: - ret = to_buf(static_cast(vec), selector, size, head, pos); - break; - case VEC_UNIFORM: - ret = to_buf(static_cast(vec), selector, size, UNFIXED_LENGTH, head, pos); - break; - case VEC_UNIFORM_CONST: - ret = to_buf(static_cast(vec), selector, size, UNFIXED_LENGTH, head, pos); - break; - default: - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected vector format", K(ret), K(format)); + if (OB_FAIL(vector_to_buf(vec, selector, UNFIXED_LENGTH, size, head, pos))) { + LOG_WARN("vector to buf failed", KR(ret)); } } return ret; @@ -124,38 +194,36 @@ int ObTempColumnStore::ColumnBlock::calc_rows_size(ObEvalCtx &ctx, batch_mem_size = get_header_size(vectors.count()); for (int64_t i = 0; OB_SUCC(ret) && i < vectors.count(); ++i) { const ObIVector *vec = vectors.at(i); - const VectorFormat format = vec->get_format(); if (exprs.at(i)->is_nested_expr()) { if (OB_FAIL(ColumnBlock::calc_nested_size(*exprs.at(i), ctx, selector, lengths, size, batch_mem_size))) { LOG_WARN("calc nested expr size failed", K(ret), K(size)); } } else { - switch (format) { - case VEC_FIXED: - batch_mem_size += calc_size(static_cast(vec), selector, size); - break; - case VEC_DISCRETE: - batch_mem_size += calc_size(static_cast(vec), selector, size); - break; - case VEC_CONTINUOUS: - batch_mem_size += calc_size(static_cast(vec), selector, size); - break; - case VEC_UNIFORM: - batch_mem_size += calc_size(static_cast(vec), - selector, size, lengths[i]); - break; - case VEC_UNIFORM_CONST: - batch_mem_size += calc_size(static_cast(vec), - selector, size, lengths[i]); - break; - default: - ret = OB_ERR_UNEXPECTED; + if (OB_FAIL(calc_vector_size(vec, selector, lengths[i], size, batch_mem_size))) { + LOG_WARN("fail to calc vector size", KR(ret)); } } } return ret; } +int ObTempColumnStore::ColumnBlock::calc_rows_size(const IVectorPtrs &vectors, + const uint16_t *selector, + const ObArray &lengths, + const int64_t size, + int64_t &batch_mem_size) +{ + int ret = OB_SUCCESS; + batch_mem_size = get_header_size(vectors.count()); + for (int64_t i = 0; OB_SUCC(ret) && i < vectors.count(); ++i) { + const ObIVector *vec = vectors.at(i); + if (OB_FAIL(calc_vector_size(vec, selector, lengths[i], size, batch_mem_size))) { + LOG_WARN("fail to calc vector size", KR(ret)); + } + } + return ret; +} + int ObTempColumnStore::ColumnBlock::add_batch(ObEvalCtx &ctx, const ObExprPtrIArray &exprs, ShrinkBuffer &buf, @@ -176,33 +244,14 @@ int ObTempColumnStore::ColumnBlock::add_batch(ObEvalCtx &ctx, int64_t pos = get_header_size(vectors.count()); for (int64_t i = 0; OB_SUCC(ret) && i < vectors.count(); ++i) { const ObIVector *vec = vectors.at(i); - const VectorFormat format = vec->get_format(); vec_offsets[i] = pos; if (exprs.at(i)->is_nested_expr()) { if (OB_FAIL(ColumnBlock::add_nested_batch(*exprs.at(i), ctx, selector, size, head, pos))) { LOG_WARN("calc nested expr size failed", K(ret), K(size)); } } else { - switch (format) { - case VEC_FIXED: - ret = to_buf(static_cast(vec), selector, size, head, pos); - break; - case VEC_DISCRETE: - ret = to_buf(static_cast(vec), selector, size, head, pos); - break; - case VEC_CONTINUOUS: - ret = to_buf(static_cast(vec), selector, size, head, pos); - break; - case VEC_UNIFORM: - ret = to_buf(static_cast(vec), selector, size, lengths[i], - head, pos); - break; - case VEC_UNIFORM_CONST: - ret = to_buf(static_cast(vec), selector, size, lengths[i], - head, pos); - break; - default: - ret = OB_ERR_UNEXPECTED; + if (OB_FAIL(vector_to_buf(vec, selector, lengths[i], size, head, pos))) { + LOG_WARN("vector to buf failed", KR(ret)); } } } @@ -219,25 +268,49 @@ int ObTempColumnStore::ColumnBlock::add_batch(ObEvalCtx &ctx, return ret; } +int ObTempColumnStore::ColumnBlock::add_batch(ShrinkBuffer &buf, + const IVectorPtrs &vectors, + const uint16_t *selector, + const ObArray &lengths, + const int64_t size, + const int64_t batch_mem_size) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(batch_mem_size > buf.remain())) { + ret = OB_BUF_NOT_ENOUGH; + LOG_WARN("block is not enough", K(ret), K(batch_mem_size), K(buf)); + } else { + char *head = buf.head(); + *reinterpret_cast(head) = static_cast(size); // row_count + int32_t *vec_offsets = reinterpret_cast(head + sizeof(int32_t)); + int64_t pos = get_header_size(vectors.count()); + for (int64_t i = 0; OB_SUCC(ret) && i < vectors.count(); ++i) { + const ObIVector *vec = vectors.at(i); + vec_offsets[i] = pos; + if (OB_FAIL(vector_to_buf(vec, selector, lengths[i], size, head, pos))) { + LOG_WARN("vector to buf failed", KR(ret)); + } + } + vec_offsets[vectors.count()] = pos; // last offset, the size of vector + buf.fast_advance(pos); + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(pos != batch_mem_size)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected memory size", K(ret), K(pos), K(batch_mem_size)); + } else { + cnt_ += size; + } + } + return ret; +} + int ObTempColumnStore::ColumnBlock::get_nested_batch(ObExpr &expr, ObEvalCtx &ctx, char *buf, int64_t &pos, const int64_t size) const { int ret = OB_SUCCESS; for (uint32_t i = 0; i < expr.attrs_cnt_ && OB_SUCC(ret); ++i) { ObIVector *vec = expr.attrs_[i]->get_vector(ctx); - const VectorFormat format = vec->get_format(); - switch (format) { - case VEC_FIXED: - ret = from_buf(buf, pos, size, static_cast(vec)); - break; - case VEC_CONTINUOUS: - ret = from_buf(buf, pos, size, static_cast(vec)); - break; - case VEC_UNIFORM: - static_cast(vec)->set_all_null(size); - break; - default: - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected vector format", K(ret), K(format), K(i), K(expr.attrs_cnt_)); + if (OB_FAIL(vector_from_buf(buf, pos ,size, vec))) { + LOG_WARN("vector from buf failed", KR(ret)); } } @@ -265,20 +338,8 @@ int ObTempColumnStore::ColumnBlock::get_next_batch(const ObExprPtrIArray &exprs, LOG_WARN("calc nested expr size failed", K(ret), K(size)); } } else { - const VectorFormat format = vec->get_format(); - switch (format) { - case VEC_FIXED: - ret = from_buf(buf, pos, size, static_cast(vec)); - break; - case VEC_CONTINUOUS: - ret = from_buf(buf, pos, size, static_cast(vec)); - break; - case VEC_UNIFORM: - static_cast(vec)->set_all_null(size); - break; - default: - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected format", K(ret), K(format)); + if (OB_FAIL(vector_from_buf(buf, pos ,size, vec))) { + LOG_WARN("vector from buf failed", KR(ret)); } } } @@ -289,6 +350,30 @@ int ObTempColumnStore::ColumnBlock::get_next_batch(const ObExprPtrIArray &exprs, return ret; } +int ObTempColumnStore::ColumnBlock::get_next_batch(const IVectorPtrs &vectors, + const ObArray &lengths, + const int32_t start_read_pos, + int32_t &batch_rows, + int32_t &batch_pos) const +{ + int ret = OB_SUCCESS; + char* buf = const_cast(payload_ + start_read_pos); + const int32_t size = *reinterpret_cast(buf); + const int32_t *vec_offsets = reinterpret_cast(buf + sizeof(int32_t)); + for (int64_t i = 0; OB_SUCC(ret) && i < vectors.count(); ++i) { + ObIVector *vec = vectors.at(i); + int64_t pos = vec_offsets[i]; + if (OB_FAIL(vector_from_buf(buf, pos ,size, vec))) { + LOG_WARN("vector from buf failed", KR(ret)); + } + } + if (OB_SUCC(ret)) { + batch_rows = size; + batch_pos = vec_offsets[vectors.count()]; + } + return ret; +} + int ObTempColumnStore::Iterator::init(ObTempColumnStore *store) { reset(); @@ -368,6 +453,35 @@ int ObTempColumnStore::Iterator::get_next_batch(const ObExprPtrIArray &exprs, return ret; } +int ObTempColumnStore::Iterator::get_next_batch(const IVectorPtrs &vectors, + int64_t &read_rows) +{ + int ret = OB_SUCCESS; + read_rows = 0; + if (OB_UNLIKELY(NULL == cur_blk_ || !cur_blk_->contain(cur_blk_id_))) { + if (OB_FAIL(next_block())) { + if (ret != OB_ITER_END) { + LOG_WARN("fail to get next block", K(ret)); + } + } + } + int32_t batch_rows = 0; + int32_t batch_pos = 0; + if (OB_FAIL(ret)) { + } else if (OB_FAIL(cur_blk_->get_next_batch(vectors, + column_store_->batch_ctx_->lengths_, + read_pos_, + batch_rows, + batch_pos))) { + LOG_WARN("fail to get next batch from column block", K(ret)); + } else { + cur_blk_id_ += batch_rows; + read_pos_ += batch_pos; + read_rows = batch_rows; + } + return ret; +} + int ObTempColumnStore::Iterator::next_block() { int ret = OB_SUCCESS; @@ -463,6 +577,123 @@ int ObTempColumnStore::init(const ObExprPtrIArray &exprs, return ret; } +int ObTempColumnStore::init(const IVectorPtrs &vectors, + const int64_t max_batch_size, + const ObMemAttr &mem_attr, + const int64_t mem_limit, + const bool enable_dump, + const ObCompressorType compressor_type) +{ + int ret = OB_SUCCESS; + mem_attr_ = mem_attr; + col_cnt_ = vectors.count(); + max_batch_size_ = max_batch_size; + ObTempBlockStore::set_inner_allocator_attr(mem_attr); + OZ(ObTempBlockStore::init(mem_limit, enable_dump, mem_attr.tenant_id_, mem_attr.ctx_id_, + mem_attr_.label_, compressor_type)); + OZ(init_batch_ctx(vectors)); + reuse_vector_array_ = false; + inited_ = true; + return ret; +} + +int ObTempColumnStore::init_vectors(const ObIArray &col_array, + ObIAllocator &allocator, + IVectorPtrs &vectors) +{ + int ret = OB_SUCCESS; + vectors.reset(); + if (OB_UNLIKELY(col_array.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(col_array)); + } else if (OB_FAIL(vectors.prepare_allocate(col_array.count()))) { + LOG_WARN("fail to prepare allocate vectors", K(ret), K(col_array.count())); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < col_array.count(); ++i) { + const ObColumnSchemaItem &column_schema = col_array.at(i); + VecValueTypeClass value_tc = get_vec_value_tc(column_schema.col_type_.get_type(), + column_schema.col_type_.get_scale(), + column_schema.col_accuracy_.get_precision()); + const bool is_fixed = is_fixed_length_vec(value_tc); + ObIVector *vector = nullptr; + if (is_fixed) { // fixed format + switch (value_tc) { + #define FIXED_VECTOR_INIT_SWITCH(value_tc) \ + case value_tc: { \ + using VecType = RTVectorType; \ + static_assert(sizeof(VecType) <= ObIVector::MAX_VECTOR_STRUCT_SIZE, \ + "vector size exceeds MAX_VECTOR_STRUCT_SIZE"); \ + vector = OB_NEWx(VecType, &allocator, nullptr, nullptr); \ + break; \ + } + FIXED_VECTOR_INIT_SWITCH(VEC_TC_INTEGER); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_UINTEGER); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_FLOAT); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_DOUBLE); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_FIXED_DOUBLE); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_DATETIME); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_DATE); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_TIME); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_YEAR); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_UNKNOWN); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_BIT); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_ENUM_SET); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_TIMESTAMP_TZ); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_TIMESTAMP_TINY); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_INTERVAL_YM); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_INTERVAL_DS); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT32); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT64); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT128); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT256); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT512); + #undef FIXED_VECTOR_INIT_SWITCH + default: + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid fixed vector value type class", K(ret), K(i), K(column_schema), K(value_tc)); + break; + } + } else { // continuous format + switch (value_tc) { + #define CONTINUOUS_VECTOR_INIT_SWITCH(value_tc) \ + case value_tc: { \ + using VecType = RTVectorType; \ + static_assert(sizeof(VecType) <= ObIVector::MAX_VECTOR_STRUCT_SIZE, \ + "vector size exceeds MAX_VECTOR_STRUCT_SIZE"); \ + vector = OB_NEWx(VecType, &allocator, nullptr, nullptr, nullptr); \ + break; \ + } + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_NUMBER); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_EXTEND); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_STRING); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_ENUM_SET_INNER); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_RAW); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_ROWID); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_LOB); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_JSON); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_GEO); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_UDT); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_COLLECTION); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_ROARINGBITMAP); + #undef CONTINUOUS_VECTOR_INIT_SWITCH + default: + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid continuous vector value type class", K(ret), K(i), K(column_schema), K(value_tc)); + break; + } + } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(vector)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc vector", KR(ret)); + } else { + vectors.at(i) = vector; + } + } + } + return ret; +} + int ObTempColumnStore::init_batch_ctx(const ObExprPtrIArray &exprs) { int ret = OB_SUCCESS; @@ -512,6 +743,58 @@ int ObTempColumnStore::init_batch_ctx(const ObExprPtrIArray &exprs) return ret; } +int ObTempColumnStore::init_batch_ctx(const IVectorPtrs &vectors) +{ + int ret = OB_SUCCESS; + const int64_t max_batch_size = max_batch_size_; + if (OB_LIKELY(NULL == batch_ctx_)) { + const int64_t size = sizeof(*batch_ctx_) + sizeof(*batch_ctx_->selector_) * max_batch_size; + char *mem = static_cast(allocator_->alloc(size, mem_attr_)); + if (OB_UNLIKELY(max_batch_size <= 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("max batch size is not positive when init batch ctx", K(ret), K(max_batch_size)); + } else if (NULL == mem) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory failed", K(ret), K(size), K(col_cnt_), K(max_batch_size)); + } else { + char *begin = mem; + batch_ctx_ = new (mem) BatchCtx(); + batch_ctx_->vectors_.set_attr(mem_attr_); + batch_ctx_->lengths_.set_attr(mem_attr_); + batch_ctx_->max_batch_size_ = max_batch_size; + if (OB_FAIL(batch_ctx_->lengths_.prepare_allocate(col_cnt_))) { + LOG_WARN("fail to prepare allocate lengths", K(ret), K(col_cnt_)); + } else { + mem += sizeof(*batch_ctx_); + batch_ctx_->selector_ = reinterpret_castselector_)>(mem); + mem += sizeof(*batch_ctx_->selector_) * max_batch_size; + if (mem - begin != size) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("size mismatch", K(ret), K(mem - begin), K(size), K(col_cnt_), + K(max_batch_size)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < vectors.count(); ++i) { + ObIVector *vector = vectors.at(i); + const VectorFormat format = vector->get_format(); + switch (format) { + case VEC_FIXED: + batch_ctx_->lengths_.at(i) = static_cast(vector)->get_length(); + break; + case VEC_CONTINUOUS: + batch_ctx_->lengths_.at(i) = UNFIXED_LENGTH; + break; + default: + ret = OB_ERR_UNDEFINED; + LOG_WARN("unexpected vector format", KR(ret), K(i), K(format)); + break; + } + } + } + } + } + return ret; +} + int ObTempColumnStore::add_batch(const common::ObIArray &exprs, ObEvalCtx &ctx, const ObBatchRows &brs, int64_t &stored_rows_count) { @@ -569,5 +852,58 @@ int ObTempColumnStore::add_batch(const common::ObIArray &exprs, ObEval return ret; } +int ObTempColumnStore::add_batch(const IVectorPtrs &vectors, + const ObBatchRows &brs, + int64_t &stored_rows_count) +{ + int ret = OB_SUCCESS; + int16_t size = 0; + const uint16_t *selector = NULL; + if (OB_UNLIKELY(vectors.count() != get_col_cnt())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("column count mismatch", K(ret), K(vectors.count()), K(get_col_cnt())); + } else if (OB_ISNULL(batch_ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected batch ctx not init", K(ret)); + } else if (brs.all_rows_active_ || (0 == brs.skip_->accumulate_bit_cnt(brs.size_))) { + // all skipped, set selector point to null + size = brs.size_; + } else { + for (int64_t i = 0; i < brs.size_; i++) { + if (brs.skip_->at(i)) { + continue; + } else { + batch_ctx_->selector_[size++] = i; + } + } + selector = batch_ctx_->selector_; + } + if (OB_SUCC(ret) && size > 0) { + int64_t batch_mem_size = 0; + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ColumnBlock::calc_rows_size(vectors, + selector, + batch_ctx_->lengths_, + size, + batch_mem_size))) { + LOG_WARN("fail to calc rows size", K(ret)); + } else if (OB_FAIL(ensure_write_blk(batch_mem_size))) { + LOG_WARN("ensure write block failed", K(ret)); + } else if (OB_FAIL(cur_blk_->add_batch(blk_buf_, + vectors, + selector, + batch_ctx_->lengths_, + size, + batch_mem_size))) { + LOG_WARN("fail to add batch to column store", K(ret)); + } else { + block_id_cnt_ += size; + inc_mem_used(batch_mem_size); + } + } + stored_rows_count = size; + return ret; +} + } // end namespace sql } // end namespace oceanbase diff --git a/src/sql/engine/basic/ob_temp_column_store.h b/src/sql/engine/basic/ob_temp_column_store.h index bc3a5f296d..b664db52e3 100644 --- a/src/sql/engine/basic/ob_temp_column_store.h +++ b/src/sql/engine/basic/ob_temp_column_store.h @@ -21,6 +21,10 @@ namespace oceanbase { +namespace storage +{ +class ObColumnSchemaItem; +} // namespace storage namespace sql { @@ -55,6 +59,22 @@ public: */ struct ColumnBlock : public Block { + static int calc_vector_size(const ObIVector *vector, + const uint16_t *selector, + const ObLength length, + const int64_t size, + int64_t &batch_mem_size); + static int vector_to_buf(const ObIVector *vector, + const uint16_t *selector, + const ObLength length, + const int64_t size, + char *head, + int64_t &pos); + static int vector_from_buf(char *buf, + int64_t &pos, + const int64_t size, + ObIVector *vector); + static int calc_rows_size(ObEvalCtx &ctx, const ObExprPtrIArray &exprs, const IVectorPtrs &vectors, @@ -62,6 +82,11 @@ public: const ObArray &lengths, const int64_t size, int64_t &batch_mem_size); + static int calc_rows_size(const IVectorPtrs &vectors, + const uint16_t *selector, + const ObArray &lengths, + const int64_t size, + int64_t &batch_mem_size); static int calc_nested_size(ObExpr &expr, ObEvalCtx &ctx, const uint16_t *selector, const ObArray &lengths, const int64_t size, int64_t &batch_mem_size); @@ -77,6 +102,12 @@ public: const ObArray &lengths, const int64_t size, const int64_t batch_mem_size); + int add_batch(ShrinkBuffer &buf, + const IVectorPtrs &vectors, + const uint16_t *selector, + const ObArray &lengths, + const int64_t size, + const int64_t batch_mem_size); int get_next_batch(const ObExprPtrIArray &exprs, ObEvalCtx &ctx, @@ -85,6 +116,12 @@ public: const int32_t start_read_pos, int32_t &batch_rows, int32_t &batch_pos) const; + int get_next_batch(const IVectorPtrs &vectors, + const ObArray &lengths, + const int32_t start_read_pos, + int32_t &batch_rows, + int32_t &batch_pos) const; + int get_nested_batch(ObExpr &expr, ObEvalCtx &ctx, char *buf, int64_t &pos, const int64_t size) const; private: inline static int64_t get_header_size(const int64_t vec_cnt) @@ -112,6 +149,8 @@ public: ObEvalCtx &ctx, const int64_t max_rows, int64_t &read_rows); + int get_next_batch(const IVectorPtrs &vectors, + int64_t &read_rows); inline bool has_rest_row_in_batch() const { return rest_row_cnt_ > 0; } void reset() @@ -169,8 +208,19 @@ public: const bool enable_dump, const bool reuse_vector_array, const common::ObCompressorType compressor_type); + // for vector interface + int init(const IVectorPtrs &vectors, + const int64_t max_batch_size, + const lib::ObMemAttr &mem_attr, + const int64_t mem_limit, + const bool enable_dump, + const common::ObCompressorType compressor_type); + static int init_vectors(const common::ObIArray &col_array, + common::ObIAllocator &allocator, + IVectorPtrs &vectors); int init_batch_ctx(const ObExprPtrIArray &exprs); + int init_batch_ctx(const IVectorPtrs &vectors); int begin(Iterator &it) { @@ -178,6 +228,9 @@ public: } int add_batch(const common::ObIArray &exprs, ObEvalCtx &ctx, const ObBatchRows &brs, int64_t &stored_rows_count); + int add_batch(const IVectorPtrs &vectors, + const ObBatchRows &brs, + int64_t &stored_rows_count); inline int64_t get_col_cnt() const { return col_cnt_; } private: diff --git a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp index db0b71d2ab..27bbafb65c 100644 --- a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp +++ b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp @@ -2373,9 +2373,10 @@ int ObLoadDataDirectImpl::init_execute_param() int64_t hint_batch_size = 0; if (OB_FAIL(hint.get_value(ObLoadDataHint::BATCH_SIZE, hint_batch_size))) { LOG_WARN("fail to get value of BATCH_SIZE", KR(ret), K(hint)); + } else if (hint_batch_size > 0) { + execute_param_.batch_row_count_ = MIN(hint_batch_size, ObTableLoadParam::MAX_BATCH_SIZE); } else { - execute_param_.batch_row_count_ = - hint_batch_size > 0 ? hint_batch_size : DEFAULT_BUFFERRED_ROW_COUNT; + execute_param_.batch_row_count_ = ObTableLoadParam::DEFAULT_BATCH_SIZE; } } // online_opt_stat_gather_ diff --git a/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp b/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp index 969fea20e4..8b951da520 100644 --- a/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp +++ b/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp @@ -68,6 +68,10 @@ int ObTableDirectInsertCtx::init( } else if (OB_UNLIKELY(session_info->get_ddl_info().is_mview_complete_refresh() && enable_inc_replace)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected mview complete refresh enable inc replace", KR(ret)); + } else if (OB_UNLIKELY(phy_plan.is_vectorized() && + phy_plan.get_batch_size() > ObTableLoadParam::MAX_BATCH_SIZE)) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "batch size exceeds 65536 in direct load is"); } else { is_direct_ = true; if (OB_ISNULL(load_exec_ctx_ = OB_NEWx(ObTableLoadExecCtx, &exec_ctx->get_allocator()))) { @@ -112,10 +116,11 @@ int ObTableDirectInsertCtx::init( param.table_id_ = table_id; param.parallel_ = parallel; param.session_count_ = parallel; - param.batch_size_ = 100; + param.batch_size_ = phy_plan.is_vectorized() ? phy_plan.get_batch_size() + : ObTableLoadParam::DEFAULT_BATCH_SIZE; param.max_error_row_count_ = 0; param.column_count_ = column_ids.count(); - param.need_sort_ = table_schema->is_heap_table() ? phy_plan.get_direct_load_need_sort() : true; + param.need_sort_ = phy_plan.get_direct_load_need_sort(); param.px_mode_ = true; param.online_opt_stat_gather_ = is_online_gather_statistics_; param.dup_action_ = (enable_inc_replace ? sql::ObLoadDupActionType::LOAD_REPLACE diff --git a/src/sql/engine/direct_load/ob_table_direct_insert_op.cpp b/src/sql/engine/direct_load/ob_table_direct_insert_op.cpp new file mode 100644 index 0000000000..eede687bd8 --- /dev/null +++ b/src/sql/engine/direct_load/ob_table_direct_insert_op.cpp @@ -0,0 +1,460 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_ENG +#include "ob_table_direct_insert_op.h" +#include "storage/access/ob_dml_param.h" +#include "storage/tx_storage/ob_access_service.h" +#include "sql/engine/dml/ob_dml_service.h" +#include "sql/engine/cmd/ob_table_direct_insert_service.h" +#include "observer/table_load/ob_table_load_service.h" +#include "share/schema/ob_table_param.h" +#include "share/schema/ob_table_dml_param.h" +#include "storage/blocksstable/ob_datum_row_utils.h" + +using namespace oceanbase::common; +using namespace oceanbase::sql; +using namespace oceanbase::storage; +using namespace oceanbase::common::serialization; +using namespace oceanbase::observer; + +namespace oceanbase +{ +namespace sql +{ +OB_SERIALIZE_MEMBER((ObTableDirectInsertOpInput, ObPxMultiPartModifyOpInput)); + +OB_SERIALIZE_MEMBER((ObTableDirectInsertSpec, ObTableModifySpec), + row_desc_, + ins_ctdef_); + +//////////////////////ObTableDirectInsertOp/////////////////// +ObTableDirectInsertOp::ObTableDirectInsertOp( + ObExecContext &exec_ctx, + const ObOpSpec &spec, + ObOpInput *input) + : ObTableModifyOp(exec_ctx, spec, input), + ins_rtdef_(), + allocator_("DirectInsertOp", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), + px_task_id_(0), + ddl_task_id_(0), + table_ctx_(nullptr), + datum_row_(nullptr), + px_writer_(nullptr), + tablet_id_(), + is_partitioned_table_(false) +{ +} + +int ObTableDirectInsertOp::inner_open() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObTableModifyOp::inner_open())) { + LOG_WARN("failed to inner open", K(ret)); + } else if (OB_FAIL(ObDMLService::init_ins_rtdef(dml_rtctx_, + ins_rtdef_, + MY_SPEC.ins_ctdef_, + trigger_clear_exprs_, + fk_checkers_))) { + LOG_WARN("init insert rtdef failed", K(ret)); + } else if (OB_UNLIKELY(!MY_SPEC.row_desc_.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table or row desc is invalid", K(ret), K(MY_SPEC.row_desc_)); + } else if (OB_FAIL(init_px_writer())) { + LOG_WARN("failed to init px writer", KR(ret)); + } + LOG_TRACE("table direct insert op", KR(ret), K_(MY_SPEC.row_desc), K_(MY_SPEC.ins_ctdef), + K(MY_SPEC.is_vectorized()), K_(MY_SPEC.use_rich_format), K_(is_partitioned_table), + K_(px_task_id), K_(ddl_task_id)); + return ret; +} + +int ObTableDirectInsertOp::inner_close() +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + const ObPhysicalPlan *plan = GET_PHY_PLAN_CTX(ctx_)->get_phy_plan(); + int error_code = (static_cast(input_))->get_error_code(); + if (OB_NOT_NULL(px_writer_)) { + if (OB_LIKELY(OB_SUCCESS == error_code) && OB_TMP_FAIL(px_writer_->close())) { + LOG_WARN("failed to close px writer", KR(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + px_writer_->reset(); + } + if (OB_TMP_FAIL(ObTableDirectInsertService::close_task(plan->get_append_table_id(), + px_task_id_, + ddl_task_id_, + table_ctx_, + error_code))) { + LOG_WARN("failed to close table direct insert task", KR(tmp_ret), + K(plan->get_append_table_id()), K(px_task_id_), K(ddl_task_id_), K(error_code)); + ret = COVER_SUCC(tmp_ret); + } + if (OB_TMP_FAIL(ObTableModifyOp::inner_close())) { + LOG_WARN("failed to inner close table modify", KR(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + return ret; +} + +void ObTableDirectInsertOp::destroy() +{ + if (OB_NOT_NULL(px_writer_)) { + px_writer_->~ObTableLoadStoreTransPXWriter(); + px_writer_ = nullptr; + } + if (OB_NOT_NULL(datum_row_)) { + datum_row_->~ObDatumRow(); + datum_row_ = nullptr; + } + allocator_.reset(); + ObTableModifyOp::destroy(); +} + +int ObTableDirectInsertOp::inner_get_next_row() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(child_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("child op is null", K(ret)); + } else if (MY_SPEC.is_returning_) { + if (OB_FAIL(child_->get_next_row())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail get next row from child", K(ret)); + } + } else if (OB_FAIL(next_row())) { + LOG_WARN("failed to process next row", KR(ret)); + } + } else { + do { + if (OB_FAIL(child_->get_next_row())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail get next row from child", K(ret)); + } + } else if (OB_FAIL(next_row())) { + LOG_WARN("failed to process next row", KR(ret)); + } + } while (OB_SUCC(ret)); + } + return ret; +} + +int ObTableDirectInsertOp::inner_get_next_batch(const int64_t max_row_cnt) +{ + return MY_SPEC.use_rich_format_ ? next_vector(max_row_cnt) : next_batch(max_row_cnt); +} + +// 向量化2.0 +int ObTableDirectInsertOp::next_vector(const int64_t max_row_cnt) +{ + int ret = OB_SUCCESS; + ObEvalCtx &eval_ctx = get_eval_ctx(); + while (OB_SUCC(ret) && !brs_.end_) { + clear_evaluated_flag(); + const ObBatchRows *child_brs = nullptr; + if (OB_FAIL(child_->get_next_batch(max_row_cnt, child_brs))) { + if (OB_ITER_END != ret) { + LOG_WARN("failed to get next batch", K(ret)); + } else { + brs_.size_ = 0; + brs_.end_ = true; + ret = OB_SUCCESS; + } + } else { + brs_.copy(child_brs); + } + + if (OB_SUCC(ret) && !brs_.end_) { + if (OB_FAIL(process_insert_batch())) { + LOG_WARN("failed to process insert batch", KR(ret)); + } else { + const ExprFixedArray &dml_expr_array = MY_SPEC.ins_ctdef_.new_row_; + const ExprFixedArray &expr_array = child_->get_spec().output_; + ObIVector *tablet_id_vector = is_partitioned_table_ ? expr_array.at(0)->get_vector(eval_ctx) : nullptr; + ObArray vectors; + int64_t affected_rows = 0; + if (OB_FAIL(vectors.prepare_allocate(dml_expr_array.count()))) { + LOG_WARN("fail to prepare allocate", KR(ret), K(dml_expr_array.count())); + } else { + for (int64_t i = 0; i < dml_expr_array.count(); ++i) { + vectors.at(i) = dml_expr_array.at(i)->get_vector(eval_ctx); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(write_vectors(tablet_id_vector, vectors, brs_))) { + LOG_WARN("failed to write vectors", KR(ret)); + } + } + } + } // end while + return ret; +} + +// 向量化1.0 +int ObTableDirectInsertOp::next_batch(const int64_t max_row_cnt) +{ + int ret = OB_SUCCESS; + ObEvalCtx &eval_ctx = get_eval_ctx(); + while (OB_SUCC(ret) && !brs_.end_) { + clear_evaluated_flag(); + const ObBatchRows *child_brs = nullptr; + if (OB_FAIL(child_->get_next_batch(max_row_cnt, child_brs))) { + if (OB_ITER_END != ret) { + LOG_WARN("failed to get next batch", K(ret)); + } else { + brs_.size_ = 0; + brs_.end_ = true; + ret = OB_SUCCESS; + } + } else { + brs_.copy(child_brs); + } + + if (OB_SUCC(ret) && !brs_.end_) { + if (OB_FAIL(process_insert_batch())) { + LOG_WARN("failed to process insert batch", KR(ret)); + } else { + const ExprFixedArray &dml_expr_array = MY_SPEC.ins_ctdef_.new_row_; + const ExprFixedArray &expr_array = child_->get_spec().output_; + ObDatumVector tablet_id_datum_vector; + ObArray datum_vectors; + int64_t affected_rows = 0; + if (is_partitioned_table_) { + tablet_id_datum_vector = expr_array.at(0)->locate_expr_datumvector(eval_ctx); + } + if (OB_FAIL(datum_vectors.prepare_allocate(dml_expr_array.count()))) { + LOG_WARN("fail to prepare allocate", KR(ret), K(dml_expr_array.count())); + } else { + for (int64_t i = 0; i < dml_expr_array.count(); ++i) { + datum_vectors.at(i) = dml_expr_array.at(i)->locate_expr_datumvector(eval_ctx); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(write_batch(tablet_id_datum_vector, datum_vectors, brs_))) { + LOG_WARN("failed to write batch", KR(ret)); + } + } + } + } // end while + return ret; +} + +int ObTableDirectInsertOp::next_row() +{ + int ret = OB_SUCCESS; + bool is_skipped = false; + clear_evaluated_flag(); + if (OB_FAIL(ObDMLService::process_insert_row(MY_SPEC.ins_ctdef_, ins_rtdef_, *this, is_skipped))) { + LOG_WARN("failed to process insert row", KR(ret), K(ins_rtdef_.cur_row_num_)); + } else if (!is_skipped) { + ObEvalCtx &eval_ctx = get_eval_ctx(); + const ExprFixedArray &dml_expr_array = MY_SPEC.ins_ctdef_.new_row_; + const ObTabletID *tablet_id_ptr = nullptr; + ObTabletID tablet_id; + if (is_partitioned_table_) { + const ExprFixedArray &expr_array = child_->get_spec().output_; + ObExpr *expr = expr_array.at(0); + ObDatum &expr_datum = expr->locate_expr_datum(eval_ctx); + tablet_id = expr_datum.get_int(); + tablet_id_ptr = &tablet_id; + } + if (OB_FAIL(write_row(tablet_id_ptr, dml_expr_array))) { + LOG_WARN("failed to write row", KR(ret)); + } + } + if (OB_SUCC(ret)) { + ins_rtdef_.cur_row_num_++; + } + return ret; +} + +int ObTableDirectInsertOp::write_vectors( + ObIVector *tablet_id_vector, + const ObIArray &vectors, + const ObBatchRows &batch_rows) +{ + int ret = OB_SUCCESS; + int64_t affected_rows = 0; + if (OB_FAIL(px_writer_->write_vector(tablet_id_vector, vectors, batch_rows, affected_rows))) { + LOG_WARN("failed to write vector", KR(ret)); + } else { + GET_PHY_PLAN_CTX(ctx_)->add_row_matched_count(affected_rows); + GET_PHY_PLAN_CTX(ctx_)->add_affected_rows(affected_rows); + } + return ret; +} + +int ObTableDirectInsertOp::write_batch( + const ObDatumVector &tablet_id_datum_vector, + const ObIArray &datum_vectors, + const ObBatchRows &batch_rows) +{ + int ret = OB_SUCCESS; + int64_t affected_rows = 0; + if (OB_FAIL(px_writer_->write_batch(tablet_id_datum_vector, datum_vectors, batch_rows, affected_rows))) { + LOG_WARN("failed to write batch", KR(ret)); + } else { + GET_PHY_PLAN_CTX(ctx_)->add_row_matched_count(affected_rows); + GET_PHY_PLAN_CTX(ctx_)->add_affected_rows(affected_rows); + } + return ret; +} + +int ObTableDirectInsertOp::write_row( + const ObTabletID *tablet_id_ptr, + const ExprFixedArray &expr_array) +{ + int ret = OB_SUCCESS; + int64_t affected_rows = 0; + ObEvalCtx &eval_ctx = get_eval_ctx(); + if (OB_ISNULL(datum_row_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null datum row", KR(ret), KP(datum_row_)); + } else if (is_partitioned_table_ && OB_ISNULL(tablet_id_ptr)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tablet_id_expr should not be null if partitioned table", KR(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && (i < expr_array.count()); ++i) { + ObExpr *expr = expr_array.at(i); + ObDatum &expr_datum = expr->locate_expr_datum(eval_ctx); + datum_row_->storage_datums_[i].shallow_copy_from_datum(expr_datum); + } + + if (OB_SUCC(ret)) { + if (is_partitioned_table_) { + if (OB_FAIL(px_writer_->write_row(*tablet_id_ptr, *datum_row_))) { + LOG_WARN("failed to write row", KR(ret), KPC(datum_row_)); + } + } else if (OB_FAIL(px_writer_->write_row(tablet_id_, *datum_row_))) { + LOG_WARN("failed to write row", KR(ret), K(tablet_id_), KPC(datum_row_)); + } + if (OB_SUCC(ret)) { + affected_rows++; + } + } + } // end if + + if (OB_SUCC(ret)) { + GET_PHY_PLAN_CTX(ctx_)->add_row_matched_count(affected_rows); + GET_PHY_PLAN_CTX(ctx_)->add_affected_rows(affected_rows); + } + return ret; +} + +int ObTableDirectInsertOp::init_px_writer() +{ + int ret = OB_SUCCESS; + const ObPhysicalPlan *plan = GET_PHY_PLAN_CTX(ctx_)->get_phy_plan(); + const ObIArray &column_ids = MY_SPEC.ins_ctdef_.das_ctdef_.column_ids_; + px_task_id_ = ctx_.get_px_task_id() + 1; + ddl_task_id_ = plan->get_ddl_task_id(); + is_partitioned_table_ = !(NO_PARTITION_ID_FLAG == MY_SPEC.row_desc_.get_part_id_index()); + if (OB_FAIL(ObTableDirectInsertService::open_task( + plan->get_append_table_id(), px_task_id_, ddl_task_id_, table_ctx_))) { + LOG_WARN("failed to open table direct insert task", KR(ret), + K(plan->get_append_table_id()), K(px_task_id_), K(ddl_task_id_)); + } else if (!MY_SPEC.is_vectorized() && OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row( + allocator_, column_ids.count(), datum_row_))) { + LOG_WARN("failed to create datum row", KR(ret), K(column_ids.count())); + } else if (OB_ISNULL(px_writer_ = OB_NEWx(ObTableLoadStoreTransPXWriter, (&allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to new ObTableLoadStoreTransPXWriter", KR(ret)); + } else { + ObTableLoadStore store(table_ctx_); + table::ObTableLoadTransId trans_id; + trans_id.segment_id_ = px_task_id_; + trans_id.trans_gid_ = 1; + if (OB_FAIL(store.init())) { + LOG_WARN("failed to init store", KR(ret)); + } else if (OB_FAIL(store.px_get_trans_writer(trans_id, *px_writer_))) { + LOG_WARN("failed to get trans writer", KR(ret), K(trans_id)); + } else if (!is_partitioned_table_) { + ObDASTableLoc *table_loc = ins_rtdef_.das_rtdef_.table_loc_; + if (OB_ISNULL(table_loc) || table_loc->get_tablet_locs().size() != 1) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("insert table location is invalid", K(ret), KPC(table_loc)); + } else { + tablet_id_ = table_loc->get_first_tablet_loc()->tablet_id_; + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(px_writer_->prepare_write(column_ids, + MY_SPEC.is_vectorized(), + MY_SPEC.use_rich_format_))) { + LOG_WARN("failed to prepare write", KR(ret), K(column_ids), + K(MY_SPEC.is_vectorized()), K(MY_SPEC.use_rich_format_)); + } + } + } + return ret; +} + +int ObTableDirectInsertOp::process_insert_batch() +{ + int ret = OB_SUCCESS; + ObEvalCtx &eval_ctx = get_eval_ctx(); + const ExprFixedArray &dml_expr_array = MY_SPEC.ins_ctdef_.new_row_; + const ObIArray &column_infos = MY_SPEC.ins_ctdef_.column_infos_; + const int64_t column_offset = MY_SPEC.ins_ctdef_.is_heap_table_ ? 1: 0; + const int64_t row_num = 0; // no sense + ObUserLoggingCtx::Guard logging_ctx_guard(*(ctx_.get_user_logging_ctx())); + ctx_.set_cur_rownum(row_num); + CK (dml_expr_array.count() == column_infos.count() + column_offset); + for (int64_t i = 0; OB_SUCC(ret) && (i < dml_expr_array.count()); ++i) { + if (MY_SPEC.ins_ctdef_.is_heap_table_ && (0 == i)) { + continue; // skip hidden table pk column + } else { + const ColumnContent &column_info = column_infos.at(i - column_offset); + common::ObString column_name = column_info.column_name_; + ctx_.set_cur_column_name(&column_name); + ObExpr *expr = dml_expr_array.at(column_info.projector_index_); + if (MY_SPEC.use_rich_format_) { + if (OB_FAIL(expr->eval_vector(eval_ctx, brs_))) { + LOG_WARN("failed to eval vector", KR(ret)); + } + } else { + if (OB_FAIL(expr->eval_batch(eval_ctx, *(brs_.skip_), brs_.size_))) { + LOG_WARN("failed to eval batch", KR(ret)); + } + } + if (OB_FAIL(ret)) { + ret = ObDMLService::log_user_error_inner(ret, row_num, column_name, ctx_); + } + } + } + + if (OB_SUCC(ret)) { + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(eval_ctx); + batch_info_guard.set_batch_size(brs_.size_); + batch_info_guard.set_batch_idx(0); + for (int64_t i = 0; OB_SUCC(ret) && (i < brs_.size_); ++i) { + bool is_skipped = brs_.skip_->at(i); + batch_info_guard.set_batch_idx(i); + if (!is_skipped) { + if (OB_FAIL(ObDMLService::process_insert_row(MY_SPEC.ins_ctdef_, ins_rtdef_, *this, is_skipped))) { + LOG_WARN("failed to process insert row", KR(ret), K(ins_rtdef_.cur_row_num_)); + } else if (is_skipped) { + brs_.skip_->set(i); + } + } + if (OB_SUCC(ret)) { + ins_rtdef_.cur_row_num_++; + } + } + } + return ret; +} + +} // namespace sql +} // namespace oceanbase \ No newline at end of file diff --git a/src/sql/engine/direct_load/ob_table_direct_insert_op.h b/src/sql/engine/direct_load/ob_table_direct_insert_op.h new file mode 100644 index 0000000000..301b00290b --- /dev/null +++ b/src/sql/engine/direct_load/ob_table_direct_insert_op.h @@ -0,0 +1,124 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_TABLE_DIRECT_INSERT_OP_H_ +#define OB_TABLE_DIRECT_INSERT_OP_H_ + +#include "lib/container/ob_fixed_array.h" +#include "sql/engine/dml/ob_table_modify_op.h" +#include "sql/engine/pdml/static/ob_px_multi_part_modify_op.h" +#include "storage/ddl/ob_direct_load_struct.h" +#include "observer/table_load/ob_table_load_store.h" +#include "observer/table_load/ob_table_load_store_trans_px_writer.h" + +namespace oceanbase +{ +namespace observer +{ +class ObTableLoadTableCtx; +} +namespace sql +{ +class ObTableDirectInsertOpInput : public ObPxMultiPartModifyOpInput +{ + OB_UNIS_VERSION_V(1); +public: + ObTableDirectInsertOpInput(ObExecContext &ctx, const ObOpSpec &spec) + : ObPxMultiPartModifyOpInput(ctx, spec), + error_code_(OB_SUCCESS) + {} + int init(ObTaskInfo &task_info) override + { + return ObPxMultiPartModifyOpInput::init(task_info); + } + void reset() override + { + ObPxMultiPartModifyOpInput::reset(); + error_code_ = OB_SUCCESS; + } + inline void set_error_code(const int error_code) { error_code_ = error_code; } + inline int get_error_code() const { return error_code_; } + TO_STRING_KV(K_(error_code)); +private: + int error_code_; + DISALLOW_COPY_AND_ASSIGN(ObTableDirectInsertOpInput); +}; + +class ObTableDirectInsertSpec : public ObTableModifySpec +{ + OB_UNIS_VERSION_V(1); +public: + ObTableDirectInsertSpec(common::ObIAllocator &alloc, const ObPhyOperatorType type) + : ObTableModifySpec(alloc, type), + row_desc_(), + ins_ctdef_(alloc) + { + } + //This interface is only allowed to be used in a single-table DML operator, + //it is invalid when multiple tables are modified in one DML operator + virtual int get_single_dml_ctdef(const ObDMLBaseCtDef *&dml_ctdef) const override + { + dml_ctdef = &ins_ctdef_; + return common::OB_SUCCESS; + } + virtual bool is_pdml_operator() const override { return true; } +public: + ObDMLOpRowDesc row_desc_; // 记录partition id column所在row的第几个cell + ObInsCtDef ins_ctdef_; + DISALLOW_COPY_AND_ASSIGN(ObTableDirectInsertSpec); +}; + +class ObTableDirectInsertOp : public ObTableModifyOp +{ + OB_UNIS_VERSION(1); +public: + ObTableDirectInsertOp(ObExecContext &exec_ctx, + const ObOpSpec &spec, + ObOpInput *input); + virtual bool has_foreign_key() const { return false; } // 默认实现,先不考虑外键的问题 + + virtual int inner_open() override; + virtual int inner_close() override; + virtual int inner_get_next_row() override; + virtual int inner_get_next_batch(const int64_t max_row_cnt) override; + virtual void destroy() override; +private: + int init_px_writer(); + int next_vector(const int64_t max_row_cnt); + int next_batch(const int64_t max_row_cnt); + int next_row(); + int write_vectors(common::ObIVector *tablet_id_vector, + const common::IVectorPtrs &vectors, + const sql::ObBatchRows &batch_rows); + int write_batch(const common::ObDatumVector &tablet_id_datum_vector, + const common::ObIArray &datum_vectors, + const sql::ObBatchRows &batch_rows); + int write_row(const common::ObTabletID *tablet_id_ptr, + const sql::ExprFixedArray &expr_array); + int process_insert_batch(); +protected: + ObInsRtDef ins_rtdef_; + common::ObArenaAllocator allocator_; + int64_t px_task_id_; + int64_t ddl_task_id_; + observer::ObTableLoadTableCtx *table_ctx_; + blocksstable::ObDatumRow *datum_row_; + observer::ObTableLoadStoreTransPXWriter *px_writer_; + ObTabletID tablet_id_; + bool is_partitioned_table_; + DISALLOW_COPY_AND_ASSIGN(ObTableDirectInsertOp); +}; + + +} // namespace sql +} // namespace oceanbase +#endif /* OB_TABLE_DIRECT_INSERT_OP_H_*/ diff --git a/src/sql/engine/expr/ob_expr_column_conv.cpp b/src/sql/engine/expr/ob_expr_column_conv.cpp index 8b9a7b62d9..1ea0ab86bf 100644 --- a/src/sql/engine/expr/ob_expr_column_conv.cpp +++ b/src/sql/engine/expr/ob_expr_column_conv.cpp @@ -417,8 +417,9 @@ int ObExprColumnConv::cg_expr(ObExprCGCtx &op_cg_ctx, rt_expr.eval_func_ = column_convert; if (rt_expr.args_[4]->is_batch_result() && !ob_is_enum_or_set_type(rt_expr.datum_meta_.type_) - && !is_lob_storage(rt_expr.datum_meta_.type_) && GET_MIN_CLUSTER_VERSION() >= CLUSTER_VERSION_4_3_3_0) { + if (!is_lob_storage(rt_expr.datum_meta_.type_) + || GET_MIN_CLUSTER_VERSION() >= CLUSTER_VERSION_4_3_5_0) rt_expr.eval_batch_func_ = column_convert_batch; } } @@ -662,6 +663,9 @@ int ObExprColumnConv::column_convert_batch(const ObExpr &expr, const ObEnumSetInfo *enumset_info = static_cast(expr.extra_info_); const uint64_t cast_mode = enumset_info->cast_mode_; bool is_strict = CM_IS_STRICT_MODE(cast_mode); + ObEvalInfo &eval_info = expr.args_[4]->get_eval_info(ctx); + bool param_not_eval = !expr.args_[4]->get_eval_info(ctx).evaluated_ + && !expr.args_[4]->get_eval_info(ctx).projected_; if (OB_FAIL(expr.args_[4]->eval_batch(ctx, skip, batch_size))) { LOG_WARN("failed to eval batch vals", K(ret)); } else { @@ -669,6 +673,8 @@ int ObExprColumnConv::column_convert_batch(const ObExpr &expr, ObDatum *results = expr.locate_batch_datums(ctx); ObBitVector &eval_flags = expr.get_evaluated_flags(ctx); bool is_string_type = ob_is_string_type(out_type); + bool is_int_tc = ob_is_int_uint_tc(out_type); + bool is_decimal_int_tc = ob_is_decimal_int_tc(out_type); ObAccuracy accuracy; accuracy.set_length(expr.max_length_); accuracy.set_scale(expr.datum_meta_.scale_); @@ -679,31 +685,138 @@ int ObExprColumnConv::column_convert_batch(const ObExpr &expr, } ObEvalCtx::BatchInfoScopeGuard batch_info_guard(ctx); batch_info_guard.set_batch_size(batch_size); - for (int64_t i = 0; OB_SUCC(ret) && i < batch_size; ++i) { - if (skip.at(i) || eval_flags.at(i)) { - continue; + if (!is_lob_storage(out_type)) { + if (is_string_type + && OB_SUCCESS != (ret = inner_loop_for_convert_batch(expr, ctx, skip, batch_size, + is_strict, max_accuracy_len, cast_mode, + eval_flags, vals, results, + batch_info_guard))) { + LOG_WARN("failed to convert batch", K(ret)); + } else if (is_int_tc + && OB_SUCCESS != (ret = inner_loop_for_convert_batch(expr, ctx, skip, batch_size, + is_strict, max_accuracy_len, cast_mode, + eval_flags, vals, results, + batch_info_guard))) { + LOG_WARN("failed to convert batch", K(ret)); + } else if (is_decimal_int_tc + && OB_SUCCESS != (ret = inner_loop_for_convert_batch(expr, ctx, skip, batch_size, + is_strict, max_accuracy_len, cast_mode, + eval_flags, vals, results, + batch_info_guard))) { + LOG_WARN("failed to convert batch", K(ret)); + } else if (OB_SUCCESS != (ret = inner_loop_for_convert_batch(expr, ctx, skip, batch_size, + is_strict, max_accuracy_len, cast_mode, + eval_flags, vals, results, + batch_info_guard))) { + LOG_WARN("failed to convert batch", K(ret)); } - if (vals[i].is_null()) { - results[i].set_null(); - } else { - batch_info_guard.set_batch_idx(i); - if (is_string_type) { - ObString str = vals[i].get_string(); - if (OB_FAIL(string_collation_check(is_strict, out_cs_type, out_type, str))) { - LOG_WARN("fail to check collation", K(ret), K(str), K(is_strict), K(expr)); + } else if (expr.args_[4]->obj_meta_.is_user_defined_sql_type()) { + ret = OB_ERR_INVALID_XML_DATATYPE; + LOG_USER_ERROR(OB_ERR_INVALID_XML_DATATYPE, ob_obj_type_str(out_type), "ANYDATA"); + LOG_WARN("convert xmltype to character type is not supported in PL", + K(ret), K(expr.args_[4]->obj_meta_), K(out_type)); + } else { + ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); // temp alloc only used for lob types + common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); + ObObjType in_type = expr.args_[4]->obj_meta_.get_type(); + ObCollationType in_cs_type = expr.args_[4]->obj_meta_.get_collation_type(); + bool has_lob_header = expr.args_[4]->obj_meta_.has_lob_header(); + bool has_lob_header_for_check = has_lob_header; + bool can_use_raw_str = T_FUN_SYS_CAST == expr.args_[4]->type_ + && param_not_eval + && ob_is_string_tc(expr.args_[4]->args_[0]->datum_meta_.type_) + && !ob_is_geometry(out_type); + for (int64_t i = 0; OB_SUCC(ret) && i < batch_size; ++i) { + if (skip.at(i) || eval_flags.at(i)) { + continue; + } + if (vals[i].is_null()) { + results[i].set_null(); + } else { + batch_info_guard.set_batch_idx(i); + ObString raw_str = vals[i].get_string(); + ObLobLocatorV2 input_lob(raw_str.ptr(), raw_str.length(), has_lob_header); + bool is_delta = input_lob.is_valid() && input_lob.is_delta_temp_lob(); + if (is_delta) { // delta lob + if (!(ob_is_text_tc(in_type) || ob_is_json(in_type))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("delta lob can not convert to non-text type", K(ret), K(out_type)); + } else { + results[i].set_string(raw_str); + } } else { - vals[i].set_string(str); + ObDatum datum_for_check = vals[i]; + ObString str; + ObTextStringIter striter(in_type, in_cs_type, vals[i].get_string(), has_lob_header); + if (can_use_raw_str) { + str = expr.args_[4]->args_[0]->locate_expr_datum(ctx).get_string(); + if (str.length() >= max_accuracy_len) { + can_use_raw_str = false; + } + } + if (!can_use_raw_str) { + if (OB_FAIL(striter.init(0, ctx.exec_ctx_.get_my_session(), &temp_allocator))) { + LOG_WARN("fail to init string iter", K(ret), K(is_strict), K(expr)); + } else if (OB_FAIL(striter.get_full_data(str))) { + LOG_WARN("fail to get full data from string iter", K(ret), K(is_strict), K(expr)); + } else if (ob_is_geometry(out_type)) { + ObGeoType geo_type = ObGeoCastUtils::get_geo_type_from_cast_mode(cast_mode); + if (OB_FAIL(ObGeoTypeUtil::check_geo_type(geo_type, str))) { + LOG_WARN("fail to check geo type", K(ret), K(str), K(geo_type), K(expr)); + ret = OB_ERR_CANT_CREATE_GEOMETRY_OBJECT; + LOG_USER_ERROR(OB_ERR_CANT_CREATE_GEOMETRY_OBJECT); + } + } + } + if (OB_FAIL(ret)) { + } else if (!check_is_ascii(str) + && OB_FAIL(string_collation_check(is_strict, out_cs_type, out_type, str))) { + LOG_WARN("fail to check collation", K(ret), K(str), K(is_strict), K(expr)); + } + if (OB_SUCC(ret)) { + has_lob_header_for_check = false; // datum_for_check must have no lob header + datum_for_check.set_string(str); + } + int warning = OB_SUCCESS; + const int64_t str_len_byte = static_cast(datum_for_check.len_); + bool need_check_length = true; + if (OB_FAIL(ret)) { + } else if (max_accuracy_len > 0 && str.length() < max_accuracy_len) { + need_check_length = false; + results[i].set_datum(datum_for_check); + } else if (OB_FAIL(column_convert_datum_accuracy_check(expr, ctx, has_lob_header_for_check, results[i], + cast_mode, datum_for_check))) { + LOG_WARN("fail do datum_accuracy_check for lob res", K(ret), K(expr), K(datum_for_check)); + } + if (OB_SUCC(ret)) { + // in type is the same with out type, if length changed, build a new lob + ObLobLocatorV2 loc(raw_str, has_lob_header); + int64_t old_data_byte_len = 0; + if (need_check_length && raw_str.length() > 0) { + if (OB_FAIL(loc.get_lob_data_byte_len(old_data_byte_len))) { + LOG_WARN("Lob: failed to get data byte len", K(ret)); + } + } + if (OB_SUCC(ret)) { + int64_t new_data_byte_len = results[i].get_string().length(); + if (!need_check_length || new_data_byte_len == old_data_byte_len) { + results[i].set_string(raw_str); + } else { + ObTextStringDatumResult str_result(expr.datum_meta_.type_, &expr, &ctx, &results[i]); + if (OB_FAIL(str_result.init(new_data_byte_len))) { + LOG_WARN("Lob: init lob result failed", K(ret)); + } else if (OB_FAIL(str_result.append(results[i].get_string().ptr(), new_data_byte_len))) { + LOG_WARN("Lob: append lob result failed", K(ret)); + } else { + str_result.set_result(); + } + } + } + } } } - bool no_need_check_length = is_string_type && max_accuracy_len > 0 && vals[i].len_ < max_accuracy_len; - if (OB_FAIL(ret)) { - } else if (no_need_check_length) { - results[i].set_datum(vals[i]); - } else if (OB_FAIL(column_convert_datum_accuracy_check(expr, ctx, false, results[i], cast_mode, vals[i]))) { - LOG_WARN("fail do datum_accuracy_check for lob res", K(ret), K(expr)); - } + eval_flags.set(i); } - eval_flags.set(i); } } return ret; @@ -782,6 +895,80 @@ int ObExprColumnConv::ObExprColumnConvCtx::setup_eval_expr(ObIAllocator &allocat return ret; } +bool ObExprColumnConv::check_is_ascii(ObString &str) +{ + bool is_ascii = true; + for (int64_t i = 0; is_ascii && i < str.length(); ++i) { + is_ascii &= (static_cast (str[i]) <= 127); + } + return is_ascii; +} + +template +int ObExprColumnConv::inner_loop_for_convert_batch(const ObExpr &expr, + ObEvalCtx &ctx, + const ObBitVector &skip, + const int64_t batch_size, + const bool is_strict, + const ObLength max_accuracy_len, + const uint64_t cast_mode, + ObBitVector &eval_flags, + ObDatum *vals, + ObDatum *results, + ObEvalCtx::BatchInfoScopeGuard &batch_info_guard) +{ + int ret = OB_SUCCESS; + ObObjType out_type = expr.datum_meta_.type_; + ObCollationType out_cs_type = expr.datum_meta_.cs_type_; + int32_t int_bytes = 0; + const ObDecimalInt *min_decint = nullptr, *max_decint = nullptr; + common::ObPrecision precision = expr.datum_meta_.precision_; + decint_cmp_fp cmp_fp; + if (TC == PARAM_TC::DECIMAL_INT_TC) { + min_decint = wide::ObDecimalIntConstValue::get_min_value(precision); + max_decint = wide::ObDecimalIntConstValue::get_max_value(precision); + int_bytes = wide::ObDecimalIntConstValue::get_int_bytes_by_precision(precision); + cmp_fp = + wide::ObDecimalIntCmpSet::get_decint_decint_cmp_func(int_bytes, int_bytes); + } + for (int64_t i = 0; OB_SUCC(ret) && i < batch_size; ++i) { + if (skip.at(i) || eval_flags.at(i)) { + continue; + } + if (vals[i].is_null()) { + results[i].set_null(); + } else { + batch_info_guard.set_batch_idx(i); + if (TC == PARAM_TC::STRING_TC) { + ObString str = vals[i].get_string(); + if (!check_is_ascii(str) + && OB_FAIL(string_collation_check(is_strict, out_cs_type, out_type, str))) { + LOG_WARN("fail to check collation", K(ret), K(str), K(is_strict), K(expr)); + } else { + vals[i].set_string(str); + } + } + if (OB_FAIL(ret)) { + } else if (TC == PARAM_TC::INT_TC + || (TC == PARAM_TC::STRING_TC + && max_accuracy_len > 0 + && vals[i].len_ < max_accuracy_len)) { + results[i].set_datum(vals[i]); + } else if (TC == PARAM_TC::DECIMAL_INT_TC + && cmp_fp(vals[i].get_decimal_int(), min_decint) >= 0 + && cmp_fp(vals[i].get_decimal_int(), max_decint) <= 0) { + results[i].set_datum(vals[i]); + } else if (OB_FAIL(column_convert_datum_accuracy_check(expr, ctx, + false, results[i], + cast_mode, vals[i]))) { + LOG_WARN("fail do datum_accuracy_check for lob res", K(ret), K(expr)); + } + } + eval_flags.set(i); + } + return ret; +} + //TODO(yaoying.yyy):(duplicate_with_type_to_str, remove later int ObBaseExprColumnConv::shallow_copy_str_values(const common::ObIArray &str_values) { diff --git a/src/sql/engine/expr/ob_expr_column_conv.h b/src/sql/engine/expr/ob_expr_column_conv.h index 5ef894b9c7..58feb71ad3 100644 --- a/src/sql/engine/expr/ob_expr_column_conv.h +++ b/src/sql/engine/expr/ob_expr_column_conv.h @@ -90,6 +90,13 @@ public: PARAMS_MAX }; + enum PARAM_TC { + INT_TC, + STRING_TC, + DECIMAL_INT_TC, + OTHER_TC, + }; + static const int64_t PARAMS_COUNT_WITHOUT_COLUMN_INFO = 5; static const int64_t PARAMS_COUNT_WITH_COLUMN_INFO = 6; class ObExprColumnConvCtx : public ObExprOperatorCtx @@ -146,6 +153,21 @@ public: const ObBitVector &skip, const int64_t batch_size); + inline static bool check_is_ascii(ObString &str); + + template + static int inner_loop_for_convert_batch(const ObExpr &expr, + ObEvalCtx &ctx, + const ObBitVector &skip, + const int64_t batch_size, + const bool is_strict, + const ObLength max_accuracy_len, + const uint64_t cast_mode, + ObBitVector &eval_flags, + ObDatum *vals, + ObDatum *results, + ObEvalCtx::BatchInfoScopeGuard &batch_info_guard); + virtual bool need_rt_ctx() const override { return ob_is_enum_or_set_type(result_type_.get_type()); } diff --git a/src/sql/engine/ob_operator.cpp b/src/sql/engine/ob_operator.cpp index c71bd8fde6..d4d9545a4d 100644 --- a/src/sql/engine/ob_operator.cpp +++ b/src/sql/engine/ob_operator.cpp @@ -1850,7 +1850,7 @@ bool ObOperator::enable_get_next_row() const ret = true; } else { // if new operator is registered, please update this check and phy operator lists below - static_assert(PHY_END == PHY_VEC_SUBPLAN_FILTER + 1, ""); + static_assert(PHY_END == PHY_TABLE_DIRECT_INSERT + 1, ""); switch (spec_.type_) { case PHY_TABLE_SCAN: // table scan with multi value index/geometry type case PHY_BLOCK_SAMPLE_SCAN: // sample scan with geometry type diff --git a/src/sql/engine/ob_operator_factory.cpp b/src/sql/engine/ob_operator_factory.cpp index 0c602949b4..5c29995e23 100644 --- a/src/sql/engine/ob_operator_factory.cpp +++ b/src/sql/engine/ob_operator_factory.cpp @@ -188,9 +188,10 @@ #include "sql/engine/set/ob_merge_intersect_vec_op.h" #include "sql/engine/set/ob_merge_except_vec_op.h" #include "sql/engine/expand/ob_expand_vec_op.h" - #include "sql/engine/join/ob_nested_loop_join_vec_op.h" #include "sql/engine/subquery/ob_subplan_filter_vec_op.h" +#include "sql/engine/direct_load/ob_table_direct_insert_op.h" + namespace oceanbase { using namespace common; diff --git a/src/sql/engine/ob_operator_reg.h b/src/sql/engine/ob_operator_reg.h index 96ce541e3c..9e734184ac 100644 --- a/src/sql/engine/ob_operator_reg.h +++ b/src/sql/engine/ob_operator_reg.h @@ -786,6 +786,13 @@ REGISTER_OPERATOR(ObLogExpand, PHY_EXPAND, ObExpandVecSpec, ObExpandVecOp, NOINP 0, /*version*/ SUPPORT_RICH_FORMAT, "PHY_VEC_EXPAND"); +// table-direct-insert +class ObLogInsert; +class ObTableDirectInsertSpec; +class ObTableDirectInsertOp; +class ObTableDirectInsertOpInput; +REGISTER_OPERATOR(ObLogInsert, PHY_TABLE_DIRECT_INSERT, ObTableDirectInsertSpec, ObTableDirectInsertOp, ObTableDirectInsertOpInput, VECTORIZED_OP, 0 /*version*/, SUPPORT_RICH_FORMAT, "PHY_TABLE_DIRECT_INSERT"); + #undef REGISTER_OPERATOR #undef REGISTER_OPERATOR_FULL #undef CHECK_IS_CHAR diff --git a/src/sql/engine/ob_phy_operator_type.h b/src/sql/engine/ob_phy_operator_type.h index afcadf5691..5ba5b61527 100644 --- a/src/sql/engine/ob_phy_operator_type.h +++ b/src/sql/engine/ob_phy_operator_type.h @@ -147,6 +147,7 @@ PHY_OP_DEF(PHY_VEC_MERGE_EXCEPT) PHY_OP_DEF(PHY_EXPAND) PHY_OP_DEF(PHY_VEC_NESTED_LOOP_JOIN) PHY_OP_DEF(PHY_VEC_SUBPLAN_FILTER) +PHY_OP_DEF(PHY_TABLE_DIRECT_INSERT) PHY_OP_DEF(PHY_END) #endif /*PHY_OP_DEF*/ diff --git a/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp b/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp index 4b2b26c5f3..13beada510 100644 --- a/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp +++ b/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp @@ -15,7 +15,6 @@ #include "storage/access/ob_dml_param.h" #include "storage/tx_storage/ob_access_service.h" #include "sql/engine/dml/ob_dml_service.h" -#include "sql/engine/cmd/ob_table_direct_insert_service.h" using namespace oceanbase::common; using namespace oceanbase::sql; @@ -48,21 +47,10 @@ int ObPxMultiPartInsertOp::inner_open() } else if (OB_FAIL(data_driver_.init(get_spec(), ctx_.get_allocator(), ins_rtdef_, this, this, MY_SPEC.ins_ctdef_.is_heap_table_))) { LOG_WARN("failed to init data driver", K(ret)); - } - if (OB_SUCC(ret)) { - const ObPhysicalPlan *plan = GET_PHY_PLAN_CTX(ctx_)->get_phy_plan(); - if (GET_PHY_PLAN_CTX(ctx_)->get_is_direct_insert_plan()) { - int64_t px_task_id = ctx_.get_px_task_id() + 1; - int64_t ddl_task_id = plan->get_ddl_task_id(); - if (OB_FAIL(ObTableDirectInsertService::open_task( - plan->get_append_table_id(), px_task_id, ddl_task_id, table_ctx_))) { - LOG_WARN("failed to open table direct insert task", KR(ret), - K(plan->get_append_table_id()), K(px_task_id), K(ddl_task_id)); - } else { - ins_rtdef_.das_rtdef_.direct_insert_task_id_ = px_task_id; - ins_rtdef_.das_rtdef_.ddl_task_id_ = ddl_task_id; - } - } + } else if (OB_UNLIKELY(GET_PHY_PLAN_CTX(ctx_)->get_is_direct_insert_plan())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("direct-insert plan should not use pdml op", + KR(ret), K(GET_PHY_PLAN_CTX(ctx_)->get_is_direct_insert_plan())); } LOG_TRACE("pdml static insert op", K(ret), K_(MY_SPEC.row_desc), K_(MY_SPEC.ins_ctdef)); return ret; @@ -108,20 +96,6 @@ int ObPxMultiPartInsertOp::inner_close() { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - const ObPhysicalPlan *plan = GET_PHY_PLAN_CTX(ctx_)->get_phy_plan(); - if (GET_PHY_PLAN_CTX(ctx_)->get_is_direct_insert_plan()) { - int64_t px_task_id = ctx_.get_px_task_id() + 1; - int64_t ddl_task_id = plan->get_ddl_task_id(); - int error_code = (static_cast(input_))->get_error_code(); - if (OB_TMP_FAIL(ObTableDirectInsertService::close_task(plan->get_append_table_id(), - px_task_id, - ddl_task_id, - table_ctx_, - error_code))) { - LOG_WARN("failed to close table direct insert task", KR(tmp_ret), - K(plan->get_append_table_id()), K(px_task_id), K(ddl_task_id), K(error_code)); - } - } if (OB_FAIL(ObTableModifyOp::inner_close())) { LOG_WARN("failed to inner close table modify", K(ret)); } else { diff --git a/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.h b/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.h index 6c7f4ffbd8..9c48351922 100644 --- a/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.h +++ b/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.h @@ -107,7 +107,7 @@ public: protected: ObPDMLOpDataDriver data_driver_; ObInsRtDef ins_rtdef_; - observer::ObTableLoadTableCtx *table_ctx_; + observer::ObTableLoadTableCtx *table_ctx_; // deprecated DISALLOW_COPY_AND_ASSIGN(ObPxMultiPartInsertOp); }; diff --git a/src/sql/engine/px/ob_px_basic_info.h b/src/sql/engine/px/ob_px_basic_info.h index 9274600591..532c6a4f26 100644 --- a/src/sql/engine/px/ob_px_basic_info.h +++ b/src/sql/engine/px/ob_px_basic_info.h @@ -51,7 +51,8 @@ namespace sql #define IS_PX_MODIFY(type) \ ((type) == PHY_PX_MULTI_PART_UPDATE || \ (type) == PHY_PX_MULTI_PART_DELETE || \ - (type) == PHY_PX_MULTI_PART_INSERT) + (type) == PHY_PX_MULTI_PART_INSERT || \ + (type) == PHY_TABLE_DIRECT_INSERT) @@ -92,7 +93,8 @@ namespace sql (type) == PHY_PX_MULTI_PART_UPDATE || \ (type) == PHY_MERGE || \ (type) == PHY_PX_MULTI_PART_SSTABLE_INSERT || \ - (type) == PHY_LOCK) + (type) == PHY_LOCK || \ + (type) == PHY_TABLE_DIRECT_INSERT) enum JoinFilterMode { diff --git a/src/sql/engine/px/ob_px_task_process.cpp b/src/sql/engine/px/ob_px_task_process.cpp index 2b3a68605d..8b2be5b5fc 100644 --- a/src/sql/engine/px/ob_px_task_process.cpp +++ b/src/sql/engine/px/ob_px_task_process.cpp @@ -41,6 +41,7 @@ #include "observer/mysql/obmp_base.h" #include "lib/alloc/ob_malloc_callback.h" #include "sql/engine/window_function/ob_window_function_vec_op.h" +#include "sql/engine/direct_load/ob_table_direct_insert_op.h" using namespace oceanbase::common; using namespace oceanbase::share; @@ -901,6 +902,22 @@ int ObPxTaskProcess::OpPostparation::apply(ObExecContext &ctx, ObOpSpec &op) LOG_TRACE("debug post apply info", K(ret_)); } } + } else if (PHY_TABLE_DIRECT_INSERT == op.get_type()) { + if (OB_ISNULL(kit->input_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("operator is NULL", K(ret), KP(kit)); + } else { + ObTableDirectInsertOpInput *input = static_cast(kit->input_); + if (OB_ISNULL(input)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("input not found for op", "op_id", op.id_, K(ret)); + } else if (OB_SUCCESS != ret_) { + input->set_error_code(ret_); + LOG_TRACE("debug post apply info", K(ret_)); + } else { + LOG_TRACE("debug post apply info", K(ret_)); + } + } } return ret; } diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index c54b09bae8..1e664470d6 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -26,6 +26,7 @@ ob_set_subtarget(ob_storage blocksstable blocksstable/ob_macro_block_writer.cpp blocksstable/ob_data_macro_block_merge_writer.cpp blocksstable/ob_micro_block_cache.cpp + blocksstable/ob_micro_block_checksum_helper.cpp blocksstable/ob_micro_block_hash_index.cpp blocksstable/ob_micro_block_reader.cpp blocksstable/ob_micro_block_row_exister.cpp @@ -58,6 +59,7 @@ ob_set_subtarget(ob_storage blocksstable blocksstable/ob_column_checksum_struct.cpp blocksstable/ob_table_flag.cpp blocksstable/ob_datum_rowkey_vector.cpp + blocksstable/ob_batch_datum_rows.cpp ) ob_set_subtarget(ob_storage blocksstable_encoding @@ -104,6 +106,7 @@ ob_set_subtarget(ob_storage blocksstable_cs_encoding blocksstable/cs_encoding/ob_cs_decoding_util.cpp blocksstable/cs_encoding/ob_cs_encoding_util.cpp blocksstable/cs_encoding/ob_icolumn_cs_encoder.cpp + blocksstable/cs_encoding/ob_dict_encoding_hash_table.cpp blocksstable/cs_encoding/ob_dict_column_encoder.cpp blocksstable/cs_encoding/ob_int_dict_column_encoder.cpp blocksstable/cs_encoding/ob_str_dict_column_encoder.cpp @@ -834,6 +837,7 @@ ob_set_subtarget(ob_storage tablelock ) ob_set_subtarget(ob_storage direct_load + direct_load/ob_direct_load_batch_row_buffer.cpp direct_load/ob_direct_load_compare.cpp direct_load/ob_direct_load_conflict_check.cpp direct_load/ob_direct_load_data_block.cpp @@ -847,10 +851,10 @@ ob_set_subtarget(ob_storage direct_load direct_load/ob_direct_load_external_table.cpp direct_load/ob_direct_load_external_table_builder.cpp direct_load/ob_direct_load_external_table_compactor.cpp - direct_load/ob_direct_load_fast_heap_table.cpp - direct_load/ob_direct_load_fast_heap_table_builder.cpp direct_load/ob_direct_load_insert_table_ctx.cpp + direct_load/ob_direct_load_insert_table_row_handler.cpp direct_load/ob_direct_load_insert_table_row_iterator.cpp + direct_load/ob_direct_load_insert_table_row_writer.cpp direct_load/ob_direct_load_merge_ctx.cpp direct_load/ob_direct_load_lob_builder.cpp direct_load/ob_direct_load_merge_task_iterator.cpp @@ -886,6 +890,7 @@ ob_set_subtarget(ob_storage direct_load direct_load/ob_direct_load_partition_rescan_task.cpp direct_load/ob_direct_load_range_splitter.cpp direct_load/ob_direct_load_rowkey_iterator.cpp + direct_load/ob_direct_load_row_iterator.cpp direct_load/ob_direct_load_sstable_builder.cpp direct_load/ob_direct_load_sstable.cpp direct_load/ob_direct_load_sstable_compactor.cpp @@ -898,7 +903,6 @@ ob_set_subtarget(ob_storage direct_load direct_load/ob_direct_load_sstable_scanner.cpp direct_load/ob_direct_load_struct.cpp direct_load/ob_direct_load_table_data_desc.cpp - direct_load/ob_direct_load_table_store.cpp direct_load/ob_direct_load_tmp_file.cpp direct_load/ob_direct_load_mem_dump.cpp direct_load/ob_direct_load_mem_loader.cpp @@ -907,6 +911,7 @@ ob_set_subtarget(ob_storage direct_load direct_load/ob_direct_load_multiple_heap_table_map.cpp direct_load/ob_direct_load_multiple_heap_table_sorter.cpp direct_load/ob_direct_load_lob_meta_row_iter.cpp + direct_load/ob_direct_load_vector_utils.cpp ) ob_set_subtarget(ob_storage lob diff --git a/src/storage/access/ob_dml_param.h b/src/storage/access/ob_dml_param.h index 4d6566039d..5f199327b2 100644 --- a/src/storage/access/ob_dml_param.h +++ b/src/storage/access/ob_dml_param.h @@ -243,7 +243,6 @@ struct ObDMLBaseParam mutable ObArenaAllocator lob_allocator_; const blocksstable::ObDatumRow *data_row_for_lob_; // for tablet split bool is_valid() const { return (timeout_ > 0 && schema_version_ >= 0) && nullptr != store_ctx_guard_; } - bool is_direct_insert() const { return (direct_insert_task_id_ > 0); } DECLARE_TO_STRING; }; diff --git a/src/storage/blocksstable/cs_encoding/ob_column_datum_iter.cpp b/src/storage/blocksstable/cs_encoding/ob_column_datum_iter.cpp index 35338f513a..be5f76751d 100644 --- a/src/storage/blocksstable/cs_encoding/ob_column_datum_iter.cpp +++ b/src/storage/blocksstable/cs_encoding/ob_column_datum_iter.cpp @@ -40,7 +40,7 @@ int ObDictDatumIter::get_next(const ObDatum *&datum) if (OB_UNLIKELY(iter_ == ht_.end())) { ret = OB_ITER_END; } else { - datum = iter_->header_->datum_; + datum = &iter_->datum_; iter_++; } diff --git a/src/storage/blocksstable/cs_encoding/ob_column_datum_iter.h b/src/storage/blocksstable/cs_encoding/ob_column_datum_iter.h index 7cd607b220..f17518c310 100644 --- a/src/storage/blocksstable/cs_encoding/ob_column_datum_iter.h +++ b/src/storage/blocksstable/cs_encoding/ob_column_datum_iter.h @@ -14,8 +14,8 @@ #define OCEANBASE_ENCODING_OB_COLUMN_DATUM_ITER_H_ #include "ob_column_encoding_struct.h" +#include "ob_dict_encoding_hash_table.h" #include "storage/blocksstable/encoding/ob_encoding_util.h" -#include "storage/blocksstable/encoding/ob_encoding_hash_util.h" namespace oceanbase { @@ -52,19 +52,19 @@ private: class ObDictDatumIter : public ObIDatumIter { public: - explicit ObDictDatumIter(const ObEncodingHashTable &ht) + explicit ObDictDatumIter(const ObDictEncodingHashTable &ht) : ht_(ht), iter_(ht_.begin()) { } ~ObDictDatumIter() {} ObDictDatumIter(const ObDictDatumIter&) = delete; ObDictDatumIter &operator=(const ObDictDatumIter&) = delete; int get_next(const ObDatum *&datum) override; - int64_t size() const override { return ht_.size(); } + int64_t size() const override { return ht_.distinct_node_cnt(); } virtual void reset() override { iter_ = ht_.begin(); } private: - const ObEncodingHashTable &ht_; - ObEncodingHashTable::ConstIterator iter_; + const ObDictEncodingHashTable &ht_; + ObDictEncodingHashTable::ConstIterator iter_; }; diff --git a/src/storage/blocksstable/cs_encoding/ob_column_encoding_struct.h b/src/storage/blocksstable/cs_encoding/ob_column_encoding_struct.h index 685819efc4..be083d9b2e 100644 --- a/src/storage/blocksstable/cs_encoding/ob_column_encoding_struct.h +++ b/src/storage/blocksstable/cs_encoding/ob_column_encoding_struct.h @@ -302,7 +302,7 @@ struct ObCSEncodingOpt }; -class ObEncodingHashTable; +class ObDictEncodingHashTable; class ObMicroBlockEncodingCtx; struct ObColumnCSEncodingCtx { @@ -314,7 +314,7 @@ struct ObColumnCSEncodingCtx int64_t fix_data_size_; int64_t max_string_size_; const ObPodFix2dArray *col_datums_; - ObEncodingHashTable *ht_; + ObDictEncodingHashTable *ht_; const ObMicroBlockEncodingCtx *encoding_ctx_; ObMicroBufferWriter *all_string_buf_writer_; diff --git a/src/storage/blocksstable/cs_encoding/ob_cs_encoding_util.cpp b/src/storage/blocksstable/cs_encoding/ob_cs_encoding_util.cpp index aea7467d4c..6a4b7cf17c 100644 --- a/src/storage/blocksstable/cs_encoding/ob_cs_encoding_util.cpp +++ b/src/storage/blocksstable/cs_encoding/ob_cs_encoding_util.cpp @@ -36,17 +36,18 @@ int64_t ObCSEncodingUtil::get_bit_size(const uint64_t v) } return bit_size; } -int ObCSEncodingUtil::build_cs_column_encoding_ctx(ObEncodingHashTable *ht, - const ObObjTypeStoreClass store_class, const int64_t precision_bytes, - ObColumnCSEncodingCtx &col_ctx) +int ObCSEncodingUtil::build_cs_column_encoding_ctx( + ObDictEncodingHashTable *ht, + const ObObjTypeStoreClass store_class, + const int64_t precision_bytes, + ObColumnCSEncodingCtx &col_ctx) { int ret = OB_SUCCESS; if (OB_ISNULL(ht)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("ht is null", K(ret)); } else { - col_ctx.null_cnt_ = ht->get_null_list().size_; - col_ctx.nope_cnt_ = ht->get_nope_list().size_; + col_ctx.null_cnt_ = ht->get_null_cnt(); col_ctx.ht_ = ht; switch (store_class) { @@ -55,17 +56,13 @@ int ObCSEncodingUtil::build_cs_column_encoding_ctx(ObEncodingHashTable *ht, int64_t int_min = INT64_MAX; int64_t int_max = INT64_MIN; int64_t value = 0; - const int64_t row_count = ht->get_node_cnt(); - for (int64_t i = 0; i < row_count; ++i) { - const ObDatum &datum = *ht->get_node_list()[i].datum_; - if (!datum.is_null()) { - value = datum.get_int(); - if (value < int_min) { - int_min = value; - } - if (value > int_max) { - int_max = value; - } + FOREACH(node, *ht) { + value = node->datum_.get_int(); + if (value < int_min) { + int_min = value; + } + if (value > int_max) { + int_max = value; } } col_ctx.integer_min_ = static_cast(int_min); @@ -78,17 +75,13 @@ int ObCSEncodingUtil::build_cs_column_encoding_ctx(ObEncodingHashTable *ht, int64_t uint_min = UINT64_MAX; int64_t uint_max = 0; uint64_t value = 0; - const int64_t row_count = ht->get_node_cnt(); - for (int64_t i = 0; i < row_count; ++i) { - const ObDatum &datum = *ht->get_node_list()[i].datum_; - if (!datum.is_null()) { - value = datum.get_uint64(); - if (value < uint_min) { - uint_min = value; - } - if (value > uint_max) { - uint_max = value; - } + FOREACH(node, *ht) { + value = node->datum_.get_uint64(); + if (value < uint_min) { + uint_min = value; + } + if (value > uint_max) { + uint_max = value; } } col_ctx.integer_min_ = uint_min; @@ -106,30 +99,26 @@ int ObCSEncodingUtil::build_cs_column_encoding_ctx(ObEncodingHashTable *ht, col_ctx.fix_data_size_ = -1; col_ctx.is_wide_int_ = false; decint_cmp_fp cmp = wide::ObDecimalIntCmpSet::get_decint_decint_cmp_func(precision_bytes, sizeof(int64_t)); - const int64_t row_count = ht->get_node_cnt(); - for (int64_t i = 0; OB_SUCC(ret) && i < row_count; ++i) { - const ObDatum &datum = *ht->get_node_list()[i].datum_; - if (!datum.is_null()) { - if (OB_UNLIKELY(datum.len_ != precision_bytes)) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("datum len is not match with precision bytes", - K(ret), K(datum), K(precision_bytes), K(i), K(row_count)); - } else if (cmp(datum.get_decimal_int(), (ObDecimalInt*)&int64_min) < 0 || cmp(datum.get_decimal_int(), (ObDecimalInt*)&int64_max) > 0) { - col_ctx.is_wide_int_ = true; - break; - } else { // value range is not over int64_t, store as integer - int64_t value = 0; - if (sizeof(int32_t) == precision_bytes) { - value = datum.get_decimal_int32(); - } else { - value = datum.get_decimal_int64(); - } - if (value < int_min) { - int_min = value; - } - if (value > int_max) { - int_max = value; - } + FOREACH(node, *ht) { + const ObDatum &datum = node->datum_; + if (OB_UNLIKELY(datum.len_ != precision_bytes)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("datum len is not match with precision bytes", K(ret), K(datum), K(precision_bytes)); + } else if (cmp(datum.get_decimal_int(), (ObDecimalInt*)&int64_min) < 0 || cmp(datum.get_decimal_int(), (ObDecimalInt*)&int64_max) > 0) { + col_ctx.is_wide_int_ = true; + break; + } else { // value range is not over int64_t, store as integer + int64_t value = 0; + if (sizeof(int32_t) == precision_bytes) { + value = datum.get_decimal_int32(); + } else { + value = datum.get_decimal_int64(); + } + if (value < int_min) { + int_min = value; + } + if (value > int_max) { + int_max = value; } } } @@ -140,10 +129,9 @@ int ObCSEncodingUtil::build_cs_column_encoding_ctx(ObEncodingHashTable *ht, if (col_ctx.is_wide_int_) { // store as fixed len string col_ctx.fix_data_size_ = precision_bytes; - FOREACH(l, *ht) - { - const int64_t len = l->header_->datum_->len_; - col_ctx.var_data_size_ += len * l->size_; + FOREACH(node, *ht) { + const int64_t len = node->datum_.len_; + col_ctx.var_data_size_ += len * node->duplicate_cnt_; col_ctx.dict_var_data_size_ += len; } } @@ -154,12 +142,11 @@ int ObCSEncodingUtil::build_cs_column_encoding_ctx(ObEncodingHashTable *ht, case ObNumberSC: { col_ctx.fix_data_size_ = -1; bool var_store = false; - FOREACH(l, *ht) - { - const ObDatum &datum = *l->header_->datum_; + FOREACH(node, *ht) { + const ObDatum &datum = node->datum_; const int64_t len = sizeof(ObNumberDesc) + datum.num_->desc_.len_ * sizeof(datum.num_->digits_[0]); - col_ctx.var_data_size_ += len * l->size_; + col_ctx.var_data_size_ += len * node->duplicate_cnt_; col_ctx.dict_var_data_size_ += len; if (!var_store) { if (col_ctx.fix_data_size_ < 0) { @@ -180,11 +167,11 @@ int ObCSEncodingUtil::build_cs_column_encoding_ctx(ObEncodingHashTable *ht, col_ctx.fix_data_size_ = -1; col_ctx.max_string_size_ = -1; bool var_store = false; - FOREACH(l, *ht) + FOREACH(node, *ht) { - const int64_t len = l->header_->datum_->len_; + const int64_t len = node->datum_.len_; col_ctx.max_string_size_ = len > col_ctx.max_string_size_ ? len : col_ctx.max_string_size_; - col_ctx.var_data_size_ += len * l->size_; + col_ctx.var_data_size_ += len * node->duplicate_cnt_; col_ctx.dict_var_data_size_ += len; if (!col_ctx.has_zero_length_datum_ && 0 == len) { col_ctx.has_zero_length_datum_ = true; @@ -204,10 +191,9 @@ int ObCSEncodingUtil::build_cs_column_encoding_ctx(ObEncodingHashTable *ht, case ObOTimestampSC: { col_ctx.fix_data_size_ = -1; bool var_store = false; - FOREACH(l, *ht) - { - const int64_t len = l->header_->datum_->len_; - col_ctx.var_data_size_ += len * l->size_; + FOREACH(node, *ht) { + const int64_t len = node->datum_.len_; + col_ctx.var_data_size_ += len * node->duplicate_cnt_; col_ctx.dict_var_data_size_ += len; if (!var_store) { if (col_ctx.fix_data_size_ < 0) { @@ -224,10 +210,9 @@ int ObCSEncodingUtil::build_cs_column_encoding_ctx(ObEncodingHashTable *ht, case ObIntervalSC: { col_ctx.fix_data_size_ = -1; bool var_store = false; - FOREACH(l, *ht) - { - const int64_t len = l->header_->datum_->len_; - col_ctx.var_data_size_ += len * l->size_; + FOREACH(node, *ht) { + const int64_t len = node->datum_.len_; + col_ctx.var_data_size_ += len * node->duplicate_cnt_; col_ctx.dict_var_data_size_ += len; if (!var_store) { if (col_ctx.fix_data_size_ < 0) { diff --git a/src/storage/blocksstable/cs_encoding/ob_cs_encoding_util.h b/src/storage/blocksstable/cs_encoding/ob_cs_encoding_util.h index 3b1ef4e5c7..40bb110df6 100644 --- a/src/storage/blocksstable/cs_encoding/ob_cs_encoding_util.h +++ b/src/storage/blocksstable/cs_encoding/ob_cs_encoding_util.h @@ -14,7 +14,7 @@ #define OCEANBASE_ENCODING_OB_CS_ENCODING_UTIL_H_ #include "ob_stream_encoding_struct.h" -#include "storage/blocksstable/encoding/ob_encoding_hash_util.h" +#include "ob_dict_encoding_hash_table.h" #include "storage/blocksstable/encoding/ob_encoding_util.h" #include "ob_cs_micro_block_transformer.h" #include "ob_icolumn_cs_decoder.h" @@ -67,9 +67,17 @@ public: return sc == ObLobSC || sc == ObJsonSC || sc == ObGeometrySC || ObRoaringBitmapSC == sc; } - static int build_cs_column_encoding_ctx(ObEncodingHashTable *ht, - const ObObjTypeStoreClass store_class, const int64_t type_store_size, ObColumnCSEncodingCtx &ctx); - + static int build_cs_column_encoding_ctx(ObDictEncodingHashTable *ht, + const ObObjTypeStoreClass store_class, + const int64_t type_store_size, + ObColumnCSEncodingCtx &ctx); + static OB_INLINE bool is_int64_vec_value_tc(const VecValueTypeClass tc) + { + return tc == VEC_TC_INTEGER || tc == VEC_TC_DATETIME || tc == VEC_TC_TIME || + tc == VEC_TC_UNKNOWN || tc == VEC_TC_INTERVAL_YM || tc == VEC_TC_DEC_INT64 || + tc == VEC_TC_UINTEGER || tc == VEC_TC_BIT || tc == VEC_TC_ENUM_SET || + tc == VEC_TC_DOUBLE || tc == VEC_TC_FIXED_DOUBLE; + } }; diff --git a/src/storage/blocksstable/cs_encoding/ob_dict_column_encoder.cpp b/src/storage/blocksstable/cs_encoding/ob_dict_column_encoder.cpp index e6404e6d8c..f26a08973e 100644 --- a/src/storage/blocksstable/cs_encoding/ob_dict_column_encoder.cpp +++ b/src/storage/blocksstable/cs_encoding/ob_dict_column_encoder.cpp @@ -26,32 +26,13 @@ namespace blocksstable using namespace common; -bool ObDictColumnEncoder::DictCmp::operator()( - const ObEncodingHashNodeList &lhs, - const ObEncodingHashNodeList &rhs) -{ - bool res = false; - int &ret = ret_; - int cmp_ret = 0; - if (OB_FAIL(ret)) { - } else if (OB_UNLIKELY(nullptr == lhs.header_ || nullptr == rhs.header_)) { - ret_ = OB_INVALID_ARGUMENT; - LOG_WARN_RET(ret_, "invalid argument", KP(lhs.header_), KP(rhs.header_)); - } else if (OB_FAIL(cmp_func_.cmp_func_(*lhs.header_->datum_, *rhs.header_->datum_, cmp_ret))) { - LOG_WARN("failed to compare datums", K(ret), K(*lhs.header_->datum_), K(*rhs.header_->datum_)); - } else { - res = cmp_ret < 0; - } - return res; -} - ObDictColumnEncoder::ObDictColumnEncoder() : dict_encoding_meta_(), ref_enc_ctx_(), max_ref_(0), ref_stream_max_value_(0), int_stream_idx_(0), - const_list_header_(nullptr), + const_node_(), ref_exception_cnt_(0) { } @@ -66,7 +47,6 @@ void ObDictColumnEncoder::reuse() ref_enc_ctx_.reset(); max_ref_ = 0; int_stream_idx_ = 0; - const_list_header_ = nullptr; ref_exception_cnt_ = 0; } @@ -153,16 +133,16 @@ int ObDictColumnEncoder::try_const_encoding_ref_() int ret = OB_SUCCESS; int64_t max_const_cnt = 0; - FOREACH(l, *ctx_->ht_) { // choose the const value - if (l->size_ > max_const_cnt) { - max_const_cnt = l->size_; - const_list_header_ = l->header_; + FOREACH(node, *ctx_->ht_) { // choose the const value + if (node->duplicate_cnt_ > max_const_cnt) { + max_const_cnt = node->duplicate_cnt_; + const_node_ = *node; //copy the node for the node ptr will change when sort. } } if (ctx_->null_cnt_ > 0) { if (ctx_->null_cnt_ > max_const_cnt) { max_const_cnt = ctx_->null_cnt_; - const_list_header_ = ctx_->ht_->get_null_list().header_; + const_node_ = *ctx_->ht_->get_null_node(); } } const int64_t exception_cnt = row_count_ - max_const_cnt; @@ -172,14 +152,14 @@ int ObDictColumnEncoder::try_const_encoding_ref_() } else if (0 == exception_cnt) { ref_exception_cnt_ = 0; dict_encoding_meta_.set_const_encoding_ref(exception_cnt); - ref_stream_max_value_ = MAX(exception_cnt, const_list_header_->dict_ref_); + ref_stream_max_value_ = MAX(exception_cnt, const_node_.dict_ref_); } else if (exception_cnt <= MAX_EXCEPTION_COUNT && exception_cnt < row_count_ * MAX_CONST_EXCEPTION_PCT / 100) { dict_encoding_meta_.set_const_encoding_ref(exception_cnt); - const ObEncodingHashNode *node_list = ctx_->ht_->get_node_list(); + const int32_t *row_refs = ctx_->ht_->get_row_refs(); uint32_t exception_max_row_id = 0; - for (int64_t i = ctx_->ht_->get_node_cnt() - 1; i >= 0; --i) { - if (const_list_header_->dict_ref_ != node_list[i].dict_ref_) { + for (int64_t i = row_count_ - 1; i >= 0; --i) { + if (const_node_.dict_ref_ != row_refs[i]) { exception_max_row_id = i; break; } @@ -201,17 +181,15 @@ int ObDictColumnEncoder::do_sort_dict_() ObCmpFunc cmp_func; cmp_func.cmp_func_ = lib::is_oracle_mode() ? basic_funcs->null_last_cmp_ : basic_funcs->null_first_cmp_; - lib::ob_sort(ctx_->ht_->begin(), ctx_->ht_->end(), DictCmp(ret, cmp_func)); - // calc new dict_ref if dict is sorted - int64_t i = 0; - FOREACH(l, *ctx_->ht_) { - FOREACH(n, *l) { - n->dict_ref_ = i; + if (OB_FAIL(ctx_->ht_->sort_dict(cmp_func))) { + LOG_WARN("fail to sort dict", K(ret)); + } else { + // update dict ref for const node + if (dict_encoding_meta_.is_const_encoding_ref() && !const_node_.datum_.is_null()) { + const_node_.dict_ref_ = ctx_->ht_->get_refs_permutation()[const_node_.dict_ref_]; } - ++i; + dict_encoding_meta_.attrs_ |= ObDictEncodingMeta::Attribute::IS_SORTED; } - dict_encoding_meta_.attrs_ |= ObDictEncodingMeta::Attribute::IS_SORTED; - return ret; } @@ -232,10 +210,10 @@ int ObDictColumnEncoder::store_dict_encoding_meta_(ObMicroBufferWriter &buf_writ int ObDictColumnEncoder::store_dict_ref_(ObMicroBufferWriter &buf_writer) { int ret = OB_SUCCESS; - int64_t node_cnt = ctx_->ht_->get_node_cnt(); - if (OB_UNLIKELY(node_cnt != row_count_)) { + const int64_t row_cnt = ctx_->ht_->get_row_count(); + if (OB_UNLIKELY(row_cnt != row_count_)) { ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "dict node count must equal to row_count", K(ret), K(node_cnt), K_(row_count)); + STORAGE_LOG(WARN, "row_count mismatch", K(ret), K(row_cnt), K_(row_count)); } else if (0 == dict_encoding_meta_.distinct_val_cnt_) { // has no dict value, means all datums are null, so don't need to store ref } else { diff --git a/src/storage/blocksstable/cs_encoding/ob_dict_column_encoder.h b/src/storage/blocksstable/cs_encoding/ob_dict_column_encoder.h index 8e292bc72d..9b478e42a3 100644 --- a/src/storage/blocksstable/cs_encoding/ob_dict_column_encoder.h +++ b/src/storage/blocksstable/cs_encoding/ob_dict_column_encoder.h @@ -15,7 +15,7 @@ #include "ob_icolumn_cs_encoder.h" #include "ob_stream_encoding_struct.h" -#include "storage/blocksstable/encoding/ob_encoding_hash_util.h" +#include "ob_dict_encoding_hash_table.h" #include "ob_integer_stream_encoder.h" namespace oceanbase @@ -30,16 +30,6 @@ public: ObDictColumnEncoder(); virtual ~ObDictColumnEncoder(); - - struct DictCmp - { - explicit DictCmp(int &ret, const ObCmpFunc &cmp_func) : ret_(ret), cmp_func_(cmp_func) { } - bool operator()(const ObEncodingHashNodeList &left, const ObEncodingHashNodeList &right); - - private: - int &ret_; - const ObCmpFunc &cmp_func_; - }; void reuse() override; int get_identifier_and_stream_types( ObColumnEncodingIdentifier &identifier, const ObIntegerStream::EncodingType *&types) const override; @@ -66,7 +56,7 @@ protected: int64_t max_ref_; int64_t ref_stream_max_value_; int32_t int_stream_idx_; - ObEncodingHashNode *const_list_header_; + ObDictHashNode const_node_; uint16_t ref_exception_cnt_; // total non-const ref common::ObCompressor *compressor_; }; @@ -79,26 +69,27 @@ int ObDictColumnEncoder::do_store_dict_ref_(ObMicroBufferWriter &buf_writer) int ret = OB_SUCCESS; T *ref_arr = nullptr; int64_t ref_arr_size = sizeof(T) * dict_encoding_meta_.ref_row_cnt_; + const int32_t *row_refs = ctx_->ht_->get_row_refs(); if (OB_ISNULL(ref_arr = static_cast(ctx_->allocator_->alloc(ref_arr_size)))) { ret = OB_ALLOCATE_MEMORY_FAILED; STORAGE_LOG(WARN, "fail to alloc", K(ret), K(ref_arr_size), K_(dict_encoding_meta)); } else if (dict_encoding_meta_.is_const_encoding_ref()) { - const ObEncodingHashNode *node_list = ctx_->ht_->get_node_list(); ref_arr[0] = ref_exception_cnt_; - ref_arr[1] = const_list_header_->dict_ref_; + ref_arr[1] = const_node_.dict_ref_; if (dict_encoding_meta_.ref_row_cnt_ > 2) { T *row_id_arr = ref_arr + 2; T *ref_exception_arr = ref_arr + 2 + ref_exception_cnt_; int64_t idx = 0; for (int64_t row_id = 0; OB_SUCC(ret) && row_id < row_count_; ++row_id) { - const ObEncodingHashNode &node = node_list[row_id]; - if (const_list_header_->dict_ref_ != node.dict_ref_) { + const int32_t dict_ref = row_refs[row_id]; + if (const_node_.dict_ref_ != dict_ref) { if (OB_UNLIKELY(idx >= ref_exception_cnt_)) { ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected idx", K(ret), K(idx), K_(ref_exception_cnt)); + STORAGE_LOG(WARN, "unexpected idx", K(ret), K(idx), K_(ref_exception_cnt), + K_(const_node), K(row_id), K(row_count_), K(dict_ref)); } else { row_id_arr[idx] = row_id; - ref_exception_arr[idx] = node.dict_ref_; + ref_exception_arr[idx] = dict_ref; ++idx; } } @@ -106,7 +97,7 @@ int ObDictColumnEncoder::do_store_dict_ref_(ObMicroBufferWriter &buf_writer) } } else { for (int64_t i = 0; i < row_count_; i++) { - ref_arr[i] = ctx_->ht_->get_node_list()[i].dict_ref_; + ref_arr[i] = row_refs[i]; } } ObIntegerStreamEncoder integer_encoder; diff --git a/src/storage/blocksstable/cs_encoding/ob_dict_encoding_hash_table.cpp b/src/storage/blocksstable/cs_encoding/ob_dict_encoding_hash_table.cpp new file mode 100644 index 0000000000..fbe5cd6576 --- /dev/null +++ b/src/storage/blocksstable/cs_encoding/ob_dict_encoding_hash_table.cpp @@ -0,0 +1,309 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "ob_dict_encoding_hash_table.h" +#include +#include "lib/container/ob_array.h" +#include "lib/container/ob_array_iterator.h" + +namespace oceanbase +{ +namespace blocksstable +{ +using namespace common; + +bool ObDictNodeCmp::operator()(const ObDictHashNode &lhs, const ObDictHashNode &rhs) +{ + bool res = false; + int &ret = ret_; + int cmp_ret = 0; + if (OB_FAIL(ret)) { + } else if (OB_FAIL(cmp_func_.cmp_func_(lhs.datum_, rhs.datum_, cmp_ret))) { + LOG_WARN("failed to compare datums", K(ret), K(lhs.datum_), K(rhs.datum_)); + } else { + res = cmp_ret < 0; + } + return res; +} + +ObDictEncodingHashTable::ObDictEncodingHashTable() + : is_created_(false), is_sorted_(false), + bucket_num_(0), node_num_(0), distinct_node_cnt_(0), + buckets_(NULL), nodes_(NULL), + row_refs_(nullptr), refs_permutation_(nullptr), null_node_(nullptr), + alloc_(blocksstable::OB_ENCODING_LABEL_HASH_TABLE, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()) +{ +} + +ObDictEncodingHashTable::~ObDictEncodingHashTable() +{ + reset(); +} + +int ObDictEncodingHashTable::create(const int64_t bucket_num, const int64_t node_num) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_created_)) { + ret = OB_INIT_TWICE; + LOG_WARN("already created", K(ret)); + } else if (OB_UNLIKELY(0 >= bucket_num || 0 >= node_num)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), K(bucket_num), K(node_num)); + } else if (0 != (bucket_num & (bucket_num - 1))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("bucket number must be power of 2", K(ret), K(bucket_num)); + } else { + bucket_num_ = bucket_num; + // if node_num only increase little, can still reuse hashtable + node_num_ = max(bucket_num, node_num); + + const int64_t bucket_size = bucket_num_ * static_cast(sizeof(HashBucket)); + const int64_t nodes_size = node_num_ * static_cast(sizeof(HashNode)); + const int64_t refs_size = node_num_ * sizeof(*row_refs_); + const int64_t refs_permutation_size = node_num_ * sizeof(*refs_permutation_); + + if (OB_ISNULL(buckets_ = reinterpret_cast(alloc_.alloc(bucket_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for bucket", K(ret), K(bucket_size)); + } else if (OB_ISNULL(nodes_ = reinterpret_cast(alloc_.alloc(nodes_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for nodes", K(ret), K(nodes_size)); + } else if (OB_ISNULL(row_refs_ = reinterpret_cast(alloc_.alloc(refs_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for row refs", K(ret), K(refs_size)); + } else if (OB_ISNULL(refs_permutation_ = reinterpret_cast(alloc_.alloc(refs_permutation_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for refs permutation", K(ret), K(refs_permutation_size)); + } else { + MEMSET(buckets_, 0, bucket_size); + // nodes_ no need to memset; + null_node_ = &nodes_[node_num_ -1]; // last node in nodes_ is used for null datum + null_node_->reset(); + is_created_ = true; + } + } + return ret; +} + +void ObDictEncodingHashTable::reset() +{ + alloc_.reuse(); + bucket_num_ = 0; + node_num_ = 0; + distinct_node_cnt_ = 0; + buckets_ = NULL; + nodes_ = NULL; + row_refs_ = nullptr; + null_node_ = nullptr; + is_created_ = false; + is_sorted_ = false; +} + +void ObDictEncodingHashTable::reuse() +{ + MEMSET(buckets_, 0, bucket_num_ * sizeof(HashBucket)); + // nodes_ no need to reset + // row_refs_ no need to reset + null_node_->reset(); + distinct_node_cnt_ = 0; + is_sorted_ = false; +} + +int ObDictEncodingHashTable::sort_dict(ObCmpFunc &cmp_func) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_created_)) { + ret = common::OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (OB_UNLIKELY(is_sorted_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dict has been sorted", K(ret), K_(row_count), K_(distinct_node_cnt)); + } else { + lib::ob_sort(begin(), end(), ObDictNodeCmp(ret, cmp_func)); + } + + if (OB_SUCC(ret)) { + for (int64_t i = 0; i < distinct_node_cnt_; i++) { + refs_permutation_[nodes_[i].dict_ref_] = i; + } + // calc new dict_ref if dict is sorted + if (null_node_->duplicate_cnt_ > 0) { // has null + for(int64_t i = 0; i < row_count_; i++) { + if (row_refs_[i] < distinct_node_cnt_) { + row_refs_[i] = refs_permutation_[row_refs_[i]]; + } + } + } else { + for(int64_t i = 0; i < row_count_; i++) { + row_refs_[i] = refs_permutation_[row_refs_[i]]; + } + } + is_sorted_ = true; + } + return ret; +} + +int ObDictEncodingHashTableBuilder::build(const ObColDatums &col_datums, const ObColDesc &col_desc) +{ + int ret = common::OB_SUCCESS; + if (OB_UNLIKELY(!is_created_)) { + ret = common::OB_NOT_INIT; + STORAGE_LOG(WARN, "not inited", K(ret)); + } else if (OB_UNLIKELY(col_datums.empty() || node_num_ < col_datums.count())) { + ret = common::OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", K(ret), K_(node_num), "row_count", col_datums.count()); + } else { + row_count_ = col_datums.count(); + const uint64_t mask = (bucket_num_ - 1); + + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < row_count_; ++row_idx) { + const ObDatum &datum = col_datums.at(row_idx); + if (datum.is_null()) { + null_node_->duplicate_cnt_++; + row_refs_[row_idx] = NULL_REF; + } else if (datum.is_ext()) { + ret = common::OB_NOT_SUPPORTED; + STORAGE_LOG(WARN, "not supported extend object type", + K(ret), K(row_idx), K(datum), K(*datum.extend_obj_)); + } else { + // add to table + uint64_t pos = ::murmurhash2(datum.ptr_, datum.len_, 0/*seed*/); + pos = pos & mask; + HashNode *node = buckets_[pos]; + bool is_equal = false; + while (OB_SUCC(ret) && nullptr != node) { + // binary equal for store types need char case or precision handling + if (node->datum_.pack_ != datum.pack_) { + is_equal = false; + } else { + is_equal = (0 == MEMCMP(node->datum_.ptr_, datum.ptr_, datum.len_)); + } + if (is_equal) { + node->duplicate_cnt_++; + row_refs_[row_idx] = node->dict_ref_; + break; + } else { + node = node->next_; + } + } + if (OB_SUCC(ret) && nullptr == node) { + node = &nodes_[distinct_node_cnt_]; + node->init(datum, distinct_node_cnt_, buckets_[pos]); + distinct_node_cnt_++; + buckets_[pos] = node; + row_refs_[row_idx] = node->dict_ref_; + } + } + } + + if (OB_SUCC(ret) && null_node_->duplicate_cnt_ > 0) { + null_node_->datum_.set_null(); + null_node_->dict_ref_ = distinct_node_cnt_; + for (int64_t i = 0; i < row_count_; i++) { + if (row_refs_[i] == NULL_REF) { + row_refs_[i] = distinct_node_cnt_; // use distinct_node_cnt_ as null replaced ref + } + } + } + } + return ret; +} + +ObDictEncodingHashTableFactory::ObDictEncodingHashTableFactory() + : allocator_(OB_MALLOC_NORMAL_BLOCK_SIZE, ObMalloc(blocksstable::OB_ENCODING_LABEL_HT_FACTORY)), + hashtables_() +{ + lib::ObMemAttr attr(MTL_ID(), blocksstable::OB_ENCODING_LABEL_HT_FACTORY); + allocator_.set_attr(attr); + hashtables_.set_attr(attr); +} + +ObDictEncodingHashTableFactory::~ObDictEncodingHashTableFactory() +{ + clear(); +} + +int ObDictEncodingHashTableFactory::create(const int64_t bucket_num, + const int64_t node_num, + ObDictEncodingHashTable *&hashtable) +{ + int ret = OB_SUCCESS; + hashtable = NULL; + if (bucket_num <= 0 || node_num <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), K(bucket_num), K(node_num)); + } else if (hashtables_.count() > 0) { + // we assume most time hashtables cached with same size, so only check one hashtable + ObDictEncodingHashTable *cache_hashtable = hashtables_[hashtables_.count() - 1]; + if (NULL == cache_hashtable) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("cache_hashtable is null", K(ret)); + } else if (!cache_hashtable->created()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("all hashtable assume already created", K(ret)); + } else if (cache_hashtable->get_bucket_num() >= bucket_num + && cache_hashtable->get_node_num() >= node_num) { + cache_hashtable->reuse(); + hashtable = cache_hashtable; + hashtables_.pop_back(); + } else { + // clear all cached hashtable + clear(); + } + } + + if (OB_SUCC(ret) && NULL == hashtable) { + if (NULL == (hashtable = allocator_.alloc())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc failed", K(ret)); + } else if (OB_FAIL(hashtable->create(bucket_num, node_num))) { + LOG_WARN("hashtable create failed", K(ret), K(bucket_num), K(node_num)); + // free it directly + allocator_.free(hashtable); + } + } + return ret; +} + +int ObDictEncodingHashTableFactory::recycle(const bool force_cache, ObDictEncodingHashTable *hashtable) +{ + int ret = OB_SUCCESS; + if (NULL == hashtable) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KP(hashtable), K(ret)); + } else if (!hashtable->created()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("hashtable not created", K(ret)); + } else if (!force_cache && hashtable->distinct_node_cnt() >= MAX_CACHED_HASHTABLE_SIZE) { + allocator_.free(hashtable); + } else { + if (OB_FAIL(hashtables_.push_back(hashtable))) { + LOG_WARN("push_back failed", K(ret)); + // free it + allocator_.free(hashtable); + } + } + return ret; +} + +void ObDictEncodingHashTableFactory::clear() +{ + FOREACH(hashtable, hashtables_) { + allocator_.free(*hashtable); + } + hashtables_.reuse(); +} + +} // end namespace blocksstable +} // end namespace oceanbase diff --git a/src/storage/blocksstable/cs_encoding/ob_dict_encoding_hash_table.h b/src/storage/blocksstable/cs_encoding/ob_dict_encoding_hash_table.h new file mode 100644 index 0000000000..1ec6dd6b3e --- /dev/null +++ b/src/storage/blocksstable/cs_encoding/ob_dict_encoding_hash_table.h @@ -0,0 +1,155 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_CS_ENCODING_OB_DICT_ENCODING_HASH_TABLE_H_ +#define OCEANBASE_CS_ENCODING_OB_DICT_ENCODING_HASH_TABLE_H_ + +#include "lib/hash/xxhash.h" +#include +#include "lib/allocator/ob_allocator.h" +#include "lib/allocator/ob_pooled_allocator.h" +#include "common/object/ob_object.h" +#include "storage/blocksstable/ob_block_sstable_struct.h" + +namespace oceanbase +{ +namespace blocksstable +{ +struct ObDictHashNode +{ + ObDatum datum_; + int32_t dict_ref_; + int32_t duplicate_cnt_; + ObDictHashNode *next_; + + OB_INLINE void init(const ObDatum datum, const int32_t dict_ref, ObDictHashNode *next) + { + datum_ = datum; + dict_ref_ = dict_ref; + duplicate_cnt_ = 1; + next_ = next; + } + OB_INLINE void reset() + { + datum_.reset(); + dict_ref_ = -1; + duplicate_cnt_ = 0; + next_ = nullptr; + } + TO_STRING_KV(K_(datum), K_(dict_ref), K_(duplicate_cnt), KP_(next)); +}; + +struct ObDictNodeCmp +{ + explicit ObDictNodeCmp(int &ret, const ObCmpFunc &cmp_func) : ret_(ret), cmp_func_(cmp_func) { } + bool operator()(const ObDictHashNode &left, const ObDictHashNode &right); + +private: + int &ret_; + const ObCmpFunc &cmp_func_; +}; + + +class ObDictEncodingHashTable +{ +public: + typedef ObDictHashNode *HashBucket; + typedef ObDictHashNode HashNode; + typedef HashNode *Iterator; + typedef const HashNode *ConstIterator; + + ObDictEncodingHashTable(); + ~ObDictEncodingHashTable(); + + int create(const int64_t bucket_num, const int64_t node_num); + inline bool created() const { return is_created_; } + void reset(); + void reuse(); + + OB_INLINE int64_t get_node_num() const { return node_num_; } + + // return distinct cell count, include NULL + OB_INLINE int64_t distinct_cnt() const + { + return distinct_val_cnt() + (null_node_->duplicate_cnt_ > 0 ? 1 : 0); + } + // return distinct value count, exclude NULL, NOPE + OB_INLINE int64_t distinct_val_cnt() const { return distinct_node_cnt_; } + OB_INLINE int64_t distinct_node_cnt() const { return distinct_node_cnt_; } + OB_INLINE int64_t get_bucket_num() const { return bucket_num_; } + OB_INLINE int64_t get_null_cnt() const { return null_node_->duplicate_cnt_; } + OB_INLINE HashNode *get_null_node() const { return null_node_; } + OB_INLINE int32_t *get_row_refs() const { return row_refs_; } + OB_INLINE int32_t *get_refs_permutation() const { return refs_permutation_; } + OB_INLINE int64_t get_row_count() const { return row_count_; } + + + Iterator begin() { return nodes_; } + + Iterator end() { return nodes_ + distinct_node_cnt_; } + + ConstIterator begin() const { return nodes_; } + + ConstIterator end() const { return nodes_ + distinct_node_cnt_; } + int sort_dict(ObCmpFunc &cmp_func); + + TO_STRING_KV(K_(is_created), K_(is_sorted), K_(bucket_num), K_(node_num), + K_(distinct_node_cnt), KP_(buckets), KP_(nodes), KP_(null_node)); + +protected: + static const int32_t NULL_REF = INT32_MAX; + + bool is_created_; + bool is_sorted_; + int64_t bucket_num_; + int64_t node_num_; + int32_t distinct_node_cnt_; + int32_t row_count_; + HashBucket *buckets_; + HashNode *nodes_; + int32_t *row_refs_; + int32_t *refs_permutation_; + HashNode *null_node_; + common::ObArenaAllocator alloc_; + +private: + DISALLOW_COPY_AND_ASSIGN(ObDictEncodingHashTable); +}; + +class ObDictEncodingHashTableBuilder : public ObDictEncodingHashTable +{ +public: + int build(const ObColDatums &col_datums, const ObColDesc &col_desc); +}; + +class ObDictEncodingHashTableFactory +{ +public: + ObDictEncodingHashTableFactory(); + virtual ~ObDictEncodingHashTableFactory(); + + int create(const int64_t bucket_num, const int64_t node_num, + ObDictEncodingHashTable *&hashtable); + int recycle(const bool force_cache, ObDictEncodingHashTable *hashtable); +private: + void clear(); + + // release hashtable too large to reduce memory usage + static const int64_t MAX_CACHED_HASHTABLE_SIZE = UINT16_MAX; + common::ObPooledAllocator allocator_; + common::ObArray hashtables_; +}; + +} // end namespace blocksstable +} // end namespace oceanbase + +#endif // OCEANBASE_CS_ENCODING_OB_DICT_ENCODING_HASH_TABLE_H_ diff --git a/src/storage/blocksstable/cs_encoding/ob_int_dict_column_encoder.cpp b/src/storage/blocksstable/cs_encoding/ob_int_dict_column_encoder.cpp index 89f5e9e3dd..6f4879d9a1 100644 --- a/src/storage/blocksstable/cs_encoding/ob_int_dict_column_encoder.cpp +++ b/src/storage/blocksstable/cs_encoding/ob_int_dict_column_encoder.cpp @@ -38,7 +38,7 @@ int ObIntDictColumnEncoder::init( LOG_WARN("init base column encoder failed", K(ret), K(ctx), K(column_index), K(row_count)); } else { column_header_.type_ = type_; - dict_encoding_meta_.distinct_val_cnt_ = ctx.ht_->size(); + dict_encoding_meta_.distinct_val_cnt_ = ctx.ht_->distinct_val_cnt(); dict_encoding_meta_.ref_row_cnt_ = row_count_; if (ctx_->null_cnt_ > 0) { dict_encoding_meta_.set_has_null(); @@ -146,15 +146,15 @@ int ObIntDictColumnEncoder::sort_dict_() uint64_t min = 0; uint64_t max = 0; if (ObIntTC == tc || ObUIntTC == tc) { - min = ctx_->ht_->begin()->header_->datum_->get_uint64(); - max = (ctx_->ht_->end() - 1)->header_->datum_->get_uint64(); + min = ctx_->ht_->begin()->datum_.get_uint64(); + max = (ctx_->ht_->end() - 1)->datum_.get_uint64(); } else if (ObDecimalIntSC == store_class_) { if (precision_width_size_ == sizeof(uint32_t)) { - min = ctx_->ht_->begin()->header_->datum_->get_decimal_int32(); - max = (ctx_->ht_->end() - 1)->header_->datum_->get_decimal_int32(); + min = ctx_->ht_->begin()->datum_.get_decimal_int32(); + max = (ctx_->ht_->end() - 1)->datum_.get_decimal_int32(); } else { - min = ctx_->ht_->begin()->header_->datum_->get_decimal_int64(); - max = (ctx_->ht_->end() - 1)->header_->datum_->get_decimal_int64(); + min = ctx_->ht_->begin()->datum_.get_decimal_int64(); + max = (ctx_->ht_->end() - 1)->datum_.get_decimal_int64(); } } if (ObIntTC == tc || ObUIntTC == tc || ObDecimalIntSC == store_class_) { diff --git a/src/storage/blocksstable/cs_encoding/ob_micro_block_cs_encoder.cpp b/src/storage/blocksstable/cs_encoding/ob_micro_block_cs_encoder.cpp index f33af1c0c4..ca2a7c3be8 100644 --- a/src/storage/blocksstable/cs_encoding/ob_micro_block_cs_encoder.cpp +++ b/src/storage/blocksstable/cs_encoding/ob_micro_block_cs_encoder.cpp @@ -31,6 +31,29 @@ namespace blocksstable { using namespace common; +int ObVecBatchInfo::init(const int32_t row_count, + const int32_t fixed_len, + const uint32_t *offsets, + sql::ObBitVector *nulls, + const uint32_t start_offset, + const bool is_integer) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(row_count <= 0 || (fixed_len < 0 && nullptr == offsets) + || (fixed_len >= 0 && nullptr != offsets))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", K(ret), K(row_count), K(fixed_len), KP(offsets)); + } else { + row_count_ = row_count; + fixed_len_ = fixed_len; + offsets_ = offsets; + nulls_ = nulls; + start_offset_ = start_offset; + is_integer_ = is_integer; + } + return ret; +} + template T *ObMicroBlockCSEncoder::alloc_encoder_() { @@ -67,7 +90,7 @@ int ObMicroBlockCSEncoder::alloc_and_init_encoder_(const int64_t column_index, O LOG_WARN("alloc encoder failed", K(ret)); } else { col_ctxs_.at(column_index).try_set_need_sort(e->get_type(), column_index, has_lob_out_row_); - if (OB_FAIL(e->init(col_ctxs_.at(column_index), column_index, datum_row_offset_arr_.count()))) { + if (OB_FAIL(e->init(col_ctxs_.at(column_index), column_index, appended_row_count_))) { LOG_WARN("init column encoder failed", K(ret), K(column_index)); } if (OB_FAIL(ret)) { @@ -87,6 +110,8 @@ ObMicroBlockCSEncoder::ObMicroBlockCSEncoder() all_col_datums_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator("CSBlkEnc", MTL_ID())), pivot_allocator_(lib::ObMemAttr(MTL_ID(), blocksstable::OB_ENCODING_LABEL_PIVOT), OB_MALLOC_MIDDLE_BLOCK_SIZE), datum_row_offset_arr_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator("CSBlkEnc", MTL_ID())), + vec_batch_info_arrs_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator("CSBlkEnc", MTL_ID())), + appended_batch_count_(0), appended_row_count_(0), estimate_size_(0), estimate_size_limit_(0), all_headers_size_(0), expand_pct_(DEFAULT_ESTIMATE_REAL_SIZE_PCT), encoders_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator("CSBlkEnc", MTL_ID())), @@ -193,6 +218,18 @@ void ObMicroBlockCSEncoder::reset() } } all_col_datums_.reset(); + + FOREACH(iter, vec_batch_info_arrs_) + { + ObVecBatchInfoArr *arr = *iter; + if (nullptr != arr) { + arr->~ObVecBatchInfoArr(); + pivot_allocator_.free(arr); + } + } + vec_batch_info_arrs_.reset(); + appended_batch_count_ = 0; + appended_row_count_ = 0; pivot_allocator_.reset(); datum_row_offset_arr_.reset(); estimate_size_ = 0; @@ -222,6 +259,13 @@ void ObMicroBlockCSEncoder::reuse() { (*c)->reuse(); } + + FOREACH(arr, vec_batch_info_arrs_) + { + (*arr)->reuse(); + } + appended_batch_count_ = 0; + appended_row_count_ = 0; // pivot_allocator_ pivot array memory is cached until encoder reset() row_buf_holder_.reuse(); all_string_data_buffer_.reuse(); @@ -266,8 +310,7 @@ void ObMicroBlockCSEncoder::dump_diagnose_info() const } else if (OB_FAIL(datum_row.init(column_cnt))) { LOG_WARN("fail to init datum row", K_(ctx), K(column_cnt)); } else { - int64_t appended_row_count = datum_row_offset_arr_.count(); - for (int64_t row_id = 0; OB_SUCC(ret) && row_id < appended_row_count; ++row_id) { + for (int64_t row_id = 0; OB_SUCC(ret) && row_id < appended_row_count_; ++row_id) { for (int64_t col_idx = 0; col_idx < column_cnt; ++col_idx) { datum_row.storage_datums_[col_idx].ptr_ = all_col_datums_.at(col_idx)->at(row_id).ptr_; datum_row.storage_datums_[col_idx].pack_ = all_col_datums_.at(col_idx)->at(row_id).pack_; @@ -310,6 +353,24 @@ int ObMicroBlockCSEncoder::init_all_col_values_(const ObMicroBlockEncodingCtx &c return ret; } +int ObMicroBlockCSEncoder::init_vec_batch_info_arrs_(const ObMicroBlockEncodingCtx &ctx) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(vec_batch_info_arrs_.reserve(ctx.column_cnt_))) { + LOG_WARN("reserve array failed", K(ret), "size", ctx.column_cnt_); + } + for (int64_t i = vec_batch_info_arrs_.count(); i < ctx.column_cnt_ && OB_SUCC(ret); ++i) { + ObVecBatchInfoArr *arr = OB_NEWx(ObVecBatchInfoArr, &pivot_allocator_, pivot_allocator_); + if (OB_ISNULL(arr)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed", K(ret), K(ctx)); + } else if (OB_FAIL(vec_batch_info_arrs_.push_back(arr))) { + LOG_WARN("push back vec_batch_info_arrs_ failed", K(ret)); + } + } + return ret; +} + void ObMicroBlockCSEncoder::print_micro_block_encoder_status_() { FLOG_INFO("Build micro block failed, print encoder status: ", K_(estimate_size), @@ -369,7 +430,7 @@ int ObMicroBlockCSEncoder::append_row(const ObDatumRow &row) LOG_WARN("column count mismatch", K(ret), "ctx", ctx_, K(row)); } else if (OB_FAIL(inner_init_())) { LOG_WARN("failed to inner init", K(ret)); - } else if (0 == datum_row_offset_arr_.count()) { + } else if (0 == appended_row_count_) { if (OB_FAIL(init_column_ctxs_())) { LOG_WARN("fail to init_column_ctxs_", K(ret), K_(ctx)); } @@ -377,16 +438,11 @@ int ObMicroBlockCSEncoder::append_row(const ObDatumRow &row) if (OB_SUCC(ret)) { int64_t store_size = 0; - uint64_t row_count = (uint64_t)(datum_row_offset_arr_.count()); - if (OB_UNLIKELY(row_count >= ObCSEncodingUtil::MAX_MICRO_BLOCK_ROW_CNT - || row_count > ctx_.encoding_granularity_)) { + if (OB_UNLIKELY(appended_row_count_ >= ObCSEncodingUtil::MAX_MICRO_BLOCK_ROW_CNT + || appended_row_count_ > ctx_.encoding_granularity_)) { ret = OB_BUF_NOT_ENOUGH; LOG_INFO("Try to encode more rows than maximum of row cnt in header, force to build a block", - K(datum_row_offset_arr_.count()), K(row), K(ctx_.encoding_granularity_)); - } else if (OB_FAIL(process_out_row_columns_(row))) { - if (OB_UNLIKELY(OB_BUF_NOT_ENOUGH != ret)) { - LOG_WARN("failed to process out row columns", K(ret)); - } + K_(appended_row_count), K(row), K(ctx_.encoding_granularity_)); } else if (OB_FAIL(copy_and_append_row_(row, store_size))) { if (OB_UNLIKELY(OB_BUF_NOT_ENOUGH != ret)) { LOG_WARN("copy and append row failed", K(ret)); @@ -401,13 +457,595 @@ int ObMicroBlockCSEncoder::append_row(const ObDatumRow &row) } cal_row_stat(row); estimate_size_ += store_size; - LOG_DEBUG("cs encoder append row", K_(estimate_size), K(store_size), K(datum_row_offset_arr_.count())); + LOG_DEBUG("cs encoder append row", K_(estimate_size), K(store_size), K_(appended_row_count)); } } return ret; } +int ObMicroBlockCSEncoder::append_batch(const ObBatchDatumRows &vec_batch, + const int64_t start, + const int64_t row_count) + +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_UNLIKELY(vec_batch.vectors_.count() != ctx_.column_cnt_ || row_count <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("column count mismatch", K(ret), K_(ctx), K(vec_batch), K(row_count)); + } else if (OB_FAIL(inner_init_())) { + LOG_WARN("failed to inner init", K(ret)); + } else if (0 == appended_batch_count_) { + if (OB_FAIL(init_column_ctxs_())) { + LOG_WARN("fail to init_column_ctxs_", K(ret), K_(ctx)); + } else if (vec_batch_info_arrs_.count() < ctx_.column_cnt_ && + OB_FAIL(init_vec_batch_info_arrs_(ctx_))) { + LOG_WARN("init vec_batch_info_arrs_ failed", K(ret), K_(ctx)); + } + } + if (OB_SUCC(ret)) { + int64_t store_size = 0; + if (OB_UNLIKELY(appended_row_count_ >= ObCSEncodingUtil::MAX_MICRO_BLOCK_ROW_CNT + || appended_row_count_ > ctx_.encoding_granularity_)) { + ret = OB_BUF_NOT_ENOUGH; + LOG_INFO("Try to encode more rows than maximum of row cnt in header, force to build a block", + K_(appended_row_count), K_(appended_batch_count), K(ctx_.encoding_granularity_)); + } else if (OB_FAIL(copy_and_append_batch_(vec_batch, start, row_count, store_size))) { + if (OB_UNLIKELY(OB_BUF_NOT_ENOUGH != ret)) { + LOG_WARN("copy and append row failed", K(ret)); + } else if (0 == appended_batch_count_ && 1 == row_count) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("At least encode one row", K(ret), K(vec_batch), K(start), K(row_count)); + } else if (OB_UNLIKELY(0 == appended_batch_count_)) { + reuse(); + } + } else { + // The statistics here are only meaningful for the mini/minor sstable, + // and cs-encoding is only used for major sstable, so there is no need to call this func. + // cal_row_stat(row); + estimate_size_ += store_size; + LOG_DEBUG("cs encoder append batch", K_(estimate_size), K(store_size), K_(appended_row_count)); + } + } + + return ret; +} + +int ObMicroBlockCSEncoder::copy_and_append_batch_(const ObBatchDatumRows &vec_batch, + const int64_t start, + const int64_t row_count, + int64_t &store_size) +{ + int ret = OB_SUCCESS; + // performance critical, do not double check parameters in private method + const int64_t column_cnt = vec_batch.vectors_.count(); + const int64_t start_offset = length_; + if (appended_row_count_ > 0 && estimate_size_ >= estimate_size_limit_) { + ret = OB_BUF_NOT_ENOUGH; + } else { + bool is_finish = false; + while(OB_SUCC(ret) && !is_finish) { + store_size = 0; + length_ = start_offset; + is_finish = true; + bool is_row_holder_not_enough = false; + for (int64_t col_idx = 0; OB_SUCC(ret) && col_idx < column_cnt; ++col_idx) { + if (OB_FAIL(copy_vector_(vec_batch.vectors_.at(col_idx), + start, + row_count, + col_idx, + store_size, + is_row_holder_not_enough))) { + if (OB_UNLIKELY(OB_BUF_NOT_ENOUGH != ret)) { + LOG_WARN("fail to copy vector", K(ret), K(col_idx), K(store_size)); + } else { + // failed to append batch due to buf not enough, but some vec batch may has been push back, need rollback + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(remove_invalid_vec_batch_info_(col_idx))) { + LOG_WARN("fail to remove_invalid_vec_batch_info_", K(ret), K(tmp_ret), K(column_cnt)); + ret = tmp_ret; + } + } + } else if (is_row_holder_not_enough) { + int64_t need_size = 0; + if (OB_FAIL(calc_batch_data_size_(vec_batch.vectors_, start, row_count, need_size))) { + LOG_WARN("fail to calc_vec_batch_data_size", K(ret), K(row_count)); + } else if (OB_UNLIKELY(need_size <= row_buf_holder_.size() - row_buf_holder_.length())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected row holder buf not enough", K(ret), K(need_size), K(row_buf_holder_)); + } else if (OB_FAIL(row_buf_holder_.ensure_space(need_size))) { + LOG_WARN("failed to ensure space", K(ret), K(need_size), K(row_buf_holder_)); + } else if (OB_FAIL(remove_invalid_vec_batch_info_(col_idx))) { + LOG_WARN("fail to remove_invalid_vec_batch_info_", K(ret), K(column_cnt)); + } else { + is_finish = false; + break; + } + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(row_buf_holder_.write_nop(length_ - start_offset))) { + STORAGE_LOG(WARN, "fail to write nop", K(ret), K(length_), K(start_offset), K(row_buf_holder_)); + } + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(try_to_append_row_(store_size))) { + if (OB_UNLIKELY(OB_BUF_NOT_ENOUGH != ret)) { + LOG_WARN("fail to try append row", K(ret)); + } else { + // failed to try_to_append_row due to buf not enough, but this row has been push back, need rollback + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(remove_invalid_vec_batch_info_(column_cnt))) { + LOG_WARN("fail to remove_invalid_vec_batch_info_", K(ret), K(tmp_ret), K(column_cnt)); + ret = tmp_ret; + } + } + } else { + appended_batch_count_++; + appended_row_count_ += row_count; + } + return ret; +} + +int ObMicroBlockCSEncoder::copy_vector_(const ObIVector *vector, + const int64_t start, + const int64_t row_count, + const int64_t col_idx, + int64_t &store_size, + bool &is_row_holder_not_enough) +{ + int ret = OB_SUCCESS; + const ObColDesc &col_desc = ctx_.col_descs_->at(col_idx); + ObObjTypeStoreClass store_class = get_store_class_map()[col_desc.col_type_.get_type_class()]; + const bool is_int_sc = store_class == ObIntSC || store_class == ObUIntSC; + is_row_holder_not_enough = false; + int64_t extra_store_size_for_var_string = 0; + const VectorFormat vec_format = vector->get_format(); + const int16_t precision = col_desc.col_type_.is_decimal_int() ? + col_desc.col_type_.get_stored_precision() : PRECISION_UNKNOWN_YET; + const VecValueTypeClass vec_tc = common::get_vec_value_tc( + col_desc.col_type_.get_type(), col_desc.col_type_.get_scale(), precision); + int64_t vec_data_size = 0; + ObDataFormatType data_format_type = ObDataFormatType::MAX; + + if (OB_FAIL(calc_col_batch_data_size_(vector, start, row_count, col_idx, vec_data_size))) { + LOG_WARN("fail to calc_col_batch_data_size_", K(ret), K(col_desc)); + } else if (is_int_sc) { + store_size += sizeof(uint64_t) * row_count; + if (VectorFormat::VEC_FIXED == vec_format) { + if (ObCSEncodingUtil::is_int64_vec_value_tc(vec_tc)) { + data_format_type = INT_64BIT_FIXED; + } else { + data_format_type = INT_NOT_64BIT_FIXED; + } + } else if (VectorFormat::VEC_CONTINUOUS == vec_format) { + data_format_type = INT_CONTINUOUS; + } else if (VectorFormat::VEC_DISCRETE == vec_format) { + data_format_type = INT_DISCRETE; + } else if (VectorFormat::VEC_UNIFORM == vec_format) { + data_format_type = INT_UNIFORM; + } else if (VectorFormat::VEC_UNIFORM_CONST == vec_format) { + data_format_type = INT_UNIFORM_CONST; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", K(ret), K(vec_format)); + } + } else { // for string + if (ObCSEncodingUtil::is_variable_len_store_class(store_class)) { + extra_store_size_for_var_string = sizeof(uint64_t) * row_count; // for var length string offset array + } + store_size += vec_data_size + extra_store_size_for_var_string; + + if (VectorFormat::VEC_FIXED == vec_format) { + data_format_type = STR_FIXED; + } else if (VectorFormat::VEC_CONTINUOUS == vec_format) { + data_format_type = STR_CONTINUOUS; + } else if (VectorFormat::VEC_DISCRETE == vec_format) { + data_format_type = STR_DISCRETE; + } else if (VectorFormat::VEC_UNIFORM == vec_format) { + data_format_type = STR_UNIFORM; + } else if (VectorFormat::VEC_UNIFORM_CONST == vec_format) { + data_format_type = STR_UNIFORM_CONST; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", K(ret), K(vec_format)); + } + } + + if (OB_FAIL(ret)) { + } else if (appended_batch_count_ > 0 && estimate_size_ + store_size >= estimate_size_limit_) { + ret = OB_BUF_NOT_ENOUGH; + } else if (row_count > 1 && estimate_size_ + store_size >= estimate_size_limit_) { + ret = OB_BUF_NOT_ENOUGH; + // appended_batch_count_ == 0 && row_count == 1 represent a large row, do not return OB_BUF_NOT_ENOUGH + } else if (row_buf_holder_.size() < length_ + vec_data_size) { + is_row_holder_not_enough = true; + } else if (OB_FAIL(do_copy_vector_(vector, + start, + row_count, + col_idx, + vec_data_size, + data_format_type, + store_class == ObIntSC/*is_signed*/))) { + LOG_WARN("fail to do copy vector", K(ret), + K(data_format_type), K(col_desc), K(row_count), K(vec_data_size)); + } + return ret; +} + +int ObMicroBlockCSEncoder::do_copy_vector_(const ObIVector *vector, + const int64_t start, + const int64_t row_count, + const int64_t col_idx, + const int64_t vec_data_size, + const ObDataFormatType type, + const bool is_signed) +{ + +#define DEFINE_MASK_FOR_SIGNED_INT \ + uint64_t value = 0; \ + uint64_t mask = 0; \ + uint64_t reverse_mask = 0; \ + if (is_signed) { \ + mask = INTEGER_MASK_TABLE[type_store_size]; \ + reverse_mask = ~mask; \ + } + +#define COPY_INT_AND_COMPLETE_SIGNED_BIT(src, src_len) \ + ENCODING_ADAPT_MEMCPY(&value, src, src_len) \ + if (is_signed) { \ + value = value & mask; \ + if (0 != reverse_mask && (value & (reverse_mask >> 1))) { \ + value |= reverse_mask; \ + } \ + } \ + ENCODING_ADAPT_MEMCPY(buf + length_, &value, sizeof(uint64_t)); + +#define DEEP_COPY_NULLS_BITMAP(vector) \ + if (vector->has_null()) { \ + if (OB_ISNULL(nulls_buf = allocator_.alloc(sql::ObBitVector::memory_size(row_count)))) { \ + ret = OB_ALLOCATE_MEMORY_FAILED; \ + LOG_WARN("fail to allocate mem", K(ret), K(row_count)); \ + } else { \ + nulls = sql::to_bit_vector(nulls_buf); \ + const sql::ObBitVector *src_nulls = vector->get_nulls(); \ + if (start == 0) { \ + nulls->deep_copy(*src_nulls, row_count); \ + } else { \ + MEMSET(nulls_buf, 0, sql::ObBitVector::memory_size(row_count)); \ + for (int64_t i = start, j = 0; j < row_count; i++, j++) { \ + if (src_nulls->at(i)) { \ + nulls->set(j); \ + } \ + } \ + } \ + } \ + } + + int ret = OB_SUCCESS; + ObVecBatchInfo batch_info; + const ObColDesc &col_desc = ctx_.col_descs_->at(col_idx); + char *buf = row_buf_holder_.data(); + const int64_t vec_start_offset = length_; + uint32_t *offsets = nullptr; + int32_t fixed_len = -1; + sql::ObBitVector *nulls = nullptr; + void *nulls_buf = nullptr; + const int64_t type_store_size = get_type_size_map()[col_desc.col_type_.get_type()]; + bool is_integer =false; + const int64_t end = start + row_count; + + switch (type) { + case INT_64BIT_FIXED: { + is_integer = true; + const ObFixedLengthBase *fix_vec = static_cast(vector); + fixed_len = fix_vec->get_length(); + OB_ASSERT(fixed_len == sizeof(uint64_t)); + const char *src = fix_vec->get_data() + start * fixed_len; + MEMCPY(buf + length_, src, vec_data_size); + length_ += fixed_len * row_count; + DEEP_COPY_NULLS_BITMAP(fix_vec); + break; + } + + case INT_NOT_64BIT_FIXED: { + is_integer = true; + const ObFixedLengthBase *fix_vec = static_cast(vector); + fixed_len = fix_vec->get_length(); + const char *src = fix_vec->get_data() + start * fixed_len; + DEFINE_MASK_FOR_SIGNED_INT; + for (int64_t i = 0; i < row_count; i++) { + COPY_INT_AND_COMPLETE_SIGNED_BIT(src + i * fixed_len, fixed_len); + length_ += sizeof(uint64_t); + } + DEEP_COPY_NULLS_BITMAP(fix_vec); + break; + } + + case INT_CONTINUOUS: { // INT don't use continuous format? + is_integer = true; + fixed_len = 0; + const ObContinuousFormat *conti_vec = static_cast(vector); + DEFINE_MASK_FOR_SIGNED_INT; + for (int64_t i = start; i < end; i++) { + const uint32_t len = conti_vec->get_length(i); + if (len != 0) { // len == 0 if the element is null, no need copy + fixed_len = len; + COPY_INT_AND_COMPLETE_SIGNED_BIT(conti_vec->get_payload(i), len); + } + length_ += sizeof(uint64_t); + } + DEEP_COPY_NULLS_BITMAP(conti_vec); + break; + } + + case INT_DISCRETE: { // INT don't use discrete format? + is_integer = true; + fixed_len = 0; + const ObDiscreteFormat *disc_vec = static_cast(vector); + DEFINE_MASK_FOR_SIGNED_INT; + for (int64_t i = start; i < end; i++) { + if (!disc_vec->has_null() || !disc_vec->is_null(i)) { // not null + const uint32_t len = disc_vec->get_length(i); + fixed_len = len; + COPY_INT_AND_COMPLETE_SIGNED_BIT(disc_vec->get_payload(i), len); + } + length_ += sizeof(uint64_t); + } + DEEP_COPY_NULLS_BITMAP(disc_vec); + break; + } + + case INT_UNIFORM: { + is_integer = true; + fixed_len = 0; + const ObUniformFormat *uni_vec = static_cast *>(vector); + DEFINE_MASK_FOR_SIGNED_INT; + for (int64_t i = start, j = 0; OB_SUCC(ret) && i < end; i++, j++) { + if (!uni_vec->is_null(i)) { + const uint32_t len = uni_vec->get_length(i); + fixed_len = len; + COPY_INT_AND_COMPLETE_SIGNED_BIT(uni_vec->get_payload(i), len); + } else { + if (OB_ISNULL(nulls_buf)) { + if (OB_ISNULL(nulls_buf = allocator_.alloc(sql::ObBitVector::memory_size(row_count)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate mem", K(ret), K(row_count)); + } else { + MEMSET(nulls_buf, 0, sql::ObBitVector::memory_size(row_count)); + nulls = sql::to_bit_vector(nulls_buf); + } + } + if (OB_SUCC(ret)) { + nulls->set(j); + } + } + length_ += sizeof(uint64_t); + } + break; + } + + case INT_UNIFORM_CONST: { + is_integer = true; + fixed_len = 0; + const ObUniformFormat *uni_vec = static_cast *>(vector); + DEFINE_MASK_FOR_SIGNED_INT; + if (!uni_vec->is_null(start)) { + const uint32_t len = uni_vec->get_length(start); + fixed_len = len; + const char *payload = uni_vec->get_payload(start); + for (int64_t i = start; i < end; i++) { + COPY_INT_AND_COMPLETE_SIGNED_BIT(payload, len); + length_ += sizeof(uint64_t); + } + } else { // const is null + if (OB_ISNULL(nulls_buf = allocator_.alloc(sql::ObBitVector::memory_size(row_count)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate mem", K(ret), K(row_count)); + } else { + MEMSET(nulls_buf, 0xFF, sql::ObBitVector::memory_size(row_count)); + nulls = sql::to_bit_vector(nulls_buf); + length_ += sizeof(uint64_t) * row_count; + } + } + break; + } + + case STR_FIXED: { + const ObFixedLengthBase *fix_vec = static_cast(vector); + fixed_len = fix_vec->get_length(); + const char *src = fix_vec->get_data() + start * fixed_len; + MEMCPY(buf + length_, src, vec_data_size); + length_ += fixed_len * row_count; + DEEP_COPY_NULLS_BITMAP(fix_vec); + break; + } + + case STR_CONTINUOUS: { + const ObContinuousFormat *conti_vec = static_cast(vector); + if (OB_ISNULL(offsets = (uint32_t*)allocator_.alloc(sizeof(uint32_t) * (row_count + 1)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate mem", K(ret), K(row_count)); + } else { + MEMCPY(buf + length_, conti_vec->get_payload(start), vec_data_size); + for (int64_t i = start, j = 0; i < end; i++, j++) { + const uint32_t len = conti_vec->get_length(i); + offsets[j] = length_; + length_ += len; + } + offsets[row_count] = length_; + DEEP_COPY_NULLS_BITMAP(conti_vec); + } + break; + } + + case STR_DISCRETE: { + const ObDiscreteFormat *disc_vec = static_cast(vector); + if (OB_ISNULL(offsets = (uint32_t*)allocator_.alloc(sizeof(uint32_t) * (row_count + 1)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate mem", K(ret), K(row_count)); + } else { + for (int64_t i = start, j = 0; i < end; i++, j++) { + if (!disc_vec->has_null() || !disc_vec->is_null(i)) { // not null + const uint32_t len = disc_vec->get_length(i); + MEMCPY(buf + length_, disc_vec->get_payload(i), len); + offsets[j] = length_; + length_ += len; + } else { + offsets[i] = length_; + } + } + offsets[row_count] = length_; + DEEP_COPY_NULLS_BITMAP(disc_vec); + } + break; + } + + case STR_UNIFORM: { + const ObUniformFormat *uni_vec = static_cast *>(vector); + if (OB_ISNULL(offsets = (uint32_t*)allocator_.alloc(sizeof(uint32_t) * (row_count + 1)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate mem", K(ret), K(row_count)); + } else { + for (int64_t i = start, j = 0; OB_SUCC(ret) && i < end; i++, j++) { + if (!uni_vec->is_null(i)) { + const uint32_t len = uni_vec->get_length(i); + MEMCPY(buf + length_, uni_vec->get_payload(i), len); + offsets[j] = length_; + length_ += len; + } else { + if (OB_ISNULL(nulls_buf)) { + if (OB_ISNULL(nulls_buf = allocator_.alloc(sql::ObBitVector::memory_size(row_count)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate mem", K(ret), K(row_count)); + } else { + MEMSET(nulls_buf, 0, sql::ObBitVector::memory_size(row_count)); + nulls = sql::to_bit_vector(nulls_buf); + } + } + if (OB_SUCC(ret)) { + nulls->set(j); + offsets[j] = length_; + } + } + } + offsets[row_count] = length_; + } + break; + } + + case STR_UNIFORM_CONST: { + const ObUniformFormat *uni_vec = static_cast *>(vector); + fixed_len = 0; + if (!uni_vec->is_null(start)) { + const uint32_t len = uni_vec->get_length(start); + const char *payload = uni_vec->get_payload(start); + fixed_len = len; + for (int64_t i = start; i < end; i++) { + MEMCPY(buf + length_, payload, len); + length_ += len; + } + } else { // const is null + if (OB_ISNULL(nulls_buf = allocator_.alloc(sql::ObBitVector::memory_size(row_count)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate mem", K(ret), K(row_count)); + } else { + MEMSET(nulls_buf, 0xFF, sql::ObBitVector::memory_size(row_count)); + nulls = sql::to_bit_vector(nulls_buf); + } + } + break; + } + + default: { + ret = OB_ERR_UNDEFINED; + LOG_ERROR("unexpected data format type", K(type), K(col_desc), K(is_signed)); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(length_ - vec_start_offset != vec_data_size)) { + ret = OB_INNER_STAT_ERROR; + LOG_WARN("vector data len not match", K(ret), K(length_), K(vec_start_offset), K(vec_data_size)); + } else if (OB_FAIL(batch_info.init(row_count, + fixed_len, + offsets, + nulls, + vec_start_offset, + is_integer))) { + LOG_ERROR("fail to init vector batch info", K(ret), K(row_count), K(fixed_len), KP(offsets), + KP(nulls), K(vec_start_offset), K(is_integer)); + } else if (OB_FAIL(vec_batch_info_arrs_.at(col_idx)->push_back(batch_info))) { + LOG_WARN("fail to push batch info", K(ret), K(batch_info)); + } + return ret; +} + +int ObMicroBlockCSEncoder::build_all_col_datums_() +{ + int ret = OB_SUCCESS; + const char *data_buf = row_buf_holder_.data(); + + for (int64_t col_idx = 0; OB_SUCC(ret) && col_idx < ctx_.column_cnt_; col_idx++) { + const ObVecBatchInfoArr &batch_info_arr = *vec_batch_info_arrs_[col_idx]; + int64_t total_row_count = 0; + + for (int64_t batch_idx = 0; OB_SUCC(ret) && batch_idx < appended_batch_count_; batch_idx++) { + const ObVecBatchInfo &batch_info = batch_info_arr.at(batch_idx); + ObColDatums &col_datums = *all_col_datums_[col_idx]; + total_row_count += batch_info.row_count_; + + if (batch_info.offsets_ != nullptr) { // continuous format + const uint32_t *offsets = batch_info.offsets_; + OB_ASSERT(!batch_info.is_integer_); + OB_ASSERT(offsets[0] == batch_info.start_offset_); + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < batch_info.row_count_; row_idx++) { + ObDatum datum; + if (batch_info.nulls_ == nullptr || !batch_info.nulls_->at(row_idx)) { + datum.ptr_ = data_buf + offsets[row_idx]; + datum.len_ = offsets[row_idx + 1] - offsets[row_idx]; + } else { + datum.set_null(); + } + if (OB_FAIL(col_datums.push_back(datum))) { + LOG_WARN("fail to push back datum", K(ret), K(datum), K(col_idx), K(batch_idx), K(row_idx)); + } + } + } else if (OB_UNLIKELY(batch_info.fixed_len_ < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected fixed_len when offsets in nullptr", K(ret), K(batch_info)); + } else { // fixed format + // integer is formatted to 8byte int row_buf_holder_ + const int32_t buf_step = batch_info.is_integer_ ? sizeof(uint64_t) : batch_info.fixed_len_; + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < batch_info.row_count_; row_idx++) { + ObDatum datum; + if (batch_info.nulls_ == nullptr || !batch_info.nulls_->at(row_idx)) { + datum.ptr_ = data_buf + batch_info.start_offset_ + row_idx * buf_step; + datum.len_ = batch_info.fixed_len_; + } else { + datum.set_null(); + } + if (OB_FAIL(col_datums.push_back(datum))) { + LOG_WARN("fail to push back datum", K(ret), K(datum), K(col_idx), K(batch_idx), K(row_idx)); + } + } + } + } // end of for build one column + + if (OB_SUCC(ret) && total_row_count != appended_row_count_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("row_count mismatch", K(ret), K(total_row_count), K_(appended_row_count)); + } + } + return ret; +} + int ObMicroBlockCSEncoder::reserve_header_(const ObMicroBlockEncodingCtx &ctx) { int ret = OB_SUCCESS; @@ -591,6 +1229,32 @@ int ObMicroBlockCSEncoder::store_stream_offsets_(int64_t &stream_offsets_length) return ret; } +int ObMicroBlockCSEncoder::prepare_for_build_block_() +{ + int ret = OB_SUCCESS; + if (appended_batch_count_ == 0) { // use singel-row interface + if (OB_FAIL(set_datum_rows_ptr_())) { + LOG_WARN("fail to set datum rows ptr", K(ret)); + } else if (OB_FAIL(process_out_row_columns_())) { + LOG_WARN("failed to process out row columns", K(ret)); + } + } else { // use batch-row interface + if (OB_FAIL(build_all_col_datums_())) { + LOG_WARN("fail to build all col datums", K(ret)); + } else if (OB_FAIL(process_out_row_columns_())) { + LOG_WARN("failed to process out row columns", K(ret)); + } else if (get_header(data_buffer_)->has_column_checksum_ && + OB_FAIL(checksum_helper_.cal_column_checksum(all_col_datums_, + appended_row_count_, get_header(data_buffer_)->column_checksums_))) { + LOG_WARN("cal column checksum failed", K(ret), K_(appended_row_count)); + } else if (need_cal_row_checksum() && + OB_FAIL(checksum_helper_.cal_rows_checksum(all_col_datums_, appended_row_count_))) { + LOG_WARN("fail to cal row chksum", K(ret), K_(appended_row_count), K_(checksum_helper)); + } + } + return ret; +} + int ObMicroBlockCSEncoder::build_block(char *&buf, int64_t &size) { int ret = OB_SUCCESS; @@ -599,18 +1263,18 @@ int ObMicroBlockCSEncoder::build_block(char *&buf, int64_t &size) if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (OB_UNLIKELY(0 == datum_row_offset_arr_.count())) { + } else if (OB_UNLIKELY(0 == appended_row_count_)) { ret = OB_INNER_STAT_ERROR; LOG_WARN("empty micro block", K(ret)); - } else if (OB_FAIL(set_datum_rows_ptr_())) { - LOG_WARN("fail to set datum rows ptr", K(ret)); + } else if (OB_FAIL(prepare_for_build_block_())) { + LOG_WARN("fail to prepare_for_build_block", K(ret)); } else if (OB_FAIL(encoder_detection_())) { LOG_WARN("detect column encoding failed", K(ret)); } else if (OB_FAIL(data_buffer_.write_nop(all_column_header_size + column_headers_size))) { LOG_WARN("fail to ensure space", K(ret), K_(data_buffer), K(all_column_header_size), K(column_headers_size)); } else { LOG_DEBUG("build micro block", K_(estimate_size), K_(all_headers_size), - K(column_headers_size), K_(expand_pct), K(datum_row_offset_arr_.count()), K(ctx_)); + K(column_headers_size), K_(expand_pct), K_(appended_row_count), K(ctx_)); int64_t column_data_offset = 0; int64_t stream_offsets_length = 0; @@ -618,7 +1282,7 @@ int ObMicroBlockCSEncoder::build_block(char *&buf, int64_t &size) uint32_t all_string_data_size = 0; // <1> store all columns, include column meta and column data if (OB_FAIL(store_columns_(column_data_offset))) { - LOG_WARN("fail to store columns", K(ret), K_(ctx), K(datum_row_offset_arr_.count()), K(estimate_size_)); + LOG_WARN("fail to store columns", K(ret), K_(ctx), K_(appended_row_count), K(estimate_size_)); // <2> store all string data } else if (OB_FAIL(store_all_string_data_(all_string_data_size, is_all_string_compress))) { LOG_WARN("fail to store_all_string_data_", K(ret)); @@ -646,7 +1310,7 @@ int ObMicroBlockCSEncoder::build_block(char *&buf, int64_t &size) all_column_header->set_is_all_string_compressed(); } // <6> fill micro header - header->row_count_ = datum_row_offset_arr_.count(); + header->row_count_ = appended_row_count_; header->has_string_out_row_ = has_string_out_row_; header->all_lob_in_row_ = !has_lob_out_row_; header->max_merged_trans_version_ = max_merged_trans_version_; @@ -660,7 +1324,7 @@ int ObMicroBlockCSEncoder::build_block(char *&buf, int64_t &size) size = data_buffer_.length(); LOG_DEBUG("finish build one micro block", KP(this), K(encoders_.count()), - K(datum_row_offset_arr_.count()), K(column_data_offset), K(size), K_(estimate_size), K_(estimate_size_limit), + K_(appended_row_count), K(column_data_offset), K(size), K_(estimate_size), K_(estimate_size_limit), K_(expand_pct), K(ctx_.micro_block_cnt_), K_(block_size_upper_bound), K(stream_offsets_.count()), K(ctx_.compressor_type_), KP(buf), K(size), K_(all_string_data_len)); } @@ -700,10 +1364,14 @@ int ObMicroBlockCSEncoder::init_column_ctxs_() int ObMicroBlockCSEncoder::set_datum_rows_ptr_() { int ret = OB_SUCCESS; - if (OB_UNLIKELY(row_buf_holder_.has_expand())) { + if (OB_UNLIKELY(appended_row_count_ != datum_row_offset_arr_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("appended_row_count is unexpected", K(ret), + K_(appended_row_count), K(datum_row_offset_arr_.count())); + } else if (OB_UNLIKELY(row_buf_holder_.has_expand())) { char *data = row_buf_holder_.data(); const int64_t column_cnt = ctx_.column_cnt_; - for (int64_t row_id = 0; OB_SUCC(ret) && row_id < datum_row_offset_arr_.count(); ++row_id) { + for (int64_t row_id = 0; OB_SUCC(ret) && row_id < appended_row_count_; ++row_id) { const char *orig_row_start_ptr = all_col_datums_.at(0)->at(row_id).ptr_; char *curr_row_start_ptr = data + datum_row_offset_arr_.at(row_id); // ptr is invalid and need update @@ -724,26 +1392,35 @@ int ObMicroBlockCSEncoder::set_datum_rows_ptr_() return ret; } -int ObMicroBlockCSEncoder::process_out_row_columns_(const ObDatumRow &row) +int ObMicroBlockCSEncoder::process_out_row_columns_() { // make sure in&out row status of all values in a column are same int ret = OB_SUCCESS; if (!need_check_lob_) { - } else if (OB_UNLIKELY(row.get_column_count() != col_ctxs_.count())) { + } else if (OB_UNLIKELY(all_col_datums_.count() != col_ctxs_.count())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected column count not match", K(ret)); - } else if (!has_lob_out_row_) { - for (int64_t i = 0; !has_lob_out_row_ && OB_SUCC(ret) && i < row.get_column_count(); ++i) { - ObStorageDatum &datum = row.storage_datums_[i]; - if (ctx_.col_descs_->at(i).col_type_.is_lob_storage()) { - if (datum.is_nop() || datum.is_null()) { - } else if (datum.len_ < sizeof(ObLobCommon)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("Unexpected lob datum len", K(ret), K(i), K(ctx_.col_descs_->at(i).col_type_), K(datum)); - } else { - const ObLobCommon &lob_common = datum.get_lob_data(); - has_lob_out_row_ = !lob_common.in_row_; - LOG_DEBUG("chaser debug lob out row", K(has_lob_out_row_), K(lob_common), K(datum)); + } else { + const int64_t col_count = all_col_datums_.count(); + for (int64_t col_idx = 0; + !has_lob_out_row_ && OB_SUCC(ret) && col_idx < col_count; col_idx++) { + if (!ctx_.col_descs_->at(col_idx).col_type_.is_lob_storage()) { + // skip non-lob column + } else { + ObColDatums &col_datums = *all_col_datums_.at(col_idx); + for (int64_t row_idx = 0; + !has_lob_out_row_ && OB_SUCC(ret) && row_idx < appended_row_count_; ++row_idx) { + ObDatum &datum = col_datums.at(row_idx); + if (datum.is_nop() || datum.is_null()) { + } else if (datum.len_ < sizeof(ObLobCommon)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected lob datum len", K(ret), K(row_idx), + K(ctx_.col_descs_->at(col_idx).col_type_), K(datum)); + } else { + const ObLobCommon &lob_common = datum.get_lob_data(); + has_lob_out_row_ = !lob_common.in_row_; + LOG_DEBUG("chaser debug lob out row", K(has_lob_out_row_), K(lob_common), K(datum)); + } } } } @@ -764,7 +1441,7 @@ int ObMicroBlockCSEncoder::copy_and_append_row_(const ObDatumRow &src, int64_t & const int64_t column_cnt = src.get_column_count(); const int64_t datum_row_offset = length_; ObDatum dst_datum; - if (datum_row_offset_arr_.count() > 0 && estimate_size_ >= estimate_size_limit_) { + if (appended_row_count_ > 0 && estimate_size_ >= estimate_size_limit_) { ret = OB_BUF_NOT_ENOUGH; } else { bool is_finish = false; @@ -831,6 +1508,8 @@ int ObMicroBlockCSEncoder::copy_and_append_row_(const ObDatumRow &src, int64_t & } } else if (OB_FAIL(datum_row_offset_arr_.push_back(datum_row_offset))) { LOG_WARN("fail to push back datum_row_offset", K(ret), K(datum_row_offset)); + } else { + appended_row_count_++; } return ret; @@ -840,8 +1519,19 @@ int ObMicroBlockCSEncoder::remove_invalid_datums_(const int32_t column_cnt) { int ret = OB_SUCCESS; for (int64_t i = 0; OB_SUCC(ret) && i < column_cnt; i++) { - if (OB_FAIL(all_col_datums_.at(i)->resize(datum_row_offset_arr_.count()))) { - LOG_ERROR("fail to resize all_col_datums_", K(ret), K(datum_row_offset_arr_.count()), K(i), K(column_cnt)); + if (OB_FAIL(all_col_datums_.at(i)->resize(appended_row_count_))) { + LOG_ERROR("fail to resize all_col_datums_", K(ret), K_(appended_row_count), K(i), K(column_cnt)); + } + } + return ret; +} + +int ObMicroBlockCSEncoder::remove_invalid_vec_batch_info_(const int32_t column_cnt) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < column_cnt; i++) { + if (OB_FAIL(vec_batch_info_arrs_.at(i)->resize(appended_batch_count_))) { + LOG_ERROR("fail to resize vec_batch_info_arrs_", K(ret), K_(appended_batch_count), K(i), K(column_cnt)); } } return ret; @@ -877,9 +1567,9 @@ int ObMicroBlockCSEncoder::copy_cell_(const ObColDesc &col_desc, const if (OB_FAIL(ret)) { } else if (FALSE_IT(store_size += datum_size + extra_store_size_for_var_string)) { - } else if (datum_row_offset_arr_.count() > 0 && estimate_size_ + store_size >= estimate_size_limit_) { + } else if (appended_row_count_ > 0 && estimate_size_ + store_size >= estimate_size_limit_) { ret = OB_BUF_NOT_ENOUGH; - // datum_row_offset_arr_.count() == 0 represent a large row, do not return OB_BUF_NOT_ENOUGH + // appended_row_count_ == 0 represent a large row, do not return OB_BUF_NOT_ENOUGH } else if (row_buf_holder_.size() < length_ + datum_size) { is_row_holder_not_enough = true; } else { @@ -929,6 +1619,86 @@ int64_t ObMicroBlockCSEncoder::calc_datum_row_size_(const ObDatumRow &src) const return need_size; } +int ObMicroBlockCSEncoder::calc_col_batch_data_size_(const ObIVector *vector, + const int64_t start, + const int64_t row_count, + const int64_t col_idx, + int64_t &size) const +{ + int ret = OB_SUCCESS; + size = 0; + const ObColDesc &col_desc = ctx_.col_descs_->at(col_idx); + ObObjTypeStoreClass store_class = get_store_class_map()[col_desc.col_type_.get_type_class()]; + const VectorFormat vec_format = vector->get_format(); + const int64_t end = start + row_count; + + if (store_class == ObIntSC || store_class == ObUIntSC) { + size += row_count * sizeof(uint64_t); // Whether it's null or not, occupy 8 bytes for one integer + } else { + switch (vec_format) { + case VEC_FIXED : { + const ObFixedLengthBase *fix_vec = static_cast(vector); + size += row_count * fix_vec->get_length(); + break; + } + case VEC_CONTINUOUS: { + const ObContinuousFormat *conti_vec = static_cast(vector); + const uint32_t *offsets = conti_vec->get_offsets(); + size += offsets[end] - offsets[start]; + break; + } + case VEC_DISCRETE: { + const ObDiscreteFormat *disc_vec = static_cast(vector); + for (int64_t i = start; i < end; i++) { + if (!disc_vec->has_null() || !disc_vec->is_null(i)) { + size += disc_vec->get_length(i); + } + } + break; + } + case VEC_UNIFORM:{ + const ObUniformFormat *uni_vec = static_cast *>(vector); + for (int64_t i = start; i < end; i++) { + if (!uni_vec->is_null(i)) { + size += uni_vec->get_length(i); + } + } + break; + } + case VEC_UNIFORM_CONST:{ + const ObUniformFormat *uni_vec = static_cast *>(vector); + if (!uni_vec->is_null(start)) { + size += uni_vec->get_length(start) * row_count; + } + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", K(ret), K(vec_format)); + } + } + } + return ret; +} + +int ObMicroBlockCSEncoder::calc_batch_data_size_(const ObIArray &vectors, + const int64_t start, + const int64_t row_count, + int64_t &size) const +{ + int ret = OB_SUCCESS; + size = 0; + int64_t col_data_size = 0; + for (int64_t col_idx = 0; OB_SUCC(ret) && col_idx < vectors.count(); ++col_idx) { + if (OB_FAIL(calc_col_batch_data_size_(vectors.at(col_idx), start, row_count, col_idx, col_data_size))) { + LOG_WARN("fail to calc_col_batch_data_size_", K(ret), K(col_idx)); + } else { + size += col_data_size; + } + } + return ret; +} + int ObMicroBlockCSEncoder::prescan_(const int64_t column_index) { int ret = OB_SUCCESS; @@ -945,24 +1715,24 @@ int ObMicroBlockCSEncoder::prescan_(const int64_t column_index) col_ctx.col_datums_ = all_col_datums_.at(column_index); // build hashtable - ObEncodingHashTable *ht = nullptr; - ObEncodingHashTableBuilder *builder = nullptr; + ObDictEncodingHashTable *ht = nullptr; + ObDictEncodingHashTableBuilder *builder = nullptr; // next power of 2 - uint64_t bucket_num = datum_row_offset_arr_.count() << 1; + uint64_t bucket_num = appended_row_count_ << 1; if (0 != (bucket_num & (bucket_num - 1))) { while (0 != (bucket_num & (bucket_num - 1))) { bucket_num = bucket_num & (bucket_num - 1); } bucket_num = bucket_num << 1; } - const int64_t node_num = datum_row_offset_arr_.count(); + const int64_t node_num = appended_row_count_; if (OB_UNLIKELY(node_num != col_ctx.col_datums_->count())) { ret = OB_INNER_STAT_ERROR; LOG_ERROR("row_count and col_datums_count is not requal", K(ret), K(node_num), KPC(col_ctx.col_datums_)); } else if (OB_FAIL(hashtable_factory_.create(bucket_num, node_num, ht))) { LOG_WARN("create hashtable failed", K(ret), K(bucket_num), K(node_num)); - } else if (FALSE_IT(builder = static_cast(ht))) { + } else if (FALSE_IT(builder = static_cast(ht))) { } else if (OB_FAIL(builder->build(*col_ctx.col_datums_, col_desc))) { LOG_WARN("build hash table failed", K(ret), K(column_index), K(column_type)); } diff --git a/src/storage/blocksstable/cs_encoding/ob_micro_block_cs_encoder.h b/src/storage/blocksstable/cs_encoding/ob_micro_block_cs_encoder.h index 594e0ec46b..80240fda01 100644 --- a/src/storage/blocksstable/cs_encoding/ob_micro_block_cs_encoder.h +++ b/src/storage/blocksstable/cs_encoding/ob_micro_block_cs_encoder.h @@ -17,11 +17,12 @@ #include "ob_cs_encoding_allocator.h" #include "ob_column_encoding_struct.h" #include "ob_icolumn_cs_encoder.h" -#include "storage/blocksstable/encoding/ob_encoding_hash_util.h" +#include "ob_dict_encoding_hash_table.h" #include "storage/blocksstable/ob_block_sstable_struct.h" #include "storage/blocksstable/ob_data_buffer.h" #include "storage/blocksstable/ob_imicro_block_writer.h" #include "storage/compaction/ob_compaction_memory_context.h" +#include "storage/blocksstable/ob_batch_datum_rows.h" namespace oceanbase { @@ -31,11 +32,42 @@ namespace blocksstable class ObIColumnCSEncoder; class ObEncodingHashTable; +class ObVecBatchInfo +{ +public: + ObVecBatchInfo() + : row_count_(0), + fixed_len_(-1), + offsets_(nullptr) , + nulls_(nullptr), + start_offset_(0), + is_integer_(false) + { + } + int init(const int32_t row_count, + const int32_t fixed_len, + const uint32_t *offsets, + sql::ObBitVector *nulls_, + const uint32_t start_offset, + bool is_integer); + + TO_STRING_KV(K_(row_count), K_(fixed_len), KP_(offsets), KP_(nulls), K_(start_offset), K_(is_integer)); + + int32_t row_count_; + int32_t fixed_len_; // fixed len for fixed format < 0 represent continuous format + const uint32_t *offsets_; // offset arr for continuous format, ==nullptr represent fixed format + sql::ObBitVector *nulls_; // null bitmap, ==nullptr represent no null + uint32_t start_offset_; // start offset in row_buf_holder_ of this vec batch + bool is_integer_; // the integer is format to 8Bytes in row_buf_holder_ which may be larger then fixed_len_ +}; +typedef ObPodFix2dArray ObVecBatchInfoArr; + class ObMicroBlockCSEncoder : public ObIMicroBlockWriter { public: static const int64_t DEFAULT_ESTIMATE_REAL_SIZE_PCT = 150; static const int64_t RESERVE_SIZE_FOR_ESTIMATE_LIMIT = 200 << 10; // 200K + ObMicroBlockCSEncoder(); virtual ~ObMicroBlockCSEncoder(); @@ -43,6 +75,9 @@ public: int init(const ObMicroBlockEncodingCtx &ctx); // return OB_BUF_NOT_ENOUGH if exceed micro block size virtual int append_row(const ObDatumRow &row); + virtual int append_batch(const ObBatchDatumRows &vec_batch, + const int64_t start, + const int64_t row_count) override; virtual int build_block(char *&buf, int64_t &size); // clear status and release memory, reset along with macro block writer virtual void reset(); @@ -51,7 +86,7 @@ public: virtual void reuse(); virtual int64_t get_row_count() const override { - return datum_row_offset_arr_.count(); + return appended_row_count_; } virtual int64_t get_block_size() const override { @@ -65,22 +100,67 @@ public: virtual void dump_diagnose_info() const override; private: + enum ObDataFormatType : uint8_t + { + INT_64BIT_FIXED, + INT_NOT_64BIT_FIXED, + INT_CONTINUOUS, + INT_DISCRETE, + INT_UNIFORM, + INT_UNIFORM_CONST, + STR_FIXED, + STR_CONTINUOUS, + STR_DISCRETE, + STR_UNIFORM, + STR_UNIFORM_CONST, + MAX + }; + int inner_init_(); int reserve_header_(const ObMicroBlockEncodingCtx &ctx); int try_to_append_row_(const int64_t &store_size); int init_column_ctxs_(); // only deep copy the cell part - int process_out_row_columns_(const ObDatumRow &row); + int process_out_row_columns_(); int copy_and_append_row_(const ObDatumRow &src, int64_t &store_size); + int copy_and_append_batch_(const ObBatchDatumRows &vec_batch, + const int64_t start, + const int64_t row_count, + int64_t &store_size); int copy_cell_(const ObColDesc &col_desc, const ObStorageDatum &src, ObDatum &dest, int64_t &store_size, bool &is_row_holder_not_enough); + int copy_vector_(const ObIVector *vector, + const int64_t start, + const int64_t row_count, + const int64_t col_idx, + int64_t &store_size, + bool &is_row_holder_not_enough); + int do_copy_vector_(const ObIVector *vector, + const int64_t start, + const int64_t row_count, + const int64_t col_idx, + const int64_t vec_data_size, + const ObDataFormatType type, + const bool is_signed); + int build_all_col_datums_(); + int prepare_for_build_block_(); int process_large_row_(const ObDatumRow &src, int64_t &store_size); int set_datum_rows_ptr_(); int remove_invalid_datums_(const int32_t column_cnt); + int remove_invalid_vec_batch_info_(const int32_t column_cnt); int encoder_detection_(); // detect encoder with pre-scan result int fast_encoder_detect_(const int64_t column_idx); int64_t calc_datum_row_size_(const ObDatumRow &src) const; + int calc_batch_data_size_(const ObIArray &vectors, + const int64_t start, + const int64_t row_count, + int64_t &size) const; + int calc_col_batch_data_size_(const ObIVector *vector, + const int64_t start, + const int64_t row_count, + const int64_t col_idx, + int64_t &size) const; int prescan_(const int64_t column_index); int choose_encoder_(const int64_t column_idx); int choose_encoder_for_integer_(const int64_t column_idx, ObIColumnCSEncoder *&e); @@ -101,6 +181,7 @@ private: int alloc_and_init_encoder_(const int64_t column_index, ObIColumnCSEncoder *&e); void update_estimate_size_limit_(const ObMicroBlockEncodingCtx &ctx); int init_all_col_values_(const ObMicroBlockEncodingCtx &ctx); + int init_vec_batch_info_arrs_(const ObMicroBlockEncodingCtx &ctx); void print_micro_block_encoder_status_(); int store_columns_(int64_t &column_data_offset); int store_all_string_data_(uint32_t &data_size, bool &use_compress); @@ -126,7 +207,10 @@ private: common::ObArray all_col_datums_; ObArenaAllocator pivot_allocator_; - common::ObSEArray datum_row_offset_arr_; + common::ObSEArray datum_row_offset_arr_; + common::ObArray vec_batch_info_arrs_; + int64_t appended_batch_count_; + int64_t appended_row_count_; int64_t estimate_size_; int64_t estimate_size_limit_; int64_t all_headers_size_; @@ -134,8 +218,8 @@ private: common::ObArray encoders_; common::ObArray stream_offsets_; ObCSEncoderAllocator encoder_allocator_; - common::ObArray hashtables_; - ObEncodingHashTableFactory hashtable_factory_; + common::ObArray hashtables_; + ObDictEncodingHashTableFactory hashtable_factory_; common::ObArray col_ctxs_; int64_t length_; bool is_inited_; diff --git a/src/storage/blocksstable/cs_encoding/ob_str_dict_column_encoder.cpp b/src/storage/blocksstable/cs_encoding/ob_str_dict_column_encoder.cpp index d693204c27..01821d48db 100644 --- a/src/storage/blocksstable/cs_encoding/ob_str_dict_column_encoder.cpp +++ b/src/storage/blocksstable/cs_encoding/ob_str_dict_column_encoder.cpp @@ -37,7 +37,7 @@ int ObStrDictColumnEncoder::init( LOG_WARN("init base column encoder failed", K(ret), K(ctx), K(column_index), K(row_count)); } else { column_header_.type_ = type_; - dict_encoding_meta_.distinct_val_cnt_ = ctx.ht_->size(); + dict_encoding_meta_.distinct_val_cnt_ = ctx.ht_->distinct_val_cnt(); dict_encoding_meta_.ref_row_cnt_ = row_count_; if (ctx_->null_cnt_ > 0) { dict_encoding_meta_.set_has_null(); diff --git a/src/storage/blocksstable/cs_encoding/ob_string_column_encoder.cpp b/src/storage/blocksstable/cs_encoding/ob_string_column_encoder.cpp index 8e1727936b..3ceccacf70 100644 --- a/src/storage/blocksstable/cs_encoding/ob_string_column_encoder.cpp +++ b/src/storage/blocksstable/cs_encoding/ob_string_column_encoder.cpp @@ -13,7 +13,7 @@ #define USING_LOG_PREFIX STORAGE #include "ob_string_column_encoder.h" -#include "storage/blocksstable/encoding/ob_encoding_hash_util.h" +#include "ob_dict_encoding_hash_table.h" #include "ob_column_datum_iter.h" #include "ob_cs_encoding_util.h" #include "lib/codec/ob_codecs.h" diff --git a/src/storage/blocksstable/ob_batch_datum_rows.cpp b/src/storage/blocksstable/ob_batch_datum_rows.cpp new file mode 100644 index 0000000000..d01fa65fe4 --- /dev/null +++ b/src/storage/blocksstable/ob_batch_datum_rows.cpp @@ -0,0 +1,67 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "storage/blocksstable/ob_batch_datum_rows.h" + +namespace oceanbase +{ +namespace blocksstable +{ + +void ObBatchDatumRows::reset() +{ + row_flag_.reset(); + mvcc_row_flag_.reset(); + trans_id_.reset(); + vectors_.reset(); + row_count_ = 0; +} + +int ObBatchDatumRows::to_datum_row(int64_t idx, ObDatumRow &datum_row) const { + int ret = OB_SUCCESS; + + if ((idx < 0 || idx >= row_count_) || datum_row.count_ != vectors_.count()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", K(idx), K(datum_row.count_), K(vectors_.count()), KR(ret)); + } + + if (OB_SUCC(ret)) { + datum_row.row_flag_ = row_flag_; + datum_row.mvcc_row_flag_ = mvcc_row_flag_; + datum_row.trans_id_ = trans_id_; + } + + const char *pay_load = nullptr; + bool is_null = false; + ObLength length = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < vectors_.count(); i ++) { + common::ObIVector *vec = vectors_.at(i); + if (vec == nullptr) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("vec should not be null", KR(ret)); + } else { + vec->get_payload(idx, is_null, pay_load, length); + if (is_null) { + datum_row.storage_datums_[i].set_null(); + } else { + datum_row.storage_datums_[i].shallow_copy_from_datum(ObDatum(pay_load, length, is_null)); + } + } + } + + return ret; +} + +} // namespace blocksstable +} // namespace oceanbase diff --git a/src/storage/blocksstable/ob_batch_datum_rows.h b/src/storage/blocksstable/ob_batch_datum_rows.h new file mode 100644 index 0000000000..c1c4458c2c --- /dev/null +++ b/src/storage/blocksstable/ob_batch_datum_rows.h @@ -0,0 +1,52 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_BATCH_DATUM_ROWS_H_ +#define OB_BATCH_DATUM_ROWS_H_ + +#include "lib/container/ob_array.h" +#include "share/vector/ob_i_vector.h" +#include "storage/blocksstable/ob_datum_row.h" + +namespace oceanbase +{ +namespace blocksstable +{ + +class ObBatchDatumRows +{ +public: + ObBatchDatumRows() : row_count_(0) {} + ~ObBatchDatumRows() {} + void reset(); + + OB_INLINE int64_t get_column_count() const { return vectors_.count(); } + + TO_STRING_KV(K_(row_flag), K_(mvcc_row_flag), K_(trans_id), K(vectors_.count()), K_(row_count)); + +public: + // convert vectors_ to datum_row at row idx = idx + // performance is low, use it in performance non sensitive position + int to_datum_row(int64_t idx, ObDatumRow &datum_row) const; + +public: + ObDmlRowFlag row_flag_; + ObMultiVersionRowFlag mvcc_row_flag_; + transaction::ObTransID trans_id_; + common::ObArray vectors_; + int64_t row_count_; +}; + +} // namespace blocksstable +} // namespace oceanbase + +#endif /* OB_BATCH_DATUM_ROWS_H_ */ diff --git a/src/storage/blocksstable/ob_imicro_block_writer.h b/src/storage/blocksstable/ob_imicro_block_writer.h index c6f37cfee8..20fa486edf 100644 --- a/src/storage/blocksstable/ob_imicro_block_writer.h +++ b/src/storage/blocksstable/ob_imicro_block_writer.h @@ -25,6 +25,7 @@ #include "ob_micro_block_checksum_helper.h" #include "storage/compaction/ob_compaction_memory_context.h" #include "storage/blocksstable/ob_logic_macro_id.h" +#include "storage/blocksstable/ob_batch_datum_rows.h" namespace oceanbase { @@ -129,6 +130,14 @@ public: } virtual ~ObIMicroBlockWriter() {} virtual int append_row(const ObDatumRow &row) = 0; + virtual int append_batch(const ObBatchDatumRows &vec_batch, + const int64_t start, + const int64_t row_count) + { + int ret = OB_NOT_SUPPORTED; + STORAGE_LOG(ERROR, "Unspport append_batch", K(ret)); + return ret; + } virtual int build_block(char *&buf, int64_t &size) = 0; virtual int64_t get_row_count() const = 0; virtual int64_t get_block_size() const = 0; // estimate block size after encoding diff --git a/src/storage/blocksstable/ob_macro_block_writer.cpp b/src/storage/blocksstable/ob_macro_block_writer.cpp index d3dde3057c..aa231d8c4f 100644 --- a/src/storage/blocksstable/ob_macro_block_writer.cpp +++ b/src/storage/blocksstable/ob_macro_block_writer.cpp @@ -432,7 +432,8 @@ ObMacroBlockWriter::ObMacroBlockWriter(const bool is_need_macro_buffer) pre_warmer_(NULL), object_cleaner_(nullptr), io_buf_(nullptr), - validator_(NULL) + validator_(NULL), + is_cs_encoding_writer_(false) { } @@ -487,6 +488,7 @@ void ObMacroBlockWriter::reset() rowkey_allocator_.reset(); io_buf_ = nullptr; validator_ = nullptr; + is_cs_encoding_writer_ = false; } @@ -544,6 +546,7 @@ int ObMacroBlockWriter::open( } else if (OB_FAIL(init_pre_agg_util(data_store_desc))) { STORAGE_LOG(WARN, "Failed to init pre aggregate utilities", K(ret)); } else { + is_cs_encoding_writer_ = data_store_desc_->encoding_enabled() && ObStoreFormat::is_row_store_type_with_cs_encoding(data_store_desc_->get_row_store_type()); const bool is_use_adaptive = !data_store_desc_->is_major_merge_type() || data_store_desc_->get_major_working_cluster_version() >= DATA_VERSION_4_1_0_0; if (OB_FAIL(micro_block_adaptive_splitter_.init(data_store_desc.get_macro_store_size(), 0/*min_micro_row_count*/, is_use_adaptive))) { @@ -612,6 +615,35 @@ int ObMacroBlockWriter::append_row(const ObDatumRow &row, const ObMacroBlockDesc return ret; } +int ObMacroBlockWriter::append_batch(const ObBatchDatumRows &datum_rows, + const ObMacroBlockDesc *curr_macro_desc) +{ + int ret = OB_SUCCESS; + + if (!is_cs_encoding_writer_) { + ObDatumRow &row = datum_row_; + for (int64_t i = 0; OB_SUCC(ret) && i < datum_rows.row_count_; i ++) { + if (OB_FAIL(datum_rows.to_datum_row(i, row))) { + LOG_WARN("fail to to datum row", KR(ret), K(i)); + } else if (OB_FAIL(append_row(row, curr_macro_desc))) { + LOG_WARN("fail to append row", K(row), KR(ret)); + } + } + } else { + if (OB_UNLIKELY(nullptr == data_store_desc_)) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "The ObMacroBlockWriter has not been opened", K(ret)); + } else if (OB_FAIL(append_batch(datum_rows, data_store_desc_->get_micro_block_size()))) { + STORAGE_LOG(WARN, "Fail to append row", K(ret)); + } else if (OB_FAIL(try_active_flush_macro_block())) { + STORAGE_LOG(WARN, "Fail to try_active_flush_macro_block", K(ret)); + } + } + + return ret; +} + + int ObMacroBlockWriter::append(const ObDataMacroBlockMeta ¯o_meta, const ObMicroBlockData *micro_block_data) { @@ -653,6 +685,60 @@ int ObMacroBlockWriter::append(const ObDataMacroBlockMeta ¯o_meta, return ret; } + +int ObMacroBlockWriter::append(const ObBatchDatumRows &datum_rows, const int64_t start, const int64_t write_row_count) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(append_batch_to_micro_block(datum_rows, start, write_row_count))) { + if (OB_BUF_NOT_ENOUGH == ret) { + if (0 == micro_writer_->get_row_count()) { + ret = OB_NOT_SUPPORTED; + STORAGE_LOG(ERROR, "The single row is too large, ", K(ret)); + } else if (OB_FAIL(build_micro_block())) { + STORAGE_LOG(WARN, "Fail to build micro block, ", K(ret)); + } else if (OB_FAIL(append_batch_to_micro_block(datum_rows, start, write_row_count))) { + STORAGE_LOG(ERROR, "Fail to append row to micro block, ", K(ret)); + } + } + } + + return ret; +} + +int ObMacroBlockWriter::append(const ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(append_batch_to_micro_block(datum_rows, 0, datum_rows.row_count_))) { + if (OB_BUF_NOT_ENOUGH == ret) { + if (0 == micro_writer_->get_row_count()) { + ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < datum_rows.row_count_; i ++) { + if (OB_FAIL(append(datum_rows, i, 1))) { + LOG_WARN("fail to append row", KR(ret)); + } + } + } else if (OB_FAIL(build_micro_block())) { + STORAGE_LOG(WARN, "Fail to build micro block, ", K(ret)); + } else if (OB_FAIL(append_batch_to_micro_block(datum_rows, 0, datum_rows.row_count_))) { + if (OB_BUF_NOT_ENOUGH == ret) { + ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < datum_rows.row_count_; i ++) { + if (OB_FAIL(append(datum_rows, i, 1))) { + LOG_WARN("fail to append row", KR(ret)); + } + } + } else { + STORAGE_LOG(ERROR, "Fail to append row to micro block, ", K(ret)); + } + } + } + } + + return ret; +} + int ObMacroBlockWriter::append(const ObDatumRow &row) { int ret = OB_SUCCESS; @@ -673,6 +759,48 @@ int ObMacroBlockWriter::append(const ObDatumRow &row) return ret; } + +int ObMacroBlockWriter::data_aggregator_eval(const ObBatchDatumRows &datum_rows, const int64_t start, const int64_t write_row_count) +{ + + int ret = OB_SUCCESS; + ObDatumRow &row = datum_row_; + for (int64_t i = 0; OB_SUCC(ret) && i < write_row_count; i ++) { + if (OB_FAIL(datum_rows.to_datum_row(i + start, row))) { + LOG_WARN("fail to get row", K(i), KR(ret)); + } else if (OB_FAIL(data_aggregator_->eval(row))) { + LOG_WARN("fail to eval row", KR(ret)); + } + } + return ret; +} + +int ObMacroBlockWriter::append_batch(const ObBatchDatumRows &datum_rows, const int64_t split_size) +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(data_store_desc_)) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "The ObMacroBlockWriter has not been opened, ", K(ret), KP(data_store_desc_)); + } else if (OB_UNLIKELY(split_size < data_store_desc_->get_micro_block_size())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid split_size", K(ret), K(split_size)); + } + + if (OB_SUCC(ret)) { + bool is_split = false; + if (OB_FAIL(append(datum_rows))) { + STORAGE_LOG(WARN, "Fail to append row to micro block", K(ret)); + } else if (OB_FAIL(micro_block_adaptive_splitter_.check_need_split(micro_writer_->get_block_size(), micro_writer_->get_row_count(), + split_size, macro_blocks_[current_index_].get_data_size(), is_keep_freespace(), is_split))) { + STORAGE_LOG(WARN, "Failed to check need split", K(ret), KPC(micro_writer_)); + } else if (is_split && OB_FAIL(build_micro_block())) { + STORAGE_LOG(WARN, "Fail to build micro block, ", K(ret)); + } + } + return ret; +} + int ObMacroBlockWriter::append_row(const ObDatumRow &row, const int64_t split_size) { int ret = OB_SUCCESS; @@ -1008,6 +1136,50 @@ int ObMacroBlockWriter::check_order(const ObDatumRow &row) return ret; } +int ObMacroBlockWriter::update_micro_commit_info(const ObBatchDatumRows &datum_rows, const int64_t start, const int64_t write_row_count) +{ + int ret = OB_SUCCESS; + bool is_ghost_row_flag = false; + if (OB_FAIL(blocksstable::ObGhostRowUtil::is_ghost_row(datum_rows.mvcc_row_flag_, is_ghost_row_flag))) { + STORAGE_LOG(ERROR, "failed to check ghost row", K(ret), K(datum_rows)); + } else if (data_store_desc_->is_cg() || is_ghost_row_flag) { //skip cg block & ghost row + } else if (datum_rows.mvcc_row_flag_.is_uncommitted_row()) { + micro_writer_->set_contain_uncommitted_row(); + LOG_TRACE("meet uncommited trans row", K(datum_rows)); + } else { + const int64_t trans_version_col_idx = data_store_desc_->get_schema_rowkey_col_cnt(); + ObIVector *vec = nullptr; + if (trans_version_col_idx >= datum_rows.vectors_.count()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", K(trans_version_col_idx), K(datum_rows.vectors_.count()), KR(ret)); + } else { + vec = datum_rows.vectors_.at(trans_version_col_idx); + if (vec == nullptr) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("vec should not be nullptr", KR(ret)); + } + } + + const char *pay_load = nullptr; + bool is_null = false; + ObLength length = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < write_row_count; i ++) { + vec->get_payload(i + start, is_null, pay_load, length); + const int64_t cur_row_version = ObDatum(pay_load, length, is_null).get_int(); + // data_store_desc_->get_major_working_cluster_version() only set in major merge. it is 0 for mini/minor + const int64_t cluster_version = data_store_desc_->get_major_working_cluster_version(); + if (!data_store_desc_->is_major_merge_type() || cluster_version >= DATA_VERSION_4_3_0_0) { + // see ObMicroBlockWriter::build_block, column_checksums_ and min_merged_trans_version_ share the same memory space. + // Only major merge set column_checksums_, so we can set min_merged_trans_version_ regardless of data version. + micro_writer_->update_merged_trans_version(-cur_row_version); + } else if (!not_compat_for_queuing_mode_42x(cluster_version)) { + micro_writer_->update_max_merged_trans_version(-cur_row_version); + } + } + } + return ret; +} + int ObMacroBlockWriter::update_micro_commit_info(const ObDatumRow &row) { int ret = OB_SUCCESS; @@ -1071,6 +1243,27 @@ int ObMacroBlockWriter::init_macro_seq_generator(const blocksstable::ObMacroSeqP return ret; } +int ObMacroBlockWriter::append_batch_to_micro_block(const ObBatchDatumRows &datum_rows, const int64_t start, const int64_t write_row_count) +{ + int ret = OB_SUCCESS; + ObDatumRow &last_row = datum_row_; + + if (OB_FAIL(datum_rows.to_datum_row(start + write_row_count - 1, last_row))) { + LOG_WARN("fail to get last row", KR(ret)); + } else if (OB_FAIL(micro_writer_->append_batch(datum_rows, start, write_row_count))) { + if (ret != OB_BUF_NOT_ENOUGH) { + STORAGE_LOG(WARN, "Failed to append row in micro writer", K(ret)); + } + } else if (OB_FAIL(update_micro_commit_info(datum_rows, start, write_row_count))) { + STORAGE_LOG(WARN, "Fail to update_micro_commit_info", K(ret), K(datum_rows), K(start), K(write_row_count)); + } else if (OB_FAIL(save_last_key(last_row))) { + STORAGE_LOG(WARN, "Fail to save last key, ", K(ret), K(last_row)); + } else if (nullptr != data_aggregator_ && OB_FAIL(data_aggregator_eval(datum_rows, start, write_row_count))) { + STORAGE_LOG(WARN, "Fail to evaluate aggregate data", K(ret)); + } + return ret; +} + int ObMacroBlockWriter::append_row_and_hash_index(const ObDatumRow &row) { int ret = OB_SUCCESS; diff --git a/src/storage/blocksstable/ob_macro_block_writer.h b/src/storage/blocksstable/ob_macro_block_writer.h index 449bf7ebbf..a30388d071 100644 --- a/src/storage/blocksstable/ob_macro_block_writer.h +++ b/src/storage/blocksstable/ob_macro_block_writer.h @@ -37,6 +37,7 @@ #include "storage/blocksstable/ob_macro_seq_generator.h" #include "storage/compaction/ob_compaction_util.h" #include "storage/compaction/ob_sstable_merge_history.h" +#include "storage/blocksstable/ob_batch_datum_rows.h" namespace oceanbase { @@ -160,6 +161,9 @@ public: const ObMicroBlockData *micro_block_data); virtual int append_micro_block(const ObMicroBlock µ_block, const ObMacroBlockDesc *curr_macro_desc = nullptr); virtual int append_row(const ObDatumRow &row, const ObMacroBlockDesc *curr_macro_desc = nullptr); + int data_aggregator_eval(const ObBatchDatumRows &datum_rows, const int64_t start, const int64_t write_row_count); + virtual int append_batch(const ObBatchDatumRows &datum_rows, + const ObMacroBlockDesc *curr_macro_desc = nullptr); // TODO(baichangmin): SSTableRebuilder disabled in SS mode. Finish SN route later. int append_macro_block(const ObDataMacroBlockMeta ¯o_meta) { @@ -197,12 +201,17 @@ protected: private: int append_row(const ObDatumRow &row, const int64_t split_size); + int append_batch(const ObBatchDatumRows &datum_rows, + const int64_t split_size); int append(const ObDatumRow &row); + int append(const ObBatchDatumRows &datum_rows); + int append(const ObBatchDatumRows &datum_rows, const int64_t start, const int64_t write_row_count); int append(const ObDataMacroBlockMeta ¯o_meta, const ObMicroBlockData *micro_block_data); int check_order(const ObDatumRow &row); int init_macro_seq_generator(const blocksstable::ObMacroSeqParam ¯o_seq_param); int init_hash_index_builder(); int append_row_and_hash_index(const ObDatumRow &row); + int append_batch_to_micro_block(const ObBatchDatumRows &datum_rows, const int64_t start, const int64_t write_row_count); int init_pre_agg_util(const ObDataStoreDesc &data_store_desc); void release_pre_agg_util(); int agg_micro_block(const ObMicroIndexInfo µ_index_info); @@ -233,6 +242,7 @@ private: int64_t *column_checksum); int flush_reuse_macro_block(const ObDataMacroBlockMeta ¯o_meta); int update_micro_commit_info(const ObDatumRow &row); + int update_micro_commit_info(const ObBatchDatumRows &datum_rows, const int64_t start, const int64_t write_row_count); void dump_micro_block(ObIMicroBlockWriter µ_writer); void dump_macro_block(ObMacroBlock ¯o_block); int exec_callback(const ObStorageObjectHandle ¯o_handle, ObMacroBlock *macro_block); @@ -281,6 +291,7 @@ private: ObSSTablePrivateObjectCleaner *object_cleaner_; char *io_buf_; ObIMacroBlockValidator *validator_; + bool is_cs_encoding_writer_; }; struct ObIMacroBlockValidator diff --git a/src/storage/blocksstable/ob_micro_block_checksum_helper.cpp b/src/storage/blocksstable/ob_micro_block_checksum_helper.cpp new file mode 100644 index 0000000000..9731002fe0 --- /dev/null +++ b/src/storage/blocksstable/ob_micro_block_checksum_helper.cpp @@ -0,0 +1,151 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "storage/blocksstable/ob_micro_block_checksum_helper.h" +namespace oceanbase +{ +namespace blocksstable +{ +int ObMicroBlockChecksumHelper::init( + const common::ObIArray *col_descs, + const bool need_opt_row_chksum) +{ + int ret = OB_SUCCESS; + reset(); + if (OB_ISNULL(col_descs)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", K(ret), KP(col_descs)); + } else { + if (need_opt_row_chksum) { + for (int64_t i = 0; i < col_descs->count(); ++i) { + if (col_descs->at(i).col_type_.is_integer_type()) { + ++integer_col_cnt_; + } + } + } + if (NEED_INTEGER_BUF_CNT < integer_col_cnt_) { + if (LOCAL_INTEGER_COL_CNT >= integer_col_cnt_) { + integer_col_buf_ = local_integer_col_buf_; + integer_col_idx_ = local_integer_col_idx_; + } else if (OB_ISNULL(integer_col_buf_ = + static_cast(allocator_.alloc(sizeof(int64_t) * integer_col_cnt_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "failed to alloc integer_col_buf", K(ret), K_(integer_col_cnt)); + } else if (OB_ISNULL(integer_col_idx_ = + static_cast(allocator_.alloc(sizeof(int16_t) * integer_col_cnt_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "failed to alloc integer_col_idx", K(ret), K_(integer_col_cnt)); + } + } + } + if (OB_SUCC(ret)) { + // traverse once again to fill idx + if (OB_NOT_NULL(integer_col_idx_)) { + for (int64_t i = 0, idx = 0; i < col_descs->count(); ++i) { + if (idx < integer_col_cnt_ && col_descs->at(i).col_type_.is_integer_type()) { + integer_col_idx_[idx] = i; + ++idx; + } + } + } + micro_block_row_checksum_ = 0; + col_descs_ = col_descs; + } else { + reset(); + } + return ret; +} + +void ObMicroBlockChecksumHelper::reset() +{ + col_descs_ = nullptr; + integer_col_cnt_ = 0; + micro_block_row_checksum_ = 0; + if (OB_NOT_NULL(integer_col_buf_)) { + if (!is_local_buf()) { + allocator_.free(integer_col_buf_); + } + integer_col_buf_ = nullptr; + } + if (OB_NOT_NULL(integer_col_idx_)) { + if (!is_local_idx()) { + allocator_.free(integer_col_idx_); + } + integer_col_idx_ = nullptr; + } + allocator_.reset(); +} + +int ObMicroBlockChecksumHelper::cal_rows_checksum( + const common::ObArray &all_col_datums, + const int64_t row_count) +{ + int ret = OB_SUCCESS; + const int64_t col_cnt = all_col_datums.count(); + if (OB_ISNULL(integer_col_buf_) || OB_ISNULL(integer_col_idx_)) { + for (int64_t row_idx = 0; row_idx < row_count; row_idx++) { + for (int64_t i = 0; i < col_cnt; ++i) { + micro_block_row_checksum_ = all_col_datums[i]->at(row_idx).checksum(micro_block_row_checksum_); + } + } + } else if (OB_ISNULL(col_descs_) || col_cnt != col_descs_->count()) { // defense + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpect error", K(ret), KPC(col_descs_), K_(integer_col_cnt), K(col_cnt)); + } else { + for (int64_t row_idx = 0; row_idx < row_count; row_idx++) { + for (int64_t i = 0, idx = 0; i < col_cnt; ++i) { + if (idx < integer_col_cnt_ && integer_col_idx_[idx] == i) { + if (all_col_datums[i]->at(row_idx).is_nop()) { + integer_col_buf_[idx] = MAGIC_NOP_NUMBER; + } else if (all_col_datums[i]->at(row_idx).is_null()) { + integer_col_buf_[idx] = MAGIC_NULL_NUMBER; + } else { + integer_col_buf_[idx] = all_col_datums[i]->at(row_idx).get_int(); + } + ++idx; + } else { + micro_block_row_checksum_ = all_col_datums[i]->at(row_idx).checksum(micro_block_row_checksum_); + } + } + micro_block_row_checksum_ = ob_crc64_sse42(micro_block_row_checksum_, + static_cast(integer_col_buf_), sizeof(int64_t) * integer_col_cnt_); + } + } + return ret; +} + +int ObMicroBlockChecksumHelper::cal_column_checksum( + const common::ObArray &all_col_datums, + const int64_t row_count, + int64_t *curr_micro_column_checksum) +{ + int ret = OB_SUCCESS; + const int64_t col_cnt = all_col_datums.count(); + + if (OB_UNLIKELY(nullptr == curr_micro_column_checksum)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid arguments", K(ret), KP(curr_micro_column_checksum)); + } else if (OB_ISNULL(col_descs_) || col_cnt != col_descs_->count()) { // defense + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpect error", K(ret), KPC(col_descs_), K_(integer_col_cnt), K(col_cnt)); + } else { + for (int64_t col_idx = 0; col_idx < col_cnt; col_idx++) { + for (int64_t row_idx = 0; row_idx < row_count; row_idx++) { + curr_micro_column_checksum[col_idx] += all_col_datums[col_idx]->at(row_idx).checksum(0); + } + } + } + return ret; +} + +}//end namespace blocksstable +}//end namespace oceanbase diff --git a/src/storage/blocksstable/ob_micro_block_checksum_helper.h b/src/storage/blocksstable/ob_micro_block_checksum_helper.h index 9e207e6c4c..b98a69aff1 100644 --- a/src/storage/blocksstable/ob_micro_block_checksum_helper.h +++ b/src/storage/blocksstable/ob_micro_block_checksum_helper.h @@ -15,6 +15,7 @@ #include "storage/blocksstable/ob_datum_row.h" #include "share/schema/ob_table_param.h" #include "storage/compaction/ob_compaction_memory_context.h" +#include "storage/blocksstable/encoding/ob_encoding_util.h" namespace oceanbase { @@ -32,10 +33,10 @@ public: micro_block_row_checksum_(0) {} ~ObMicroBlockChecksumHelper() { reset(); } - inline int init( - const common::ObIArray *col_descs, - const bool need_opt_row_chksum); - inline void reset(); + int init( + const common::ObIArray *col_descs, + const bool need_opt_row_chksum); + void reset(); inline void reuse() { micro_block_row_checksum_ = 0; } inline int64_t get_row_checksum() const { return micro_block_row_checksum_; } inline bool is_local_buf() const { return integer_col_buf_ == local_integer_col_buf_; } @@ -57,7 +58,16 @@ public: } return ret; } - TO_STRING_KV(KPC_(col_descs), KP_(integer_col_idx), KP_(integer_col_buf), K_(integer_col_cnt), K_(micro_block_row_checksum)); + int cal_rows_checksum(const common::ObArray &all_col_datums, + const int64_t row_count); + + int cal_column_checksum(const common::ObArray &all_col_datums, + const int64_t row_count, + int64_t *curr_micro_column_checksum); + + TO_STRING_KV(KPC_(col_descs), KP_(integer_col_idx), KP_(integer_col_buf), + K_(integer_col_cnt), K_(micro_block_row_checksum)); + private: static const int64_t MAGIC_NOP_NUMBER = 0xa1b; static const int64_t MAGIC_NULL_NUMBER = 0xce75; @@ -74,83 +84,12 @@ private: int64_t local_integer_col_buf_[LOCAL_INTEGER_COL_CNT]; }; -int ObMicroBlockChecksumHelper::init( - const common::ObIArray *col_descs, - const bool need_opt_row_chksum) -{ - int ret = OB_SUCCESS; - reset(); - if (OB_ISNULL(col_descs)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), KP(col_descs)); - } else { - if (need_opt_row_chksum) { - for (int64_t i = 0; i < col_descs->count(); ++i) { - if (col_descs->at(i).col_type_.is_integer_type()) { - ++integer_col_cnt_; - } - } - } - if (NEED_INTEGER_BUF_CNT < integer_col_cnt_) { - if (LOCAL_INTEGER_COL_CNT >= integer_col_cnt_) { - integer_col_buf_ = local_integer_col_buf_; - integer_col_idx_ = local_integer_col_idx_; - } else if (OB_ISNULL(integer_col_buf_ = - static_cast(allocator_.alloc(sizeof(int64_t) * integer_col_cnt_)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "failed to alloc integer_col_buf", K(ret), K_(integer_col_cnt)); - } else if (OB_ISNULL(integer_col_idx_ = - static_cast(allocator_.alloc(sizeof(int16_t) * integer_col_cnt_)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "failed to alloc integer_col_idx", K(ret), K_(integer_col_cnt)); - } - } - } - if (OB_SUCC(ret)) { - // traverse once again to fill idx - if (OB_NOT_NULL(integer_col_idx_)) { - for (int64_t i = 0, idx = 0; i < col_descs->count(); ++i) { - if (idx < integer_col_cnt_ && col_descs->at(i).col_type_.is_integer_type()) { - integer_col_idx_[idx] = i; - ++idx; - } - } - } - micro_block_row_checksum_ = 0; - col_descs_ = col_descs; - } else { - reset(); - } - return ret; -} - -void ObMicroBlockChecksumHelper::reset() -{ - col_descs_ = nullptr; - integer_col_cnt_ = 0; - micro_block_row_checksum_ = 0; - if (OB_NOT_NULL(integer_col_buf_)) { - if (!is_local_buf()) { - allocator_.free(integer_col_buf_); - } - integer_col_buf_ = nullptr; - } - if (OB_NOT_NULL(integer_col_idx_)) { - if (!is_local_idx()) { - allocator_.free(integer_col_idx_); - } - integer_col_idx_ = nullptr; - } - allocator_.reset(); -} - template int ObMicroBlockChecksumHelper::cal_row_checksum( const T* datums, const int64_t row_col_cnt) { int ret = OB_SUCCESS; - bool need_free = false; if (OB_ISNULL(datums)) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "invalid argument", K(ret), KP(datums)); @@ -181,6 +120,7 @@ int ObMicroBlockChecksumHelper::cal_row_checksum( } return ret; } + }//end namespace blocksstable }//end namespace oceanbase diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp index 28c57e525c..f351db28f8 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp @@ -1527,6 +1527,7 @@ int ObTabletDirectLoadMgr::prepare_schema_item_on_demand(const uint64_t table_id ObColumnSchemaItem column_item; if (i >= table_schema->get_rowkey_column_num() && i < table_schema->get_rowkey_column_num() + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt()) { // skip multi version column, keep item invalid + column_item.col_type_ = col_desc.col_type_; // for append_batch } else if (OB_ISNULL(column_schema = table_schema->get_column_schema(col_desc.col_id_))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("column schema is null", K(ret), K(i), K(data_desc.get_col_desc_array()), K(col_desc.col_id_)); @@ -1625,6 +1626,71 @@ int ObTabletDirectLoadMgr::fill_sstable_slice( return ret; } +int ObTabletDirectLoadMgr::fill_sstable_slice( + const ObDirectLoadSliceInfo &slice_info, + const SCN &start_scn, + const ObBatchDatumRows &datum_rows, + ObInsertMonitor *insert_monitor) +{ + int ret = OB_SUCCESS; + share::SCN commit_scn; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_UNLIKELY(!slice_info.is_valid() || !start_scn.is_valid_and_not_min()) || !sqc_build_ctx_.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", K(ret), K(slice_info), K(start_scn), K(sqc_build_ctx_)); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected err", K(ret), KPC(this)); + } else if (is_full_direct_load(direct_load_type_)) { + if (sqc_build_ctx_.commit_scn_.is_valid_and_not_min()) { + ret = OB_TRANS_COMMITED; + FLOG_INFO("already committed", K(commit_scn), KPC(this)); + } else if (start_scn != get_start_scn()) { + ret = OB_TASK_EXPIRED; + LOG_WARN("task expired", K(ret), "start_scn of current execution", start_scn, "start_scn latest", get_start_scn()); + } + } + if (OB_SUCC(ret)) { + ObDirectLoadSliceWriter *slice_writer = nullptr; + ObTabletDirectLoadBuildCtx::SliceKey slice_key(slice_info.context_id_, slice_info.slice_id_); + if (OB_FAIL(sqc_build_ctx_.slice_mgr_map_.get_refactored(slice_key, slice_writer))) { + LOG_WARN("get refactored failed", K(ret), K(slice_info)); + } else if (OB_ISNULL(slice_writer) || OB_UNLIKELY(!ATOMIC_LOAD(&is_schema_item_ready_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected err", K(ret), K(slice_info), K(is_schema_item_ready_)); + } else if (OB_FAIL(slice_writer->fill_sstable_slice(start_scn, + sqc_build_ctx_.build_param_.runtime_only_param_.table_id_, + tablet_id_, + sqc_build_ctx_.storage_schema_, + datum_rows, + schema_item_, + direct_load_type_, + column_items_, + dir_id_, + sqc_build_ctx_.build_param_.runtime_only_param_.parallel_, + insert_monitor))) { + LOG_WARN("fill sstable slice failed", K(ret), KPC(this)); + } + } + if (OB_FAIL(ret) && (OB_TRANS_COMMITED != ret)) { + // cleanup when failed. + int tmp_ret = OB_SUCCESS; + ObDirectLoadSliceWriter *slice_writer = nullptr; + ObTabletDirectLoadBuildCtx::SliceKey slice_key(slice_info.context_id_, slice_info.slice_id_); + if (OB_TMP_FAIL(sqc_build_ctx_.slice_mgr_map_.erase_refactored(slice_key, &slice_writer))) { + LOG_ERROR("erase failed", K(ret), K(tmp_ret), K(slice_info)); + } else { + LOG_INFO("erase a slice writer", KP(slice_writer), K(slice_key), K(sqc_build_ctx_.slice_mgr_map_.size())); + slice_writer->~ObDirectLoadSliceWriter(); + sqc_build_ctx_.slice_writer_allocator_.free(slice_writer); + slice_writer = nullptr; + } + } + return ret; +} + int ObTabletDirectLoadMgr::fill_lob_sstable_slice( ObIAllocator &allocator, const ObDirectLoadSliceInfo &slice_info, @@ -1686,6 +1752,67 @@ int ObTabletDirectLoadMgr::fill_lob_sstable_slice( return ret; } +int ObTabletDirectLoadMgr::fill_lob_sstable_slice( + ObIAllocator &allocator, + const ObDirectLoadSliceInfo &slice_info, + const SCN &start_scn, + share::ObTabletCacheInterval &pk_interval, + blocksstable::ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + share::SCN commit_scn; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_UNLIKELY(!slice_info.is_valid() || !sqc_build_ctx_.is_valid() || !start_scn.is_valid_and_not_min() || + !lob_mgr_handle_.is_valid() || !lob_mgr_handle_.get_obj()->get_sqc_build_ctx().is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(slice_info), "lob_direct_load_mgr is valid", lob_mgr_handle_.is_valid(), KPC(this), K(start_scn)); + } else if (is_full_direct_load(direct_load_type_)) { + if (sqc_build_ctx_.commit_scn_.is_valid_and_not_min()) { + ret = OB_TRANS_COMMITED; + FLOG_INFO("already committed", K(commit_scn), KPC(this)); + } else if (start_scn != get_start_scn()) { + ret = OB_TASK_EXPIRED; + LOG_WARN("task expired", K(ret), "start_scn of current execution", start_scn, "start_scn latest", get_start_scn()); + } + } + + if (OB_SUCC(ret)) { + ObDirectLoadSliceWriter *slice_writer = nullptr; + ObTabletDirectLoadBuildCtx::SliceKey slice_key(slice_info.context_id_, slice_info.slice_id_); + const int64_t trans_version = is_full_direct_load(direct_load_type_) ? table_key_.get_snapshot_version() : INT64_MAX; + ObBatchSliceWriteInfo info(tablet_id_, ls_id_, trans_version, direct_load_type_, sqc_build_ctx_.build_param_.runtime_only_param_.trans_id_, + sqc_build_ctx_.build_param_.runtime_only_param_.seq_no_, slice_info.src_tenant_id_, + sqc_build_ctx_.build_param_.runtime_only_param_.tx_desc_); + + if (OB_FAIL(lob_mgr_handle_.get_obj()->get_sqc_build_ctx().slice_mgr_map_.get_refactored(slice_key, slice_writer))) { + LOG_WARN("get refactored failed", K(ret), K(slice_info), K(sqc_build_ctx_.slice_mgr_map_.size())); + } else if (OB_ISNULL(slice_writer) || OB_UNLIKELY(!ATOMIC_LOAD(&(lob_mgr_handle_.get_obj()->is_schema_item_ready_)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected err", K(ret), K(slice_info), K(lob_mgr_handle_.get_obj()->is_schema_item_ready_)); + } else if (OB_FAIL(slice_writer->fill_lob_sstable_slice(lob_mgr_handle_.get_obj()->sqc_build_ctx_.build_param_.runtime_only_param_.table_id_, allocator, sqc_build_ctx_.allocator_, + start_scn, info, pk_interval, lob_column_idxs_, lob_col_types_, schema_item_.lob_inrow_threshold_, datum_rows))) { + LOG_WARN("fail to fill batch sstable slice", K(ret), K(start_scn), K(tablet_id_), K(pk_interval)); + } + } + if (OB_FAIL(ret) && lob_mgr_handle_.is_valid()) { + // cleanup when failed. + int tmp_ret = OB_SUCCESS; + ObDirectLoadSliceWriter *slice_writer = nullptr; + ObTabletDirectLoadBuildCtx::SliceKey slice_key(slice_info.context_id_, slice_info.slice_id_); + if (OB_TMP_FAIL(lob_mgr_handle_.get_obj()->get_sqc_build_ctx().slice_mgr_map_.erase_refactored(slice_key, &slice_writer))) { + LOG_ERROR("erase failed", K(ret), K(tmp_ret), K(slice_info)); + } else { + LOG_INFO("erase a slice writer", KP(slice_writer), K(slice_key), K(sqc_build_ctx_.slice_mgr_map_.size())); + slice_writer->~ObDirectLoadSliceWriter(); + lob_mgr_handle_.get_obj()->get_sqc_build_ctx().slice_writer_allocator_.free(slice_writer); + slice_writer = nullptr; + } + } + return ret; +} + int ObTabletDirectLoadMgr::fill_lob_meta_sstable_slice( const ObDirectLoadSliceInfo &slice_info, const share::SCN &start_scn, @@ -1819,14 +1946,10 @@ public: ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid cs slice writer", K(ret), KPC(left), KPC(right)); } else { - const ObChunkSliceStore *left_slice_store = left->is_cs_replica_write() - ? static_cast(left->get_slice_store())->get_column_slice_store() - : static_cast(left->get_slice_store()); - const ObChunkSliceStore *right_slice_store = right->is_cs_replica_write() - ? static_cast(right->get_slice_store())->get_column_slice_store() - : static_cast(right->get_slice_store()); + const ObDatumRowkey left_key = left->get_slice_store()->get_compare_key(); + const ObDatumRowkey right_key = right->get_slice_store()->get_compare_key(); int cmp_ret = 0; - if (OB_FAIL(left_slice_store->endkey_.compare(right_slice_store->endkey_, datum_utils_, cmp_ret))) { + if (OB_FAIL(left_key.compare(right_key, datum_utils_, cmp_ret))) { LOG_WARN("endkey compare failed", K(ret)); } else { bret = cmp_ret < 0; @@ -2247,8 +2370,6 @@ int ObTabletDirectLoadMgr::fill_aggregated_column_group( const ObIArray &cg_schemas = storage_schema->get_column_groups(); for (int64_t cg_idx = 0; OB_SUCC(ret) && cg_idx < cg_schemas.count(); ++cg_idx) { cur_writer->reset(); - common::ObArray datum_stores; - datum_stores.set_attr(ObMemAttr(MTL_ID(), "TDL_Agg_CG")); if (start_idx == last_idx || start_idx >= sqc_build_ctx_.sorted_slice_writers_.count() || last_idx > sqc_build_ctx_.sorted_slice_writers_.count()) { // skip } else { @@ -2267,7 +2388,7 @@ int ObTabletDirectLoadMgr::fill_aggregated_column_group( if (OB_ISNULL(slice_writer) || !slice_writer->need_column_store()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("wrong slice writer", K(ret), KPC(slice_writer)); - } else if (OB_FAIL(slice_writer->fill_aggregated_column_group(cg_idx, cur_writer, datum_stores))) { + } else if (OB_FAIL(slice_writer->fill_aggregated_column_group(cg_idx, cur_writer))) { LOG_WARN("slice writer rescan failed", K(ret), K(cg_idx), KPC(cur_writer)); } else if (cg_idx == cg_schemas.count() - 1) { // after fill last cg, inc finish cnt @@ -2282,13 +2403,6 @@ int ObTabletDirectLoadMgr::fill_aggregated_column_group( if (OB_SUCC(ret)) { if (cur_writer->is_inited() && OB_FAIL(cur_writer->close())) { LOG_WARN("close co ddl writer failed", K(ret)); - } else { - for (int64_t i = 0; i < datum_stores.count(); ++i) { - if (OB_NOT_NULL(datum_stores.at(i))) { - datum_stores.at(i)->~ObCompactStore(); - } - } - datum_stores.reset(); } } // next cg diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h index 636953ba6e..7c04bd4350 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h @@ -340,12 +340,23 @@ public: ObIStoreRowIterator *iter, int64_t &affected_rows, ObInsertMonitor *insert_monitor = NULL); + virtual int fill_sstable_slice( + const ObDirectLoadSliceInfo &slice_info, + const share::SCN &start_scn, + const blocksstable::ObBatchDatumRows &datum_rows, + ObInsertMonitor *insert_monitor = NULL); virtual int fill_lob_sstable_slice( ObIAllocator &allocator, const ObDirectLoadSliceInfo &slice_info /*contains data_tablet_id, lob_slice_id, start_seq*/, const share::SCN &start_scn, share::ObTabletCacheInterval &pk_interval, blocksstable::ObDatumRow &datum_row); + virtual int fill_lob_sstable_slice( + ObIAllocator &allocator, + const ObDirectLoadSliceInfo &slice_info /*contains data_tablet_id, lob_slice_id, start_seq*/, + const share::SCN &start_scn, + share::ObTabletCacheInterval &pk_interval, + blocksstable::ObBatchDatumRows &datum_rows); // for delete lob in incremental direct load only virtual int fill_lob_meta_sstable_slice( const ObDirectLoadSliceInfo &slice_info /*contains data_tablet_id, lob_slice_id, start_seq*/, diff --git a/src/storage/ddl/ob_direct_load_mgr_agent.cpp b/src/storage/ddl/ob_direct_load_mgr_agent.cpp index c536163551..d1748fc497 100644 --- a/src/storage/ddl/ob_direct_load_mgr_agent.cpp +++ b/src/storage/ddl/ob_direct_load_mgr_agent.cpp @@ -275,6 +275,62 @@ int ObDirectLoadMgrAgent::fill_sstable_slice_for_ss( return ret; } +int ObDirectLoadMgrAgent::fill_sstable_slice( + const ObDirectLoadSliceInfo &slice_info, + const ObBatchDatumRows &datum_rows, + ObInsertMonitor *insert_monitor) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_UNLIKELY(!slice_info.is_valid() || 0 == datum_rows.row_count_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", K(ret), K(slice_info), K(datum_rows.row_count_)); + } else if (!is_shared_storage_dempotent_mode(direct_load_type_)) { + if (OB_FAIL(fill_sstable_slice_for_sn(slice_info, datum_rows, insert_monitor))) { + LOG_WARN("fill slice for sn failed", K(ret), K(slice_info)); + } + } else if (OB_FAIL(fill_sstable_slice_for_ss(slice_info, datum_rows, insert_monitor))) { + LOG_WARN("fill slice for ss failed", K(ret), K(slice_info)); + } + return ret; +} + +int ObDirectLoadMgrAgent::fill_sstable_slice_for_sn( + const ObDirectLoadSliceInfo &slice_info, + const ObBatchDatumRows &datum_rows, + ObInsertMonitor *insert_monitor) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!mgr_handle_.is_valid())) { + // already committed, do nothing + } else if (OB_FAIL(mgr_handle_.get_obj()->fill_sstable_slice(slice_info, start_scn_, datum_rows, insert_monitor))) { + if (OB_TRANS_COMMITED == ret && slice_info.is_full_direct_load_) { + ret = OB_SUCCESS; + LOG_INFO("trans commited", K(slice_info)); + } else { + LOG_WARN("fill slice failed", K(ret), K(slice_info)); + } + } + return ret; +} + +int ObDirectLoadMgrAgent::fill_sstable_slice_for_ss( + const ObDirectLoadSliceInfo &slice_info, + const ObBatchDatumRows &datum_rows, + ObInsertMonitor *insert_monitor) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!mgr_handle_.is_valid())) { + ret = OB_ERR_SYS; + LOG_WARN("error sys", K(ret)); + } else if (OB_FAIL(mgr_handle_.get_obj()->fill_sstable_slice(slice_info, start_scn_, datum_rows, insert_monitor))) { + LOG_WARN("fill slice failed ss", K(ret), K(slice_info)); + } + return ret; +} + int ObDirectLoadMgrAgent::fill_lob_sstable_slice( ObIAllocator &allocator, const ObDirectLoadSliceInfo &slice_info /*contains data_tablet_id, lob_slice_id, start_seq*/, @@ -329,6 +385,60 @@ int ObDirectLoadMgrAgent::fill_lob_sstable_slice_for_ss( return ret; } +int ObDirectLoadMgrAgent::fill_lob_sstable_slice( + ObIAllocator &allocator, + const ObDirectLoadSliceInfo &slice_info /*contains data_tablet_id, lob_slice_id, start_seq*/, + share::ObTabletCacheInterval &pk_interval, + blocksstable::ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_UNLIKELY(!slice_info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", K(ret), K(slice_info)); + } else if (!is_shared_storage_dempotent_mode(direct_load_type_)) { + if (OB_FAIL(fill_lob_sstable_slice_for_sn(allocator, slice_info, pk_interval, datum_rows))) { + LOG_WARN("fill slice for sn failed", K(ret), K(slice_info)); + } + } else if (OB_FAIL(fill_lob_sstable_slice_for_ss(allocator, slice_info, pk_interval, datum_rows))) { + LOG_WARN("fill slice for ss failed", K(ret), K(slice_info)); + } + return ret; +} + +int ObDirectLoadMgrAgent::fill_lob_sstable_slice_for_sn( + ObIAllocator &allocator, + const ObDirectLoadSliceInfo &slice_info, + share::ObTabletCacheInterval &pk_interval, + blocksstable::ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!mgr_handle_.is_valid())) { + // already committed, do nothing. + } else if (OB_FAIL(mgr_handle_.get_obj()->fill_lob_sstable_slice(allocator, slice_info, start_scn_, pk_interval, datum_rows))) { + LOG_WARN("fail to fill batch sstable slice", K(ret), K(slice_info)); + } + return ret; +} + +int ObDirectLoadMgrAgent::fill_lob_sstable_slice_for_ss( + ObIAllocator &allocator, + const ObDirectLoadSliceInfo &slice_info, + share::ObTabletCacheInterval &pk_interval, + blocksstable::ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!mgr_handle_.is_valid())) { + ret = OB_ERR_SYS; + LOG_WARN("error sys", K(ret)); + } else if (OB_FAIL(mgr_handle_.get_obj()->fill_lob_sstable_slice(allocator, slice_info, start_scn_, pk_interval, datum_rows))) { + LOG_WARN("fail to fill batch sstable slice", K(ret), K(slice_info)); + } + return ret; +} + int ObDirectLoadMgrAgent::fill_lob_meta_sstable_slice( const ObDirectLoadSliceInfo &slice_info /*contains data_tablet_id, lob_slice_id, start_seq*/, ObIStoreRowIterator *iter, diff --git a/src/storage/ddl/ob_direct_load_mgr_agent.h b/src/storage/ddl/ob_direct_load_mgr_agent.h index fc7a42f19e..465e885d77 100644 --- a/src/storage/ddl/ob_direct_load_mgr_agent.h +++ b/src/storage/ddl/ob_direct_load_mgr_agent.h @@ -45,11 +45,20 @@ public: ObIStoreRowIterator *iter, int64_t &affected_rows, ObInsertMonitor *insert_monitor = nullptr); + int fill_sstable_slice( + const ObDirectLoadSliceInfo &slice_info, + const blocksstable::ObBatchDatumRows &datum_rows, + ObInsertMonitor *insert_monitor = nullptr); int fill_lob_sstable_slice( ObIAllocator &allocator, const ObDirectLoadSliceInfo &slice_info /*contains data_tablet_id, lob_slice_id, start_seq*/, share::ObTabletCacheInterval &pk_interval, blocksstable::ObDatumRow &datum_row); + int fill_lob_sstable_slice( + ObIAllocator &allocator, + const ObDirectLoadSliceInfo &slice_info /*contains data_tablet_id, lob_slice_id, start_seq*/, + share::ObTabletCacheInterval &pk_interval, + blocksstable::ObBatchDatumRows &datum_rows); int fill_lob_meta_sstable_slice( const ObDirectLoadSliceInfo &slice_info /*contains data_tablet_id, lob_slice_id, start_seq*/, ObIStoreRowIterator *iter, @@ -81,11 +90,20 @@ private: ObIStoreRowIterator *iter, int64_t &affected_rows, ObInsertMonitor *insert_monitor = nullptr); + int fill_sstable_slice_for_ss( + const ObDirectLoadSliceInfo &slice_info, + const blocksstable::ObBatchDatumRows &datum_rows, + ObInsertMonitor *insert_monitor = nullptr); int fill_lob_sstable_slice_for_ss( ObIAllocator &allocator, const ObDirectLoadSliceInfo &slice_info /*contains data_tablet_id, lob_slice_id, start_seq*/, share::ObTabletCacheInterval &pk_interval, blocksstable::ObDatumRow &datum_row); + int fill_lob_sstable_slice_for_ss( + ObIAllocator &allocator, + const ObDirectLoadSliceInfo &slice_info /*contains data_tablet_id, lob_slice_id, start_seq*/, + share::ObTabletCacheInterval &pk_interval, + blocksstable::ObBatchDatumRows &datum_rows); int fill_lob_meta_sstable_slice_for_ss( const ObDirectLoadSliceInfo &slice_info /*contains data_tablet_id, lob_slice_id, start_seq*/, ObIStoreRowIterator *iter, @@ -103,11 +121,20 @@ private: ObIStoreRowIterator *iter, int64_t &affected_rows, ObInsertMonitor *insert_monitor = nullptr); + int fill_sstable_slice_for_sn( + const ObDirectLoadSliceInfo &slice_info, + const blocksstable::ObBatchDatumRows &datum_rows, + ObInsertMonitor *insert_monitor = nullptr); int fill_lob_sstable_slice_for_sn( ObIAllocator &allocator, const ObDirectLoadSliceInfo &slice_info /*contains data_tablet_id, lob_slice_id, start_seq*/, share::ObTabletCacheInterval &pk_interval, blocksstable::ObDatumRow &datum_row); + int fill_lob_sstable_slice_for_sn( + ObIAllocator &allocator, + const ObDirectLoadSliceInfo &slice_info /*contains data_tablet_id, lob_slice_id, start_seq*/, + share::ObTabletCacheInterval &pk_interval, + blocksstable::ObBatchDatumRows &datum_rows); int fill_lob_meta_sstable_slice_for_sn( const ObDirectLoadSliceInfo &slice_info /*contains data_tablet_id, lob_slice_id, start_seq*/, ObIStoreRowIterator *iter, diff --git a/src/storage/ddl/ob_direct_load_struct.cpp b/src/storage/ddl/ob_direct_load_struct.cpp index 1da288bf5b..cfb441acf2 100755 --- a/src/storage/ddl/ob_direct_load_struct.cpp +++ b/src/storage/ddl/ob_direct_load_struct.cpp @@ -570,6 +570,7 @@ void ObChunkSliceStore::reset() target_store_idx_ = -1; row_cnt_ = 0; arena_allocator_ = nullptr; + is_canceled_ = false; is_inited_ = false; } @@ -723,6 +724,340 @@ int ObChunkSliceStore::close() return ret; } +int ObChunkSliceStore::fill_column_group(const int64_t cg_idx, + ObCOSliceWriter *cur_writer, + ObInsertMonitor *insert_monitor) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObChunkSliceStore not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(cg_idx < 0 || cg_idx >= datum_stores_.count() || nullptr == cur_writer)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(datum_stores_.count()), K(cg_idx), KP(cur_writer)); + } else { + sql::ObCompactStore *cur_datum_store = datum_stores_.at(cg_idx); + const ObChunkDatumStore::StoredRow *stored_row = nullptr; + bool has_next = false; + int64_t cg_row_inserted_cnt = 0; + while (OB_SUCC(ret) && OB_SUCC(cur_datum_store->has_next(has_next)) && has_next) { + if (OB_FAIL(cur_datum_store->get_next_row(stored_row))) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; + } else { + LOG_WARN("get next row failed", K(ret)); + } + } else { + if (OB_FAIL(cur_writer->append_row(stored_row))) { + LOG_WARN("append row failed", K(ret), KPC(stored_row)); + } else { + ++cg_row_inserted_cnt; + if (0 == cg_row_inserted_cnt % 100) { + if (OB_NOT_NULL(insert_monitor)) { + (void) ATOMIC_AAF(&insert_monitor->inserted_cg_row_cnt_, 100); + } + if (OB_UNLIKELY(is_canceled_)) { + ret = OB_CANCELED; + LOG_WARN("fill column group is canceled", KR(ret)); + } else if (OB_FAIL(share::dag_yield())) { + LOG_WARN("dag yield failed", K(ret)); // exit for dag task as soon as possible after canceled. + } else if (OB_FAIL(THIS_WORKER.check_status())) { + LOG_WARN("check status failed", K(ret)); + } + } + } + } + } + if (OB_SUCC(ret)) { + if (OB_NOT_NULL(insert_monitor)) { + (void) ATOMIC_AAF(&insert_monitor->inserted_cg_row_cnt_, cg_row_inserted_cnt % 100); + } + cur_datum_store->reset(); + } + } + return ret; +} + +/** + * ObChunkBatchSliceStore + */ + +void ObChunkBatchSliceStore::reset() +{ + is_inited_ = false; + if (OB_NOT_NULL(arena_allocator_)) { + for (int64_t i = 0; i < cg_ctxs_.count(); ++i) { + ColumnGroupCtx *cg_ctx = cg_ctxs_.at(i); + cg_ctx->~ColumnGroupCtx(); + arena_allocator_->free(cg_ctx); + cg_ctx = nullptr; + } + } + cg_ctxs_.reset(); + arena_allocator_ = nullptr; + column_count_ = 0; + rowkey_column_count_ = 0; + row_cnt_ = 0; + start_key_.reset(); + is_canceled_ = false; +} + +int ObChunkBatchSliceStore::init(const int64_t rowkey_column_count, + const ObStorageSchema *storage_schema, + ObArenaAllocator &allocator, + const ObIArray &col_array, + const int64_t dir_id, + const int64_t parallelism, + const int64_t max_batch_size) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObChunkBatchSliceStore init twice", KR(ret), KP(this)); + } else if (OB_ISNULL(storage_schema)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("null schema", KR(ret), K(*this)); + } else if (OB_UNLIKELY(rowkey_column_count <= 0 || max_batch_size <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(rowkey_column_count), K(max_batch_size)); + } else if (OB_FAIL(prepare_column_group_ctxs(MTL_ID(), storage_schema, allocator, col_array, dir_id, parallelism, max_batch_size))) { + LOG_WARN("fail to prepare datum stores"); + } else { + arena_allocator_ = &allocator; + column_count_ = col_array.count(); + rowkey_column_count_ = rowkey_column_count; + is_inited_ = true; + } + LOG_DEBUG("init chunk batch slice store", KR(ret), KPC(this)); + return ret; +} + +int ObChunkBatchSliceStore::prepare_column_group_ctxs( + const uint64_t tenant_id, + const ObStorageSchema *storage_schema, + ObIAllocator &allocator, + const ObIArray &col_array, + const int64_t dir_id, + const int64_t parallelism, + const int64_t max_batch_size) +{ + int ret = OB_SUCCESS; + const int64_t chunk_mem_limit = 64 * 1024L; // 64K + if (OB_UNLIKELY(tenant_id <= 0 || nullptr == storage_schema)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(tenant_id), KP(storage_schema)); + } else { + const ObIArray &cg_schemas = storage_schema->get_column_groups(); + ObArray column_items; + column_items.set_attr(ObMemAttr(tenant_id, "tmp_cg_item")); + for (int64_t i = 0; OB_SUCC(ret) && i < cg_schemas.count(); ++i) { + const ObStorageColumnGroupSchema &cg_schema = cg_schemas.at(i); + ObCompressorType compressor_type = cg_schema.compressor_type_; + compressor_type = NONE_COMPRESSOR == compressor_type ? (CS_ENCODING_ROW_STORE == cg_schema.row_store_type_ ? ZSTD_1_3_8_COMPRESSOR : NONE_COMPRESSOR) : compressor_type; + column_items.reuse(); + if (OB_FAIL(ObDDLUtil::get_temp_store_compress_type(compressor_type, + parallelism, + compressor_type))) { + LOG_WARN("fail to get temp store compress type", KR(ret)); + } + for (int64_t j = 0; OB_SUCC(ret) && j < cg_schema.column_cnt_; ++j) { + const int64_t column_idx = cg_schema.get_column_idx(j); // all_cg column_idxs_ = null + if (OB_UNLIKELY(column_idx >= col_array.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid column idex", KR(ret), K(i), K(j), K(column_idx), K(cg_schema), K(col_array.count())); + } else if (OB_FAIL(column_items.push_back(col_array.at(column_idx)))) { + LOG_WARN("fail to push_back col_item", K(ret)); + } + } + if (OB_SUCC(ret)) { + const int64_t skip_size = ObBitVector::memory_size(max_batch_size); + void *skip_mem = nullptr; + ColumnGroupCtx *cg_ctx = nullptr; + if (OB_ISNULL(cg_ctx = OB_NEWx(ColumnGroupCtx, &allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ColumnGroupCtx", KR(ret)); + } else if (OB_FAIL(ObTempColumnStore::init_vectors(column_items, cg_ctx->allocator_, cg_ctx->vectors_))) { + LOG_WARN("fail to init vectors", KR(ret), K(i), K(column_items)); + } else if (OB_FAIL(cg_ctx->store_.init(cg_ctx->vectors_, + max_batch_size, + ObMemAttr(tenant_id, "DL_VEC_STORE"), + chunk_mem_limit, + true/*enable_dump*/, + compressor_type))) { + LOG_WARN("failed to init temp column store", KR(ret), K(i), K(column_items)); + } else if (OB_FAIL(cg_ctx->append_vectors_.prepare_allocate(cg_schema.column_cnt_))) { + LOG_WARN("fail to prepare allocate", KR(ret), K(cg_schema.column_cnt_)); + } else if (OB_ISNULL(skip_mem = allocator.alloc(skip_size))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc skip buf", KR(ret)); + } else { + cg_ctx->cg_schema_ = cg_schema; + cg_ctx->store_.set_dir_id(dir_id); + cg_ctx->store_.get_inner_allocator().set_tenant_id(tenant_id); + cg_ctx->brs_.skip_ = to_bit_vector(skip_mem); + cg_ctx->brs_.skip_->reset(max_batch_size); + cg_ctx->brs_.size_ = 0; + cg_ctx->brs_.set_all_rows_active(true); + LOG_INFO("set dir id", K(dir_id)); + } + if (OB_SUCC(ret)) { + if (OB_FAIL(cg_ctxs_.push_back(cg_ctx))) { + LOG_WARN("fail to push back", KR(ret)); + } + } + if (OB_FAIL(ret)) { + if (nullptr != cg_ctx) { + cg_ctx->~ColumnGroupCtx(); + allocator.free(cg_ctx); + cg_ctx = nullptr; + } + } + } + } + } + return ret; +} + +int ObChunkBatchSliceStore::init_start_key() +{ + int ret = OB_SUCCESS; + void *buf = nullptr; + if (OB_ISNULL(buf = arena_allocator_->alloc(sizeof(ObStorageDatum) * rowkey_column_count_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc datums", KR(ret), K(rowkey_column_count_)); + } else { + start_key_.datums_ = new (buf) ObStorageDatum[rowkey_column_count_]; + start_key_.datum_cnt_ = rowkey_column_count_; + } + return ret; +} + +int ObChunkBatchSliceStore::append_batch(const ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObChunkBatchSliceStore not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(datum_rows.get_column_count() < column_count_ || datum_rows.row_count_ <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(column_count_), K(datum_rows.get_column_count()), K(datum_rows.row_count_)); + } else { + int64_t stored_rows_count = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < cg_ctxs_.count(); ++i) { + ColumnGroupCtx *cg_ctx = cg_ctxs_.at(i); + const ObStorageColumnGroupSchema &cg_schema = cg_ctx->cg_schema_; + for (int64_t j = 0; j < cg_schema.column_cnt_; ++j) { + const int64_t column_idx = cg_schema.get_column_idx(j); + cg_ctx->append_vectors_.at(j) = datum_rows.vectors_.at(column_idx); + } + cg_ctx->brs_.size_ = datum_rows.row_count_; + if (OB_FAIL(cg_ctx->store_.add_batch(cg_ctx->append_vectors_, cg_ctx->brs_, stored_rows_count))) { + LOG_WARN("fail to add batch", KR(ret), K(i), K(cg_schema)); + } else if (OB_UNLIKELY(stored_rows_count != datum_rows.row_count_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected rows count", KR(ret), K(stored_rows_count), K(datum_rows.row_count_)); + } + } + // save start_key_ + if (OB_SUCC(ret) && 0 == row_cnt_) { + bool is_null = false; + const char *payload = nullptr; + ObLength length = 0; + ObStorageDatum tmp_datum; + if (OB_FAIL(init_start_key())) { + LOG_WARN("fail to init start key", KR(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_column_count_; ++i) { + ObIVector *vec = datum_rows.vectors_.at(i); + vec->get_payload(0, is_null, payload, length); + tmp_datum.shallow_copy_from_datum(ObDatum(payload, length, is_null)); + if (OB_FAIL(start_key_.datums_[i].deep_copy(tmp_datum, *arena_allocator_))) { + LOG_WARN("fail to deep copy storage datum", KR(ret), K(tmp_datum)); + } + } + } + if (OB_SUCC(ret)) { + row_cnt_ += datum_rows.row_count_; + } + } + return ret; +} + +int ObChunkBatchSliceStore::close() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObChunkBatchSliceStore not init", KR(ret), KP(this)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < cg_ctxs_.count(); ++i) { + ColumnGroupCtx *cg_ctx = cg_ctxs_.at(i); + if (OB_FAIL(cg_ctx->store_.finish_add_row(true/*need_dump*/))) { + LOG_WARN("fail to finish add row", KR(ret)); + } else { + cg_ctx->append_vectors_.reset(); + } + } + } + LOG_DEBUG("chunk batch slice store closed", KR(ret), K(start_key_)); + return ret; +} + +int ObChunkBatchSliceStore::fill_column_group(const int64_t cg_idx, + ObCOSliceWriter *writer, + ObInsertMonitor *insert_monitor) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObChunkBatchSliceStore not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(cg_idx < 0 || cg_idx >= cg_ctxs_.count() || nullptr == writer)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(cg_ctxs_.count()), K(cg_idx), KP(writer)); + } else { + ColumnGroupCtx *cg_ctx = cg_ctxs_.at(cg_idx); + ObTempColumnStore::Iterator iterator; + int64_t read_rows = 0; + if (OB_FAIL(cg_ctx->store_.begin(iterator))) { + LOG_WARN("fail to begin iterator", KR(ret)); + } + while (OB_SUCC(ret)) { + if (OB_UNLIKELY(is_canceled_)) { + ret = OB_CANCELED; + LOG_WARN("fill column group is canceled", KR(ret)); + } else if (OB_FAIL(share::dag_yield())) { + LOG_WARN("dag yield failed", K(ret)); // exit for dag task as soon as possible after canceled. + } else if (OB_FAIL(THIS_WORKER.check_status())) { + LOG_WARN("check status failed", K(ret)); + } else if (OB_FAIL(iterator.get_next_batch(cg_ctx->vectors_, read_rows))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail to get next batch", KR(ret)); + } else { + ret = OB_SUCCESS; + break; + } + } else if (OB_FAIL(writer->append_batch(cg_ctx->vectors_, read_rows))) { + LOG_WARN("fail to append batch", KR(ret)); + } else { + if (OB_NOT_NULL(insert_monitor)) { + (void) ATOMIC_AAF(&insert_monitor->inserted_cg_row_cnt_, read_rows); + } + } + } + iterator.reset(); + if (OB_SUCC(ret)) { + cg_ctx->store_.reset(); + cg_ctx->vectors_.reset(); + cg_ctx->allocator_.reset(); + } + } + return ret; +} + +/** + * ObMacroBlockSliceStore + */ int ObMacroBlockSliceStore::init( ObTabletDirectLoadMgr *tablet_direct_load_mgr, @@ -811,6 +1146,18 @@ int ObMacroBlockSliceStore::append_row(const blocksstable::ObDatumRow &datum_row return ret; } +int ObMacroBlockSliceStore::append_batch(const blocksstable::ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObMacroBlockSliceStore not init", KR(ret), KP(this)); + } else if (OB_FAIL(macro_block_writer_.append_batch(datum_rows))) { + LOG_WARN("macro block writer append batch failed", K(ret)); + } + return ret; +} + int ObMacroBlockSliceStore::close() { int ret = OB_SUCCESS; @@ -845,7 +1192,9 @@ int ObMultiSliceStore::init( const ObStorageSchema *storage_schema, const ObIArray &col_schema, const int64_t dir_id, - const int64_t parallelism) + const int64_t parallelism, + const bool use_batch_store, + const int64_t max_batch_size) { int ret = OB_SUCCESS; if (IS_INIT) { @@ -868,12 +1217,26 @@ int ObMultiSliceStore::init( LOG_WARN("fail to allocate memory for row slice store", K(ret)); } else if (OB_FAIL(row_slice_store_->init(tablet_direct_load_mgr, data_seq, start_scn, true /*need_process_cs_replica*/))) { LOG_WARN("fail to init row slice store", K(ret), KPC(tablet_direct_load_mgr), K(data_seq), K(start_scn)); - } else if (OB_ISNULL(column_slice_store_ = OB_NEWx(ObChunkSliceStore, &allocator))){ - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate memory for column slice store", K(ret)); - } else if (OB_FAIL(column_slice_store_->init(rowkey_column_count, cs_replica_schema_, allocator, col_schema, dir_id, parallelism))) { - LOG_WARN("fail to init column slice store", K(ret), K(dir_id), K(parallelism), KPC(storage_schema), K(col_schema), K(rowkey_column_count)); } else { + if (use_batch_store) { + ObChunkBatchSliceStore *column_slice_store = nullptr; + if (OB_ISNULL(column_slice_store_ = column_slice_store = OB_NEWx(ObChunkBatchSliceStore, &allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObChunkBatchSliceStore", KR(ret)); + } else if (OB_FAIL(column_slice_store->init(rowkey_column_count, cs_replica_schema_, allocator, col_schema, dir_id, parallelism, max_batch_size))) { + LOG_WARN("fail to init chunk batch slice store", K(ret), K(dir_id), K(parallelism), KPC(storage_schema), K(col_schema), K(rowkey_column_count), K(max_batch_size)); + } + } else { + ObChunkSliceStore *column_slice_store = nullptr; + if (OB_ISNULL(column_slice_store_ = column_slice_store = OB_NEWx(ObChunkSliceStore, &allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObChunkSliceStore", KR(ret)); + } else if (OB_FAIL(column_slice_store->init(rowkey_column_count, cs_replica_schema_, allocator, col_schema, dir_id, parallelism))) { + LOG_WARN("fail to init chunk slice store", K(ret), K(dir_id), K(parallelism), KPC(storage_schema), K(col_schema), K(rowkey_column_count)); + } + } + } + if (OB_SUCC(ret)) { is_inited_ = true; arena_allocator_ = &allocator; LOG_DEBUG("[CS-Replica] Successfully init multi slice store", K(ret), KPC(this)); @@ -902,6 +1265,23 @@ int ObMultiSliceStore::append_row(const blocksstable::ObDatumRow &datum_row) return ret; } +int ObMultiSliceStore::append_batch(const blocksstable::ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObMultiSliceStore not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(nullptr == row_slice_store_ || nullptr == column_slice_store_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected slice store", KR(ret), KP(row_slice_store_), KP(column_slice_store_)); + } else if (OB_FAIL(row_slice_store_->append_batch(datum_rows))) { + LOG_WARN("fail to append batch to row slice store", KR(ret)); + } else if (OB_FAIL(column_slice_store_->append_batch(datum_rows))) { + LOG_WARN("fail to append batch to column slice store", KR(ret)); + } + return ret; +} + int ObMultiSliceStore::close() { int ret = OB_SUCCESS; @@ -921,6 +1301,33 @@ int ObMultiSliceStore::close() return ret; } +int ObMultiSliceStore::fill_column_group(const int64_t cg_idx, + ObCOSliceWriter *writer, + ObInsertMonitor *insert_monitor) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObMultiSliceStore not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(nullptr == column_slice_store_)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected column slice store is null", KR(ret), KP(column_slice_store_)); + } else if (OB_FAIL(column_slice_store_->fill_column_group(cg_idx, writer, insert_monitor))) { + LOG_WARN("fail to fill column group", KR(ret), K(cg_idx), KP(writer)); + } + return ret; +} + +void ObMultiSliceStore::cancel() +{ + if (OB_NOT_NULL(row_slice_store_)) { + row_slice_store_->cancel(); + } + if (OB_NOT_NULL(column_slice_store_)) { + column_slice_store_->cancel(); + } +} + int64_t ObMultiSliceStore::get_row_count() const { return column_slice_store_->get_row_count(); @@ -931,6 +1338,15 @@ int64_t ObMultiSliceStore::get_next_block_start_seq() const return row_slice_store_->get_next_block_start_seq(); } +ObDatumRowkey ObMultiSliceStore::get_compare_key() const +{ + ObDatumRowkey rowkey; + if (OB_NOT_NULL(column_slice_store_)) { + rowkey = column_slice_store_->get_compare_key(); + } + return rowkey; +} + void ObMultiSliceStore::reset() { if (OB_NOT_NULL(arena_allocator_)) { @@ -948,7 +1364,7 @@ void ObMultiSliceStore::reset() void ObMultiSliceStore::free_memory(ObArenaAllocator &allocator) { if (OB_NOT_NULL(column_slice_store_)) { - column_slice_store_->~ObChunkSliceStore(); + column_slice_store_->~ObTabletSliceStore(); allocator.free(column_slice_store_); column_slice_store_ = nullptr; } @@ -979,7 +1395,7 @@ bool ObTabletDDLParam::is_valid() const ObDirectLoadSliceWriter::ObDirectLoadSliceWriter() : is_inited_(false), writer_type_(ObDirectLoadSliceWriterType::WRITER_TYPE_MAX), is_canceled_(false), start_seq_(), tablet_direct_load_mgr_(nullptr), slice_store_(nullptr), meta_write_iter_(nullptr), row_iterator_(nullptr), - allocator_(lib::ObLabel("SliceWriter"), OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), row_offset_(-1) + allocator_(lib::ObLabel("SliceWriter"), OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), lob_allocator_(nullptr), row_offset_(-1) { } @@ -1000,6 +1416,11 @@ ObDirectLoadSliceWriter::~ObDirectLoadSliceWriter() allocator_.free(row_iterator_); row_iterator_ = nullptr; } + if (nullptr != lob_allocator_) { + lob_allocator_->reset(); + allocator_.free(lob_allocator_); + lob_allocator_= nullptr; + } allocator_.reset(); row_offset_ = -1; writer_type_ = ObDirectLoadSliceWriterType::WRITER_TYPE_MAX; @@ -1038,7 +1459,9 @@ int ObDirectLoadSliceWriter::prepare_slice_store_if_need( const ObStorageSchema *storage_schema, const SCN &start_scn, const ObString vec_idx_param, - const int64_t vec_dim) + const int64_t vec_dim, + const bool use_batch_store, + const int64_t max_batch_size) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!is_inited_)) { @@ -1075,9 +1498,17 @@ int ObDirectLoadSliceWriter::prepare_slice_store_if_need( } else if (OB_ISNULL(multi_slice_store = OB_NEWx(ObMultiSliceStore, &allocator_))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate memory for multi slice store failed", K(ret)); - } else if (OB_FAIL(multi_slice_store->init(allocator_, tablet_direct_load_mgr_, start_seq_, start_scn, - schema_rowkey_column_num + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(), - storage_schema, tablet_direct_load_mgr_->get_column_info(), dir_id, parallelism))) { + } else if (OB_FAIL(multi_slice_store->init(allocator_, + tablet_direct_load_mgr_, + start_seq_, + start_scn, + schema_rowkey_column_num + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(), + storage_schema, + tablet_direct_load_mgr_->get_column_info(), + dir_id, + parallelism, + use_batch_store, + max_batch_size))) { LOG_WARN("init multi slice store failed", K(ret), KPC_(tablet_direct_load_mgr), KPC(storage_schema)); } else { slice_store_ = multi_slice_store; @@ -1088,22 +1519,38 @@ int ObDirectLoadSliceWriter::prepare_slice_store_if_need( } } else if (is_full_direct_load(tablet_direct_load_mgr_->get_direct_load_type()) && is_column_store) { writer_type_ = ObDirectLoadSliceWriterType::COL_STORE_WRITER; - ObChunkSliceStore *chunk_slice_store = nullptr; if (OB_ISNULL(storage_schema)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("null schema", K(ret), K(*this)); - } else if (OB_ISNULL(chunk_slice_store = OB_NEWx(ObChunkSliceStore, &allocator_))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("allocate memory for chunk slice store failed", K(ret)); - } else if (OB_FAIL(chunk_slice_store->init(schema_rowkey_column_num + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(), - storage_schema, allocator_, tablet_direct_load_mgr_->get_column_info(), dir_id, parallelism))) { - LOG_WARN("init chunk slice store failed", K(ret), KPC(storage_schema)); } else { - slice_store_ = chunk_slice_store; - } - if (OB_FAIL(ret) && nullptr != chunk_slice_store) { - chunk_slice_store->~ObChunkSliceStore(); - allocator_.free(chunk_slice_store); + if (use_batch_store) { + ObChunkBatchSliceStore *chunk_slice_store = nullptr; + if (OB_ISNULL(slice_store_ = chunk_slice_store = OB_NEWx(ObChunkBatchSliceStore, &allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObChunkBatchSliceStore", KR(ret)); + } else if (OB_FAIL(chunk_slice_store->init(schema_rowkey_column_num + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(), + storage_schema, + allocator_, + tablet_direct_load_mgr_->get_column_info(), + dir_id, + parallelism, + max_batch_size))) { + LOG_WARN("fail to init chunk batch slice store", K(ret), KPC(storage_schema)); + } + } else { + ObChunkSliceStore *chunk_slice_store = nullptr; + if (OB_ISNULL(slice_store_ = chunk_slice_store = OB_NEWx(ObChunkSliceStore, &allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObChunkSliceStore", KR(ret)); + } else if (OB_FAIL(chunk_slice_store->init(schema_rowkey_column_num + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(), + storage_schema, + allocator_, + tablet_direct_load_mgr_->get_column_info(), + dir_id, + parallelism))) { + LOG_WARN("fail to init chunk slice store", K(ret), KPC(storage_schema)); + } + } } } else { writer_type_ = ObDirectLoadSliceWriterType::ROW_STORE_WRITER; @@ -1165,14 +1612,24 @@ int ObDirectLoadSliceWriter::prepare_iters( int ret = OB_SUCCESS; row_iter = nullptr; - if (OB_ISNULL(meta_write_iter_)) { + if (OB_ISNULL(lob_allocator_)) { + void *buf = nullptr; + if (OB_ISNULL(buf = allocator_.alloc(sizeof(common::ObArenaAllocator)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc lob allocator failed", K(ret)); + } else { + lob_allocator_ = new (buf) common::ObArenaAllocator("LobWriter", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); + } + } + + if (OB_SUCC(ret) && OB_ISNULL(meta_write_iter_)) { void *buf = nullptr; if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObLobMetaWriteIter)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc lob meta write iter failed", K(ret)); } else { // keep allocator is same as insert_lob_column - meta_write_iter_ = new (buf) ObLobMetaWriteIter(&allocator, ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE); + meta_write_iter_ = new (buf) ObLobMetaWriteIter(lob_allocator_, ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE); } } @@ -1197,7 +1654,7 @@ int ObDirectLoadSliceWriter::prepare_iters( LOG_WARN("tx_desc should not be null if is incremental_direct_load", K(ret), K(direct_load_type), K(ls_id), K(tablet_id), K(trans_version), K(seq_no), K(obj_type), K(cs_type), K(trans_id)); } else if (OB_FAIL(ObInsertLobColumnHelper::insert_lob_column( - allocator, tx_desc, pk_interval, ls_id, tablet_id/* tablet_id of main table */, tablet_direct_load_mgr_->get_tablet_id()/*tablet id of lob meta table*/, + allocator, *lob_allocator_, tx_desc, pk_interval, ls_id, tablet_id/* tablet_id of main table */, tablet_direct_load_mgr_->get_tablet_id()/*tablet id of lob meta table*/, obj_type, cs_type, lob_storage_param, datum, timeout_ts, true/*has_lob_header*/, src_tenant_id, *meta_write_iter_))) { LOG_WARN("fail to insert_lob_col", K(ret), K(ls_id), K(tablet_id), K(src_tenant_id)); } else if (OB_FAIL(row_iterator_->init(meta_write_iter_, trans_id, @@ -1227,13 +1684,22 @@ int ObDirectLoadSliceWriter::fill_lob_sstable_slice( if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObDirectLoadSliceWriter not init", KR(ret), KP(this)); - } else if (DATA_VERSION_4_3_0_0 > data_format_version) { - if (OB_FAIL(fill_lob_into_memtable(allocator, info, lob_column_idxs, col_types, lob_inrow_threshold, datum_row))) { - LOG_WARN("fill lob into memtable failed", K(ret), K(data_format_version)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < lob_column_idxs.count(); i++) { + const int64_t idx = lob_column_idxs.at(i); + const ObObjMeta &col_type = col_types.at(i); + ObStorageDatum &datum = datum_row.storage_datums_[idx]; + if (DATA_VERSION_4_3_0_0 > data_format_version) { + if (OB_FAIL(fill_lob_into_memtable(allocator, info, col_type, lob_inrow_threshold, datum))) { + LOG_WARN("fill lob into memtable failed", K(ret), K(data_format_version)); + } + } else { + if (OB_FAIL(fill_lob_into_macro_block(allocator, iter_allocator, start_scn, info, + pk_interval, col_type, lob_inrow_threshold, datum))) { + LOG_WARN("fill lob into macro block failed", K(ret), K(data_format_version)); + } + } } - } else if (OB_FAIL(fill_lob_into_macro_block(allocator, iter_allocator, start_scn, info, - pk_interval, lob_column_idxs, col_types, lob_inrow_threshold, datum_row))) { - LOG_WARN("fill lob into macro block failed", K(ret), K(data_format_version)); } return ret; } @@ -1241,25 +1707,19 @@ int ObDirectLoadSliceWriter::fill_lob_sstable_slice( int ObDirectLoadSliceWriter::fill_lob_into_memtable( ObIAllocator &allocator, const ObBatchSliceWriteInfo &info, - const ObArray &lob_column_idxs, - const ObArray &col_types, + const common::ObObjMeta &col_type, const int64_t lob_inrow_threshold, - blocksstable::ObDatumRow &datum_row) + blocksstable::ObStorageDatum &datum) { // to insert lob data into memtable. int ret = OB_SUCCESS; - const int64_t timeout_ts = - ObTimeUtility::fast_current_time() + (ObInsertLobColumnHelper::LOB_ACCESS_TX_TIMEOUT * lob_column_idxs.count()); - for (int64_t i = 0; OB_SUCC(ret) && i < lob_column_idxs.count(); i++) { - const int64_t idx = lob_column_idxs.at(i); - ObStorageDatum &datum = datum_row.storage_datums_[idx]; - ObLobStorageParam lob_storage_param; - lob_storage_param.inrow_threshold_ = lob_inrow_threshold; - if (OB_FAIL(ObInsertLobColumnHelper::insert_lob_column( - allocator, info.ls_id_, info.data_tablet_id_, col_types.at(i).get_type(), col_types.at(i).get_collation_type(), - lob_storage_param, datum, timeout_ts, true/*has_lob_header*/, info.src_tenant_id_))) { - LOG_WARN("fail to insert_lob_col", K(ret), K(datum)); - } + const int64_t timeout_ts = ObTimeUtility::fast_current_time() + ObInsertLobColumnHelper::LOB_ACCESS_TX_TIMEOUT; + ObLobStorageParam lob_storage_param; + lob_storage_param.inrow_threshold_ = lob_inrow_threshold; + if (OB_FAIL(ObInsertLobColumnHelper::insert_lob_column( + allocator, info.ls_id_, info.data_tablet_id_, col_type.get_type(), col_type.get_collation_type(), + lob_storage_param, datum, timeout_ts, true/*has_lob_header*/, info.src_tenant_id_))) { + LOG_WARN("fail to insert_lob_col", K(ret), K(datum)); } return ret; } @@ -1270,71 +1730,312 @@ int ObDirectLoadSliceWriter::fill_lob_into_macro_block( const SCN &start_scn, const ObBatchSliceWriteInfo &info, share::ObTabletCacheInterval &pk_interval, - const ObArray &lob_column_idxs, - const ObArray &col_types, + const common::ObObjMeta &col_type, const int64_t lob_inrow_threshold, - blocksstable::ObDatumRow &datum_row) + blocksstable::ObStorageDatum &datum) { // to insert lob data into macro block. int ret = OB_SUCCESS; int64_t unused_affected_rows = 0; - const int64_t timeout_ts = - ObTimeUtility::fast_current_time() + (ObInsertLobColumnHelper::LOB_ACCESS_TX_TIMEOUT * lob_column_idxs.count()); - for (int64_t i = 0; OB_SUCC(ret) && i < lob_column_idxs.count(); i++) { - int64_t idx = lob_column_idxs.at(i); - ObStorageDatum &datum = datum_row.storage_datums_[idx]; - if (!datum.is_nop() && !datum.is_null()) { - { - ObLobMetaRowIterator *row_iter = nullptr; - if (OB_FAIL(prepare_iters(allocator, iter_allocator, datum, info.ls_id_, - info.data_tablet_id_, info.trans_version_, col_types.at(i).get_type(), col_types.at(i).get_collation_type(), - info.trans_id_, info.seq_no_, timeout_ts, lob_inrow_threshold, info.src_tenant_id_, info.direct_load_type_, - info.tx_desc_, pk_interval, row_iter))) { - LOG_WARN("fail to prepare iters", K(ret), KP(row_iter), K(datum)); - } else { - while (OB_SUCC(ret)) { - const blocksstable::ObDatumRow *cur_row = nullptr; - if (OB_FAIL(THIS_WORKER.check_status())) { - LOG_WARN("check status failed", K(ret)); - } else if (ATOMIC_LOAD(&is_canceled_)) { - ret = OB_CANCELED; - LOG_WARN("fil lob task canceled", K(ret), K(is_canceled_)); - } else if (OB_FAIL(row_iter->get_next_row(cur_row))) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - break; - } else { - LOG_WARN("get next row failed", K(ret)); - } - } else if (OB_ISNULL(cur_row) || !cur_row->is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), KPC(cur_row)); - } else if (OB_FAIL(check_null(false/*is_index_table*/, ObLobMetaUtil::LOB_META_SCHEMA_ROWKEY_COL_CNT, *cur_row))) { - LOG_WARN("fail to check null value in row", KR(ret), KPC(cur_row)); - } else if (OB_FAIL(prepare_slice_store_if_need(ObLobMetaUtil::LOB_META_SCHEMA_ROWKEY_COL_CNT, - false/*is_column_store*/, 1L/*unsued*/, 1L/*unused*/, nullptr /*storage_schema*/, start_scn, - ObString()/*unsued*/, 0/*unsued*/))) { - LOG_WARN("prepare macro block writer failed", K(ret)); - } else if (OB_FAIL(slice_store_->append_row(*cur_row))) { - LOG_WARN("macro block writer append row failed", K(ret), KPC(cur_row)); + const int64_t timeout_ts = ObTimeUtility::fast_current_time() + ObInsertLobColumnHelper::LOB_ACCESS_TX_TIMEOUT; + if (!datum.is_nop() && !datum.is_null()) { + { + ObLobMetaRowIterator *row_iter = nullptr; + if (OB_FAIL(prepare_iters(allocator, iter_allocator, datum, info.ls_id_, + info.data_tablet_id_, info.trans_version_, col_type.get_type(), col_type.get_collation_type(), + info.trans_id_, info.seq_no_, timeout_ts, lob_inrow_threshold, info.src_tenant_id_, info.direct_load_type_, + info.tx_desc_, pk_interval, row_iter))) { + LOG_WARN("fail to prepare iters", K(ret), KP(row_iter), K(datum)); + } else { + while (OB_SUCC(ret)) { + const blocksstable::ObDatumRow *cur_row = nullptr; + if (OB_FAIL(THIS_WORKER.check_status())) { + LOG_WARN("check status failed", K(ret)); + } else if (ATOMIC_LOAD(&is_canceled_)) { + ret = OB_CANCELED; + LOG_WARN("fil lob task canceled", K(ret), K(is_canceled_)); + } else if (OB_FAIL(row_iter->get_next_row(cur_row))) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; + } else { + LOG_WARN("get next row failed", K(ret)); } - if (OB_SUCC(ret)) { - ++unused_affected_rows; - LOG_DEBUG("sstable insert op append row", K(unused_affected_rows), KPC(cur_row)); - } - } - if (OB_SUCC(ret) && OB_NOT_NULL(meta_write_iter_) && OB_FAIL(meta_write_iter_->check_write_length())) { - LOG_WARN("check_write_length fail", K(ret), KPC(meta_write_iter_)); + } else if (OB_ISNULL(cur_row) || !cur_row->is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KPC(cur_row)); + } else if (OB_FAIL(check_null(false/*is_index_table*/, ObLobMetaUtil::LOB_META_SCHEMA_ROWKEY_COL_CNT, *cur_row))) { + LOG_WARN("fail to check null value in row", KR(ret), KPC(cur_row)); + } else if (OB_FAIL(prepare_slice_store_if_need(ObLobMetaUtil::LOB_META_SCHEMA_ROWKEY_COL_CNT, + false/*is_column_store*/, 1L/*unsued*/, 1L/*unused*/, nullptr /*storage_schema*/, start_scn, + ObString()/*unsued*/, 0/*unsued*/))) { + LOG_WARN("prepare macro block writer failed", K(ret)); + } else if (OB_FAIL(slice_store_->append_row(*cur_row))) { + LOG_WARN("macro block writer append row failed", K(ret), KPC(cur_row)); } if (OB_SUCC(ret)) { - if (OB_NOT_NULL(meta_write_iter_)) { - meta_write_iter_->reuse(); - } - if (OB_NOT_NULL(row_iterator_)) { - row_iterator_->reuse(); - } + ++unused_affected_rows; + LOG_DEBUG("sstable insert op append row", K(unused_affected_rows), KPC(cur_row)); } } + if (OB_SUCC(ret) && OB_NOT_NULL(meta_write_iter_) && OB_FAIL(meta_write_iter_->check_write_length())) { + LOG_WARN("check_write_length fail", K(ret), KPC(meta_write_iter_)); + } + if (OB_SUCC(ret)) { + if (OB_NOT_NULL(meta_write_iter_)) { + meta_write_iter_->reuse(); + } + if (OB_NOT_NULL(row_iterator_)) { + row_iterator_->reuse(); + } + if (OB_NOT_NULL(lob_allocator_)) { + lob_allocator_->reuse(); + } + } + } + } + } + return ret; +} + +static bool fast_check_vector_is_all_null(ObIVector *vector, const int64_t batch_size) +{ + bool is_all_null = false; + VectorFormat format = vector->get_format(); + switch (format) { + case VEC_FIXED: + case VEC_DISCRETE: + case VEC_CONTINUOUS: + is_all_null = static_cast(vector)->get_nulls()->is_all_true(batch_size); + break; + default: + break; + } + return is_all_null; +} + +static int new_discrete_vector(VecValueTypeClass value_tc, + const int64_t max_batch_size, + ObIAllocator &allocator, + ObDiscreteBase *&result_vec) +{ + int ret = OB_SUCCESS; + result_vec = nullptr; + ObIVector *vector = nullptr; + switch (value_tc) { +#define DISCRETE_VECTOR_INIT_SWITCH(value_tc) \ + case value_tc: { \ + using VecType = RTVectorType; \ + static_assert(sizeof(VecType) <= ObIVector::MAX_VECTOR_STRUCT_SIZE, \ + "vector size exceeds MAX_VECTOR_STRUCT_SIZE"); \ + vector = OB_NEWx(VecType, &allocator, nullptr, nullptr, nullptr); \ + break; \ + } + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_NUMBER); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_EXTEND); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_STRING); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_ENUM_SET_INNER); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_RAW); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_ROWID); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_LOB); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_JSON); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_GEO); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_UDT); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_COLLECTION); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_ROARINGBITMAP); +#undef DISCRETE_VECTOR_INIT_SWITCH + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected discrete vector value type class", KR(ret), K(value_tc)); + break; + } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(vector)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc vecttor", KR(ret)); + } else { + ObDiscreteBase *discrete_vec = static_cast(vector); + const int64_t nulls_size = ObBitVector::memory_size(max_batch_size); + const int64_t lens_size = sizeof(int32_t) * max_batch_size; + const int64_t ptrs_size = sizeof(char *) * max_batch_size; + ObBitVector *nulls = nullptr; + int32_t *lens = nullptr; + char **ptrs = nullptr; + if (OB_ISNULL(nulls = to_bit_vector(allocator.alloc(nulls_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", KR(ret), K(nulls_size)); + } else if (OB_ISNULL(lens = static_cast(allocator.alloc(lens_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", KR(ret), K(lens_size)); + } else if (OB_ISNULL(ptrs = static_cast(allocator.alloc(ptrs_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", KR(ret), K(ptrs_size)); + } else { + nulls->reset(max_batch_size); + discrete_vec->set_nulls(nulls); + discrete_vec->set_lens(lens); + discrete_vec->set_ptrs(ptrs); + result_vec = discrete_vec; + } + } + return ret; +} + +int ObDirectLoadSliceWriter::fill_lob_sstable_slice( + const uint64_t table_id, + ObIAllocator &allocator, + ObIAllocator &iter_allocator, + const SCN &start_scn, + const ObBatchSliceWriteInfo &info, + share::ObTabletCacheInterval &pk_interval, + const ObArray &lob_column_idxs, + const ObArray &col_types, + const int64_t lob_inrow_threshold, + blocksstable::ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + const uint64_t data_format_version = tablet_direct_load_mgr_->get_data_format_version(); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadSliceWriter not init", KR(ret), KP(this)); + } else { + ObStorageDatum temp_datum; + for (int64_t i = 0; OB_SUCC(ret) && i < lob_column_idxs.count(); i++) { + const int64_t idx = lob_column_idxs.at(i); + const ObObjMeta &col_type = col_types.at(i); + ObIVector *vector = datum_rows.vectors_.at(idx); + const VectorFormat format = vector->get_format(); + if (fast_check_vector_is_all_null(vector, datum_rows.row_count_)) { + // do nothing + continue; + } + switch (format) { + case VEC_CONTINUOUS: + { + ObContinuousBase *continuous_vec = static_cast(vector); + ObDiscreteBase *discrete_vec = nullptr; + char *data = continuous_vec->get_data(); + uint32_t *offsets = continuous_vec->get_offsets(); + char **ptrs = nullptr; + ObLength *lens = nullptr; + VecValueTypeClass value_tc = get_vec_value_tc(col_type.get_type(), + col_type.get_scale(), + col_type.get_stored_precision()); + if (OB_FAIL(new_discrete_vector(value_tc, datum_rows.row_count_, allocator, discrete_vec))) { + LOG_WARN("fail to new discrete vector", KR(ret)); + } else { + ptrs = discrete_vec->get_ptrs(); + lens = discrete_vec->get_lens(); + } + for (int64_t j = 0; OB_SUCC(ret) && j < datum_rows.row_count_; ++j) { + if (continuous_vec->is_null(j)) { + discrete_vec->set_null(j); + } else { + temp_datum.ptr_ = data + offsets[j]; + temp_datum.len_ = offsets[j + 1] - offsets[j]; + if (DATA_VERSION_4_3_0_0 > data_format_version) { + if (OB_FAIL(fill_lob_into_memtable(allocator, info, col_type, lob_inrow_threshold, temp_datum))) { + LOG_WARN("fill lob into memtable failed", K(ret), K(data_format_version)); + } + } else { + if (OB_FAIL(fill_lob_into_macro_block(allocator, iter_allocator, start_scn, info, + pk_interval, col_type, lob_inrow_threshold, temp_datum))) { + LOG_WARN("fill lob into macro block failed", K(ret), K(data_format_version)); + } + } + if (OB_SUCC(ret)) { + ptrs[j] = const_cast(temp_datum.ptr_); + lens[j] = temp_datum.len_; + } + } + } + if (OB_SUCC(ret)) { + datum_rows.vectors_.at(idx) = discrete_vec; + } + break; + } + case VEC_DISCRETE: + { + ObDiscreteBase *discrete_vec = static_cast(vector); + char **ptrs = discrete_vec->get_ptrs(); + ObLength *lens =discrete_vec->get_lens(); + for (int64_t j = 0; OB_SUCC(ret) && j < datum_rows.row_count_; ++j) { + if (!discrete_vec->is_null(j)) { + temp_datum.ptr_ = ptrs[j]; + temp_datum.len_ = lens[j]; + if (DATA_VERSION_4_3_0_0 > data_format_version) { + if (OB_FAIL(fill_lob_into_memtable(allocator, info, col_type, lob_inrow_threshold, temp_datum))) { + LOG_WARN("fill lob into memtable failed", K(ret), K(data_format_version)); + } + } else { + if (OB_FAIL(fill_lob_into_macro_block(allocator, iter_allocator, start_scn, info, + pk_interval, col_type, lob_inrow_threshold, temp_datum))) { + LOG_WARN("fill lob into macro block failed", K(ret), K(data_format_version)); + } + } + if (OB_SUCC(ret)) { + ptrs[j] = const_cast(temp_datum.ptr_); + lens[j] = temp_datum.len_; + } + } + } + break; + } + case VEC_UNIFORM: + { + ObUniformBase *uniform_vec = static_cast(vector); + ObDatum *datums = uniform_vec->get_datums(); + for (int64_t j = 0; OB_SUCC(ret) && j < datum_rows.row_count_; ++j) { + ObDatum &datum = datums[j]; + if (!datum.is_null()) { + temp_datum.ptr_ = datum.ptr_; + temp_datum.len_ = datum.len_; + if (DATA_VERSION_4_3_0_0 > data_format_version) { + if (OB_FAIL(fill_lob_into_memtable(allocator, info, col_type, lob_inrow_threshold, temp_datum))) { + LOG_WARN("fill lob into memtable failed", K(ret), K(data_format_version)); + } + } else { + if (OB_FAIL(fill_lob_into_macro_block(allocator, iter_allocator, start_scn, info, + pk_interval, col_type, lob_inrow_threshold, temp_datum))) { + LOG_WARN("fill lob into macro block failed", K(ret), K(data_format_version)); + } + } + if (OB_SUCC(ret)) { + datum.ptr_ = temp_datum.ptr_; + datum.len_ = temp_datum.len_; + } + } + } + break; + } + case VEC_UNIFORM_CONST: + { + ObUniformBase *uniform_vec = static_cast(vector); + ObDatum &datum = uniform_vec->get_datums()[0]; + if (!datum.is_null()) { + temp_datum.ptr_ = datum.ptr_; + temp_datum.len_ = datum.len_; + if (DATA_VERSION_4_3_0_0 > data_format_version) { + if (OB_FAIL(fill_lob_into_memtable(allocator, info, col_type, lob_inrow_threshold, temp_datum))) { + LOG_WARN("fill lob into memtable failed", K(ret), K(data_format_version)); + } + } else { + if (OB_FAIL(fill_lob_into_macro_block(allocator, iter_allocator, start_scn, info, + pk_interval, col_type, lob_inrow_threshold, temp_datum))) { + LOG_WARN("fill lob into macro block failed", K(ret), K(data_format_version)); + } + } + if (OB_SUCC(ret)) { + datum.ptr_ = temp_datum.ptr_; + datum.len_ = temp_datum.len_; + } + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected lob vector format", KR(ret), K(i), K(format)); + break; } } } @@ -1481,12 +2182,12 @@ int ObDirectLoadSliceWriter::fill_sstable_slice( ret = OB_ERR_DUPLICATED_UNIQUE_KEY; } } else { - LOG_WARN("macro block writer append row failed", K(ret), KPC(cur_row), KPC(cur_row)); + LOG_WARN("macro block writer append row failed", K(ret), KPC(cur_row)); } } if (OB_SUCC(ret)) { ++affected_rows; - LOG_DEBUG("sstable insert op append row", KPC(cur_row), KPC(cur_row)); + LOG_DEBUG("sstable insert op append row", KPC(cur_row)); if ((affected_rows % 100 == 0) && OB_NOT_NULL(insert_monitor)) { (void) ATOMIC_AAF(&insert_monitor->scanned_row_cnt_, 100); (void) ATOMIC_AAF(&insert_monitor->inserted_row_cnt_, 100); @@ -1501,6 +2202,92 @@ int ObDirectLoadSliceWriter::fill_sstable_slice( return ret; } +int ObDirectLoadSliceWriter::fill_sstable_slice( + const SCN &start_scn, + const uint64_t table_id, + const ObTabletID &tablet_id, + const ObStorageSchema *storage_schema, + const blocksstable::ObBatchDatumRows &datum_rows, + const ObTableSchemaItem &schema_item, + const ObDirectLoadType &direct_load_type, + const ObArray &column_items, + const int64_t dir_id, + const int64_t parallelism, + ObInsertMonitor *insert_monitor) +{ + int ret = OB_SUCCESS; + const bool is_full_direct_load_task = is_full_direct_load(direct_load_type); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadSliceWriter not init", KR(ret), KP(this)); + } else if (OB_ISNULL(storage_schema)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("null schema", K(ret), K(*this)); + } else if (OB_UNLIKELY(ATOMIC_LOAD(&is_canceled_))) { + ret = OB_CANCELED; + LOG_WARN("fil sstable task canceled", K(ret), K(is_canceled_)); + } else { + ObArenaAllocator arena("SliceW_sst", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); + const ObDataStoreDesc &data_desc = tablet_direct_load_mgr_->get_sqc_build_ctx().data_block_desc_.get_desc(); + const int64_t max_batch_size = tablet_direct_load_mgr_->get_sqc_build_ctx().build_param_.runtime_only_param_.max_batch_size_; + if (OB_UNLIKELY(datum_rows.get_column_count() != data_desc.get_col_desc_array().count())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(datum_rows.get_column_count()), K(data_desc.get_col_desc_array())); + } else { // row reshape + for (int64_t i = 0; OB_SUCC(ret) && i < datum_rows.get_column_count(); ++i) { + const ObColDesc &col_desc = data_desc.get_col_desc_array().at(i); + ObIVector *vector = datum_rows.vectors_.at(i); + if (i >= schema_item.rowkey_column_num_ && i < schema_item.rowkey_column_num_ + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt()) { + // skip multi version column + } else if (OB_UNLIKELY(i >= column_items.count()) || OB_UNLIKELY(!column_items.at(i).is_valid_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column schema is wrong", K(ret), K(i), K(column_items)); + } else if (OB_FAIL(ObDASUtils::reshape_vector_value(column_items.at(i).col_type_, + column_items.at(i).col_accuracy_, + arena, + vector, + datum_rows.row_count_))) { + LOG_WARN("fail to reshape vector value", K(ret)); + } + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(check_null(schema_item.is_index_table_, schema_item.rowkey_column_num_, datum_rows))) { + LOG_WARN("fail to check null value in row", KR(ret)); + } else if (OB_FAIL(prepare_slice_store_if_need(schema_item.rowkey_column_num_, + schema_item.is_column_store_, + dir_id, + parallelism, + storage_schema, + start_scn, + schema_item.vec_idx_param_, + schema_item.vec_dim_, + true/*use_batch_store*/, + max_batch_size))) { + LOG_WARN("prepare macro block writer failed", K(ret)); + } else if (OB_FAIL(slice_store_->append_batch(datum_rows))) { + if (is_full_direct_load_task && OB_ERR_PRIMARY_KEY_DUPLICATE == ret && schema_item.is_unique_index_) { + int report_ret_code = OB_SUCCESS; + LOG_USER_ERROR(OB_ERR_PRIMARY_KEY_DUPLICATE, "", static_cast(sizeof("UNIQUE IDX") - 1), "UNIQUE IDX"); + (void) report_unique_key_dumplicated(ret, table_id, datum_rows, tablet_direct_load_mgr_->get_tablet_id(), report_ret_code); // ignore ret + if (OB_ERR_DUPLICATED_UNIQUE_KEY == report_ret_code) { + //error message of OB_ERR_PRIMARY_KEY_DUPLICATE is not compatiable with oracle, so use a new error code + ret = OB_ERR_DUPLICATED_UNIQUE_KEY; + } + } else { + LOG_WARN("macro block writer append batch failed", K(ret)); + } + } else { + LOG_DEBUG("sstable insert op append batch", K(datum_rows.row_count_)); + if (OB_NOT_NULL(insert_monitor)) { + (void) ATOMIC_AAF(&insert_monitor->scanned_row_cnt_, datum_rows.row_count_); + (void) ATOMIC_AAF(&insert_monitor->inserted_row_cnt_, datum_rows.row_count_); + } + } + } + return ret; +} + int ObDirectLoadSliceWriter::report_unique_key_dumplicated( const int ret_code, const uint64_t table_id, const ObDatumRow &datum_row, const ObTabletID &tablet_id, int &report_ret_code) @@ -1537,6 +2324,118 @@ int ObDirectLoadSliceWriter::report_unique_key_dumplicated( return ret; } +int ObDirectLoadSliceWriter::report_unique_key_dumplicated( + const int ret_code, + const uint64_t table_id, + const ObBatchDatumRows &datum_rows, + const ObTabletID &tablet_id, + int &report_ret_code) +{ + int ret = OB_SUCCESS; + report_ret_code = OB_SUCCESS; + ObSchemaGetterGuard schema_guard; + const ObTableSchema *table_schema = nullptr; + if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard( + MTL_ID(), schema_guard))) { + LOG_WARN("get tenant schema failed", K(ret), K(table_id), K(MTL_ID()), K(table_id)); + } else if (OB_FAIL(schema_guard.get_table_schema(MTL_ID(), + table_id, table_schema))) { + LOG_WARN("get table schema failed", K(ret), K(MTL_ID()), K(table_id)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("table not exist", K(ret), K(MTL_ID()), K(table_id)); + } else { + const int64_t rowkey_column_num = table_schema->get_rowkey_column_num(); + ObMemAttr mem_attr(MTL_ID(), "DL_Temp"); + ObArenaAllocator allocator(mem_attr); + ObArray col_descs; + blocksstable::ObStorageDatumUtils datum_utils; + char index_key_buffer[OB_TMP_BUF_SIZE_256] = { 0 }; + int64_t task_id = 0; + ObDatumRowkey key1, key2; + ObDatumRowkey *index_key = nullptr, *prev_key = nullptr, *next_key = nullptr; + ObDDLErrorMessageTableOperator::ObDDLErrorInfo error_info; + col_descs.set_block_allocator(ModulePageAllocator(allocator)); + + if (OB_FAIL(table_schema->get_rowkey_column_ids(col_descs))) { + LOG_WARN("fail to get rowkey column ids", KR(ret)); + } else if (OB_FAIL(datum_utils.init(col_descs, + rowkey_column_num, + lib::is_oracle_mode(), + allocator, + true/*no need compare multiple version cols*/))) { + LOG_WARN("fail to init datum utils", KR(ret), K(col_descs), K(rowkey_column_num)); + } + + // init keys for compare + if (OB_SUCC(ret)) { + const int64_t datum_cnt = rowkey_column_num * 2; + ObStorageDatum *datums = nullptr; + if (OB_ISNULL(datums = static_cast(allocator.alloc(sizeof(ObStorageDatum) * datum_cnt)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc datums", KR(ret), K(datum_cnt)); + } else { + key1.assign(datums, rowkey_column_num); + key2.assign(datums + rowkey_column_num, rowkey_column_num); + prev_key = &key1; + next_key = &key2; + } + } + // find dumplicated key + if (OB_SUCC(ret)) { + bool is_null = false; + const char *payload = nullptr; + ObLength length = 0; + ObStorageDatum tmp_datum; + int cmp_ret = 0; + bool find_dumplicated_key = false; + // init prev key + for (int64_t j = 1; OB_SUCC(ret) && j < rowkey_column_num; ++j) { + ObIVector *vector = datum_rows.vectors_.at(j); + vector->get_payload(0, is_null, payload, length); + prev_key->datums_[j].shallow_copy_from_datum(ObDatum(payload, length, is_null)); + } + for (int64_t i = 1; OB_SUCC(ret) && !find_dumplicated_key && i < datum_rows.row_count_; ++i) { + // set next key + for (int64_t j = 1; OB_SUCC(ret) && j < rowkey_column_num; ++j) { + ObIVector *vector = datum_rows.vectors_.at(j); + vector->get_payload(i, is_null, payload, length); + next_key->datums_[j].shallow_copy_from_datum(ObDatum(payload, length, is_null)); + } + // compare key + if (OB_FAIL(next_key->compare(*prev_key, datum_utils, cmp_ret))) { + LOG_WARN("fail to compare rowkey", KR(ret), KPC(prev_key), KPC(next_key)); + } else if (0 == cmp_ret) { + find_dumplicated_key = true; + } else { + std::swap(prev_key, next_key); + } + } + if (OB_SUCC(ret)) { + index_key = prev_key; + if (!find_dumplicated_key) { + // first key is dumplicated key + for (int64_t j = 1; OB_SUCC(ret) && j < rowkey_column_num; ++j) { + ObIVector *vector = datum_rows.vectors_.at(j); + vector->get_payload(0, is_null, payload, length); + index_key->datums_[j].shallow_copy_from_datum(ObDatum(payload, length, is_null)); + } + } + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ObDDLErrorMessageTableOperator::extract_index_key(*table_schema, *index_key, index_key_buffer, OB_TMP_BUF_SIZE_256))) { // read the unique key that violates the unique constraint + LOG_WARN("extract unique index key failed", K(ret), KPC(index_key), K(index_key_buffer)); + } else if (OB_FAIL(ObDDLErrorMessageTableOperator::get_index_task_info(*GCTX.sql_proxy_, *table_schema, error_info))) { + LOG_WARN("get task id of index table failed", K(ret), K(task_id), K(table_schema)); + } else if (OB_FAIL(ObDDLErrorMessageTableOperator::generate_index_ddl_error_message(ret_code, *table_schema, ObCurTraceId::get_trace_id_str(), + error_info.task_id_, error_info.parent_task_id_, tablet_id.id(), GCTX.self_addr(), *GCTX.sql_proxy_, index_key_buffer, report_ret_code))) { + LOG_WARN("generate index ddl error message", K(ret), K(ret), K(report_ret_code)); + } + } + return ret; +} + int ObDirectLoadSliceWriter::check_null( const bool is_index_table, const int64_t rowkey_column_num, @@ -1560,50 +2459,48 @@ int ObDirectLoadSliceWriter::check_null( return ret; } -int ObDirectLoadSliceWriter::fill_aggregated_column_group( - const int64_t cg_idx, - ObCOSliceWriter *cur_writer, - ObIArray &datum_stores) +int ObDirectLoadSliceWriter::check_null( + const bool is_index_table, + const int64_t rowkey_column_num, + const ObBatchDatumRows &datum_rows) const +{ + int ret = OB_SUCCESS; + if (is_index_table) { + // index table is index-organized but can have null values in index column + } else if (OB_UNLIKELY(rowkey_column_num > datum_rows.get_column_count())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid rowkey column number", KR(ret), K(rowkey_column_num), K(datum_rows.get_column_count())); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_column_num; ++i) { + ObIVector *vector = datum_rows.vectors_.at(i); + for (int64_t j = 0; OB_SUCC(ret) && j < datum_rows.row_count_; ++j) { + if (OB_UNLIKELY(vector->is_null(j))) { + ret = OB_ER_INVALID_USE_OF_NULL; + LOG_WARN("invalid null cell for row key column", KR(ret), K(i), K(j), KPC(vector)); + } + } + } + } + return ret; +} + +int ObDirectLoadSliceWriter::fill_aggregated_column_group( + const int64_t cg_idx, + ObCOSliceWriter *cur_writer) { int ret = OB_SUCCESS; - datum_stores.reset(); - ObChunkSliceStore *chunk_slice_store = static_cast(slice_store_); if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (nullptr == chunk_slice_store || is_empty()) { + } else if (nullptr == slice_store_ || is_empty()) { // do nothing LOG_INFO("chunk slice store is null or empty", K(ret), - KPC(chunk_slice_store), KPC(tablet_direct_load_mgr_)); + KPC(slice_store_), KPC(tablet_direct_load_mgr_)); } else if (ATOMIC_LOAD(&is_canceled_)) { ret = OB_CANCELED; LOG_WARN("fil cg task canceled", K(ret), K(is_canceled_)); - } else if (cg_idx < 0 || cg_idx > chunk_slice_store->datum_stores_.count()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid cg idx", K(ret), K(cg_idx), K(chunk_slice_store->datum_stores_)); - } else { - sql::ObCompactStore *cur_datum_store = chunk_slice_store->datum_stores_.at(cg_idx); - const ObChunkDatumStore::StoredRow *stored_row = nullptr; - bool has_next = false; - while (OB_SUCC(ret) && OB_SUCC(cur_datum_store->has_next(has_next)) && has_next) { - if (OB_FAIL(cur_datum_store->get_next_row(stored_row))) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - break; - } else { - LOG_WARN("get next row failed", K(ret)); - } - } else { - if (OB_FAIL(cur_writer->append_row(stored_row))) { - LOG_WARN("append row failed", K(ret), KPC(stored_row)); - } - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(datum_stores.push_back(cur_datum_store))) { - LOG_WARN("fail to push datum store", K(ret)); - } - } + } else if (OB_FAIL(slice_store_->fill_column_group(cg_idx, cur_writer, nullptr/*insert_monitor*/))) { + LOG_WARN("fail to fill column group", KR(ret), KPC(slice_store_), K(cg_idx)); } return ret; } @@ -1699,7 +2596,6 @@ int ObDirectLoadSliceWriter::fill_column_group(const ObStorageSchema *storage_sc { int ret = OB_SUCCESS; const bool need_process_cs_replica = tablet_direct_load_mgr_->need_process_cs_replica(); - const ObChunkSliceStore *chunk_slice_store = nullptr; ObStorageSchema *cs_replica_storage_schema = nullptr; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; @@ -1710,13 +2606,9 @@ int ObDirectLoadSliceWriter::fill_column_group(const ObStorageSchema *storage_sc } else if (OB_UNLIKELY(row_offset_ < 0)) { ret = OB_ERR_SYS; LOG_WARN("row offset not set", K(ret), K(row_offset_)); - } else if (OB_ISNULL(slice_store_) || is_empty() - || OB_ISNULL(chunk_slice_store = need_process_cs_replica - ? static_cast(slice_store_)->get_column_slice_store() - : static_cast(slice_store_))) { + } else if (OB_ISNULL(slice_store_) || is_empty()) { // do nothing - LOG_INFO("slice_store_ is null or empty", K(ret), KPC_(slice_store), - KPC(chunk_slice_store), KPC(tablet_direct_load_mgr_)); + LOG_INFO("slice_store_ is null or empty", K(ret), KPC_(slice_store), KPC(tablet_direct_load_mgr_)); } else if (ATOMIC_LOAD(&is_canceled_)) { ret = OB_CANCELED; LOG_WARN("fil cg task canceled", K(ret), K(is_canceled_)); @@ -1725,7 +2617,7 @@ int ObDirectLoadSliceWriter::fill_column_group(const ObStorageSchema *storage_sc } else if (need_process_cs_replica && OB_FAIL(cs_replica_storage_schema->init(allocator_, *storage_schema, false /*skip_column_info*/, nullptr /*column_group_schema*/, true /*generate_cs_replica_cg_array*/))) { LOG_WARN("failed to init storage schema for cs replica", K(ret), KPC(storage_schema)); - } else if (OB_FAIL(inner_fill_column_group(chunk_slice_store, need_process_cs_replica ? cs_replica_storage_schema : storage_schema, start_scn, insert_monitor))) { + } else if (OB_FAIL(inner_fill_column_group(slice_store_, need_process_cs_replica ? cs_replica_storage_schema : storage_schema, start_scn, insert_monitor))) { LOG_WARN("failed to fill column group", K(ret)); } @@ -1739,7 +2631,7 @@ int ObDirectLoadSliceWriter::fill_column_group(const ObStorageSchema *storage_sc int ObDirectLoadSliceWriter::inner_fill_column_group( - const ObChunkSliceStore *chunk_slice_store, + ObTabletSliceStore *slice_store, const ObStorageSchema *storage_schema, const SCN &start_scn, ObInsertMonitor* insert_monitor) @@ -1749,7 +2641,7 @@ int ObDirectLoadSliceWriter::inner_fill_column_group( const ObIArray &cg_schemas = storage_schema->get_column_groups(); FLOG_INFO("[DDL_FILL_CG] fill column group start", "tablet_id", tablet_direct_load_mgr_->get_tablet_id(), - "row_count", chunk_slice_store->get_row_count(), + "row_count", slice_store->get_row_count(), "column_group_count", cg_schemas.count()); // 1. reserve writers @@ -1763,42 +2655,10 @@ int ObDirectLoadSliceWriter::inner_fill_column_group( cur_writer->reset(); if (OB_FAIL(cur_writer->init(storage_schema, cg_idx, tablet_direct_load_mgr_, start_seq_, row_offset_, start_scn, tablet_direct_load_mgr_->need_process_cs_replica()))) { LOG_WARN("init co ddl writer failed", K(ret), KPC(cur_writer), K(cg_idx), KPC(this)); - } else { - sql::ObCompactStore *cur_datum_store = chunk_slice_store->datum_stores_.at(cg_idx); - const ObChunkDatumStore::StoredRow *stored_row = nullptr; - bool has_next = false; - int64_t cg_row_inserted_cnt = 0; - while (OB_SUCC(ret) && OB_SUCC(cur_datum_store->has_next(has_next)) && has_next) { - if (OB_FAIL(cur_datum_store->get_next_row(stored_row))) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - break; - } else { - LOG_WARN("get next row failed", K(ret)); - } - } else { - if (OB_FAIL(cur_writer->append_row(stored_row))) { - LOG_WARN("append row failed", K(ret), KPC(stored_row)); - } else { - ++cg_row_inserted_cnt; - if ((0 == cg_row_inserted_cnt % 100) && OB_NOT_NULL(insert_monitor)) { - (void) ATOMIC_AAF(&insert_monitor->inserted_cg_row_cnt_, 100); - } - } - } - } - if (OB_SUCC(ret)) { - // 3. close writers - if (OB_FAIL(cur_writer->close())) { - LOG_WARN("close co ddl writer failed", K(ret)); - } else { - // 4. reset datum store (if fail, datum store will free when ~ObDirectLoadSliceWriter()) - cur_datum_store->reset(); - if (OB_NOT_NULL(insert_monitor)) { - (void) ATOMIC_AAF(&insert_monitor->inserted_cg_row_cnt_, cg_row_inserted_cnt % 100); - } - } - } + } else if (OB_FAIL(slice_store->fill_column_group(cg_idx, cur_writer, insert_monitor))) { + LOG_WARN("fail to fill column group", KR(ret), K(cg_idx)); + } else if (OB_FAIL(cur_writer->close())) { + LOG_WARN("close co ddl writer failed", K(ret)); } } } @@ -1808,17 +2668,25 @@ int ObDirectLoadSliceWriter::inner_fill_column_group( } FLOG_INFO("[DDL_FILL_CG] fill column group finished", "tablet_id", tablet_direct_load_mgr_->get_tablet_id(), - "row_count", chunk_slice_store->get_row_count(), + "row_count", slice_store->get_row_count(), "column_group_count", cg_schemas.count()); } return ret; } +void ObDirectLoadSliceWriter::cancel() +{ + ATOMIC_SET(&is_canceled_, true); + if (OB_NOT_NULL(slice_store_)) { + slice_store_->cancel(); + } +} void ObCOSliceWriter::reset() { is_inited_ = false; cg_row_.reset(); + datum_rows_.reset(); macro_block_writer_.reset(); flush_callback_.reset(); index_builder_.reset(); @@ -1917,7 +2785,10 @@ int ObCOSliceWriter::init(const ObStorageSchema *storage_schema, const int64_t c if (OB_SUCC(ret)) { if (OB_FAIL(cg_row_.init(cg_schema.column_cnt_))) { LOG_WARN("init column group row failed", K(ret)); + } else if (OB_FAIL(datum_rows_.vectors_.prepare_allocate(cg_schema.column_cnt_))) { + LOG_WARN("fail to prepare allocate", K(ret)); } else { + datum_rows_.row_flag_.set_flag(ObDmlFlag::DF_INSERT); cg_idx_ = cg_idx; cg_schema_ = &cg_schema; is_inited_ = true; @@ -1942,6 +2813,27 @@ int ObCOSliceWriter::append_row(const sql::ObChunkDatumStore::StoredRow *stored_ return ret; } +int ObCOSliceWriter::append_batch(const ObIArray &vectors, const int64_t batch_size) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_UNLIKELY(vectors.count() != datum_rows_.get_column_count() || batch_size <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(vectors.count()), K(batch_size), K(datum_rows_.get_column_count())); + } else { + for (int64_t i = 0; i < datum_rows_.get_column_count(); ++i) { + datum_rows_.vectors_.at(i) = vectors.at(i); + } + datum_rows_.row_count_ = batch_size; + if (OB_FAIL(macro_block_writer_.append_batch(datum_rows_))) { + LOG_WARN("write column group row failed", K(ret)); + } + } + return ret; +} + int ObCOSliceWriter::project_cg_row(const ObStorageColumnGroupSchema &cg_schema, const ObChunkDatumStore::StoredRow *stored_row, ObDatumRow &cg_row) diff --git a/src/storage/ddl/ob_direct_load_struct.h b/src/storage/ddl/ob_direct_load_struct.h index a9c3e99e41..caf5d0010a 100755 --- a/src/storage/ddl/ob_direct_load_struct.h +++ b/src/storage/ddl/ob_direct_load_struct.h @@ -23,10 +23,12 @@ #include "share/vector_index/ob_vector_index_util.h" #include "share/ob_ddl_common.h" #include "sql/engine/basic/ob_chunk_datum_store.h" +#include "sql/engine/basic/ob_temp_column_store.h" #include "sql/engine/basic/chunk_store/ob_compact_store.h" #include "storage/ob_i_table.h" #include "storage/access/ob_store_row_iterator.h" #include "storage/blocksstable/index_block/ob_index_block_builder.h" +#include "storage/blocksstable/ob_batch_datum_rows.h" #include "storage/blocksstable/ob_macro_block_struct.h" #include "storage/blocksstable/ob_imacro_block_flush_callback.h" #include "storage/ddl/ob_ddl_redo_log_writer.h" @@ -46,6 +48,8 @@ class ObExecContext; namespace storage { class ObTablet; +class ObInsertMonitor; +class ObCOSliceWriter; struct ObBatchSliceWriteInfo final { @@ -263,11 +267,32 @@ struct ObDirectInsertRuntimeOnlyParam final { public: ObDirectInsertRuntimeOnlyParam() - : exec_ctx_(nullptr), task_id_(0), table_id_(OB_INVALID_ID), schema_version_(0), task_cnt_(0), need_online_opt_stat_gather_(false), tx_desc_(nullptr), trans_id_(), seq_no_(0), parallel_(1) - {} + : exec_ctx_(nullptr), + task_id_(0), + table_id_(OB_INVALID_ID), + schema_version_(0), + task_cnt_(0), + need_online_opt_stat_gather_(false), + tx_desc_(nullptr), + trans_id_(), + seq_no_(0), + parallel_(1), + max_batch_size_(0) + { + } ~ObDirectInsertRuntimeOnlyParam() = default; bool is_valid() const { return OB_INVALID_ID != task_id_ && OB_INVALID_ID != table_id_ && schema_version_ > 0 && task_cnt_ >= 0; } - TO_STRING_KV(KP_(exec_ctx), K_(task_id), K_(table_id), K_(schema_version), K_(task_cnt), K_(need_online_opt_stat_gather), KP_(tx_desc), K_(trans_id), K_(seq_no), K_(parallel)); + TO_STRING_KV(KP_(exec_ctx), + K_(task_id), + K_(table_id), + K_(schema_version), + K_(task_cnt), + K_(need_online_opt_stat_gather), + KP_(tx_desc), + K_(trans_id), + K_(seq_no), + K_(parallel), + K_(max_batch_size)); public: sql::ObExecContext *exec_ctx_; int64_t task_id_; @@ -285,6 +310,7 @@ public: // fixed 0 for the full direct load. int64_t seq_no_; // int64_t parallel_; // used to decide wehter need to use compress temp data in rescan task. + int64_t max_batch_size_; }; // full parameters used by runtime execution @@ -489,9 +515,13 @@ public: ObTabletSliceStore() {} virtual ~ObTabletSliceStore() {} virtual int append_row(const blocksstable::ObDatumRow &datum_row) = 0; + virtual int append_batch(const blocksstable::ObBatchDatumRows &datum_rows) = 0; virtual int close() = 0; + virtual int fill_column_group(const int64_t cg_idx, ObCOSliceWriter *writer, ObInsertMonitor *insert_monitor) = 0; + virtual void cancel() = 0; // for exit fill_column_group virtual int64_t get_row_count() const { return 0; } // dummy one virtual int64_t get_next_block_start_seq() const { return -1; } // invalid block start seq. + virtual ObDatumRowkey get_compare_key() const { return ObDatumRowkey(); } DECLARE_PURE_VIRTUAL_TO_STRING; }; @@ -511,7 +541,16 @@ public: const int64_t vec_dim, const ObIArray &col_array); virtual int append_row(const blocksstable::ObDatumRow &datum_row) override; + virtual int append_batch(const blocksstable::ObBatchDatumRows &datum_rows) override + { + return OB_ERR_UNEXPECTED; + } virtual int close() override; + virtual int fill_column_group(const int64_t cg_idx, ObCOSliceWriter *writer, ObInsertMonitor *insert_monitor) override + { + return OB_ERR_UNEXPECTED; + } + virtual void cancel() override {} virtual int64_t get_row_count() const { return row_cnt_; } void reset(); // vector index functions @@ -549,7 +588,7 @@ public: class ObChunkSliceStore : public ObTabletSliceStore { public: - ObChunkSliceStore() : is_inited_(false), target_store_idx_(-1), row_cnt_(0), arena_allocator_(nullptr), cg_schemas_(), datum_stores_(), rowkey_column_count_(0) + ObChunkSliceStore() : is_inited_(false), is_canceled_(false), target_store_idx_(-1), row_cnt_(0), arena_allocator_(nullptr), cg_schemas_(), datum_stores_(), rowkey_column_count_(0) { cg_schemas_.set_attr(ObMemAttr(MTL_ID(), "ChunkSlicStoreC")); datum_stores_.set_attr(ObMemAttr(MTL_ID(), "ChunkSlicStoreD")); @@ -558,16 +597,24 @@ public: int init(const int64_t rowkey_column_count, const ObStorageSchema *storage_schema, ObArenaAllocator &allocator, const ObIArray &col_schema, const int64_t dir_id, const int64_t parallelism); virtual int append_row(const blocksstable::ObDatumRow &datum_row) override; + virtual int append_batch(const blocksstable::ObBatchDatumRows &datum_rows) override + { + return OB_ERR_UNEXPECTED; + } virtual int close() override; + virtual int fill_column_group(const int64_t cg_idx, ObCOSliceWriter *writer, ObInsertMonitor *insert_monitor) override; + virtual void cancel() override { is_canceled_ = true; } void reset(); virtual int64_t get_row_count() const { return row_cnt_; } - TO_STRING_KV(K(is_inited_), K(target_store_idx_), K(row_cnt_), KP(arena_allocator_), K(datum_stores_), K(endkey_), K(rowkey_column_count_), K(cg_schemas_)); + virtual ObDatumRowkey get_compare_key() const override { return endkey_; } + TO_STRING_KV(K(is_inited_), K(is_canceled_), K(target_store_idx_), K(row_cnt_), KP(arena_allocator_), K(datum_stores_), K(endkey_), K(rowkey_column_count_), K(cg_schemas_)); private: int prepare_datum_stores(const uint64_t tenant_id, const ObStorageSchema *storage_schema, ObIAllocator &allocator, const ObIArray &col_array, const int64_t dir_id, const int64_t parallelism); int64_t calc_chunk_limit(const ObStorageColumnGroupSchema &cg_schema); public: bool is_inited_; + bool is_canceled_; int64_t target_store_idx_; int64_t row_cnt_; ObArenaAllocator *arena_allocator_; @@ -577,6 +624,95 @@ public: int64_t rowkey_column_count_; }; +class ObChunkBatchSliceStore : public ObTabletSliceStore +{ +public: + ObChunkBatchSliceStore() + : arena_allocator_(nullptr), + cg_ctxs_(), + column_count_(0), + rowkey_column_count_(0), + row_cnt_(0), + start_key_(), + is_canceled_(false), + is_inited_(false) + { + cg_ctxs_.set_attr(ObMemAttr(MTL_ID(), "DL_CG_CTXS")); + } + virtual ~ObChunkBatchSliceStore() { reset(); } + void reset(); + int init(const int64_t rowkey_column_count, + const ObStorageSchema *storage_schema, + ObArenaAllocator &allocator, + const ObIArray &col_schema, + const int64_t dir_id, + const int64_t parallelism, + const int64_t max_batch_size); + virtual int append_row(const blocksstable::ObDatumRow &datum_row) override + { + return OB_ERR_UNEXPECTED; + } + virtual int append_batch(const blocksstable::ObBatchDatumRows &datum_rows) override; + virtual int close() override; + virtual int fill_column_group(const int64_t cg_idx, ObCOSliceWriter *writer, ObInsertMonitor *insert_monitor) override; + virtual void cancel() override { is_canceled_ = true; } + virtual int64_t get_row_count() const { return row_cnt_; } + virtual ObDatumRowkey get_compare_key() const override { return start_key_; } + TO_STRING_KV(KP_(arena_allocator), + K_(cg_ctxs), + K_(column_count), + K_(rowkey_column_count), + K_(row_cnt), + K_(start_key), + K_(is_canceled), + K_(is_inited)); +private: + int init_start_key(); + int prepare_column_group_ctxs(const uint64_t tenant_id, + const ObStorageSchema *storage_schema, + ObIAllocator &allocator, + const ObIArray &col_array, + const int64_t dir_id, + const int64_t parallelism, + const int64_t max_batch_size); +private: + struct ColumnGroupCtx + { + public: + ColumnGroupCtx() + : allocator_("DL_VECTOR"), + cg_schema_(), + store_(), + vectors_(), + append_vectors_(), + brs_() + { + allocator_.set_tenant_id(MTL_ID()); + } + TO_STRING_KV(K_(cg_schema), + // K_(store), + K(vectors_.count()), + K(append_vectors_.count()), + K_(brs)); + public: + ObArenaAllocator allocator_; + ObStorageColumnGroupSchema cg_schema_; + sql::ObTempColumnStore store_; + ObArray vectors_; + ObArray append_vectors_; + sql::ObBatchRows brs_; + }; +public: + ObArenaAllocator *arena_allocator_; + ObArray cg_ctxs_; + int64_t column_count_; + int64_t rowkey_column_count_; + int64_t row_cnt_; + blocksstable::ObDatumRowkey start_key_; + bool is_canceled_; + bool is_inited_; +}; + class ObMacroBlockSliceStore: public ObTabletSliceStore { public: @@ -593,7 +729,13 @@ public: const share::SCN &start_scn, const bool need_process_cs_replica = false); virtual int append_row(const blocksstable::ObDatumRow &datum_row) override; + virtual int append_batch(const blocksstable::ObBatchDatumRows &datum_rows) override; virtual int close() override; + virtual int fill_column_group(const int64_t cg_idx, ObCOSliceWriter *writer, ObInsertMonitor *insert_monitor) override + { + return OB_ERR_UNEXPECTED; + } + virtual void cancel() override {} virtual int64_t get_next_block_start_seq() const override { return macro_block_writer_.get_last_macro_seq(); } TO_STRING_KV(K(is_inited_), K_(need_process_cs_replica), K(macro_block_writer_)); private: @@ -617,13 +759,18 @@ public: const ObStorageSchema *storage_schema, const ObIArray &col_schema, const int64_t dir_id, - const int64_t parallelism); + const int64_t parallelism, + const bool use_batch_store, + const int64_t max_batch_size); virtual int append_row(const blocksstable::ObDatumRow &datum_row) override; + virtual int append_batch(const blocksstable::ObBatchDatumRows &datum_rows) override; virtual int close() override; + virtual int fill_column_group(const int64_t cg_idx, ObCOSliceWriter *writer, ObInsertMonitor *insert_monitor) override; + virtual void cancel() override; virtual int64_t get_row_count() const override; virtual int64_t get_next_block_start_seq() const override; + virtual ObDatumRowkey get_compare_key() const override; void reset(); - const ObChunkSliceStore *get_column_slice_store() const { return column_slice_store_; } TO_STRING_KV(K_(is_inited), KPC_(cs_replica_schema), KPC_(row_slice_store), KPC_(column_slice_store)); private: void free_memory(ObArenaAllocator &allocator); @@ -632,7 +779,7 @@ private: ObArenaAllocator *arena_allocator_; ObStorageSchema *cs_replica_schema_; ObMacroBlockSliceStore *row_slice_store_; - ObChunkSliceStore *column_slice_store_; + ObTabletSliceStore *column_slice_store_; }; class ObTabletDirectLoadMgr; @@ -650,7 +797,6 @@ public: int64_t &inserted_cg_row_cnt_; }; -class ObCOSliceWriter; class ObDirectLoadSliceWriter final { public: @@ -680,6 +826,18 @@ public: const int64_t parallelism, int64_t &affected_rows, ObInsertMonitor *insert_monitor = NULL); + int fill_sstable_slice( + const share::SCN &start_scn, + const uint64_t table_id, + const ObTabletID &curr_tablet_id, + const ObStorageSchema *storage_schema, + const blocksstable::ObBatchDatumRows &datum_rows, + const ObTableSchemaItem &schema_item, + const ObDirectLoadType &direct_load_type, + const ObArray &column_items, + const int64_t dir_id, + const int64_t parallelism, + ObInsertMonitor *insert_monitor = NULL); int fill_lob_sstable_slice( const uint64_t table_id, ObIAllocator &allocator, @@ -691,6 +849,17 @@ public: const ObArray &col_types, const int64_t lob_inrow_threshold, blocksstable::ObDatumRow &datum_row); + int fill_lob_sstable_slice( + const uint64_t table_id, + ObIAllocator &allocator, + ObIAllocator &iter_allocator, + const share::SCN &start_scn, + const ObBatchSliceWriteInfo &info, + share::ObTabletCacheInterval &pk_interval, + const ObArray &lob_column_idxs, + const ObArray &col_types, + const int64_t lob_inrow_threshold, + blocksstable::ObBatchDatumRows &datum_rows); int close(); // fill lob meta row into macro block int fill_lob_meta_sstable_slice( @@ -705,8 +874,7 @@ public: ObInsertMonitor *monitor_node = NULL); int fill_aggregated_column_group( const int64_t cg_idx, - ObCOSliceWriter *cur_writer, - ObIArray &datum_stores); + ObCOSliceWriter *cur_writer); int fill_vector_index_data( const int64_t snapshot_version, const ObStorageSchema *storage_schema, @@ -724,32 +892,33 @@ public: bool need_column_store() const { return is_col_store_writer() || is_cs_replica_write(); } ObTabletSliceStore *get_slice_store() const { return slice_store_; } ObDirectLoadSliceWriterType get_writer_type() const { return writer_type_; } - void cancel() { ATOMIC_SET(&is_canceled_, true); } + void cancel(); int64_t get_next_block_start_seq() const { return nullptr == slice_store_ ? start_seq_.get_data_seq() /*slice empty*/ : slice_store_->get_next_block_start_seq(); } TO_STRING_KV(K(is_inited_), K(writer_type_), K(is_canceled_), K(start_seq_), KPC(slice_store_), K(row_offset_)); private: int fill_lob_into_memtable( // for version < 4.3.0.0 ObIAllocator &allocator, const ObBatchSliceWriteInfo &info, - const ObArray &lob_column_idxs, - const ObArray &col_types, + const common::ObObjMeta &col_type, const int64_t lob_inrow_threshold, - blocksstable::ObDatumRow &datum_row); + blocksstable::ObStorageDatum &datum); int fill_lob_into_macro_block( // for version >= 4.3.0.0 ObIAllocator &allocator, ObIAllocator &iter_allocator, const share::SCN &start_scn, const ObBatchSliceWriteInfo &info, share::ObTabletCacheInterval &pk_interval, - const ObArray &lob_column_idxs, - const ObArray &col_types, + const common::ObObjMeta &col_type, const int64_t lob_inrow_threshold, - blocksstable::ObDatumRow &datum_row); - + blocksstable::ObStorageDatum &datum); int check_null( const bool is_index_table, const int64_t rowkey_column_cnt, const blocksstable::ObDatumRow &row_val) const; + int check_null( + const bool is_index_table, + const int64_t rowkey_column_cnt, + const blocksstable::ObBatchDatumRows &datum_rows) const; int prepare_slice_store_if_need( const int64_t schema_rowkey_column_num, const bool is_slice_store, @@ -758,13 +927,21 @@ private: const ObStorageSchema *storage_schema, const share::SCN &start_scn, const ObString vec_idx_param, - const int64_t vec_dim); + const int64_t vec_dim, + const bool use_vector_store = false, + const int64_t max_batch_size = 0); int report_unique_key_dumplicated( const int ret_code, const uint64_t table_id, const blocksstable::ObDatumRow &datum_row, const common::ObTabletID &tablet_id, int &report_ret_code); + int report_unique_key_dumplicated( + const int ret_code, + const uint64_t table_id, + const blocksstable::ObBatchDatumRows &datum_rows, + const common::ObTabletID &tablet_id, + int &report_ret_code); int prepare_iters( ObIAllocator &allocator, ObIAllocator &iter_allocator, @@ -785,7 +962,7 @@ private: ObLobMetaRowIterator *&row_iter); int mock_chunk_store(const int64_t row_cnt); int inner_fill_column_group( - const ObChunkSliceStore *chunk_slice_store, + ObTabletSliceStore *slice_store, const ObStorageSchema *storage_schema, const share::SCN &start_scn, ObInsertMonitor *monitor_node = NULL); @@ -799,6 +976,7 @@ private: ObLobMetaWriteIter *meta_write_iter_; ObLobMetaRowIterator *row_iterator_; common::ObArenaAllocator allocator_; + common::ObIAllocator *lob_allocator_; int64_t row_offset_; }; @@ -819,6 +997,7 @@ public: void reset(); int append_row( const sql::ObChunkDatumStore::StoredRow *stored_row); + int append_batch(const ObIArray &vectors, const int64_t batch_size); int project_cg_row( const ObStorageColumnGroupSchema &cg_schema, const sql::ObChunkDatumStore::StoredRow *stored_row, @@ -835,6 +1014,7 @@ private: blocksstable::ObMacroBlockWriter macro_block_writer_; storage::ObDDLRedoLogWriterCallback flush_callback_; blocksstable::ObDatumRow cg_row_; + blocksstable::ObBatchDatumRows datum_rows_; }; struct ObTabletDirectLoadExecContextId final diff --git a/src/storage/direct_load/ob_direct_load_batch_row_buffer.cpp b/src/storage/direct_load/ob_direct_load_batch_row_buffer.cpp new file mode 100644 index 0000000000..596f769a8c --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_batch_row_buffer.cpp @@ -0,0 +1,371 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX STORAGE + +#include "storage/direct_load/ob_direct_load_batch_row_buffer.h" +#include "storage/blocksstable/ob_datum_row.h" +#include "storage/direct_load/ob_direct_load_row_iterator.h" +#include "storage/direct_load/ob_direct_load_vector_utils.h" + +namespace oceanbase +{ +namespace storage +{ +using namespace blocksstable; +using namespace common; +using namespace share::schema; +using namespace sql; + +ObDirectLoadBatchRowBuffer::ObDirectLoadBatchRowBuffer() + : allocator_("TLD_BatchBuf"), + vector_allocators_(), + vectors_(), + max_batch_size_(0), + row_count_(0), + is_inited_(false) +{ + allocator_.set_tenant_id(MTL_ID()); + vector_allocators_.set_block_allocator(ModulePageAllocator(allocator_)); + vectors_.set_block_allocator(ModulePageAllocator(allocator_)); +} + +ObDirectLoadBatchRowBuffer::~ObDirectLoadBatchRowBuffer() { reset(); } + +void ObDirectLoadBatchRowBuffer::reset() +{ + is_inited_ = false; + for (int64_t i = 0; i < vector_allocators_.count(); ++i) { + ObArenaAllocator *allocator = vector_allocators_.at(i); + if (nullptr != allocator) { + allocator->~ObArenaAllocator(); + allocator_.free(allocator); + } + } + vector_allocators_.reset(); + vectors_.reset(); + max_batch_size_ = 0; + row_count_ = 0; + allocator_.reset(); +} + +void ObDirectLoadBatchRowBuffer::reuse() +{ + row_count_ = 0; + for (int64_t i = 0; i < vector_allocators_.count(); ++i) { + ObArenaAllocator *allocator = vector_allocators_.at(i); + if (nullptr != allocator) { + allocator->reset_remain_one_page(); + } + } +} + +int ObDirectLoadBatchRowBuffer::init(const ObIArray &col_descs, + const int64_t max_batch_size) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObDirectLoadBatchRowBuffer init twice", KR(ret), KP(this)); + } else if (OB_UNLIKELY(col_descs.empty() || max_batch_size <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(col_descs), K(max_batch_size)); + } else { + if (OB_FAIL(init_vectors(col_descs, max_batch_size))) { + LOG_WARN("fail to init vectors", KR(ret)); + } else { + max_batch_size_ = max_batch_size; + is_inited_ = true; + } + } + return ret; +} + +int ObDirectLoadBatchRowBuffer::init_vectors(const ObIArray &col_descs, + int64_t max_batch_size) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(vector_allocators_.prepare_allocate(col_descs.count()))) { + LOG_WARN("fail to prepare allocate", KR(ret), K(col_descs.count())); + } else if (OB_FAIL(vectors_.prepare_allocate(col_descs.count()))) { + LOG_WARN("fail to prepare allocate", KR(ret), K(col_descs.count())); + } + for (int64_t i = 0; OB_SUCC(ret) && i < col_descs.count(); ++i) { + const ObColDesc &col_desc = col_descs.at(i); + VecValueTypeClass value_tc = + get_vec_value_tc(col_desc.col_type_.get_type(), col_desc.col_type_.get_scale(), + col_desc.col_type_.get_stored_precision()); + const bool is_fixed = is_fixed_length_vec(value_tc); + ObIVector *vector = nullptr; + ObArenaAllocator *allocator = nullptr; + if (is_fixed) { // fixed format + if (OB_FAIL(ObDirectLoadVectorUtils::new_vector(VEC_FIXED, value_tc, allocator_, vector))) { + LOG_WARN("fail to new fixed vector", KR(ret), K(value_tc)); + } else if (OB_FAIL( + ObDirectLoadVectorUtils::prepare_vector(vector, max_batch_size, allocator_))) { + LOG_WARN("fail to prepare fixed vector", KR(ret), K(value_tc)); + } + } else { // discrete format + if (OB_FAIL( + ObDirectLoadVectorUtils::new_vector(VEC_DISCRETE, value_tc, allocator_, vector))) { + LOG_WARN("fail to new discrete vector", KR(ret), K(value_tc)); + } else if (OB_FAIL( + ObDirectLoadVectorUtils::prepare_vector(vector, max_batch_size, allocator_))) { + LOG_WARN("fail to prepare discrete vector", KR(ret), K(value_tc)); + } else if (OB_ISNULL(allocator = OB_NEWx(ObArenaAllocator, &allocator_, "TLD_VecAlloc"))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObArenaAllocator", KR(ret)); + } else { + allocator->set_tenant_id(MTL_ID()); + } + } + if (OB_SUCC(ret)) { + vector_allocators_.at(i) = allocator; + vectors_.at(i) = vector; + int64_t fixed_len = 0; + if (is_fixed) { + fixed_len = vector->get_length(0); + } + } + } + return ret; +} + +int ObDirectLoadBatchRowBuffer::to_vector(const ObDatum &datum, ObIVector *vector, + const int64_t batch_idx, ObIAllocator *allocator) +{ + int ret = OB_SUCCESS; + const VectorFormat format = vector->get_format(); + switch (format) { + case VEC_FIXED: { + ObFixedLengthBase *fixed_vec = static_cast(vector); + const ObLength len = fixed_vec->get_length(); + if (datum.is_null()) { + fixed_vec->set_null(batch_idx); + } else if (OB_UNLIKELY(datum.len_ != len)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid datum", KR(ret), K(datum), K(len), K(batch_idx)); + } else { + MEMCPY(static_cast(fixed_vec->get_data()) + len * batch_idx, datum.ptr_, + datum.len_); + } + break; + } + case VEC_DISCRETE: { + ObDiscreteBase *discrete_vec = static_cast(vector); + if (datum.is_null()) { + discrete_vec->set_null(batch_idx); + } else { + char *buf = nullptr; + if (OB_ISNULL(allocator)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected allocator is null", KR(ret)); + } else if (OB_ISNULL(buf = static_cast( + allocator->alloc(datum.len_ > 0 ? datum.len_ : 1)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", KR(ret), K(datum.len_)); + } else { + if (datum.len_ > 0) { + MEMCPY(buf, datum.ptr_, datum.len_); + } + discrete_vec->get_lens()[batch_idx] = datum.len_; + discrete_vec->get_ptrs()[batch_idx] = buf; + } + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", KR(ret), K(format)); + break; + } + return ret; +} + +int ObDirectLoadBatchRowBuffer::to_vector(ObIVector *src_vector, const int64_t src_batch_idx, + ObIVector *vector, const int64_t batch_idx, + ObIAllocator *allocator) +{ + int ret = OB_SUCCESS; + bool is_null = false; + const char *payload = nullptr; + ObLength len = 0; + if (OB_FAIL( + ObDirectLoadVectorUtils::get_payload(src_vector, src_batch_idx, is_null, payload, len))) { + LOG_WARN("fail to get payload", KR(ret)); + } else { + const VectorFormat format = vector->get_format(); + switch (format) { + case VEC_FIXED: { + ObFixedLengthBase *fixed_vec = static_cast(vector); + const ObLength fixed_len = fixed_vec->get_length(); + if (is_null) { + fixed_vec->set_null(batch_idx); + } else { + const ObLength fixed_len = fixed_vec->get_length(); + if (OB_UNLIKELY(len != fixed_len)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid value", KR(ret), K(src_batch_idx), K(len), K(fixed_len)); + } else { + MEMCPY(static_cast(fixed_vec->get_data()) + len * batch_idx, payload, len); + } + } + break; + } + case VEC_DISCRETE: { + ObDiscreteBase *discrete_vec = static_cast(vector); + if (is_null) { + discrete_vec->set_null(batch_idx); + } else { + char *buf = nullptr; + if (OB_ISNULL(allocator)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected allocator is null", KR(ret)); + } else if (OB_ISNULL(buf = static_cast(allocator->alloc(len > 0 ? len : 1)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", KR(ret), K(len)); + } else { + if (len > 0) { + MEMCPY(buf, payload, len); + } + discrete_vec->get_lens()[batch_idx] = len; + discrete_vec->get_ptrs()[batch_idx] = buf; + } + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", KR(ret), K(format)); + break; + } + } + return ret; +} + +int ObDirectLoadBatchRowBuffer::append_row(const ObStorageDatum *datums, + const int64_t count, + const ObDirectLoadRowFlag &row_flag, + bool &is_full) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadBatchRowBuffer not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(row_count_ >= max_batch_size_)) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("size overflow", KR(ret), K(max_batch_size_), K(row_count_)); + } else if (OB_UNLIKELY(nullptr == datums || + row_flag.get_column_count(count) != vectors_.count())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(vectors_.count()), KP(datums), K(count), K(row_flag)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < count; ++i) { + const ObDatum &datum = datums[i]; + ObIVector *vec = vectors_.at(row_flag.uncontain_hidden_pk_ ? i + 1 : i); + ObIAllocator *allocator = vector_allocators_.at(row_flag.uncontain_hidden_pk_ ? i + 1 : i); + if (OB_UNLIKELY(datum.is_ext())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid datum", KR(ret), K(i), K(datum)); + } else if (OB_FAIL(to_vector(datum, vec, row_count_, allocator))) { + LOG_WARN("fail to set vector", KR(ret), K(i), K(datum), K(row_flag)); + } + } + if (OB_SUCC(ret)) { + ++row_count_; + is_full = (row_count_ == max_batch_size_); + } + } + return ret; +} + +int ObDirectLoadBatchRowBuffer::append_row(const ObDatumRow &datum_row, + const ObDirectLoadRowFlag &row_flag, + bool &is_full) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadBatchRowBuffer not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(row_count_ >= max_batch_size_)) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("size overflow", KR(ret), K(max_batch_size_), K(row_count_)); + } else if (OB_UNLIKELY(row_flag.get_column_count(datum_row.get_column_count()) != + vectors_.count())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(vectors_.count()), K(datum_row), K(row_flag)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < datum_row.count_; ++i) { + const ObDatum &datum = datum_row.storage_datums_[i]; + ObIVector *vec = vectors_.at(row_flag.uncontain_hidden_pk_ ? i + 1 : i); + ObIAllocator *allocator = vector_allocators_.at(row_flag.uncontain_hidden_pk_ ? i + 1 : i); + if (OB_UNLIKELY(datum.is_ext())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid datum", KR(ret), K(i), K(datum)); + } else if (OB_FAIL(to_vector(datum, vec, row_count_, allocator))) { + LOG_WARN("fail to set vector", KR(ret), K(i), K(datum), K(row_flag)); + } + } + if (OB_SUCC(ret)) { + ++row_count_; + is_full = (row_count_ == max_batch_size_); + } + } + return ret; +} + +int ObDirectLoadBatchRowBuffer::append_row(const IVectorPtrs &vectors, + const int64_t row_idx, + const ObDirectLoadRowFlag &row_flag, + bool &is_full) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadBatchRowBuffer not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(row_count_ >= max_batch_size_)) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("size overflow", KR(ret), K(max_batch_size_), K(row_count_)); + } else if (OB_UNLIKELY(row_flag.get_column_count(vectors.count()) != vectors_.count())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(vectors_.count()), K(vectors.count()), K(row_flag)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < vectors.count(); ++i) { + ObIVector *src_vec = vectors.at(i); + ObIVector *dest_vec = vectors_.at(row_flag.uncontain_hidden_pk_ ? i + 1 : i); + ObIAllocator *allocator = vector_allocators_.at(row_flag.uncontain_hidden_pk_ ? i + 1 : i); + if (OB_FAIL(to_vector(src_vec, row_idx, dest_vec, row_count_, allocator))) { + LOG_WARN("fail to set vector", KR(ret)); + } + } + if (OB_SUCC(ret)) { + ++row_count_; + is_full = (row_count_ == max_batch_size_); + } + } + return ret; +} + +int ObDirectLoadBatchRowBuffer::get_batch(const IVectorPtrs *&vectors, int64_t &row_count) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadBatchRowBuffer not init", KR(ret), KP(this)); + } else { + vectors = &vectors_; + row_count = row_count_; + } + return ret; +} + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_batch_row_buffer.h b/src/storage/direct_load/ob_direct_load_batch_row_buffer.h new file mode 100644 index 0000000000..875e7788d8 --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_batch_row_buffer.h @@ -0,0 +1,80 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once + +#include "share/vector/ob_i_vector.h" + +namespace oceanbase +{ +namespace share +{ +namespace schema +{ +class ObColDesc; +} // namespace schema +} // namespace share +namespace blocksstable +{ +class ObStorageDatum; +class ObDatumRow; +} // namespace blocksstable +namespace storage +{ +class ObDirectLoadRowFlag; + +class ObDirectLoadBatchRowBuffer +{ +public: + ObDirectLoadBatchRowBuffer(); + ~ObDirectLoadBatchRowBuffer(); + void reset(); + void reuse(); + int init(const common::ObIArray &col_descs, + const int64_t max_batch_size); + int append_row(const blocksstable::ObStorageDatum *datums, + const int64_t count, + const ObDirectLoadRowFlag &row_flag, + bool &is_full); + int append_row(const blocksstable::ObDatumRow &datum_row, + const ObDirectLoadRowFlag &row_flag, + bool &is_full); + int append_row(const IVectorPtrs &vectors, + const int64_t row_idx, + const ObDirectLoadRowFlag &row_flag, + bool &is_full); + int get_batch(const IVectorPtrs *&vectors, int64_t &row_count); + bool empty() const { return 0 == row_count_; } + const IVectorPtrs &get_vectors() const { return vectors_; } + int64_t get_max_batch_size() const { return max_batch_size_; } + int64_t get_row_count() const { return row_count_; } + TO_STRING_KV(K_(vectors), K_(max_batch_size), K_(row_count), K_(is_inited)); + +private: + int init_vectors(const common::ObIArray &col_descs, + int64_t max_batch_size); + int to_vector(const ObDatum &datum, ObIVector *vector, const int64_t batch_idx, + ObIAllocator *allocator); + int to_vector(ObIVector *src_vector, const int64_t src_batch_idx, ObIVector *vector, + const int64_t batch_idx, ObIAllocator *allocator); + +private: + common::ObArenaAllocator allocator_; // 常驻内存分配器 + ObArray vector_allocators_; // 动态内存分配器 + ObArray vectors_; + int64_t max_batch_size_; + int64_t row_count_; + bool is_inited_; +}; + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_conflict_check.cpp b/src/storage/direct_load/ob_direct_load_conflict_check.cpp index 0a66168d3d..d7c6ddc2a4 100644 --- a/src/storage/direct_load/ob_direct_load_conflict_check.cpp +++ b/src/storage/direct_load/ob_direct_load_conflict_check.cpp @@ -74,7 +74,7 @@ ObDirectLoadConflictCheck::ObDirectLoadConflictCheck() ObDirectLoadConflictCheck::~ObDirectLoadConflictCheck() { if (origin_iter_ != nullptr) { - origin_iter_->~ObIStoreRowIterator(); + origin_iter_->~ObDirectLoadIStoreRowIterator(); allocator_.free(origin_iter_); origin_iter_ = nullptr; } @@ -323,6 +323,10 @@ int ObDirectLoadSSTableConflictCheck::init( } else if (OB_FAIL(conflict_check_.init(param, &scan_merge_))) { LOG_WARN("fail to init conflict_check_", KR(ret)); } else { + row_flag_.uncontain_hidden_pk_ = false; + row_flag_.has_multi_version_cols_ = false; + // row_flag_.has_delete_row_ = ?; + column_count_ = param.table_data_desc_.column_count_; is_inited_ = true; } } @@ -378,6 +382,10 @@ int ObDirectLoadMultipleSSTableConflictCheck::init( } else if (OB_FAIL(conflict_check_.init(param, &scan_merge_))) { LOG_WARN("fail to init conflict_check_", KR(ret)); } else { + row_flag_.uncontain_hidden_pk_ = false; + row_flag_.has_multi_version_cols_ = false; + // row_flag_.has_delete_row_ = ?; + column_count_ = param.table_data_desc_.column_count_; is_inited_ = true; } } diff --git a/src/storage/direct_load/ob_direct_load_conflict_check.h b/src/storage/direct_load/ob_direct_load_conflict_check.h index ab6f137f6e..6fddf95a9e 100644 --- a/src/storage/direct_load/ob_direct_load_conflict_check.h +++ b/src/storage/direct_load/ob_direct_load_conflict_check.h @@ -19,6 +19,7 @@ #include "storage/direct_load/ob_direct_load_origin_table.h" #include "storage/direct_load/ob_direct_load_sstable_scan_merge.h" #include "storage/direct_load/ob_direct_load_external_multi_partition_table.h" +#include "storage/direct_load/ob_direct_load_row_iterator.h" namespace oceanbase { @@ -86,7 +87,7 @@ private: common::ObArenaAllocator range_allocator_; ObDirectLoadConflictCheckParam param_; ObIStoreRowIterator *load_iter_; - ObIStoreRowIterator *origin_iter_; + ObDirectLoadIStoreRowIterator *origin_iter_; const blocksstable::ObDatumRow *origin_row_; blocksstable::ObDatumRow append_row_; ObDatumRange new_range_; @@ -95,7 +96,7 @@ private: bool is_inited_; }; -class ObDirectLoadSSTableConflictCheck : public ObIStoreRowIterator +class ObDirectLoadSSTableConflictCheck : public ObDirectLoadIStoreRowIterator { public: ObDirectLoadSSTableConflictCheck(); @@ -110,7 +111,7 @@ private: bool is_inited_; }; -class ObDirectLoadMultipleSSTableConflictCheck : public ObIStoreRowIterator +class ObDirectLoadMultipleSSTableConflictCheck : public ObDirectLoadIStoreRowIterator { public: ObDirectLoadMultipleSSTableConflictCheck(); diff --git a/src/storage/direct_load/ob_direct_load_data_fuse.cpp b/src/storage/direct_load/ob_direct_load_data_fuse.cpp index 9d01e6a45f..d4eddf830d 100644 --- a/src/storage/direct_load/ob_direct_load_data_fuse.cpp +++ b/src/storage/direct_load/ob_direct_load_data_fuse.cpp @@ -298,7 +298,7 @@ ObDirectLoadSSTableDataFuse::ObDirectLoadSSTableDataFuse() ObDirectLoadSSTableDataFuse::~ObDirectLoadSSTableDataFuse() { if (nullptr != origin_iter_) { - origin_iter_->~ObIStoreRowIterator(); + origin_iter_->~ObDirectLoadIStoreRowIterator(); allocator_.free(origin_iter_); origin_iter_ = nullptr; } @@ -334,6 +334,10 @@ int ObDirectLoadSSTableDataFuse::init(const ObDirectLoadDataFuseParam ¶m, if (OB_FAIL(data_fuse_.init(param, origin_iter_, &scan_merge_))) { LOG_WARN("fail to init data fuse", KR(ret)); } else { + row_flag_.uncontain_hidden_pk_ = false; + row_flag_.has_multi_version_cols_ = false; + // row_flag_.has_delete_row_ = ?; + column_count_ = param.table_data_desc_.column_count_; is_inited_ = true; } } @@ -366,7 +370,7 @@ ObDirectLoadMultipleSSTableDataFuse::ObDirectLoadMultipleSSTableDataFuse() ObDirectLoadMultipleSSTableDataFuse::~ObDirectLoadMultipleSSTableDataFuse() { if (nullptr != origin_iter_) { - origin_iter_->~ObIStoreRowIterator(); + origin_iter_->~ObDirectLoadIStoreRowIterator(); allocator_.free(origin_iter_); origin_iter_ = nullptr; } @@ -405,6 +409,10 @@ int ObDirectLoadMultipleSSTableDataFuse::init( if (OB_FAIL(data_fuse_.init(param, origin_iter_, &scan_merge_))) { LOG_WARN("fail to init data fuse", KR(ret)); } else { + row_flag_.uncontain_hidden_pk_ = false; + row_flag_.has_multi_version_cols_ = false; + // row_flag_.has_delete_row_ = ?; + column_count_ = param.table_data_desc_.column_count_; is_inited_ = true; } } diff --git a/src/storage/direct_load/ob_direct_load_data_fuse.h b/src/storage/direct_load/ob_direct_load_data_fuse.h index 2185757964..d9ad84242d 100644 --- a/src/storage/direct_load/ob_direct_load_data_fuse.h +++ b/src/storage/direct_load/ob_direct_load_data_fuse.h @@ -18,6 +18,7 @@ #include "storage/direct_load/ob_direct_load_multiple_datum_range.h" #include "storage/direct_load/ob_direct_load_multiple_sstable_scan_merge.h" #include "storage/direct_load/ob_direct_load_origin_table.h" +#include "storage/direct_load/ob_direct_load_row_iterator.h" #include "storage/direct_load/ob_direct_load_sstable_scan_merge.h" #include "storage/direct_load/ob_direct_load_struct.h" @@ -98,7 +99,7 @@ protected: bool is_inited_; }; -class ObDirectLoadSSTableDataFuse : public ObIStoreRowIterator +class ObDirectLoadSSTableDataFuse : public ObDirectLoadIStoreRowIterator { public: ObDirectLoadSSTableDataFuse(); @@ -109,13 +110,13 @@ public: int get_next_row(const blocksstable::ObDatumRow *&datum_row) override; private: common::ObArenaAllocator allocator_; - ObIStoreRowIterator *origin_iter_; + ObDirectLoadIStoreRowIterator *origin_iter_; ObDirectLoadSSTableScanMerge scan_merge_; ObDirectLoadDataFuse data_fuse_; bool is_inited_; }; -class ObDirectLoadMultipleSSTableDataFuse : public ObIStoreRowIterator +class ObDirectLoadMultipleSSTableDataFuse : public ObDirectLoadIStoreRowIterator { public: ObDirectLoadMultipleSSTableDataFuse(); @@ -127,7 +128,7 @@ public: private: common::ObArenaAllocator allocator_; ObDirectLoadMultipleDatumRange range_; - ObIStoreRowIterator *origin_iter_; + ObDirectLoadIStoreRowIterator *origin_iter_; ObDirectLoadMultipleSSTableScanMerge scan_merge_; ObDirectLoadDataFuse data_fuse_; bool is_inited_; diff --git a/src/storage/direct_load/ob_direct_load_data_insert.cpp b/src/storage/direct_load/ob_direct_load_data_insert.cpp index f02fa8c307..9ff54fa7c5 100644 --- a/src/storage/direct_load/ob_direct_load_data_insert.cpp +++ b/src/storage/direct_load/ob_direct_load_data_insert.cpp @@ -140,6 +140,10 @@ int ObDirectLoadSSTableDataInsert::init( } else if (OB_FAIL(data_insert_.init(param, &scan_merge_))) { LOG_WARN("fail to init data insert", KR(ret)); } else { + row_flag_.uncontain_hidden_pk_ = false; + row_flag_.has_multi_version_cols_ = false; + // row_flag_.has_delete_row_ = ?; + column_count_ = param.table_data_desc_.column_count_; is_inited_ = true; } } @@ -197,6 +201,10 @@ int ObDirectLoadMultipleSSTableDataInsert::init( } else if (OB_FAIL(data_insert_.init(param, &scan_merge_))) { LOG_WARN("fail to init data insert", KR(ret)); } else { + row_flag_.uncontain_hidden_pk_ = false; + row_flag_.has_multi_version_cols_ = false; + // row_flag_.has_delete_row_ = ?; + column_count_ = param.table_data_desc_.column_count_; is_inited_ = true; } } diff --git a/src/storage/direct_load/ob_direct_load_data_insert.h b/src/storage/direct_load/ob_direct_load_data_insert.h index 21f38fe428..f9df46a25c 100644 --- a/src/storage/direct_load/ob_direct_load_data_insert.h +++ b/src/storage/direct_load/ob_direct_load_data_insert.h @@ -17,6 +17,7 @@ #include "storage/direct_load/ob_direct_load_dml_row_handler.h" #include "storage/direct_load/ob_direct_load_multiple_datum_range.h" #include "storage/direct_load/ob_direct_load_multiple_sstable_scan_merge.h" +#include "storage/direct_load/ob_direct_load_row_iterator.h" #include "storage/direct_load/ob_direct_load_sstable_scan_merge.h" #include "storage/direct_load/ob_direct_load_struct.h" @@ -58,7 +59,7 @@ private: bool is_inited_; }; -class ObDirectLoadSSTableDataInsert : public ObIStoreRowIterator +class ObDirectLoadSSTableDataInsert : public ObDirectLoadIStoreRowIterator { public: ObDirectLoadSSTableDataInsert(); @@ -74,7 +75,7 @@ private: bool is_inited_; }; -class ObDirectLoadMultipleSSTableDataInsert : public ObIStoreRowIterator +class ObDirectLoadMultipleSSTableDataInsert : public ObDirectLoadIStoreRowIterator { public: ObDirectLoadMultipleSSTableDataInsert(); diff --git a/src/storage/direct_load/ob_direct_load_dml_row_handler.h b/src/storage/direct_load/ob_direct_load_dml_row_handler.h index d838b286f4..299cff6137 100644 --- a/src/storage/direct_load/ob_direct_load_dml_row_handler.h +++ b/src/storage/direct_load/ob_direct_load_dml_row_handler.h @@ -11,6 +11,7 @@ */ #pragma once +#include "storage/blocksstable/ob_batch_datum_rows.h" #include "storage/blocksstable/ob_datum_row.h" #include "storage/direct_load/ob_direct_load_external_row.h" #include "storage/direct_load/ob_direct_load_multiple_datum_row.h" @@ -27,9 +28,11 @@ public: virtual ~ObDirectLoadDMLRowHandler() = default; // handle rows direct insert into sstable virtual int handle_insert_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) = 0; + virtual int handle_insert_batch(const ObTabletID &tablet_id, const blocksstable::ObBatchDatumRows &datum_rows) = 0; virtual int handle_delete_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) = 0; // only used for heap table virtual int handle_insert_row_with_multi_version(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) = 0; + virtual int handle_insert_batch_with_multi_version(const ObTabletID &tablet_id, const blocksstable::ObBatchDatumRows &datum_rows) = 0; // handle rows with the same primary key in the imported data virtual int handle_update_row(const blocksstable::ObDatumRow &row) = 0; virtual int handle_update_row(common::ObArray &rows, diff --git a/src/storage/direct_load/ob_direct_load_fast_heap_table.cpp b/src/storage/direct_load/ob_direct_load_fast_heap_table.cpp deleted file mode 100644 index 6dd8ef2693..0000000000 --- a/src/storage/direct_load/ob_direct_load_fast_heap_table.cpp +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ -#define USING_LOG_PREFIX STORAGE - -#include "storage/direct_load/ob_direct_load_fast_heap_table.h" - -namespace oceanbase -{ -namespace storage -{ -using namespace common; - -/** - * ObDirectLoadFastHeapTableCreateParam - */ - -ObDirectLoadFastHeapTableCreateParam::ObDirectLoadFastHeapTableCreateParam() - : row_count_(0) -{ -} - -ObDirectLoadFastHeapTableCreateParam::~ObDirectLoadFastHeapTableCreateParam() -{ -} - -bool ObDirectLoadFastHeapTableCreateParam::is_valid() const -{ - return tablet_id_.is_valid() && row_count_ >= 0; -} - -/** - * ObDirectLoadFastHeapTable - */ - -ObDirectLoadFastHeapTable::ObDirectLoadFastHeapTable() - : is_inited_(false) -{ -} - -ObDirectLoadFastHeapTable::~ObDirectLoadFastHeapTable() -{ -} - -int ObDirectLoadFastHeapTable::init(const ObDirectLoadFastHeapTableCreateParam ¶m) -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - LOG_WARN("ObDirectLoadFastHeapTable init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!param.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(param)); - } else { - meta_.tablet_id_ = param.tablet_id_; - meta_.row_count_ = param.row_count_; - is_inited_ = true; - } - return ret; -} - -} // namespace storage -} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_fast_heap_table.h b/src/storage/direct_load/ob_direct_load_fast_heap_table.h deleted file mode 100644 index 65772683f1..0000000000 --- a/src/storage/direct_load/ob_direct_load_fast_heap_table.h +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ -#pragma once - -#include "common/ob_tablet_id.h" -#include "storage/direct_load/ob_direct_load_i_table.h" - -namespace oceanbase -{ -namespace storage -{ - -struct ObDirectLoadFastHeapTableCreateParam -{ -public: - ObDirectLoadFastHeapTableCreateParam(); - ~ObDirectLoadFastHeapTableCreateParam(); - bool is_valid() const; - TO_STRING_KV(K_(tablet_id), K_(row_count)); -public: - common::ObTabletID tablet_id_; - int64_t row_count_; -}; - -struct ObDirectLoadFastHeapTableMeta -{ - common::ObTabletID tablet_id_; - int64_t row_count_; - TO_STRING_KV(K_(tablet_id), K_(row_count)); -}; - -class ObDirectLoadFastHeapTable : public ObIDirectLoadPartitionTable -{ -public: - ObDirectLoadFastHeapTable(); - virtual ~ObDirectLoadFastHeapTable(); - int init(const ObDirectLoadFastHeapTableCreateParam ¶m); - const common::ObTabletID &get_tablet_id() const override { return meta_.tablet_id_; } - int64_t get_row_count() const override { return meta_.row_count_; } - bool is_valid() const override { return is_inited_; } - void release_data() override { /*do nothing*/ } - const ObDirectLoadFastHeapTableMeta &get_meta() const { return meta_; } - TO_STRING_KV(K_(meta)); -private: - ObDirectLoadFastHeapTableMeta meta_; - bool is_inited_; - DISABLE_COPY_ASSIGN(ObDirectLoadFastHeapTable); -}; - -} // namespace storage -} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.cpp b/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.cpp deleted file mode 100644 index 01b0401f3f..0000000000 --- a/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.cpp +++ /dev/null @@ -1,291 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ -#define USING_LOG_PREFIX STORAGE - -#include "storage/direct_load/ob_direct_load_fast_heap_table_builder.h" -#include "share/table/ob_table_load_define.h" -#include "storage/direct_load/ob_direct_load_dml_row_handler.h" -#include "storage/direct_load/ob_direct_load_fast_heap_table.h" -#include "storage/direct_load/ob_direct_load_insert_table_ctx.h" - -namespace oceanbase -{ -namespace storage -{ -using namespace common; -using namespace blocksstable; -using namespace share; -using namespace table; - -/** - * ObDirectLoadFastHeapTableBuildParam - */ - -ObDirectLoadFastHeapTableBuildParam::ObDirectLoadFastHeapTableBuildParam() - : insert_table_ctx_(nullptr), dml_row_handler_(nullptr) -{ -} - -ObDirectLoadFastHeapTableBuildParam::~ObDirectLoadFastHeapTableBuildParam() {} - -bool ObDirectLoadFastHeapTableBuildParam::is_valid() const -{ - return tablet_id_.is_valid() && table_data_desc_.is_valid() && nullptr != insert_table_ctx_ && - nullptr != dml_row_handler_; -} - -/** - * RowIterator - */ - -ObDirectLoadFastHeapTableBuilder::RowIterator::RowIterator() : datum_row_(nullptr) {} - -ObDirectLoadFastHeapTableBuilder::RowIterator::~RowIterator() {} - -int ObDirectLoadFastHeapTableBuilder::RowIterator::init( - ObDirectLoadInsertTabletContext *insert_tablet_ctx, - ObTableLoadSqlStatistics *sql_statistics, - ObDirectLoadLobBuilder &lob_builder) -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - LOG_WARN("ObDirectLoadFastHeapTableBuilder init twice", KR(ret), KP(this)); - } else if (OB_FAIL(inner_init(insert_tablet_ctx, sql_statistics, lob_builder))) { - LOG_WARN("fail to inner init", KR(ret)); - } else { - is_inited_ = true; - } - return ret; -} - -int ObDirectLoadFastHeapTableBuilder::RowIterator::set_row(ObDatumRow &row) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("RowIterator not init", KR(ret), KP(this)); - } else if (OB_UNLIKELY(nullptr != datum_row_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected set row twice", KR(ret)); - } else { - datum_row_ = &row; - } - return ret; -} - -int ObDirectLoadFastHeapTableBuilder::RowIterator::inner_get_next_row(ObDatumRow *&row) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("RowIterator not init", KR(ret), KP(this)); - } else if (nullptr == datum_row_) { - ret = OB_ITER_END; - } else { - row = datum_row_; - datum_row_ = nullptr; - } - return ret; -} - -/** - * ObDirectLoadFastHeapTableBuilder - */ - -ObDirectLoadFastHeapTableBuilder::ObDirectLoadFastHeapTableBuilder() - : insert_tablet_ctx_(nullptr), - sql_statistics_(nullptr), - current_slice_id_(0), - row_count_(0), - has_lob_(false), - is_closed_(false), - is_inited_(false) -{ -} - -ObDirectLoadFastHeapTableBuilder::~ObDirectLoadFastHeapTableBuilder() -{ -} - -int ObDirectLoadFastHeapTableBuilder::init(const ObDirectLoadFastHeapTableBuildParam ¶m) -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - LOG_WARN("ObDirectLoadFastHeapTableBuilder init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!param.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(param)); - } else if (OB_FAIL( - param.insert_table_ctx_->get_tablet_context(param.tablet_id_, insert_tablet_ctx_))) { - LOG_WARN("fail to get tablet context", KR(ret)); - } else { - param_ = param; - const bool online_opt_stat_gather = insert_tablet_ctx_->get_online_opt_stat_gather(); - has_lob_ = insert_tablet_ctx_->has_lob_storage(); - if (online_opt_stat_gather && - OB_FAIL(param.insert_table_ctx_->get_sql_statistics(sql_statistics_))) { - LOG_WARN("fail to get sql statistics", KR(ret)); - } else if (has_lob_ && OB_FAIL(lob_builder_.init(insert_tablet_ctx_))) { - LOG_WARN("fail to inner init sql statistics", KR(ret)); - } else if (OB_FAIL(init_sstable_slice_ctx())) { - LOG_WARN("fail to init sstable slice ctx", KR(ret)); - } else if (OB_FAIL(insert_tablet_ctx_->init_datum_row(datum_row_))) { - LOG_WARN("fail to init datum row", KR(ret)); - } else if (OB_FAIL(row_iter_.init(insert_tablet_ctx_, sql_statistics_, lob_builder_))) { - LOG_WARN("fail to init row iter", KR(ret)); - } else { - is_inited_ = true; - } - } - return ret; -} - -int ObDirectLoadFastHeapTableBuilder::init_sstable_slice_ctx() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(insert_tablet_ctx_->get_write_ctx(write_ctx_))) { - LOG_WARN("fail to get write ctx", KR(ret)); - } else if (OB_FAIL( - insert_tablet_ctx_->open_sstable_slice(write_ctx_.start_seq_, current_slice_id_))) { - LOG_WARN("fail to open sstable slice", KR(ret)); - } - return ret; -} - -int ObDirectLoadFastHeapTableBuilder::switch_sstable_slice() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(insert_tablet_ctx_->close_sstable_slice(current_slice_id_))) { - LOG_WARN("fail to close sstable slice builder", KR(ret)); - } else if (OB_FAIL(init_sstable_slice_ctx())) { - LOG_WARN("fail to init sstable slice ctx", KR(ret)); - } - return ret; -} - -int ObDirectLoadFastHeapTableBuilder::append_row(const ObTabletID &tablet_id, - const table::ObTableLoadSequenceNo &seq_no, - const ObDatumRow &datum_row) -{ - UNUSED(tablet_id); - UNUSED(seq_no); - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadFastHeapTableBuilder not init", KR(ret), KP(this)); - } else if (OB_UNLIKELY(is_closed_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fast heap table builder is closed", KR(ret)); - } else if (OB_FAIL(!datum_row.is_valid() || - datum_row.get_column_count() != param_.table_data_desc_.column_count_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(datum_row), K(param_.table_data_desc_.column_count_)); - } else { - uint64_t pk_seq = OB_INVALID_ID; - int64_t affected_rows = 0; - if (OB_FAIL(write_ctx_.pk_interval_.next_value(pk_seq))) { - if (OB_UNLIKELY(OB_EAGAIN != ret)) { - LOG_WARN("fail to get next pk seq", KR(ret)); - } else if (OB_FAIL(switch_sstable_slice())) { - LOG_WARN("fail to switch sstable slice", KR(ret)); - } else if (OB_FAIL(write_ctx_.pk_interval_.next_value(pk_seq))) { - LOG_WARN("fail to get next pk seq", KR(ret)); - } - } - if (OB_SUCC(ret)) { - datum_row_.storage_datums_[0].set_int(pk_seq); - for (int64_t i = 0, j = HIDDEN_ROWKEY_COLUMN_NUM + - ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); - i < datum_row.count_; ++i, ++j) { - datum_row_.storage_datums_[j] = datum_row.storage_datums_[i]; - } - if (OB_FAIL(row_iter_.set_row(datum_row_))) { - LOG_WARN("fail to set row", KR(ret)); - } else if (OB_FAIL(insert_tablet_ctx_->fill_sstable_slice(current_slice_id_, row_iter_, - affected_rows))) { - LOG_WARN("fail to fill sstable slice", KR(ret)); - } else { - ++row_count_; - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(param_.dml_row_handler_->handle_insert_row_with_multi_version(param_.tablet_id_, datum_row_))) { - LOG_WARN("fail to handle insert row", KR(ret), K_(datum_row)); - } - } - } - return ret; -} - -int ObDirectLoadFastHeapTableBuilder::close() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadFastHeapTableBuilder not init", KR(ret), KP(this)); - } else if (OB_UNLIKELY(is_closed_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fast heap table builder is closed", KR(ret)); - } else { - if (has_lob_ && OB_FAIL(lob_builder_.close())) { - LOG_WARN("fail to close lob_builder", KR(ret)); - } else if (OB_FAIL(insert_tablet_ctx_->close_sstable_slice(current_slice_id_))) { - LOG_WARN("fail to close sstable slice builder", KR(ret)); - } else { - current_slice_id_ = 0; - is_closed_ = true; - insert_tablet_ctx_->inc_row_count(row_count_); - } - } - return ret; -} - -int ObDirectLoadFastHeapTableBuilder::get_tables( - ObIArray &table_array, ObIAllocator &allocator) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadFastHeapTableBuilder not init", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!is_closed_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fast heap table builder not closed", KR(ret)); - } else if (row_count_ == 0) { - // do nothing - } else { - ObDirectLoadFastHeapTableCreateParam create_param; - create_param.tablet_id_ = param_.tablet_id_; - create_param.row_count_ = row_count_; - ObDirectLoadFastHeapTable *fast_heap_table = nullptr; - if (OB_ISNULL(fast_heap_table = OB_NEWx(ObDirectLoadFastHeapTable, (&allocator)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObDirectLoadFastHeapTable", KR(ret)); - } else if (OB_FAIL(fast_heap_table->init(create_param))) { - LOG_WARN("fail to init sstable", KR(ret), K(create_param)); - } else if (OB_FAIL(table_array.push_back(fast_heap_table))) { - LOG_WARN("fail to push back sstable", KR(ret)); - } - if (OB_FAIL(ret)) { - if (nullptr != fast_heap_table) { - fast_heap_table->~ObDirectLoadFastHeapTable(); - allocator.free(fast_heap_table); - fast_heap_table = nullptr; - } - } - } - return ret; -} - -} // namespace storage -} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.h b/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.h deleted file mode 100644 index 7a6e143e00..0000000000 --- a/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.h +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ -#pragma once - -#include "common/ob_tablet_id.h" -#include "storage/direct_load/ob_direct_load_i_table.h" -#include "storage/direct_load/ob_direct_load_insert_table_ctx.h" -#include "storage/direct_load/ob_direct_load_insert_table_row_iterator.h" -#include "storage/direct_load/ob_direct_load_lob_builder.h" -#include "storage/direct_load/ob_direct_load_table_data_desc.h" - -namespace oceanbase -{ -namespace table -{ -class ObTableLoadSqlStatistics; -} // namespace table -namespace storage -{ -class ObDirectLoadInsertTableContext; -class ObDirectLoadDMLRowHandler; -struct ObDirectLoadFastHeapTableBuildParam -{ -public: - ObDirectLoadFastHeapTableBuildParam(); - ~ObDirectLoadFastHeapTableBuildParam(); - bool is_valid() const; - TO_STRING_KV(K_(tablet_id), - K_(table_data_desc), - KP_(insert_table_ctx), - KP_(dml_row_handler)); -public: - common::ObTabletID tablet_id_; - ObDirectLoadTableDataDesc table_data_desc_; - ObDirectLoadInsertTableContext *insert_table_ctx_; - ObDirectLoadDMLRowHandler *dml_row_handler_; -}; - -class ObDirectLoadFastHeapTableBuilder : public ObIDirectLoadPartitionTableBuilder -{ - static const int64_t HIDDEN_ROWKEY_COLUMN_NUM = 1; -public: - ObDirectLoadFastHeapTableBuilder(); - virtual ~ObDirectLoadFastHeapTableBuilder(); - int init(const ObDirectLoadFastHeapTableBuildParam ¶m); - int append_row(const common::ObTabletID &tablet_id, const table::ObTableLoadSequenceNo &seq_no, - const blocksstable::ObDatumRow &datum_row) override; - int close() override; - int64_t get_row_count() const override { return row_count_; } - int get_tables(common::ObIArray &table_array, - common::ObIAllocator &allocator) override; -private: - int init_sstable_slice_ctx(); - int switch_sstable_slice(); -private: - class RowIterator : public ObDirectLoadInsertTableRowIterator - { - public: - RowIterator(); - virtual ~RowIterator(); - int init(ObDirectLoadInsertTabletContext *insert_tablet_ctx, - table::ObTableLoadSqlStatistics *sql_statistics, - ObDirectLoadLobBuilder &lob_builder); - int set_row(blocksstable::ObDatumRow &row); - protected: - int inner_get_next_row(blocksstable::ObDatumRow *&row) override; - private: - blocksstable::ObDatumRow *datum_row_; - }; -private: - ObDirectLoadFastHeapTableBuildParam param_; - ObDirectLoadInsertTabletContext *insert_tablet_ctx_; - table::ObTableLoadSqlStatistics *sql_statistics_; - ObDirectLoadLobBuilder lob_builder_; - ObDirectLoadInsertTabletWriteCtx write_ctx_; - blocksstable::ObDatumRow datum_row_; - RowIterator row_iter_; - int64_t current_slice_id_; - int64_t row_count_; - bool has_lob_; - bool is_closed_; - bool is_inited_; - DISALLOW_COPY_AND_ASSIGN(ObDirectLoadFastHeapTableBuilder); -}; - -} // namespace storage -} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp b/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp index 87b3dd49fa..317f885f86 100644 --- a/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp +++ b/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp @@ -19,6 +19,8 @@ #include "share/table/ob_table_load_sql_statistics.h" #include "share/stat/ob_stat_item.h" #include "storage/direct_load/ob_direct_load_origin_table.h" +#include "storage/direct_load/ob_direct_load_row_iterator.h" +#include "storage/direct_load/ob_direct_load_vector_utils.h" #include "storage/lob/ob_lob_meta.h" namespace oceanbase @@ -46,7 +48,7 @@ ObDirectLoadInsertTableParam::ObDirectLoadInsertTableParam() reserved_parallel_(0), rowkey_column_count_(0), column_count_(0), - lob_column_count_(0), + lob_inrow_threshold_(-1), is_partitioned_table_(false), is_heap_table_(false), is_column_store_(false), @@ -55,8 +57,10 @@ ObDirectLoadInsertTableParam::ObDirectLoadInsertTableParam() datum_utils_(nullptr), col_descs_(nullptr), cmp_funcs_(nullptr), + lob_column_idxs_(nullptr), online_sample_percent_(1.), - is_no_logging_(false) + is_no_logging_(false), + max_batch_size_(0) { } @@ -73,11 +77,13 @@ bool ObDirectLoadInsertTableParam::is_valid() const reserved_parallel_ >= 0 && rowkey_column_count_ > 0 && column_count_ > 0 && column_count_ >= rowkey_column_count_ && - lob_column_count_ >= 0 && lob_column_count_ < column_count_ && + lob_inrow_threshold_ >= 0 && (!is_incremental_ || trans_param_.is_valid()) && nullptr != datum_utils_ && nullptr != col_descs_ && col_descs_->count() == column_count_ && - nullptr != cmp_funcs_; + nullptr != cmp_funcs_ && + nullptr != lob_column_idxs_ && + max_batch_size_ > 0; } /** @@ -218,6 +224,7 @@ int ObDirectLoadInsertTabletContext::create_tablet_direct_load() direct_load_param.runtime_only_param_.tx_desc_ = param_->trans_param_.tx_desc_; direct_load_param.runtime_only_param_.trans_id_ = param_->trans_param_.tx_id_; direct_load_param.runtime_only_param_.seq_no_ = param_->trans_param_.tx_seq_.cast_to_int(); + direct_load_param.runtime_only_param_.max_batch_size_ = param_->max_batch_size_; if (OB_FAIL(sstable_insert_mgr->create_tablet_direct_load(context_id_, context_id_ /*execution_id*/, direct_load_param))) { @@ -397,7 +404,31 @@ int ObDirectLoadInsertTabletContext::refresh_pk_cache(const ObTabletID &tablet_i return ret; } -int ObDirectLoadInsertTabletContext::init_datum_row(ObDatumRow &datum_row) +int ObDirectLoadInsertTabletContext::get_row_info(ObDirectLoadInsertTableRowInfo &row_info, const bool is_delete) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableContext not init", K(ret), KP(this)); + } else { + row_info.row_flag_.set_flag( + is_delete ? ObDmlFlag::DF_DELETE : ObDmlFlag::DF_INSERT, + // 只有增量inc_replace模式下的主表insert行需要DF_TYPE_INSERT_DELETE + // * 目前这里没有细分增量inc和增量inc_replace + // * 增量inc_replace带索引或lob会退化成增量inc + (!param_->is_incremental_ || is_delete) ? DF_TYPE_NORMAL : DF_TYPE_INSERT_DELETE); + row_info.mvcc_row_flag_.set_compacted_multi_version_row(true); + row_info.mvcc_row_flag_.set_first_multi_version_row(true); + row_info.mvcc_row_flag_.set_last_multi_version_row(true); + row_info.mvcc_row_flag_.set_uncommitted_row(param_->is_incremental_); + row_info.trans_version_ = !param_->is_incremental_ ? param_->snapshot_version_ : INT64_MAX; + row_info.trans_id_ = param_->trans_param_.tx_id_; + row_info.seq_no_ = param_->trans_param_.tx_seq_.cast_to_int(); + } + return ret; +} + +int ObDirectLoadInsertTabletContext::init_datum_row(ObDatumRow &datum_row, const bool is_delete) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -412,7 +443,12 @@ int ObDirectLoadInsertTabletContext::init_datum_row(ObDatumRow &datum_row) const int64_t trans_version = !param_->is_incremental_ ? param_->snapshot_version_ : INT64_MAX; datum_row.trans_id_ = param_->trans_param_.tx_id_; - datum_row.row_flag_.set_flag(ObDmlFlag::DF_INSERT, !param_->is_incremental_ ? DF_TYPE_NORMAL : DF_TYPE_INSERT_DELETE); + datum_row.row_flag_.set_flag( + is_delete ? ObDmlFlag::DF_DELETE : ObDmlFlag::DF_INSERT, + // 只有增量inc_replace模式下的主表insert行需要DF_TYPE_INSERT_DELETE + // * 目前这里没有细分增量inc和增量inc_replace + // * 增量inc_replace带索引或lob会退化成增量inc + (!param_->is_incremental_ || is_delete) ? DF_TYPE_NORMAL : DF_TYPE_INSERT_DELETE); datum_row.mvcc_row_flag_.set_compacted_multi_version_row(true); datum_row.mvcc_row_flag_.set_first_multi_version_row(true); datum_row.mvcc_row_flag_.set_last_multi_version_row(true); @@ -485,6 +521,32 @@ int ObDirectLoadInsertTabletContext::fill_sstable_slice(const int64_t &slice_id, return ret; } +int ObDirectLoadInsertTabletContext::fill_sstable_slice(const int64_t &slice_id, + const ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableContext not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(is_cancel_)) { + ret = OB_CANCELED; + LOG_WARN("task is cancel", KR(ret)); + } else { + ObDirectLoadInsertTabletContext *tablet_ctx = nullptr; + ObDirectLoadSliceInfo slice_info; + slice_info.is_full_direct_load_ = !param_->is_incremental_; + slice_info.is_lob_slice_ = false; + slice_info.ls_id_ = ls_id_; + slice_info.data_tablet_id_ = tablet_id_; + slice_info.slice_id_ = slice_id; + slice_info.context_id_ = context_id_; + if (OB_FAIL(ddl_agent_.fill_sstable_slice(slice_info, datum_rows))) { + LOG_WARN("fail to fill sstable slice", KR(ret)); + } + } + return ret; +} + int ObDirectLoadInsertTabletContext::fill_lob_meta_sstable_slice(const int64_t &lob_slice_id, ObIStoreRowIterator &iter, int64_t &affected_rows) @@ -548,6 +610,36 @@ int ObDirectLoadInsertTabletContext::fill_lob_sstable_slice(ObIAllocator &alloca return ret; } +int ObDirectLoadInsertTabletContext::fill_lob_sstable_slice(ObIAllocator &allocator, + const int64_t &lob_slice_id, + ObTabletCacheInterval &pk_interval, + ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableContext not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(is_cancel_)) { + ret = OB_CANCELED; + LOG_WARN("task is cancel", KR(ret)); + } else if (OB_UNLIKELY(!is_open_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected not open", KR(ret)); + } else { + ObDirectLoadSliceInfo slice_info; + slice_info.is_full_direct_load_ = !param_->is_incremental_; + slice_info.is_lob_slice_ = true; + slice_info.ls_id_ = ls_id_; + slice_info.data_tablet_id_ = tablet_id_; + slice_info.slice_id_ = lob_slice_id; + slice_info.context_id_ = context_id_; + if (OB_FAIL(ddl_agent_.fill_lob_sstable_slice(allocator, slice_info, pk_interval, datum_rows))) { + LOG_WARN("fail to fill sstable slice", KR(ret), K(slice_info)); + } + } + return ret; +} + int ObDirectLoadInsertTabletContext::open_sstable_slice(const ObMacroDataSeq &start_seq, int64_t &slice_id) { @@ -970,7 +1062,170 @@ int ObDirectLoadInsertTableContext::update_sql_statistics(ObTableLoadSqlStatisti } else if (OB_FAIL(col_stat->update_column_stat_info(&datum, col_desc.col_type_, cmp_func.cmp_func_))) { - LOG_WARN("fail to merge obj", KR(ret), KP(col_stat)); + LOG_WARN("fail to merge obj", KR(ret), K(i), K(col_desc), K(datum), KP(col_stat)); + } + } + } + } + } + return ret; +} + +int ObDirectLoadInsertTableContext::update_sql_statistics(ObTableLoadSqlStatistics &sql_statistics, + const ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + const int64_t extra_rowkey_cnt = ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableContext not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(!param_.online_opt_stat_gather_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected not gather sql stat", KR(ret), K(param_)); + } else if (OB_UNLIKELY(datum_rows.get_column_count() != param_.column_count_ + extra_rowkey_cnt)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid datum row", KR(ret), K(param_)); + } else { + bool ignore = false; + ObOptOSGColumnStat *col_stat = nullptr; + ObDatum datum; + for (int64_t row_idx = 0; OB_SUCC(ret) && row_idx < datum_rows.row_count_; ++row_idx) { + if (OB_FAIL(sql_statistics.get_sample_helper().sample_row(ignore))) { + LOG_WARN("failed to sample row", KR(ret)); + } else if (ignore) { + // do nothing + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < param_.column_count_; ++i) { + if (i < param_.rowkey_column_count_ && param_.is_heap_table_) { + // ignore heap table hidden pk + } else { + const int64_t datum_idx = i < param_.rowkey_column_count_ ? i : i + extra_rowkey_cnt; + const int64_t col_stat_idx = param_.is_heap_table_ ? i - 1 : i; + const ObCmpFunc &cmp_func = param_.cmp_funcs_->at(i).get_cmp_func(); + const ObColDesc &col_desc = param_.col_descs_->at(i); + ObIVector *vector = datum_rows.vectors_.at(datum_idx); + const bool is_valid = + ObColumnStatParam::is_valid_opt_col_type(col_desc.col_type_.get_type(), true); + if (is_valid) { + if (OB_FAIL(sql_statistics.get_col_stat(col_stat_idx, col_stat))) { + LOG_WARN("fail to get col stat", KR(ret), K(col_stat_idx)); + } else if (OB_ISNULL(col_stat)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected col stat is null", KR(ret), K(col_stat_idx)); + } else if (OB_FAIL(ObDirectLoadVectorUtils::to_datum(vector, row_idx, datum))) { + LOG_WARN("fail to get datum", KR(ret)); + } else if (OB_FAIL(col_stat->update_column_stat_info(&datum, + col_desc.col_type_, + cmp_func.cmp_func_))) { + LOG_WARN("fail to merge obj", KR(ret), KP(col_stat)); + } + } + } + } + } + } + } + return ret; +} + +int ObDirectLoadInsertTableContext::update_sql_statistics(ObTableLoadSqlStatistics &sql_statistics, + const ObDatumRow &datum_row, + const ObDirectLoadRowFlag &row_flag) +{ + int ret = OB_SUCCESS; + bool ignore = false; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableContext not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(!param_.online_opt_stat_gather_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected not gather sql stat", KR(ret), K(param_)); + } else if (OB_UNLIKELY(row_flag.get_column_count(datum_row.get_column_count()) != + param_.column_count_ || + (row_flag.uncontain_hidden_pk_ && !param_.is_heap_table_))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid datum row", KR(ret), K(param_), K(datum_row), K(row_flag)); + } else if (OB_FAIL(sql_statistics.get_sample_helper().sample_row(ignore))) { + LOG_WARN("failed to sample row", KR(ret)); + } else if (ignore) { + // do nothing + } else { + ObOptOSGColumnStat *col_stat = nullptr; + for (int64_t i = 0; OB_SUCC(ret) && i < param_.column_count_; ++i) { + if (i < param_.rowkey_column_count_ && param_.is_heap_table_) { + // ignore heap table hidden pk + } else { + const int64_t datum_idx = row_flag.uncontain_hidden_pk_ ? i - 1 : i; + const int64_t col_stat_idx = param_.is_heap_table_ ? i - 1 : i; + const ObStorageDatum &datum = datum_row.storage_datums_[datum_idx]; + const ObCmpFunc &cmp_func = param_.cmp_funcs_->at(i).get_cmp_func(); + const ObColDesc &col_desc = param_.col_descs_->at(i); + const bool is_valid = ObColumnStatParam::is_valid_opt_col_type(col_desc.col_type_.get_type(), true); + if (is_valid) { + if (OB_FAIL(sql_statistics.get_col_stat(col_stat_idx, col_stat))) { + LOG_WARN("fail to get col stat", KR(ret), K(col_stat_idx)); + } else if (OB_ISNULL(col_stat)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected col stat is null", KR(ret), K(col_stat_idx)); + } else if (OB_FAIL(col_stat->update_column_stat_info(&datum, + col_desc.col_type_, + cmp_func.cmp_func_))) { + LOG_WARN("fail to merge obj", KR(ret), K(i), K(col_desc), K(datum), KP(col_stat)); + } + } + } + } + } + return ret; +} + +int ObDirectLoadInsertTableContext::update_sql_statistics(ObTableLoadSqlStatistics &sql_statistics, + const IVectorPtrs &vectors, + const int64_t row_idx, + const ObDirectLoadRowFlag &row_flag) +{ + int ret = OB_SUCCESS; + bool ignore = false; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableContext not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(!param_.online_opt_stat_gather_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected not gather sql stat", KR(ret), K(param_)); + } else if (OB_UNLIKELY(row_flag.get_column_count(vectors.count()) != param_.column_count_ || + row_idx < 0 || + (row_flag.uncontain_hidden_pk_ && !param_.is_heap_table_))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid datum row", KR(ret), K(param_), K(vectors.count()), K(row_idx), K(row_flag)); + } else if (OB_FAIL(sql_statistics.get_sample_helper().sample_row(ignore))) { + LOG_WARN("failed to sample row", KR(ret)); + } else if (ignore) { + // do nothing + } else { + ObOptOSGColumnStat *col_stat = nullptr; + ObDatum datum; + for (int64_t i = 0; OB_SUCC(ret) && i < param_.column_count_; ++i) { + if (i < param_.rowkey_column_count_ && param_.is_heap_table_) { + // ignore heap table hidden pk + } else { + const int64_t datum_idx = row_flag.uncontain_hidden_pk_ ? i - 1 : i; + const int64_t col_stat_idx = param_.is_heap_table_ ? i - 1 : i; + const ObCmpFunc &cmp_func = param_.cmp_funcs_->at(i).get_cmp_func(); + const ObColDesc &col_desc = param_.col_descs_->at(i); + ObIVector *vector = vectors.at(datum_idx); + const bool is_valid = ObColumnStatParam::is_valid_opt_col_type(col_desc.col_type_.get_type(), true); + if (is_valid) { + if (OB_FAIL(sql_statistics.get_col_stat(col_stat_idx, col_stat))) { + LOG_WARN("fail to get col stat", KR(ret), K(col_stat_idx)); + } else if (OB_ISNULL(col_stat)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected col stat is null", KR(ret), K(col_stat_idx)); + } else if (OB_FAIL(ObDirectLoadVectorUtils::to_datum(vector, row_idx, datum))) { + LOG_WARN("fail to get datum", KR(ret)); + } else if (OB_FAIL(col_stat->update_column_stat_info(&datum, + col_desc.col_type_, + cmp_func.cmp_func_))) { + LOG_WARN("fail to merge obj", KR(ret), K(i), K(col_desc), K(datum), KP(col_stat)); } } } diff --git a/src/storage/direct_load/ob_direct_load_insert_table_ctx.h b/src/storage/direct_load/ob_direct_load_insert_table_ctx.h index 21625b7166..04c2ce0efa 100644 --- a/src/storage/direct_load/ob_direct_load_insert_table_ctx.h +++ b/src/storage/direct_load/ob_direct_load_insert_table_ctx.h @@ -33,6 +33,38 @@ class ObTableLoadSqlStatistics; namespace storage { class ObDirectLoadInsertTableContext; +class ObDirectLoadRowFlag; + +struct ObDirectLoadInsertTableRowInfo +{ +public: + ObDirectLoadInsertTableRowInfo() + : row_flag_(), + mvcc_row_flag_(), + trans_version_(INT64_MAX), + trans_id_(), + seq_no_(0) + { + } + bool is_valid() const + { + return row_flag_.is_valid() & + mvcc_row_flag_.is_valid() && + ((INT64_MAX == trans_version_ && trans_id_.is_valid() && seq_no_ > 0) || + (INT64_MAX != trans_version_ && !trans_id_.is_valid() && 0 == seq_no_)); + } + TO_STRING_KV(K_(row_flag), + K_(mvcc_row_flag), + K_(trans_version), + K_(trans_id), + K_(seq_no)); +public: + blocksstable::ObDmlRowFlag row_flag_; + blocksstable::ObMultiVersionRowFlag mvcc_row_flag_; + int64_t trans_version_; + transaction::ObTransID trans_id_; + int64_t seq_no_; +}; struct ObDirectLoadInsertTableParam { @@ -49,7 +81,7 @@ public: K_(reserved_parallel), K_(rowkey_column_count), K_(column_count), - K_(lob_column_count), + K_(lob_inrow_threshold), K_(is_partitioned_table), K_(is_heap_table), K_(is_column_store), @@ -59,8 +91,10 @@ public: KP_(datum_utils), KP_(col_descs), KP_(cmp_funcs), + KP_(lob_column_idxs), K_(online_sample_percent), - K_(is_no_logging)); + K_(is_no_logging), + K_(max_batch_size)); public: uint64_t table_id_; // dest_table_id @@ -72,7 +106,7 @@ public: int64_t reserved_parallel_; int64_t rowkey_column_count_; int64_t column_count_; // 不包含多版本列 - int64_t lob_column_count_; + int64_t lob_inrow_threshold_; bool is_partitioned_table_; bool is_heap_table_; bool is_column_store_; @@ -82,8 +116,10 @@ public: const blocksstable::ObStorageDatumUtils *datum_utils_; const common::ObIArray *col_descs_; const blocksstable::ObStoreCmpFuncs *cmp_funcs_; + const common::ObIArray *lob_column_idxs_; // 不包含多版本列 double online_sample_percent_; bool is_no_logging_; + int64_t max_batch_size_; }; struct ObDirectLoadInsertTabletWriteCtx @@ -123,7 +159,7 @@ public: DEFINE_INSERT_TABLE_PARAM_GETTER(int64_t, reserved_parallel, 0); DEFINE_INSERT_TABLE_PARAM_GETTER(int64_t, rowkey_column_count, 0); DEFINE_INSERT_TABLE_PARAM_GETTER(int64_t, column_count, 0); - DEFINE_INSERT_TABLE_PARAM_GETTER(int64_t, lob_column_count, 0); + DEFINE_INSERT_TABLE_PARAM_GETTER(int64_t, lob_inrow_threshold, -1); DEFINE_INSERT_TABLE_PARAM_GETTER(bool, is_partitioned_table, false); DEFINE_INSERT_TABLE_PARAM_GETTER(bool, is_heap_table, false); DEFINE_INSERT_TABLE_PARAM_GETTER(bool, is_column_store, false); @@ -133,12 +169,15 @@ public: DEFINE_INSERT_TABLE_PARAM_GETTER(const blocksstable::ObStorageDatumUtils *, datum_utils, nullptr); DEFINE_INSERT_TABLE_PARAM_GETTER(const common::ObIArray *, col_descs, nullptr); DEFINE_INSERT_TABLE_PARAM_GETTER(const blocksstable::ObStoreCmpFuncs *, cmp_funcs, nullptr); + DEFINE_INSERT_TABLE_PARAM_GETTER(const common::ObIArray *, lob_column_idxs, nullptr); + DEFINE_INSERT_TABLE_PARAM_GETTER(bool, is_no_logging, false); + DEFINE_INSERT_TABLE_PARAM_GETTER(int64_t, max_batch_size, 0); #undef DEFINE_INSERT_TABLE_PARAM_GETTER - OB_INLINE bool has_lob_storage() const { return get_lob_column_count() > 0; } + OB_INLINE bool has_lob_storage() const { return nullptr != param_ ? !param_->lob_column_idxs_->empty() : false; } OB_INLINE bool need_rescan() const { return nullptr != param_ ? (!param_->is_incremental_ && param_->is_column_store_) : false; } - OB_INLINE bool need_del_lob() const { return nullptr != param_ ? (param_->is_incremental_ && param_->lob_column_count_ > 0) : false; } + OB_INLINE bool need_del_lob() const { return nullptr != param_ ? (param_->is_incremental_ && !param_->lob_column_idxs_->empty()) : false; } const ObLobId &get_min_insert_lob_id() const { return min_insert_lob_id_; } @@ -154,17 +193,26 @@ public: int cancel(); int get_write_ctx(ObDirectLoadInsertTabletWriteCtx &write_ctx); int get_lob_write_ctx(ObDirectLoadInsertTabletWriteCtx &write_ctx); - int init_datum_row(blocksstable::ObDatumRow &datum_row); + int get_row_info(ObDirectLoadInsertTableRowInfo &row_info, const bool is_delete = false); + int init_datum_row(blocksstable::ObDatumRow &datum_row, const bool is_delete = false); int init_lob_datum_row(blocksstable::ObDatumRow &datum_row, const bool is_delete = true); int open_sstable_slice(const blocksstable::ObMacroDataSeq &start_seq, int64_t &slice_id); int close_sstable_slice(const int64_t slice_id); - int fill_sstable_slice(const int64_t &slice_id, ObIStoreRowIterator &iter, + int fill_sstable_slice(const int64_t &slice_id, + ObIStoreRowIterator &iter, int64_t &affected_rows); + int fill_sstable_slice(const int64_t &slice_id, + const blocksstable::ObBatchDatumRows &datum_rows); int open_lob_sstable_slice(const blocksstable::ObMacroDataSeq &start_seq, int64_t &slice_id); int close_lob_sstable_slice(const int64_t slice_id); - int fill_lob_sstable_slice(ObIAllocator &allocator, const int64_t &lob_slice_id, + int fill_lob_sstable_slice(ObIAllocator &allocator, + const int64_t &lob_slice_id, share::ObTabletCacheInterval &pk_interval, blocksstable::ObDatumRow &datum_row); + int fill_lob_sstable_slice(ObIAllocator &allocator, + const int64_t &lob_slice_id, + share::ObTabletCacheInterval &pk_interval, + blocksstable::ObBatchDatumRows &datum_rows); int fill_lob_meta_sstable_slice(const int64_t &lob_slice_id, ObIStoreRowIterator &iter, int64_t &affected_rows); @@ -255,12 +303,24 @@ public: OB_INLINE ObDirectLoadType get_direct_load_type() const { return ddl_ctrl_.direct_load_type_; } OB_INLINE bool need_rescan() const { return (!param_.is_incremental_ && param_.is_column_store_); } - OB_INLINE bool need_del_lob() const { return (param_.is_incremental_ && param_.lob_column_count_ > 0); } + OB_INLINE bool need_del_lob() const { return (param_.is_incremental_ && !param_.lob_column_idxs_->empty()); } OB_INLINE TABLET_CTX_MAP &get_tablet_ctx_map() { return tablet_ctx_map_; } + int64_t get_sql_stat_column_count() const; int get_sql_statistics(table::ObTableLoadSqlStatistics *&sql_statistics); + // 带多版本列的完整行 int update_sql_statistics(table::ObTableLoadSqlStatistics &sql_statistics, const blocksstable::ObDatumRow &datum_row); + int update_sql_statistics(table::ObTableLoadSqlStatistics &sql_statistics, + const blocksstable::ObBatchDatumRows &datum_rows); + // 中间过程数据 + int update_sql_statistics(table::ObTableLoadSqlStatistics &sql_statistics, + const blocksstable::ObDatumRow &datum_row, + const ObDirectLoadRowFlag &row_flag); + int update_sql_statistics(table::ObTableLoadSqlStatistics &sql_statistics, + const IVectorPtrs &vectors, + const int64_t row_idx, + const ObDirectLoadRowFlag &row_flag); TO_STRING_KV(K_(param), K_(ddl_ctrl)); private: diff --git a/src/storage/direct_load/ob_direct_load_insert_table_row_handler.cpp b/src/storage/direct_load/ob_direct_load_insert_table_row_handler.cpp new file mode 100644 index 0000000000..0e60046830 --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_insert_table_row_handler.cpp @@ -0,0 +1,176 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX STORAGE + +#include "storage/direct_load/ob_direct_load_insert_table_row_handler.h" +#include "storage/direct_load/ob_direct_load_insert_table_ctx.h" +#include "storage/direct_load/ob_direct_load_lob_builder.h" + +namespace oceanbase +{ +namespace storage +{ +using namespace blocksstable; + +ObDirectLoadInsertTableRowHandler::ObDirectLoadInsertTableRowHandler() + : insert_tablet_ctx_(nullptr), + sql_statistics_(nullptr), + lob_builder_(nullptr), + is_inited_(false) +{ +} + +ObDirectLoadInsertTableRowHandler::~ObDirectLoadInsertTableRowHandler() { reset(); } + +void ObDirectLoadInsertTableRowHandler::reset() +{ + is_inited_ = false; + insert_tablet_ctx_ = nullptr; + sql_statistics_ = nullptr; + if (nullptr != lob_builder_) { + ObMemAttr mem_attr(MTL_ID(), "TLD_LobBuilder"); + OB_DELETE(ObDirectLoadLobBuilder, mem_attr, lob_builder_); + lob_builder_ = nullptr; + } +} + +int ObDirectLoadInsertTableRowHandler::init(ObDirectLoadInsertTabletContext *insert_tablet_ctx, + ObIAllocator *lob_allocator) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObDirectLoadInsertTableRowHandler init twice", KR(ret), KP(this)); + } else if (OB_UNLIKELY(nullptr == insert_tablet_ctx)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(insert_tablet_ctx)); + } else { + if (insert_tablet_ctx->get_online_opt_stat_gather()) { + if (OB_FAIL(insert_tablet_ctx->get_table_ctx()->get_sql_statistics(sql_statistics_))) { + LOG_WARN("fail to get sql statistics", KR(ret)); + } + } + if (OB_SUCC(ret) && insert_tablet_ctx->has_lob_storage()) { + ObMemAttr mem_attr(MTL_ID(), "TLD_LobBuilder"); + if (OB_ISNULL(lob_builder_ = OB_NEW(ObDirectLoadLobBuilder, mem_attr))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObDirectLoadLobBuilder", KR(ret)); + } else if (OB_FAIL(lob_builder_->init(insert_tablet_ctx, lob_allocator))) { + LOG_WARN("fail to init lob builder", KR(ret)); + } + } + if (OB_SUCC(ret)) { + insert_tablet_ctx_ = insert_tablet_ctx; + is_inited_ = true; + } + } + return ret; +} + +int ObDirectLoadInsertTableRowHandler::handle_row(ObDatumRow &datum_row, const bool skip_lob) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableRowHandler not init", KR(ret), KP(this)); + } else { + if (nullptr != sql_statistics_ && + OB_FAIL(insert_tablet_ctx_->get_table_ctx()->update_sql_statistics(*sql_statistics_, + datum_row))) { + LOG_WARN("fail to update sql statistics", KR(ret)); + } else if (!skip_lob && nullptr != lob_builder_ && + OB_FAIL(lob_builder_->append_lob(datum_row))) { + LOG_WARN("fail to append lob", KR(ret)); + } + } + return ret; +} + +int ObDirectLoadInsertTableRowHandler::handle_batch(ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableRowHandler not init", KR(ret), KP(this)); + } else { + if (nullptr != sql_statistics_ && + OB_FAIL(insert_tablet_ctx_->get_table_ctx()->update_sql_statistics(*sql_statistics_, + datum_rows))) { + LOG_WARN("fail to update sql statistics", KR(ret)); + } else if (nullptr != lob_builder_ && + OB_FAIL(lob_builder_->append_lob(datum_rows))) { + LOG_WARN("fail to append lob", KR(ret)); + } + } + return ret; +} + +int ObDirectLoadInsertTableRowHandler::handle_row(ObDatumRow &datum_row, + const ObDirectLoadRowFlag &row_flag) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableRowHandler not init", KR(ret), KP(this)); + } else { + if (nullptr != sql_statistics_ && + OB_FAIL(insert_tablet_ctx_->get_table_ctx()->update_sql_statistics(*sql_statistics_, + datum_row, + row_flag))) { + LOG_WARN("fail to update sql statistics", KR(ret)); + } else if (nullptr != lob_builder_ && + OB_FAIL(lob_builder_->append_lob(datum_row, row_flag))) { + LOG_WARN("fail to append lob", KR(ret)); + } + } + return ret; +} + +int ObDirectLoadInsertTableRowHandler::handle_row(const IVectorPtrs &vectors, + const int64_t row_idx, + const ObDirectLoadRowFlag &row_flag) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableRowHandler not init", KR(ret), KP(this)); + } else { + if (nullptr != sql_statistics_ && + OB_FAIL(insert_tablet_ctx_->get_table_ctx()->update_sql_statistics(*sql_statistics_, + vectors, + row_idx, + row_flag))) { + LOG_WARN("fail to update sql statistics", KR(ret)); + } else if (nullptr != lob_builder_ && + OB_FAIL(lob_builder_->append_lob(vectors, row_idx, row_flag))) { + LOG_WARN("fail to append lob", KR(ret)); + } + } + return ret; +} + +int ObDirectLoadInsertTableRowHandler::close() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableRowHandler not init", KR(ret), KP(this)); + } else { + if (nullptr != lob_builder_ && OB_FAIL(lob_builder_->close())) { + LOG_WARN("fail to close lob", KR(ret)); + } + } + return ret; +} + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_insert_table_row_handler.h b/src/storage/direct_load/ob_direct_load_insert_table_row_handler.h new file mode 100644 index 0000000000..e45f848388 --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_insert_table_row_handler.h @@ -0,0 +1,62 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once + +#include "lib/allocator/page_arena.h" +#include "share/vector/ob_i_vector.h" + +namespace oceanbase +{ +namespace table +{ +class ObTableLoadSqlStatistics; +} // namespace table +namespace blocksstable +{ +class ObDatumRow; +class ObBatchDatumRows; +} // namespace blocksstable +namespace storage +{ +class ObDirectLoadInsertTabletContext; +class ObDirectLoadLobBuilder; +class ObDirectLoadRowFlag; + +class ObDirectLoadInsertTableRowHandler +{ +public: + ObDirectLoadInsertTableRowHandler(); + ~ObDirectLoadInsertTableRowHandler(); + void reset(); + int init(ObDirectLoadInsertTabletContext *insert_tablet_ctx, + common::ObIAllocator *lob_allocator = nullptr); + // 包含多版本列的完整行 + int handle_row(blocksstable::ObDatumRow &datum_row, const bool skip_lob); + int handle_batch(blocksstable::ObBatchDatumRows &datum_rows); + // 中间过程数据 + int handle_row(blocksstable::ObDatumRow &datum_row, + const ObDirectLoadRowFlag &row_flag); + int handle_row(const IVectorPtrs &vectors, + const int64_t row_idx, + const ObDirectLoadRowFlag &row_flag); + int close(); + +private: + ObDirectLoadInsertTabletContext *insert_tablet_ctx_; + table::ObTableLoadSqlStatistics *sql_statistics_; + ObDirectLoadLobBuilder *lob_builder_; + bool is_inited_; +}; + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_insert_table_row_iterator.cpp b/src/storage/direct_load/ob_direct_load_insert_table_row_iterator.cpp index 67ac2fab1d..018bcd3be8 100644 --- a/src/storage/direct_load/ob_direct_load_insert_table_row_iterator.cpp +++ b/src/storage/direct_load/ob_direct_load_insert_table_row_iterator.cpp @@ -13,7 +13,10 @@ #define USING_LOG_PREFIX STORAGE #include "storage/direct_load/ob_direct_load_insert_table_row_iterator.h" -#include "storage/direct_load/ob_direct_load_lob_builder.h" +#include "sql/engine/cmd/ob_load_data_utils.h" +#include "storage/direct_load/ob_direct_load_insert_table_ctx.h" +#include "storage/direct_load/ob_direct_load_row_iterator.h" +#include "storage/direct_load/ob_direct_load_dml_row_handler.h" namespace oceanbase { @@ -22,36 +25,65 @@ namespace storage using namespace common; using namespace blocksstable; using namespace table; - -/** - * ObDirectLoadInsertTableRowIterator - */ +using namespace sql; ObDirectLoadInsertTableRowIterator::ObDirectLoadInsertTableRowIterator() : insert_tablet_ctx_(nullptr), - sql_statistics_(nullptr), - lob_builder_(nullptr), - lob_allocator_(ObModIds::OB_LOB_ACCESS_BUFFER, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), + row_iters_(nullptr), + dml_row_handler_(nullptr), + job_stat_(nullptr), + rowkey_column_count_(0), + column_count_(0), + pos_(0), + row_count_(0), is_inited_(false) { } ObDirectLoadInsertTableRowIterator::~ObDirectLoadInsertTableRowIterator() {} -int ObDirectLoadInsertTableRowIterator::inner_init( +int ObDirectLoadInsertTableRowIterator::init( ObDirectLoadInsertTabletContext *insert_tablet_ctx, - ObTableLoadSqlStatistics *sql_statistics, - ObDirectLoadLobBuilder &lob_builder) + const ObIArray &row_iters, + ObDirectLoadDMLRowHandler *dml_row_handler, + ObLoadDataStat *job_stat) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(nullptr == insert_tablet_ctx || !insert_tablet_ctx->is_valid() || - (insert_tablet_ctx->get_online_opt_stat_gather() && nullptr == sql_statistics))) { + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObDirectLoadInsertTableRowIterator init twice", KR(ret), KP(this)); + } else if (OB_UNLIKELY(nullptr == insert_tablet_ctx || nullptr == job_stat)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), KPC(insert_tablet_ctx), KP(sql_statistics)); + LOG_WARN("invalid args", KR(ret), KPC(insert_tablet_ctx), KP(job_stat)); } else { - insert_tablet_ctx_ = insert_tablet_ctx; - sql_statistics_ = sql_statistics; - lob_builder_ = &lob_builder; + // check row iters + for (int64_t i = 0; OB_SUCC(ret) && i < row_iters.count(); ++i) { + ObDirectLoadIStoreRowIterator *row_iter = row_iters.at(i); + if (OB_ISNULL(row_iter)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected row iter is null", KR(ret), K(i), K(row_iters)); + } else if (OB_UNLIKELY(!row_iter->is_valid() || row_iter->get_row_flag().get_column_count( + row_iter->get_column_count()) != + insert_tablet_ctx->get_column_count())) { + LOG_WARN("unexpected row iter", KR(ret), KPC(row_iter)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(row_handler_.init(insert_tablet_ctx))) { + LOG_WARN("fail to init row handler", KR(ret)); + } else if (OB_FAIL(insert_tablet_ctx->init_datum_row(insert_datum_row_))) { + LOG_WARN("fail to init datum row", KR(ret)); + } else if (OB_FAIL(insert_tablet_ctx->init_datum_row(delete_datum_row_, true /*is_delete*/))) { + LOG_WARN("fail to init datum row", KR(ret)); + } else { + insert_tablet_ctx_ = insert_tablet_ctx; + dml_row_handler_ = dml_row_handler; + row_iters_ = &row_iters; + job_stat_ = job_stat; + rowkey_column_count_ = insert_tablet_ctx->get_rowkey_column_count(); + column_count_ = insert_tablet_ctx->get_column_count(); + is_inited_ = true; + } } return ret; } @@ -60,13 +92,14 @@ int ObDirectLoadInsertTableRowIterator::get_next_row(const ObDatumRow *&result_r { int ret = OB_SUCCESS; ret = get_next_row(false, result_row); - if (ret != OB_ITER_END && ret != OB_SUCCESS) { + if (OB_UNLIKELY(ret != OB_ITER_END && ret != OB_SUCCESS)) { LOG_WARN("fail to get next row", KR(ret)); } return ret; } -int ObDirectLoadInsertTableRowIterator::get_next_row(const bool skip_lob, const blocksstable::ObDatumRow *&result_row) +int ObDirectLoadInsertTableRowIterator::get_next_row(const bool skip_lob, + const ObDatumRow *&result_row) { int ret = OB_SUCCESS; result_row = nullptr; @@ -77,27 +110,88 @@ int ObDirectLoadInsertTableRowIterator::get_next_row(const bool skip_lob, const ObDatumRow *datum_row = nullptr; if (OB_FAIL(inner_get_next_row(datum_row))) { if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("fail to do inner get next row", KR(ret)); + LOG_WARN("fail to get next row", KR(ret)); } - } else if (insert_tablet_ctx_->get_online_opt_stat_gather() && - OB_FAIL(insert_tablet_ctx_->get_table_ctx()->update_sql_statistics(*sql_statistics_, - *datum_row))) { - LOG_WARN("fail to update sql statistics", KR(ret)); - } else if ((insert_tablet_ctx_->has_lob_storage() && !skip_lob) && OB_FAIL(handle_lob(*datum_row))) { - LOG_WARN("fail to handle lob", KR(ret)); + } else if (OB_FAIL(row_handler_.handle_row(*datum_row, skip_lob))) { + LOG_WARN("fail to handle row", KR(ret)); } else { result_row = datum_row; + ++row_count_; + ATOMIC_AAF(&job_stat_->store_.merge_stage_write_rows_, 1); } } return ret; } -int ObDirectLoadInsertTableRowIterator::handle_lob(ObDatumRow &datum_row) +int ObDirectLoadInsertTableRowIterator::inner_get_next_row(ObDatumRow *&result_row) { int ret = OB_SUCCESS; - lob_allocator_.reuse(); - if (OB_FAIL(lob_builder_->append_lob(lob_allocator_, datum_row))) { - LOG_WARN("fail to append lob", KR(ret), K(datum_row)); + result_row = nullptr; + const ObDatumRow *datum_row = nullptr; + while (OB_SUCC(ret) && nullptr == result_row) { + if (pos_ >= row_iters_->count()) { + ret = OB_ITER_END; + } else { + ObDirectLoadIStoreRowIterator *row_iter = row_iters_->at(pos_); + if (OB_FAIL(row_iter->get_next_row(datum_row))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail to get next row", KR(ret)); + } else { + ret = OB_SUCCESS; + ++pos_; // switch next row iter + } + } else if (OB_ISNULL(datum_row)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected datum row is null", KR(ret)); + } else { + ObDatumRow *datum_row2 = + datum_row->row_flag_.is_delete() ? &delete_datum_row_ : &insert_datum_row_; + // rowkey cols + if (row_iter->get_row_flag().uncontain_hidden_pk_) { + uint64_t pk_seq = OB_INVALID_ID; + if (OB_FAIL(row_iter->get_hide_pk_interval()->next_value(pk_seq))) { + LOG_WARN("fail to get next pk seq", KR(ret)); + } else { + datum_row2->storage_datums_[0].set_int(pk_seq); + } + } else { + for (int64_t i = 0; i < rowkey_column_count_; ++i) { + datum_row2->storage_datums_[i] = datum_row->storage_datums_[i]; + } + } + // normal cols + if (OB_SUCC(ret) && !datum_row->row_flag_.is_delete()) { + for (int64_t + i = row_iter->get_row_flag().uncontain_hidden_pk_ ? 0 : rowkey_column_count_, + j = rowkey_column_count_ + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); + i < datum_row->count_; ++i, ++j) { + datum_row2->storage_datums_[j] = datum_row->storage_datums_[i]; + } + } + if (OB_SUCC(ret) && nullptr != dml_row_handler_) { + if (OB_FAIL(dml_row_handler_->handle_insert_row_with_multi_version(insert_tablet_ctx_->get_tablet_id(), *datum_row2))) { + LOG_WARN("fail to handle insert row", KR(ret), KPC(datum_row2)); + } + } + if (OB_SUCC(ret)) { + result_row = datum_row2; + } + } + } + } + return ret; +} + +int ObDirectLoadInsertTableRowIterator::close() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableRowIterator not init", KR(ret), KP(this)); + } else if (OB_FAIL(row_handler_.close())) { + LOG_WARN("fail to close row handler", KR(ret)); + } else { + insert_tablet_ctx_->inc_row_count(row_count_); } return ret; } diff --git a/src/storage/direct_load/ob_direct_load_insert_table_row_iterator.h b/src/storage/direct_load/ob_direct_load_insert_table_row_iterator.h index 2f0569fbcf..8ff3a4c9a2 100644 --- a/src/storage/direct_load/ob_direct_load_insert_table_row_iterator.h +++ b/src/storage/direct_load/ob_direct_load_insert_table_row_iterator.h @@ -12,10 +12,16 @@ #pragma once -#include "storage/access/ob_store_row_iterator.h" +#include "storage/blocksstable/ob_datum_row.h" +#include "storage/ddl/ob_ddl_struct.h" +#include "storage/direct_load/ob_direct_load_insert_table_row_handler.h" namespace oceanbase { +namespace sql +{ +class ObLoadDataStat; +} // namespace sql namespace table { class ObTableLoadSqlStatistics; @@ -23,27 +29,36 @@ class ObTableLoadSqlStatistics; namespace storage { class ObDirectLoadInsertTabletContext; -class ObDirectLoadLobBuilder; +class ObDirectLoadIStoreRowIterator; +class ObDirectLoadDMLRowHandler; -class ObDirectLoadInsertTableRowIterator : public ObIDirectLoadRowIterator +class ObDirectLoadInsertTableRowIterator final : public ObIDirectLoadRowIterator { public: ObDirectLoadInsertTableRowIterator(); virtual ~ObDirectLoadInsertTableRowIterator(); + int init(ObDirectLoadInsertTabletContext *insert_tablet_ctx, + const ObIArray &row_iters, + ObDirectLoadDMLRowHandler *dml_row_handler, + sql::ObLoadDataStat *job_stat); int get_next_row(const blocksstable::ObDatumRow *&datum_row) override; int get_next_row(const bool skip_lob, const blocksstable::ObDatumRow *&row) override; -protected: - int inner_init(ObDirectLoadInsertTabletContext *insert_tablet_ctx, - table::ObTableLoadSqlStatistics *sql_statistics, - ObDirectLoadLobBuilder &lob_builder); - virtual int inner_get_next_row(blocksstable::ObDatumRow *&datum_row) = 0; + int close(); private: - int handle_lob(blocksstable::ObDatumRow &datum_row); + int inner_get_next_row(blocksstable::ObDatumRow *&datum_row); + protected: ObDirectLoadInsertTabletContext *insert_tablet_ctx_; - table::ObTableLoadSqlStatistics *sql_statistics_; - ObDirectLoadLobBuilder *lob_builder_; - common::ObArenaAllocator lob_allocator_; + const ObIArray *row_iters_; + ObDirectLoadDMLRowHandler *dml_row_handler_; + sql::ObLoadDataStat *job_stat_; + ObDirectLoadInsertTableRowHandler row_handler_; + blocksstable::ObDatumRow insert_datum_row_; + blocksstable::ObDatumRow delete_datum_row_; + int64_t rowkey_column_count_; + int64_t column_count_; + int64_t pos_; + int64_t row_count_; bool is_inited_; }; diff --git a/src/storage/direct_load/ob_direct_load_insert_table_row_writer.cpp b/src/storage/direct_load/ob_direct_load_insert_table_row_writer.cpp new file mode 100644 index 0000000000..17883938c2 --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_insert_table_row_writer.cpp @@ -0,0 +1,456 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX STORAGE + +#include "storage/direct_load/ob_direct_load_insert_table_row_writer.h" +#include "sql/engine/cmd/ob_load_data_utils.h" +#include "storage/blocksstable/ob_datum_row.h" +#include "storage/direct_load/ob_direct_load_dml_row_handler.h" +#include "storage/direct_load/ob_direct_load_row_iterator.h" +#include "storage/direct_load/ob_direct_load_vector_utils.h" + +namespace oceanbase +{ +namespace storage +{ +using namespace blocksstable; +using namespace share::schema; +using namespace sql; + +/** + * ObDirectLoadInsertTableBatchRowBufferWriter + */ + +ObDirectLoadInsertTableBatchRowBufferWriter::ObDirectLoadInsertTableBatchRowBufferWriter() + : allocator_("TLD_BatchBW"), + insert_tablet_ctx_(nullptr), + row_handler_(), + buffer_(), + datum_rows_(), + tablet_id_(), + slice_id_(0), + row_count_(0), + is_canceled_(false), + is_inited_(false) +{ + allocator_.set_tenant_id(MTL_ID()); +} + +ObDirectLoadInsertTableBatchRowBufferWriter::~ObDirectLoadInsertTableBatchRowBufferWriter() {} + +int ObDirectLoadInsertTableBatchRowBufferWriter::inner_init( + ObDirectLoadInsertTabletContext *insert_tablet_ctx, + const ObDirectLoadInsertTableRowInfo &row_info, + ObIAllocator *lob_allocator) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == insert_tablet_ctx || !row_info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(insert_tablet_ctx), K(row_info)); + } else { + const int64_t multi_version_col_cnt = ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); + const int64_t rowkey_column_count = insert_tablet_ctx->get_rowkey_column_count(); + const int64_t column_count = insert_tablet_ctx->get_column_count(); + const int64_t max_batch_size = insert_tablet_ctx->get_max_batch_size(); + const ObIArray *col_descs = insert_tablet_ctx->get_col_descs(); + ObIVector *trans_version_vector = nullptr; + ObIVector *seq_no_vector = nullptr; + if (OB_UNLIKELY(col_descs->count() != column_count)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected column count", KR(ret), KPC(col_descs), K(column_count)); + } else if (OB_FAIL(buffer_.init(*col_descs, max_batch_size))) { + LOG_WARN("fail to init buffer", KR(ret)); + } else if (OB_FAIL(ObDirectLoadVectorUtils::make_const_multi_version_vector( + -row_info.trans_version_, allocator_, trans_version_vector))) { + LOG_WARN("fail to make trans version vector", KR(ret)); + } else if (OB_FAIL(ObDirectLoadVectorUtils::make_const_multi_version_vector( + -row_info.seq_no_, allocator_, seq_no_vector))) { + LOG_WARN("fail to make seq no vector", KR(ret)); + } else if (OB_FAIL( + datum_rows_.vectors_.prepare_allocate(column_count + multi_version_col_cnt))) { + LOG_WARN("fail to prepare allocate", KR(ret)); + } else { + datum_rows_.row_flag_ = row_info.row_flag_; + datum_rows_.mvcc_row_flag_ = row_info.mvcc_row_flag_; + datum_rows_.trans_id_ = row_info.trans_id_; + const IVectorPtrs &vectors = buffer_.get_vectors(); + for (int64_t i = 0; i < vectors.count(); ++i) { + if (i < rowkey_column_count) { + datum_rows_.vectors_.at(i) = vectors.at(i); + } else { + datum_rows_.vectors_.at(i + multi_version_col_cnt) = vectors.at(i); + } + } + datum_rows_.vectors_.at(rowkey_column_count) = trans_version_vector; + datum_rows_.vectors_.at(rowkey_column_count + 1) = seq_no_vector; + } + if (OB_SUCC(ret)) { + if (OB_FAIL(row_handler_.init(insert_tablet_ctx, lob_allocator))) { + LOG_WARN("fail to init row handler", KR(ret)); + } + } + if (OB_SUCC(ret)) { + insert_tablet_ctx_ = insert_tablet_ctx; + tablet_id_ = insert_tablet_ctx->get_tablet_id(); + } + } + return ret; +} + +int ObDirectLoadInsertTableBatchRowBufferWriter::close() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableBatchRowBufferWriter not init", KR(ret), KP(this)); + } else { + if (!buffer_.empty() && OB_FAIL(flush_buffer())) { + LOG_WARN("fail to flush buffer", KR(ret)); + } else if (OB_FAIL(row_handler_.close())) { + LOG_WARN("fail to close row handler", KR(ret)); + } else { + insert_tablet_ctx_->inc_row_count(row_count_); + } + } + return ret; +} + +int ObDirectLoadInsertTableBatchRowBufferWriter::flush_buffer() +{ + int ret = OB_SUCCESS; + datum_rows_.row_count_ = buffer_.get_row_count(); + if (OB_FAIL(flush_batch(datum_rows_))) { + LOG_WARN("fail to flush batch", KR(ret)); + } else { + buffer_.reuse(); + } + return ret; +} + +int ObDirectLoadInsertTableBatchRowBufferWriter::flush_batch(ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(before_flush_batch(datum_rows))) { + LOG_WARN("fail to before flush batch", KR(ret)); + } else if (OB_FAIL(insert_tablet_ctx_->fill_sstable_slice(slice_id_, datum_rows))) { + LOG_WARN("fail to fill sstable slice", KR(ret), K(slice_id_)); + } else if (OB_FAIL(after_flush_batch(datum_rows))) { + LOG_WARN("fail to after flush batch", KR(ret)); + } else { + row_count_ += datum_rows.row_count_; + } + return ret; +} + +/** + * ObDirectLoadInsertTableBatchRowDirectWriter + */ + +int ObDirectLoadInsertTableBatchRowDirectWriter::init( + ObDirectLoadInsertTabletContext *insert_tablet_ctx, + const ObDirectLoadInsertTableRowInfo &row_info, + ObDirectLoadDMLRowHandler *dml_row_handler, + ObIAllocator *lob_allocator) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObDirectLoadInsertTableBatchRowDirectWriter init twice", KR(ret), KP(this)); + } else if (OB_FAIL(inner_init(insert_tablet_ctx, row_info, lob_allocator))) { + LOG_WARN("fail to inner init", KR(ret)); + } else if (OB_UNLIKELY(nullptr == dml_row_handler)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid agrs", KR(ret), KP(dml_row_handler)); + } else if (OB_UNLIKELY(!insert_tablet_ctx->get_is_heap_table())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected not heap table", KR(ret)); + } else if (OB_FAIL(init_sstable_slice())) { + LOG_WARN("fail to init sstable slice", KR(ret)); + } else { + const int64_t multi_version_col_cnt = ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); + const int64_t column_count = insert_tablet_ctx->get_column_count(); + if (OB_FAIL( + direct_datum_rows_.vectors_.prepare_allocate(column_count + multi_version_col_cnt))) { + LOG_WARN("fail to prepare allocate", KR(ret)); + } else { + for (int64_t i = 0; i < multi_version_col_cnt + 1; ++i) { + direct_datum_rows_.vectors_.at(i) = datum_rows_.vectors_.at(i); + } + direct_datum_rows_.row_flag_ = row_info.row_flag_; + direct_datum_rows_.mvcc_row_flag_ = row_info.mvcc_row_flag_; + direct_datum_rows_.trans_id_ = row_info.trans_id_; + dml_row_handler_ = dml_row_handler; + expect_column_count_ = column_count - 1; + row_flag_.uncontain_hidden_pk_ = true; + is_inited_ = true; + } + } + return ret; +} + +int ObDirectLoadInsertTableBatchRowDirectWriter::init_sstable_slice() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(insert_tablet_ctx_->get_write_ctx(write_ctx_))) { + LOG_WARN("fail to get write ctx", KR(ret)); + } else if (OB_FAIL(insert_tablet_ctx_->open_sstable_slice(write_ctx_.start_seq_, slice_id_))) { + LOG_WARN("fail to open sstable slice", KR(ret)); + } + return ret; +} + +int ObDirectLoadInsertTableBatchRowDirectWriter::close_sstable_slice() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(insert_tablet_ctx_->close_sstable_slice(slice_id_))) { + LOG_WARN("fail to close sstable slice", KR(ret)); + } else { + slice_id_ = 0; + } + return ret; +} + +int ObDirectLoadInsertTableBatchRowDirectWriter::switch_sstable_slice() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(close_sstable_slice())) { + LOG_WARN("fail to close sstable slice", KR(ret)); + } else if (OB_FAIL(init_sstable_slice())) { + LOG_WARN("fail to init sstable slice", KR(ret)); + } + return ret; +} + +int ObDirectLoadInsertTableBatchRowDirectWriter::before_flush_batch(ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + if (write_ctx_.pk_interval_.remain_count() < datum_rows.row_count_ && + OB_FAIL(switch_sstable_slice())) { + LOG_WARN("fail to switch sstable slice", KR(ret)); + } else if (OB_FAIL(ObDirectLoadVectorUtils::batch_fill_hidden_pk(datum_rows.vectors_.at(0), + 0 /*start*/, + datum_rows.row_count_, + write_ctx_.pk_interval_))) { + LOG_WARN("fail to batch fill hidden pk", KR(ret)); + } + return ret; +} + +int ObDirectLoadInsertTableBatchRowDirectWriter::after_flush_batch(ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(dml_row_handler_->handle_insert_batch_with_multi_version(tablet_id_, datum_rows))) { + LOG_WARN("fail to handle insert batch", KR(ret)); + } + return ret; +} + +int ObDirectLoadInsertTableBatchRowDirectWriter::append_batch(const IVectorPtrs &vectors, + const int64_t batch_size) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableBatchRowDirectWriter not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(vectors.count() != expect_column_count_ || batch_size <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(expect_column_count_), K(vectors.count()), K(batch_size)); + } else { + const int64_t multi_version_col_cnt = ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); + for (int64_t i = 0; i < vectors.count(); ++i) { + direct_datum_rows_.vectors_.at(i + multi_version_col_cnt + 1) = vectors.at(i); + } + direct_datum_rows_.row_count_ = batch_size; + if (OB_FAIL(row_handler_.handle_batch(direct_datum_rows_))) { + LOG_WARN("fail to handle batch", KR(ret)); + } else if (OB_FAIL(flush_batch(direct_datum_rows_))) { + LOG_WARN("fail to flush batch", KR(ret)); + } + } + return ret; +} + +int ObDirectLoadInsertTableBatchRowDirectWriter::append_row(const IVectorPtrs &vectors, + const int64_t row_idx) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableBatchRowDirectWriter not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(vectors.count() != expect_column_count_ || row_idx < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(expect_column_count_), K(vectors.count()), K(row_idx)); + } else { + bool is_full = false; + // 先处理lob列, 降低内存压力 + if (OB_FAIL(row_handler_.handle_row(vectors, row_idx, row_flag_))) { + LOG_WARN("fail to handle row", KR(ret)); + } else if (OB_FAIL(buffer_.append_row(vectors, row_idx, row_flag_, is_full))) { + LOG_WARN("fail to append row", KR(ret)); + } else if (is_full) { + if (OB_FAIL(flush_buffer())) { + LOG_WARN("fail to flush buffer", KR(ret)); + } + } + } + return ret; +} + +int ObDirectLoadInsertTableBatchRowDirectWriter::append_row(const ObDatumRow &datum_row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableBatchRowDirectWriter not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(datum_row.count_ != expect_column_count_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(expect_column_count_), K(datum_row.count_)); + } else { + bool is_full = false; + // 先处理lob列, 降低内存压力 + if (OB_FAIL(row_handler_.handle_row(const_cast(datum_row), row_flag_))) { + LOG_WARN("fail to handle row", KR(ret)); + } else if (OB_FAIL(buffer_.append_row(datum_row, row_flag_, is_full))) { + LOG_WARN("fail to append row", KR(ret)); + } else if (is_full) { + if (OB_FAIL(flush_buffer())) { + LOG_WARN("fail to flush buffer", KR(ret)); + } + } + } + return ret; +} + +int ObDirectLoadInsertTableBatchRowDirectWriter::close() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableBatchRowDirectWriter not init", KR(ret), KP(this)); + } else if (OB_FAIL(ObDirectLoadInsertTableBatchRowBufferWriter::close())) { + LOG_WARN("fail to close buffer writer", KR(ret)); + } else if (OB_FAIL(close_sstable_slice())) { + LOG_WARN("fail to close sstable slice", KR(ret)); + } + return ret; +} + +/* + * ObDirectLoadInsertTableBatchRowStoreWriter + */ + +int ObDirectLoadInsertTableBatchRowStoreWriter::init( + ObDirectLoadInsertTabletContext *insert_tablet_ctx, + const ObDirectLoadInsertTableRowInfo &row_info, + const int64_t slice_id, + ObDirectLoadDMLRowHandler *dml_row_handler, + ObLoadDataStat *job_stat) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObDirectLoadInsertTableBatchRowStoreWriter init twice", KR(ret), KP(this)); + } else if (OB_FAIL(inner_init(insert_tablet_ctx, row_info))) { + LOG_WARN("fail to inner init", KR(ret)); + } else if (OB_UNLIKELY(slice_id <= 0 || nullptr == job_stat)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(slice_id), KP(job_stat)); + } else { + slice_id_ = slice_id; + dml_row_handler_ = dml_row_handler; + job_stat_ = job_stat; + is_inited_ = true; + } + return ret; +} + +int ObDirectLoadInsertTableBatchRowStoreWriter::after_flush_batch(ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + ATOMIC_AAF(&job_stat_->store_.merge_stage_write_rows_, datum_rows.row_count_); + if (nullptr != dml_row_handler_ && + OB_FAIL(dml_row_handler_->handle_insert_batch_with_multi_version(tablet_id_, datum_rows))) { + LOG_WARN("fail to handle insert batch", KR(ret)); + } + return ret; +} + +int ObDirectLoadInsertTableBatchRowStoreWriter::write(ObDirectLoadIStoreRowIterator *row_iter) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadInsertTableBatchRowStoreWriter not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(nullptr == row_iter)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(row_iter)); + } else { + const ObDirectLoadRowFlag &row_flag = row_iter->get_row_flag(); + ObTabletCacheInterval *hide_pk_interval = row_iter->get_hide_pk_interval(); + int64_t start_pos = buffer_.get_row_count(); + bool is_full = false; + const ObDatumRow *datum_row = nullptr; + if (OB_UNLIKELY(row_flag.uncontain_hidden_pk_ && nullptr == hide_pk_interval)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid row iter", KR(ret), KPC(row_iter)); + } + while (OB_SUCC(ret)) { + if (OB_UNLIKELY(is_canceled_)) { + ret = OB_CANCELED; + LOG_WARN("merge task is canceled", KR(ret)); + } else if (OB_FAIL(row_iter->get_next_row(datum_row))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail to get next row", KR(ret)); + } else { + ret = OB_SUCCESS; + break; + } + } else if (OB_ISNULL(datum_row)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected datum row is null", KR(ret)); + } else if (OB_UNLIKELY(datum_row->row_flag_.is_delete())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected delete row", KR(ret), KPC(datum_row)); + } + // 先处理lob列, 降低内存压力 + else if (OB_FAIL(row_handler_.handle_row(const_cast(*datum_row), row_flag))) { + LOG_WARN("fail to handle row", KR(ret), KPC(datum_row), K(row_flag)); + } else if (OB_FAIL(buffer_.append_row(*datum_row, row_flag, is_full))) { + LOG_WARN("fail to append row", KR(ret)); + } else if (is_full) { + if (row_flag.uncontain_hidden_pk_ && + OB_FAIL(ObDirectLoadVectorUtils::batch_fill_hidden_pk(datum_rows_.vectors_.at(0), + start_pos, + buffer_.get_row_count() - start_pos, + *hide_pk_interval))) { + LOG_WARN("fail to batch fill hidden pk", KR(ret)); + } else if (OB_FAIL(flush_buffer())) { + LOG_WARN("fail to flush buffer", KR(ret)); + } else { + start_pos = 0; + } + } + } + if (OB_SUCC(ret) && !buffer_.empty() && row_flag.uncontain_hidden_pk_) { + if (OB_FAIL(ObDirectLoadVectorUtils::batch_fill_hidden_pk(datum_rows_.vectors_.at(0), + start_pos, + buffer_.get_row_count() - start_pos, + *hide_pk_interval))) { + LOG_WARN("fail to batch fill hidden pk", KR(ret)); + } + } + } + return ret; +} + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_insert_table_row_writer.h b/src/storage/direct_load/ob_direct_load_insert_table_row_writer.h new file mode 100644 index 0000000000..1c3cae92ec --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_insert_table_row_writer.h @@ -0,0 +1,132 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once + +#include "storage/blocksstable/ob_batch_datum_rows.h" +#include "storage/direct_load/ob_direct_load_batch_row_buffer.h" +#include "storage/direct_load/ob_direct_load_insert_table_ctx.h" +#include "storage/direct_load/ob_direct_load_insert_table_row_handler.h" +#include "storage/direct_load/ob_direct_load_row_iterator.h" + +namespace oceanbase +{ +namespace sql +{ +class ObLoadDataStat; +} // namespace sql +namespace blocksstable +{ +class ObDatumRow; +} // namespace blocksstable +namespace storage +{ +class ObDirectLoadDMLRowHandler; + +class ObDirectLoadInsertTableBatchRowBufferWriter +{ +public: + ObDirectLoadInsertTableBatchRowBufferWriter(); + virtual ~ObDirectLoadInsertTableBatchRowBufferWriter(); + virtual int close(); + void cancel() { is_canceled_ = true; } + int64_t get_row_count() const { return row_count_; } + VIRTUAL_TO_STRING_KV(KP_(insert_tablet_ctx), K_(slice_id), K_(row_count), K_(is_canceled), + K_(is_inited)); + +protected: + int inner_init(ObDirectLoadInsertTabletContext *insert_tablet_ctx, + const ObDirectLoadInsertTableRowInfo &row_info, + common::ObIAllocator *lob_allocator = nullptr); + int flush_buffer(); + int flush_batch(blocksstable::ObBatchDatumRows &datum_rows); + virtual int before_flush_batch(blocksstable::ObBatchDatumRows &datum_rows) { return OB_SUCCESS; } + virtual int after_flush_batch(blocksstable::ObBatchDatumRows &datum_rows) { return OB_SUCCESS; } + +protected: + ObArenaAllocator allocator_; + ObDirectLoadInsertTabletContext *insert_tablet_ctx_; + ObDirectLoadInsertTableRowHandler row_handler_; + ObDirectLoadBatchRowBuffer buffer_; + blocksstable::ObBatchDatumRows datum_rows_; + ObTabletID tablet_id_; + int64_t slice_id_; + int64_t row_count_; + bool is_canceled_; + bool is_inited_; +}; + +// only for hidden table +class ObDirectLoadInsertTableBatchRowDirectWriter + : public ObDirectLoadInsertTableBatchRowBufferWriter +{ +public: + ObDirectLoadInsertTableBatchRowDirectWriter() + : ObDirectLoadInsertTableBatchRowBufferWriter(), + dml_row_handler_(nullptr), + write_ctx_(), + direct_datum_rows_(), + expect_column_count_(0), + row_flag_() + { + } + virtual ~ObDirectLoadInsertTableBatchRowDirectWriter() = default; + int init(ObDirectLoadInsertTabletContext *insert_tablet_ctx, + const ObDirectLoadInsertTableRowInfo &row_info, + ObDirectLoadDMLRowHandler *dml_row_handler, + common::ObIAllocator *lob_allocator); + int append_batch(const IVectorPtrs &vectors, const int64_t batch_size); + int append_row(const IVectorPtrs &vectors, const int64_t row_idx); + int append_row(const blocksstable::ObDatumRow &datum_row); + int close() override; + +private: + int init_sstable_slice(); + int close_sstable_slice(); + int switch_sstable_slice(); + int before_flush_batch(blocksstable::ObBatchDatumRows &datum_rows) override; + int after_flush_batch(blocksstable::ObBatchDatumRows &datum_rows) override; + +private: + ObDirectLoadDMLRowHandler *dml_row_handler_; + ObDirectLoadInsertTabletWriteCtx write_ctx_; + blocksstable::ObBatchDatumRows direct_datum_rows_; + int64_t expect_column_count_; + ObDirectLoadRowFlag row_flag_; +}; + +class ObDirectLoadInsertTableBatchRowStoreWriter final + : public ObDirectLoadInsertTableBatchRowBufferWriter +{ +public: + ObDirectLoadInsertTableBatchRowStoreWriter() + : ObDirectLoadInsertTableBatchRowBufferWriter(), dml_row_handler_(nullptr), job_stat_(nullptr) + { + } + virtual ~ObDirectLoadInsertTableBatchRowStoreWriter() = default; + int init(ObDirectLoadInsertTabletContext *insert_tablet_ctx, + const ObDirectLoadInsertTableRowInfo &row_info, + const int64_t slice_id, + ObDirectLoadDMLRowHandler *dml_row_handler, + sql::ObLoadDataStat *job_stat); + int write(ObDirectLoadIStoreRowIterator *row_iter); + +private: + int after_flush_batch(blocksstable::ObBatchDatumRows &datum_rows) override; + +private: + ObDirectLoadDMLRowHandler *dml_row_handler_; + sql::ObLoadDataStat *job_stat_; +}; + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_lob_builder.cpp b/src/storage/direct_load/ob_direct_load_lob_builder.cpp index 7870761529..6cc662c6cf 100644 --- a/src/storage/direct_load/ob_direct_load_lob_builder.cpp +++ b/src/storage/direct_load/ob_direct_load_lob_builder.cpp @@ -13,8 +13,9 @@ #define USING_LOG_PREFIX STORAGE #include "storage/direct_load/ob_direct_load_lob_builder.h" -#include "share/table/ob_table_load_define.h" #include "storage/direct_load/ob_direct_load_insert_table_ctx.h" +#include "storage/direct_load/ob_direct_load_row_iterator.h" +#include "storage/direct_load/ob_direct_load_vector_utils.h" namespace oceanbase { @@ -23,6 +24,7 @@ namespace storage using namespace common; using namespace blocksstable; using namespace share; +using namespace sql; /** * ObDirectLoadLobBuilder @@ -30,16 +32,23 @@ using namespace share; ObDirectLoadLobBuilder::ObDirectLoadLobBuilder() : insert_tablet_ctx_(nullptr), - lob_column_count_(0), + lob_allocator_(nullptr), + inner_lob_allocator_("TLD_LobAlloc"), + lob_column_idxs_(nullptr), + lob_column_cnt_(0), + extra_rowkey_cnt_(0), + lob_inrow_threshold_(0), current_lob_slice_id_(0), is_closed_(false), is_inited_(false) { + inner_lob_allocator_.set_tenant_id(MTL_ID()); } ObDirectLoadLobBuilder::~ObDirectLoadLobBuilder() {} -int ObDirectLoadLobBuilder::init(ObDirectLoadInsertTabletContext *insert_tablet_ctx) +int ObDirectLoadLobBuilder::init(ObDirectLoadInsertTabletContext *insert_tablet_ctx, + ObIAllocator *lob_allocator) { int ret = OB_SUCCESS; if (IS_INIT) { @@ -53,7 +62,11 @@ int ObDirectLoadLobBuilder::init(ObDirectLoadInsertTabletContext *insert_tablet_ LOG_WARN("unexpected has no lob", KR(ret), KPC(insert_tablet_ctx)); } else { insert_tablet_ctx_ = insert_tablet_ctx; - lob_column_count_ = insert_tablet_ctx->get_lob_column_count(); + lob_allocator_ = (nullptr != lob_allocator ? lob_allocator : &inner_lob_allocator_); + lob_column_idxs_ = insert_tablet_ctx->get_lob_column_idxs(); + lob_column_cnt_ = lob_column_idxs_->count(); + extra_rowkey_cnt_ = ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); + lob_inrow_threshold_ = insert_tablet_ctx->get_lob_inrow_threshold(); if (OB_FAIL(init_sstable_slice_ctx())) { LOG_WARN("fail to init sstable slice ctx", KR(ret)); } else { @@ -86,7 +99,150 @@ int ObDirectLoadLobBuilder::switch_sstable_slice() return ret; } -int ObDirectLoadLobBuilder::append_lob(ObIAllocator &allocator, blocksstable::ObDatumRow &datum_row) +int ObDirectLoadLobBuilder::check_can_skip(char *ptr, uint32_t len, bool &can_skip) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == ptr || len <= 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected lob column is empty data", KR(ret), KP(ptr), K(len)); + } else { + ObLobLocatorV2 locator(ptr, len, true /*has_lob_header*/); + can_skip = (locator.is_inrow_disk_lob_locator() && (len - sizeof(ObLobCommon) <= lob_inrow_threshold_)); + } + return ret; +} + +int ObDirectLoadLobBuilder::check_can_skip(const ObDatumRow &datum_row, bool &can_skip) +{ + int ret = OB_SUCCESS; + can_skip = true; + for (int64_t i = 0; OB_SUCC(ret) && can_skip && i < lob_column_idxs_->count(); ++i) { + const int64_t column_idx = lob_column_idxs_->at(i) + extra_rowkey_cnt_; + const ObDatum &datum = datum_row.storage_datums_[column_idx]; + if (datum.is_null()) { + } else if (OB_FAIL(check_can_skip(const_cast(datum.ptr_), datum.len_, can_skip))) { + LOG_WARN("fail to check lob can skip", KR(ret), K(column_idx), K(datum)); + } + } + return ret; +} + +int ObDirectLoadLobBuilder::check_can_skip(const ObBatchDatumRows &datum_rows, bool &can_skip) +{ + int ret = OB_SUCCESS; + can_skip = true; + for (int64_t i = 0; OB_SUCC(ret) && can_skip && i < lob_column_idxs_->count(); ++i) { + const int64_t column_idx = lob_column_idxs_->at(i) + extra_rowkey_cnt_; + ObIVector *vector = datum_rows.vectors_.at(column_idx); + const VectorFormat format = vector->get_format(); + switch (format) { + case VEC_CONTINUOUS: { + ObContinuousBase *continuous_vec = static_cast(vector); + ObBitVector *nulls = continuous_vec->get_nulls(); + char *data = continuous_vec->get_data(); + uint32_t *offsets = continuous_vec->get_offsets(); + if (!nulls->is_all_true(datum_rows.row_count_)) { + for (int64_t j = 0; OB_SUCC(ret) && can_skip && j < datum_rows.row_count_; ++j) { + if (nulls->at(j)) { + } else if (OB_FAIL(check_can_skip(data + offsets[j], offsets[j + 1] - offsets[j], can_skip))) { + LOG_WARN("fail to check lob can skip", KR(ret), K(column_idx), K(j), KP(data), + K(offsets[j]), K(offsets[j + 1])); + } + } + } + break; + } + case VEC_DISCRETE: { + ObDiscreteBase *discrete_vec = static_cast(vector); + ObBitVector *nulls = discrete_vec->get_nulls(); + char **ptrs = discrete_vec->get_ptrs(); + int32_t *lens = discrete_vec->get_lens(); + if (!nulls->is_all_true(datum_rows.row_count_)) { + for (int64_t j = 0; OB_SUCC(ret) && can_skip && j < datum_rows.row_count_; ++j) { + if (nulls->at(j)) { + } else if (OB_FAIL(check_can_skip(ptrs[j], lens[j], can_skip))) { + LOG_WARN("fail to check lob can skip", KR(ret), K(column_idx), K(j), KP(ptrs[j]), + K(lens[j])); + } + } + } + break; + } + case VEC_UNIFORM: { + ObUniformBase *uniform_vec = static_cast(vector); + ObDatum *datums = uniform_vec->get_datums(); + for (int64_t j = 0; OB_SUCC(ret) && can_skip && j < datum_rows.row_count_; ++j) { + const ObDatum &datum = datums[j]; + if (datum.is_null()) { + } else if (OB_FAIL(check_can_skip(const_cast(datum.ptr_), datum.len_, can_skip))) { + LOG_WARN("fail to check lob can skip", KR(ret), K(column_idx), K(j), K(datum)); + } + } + break; + } + case VEC_UNIFORM_CONST: { + ObUniformBase *uniform_vec = static_cast(vector); + const ObDatum &datum = uniform_vec->get_datums()[0]; + if (datum.is_null()) { + } else if (OB_FAIL(check_can_skip(const_cast(datum.ptr_), datum.len_, can_skip))) { + LOG_WARN("fail to check lob can skip", KR(ret), K(column_idx), K(datum)); + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format in lob column", KR(ret), K(column_idx), K(format)); + break; + } + } + return ret; +} + +int ObDirectLoadLobBuilder::append_row(ObDatumRow &datum_row) +{ + int ret = OB_SUCCESS; + bool can_skip = false; + lob_allocator_->reuse(); + if (OB_FAIL(check_can_skip(datum_row, can_skip))) { + LOG_WARN("fail to check can skip", KR(ret), K(datum_row)); + } else if (can_skip) { + // do nothing + } else if (write_ctx_.pk_interval_.remain_count() < lob_column_cnt_ && + OB_FAIL(switch_sstable_slice())) { + LOG_WARN("fail to switch sstable slice", KR(ret)); + } else if (OB_FAIL(insert_tablet_ctx_->fill_lob_sstable_slice(*lob_allocator_, + current_lob_slice_id_, + write_ctx_.pk_interval_, + datum_row))) { + LOG_WARN("fail to fill lob sstable slice", K(ret), KP(insert_tablet_ctx_), + K(current_lob_slice_id_), K(write_ctx_.pk_interval_), K(datum_row)); + } + return ret; +} + +int ObDirectLoadLobBuilder::append_batch(ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + const int64_t lob_cnt = lob_column_cnt_ * datum_rows.row_count_; + bool can_skip = false; + lob_allocator_->reuse(); + if (OB_FAIL(check_can_skip(datum_rows, can_skip))) { + LOG_WARN("fail to check can skip", KR(ret), K(datum_rows)); + } else if (can_skip) { + // do nothing + } else if (write_ctx_.pk_interval_.remain_count() < lob_cnt && OB_FAIL(switch_sstable_slice())) { + LOG_WARN("fail to switch sstable slice", KR(ret)); + } else if (OB_FAIL(insert_tablet_ctx_->fill_lob_sstable_slice(*lob_allocator_, + current_lob_slice_id_, + write_ctx_.pk_interval_, + datum_rows))) { + LOG_WARN("fail to fill lob sstable slice batch", K(ret), KP(insert_tablet_ctx_), + K(current_lob_slice_id_), K(write_ctx_.pk_interval_), K(datum_rows)); + } + return ret; +} + +int ObDirectLoadLobBuilder::append_lob(ObDatumRow &datum_row) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -95,18 +251,133 @@ int ObDirectLoadLobBuilder::append_lob(ObIAllocator &allocator, blocksstable::Ob } else if (OB_UNLIKELY(is_closed_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("lob builder is closed", KR(ret)); + } else if (OB_FAIL(append_row(datum_row))) { + LOG_WARN("fail to append row", KR(ret), K(datum_row)); + } + return ret; +} + +int ObDirectLoadLobBuilder::append_lob(ObBatchDatumRows &datum_rows) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadLobBuilder not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(is_closed_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("lob builder is closed", KR(ret)); + } else if (OB_FAIL(append_batch(datum_rows))) { + LOG_WARN("fail to append batch", KR(ret), K(datum_rows)); + } + return ret; +} + +int ObDirectLoadLobBuilder::fill_into_datum_row(ObDatumRow &datum_row, + const ObDirectLoadRowFlag &row_flag) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; i < lob_column_idxs_->count(); ++i) { + const int64_t column_idx = lob_column_idxs_->at(i); + const int64_t src_column_idx = row_flag.uncontain_hidden_pk_ ? column_idx - 1 : column_idx; + const int64_t dest_column_idx = column_idx + extra_rowkey_cnt_; + datum_row_.storage_datums_[dest_column_idx] = datum_row.storage_datums_[src_column_idx]; + } + return ret; +} + +int ObDirectLoadLobBuilder::fetch_from_datum_row(ObDatumRow &datum_row, + const ObDirectLoadRowFlag &row_flag) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; i < lob_column_idxs_->count(); ++i) { + const int64_t column_idx = lob_column_idxs_->at(i); + const int64_t src_column_idx = row_flag.uncontain_hidden_pk_ ? column_idx - 1 : column_idx; + const int64_t dest_column_idx = column_idx + extra_rowkey_cnt_; + datum_row.storage_datums_[src_column_idx] = datum_row_.storage_datums_[dest_column_idx]; + } + return ret; +} + +int ObDirectLoadLobBuilder::append_lob(ObDatumRow &datum_row, + const ObDirectLoadRowFlag &row_flag) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadLobBuilder not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(is_closed_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("lob builder is closed", KR(ret)); + } else if (!datum_row_.is_valid() && OB_FAIL(insert_tablet_ctx_->init_datum_row(datum_row_))) { + LOG_WARN("fail to init datum row", KR(ret)); } else { - if (write_ctx_.pk_interval_.remain_count() < lob_column_count_) { - if (OB_FAIL(switch_sstable_slice())) { - LOG_WARN("fail to switch sstable slice", KR(ret)); - } + if (OB_FAIL(fill_into_datum_row(datum_row, row_flag))) { + LOG_WARN("fail to fill into datum row", KR(ret)); + } else if (OB_FAIL(append_row(datum_row_))) { + LOG_WARN("fail to append row", KR(ret), K(datum_row_)); + } else if (OB_FAIL(fetch_from_datum_row(datum_row, row_flag))) { + LOG_WARN("fail to fetch from datum row", KR(ret)); } - if (OB_SUCC(ret)) { - if (OB_FAIL(insert_tablet_ctx_->fill_lob_sstable_slice(allocator, current_lob_slice_id_, - write_ctx_.pk_interval_, datum_row))) { - LOG_WARN("fill lob sstable slice failed", K(ret), KP(insert_tablet_ctx_), - K(current_lob_slice_id_), K(write_ctx_.pk_interval_), K(datum_row)); - } + } + return ret; +} + +int ObDirectLoadLobBuilder::fill_into_datum_row(const IVectorPtrs &vectors, + const int64_t row_idx, + const ObDirectLoadRowFlag &row_flag) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < lob_column_idxs_->count(); ++i) { + const int64_t column_idx = lob_column_idxs_->at(i); + const int64_t src_column_idx = row_flag.uncontain_hidden_pk_ ? column_idx - 1 : column_idx; + const int64_t dest_column_idx = column_idx + extra_rowkey_cnt_; + ObIVector *vector = vectors.at(src_column_idx); + ObDatum &datum = datum_row_.storage_datums_[dest_column_idx]; + if (OB_FAIL(ObDirectLoadVectorUtils::to_datum(vector, row_idx, datum))) { + LOG_WARN("fail to get datum", KR(ret)); + } + } + return ret; +} + +int ObDirectLoadLobBuilder::fetch_from_datum_row(const IVectorPtrs &vectors, + const int64_t row_idx, + const ObDirectLoadRowFlag &row_flag) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < lob_column_idxs_->count(); ++i) { + const int64_t column_idx = lob_column_idxs_->at(i); + const int64_t src_column_idx = row_flag.uncontain_hidden_pk_ ? column_idx - 1 : column_idx; + const int64_t dest_column_idx = column_idx + extra_rowkey_cnt_; + ObIVector *vector = vectors.at(src_column_idx); + ObDatum &datum = datum_row_.storage_datums_[dest_column_idx]; + if (OB_FAIL(ObDirectLoadVectorUtils::set_datum(vector, row_idx, datum))) { + LOG_WARN("fail to set datum", KR(ret)); + } + } + return ret; +} + +int ObDirectLoadLobBuilder::append_lob(const IVectorPtrs &vectors, + const int64_t row_idx, + const ObDirectLoadRowFlag &row_flag) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadLobBuilder not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(is_closed_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("lob builder is closed", KR(ret)); + } else if (!datum_row_.is_valid() && OB_FAIL(insert_tablet_ctx_->init_datum_row(datum_row_))) { + LOG_WARN("fail to init datum row", KR(ret)); + } else { + if (OB_FAIL(fill_into_datum_row(vectors, row_idx, row_flag))) { + LOG_WARN("fail to fill into datum row", KR(ret)); + } else if (OB_FAIL(append_row(datum_row_))) { + LOG_WARN("fail to append row", KR(ret), K(datum_row_)); + } else if (OB_FAIL(fetch_from_datum_row(vectors, row_idx, row_flag))) { + LOG_WARN("fail to fetch from datum row", KR(ret)); } } return ret; diff --git a/src/storage/direct_load/ob_direct_load_lob_builder.h b/src/storage/direct_load/ob_direct_load_lob_builder.h index c7546cda5b..5121970ced 100644 --- a/src/storage/direct_load/ob_direct_load_lob_builder.h +++ b/src/storage/direct_load/ob_direct_load_lob_builder.h @@ -12,29 +12,67 @@ #pragma once +#include "storage/blocksstable/ob_datum_row.h" #include "storage/direct_load/ob_direct_load_insert_table_ctx.h" namespace oceanbase { namespace storage { - class ObDirectLoadLobBuilder { public: ObDirectLoadLobBuilder(); ~ObDirectLoadLobBuilder(); - int init(ObDirectLoadInsertTabletContext *insert_tablet_ctx); - int append_lob(common::ObIAllocator &allocator, blocksstable::ObDatumRow &datum_row); + int init(ObDirectLoadInsertTabletContext *insert_tablet_ctx, + common::ObIAllocator *lob_allocator = nullptr); + + // 包含多版本列的完整行 + int append_lob(blocksstable::ObDatumRow &datum_row); + int append_lob(blocksstable::ObBatchDatumRows &datum_rows); + // 中间过程数据 + int append_lob(blocksstable::ObDatumRow &datum_row, + const ObDirectLoadRowFlag &row_flag); + int append_lob(const IVectorPtrs &vectors, + const int64_t row_idx, + const ObDirectLoadRowFlag &row_flag); int close(); + private: int init_sstable_slice_ctx(); int switch_sstable_slice(); + + inline int check_can_skip(char *ptr, uint32_t len, bool &can_skip); + int check_can_skip(const blocksstable::ObDatumRow &datum_row, bool &can_skip); + int check_can_skip(const blocksstable::ObBatchDatumRows &datum_rows, bool &can_skip); + + int append_row(blocksstable::ObDatumRow &datum_row); + int append_batch(blocksstable::ObBatchDatumRows &datum_rows); + + int fill_into_datum_row(blocksstable::ObDatumRow &datum_row, + const ObDirectLoadRowFlag &row_flag); + int fetch_from_datum_row(blocksstable::ObDatumRow &datum_row, + const ObDirectLoadRowFlag &row_flag); + + int fill_into_datum_row(const IVectorPtrs &vectors, + const int64_t row_idx, + const ObDirectLoadRowFlag &row_flag); + int fetch_from_datum_row(const IVectorPtrs &vectors, + const int64_t row_idx, + const ObDirectLoadRowFlag &row_flag); + private: ObDirectLoadInsertTabletContext *insert_tablet_ctx_; - int64_t lob_column_count_; + common::ObIAllocator *lob_allocator_; + common::ObArenaAllocator inner_lob_allocator_; + // 不包含多版本列, 默认lob不会是主键列 + const ObIArray *lob_column_idxs_; + int64_t lob_column_cnt_; + int64_t extra_rowkey_cnt_; + int64_t lob_inrow_threshold_; ObDirectLoadInsertTabletWriteCtx write_ctx_; int64_t current_lob_slice_id_; + blocksstable::ObDatumRow datum_row_; bool is_closed_; bool is_inited_; DISALLOW_COPY_AND_ASSIGN(ObDirectLoadLobBuilder); diff --git a/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.cpp b/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.cpp index c766c10a45..b3d8b475b5 100644 --- a/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.cpp +++ b/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.cpp @@ -52,7 +52,7 @@ ObDirectLoadLobMetaRowIter::ObDirectLoadLobMetaRowIter() ObDirectLoadLobMetaRowIter::~ObDirectLoadLobMetaRowIter() { if (nullptr != origin_iter_) { - origin_iter_->~ObIStoreRowIterator(); + origin_iter_->~ObDirectLoadIStoreRowIterator(); allocator_.free(origin_iter_); origin_iter_ = nullptr; } diff --git a/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.h b/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.h index 656f9ac917..aa77a8321c 100644 --- a/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.h +++ b/src/storage/direct_load/ob_direct_load_lob_meta_row_iter.h @@ -32,6 +32,10 @@ public: { return OB_ERR_UNEXPECTED; } + int handle_insert_batch(const ObTabletID &tablet_id, const blocksstable::ObBatchDatumRows &datum_rows) override + { + return OB_ERR_UNEXPECTED; + } int handle_delete_row(const ObTabletID tablet_id, const blocksstable::ObDatumRow &row) { return OB_ERR_UNEXPECTED; @@ -40,6 +44,10 @@ public: { return OB_ERR_UNEXPECTED; } + int handle_insert_batch_with_multi_version(const ObTabletID &tablet_id, const blocksstable::ObBatchDatumRows &datum_rows) + { + return OB_ERR_UNEXPECTED; + } int handle_update_row(const blocksstable::ObDatumRow &row) override { return OB_ERR_UNEXPECTED; } int handle_update_row(common::ObArray &rows, const ObDirectLoadExternalRow *&row) override @@ -94,7 +102,7 @@ private: ObDirectLoadMultipleDatumRange scan_range_; ObDirectLoadMultipleSSTableScanMerge scan_merge_; ObDirectLoadOriginTable *origin_table_; - ObIStoreRowIterator *origin_iter_; + ObDirectLoadIStoreRowIterator *origin_iter_; common::ObArenaAllocator range_allocator_; blocksstable::ObDatumRange range_; ObDirectLoadLobIdConflictHandler conflict_handler_; diff --git a/src/storage/direct_load/ob_direct_load_merge_ctx.cpp b/src/storage/direct_load/ob_direct_load_merge_ctx.cpp index 8b4dc2f519..05e343b1ce 100644 --- a/src/storage/direct_load/ob_direct_load_merge_ctx.cpp +++ b/src/storage/direct_load/ob_direct_load_merge_ctx.cpp @@ -450,42 +450,61 @@ int ObDirectLoadTabletMergeCtx::build_empty_data_merge_task(const ObIArray max_parallel_degree)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected range count", KR(ret), K(max_parallel_degree), K(range_array_.count())); - } - for (int64_t i = 0; OB_SUCC(ret) && i < range_array_.count(); ++i) { - const ObDatumRange &range = range_array_.at(i); - ObDirectLoadPartitionRangeMergeTask *merge_task = nullptr; - if (OB_ISNULL(merge_task = OB_NEWx(ObDirectLoadPartitionRangeMergeTask, (&allocator_)))) { + if (!merge_with_origin_data()) { + // construct empty task for close tablet + ObDirectLoadPartitionEmptyMergeTask *merge_task = nullptr; + if (OB_ISNULL(merge_task = OB_NEWx(ObDirectLoadPartitionEmptyMergeTask, (&allocator_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObDirectLoadPartitionRangeMergeTask", KR(ret)); - } else if (OB_FAIL(merge_task->init(ctx_, param_, this, &origin_table_, sstable_array_, range, i))) { + LOG_WARN("fail to new ObDirectLoadPartitionEmptyMergeTask", KR(ret)); + } else if (OB_FAIL(merge_task->init(ctx_, param_, this))) { LOG_WARN("fail to init merge task", KR(ret)); } else if (OB_FAIL(task_array_.push_back(merge_task))) { LOG_WARN("fail to push back merge task", KR(ret)); } if (OB_FAIL(ret)) { if (nullptr != merge_task) { - merge_task->~ObDirectLoadPartitionRangeMergeTask(); + merge_task->~ObDirectLoadPartitionEmptyMergeTask(); allocator_.free(merge_task); merge_task = nullptr; } } + } else { + // only origin data, construct task by split range + ObDirectLoadMergeRangeSplitter range_splitter; + if (OB_FAIL(range_splitter.init(tablet_id_, + &origin_table_, + multiple_sstable_array_, + param_.table_data_desc_, + param_.datum_utils_, + col_descs))) { + LOG_WARN("fail to init range splitter", KR(ret)); + } else if (OB_FAIL(range_splitter.split_range(range_array_, max_parallel_degree, allocator_))) { + LOG_WARN("fail to split range", KR(ret)); + } else if (OB_UNLIKELY(range_array_.count() > max_parallel_degree)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected range count", KR(ret), K(max_parallel_degree), K(range_array_.count())); + } + for (int64_t i = 0; OB_SUCC(ret) && i < range_array_.count(); ++i) { + const ObDatumRange &range = range_array_.at(i); + ObDirectLoadPartitionOriginDataMergeTask *merge_task = nullptr; + if (OB_ISNULL(merge_task = + OB_NEWx(ObDirectLoadPartitionOriginDataMergeTask, (&allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObDirectLoadPartitionOriginDataMergeTask", KR(ret)); + } else if (OB_FAIL(merge_task->init(ctx_, param_, this, &origin_table_, range, i))) { + LOG_WARN("fail to init merge task", KR(ret)); + } else if (OB_FAIL(task_array_.push_back(merge_task))) { + LOG_WARN("fail to push back merge task", KR(ret)); + } + if (OB_FAIL(ret)) { + if (nullptr != merge_task) { + merge_task->~ObDirectLoadPartitionOriginDataMergeTask(); + allocator_.free(merge_task); + merge_task = nullptr; + } + } + } } - return ret; } @@ -610,16 +629,15 @@ int ObDirectLoadTabletMergeCtx::build_heap_table_multiple_merge_task( if (OB_FAIL(init_multiple_heap_table_array(table_array))) { LOG_WARN("fail to init multiple heap table array", KR(ret)); } - // for existing data, construct task by split range - if (OB_SUCC(ret)) { + // for origin data, construct task by split range + if (OB_SUCC(ret) && merge_with_origin_data()) { ObDirectLoadMergeRangeSplitter range_splitter; - if (OB_FAIL(range_splitter.init( - tablet_id_, - (merge_with_origin_data() ? &origin_table_ : nullptr), - multiple_sstable_array_, - param_.table_data_desc_, - param_.datum_utils_, - col_descs))) { + if (OB_FAIL(range_splitter.init(tablet_id_, + &origin_table_, + multiple_sstable_array_, + param_.table_data_desc_, + param_.datum_utils_, + col_descs))) { LOG_WARN("fail to init range splitter", KR(ret)); } else if (OB_FAIL(range_splitter.split_range(range_array_, max_parallel_degree, allocator_))) { LOG_WARN("fail to split range", KR(ret)); @@ -627,24 +645,24 @@ int ObDirectLoadTabletMergeCtx::build_heap_table_multiple_merge_task( ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected range count", KR(ret), K(max_parallel_degree), K(range_array_.count())); } - } - for (int64_t i = 0; OB_SUCC(ret) && i < range_array_.count(); ++i) { - const ObDatumRange &range = range_array_.at(i); - ObDirectLoadPartitionRangeMergeTask *merge_task = nullptr; - if (OB_ISNULL(merge_task = OB_NEWx(ObDirectLoadPartitionRangeMergeTask, (&allocator_)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObDirectLoadPartitionRangeMergeTask", KR(ret)); - } else if (OB_FAIL(merge_task->init(ctx_, param_, this, &origin_table_, sstable_array_, range, - parallel_idx++))) { - LOG_WARN("fail to init merge task", KR(ret)); - } else if (OB_FAIL(task_array_.push_back(merge_task))) { - LOG_WARN("fail to push back merge task", KR(ret)); - } - if (OB_FAIL(ret)) { - if (nullptr != merge_task) { - merge_task->~ObDirectLoadPartitionRangeMergeTask(); - allocator_.free(merge_task); - merge_task = nullptr; + for (int64_t i = 0; OB_SUCC(ret) && i < range_array_.count(); ++i) { + const ObDatumRange &range = range_array_.at(i); + ObDirectLoadPartitionOriginDataMergeTask *merge_task = nullptr; + if (OB_ISNULL(merge_task = + OB_NEWx(ObDirectLoadPartitionOriginDataMergeTask, (&allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObDirectLoadPartitionOriginDataMergeTask", KR(ret)); + } else if (OB_FAIL(merge_task->init(ctx_, param_, this, &origin_table_, range, parallel_idx++))) { + LOG_WARN("fail to init merge task", KR(ret)); + } else if (OB_FAIL(task_array_.push_back(merge_task))) { + LOG_WARN("fail to push back merge task", KR(ret)); + } + if (OB_FAIL(ret)) { + if (nullptr != merge_task) { + merge_task->~ObDirectLoadPartitionOriginDataMergeTask(); + allocator_.free(merge_task); + merge_task = nullptr; + } } } } diff --git a/src/storage/direct_load/ob_direct_load_merge_ctx.h b/src/storage/direct_load/ob_direct_load_merge_ctx.h index f2c9433bbd..1672f26f2d 100644 --- a/src/storage/direct_load/ob_direct_load_merge_ctx.h +++ b/src/storage/direct_load/ob_direct_load_merge_ctx.h @@ -19,7 +19,6 @@ #include "storage/direct_load/ob_direct_load_struct.h" #include "storage/direct_load/ob_direct_load_table_data_desc.h" #include "storage/direct_load/ob_direct_load_trans_param.h" -#include "storage/direct_load/ob_direct_load_fast_heap_table.h" #include "observer/table_load/ob_table_load_table_ctx.h" namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_origin_table.cpp b/src/storage/direct_load/ob_direct_load_origin_table.cpp index 030866df1d..78272b5e58 100644 --- a/src/storage/direct_load/ob_direct_load_origin_table.cpp +++ b/src/storage/direct_load/ob_direct_load_origin_table.cpp @@ -164,7 +164,7 @@ int ObDirectLoadOriginTable::prepare_tables() int ObDirectLoadOriginTable::scan( const ObDatumRange &key_range, ObIAllocator &allocator, - ObIStoreRowIterator *&row_iter, + ObDirectLoadIStoreRowIterator *&row_iter, bool skip_read_lob) { int ret = OB_SUCCESS; @@ -260,6 +260,10 @@ int ObDirectLoadOriginTableScanner::init(ObDirectLoadOriginTable *origin_table, scan_merge_.init(table_access_param_, table_access_ctx_, get_table_param_))) { LOG_WARN("fail to init multi merge", KR(ret)); } else { + row_flag_.uncontain_hidden_pk_ = false; + row_flag_.has_multi_version_cols_ = false; + row_flag_.has_delete_row_ = false; + column_count_ = col_ids_.count(); is_inited_ = true; } } diff --git a/src/storage/direct_load/ob_direct_load_origin_table.h b/src/storage/direct_load/ob_direct_load_origin_table.h index 80c9072dac..584cfef827 100644 --- a/src/storage/direct_load/ob_direct_load_origin_table.h +++ b/src/storage/direct_load/ob_direct_load_origin_table.h @@ -13,7 +13,7 @@ #include "share/schema/ob_table_dml_param.h" #include "storage/access/ob_multiple_scan_merge.h" -#include "storage/access/ob_store_row_iterator.h" +#include "storage/direct_load/ob_direct_load_row_iterator.h" #include "storage/direct_load/ob_direct_load_struct.h" namespace oceanbase @@ -67,7 +67,7 @@ public: int scan( const blocksstable::ObDatumRange &key_range, common::ObIAllocator &allocator, - ObIStoreRowIterator *&row_iter, + ObDirectLoadIStoreRowIterator *&row_iter, bool skip_read_lob); int rescan(const blocksstable::ObDatumRange &key_range, ObIStoreRowIterator *row_iter); bool is_valid() const { return is_inited_; } @@ -89,7 +89,7 @@ private: bool is_inited_; }; -class ObDirectLoadOriginTableScanner : public ObIStoreRowIterator +class ObDirectLoadOriginTableScanner : public ObDirectLoadIStoreRowIterator { public: ObDirectLoadOriginTableScanner(); diff --git a/src/storage/direct_load/ob_direct_load_partition_merge_task.cpp b/src/storage/direct_load/ob_direct_load_partition_merge_task.cpp index ea1e8c99a2..3bab226885 100644 --- a/src/storage/direct_load/ob_direct_load_partition_merge_task.cpp +++ b/src/storage/direct_load/ob_direct_load_partition_merge_task.cpp @@ -12,14 +12,17 @@ #define USING_LOG_PREFIX STORAGE #include "storage/direct_load/ob_direct_load_partition_merge_task.h" -#include "storage/direct_load/ob_direct_load_external_table.h" -#include "storage/direct_load/ob_direct_load_insert_table_ctx.h" -#include "storage/direct_load/ob_direct_load_merge_ctx.h" -#include "storage/direct_load/ob_direct_load_multiple_heap_table.h" -#include "storage/direct_load/ob_direct_load_origin_table.h" +#include "observer/table_load/ob_table_load_table_ctx.h" +#include "storage/direct_load/ob_direct_load_compare.h" #include "storage/direct_load/ob_direct_load_conflict_check.h" #include "storage/direct_load/ob_direct_load_data_insert.h" -#include "storage/direct_load/ob_direct_load_compare.h" +#include "storage/direct_load/ob_direct_load_data_fuse.h" +#include "storage/direct_load/ob_direct_load_external_table.h" +#include "storage/direct_load/ob_direct_load_insert_table_ctx.h" +#include "storage/direct_load/ob_direct_load_insert_table_row_iterator.h" +#include "storage/direct_load/ob_direct_load_insert_table_row_writer.h" +#include "storage/direct_load/ob_direct_load_multiple_heap_table.h" +#include "storage/direct_load/ob_direct_load_origin_table.h" namespace oceanbase { @@ -41,46 +44,17 @@ ObDirectLoadPartitionMergeTask::ObDirectLoadPartitionMergeTask() merge_ctx_(nullptr), parallel_idx_(-1), affected_rows_(0), - allocator_("TLD_ParMT"), insert_tablet_ctx_(nullptr), - sql_statistics_(nullptr), + need_handle_dml_row_(false), is_stop_(false), is_inited_(false) { - allocator_.set_tenant_id(MTL_ID()); } ObDirectLoadPartitionMergeTask::~ObDirectLoadPartitionMergeTask() { } -class ObStoreRowIteratorWrapper : public ObIDirectLoadRowIterator -{ -public: - ObStoreRowIteratorWrapper(observer::ObTableLoadTableCtx *ctx, ObIStoreRowIterator *inner_iter) : - ctx_(ctx), inner_iter_(inner_iter) { - } - - int get_next_row(const blocksstable::ObDatumRow *&row) - { - int ret = inner_iter_->get_next_row(row); - if (ret == OB_SUCCESS) { - ATOMIC_AAF(&ctx_->job_stat_->store_.merge_stage_write_rows_, 1); - } - return ret; - } - - int get_next_row(const bool skip_lob, const blocksstable::ObDatumRow *&row) - { - UNUSED(skip_lob); - return get_next_row(row); - } - -private: - observer::ObTableLoadTableCtx *ctx_; - ObIStoreRowIterator *inner_iter_; -}; - int ObDirectLoadPartitionMergeTask::process() { int ret = OB_SUCCESS; @@ -88,52 +62,56 @@ int ObDirectLoadPartitionMergeTask::process() ret = OB_NOT_INIT; LOG_WARN("ObDirectLoadPartitionMergeTask not init", KR(ret), KP(this)); } else { - int64_t slice_id = 0; const ObTabletID &tablet_id = merge_ctx_->get_tablet_id(); - ObIStoreRowIterator *row_iter = nullptr; + int64_t slice_id = 0; ObMacroDataSeq block_start_seq; - if (OB_FAIL(block_start_seq.set_parallel_degree(parallel_idx_))) { + ObArenaAllocator allocator("TLD_MergeExec"); + ObArray row_iters; + allocator.set_tenant_id(MTL_ID()); + row_iters.set_block_allocator(ModulePageAllocator(allocator)); + if (OB_FAIL( + merge_param_->insert_table_ctx_->get_tablet_context(tablet_id, insert_tablet_ctx_))) { + LOG_WARN("fail to get tablet context ", KR(ret), K(tablet_id)); + } else if (OB_FAIL(construct_row_iters(row_iters, allocator))) { + LOG_WARN("fail to construct row iters", KR(ret)); + } + // 对于全量导入, 无论一个分区有没有数据, 都需要创建ddl对象来为该分区创建major sstable + else if (OB_FAIL(block_start_seq.set_parallel_degree(parallel_idx_))) { LOG_WARN("fail to set parallel degree", KR(ret), K(parallel_idx_)); - } else if (OB_FAIL(merge_param_->insert_table_ctx_->get_tablet_context(tablet_id, - insert_tablet_ctx_))) { - LOG_WARN("fail to get tablet context ", KR(ret), K(tablet_id), K(block_start_seq)); - } else if (insert_tablet_ctx_->get_online_opt_stat_gather() && - OB_FAIL(merge_param_->insert_table_ctx_->get_sql_statistics(sql_statistics_))) { - LOG_WARN("fail to get sql statistics", KR(ret)); - } else if (insert_tablet_ctx_->has_lob_storage() && OB_FAIL(lob_builder_.init(insert_tablet_ctx_))) { - LOG_WARN("fail to inner init lob builder", KR(ret)); - } else if (OB_FAIL(construct_row_iter(allocator_, row_iter))) { - LOG_WARN("fail to construct row iter", KR(ret)); } else if (OB_FAIL(insert_tablet_ctx_->open_sstable_slice(block_start_seq, slice_id))) { - LOG_WARN("fail to construct sstable slice ", KR(ret), K(slice_id), K(block_start_seq)); + LOG_WARN("fail to open sstable slice ", KR(ret), K(block_start_seq)); + } else if (row_iters.empty()) { + // do nothing + LOG_INFO("skip empty sstable slice", K(tablet_id), K(parallel_idx_), K(block_start_seq), + K(slice_id)); } else { - LOG_INFO("add sstable slice begin", K(tablet_id), K(parallel_idx_), K(slice_id)); - ObStoreRowIteratorWrapper row_iter_wrapper(ctx_, row_iter); + const bool use_batch_mode = insert_tablet_ctx_->need_rescan(); + LOG_INFO("add sstable slice begin", K(tablet_id), K(parallel_idx_), K(block_start_seq), + K(slice_id), K(row_iters.count()), K(use_batch_mode)); if (OB_UNLIKELY(is_stop_)) { ret = OB_CANCELED; LOG_WARN("merge task canceled", KR(ret)); - } else if (OB_FAIL(insert_tablet_ctx_->fill_sstable_slice(slice_id, row_iter_wrapper, affected_rows_))) { - LOG_WARN("fail to fill sstable slice", KR(ret)); - } else if (OB_FAIL(insert_tablet_ctx_->close_sstable_slice(slice_id))) { - LOG_WARN("fail to close writer", KR(ret)); - } else if (insert_tablet_ctx_->has_lob_storage() && OB_FAIL(lob_builder_.close())) { - LOG_WARN("fail to close lob_builder", KR(ret)); } else { - insert_tablet_ctx_->inc_row_count(affected_rows_); + // batch mode不支持同时写insert和delete行 + if (use_batch_mode) { + if (OB_FAIL(fill_sstable_slice_batch(slice_id, row_iters))) { + LOG_WARN("fail to fill sstable slice batch", KR(ret), K(slice_id)); + } + } else { + if (OB_FAIL(fill_sstable_slice(slice_id, row_iters))) { + LOG_WARN("fail to fill sstable slice", KR(ret), K(slice_id)); + } + } } LOG_INFO("add sstable slice end", KR(ret), K(tablet_id), K(parallel_idx_), K(affected_rows_)); } if (OB_SUCC(ret)) { - // finish_check里面会用到row_iter, 所以要在row_iter释放前调用 - if (OB_FAIL(finish_check())) { + if (OB_FAIL(insert_tablet_ctx_->close_sstable_slice(slice_id))) { + LOG_WARN("fail to close writer", KR(ret)); + } else if (OB_FAIL(finish_check())) { LOG_WARN("fail to do finish check", KR(ret)); } } - if (OB_NOT_NULL(row_iter)) { - row_iter->~ObIStoreRowIterator(); - allocator_.free(row_iter); - row_iter = nullptr; - } if (OB_SUCC(ret)) { bool is_ready = false; if (OB_FAIL(merge_ctx_->inc_finish_count(is_ready))) { @@ -150,182 +128,169 @@ int ObDirectLoadPartitionMergeTask::process() } } } + // release row iters + for (int64_t i = 0; i < row_iters.count(); ++i) { + ObDirectLoadIStoreRowIterator *row_iter = row_iters.at(i); + row_iter->~ObDirectLoadIStoreRowIterator(); + allocator.free(row_iter); + } + row_iters.reset(); + allocator.reset(); + } + return ret; +} + +int ObDirectLoadPartitionMergeTask::fill_sstable_slice( + const int64_t slice_id, + const ObIArray &row_iters) +{ + int ret = OB_SUCCESS; + ObDirectLoadInsertTableRowIterator insert_table_row_iter; + if (OB_FAIL(insert_table_row_iter.init(insert_tablet_ctx_, + row_iters, + need_handle_dml_row_ ? merge_param_->dml_row_handler_ : nullptr, + ctx_->job_stat_))) { + LOG_WARN("fail to init insert table row iter", KR(ret)); + } else if (OB_FAIL(insert_tablet_ctx_->fill_sstable_slice(slice_id, + insert_table_row_iter, + affected_rows_))) { + LOG_WARN("fail to fill sstable slice", KR(ret)); + } else if (OB_FAIL(insert_table_row_iter.close())) { + LOG_WARN("fail to close insert table row iter", KR(ret)); + } + return ret; +} + +int ObDirectLoadPartitionMergeTask::fill_sstable_slice_batch( + const int64_t slice_id, + const ObIArray &row_iters) +{ + int ret = OB_SUCCESS; + ObDirectLoadInsertTableBatchRowStoreWriter batch_writer; + ObDirectLoadInsertTableRowInfo row_info; + if (OB_FAIL(insert_tablet_ctx_->get_row_info(row_info))) { + LOG_WARN("fail to get row info", KR(ret)); + } else if (OB_FAIL(batch_writer.init(insert_tablet_ctx_, + row_info, + slice_id, + need_handle_dml_row_ ? merge_param_->dml_row_handler_ : nullptr, + ctx_->job_stat_))) { + LOG_WARN("fail to init buffer writer", KR(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < row_iters.count(); ++i) { + ObDirectLoadIStoreRowIterator *row_iter = row_iters.at(i); + if (OB_FAIL(batch_writer.write(row_iter))) { + LOG_WARN("fail to write", KR(ret)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(batch_writer.close())) { + LOG_WARN("fail to close writer", KR(ret)); + } else { + affected_rows_ = batch_writer.get_row_count(); } return ret; } void ObDirectLoadPartitionMergeTask::stop() +{ + is_stop_ = true; + if (OB_NOT_NULL(insert_tablet_ctx_)) { + insert_tablet_ctx_->cancel(); + } +} + +/** + * ObDirectLoadPartitionEmptyMergeTask + */ + +int ObDirectLoadPartitionEmptyMergeTask::init(ObTableLoadTableCtx *ctx, + const ObDirectLoadMergeParam &merge_param, + ObDirectLoadTabletMergeCtx *merge_ctx) { int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadPartitionRescanTask not init", KR(ret), KP(this)); + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObDirectLoadPartitionEmptyMergeTask init twice", KR(ret), KP(this)); + } else if (OB_UNLIKELY(nullptr == ctx || !merge_param.is_valid() || nullptr == merge_ctx)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(merge_param), KP(merge_ctx)); } else { - ObDirectLoadInsertTabletContext *tablet_ctx = nullptr; - const ObTabletID &tablet_id = merge_ctx_->get_tablet_id(); - if (OB_FAIL(merge_param_->insert_table_ctx_->get_tablet_context(tablet_id, tablet_ctx))) { - LOG_WARN("fail to get tablet context ", KR(ret), K(tablet_id)); - } else if (OB_FAIL(tablet_ctx->cancel())) { - LOG_WARN("fail to cancel fill task", K(ret)); - } else { - is_stop_ = true; + ctx_ = ctx; + merge_param_ = &merge_param; + merge_ctx_ = merge_ctx; + parallel_idx_ = 0; + is_inited_ = true; + } + return ret; +} + +/** + * ObDirectLoadPartitionOriginDataMergeTask + */ + +ObDirectLoadPartitionOriginDataMergeTask::ObDirectLoadPartitionOriginDataMergeTask() + : origin_table_(nullptr), range_(nullptr) +{ +} + +ObDirectLoadPartitionOriginDataMergeTask::~ObDirectLoadPartitionOriginDataMergeTask() {} + +int ObDirectLoadPartitionOriginDataMergeTask::init(ObTableLoadTableCtx *ctx, + const ObDirectLoadMergeParam &merge_param, + ObDirectLoadTabletMergeCtx *merge_ctx, + ObDirectLoadOriginTable *origin_table, + const ObDatumRange &range, + int64_t parallel_idx) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObDirectLoadPartitionOriginDataMergeTask init twice", KR(ret), KP(this)); + } else if (OB_UNLIKELY(nullptr == ctx || !merge_param.is_valid() || nullptr == merge_ctx || + nullptr == origin_table || !range.is_valid() || parallel_idx < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(merge_param), KP(merge_ctx), K(range), K(parallel_idx)); + } else { + ctx_ = ctx; + merge_param_ = &merge_param; + merge_ctx_ = merge_ctx; + parallel_idx_ = parallel_idx; + origin_table_ = origin_table; + range_ = ⦥ + is_inited_ = true; + } + return ret; +} + +int ObDirectLoadPartitionOriginDataMergeTask::construct_row_iters( + ObIArray &row_iters, + ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + row_iters.reset(); + ObDirectLoadIStoreRowIterator *data_iter = nullptr; + if (OB_FAIL(origin_table_->scan(*range_, allocator, data_iter, false /*skip_read_lob*/))) { + LOG_WARN("fail to scan origin table", KR(ret)); + } else if (OB_FAIL(row_iters.push_back(data_iter))) { + LOG_WARN("fail to push back", KR(ret)); + } + if (OB_FAIL(ret)) { + if (nullptr != data_iter) { + data_iter->~ObDirectLoadIStoreRowIterator(); + allocator.free(data_iter); + data_iter = nullptr; } } - // ignore ret + return ret; } /** * ObDirectLoadPartitionRangeMergeTask */ -ObDirectLoadPartitionRangeMergeTask::RowIterator::RowIterator() - : data_iter_(nullptr), conflict_check_(nullptr), rowkey_column_num_(0) -{ -} - -ObDirectLoadPartitionRangeMergeTask::RowIterator::~RowIterator() -{ - if (data_iter_ != nullptr) { - ob_delete(data_iter_); - } -} - -int ObDirectLoadPartitionRangeMergeTask::RowIterator::init( - const ObDirectLoadMergeParam &merge_param, - ObDirectLoadTabletMergeCtx *merge_ctx, - ObDirectLoadOriginTable *origin_table, - ObTableLoadSqlStatistics *sql_statistics, - ObDirectLoadLobBuilder &lob_builder, - const ObIArray &sstable_array, - const ObDatumRange &range, - ObDirectLoadInsertTabletContext *insert_tablet_ctx) -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - LOG_WARN("ObDirectLoadPartitionRangeMergeTask::RowIterator init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!merge_param.is_valid() || nullptr == merge_ctx || - nullptr == origin_table || !range.is_valid() || - nullptr == insert_tablet_ctx)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(merge_param), KP(merge_ctx), K(sstable_array), K(range), - KP(insert_tablet_ctx)); - } else { - if (OB_FAIL(inner_init(insert_tablet_ctx, sql_statistics, lob_builder))) { - LOG_WARN("fail to inner init", KR(ret)); - } else if (OB_FAIL(insert_tablet_ctx->init_datum_row(datum_row_))) { - LOG_WARN("fail to init datum row", KR(ret)); - } else { - if (merge_ctx->merge_with_origin_data()) { - // init data_fuse_ - ObDirectLoadDataFuseParam data_fuse_param; - data_fuse_param.tablet_id_ = merge_ctx->get_tablet_id(); - data_fuse_param.store_column_count_ = merge_param.store_column_count_; - data_fuse_param.table_data_desc_ = merge_param.table_data_desc_; - data_fuse_param.datum_utils_ = merge_param.datum_utils_; - data_fuse_param.dml_row_handler_ = merge_param.dml_row_handler_; - ObDirectLoadSSTableDataFuse *data_fuse = nullptr; - if (OB_ISNULL(data_fuse = OB_NEW(ObDirectLoadSSTableDataFuse, ObMemAttr(MTL_ID(), "TLD_PRMT")))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory", K(ret)); - } else if (FALSE_IT(data_iter_ = data_fuse)) { - } else if (OB_FAIL(data_fuse->init(data_fuse_param, origin_table, sstable_array, range))) { - LOG_WARN("fail to init ObDirectLoadSSTableDataFuse", K(ret)); - } - } else if (merge_ctx->merge_with_conflict_check()) { - ObDirectLoadConflictCheckParam conflict_check_param; - conflict_check_param.tablet_id_ = merge_ctx->get_tablet_id(); - conflict_check_param.tablet_id_in_lob_id_ = insert_tablet_ctx->get_tablet_id_in_lob_id(); - conflict_check_param.store_column_count_ = merge_param.store_column_count_; - conflict_check_param.table_data_desc_ = merge_param.table_data_desc_; - conflict_check_param.origin_table_ = origin_table; - conflict_check_param.range_ = ⦥ - conflict_check_param.col_descs_ = merge_param.col_descs_; - conflict_check_param.lob_column_idxs_ = merge_param.lob_column_idxs_; - conflict_check_param.datum_utils_ = merge_param.datum_utils_; - conflict_check_param.lob_meta_datum_utils_ = merge_param.lob_meta_datum_utils_; - conflict_check_param.dml_row_handler_ = merge_param.dml_row_handler_; - ObDirectLoadSSTableConflictCheck *conflict_check = nullptr; - if (OB_FAIL(merge_ctx->get_table_builder(conflict_check_param.builder_))) { - LOG_WARN("fail to get table builder", K(ret)); - } else if (OB_ISNULL(conflict_check = OB_NEW(ObDirectLoadSSTableConflictCheck, ObMemAttr(MTL_ID(), "TLD_PRMT")))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc memory", K(ret)); - } else if (FALSE_IT(data_iter_ = conflict_check)) { - } else if (OB_FAIL(conflict_check->init(conflict_check_param, sstable_array))) { - LOG_WARN("fail to init ObDirectLoadSSTableConflictCheck", K(ret)); - } else { - conflict_check_ = conflict_check; // save conflict_check_ - } - } else { - ObDirectLoadDataInsertParam data_insert_param; - data_insert_param.tablet_id_ = merge_ctx->get_tablet_id(); - data_insert_param.store_column_count_ = merge_param.store_column_count_; - data_insert_param.table_data_desc_ = merge_param.table_data_desc_; - data_insert_param.datum_utils_ = merge_param.datum_utils_; - data_insert_param.dml_row_handler_ = merge_param.dml_row_handler_; - ObDirectLoadSSTableDataInsert *data_insert = nullptr; - if (OB_ISNULL(data_insert = OB_NEW(ObDirectLoadSSTableDataInsert, ObMemAttr(MTL_ID(), "TLD_PRMT")))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory", K(ret)); - } else if (FALSE_IT(data_iter_ = data_insert)) { - } else if (OB_FAIL(data_insert->init(data_insert_param, sstable_array, range))) { - LOG_WARN("fail to init ObDirectLoadSSTableDataInsert", K(ret)); - } - } - } - if (OB_SUCC(ret)) { - rowkey_column_num_ = merge_param.rowkey_column_num_; - is_inited_ = true; - } - } - - return ret; -} - -int ObDirectLoadPartitionRangeMergeTask::RowIterator::inner_get_next_row(ObDatumRow *&result_row) -{ - int ret = OB_SUCCESS; - result_row = nullptr; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadPartitionRangeMergeTask::RowIterator not init", KR(ret), KP(this)); - } else { - const ObDatumRow *datum_row = nullptr; - if (OB_FAIL(data_iter_->get_next_row(datum_row))) { - if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("fail to get next row", KR(ret)); - } - } else { - // copy rowkey columns - for (int64_t i = 0; i < rowkey_column_num_; ++i) { - datum_row_.storage_datums_[i] = datum_row->storage_datums_[i]; - } - // copy normal columns - for (int64_t i = rowkey_column_num_, - j = rowkey_column_num_ + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); - i < datum_row->count_; ++i, ++j) { - datum_row_.storage_datums_[j] = datum_row->storage_datums_[i]; - } - datum_row_.row_flag_.set_flag(datum_row->row_flag_.is_delete() ? blocksstable::DF_DELETE : blocksstable::DF_INSERT, - datum_row->row_flag_.is_delete() || !insert_tablet_ctx_->is_incremental() ? DF_TYPE_NORMAL : DF_TYPE_INSERT_DELETE); - result_row = &datum_row_; - } - } - return ret; -} - -ObLobId ObDirectLoadPartitionRangeMergeTask::RowIterator::get_max_del_lob_id() const -{ - ObLobId max_del_lob_id; - if (conflict_check_ != nullptr) { - max_del_lob_id = conflict_check_->get_max_del_lob_id(); - } - return max_del_lob_id; -} - ObDirectLoadPartitionRangeMergeTask::ObDirectLoadPartitionRangeMergeTask() - : origin_table_(nullptr), sstable_array_(nullptr), range_(nullptr), row_iter_(nullptr) + : origin_table_(nullptr), sstable_array_(nullptr), range_(nullptr), conflict_check_(nullptr) { } @@ -336,8 +301,7 @@ int ObDirectLoadPartitionRangeMergeTask::init(ObTableLoadTableCtx *ctx, ObDirectLoadTabletMergeCtx *merge_ctx, ObDirectLoadOriginTable *origin_table, const ObIArray &sstable_array, - const ObDatumRange &range, - int64_t parallel_idx) + const ObDatumRange &range, int64_t parallel_idx) { int ret = OB_SUCCESS; if (IS_INIT) { @@ -361,33 +325,77 @@ int ObDirectLoadPartitionRangeMergeTask::init(ObTableLoadTableCtx *ctx, return ret; } -int ObDirectLoadPartitionRangeMergeTask::construct_row_iter(ObIAllocator &allocator, - ObIStoreRowIterator *&result_row_iter) +int ObDirectLoadPartitionRangeMergeTask::construct_row_iters( + ObIArray &row_iters, + ObIAllocator &allocator) { int ret = OB_SUCCESS; - result_row_iter = nullptr; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadPartitionRangeMergeTask not init", KR(ret), KP(this)); - } else { - RowIterator *row_iter = nullptr; - if (OB_ISNULL(row_iter = OB_NEWx(RowIterator, (&allocator)))) { + row_iters.reset(); + ObDirectLoadIStoreRowIterator *data_iter = nullptr; + if (merge_ctx_->merge_with_origin_data()) { + ObDirectLoadDataFuseParam data_fuse_param; + data_fuse_param.tablet_id_ = merge_ctx_->get_tablet_id(); + data_fuse_param.store_column_count_ = merge_param_->store_column_count_; + data_fuse_param.table_data_desc_ = merge_param_->table_data_desc_; + data_fuse_param.datum_utils_ = merge_param_->datum_utils_; + data_fuse_param.dml_row_handler_ = merge_param_->dml_row_handler_; + ObDirectLoadSSTableDataFuse *data_fuse = nullptr; + if (OB_ISNULL(data_iter = data_fuse = OB_NEWx(ObDirectLoadSSTableDataFuse, &allocator))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new RowIterator", KR(ret)); - } else if (OB_FAIL(row_iter->init(*merge_param_, merge_ctx_, origin_table_, - sql_statistics_, lob_builder_, *sstable_array_, *range_, - insert_tablet_ctx_))) { - LOG_WARN("fail to init row iter", KR(ret)); - } else { - result_row_iter = row_iter; - row_iter_ = row_iter; // save row_iter_ + LOG_WARN("fail to new ObDirectLoadSSTableDataFuse", KR(ret)); + } else if (OB_FAIL(data_fuse->init(data_fuse_param, origin_table_, *sstable_array_, *range_))) { + LOG_WARN("fail to init data fuse", KR(ret)); } - if (OB_FAIL(ret)) { - if (nullptr != row_iter) { - row_iter->~RowIterator(); - allocator.free(row_iter); - row_iter = nullptr; - } + } else if (merge_ctx_->merge_with_conflict_check()) { + ObDirectLoadConflictCheckParam conflict_check_param; + conflict_check_param.tablet_id_ = merge_ctx_->get_tablet_id(); + conflict_check_param.tablet_id_in_lob_id_ = insert_tablet_ctx_->get_tablet_id_in_lob_id(); + conflict_check_param.store_column_count_ = merge_param_->store_column_count_; + conflict_check_param.table_data_desc_ = merge_param_->table_data_desc_; + conflict_check_param.origin_table_ = origin_table_; + conflict_check_param.range_ = range_; + conflict_check_param.col_descs_ = merge_param_->col_descs_; + conflict_check_param.lob_column_idxs_ = merge_param_->lob_column_idxs_; + conflict_check_param.datum_utils_ = merge_param_->datum_utils_; + conflict_check_param.lob_meta_datum_utils_ = merge_param_->lob_meta_datum_utils_; + conflict_check_param.dml_row_handler_ = merge_param_->dml_row_handler_; + ObDirectLoadSSTableConflictCheck *conflict_check = nullptr; + if (OB_FAIL(merge_ctx_->get_table_builder(conflict_check_param.builder_))) { + LOG_WARN("fail to get table builder", K(ret)); + } else if (OB_ISNULL(data_iter = conflict_check = + OB_NEWx(ObDirectLoadSSTableConflictCheck, &allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObDirectLoadSSTableConflictCheck", K(ret)); + } else if (OB_FAIL(conflict_check->init(conflict_check_param, *sstable_array_))) { + LOG_WARN("fail to init conflict check", K(ret)); + } else { + conflict_check_ = conflict_check; // save conflict_check_ + } + } else { + ObDirectLoadDataInsertParam data_insert_param; + data_insert_param.tablet_id_ = merge_ctx_->get_tablet_id(); + data_insert_param.store_column_count_ = merge_param_->store_column_count_; + data_insert_param.table_data_desc_ = merge_param_->table_data_desc_; + data_insert_param.datum_utils_ = merge_param_->datum_utils_; + data_insert_param.dml_row_handler_ = merge_param_->dml_row_handler_; + ObDirectLoadSSTableDataInsert *data_insert = nullptr; + if (OB_ISNULL(data_iter = data_insert = OB_NEWx(ObDirectLoadSSTableDataInsert, &allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to new ObDirectLoadSSTableDataInsert", K(ret)); + } else if (OB_FAIL(data_insert->init(data_insert_param, *sstable_array_, *range_))) { + LOG_WARN("fail to init data insert", K(ret)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(row_iters.push_back(data_iter))) { + LOG_WARN("fail to push back", KR(ret)); + } + } + if (OB_FAIL(ret)) { + if (nullptr != data_iter) { + data_iter->~ObDirectLoadIStoreRowIterator(); + allocator.free(data_iter); + data_iter = nullptr; } } return ret; @@ -396,22 +404,23 @@ int ObDirectLoadPartitionRangeMergeTask::construct_row_iter(ObIAllocator &alloca int ObDirectLoadPartitionRangeMergeTask::finish_check() { int ret = OB_SUCCESS; - if (OB_UNLIKELY(row_iter_ == nullptr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected row_iter_ is null", KR(ret)); - } else { - const ObLobId max_del_lob_id = row_iter_->get_max_del_lob_id(); + if (nullptr != conflict_check_) { + const ObLobId max_del_lob_id = conflict_check_->get_max_del_lob_id(); const ObLobId &min_insert_lob_id = insert_tablet_ctx_->get_min_insert_lob_id(); if (max_del_lob_id.is_valid() && min_insert_lob_id.is_valid()) { ObStorageDatum max_del_lob_id_datum, min_insert_lob_id_datum; ObDirectLoadSingleDatumCompare compare; int cmp_ret = 0; - max_del_lob_id_datum.set_string(reinterpret_cast(&max_del_lob_id), sizeof(ObLobId)); - min_insert_lob_id_datum.set_string(reinterpret_cast(&min_insert_lob_id), sizeof(ObLobId)); + max_del_lob_id_datum.set_string(reinterpret_cast(&max_del_lob_id), + sizeof(ObLobId)); + min_insert_lob_id_datum.set_string(reinterpret_cast(&min_insert_lob_id), + sizeof(ObLobId)); if (OB_FAIL(compare.init(*merge_param_->lob_meta_datum_utils_))) { LOG_WARN("fail to init compare", KR(ret)); - } else if (OB_FAIL(compare.compare(&max_del_lob_id_datum, &min_insert_lob_id_datum, cmp_ret))) { - LOG_WARN("fail to compare lob id", KR(ret), K(max_del_lob_id_datum), K(min_insert_lob_id_datum)); + } else if (OB_FAIL( + compare.compare(&max_del_lob_id_datum, &min_insert_lob_id_datum, cmp_ret))) { + LOG_WARN("fail to compare lob id", KR(ret), K(max_del_lob_id_datum), + K(min_insert_lob_id_datum)); } else if (OB_UNLIKELY(cmp_ret >= 0)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected lob id", KR(ret), K(max_del_lob_id), K(max_del_lob_id_datum), @@ -426,169 +435,17 @@ int ObDirectLoadPartitionRangeMergeTask::finish_check() * ObDirectLoadPartitionRangeMultipleMergeTask */ -ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator::RowIterator() - : data_iter_(nullptr), conflict_check_(nullptr), rowkey_column_num_(0) -{ -} - -ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator::~RowIterator() -{ - if (data_iter_ != nullptr) { - ob_delete(data_iter_); - } -} - -int ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator::init( - const ObDirectLoadMergeParam &merge_param, - ObDirectLoadTabletMergeCtx *merge_ctx, - ObDirectLoadOriginTable *origin_table, - ObTableLoadSqlStatistics *sql_statistics, - ObDirectLoadLobBuilder &lob_builder, - const ObIArray &sstable_array, - const ObDatumRange &range, - ObDirectLoadInsertTabletContext *insert_tablet_ctx) -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - LOG_WARN("ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator init twice", KR(ret), - KP(this)); - } else if (OB_UNLIKELY(!merge_param.is_valid() || nullptr == merge_ctx || - nullptr == origin_table || !range.is_valid() || - nullptr == insert_tablet_ctx)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(merge_param), KP(merge_ctx), K(sstable_array), K(range), - KP(insert_tablet_ctx)); - } else { - if (OB_FAIL(inner_init(insert_tablet_ctx, sql_statistics, lob_builder))) { - LOG_WARN("fail to inner init", KR(ret)); - } else if (OB_FAIL(insert_tablet_ctx->init_datum_row(datum_row_))) { - LOG_WARN("fail to init datum row", KR(ret)); - } else { - if (merge_ctx->merge_with_origin_data()) { - // init data_fuse_ - ObDirectLoadDataFuseParam data_fuse_param; - data_fuse_param.tablet_id_ = merge_ctx->get_tablet_id(); - data_fuse_param.store_column_count_ = merge_param.store_column_count_; - data_fuse_param.table_data_desc_ = merge_param.table_data_desc_; - data_fuse_param.datum_utils_ = merge_param.datum_utils_; - data_fuse_param.dml_row_handler_ = merge_param.dml_row_handler_; - ObDirectLoadMultipleSSTableDataFuse *data_fuse = nullptr; - if (OB_ISNULL(data_fuse = OB_NEW(ObDirectLoadMultipleSSTableDataFuse, ObMemAttr(MTL_ID(), "TLD_PRMMT")))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory", K(ret)); - } else if (FALSE_IT(data_iter_ = data_fuse)) { - } else if (OB_FAIL(data_fuse->init(data_fuse_param, origin_table, sstable_array, range))) { - LOG_WARN("fail to init ObDirectLoadMultipleSSTableDataFuse", K(ret)); - } - } else if (merge_ctx->merge_with_conflict_check()) { - ObDirectLoadConflictCheckParam conflict_check_param; - conflict_check_param.tablet_id_ = merge_ctx->get_tablet_id(); - conflict_check_param.tablet_id_in_lob_id_ = insert_tablet_ctx->get_tablet_id_in_lob_id(); - conflict_check_param.store_column_count_ = merge_param.store_column_count_; - conflict_check_param.table_data_desc_ = merge_param.table_data_desc_; - conflict_check_param.origin_table_ = origin_table; - conflict_check_param.range_ = ⦥ - conflict_check_param.col_descs_ = merge_param.col_descs_; - conflict_check_param.lob_column_idxs_ = merge_param.lob_column_idxs_; - conflict_check_param.datum_utils_ = merge_param.datum_utils_; - conflict_check_param.lob_meta_datum_utils_ = merge_param.lob_meta_datum_utils_; - conflict_check_param.dml_row_handler_ = merge_param.dml_row_handler_; - ObDirectLoadMultipleSSTableConflictCheck *conflict_check = nullptr; - if (OB_FAIL(merge_ctx->get_table_builder(conflict_check_param.builder_))) { - LOG_WARN("fail to get table builder", K(ret)); - } else if (OB_ISNULL(conflict_check = OB_NEW(ObDirectLoadMultipleSSTableConflictCheck, ObMemAttr(MTL_ID(), "TLD_PRMMT")))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc memory", K(ret)); - } else if (FALSE_IT(data_iter_ = conflict_check)) { - } else if (OB_FAIL(conflict_check->init(conflict_check_param, sstable_array))) { - LOG_WARN("fail to init ObDirectLoadMultipleSSTableConflictCheck", K(ret)); - } else { - conflict_check_ = conflict_check; // save conflict_check_ - } - } else { - ObDirectLoadDataInsertParam data_insert_param; - data_insert_param.tablet_id_ = merge_ctx->get_tablet_id(); - data_insert_param.store_column_count_ = merge_param.store_column_count_; - data_insert_param.table_data_desc_ = merge_param.table_data_desc_; - data_insert_param.datum_utils_ = merge_param.datum_utils_; - data_insert_param.dml_row_handler_ = merge_param.dml_row_handler_; - ObDirectLoadMultipleSSTableDataInsert *data_insert = nullptr; - if (OB_ISNULL(data_insert = OB_NEW(ObDirectLoadMultipleSSTableDataInsert, ObMemAttr(MTL_ID(), "TLD_PRMT")))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory", K(ret)); - } else if (FALSE_IT(data_iter_ = data_insert)) { - } else if (OB_FAIL(data_insert->init(data_insert_param, sstable_array, range))) { - LOG_WARN("fail to init ObDirectLoadMultipleSSTableDataInsert", K(ret)); - } - } - } - if (OB_SUCC(ret)) { - rowkey_column_num_ = merge_param.rowkey_column_num_; - is_inited_ = true; - } - } - - return ret; -} - -int ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator::inner_get_next_row( - ObDatumRow *&result_row) -{ - int ret = OB_SUCCESS; - result_row = nullptr; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator not init", KR(ret), - KP(this)); - } else { - const ObDatumRow *datum_row = nullptr; - if (OB_FAIL(data_iter_->get_next_row(datum_row))) { - if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("fail to get next row", KR(ret)); - } - } else { - // copy rowkey columns - for (int64_t i = 0; i < rowkey_column_num_; ++i) { - datum_row_.storage_datums_[i] = datum_row->storage_datums_[i]; - } - // copy normal columns - for (int64_t i = rowkey_column_num_, - j = rowkey_column_num_ + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); - i < datum_row->count_; ++i, ++j) { - datum_row_.storage_datums_[j] = datum_row->storage_datums_[i]; - } - datum_row_.row_flag_.set_flag(datum_row->row_flag_.is_delete() ? blocksstable::DF_DELETE : blocksstable::DF_INSERT, - datum_row->row_flag_.is_delete() || !insert_tablet_ctx_->is_incremental() ? DF_TYPE_NORMAL : DF_TYPE_INSERT_DELETE); - result_row = &datum_row_; - } - } - return ret; -} - -ObLobId ObDirectLoadPartitionRangeMultipleMergeTask::RowIterator::get_max_del_lob_id() const -{ - ObLobId max_del_lob_id; - if (conflict_check_ != nullptr) { - max_del_lob_id = conflict_check_->get_max_del_lob_id(); - } - return max_del_lob_id; -} - ObDirectLoadPartitionRangeMultipleMergeTask::ObDirectLoadPartitionRangeMultipleMergeTask() - : origin_table_(nullptr), sstable_array_(nullptr), range_(nullptr), row_iter_(nullptr) + : origin_table_(nullptr), sstable_array_(nullptr), range_(nullptr), conflict_check_(nullptr) { } ObDirectLoadPartitionRangeMultipleMergeTask::~ObDirectLoadPartitionRangeMultipleMergeTask() {} int ObDirectLoadPartitionRangeMultipleMergeTask::init( - ObTableLoadTableCtx *ctx, - const ObDirectLoadMergeParam &merge_param, - ObDirectLoadTabletMergeCtx *merge_ctx, - ObDirectLoadOriginTable *origin_table, - const ObIArray &sstable_array, - const ObDatumRange &range, + ObTableLoadTableCtx *ctx, const ObDirectLoadMergeParam &merge_param, + ObDirectLoadTabletMergeCtx *merge_ctx, ObDirectLoadOriginTable *origin_table, + const ObIArray &sstable_array, const ObDatumRange &range, int64_t parallel_idx) { int ret = OB_SUCCESS; @@ -613,33 +470,79 @@ int ObDirectLoadPartitionRangeMultipleMergeTask::init( return ret; } -int ObDirectLoadPartitionRangeMultipleMergeTask::construct_row_iter( - ObIAllocator &allocator, ObIStoreRowIterator *&result_row_iter) +int ObDirectLoadPartitionRangeMultipleMergeTask::construct_row_iters( + ObIArray &row_iters, + ObIAllocator &allocator) { int ret = OB_SUCCESS; - result_row_iter = nullptr; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadPartitionRangeMultipleMergeTask not init", KR(ret), KP(this)); - } else { - RowIterator *row_iter = nullptr; - if (OB_ISNULL(row_iter = OB_NEWx(RowIterator, (&allocator)))) { + row_iters.reset(); + ObDirectLoadIStoreRowIterator *data_iter = nullptr; + if (merge_ctx_->merge_with_origin_data()) { + ObDirectLoadDataFuseParam data_fuse_param; + data_fuse_param.tablet_id_ = merge_ctx_->get_tablet_id(); + data_fuse_param.store_column_count_ = merge_param_->store_column_count_; + data_fuse_param.table_data_desc_ = merge_param_->table_data_desc_; + data_fuse_param.datum_utils_ = merge_param_->datum_utils_; + data_fuse_param.dml_row_handler_ = merge_param_->dml_row_handler_; + ObDirectLoadMultipleSSTableDataFuse *data_fuse = nullptr; + if (OB_ISNULL(data_iter = data_fuse = + OB_NEWx(ObDirectLoadMultipleSSTableDataFuse, &allocator))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new RowIterator", KR(ret)); - } else if (OB_FAIL(row_iter->init(*merge_param_, merge_ctx_, origin_table_, - sql_statistics_, lob_builder_, *sstable_array_, *range_, - insert_tablet_ctx_))) { - LOG_WARN("fail to init row iter", KR(ret)); - } else { - result_row_iter = row_iter; - row_iter_ = row_iter; // save row_iter_ + LOG_WARN("failed to new ObDirectLoadMultipleSSTableDataFuse", KR(ret)); + } else if (OB_FAIL(data_fuse->init(data_fuse_param, origin_table_, *sstable_array_, *range_))) { + LOG_WARN("fail to init data fuse", KR(ret)); } - if (OB_FAIL(ret)) { - if (nullptr != row_iter) { - row_iter->~RowIterator(); - allocator.free(row_iter); - row_iter = nullptr; - } + } else if (merge_ctx_->merge_with_conflict_check()) { + ObDirectLoadConflictCheckParam conflict_check_param; + conflict_check_param.tablet_id_ = merge_ctx_->get_tablet_id(); + conflict_check_param.tablet_id_in_lob_id_ = insert_tablet_ctx_->get_tablet_id_in_lob_id(); + conflict_check_param.store_column_count_ = merge_param_->store_column_count_; + conflict_check_param.table_data_desc_ = merge_param_->table_data_desc_; + conflict_check_param.origin_table_ = origin_table_; + conflict_check_param.range_ = range_; + conflict_check_param.col_descs_ = merge_param_->col_descs_; + conflict_check_param.lob_column_idxs_ = merge_param_->lob_column_idxs_; + conflict_check_param.datum_utils_ = merge_param_->datum_utils_; + conflict_check_param.lob_meta_datum_utils_ = merge_param_->lob_meta_datum_utils_; + conflict_check_param.dml_row_handler_ = merge_param_->dml_row_handler_; + ObDirectLoadMultipleSSTableConflictCheck *conflict_check = nullptr; + if (OB_FAIL(merge_ctx_->get_table_builder(conflict_check_param.builder_))) { + LOG_WARN("fail to get table builder", KR(ret)); + } else if (OB_ISNULL(data_iter = conflict_check = + OB_NEWx(ObDirectLoadMultipleSSTableConflictCheck, &allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObDirectLoadMultipleSSTableConflictCheck", KR(ret)); + } else if (OB_FAIL(conflict_check->init(conflict_check_param, *sstable_array_))) { + LOG_WARN("fail to init conflict check", KR(ret)); + } else { + conflict_check_ = conflict_check; // save conflict_check_ + } + } else { + ObDirectLoadDataInsertParam data_insert_param; + data_insert_param.tablet_id_ = merge_ctx_->get_tablet_id(); + data_insert_param.store_column_count_ = merge_param_->store_column_count_; + data_insert_param.table_data_desc_ = merge_param_->table_data_desc_; + data_insert_param.datum_utils_ = merge_param_->datum_utils_; + data_insert_param.dml_row_handler_ = merge_param_->dml_row_handler_; + ObDirectLoadMultipleSSTableDataInsert *data_insert = nullptr; + if (OB_ISNULL(data_iter = data_insert = + OB_NEWx(ObDirectLoadMultipleSSTableDataInsert, &allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to new ObDirectLoadMultipleSSTableDataInsert", KR(ret)); + } else if (OB_FAIL(data_insert->init(data_insert_param, *sstable_array_, *range_))) { + LOG_WARN("fail to init data insert", KR(ret)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(row_iters.push_back(data_iter))) { + LOG_WARN("fail to push back", KR(ret)); + } + } + if (OB_FAIL(ret)) { + if (nullptr != data_iter) { + data_iter->~ObDirectLoadIStoreRowIterator(); + allocator.free(data_iter); + data_iter = nullptr; } } return ret; @@ -648,22 +551,23 @@ int ObDirectLoadPartitionRangeMultipleMergeTask::construct_row_iter( int ObDirectLoadPartitionRangeMultipleMergeTask::finish_check() { int ret = OB_SUCCESS; - if (OB_UNLIKELY(row_iter_ == nullptr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected row_iter_ is null", KR(ret)); - } else { - const ObLobId max_del_lob_id = row_iter_->get_max_del_lob_id(); + if (nullptr != conflict_check_) { + const ObLobId max_del_lob_id = conflict_check_->get_max_del_lob_id(); const ObLobId &min_insert_lob_id = insert_tablet_ctx_->get_min_insert_lob_id(); if (max_del_lob_id.is_valid() && min_insert_lob_id.is_valid()) { ObStorageDatum max_del_lob_id_datum, min_insert_lob_id_datum; ObDirectLoadSingleDatumCompare compare; int cmp_ret = 0; - max_del_lob_id_datum.set_string(reinterpret_cast(&max_del_lob_id), sizeof(ObLobId)); - min_insert_lob_id_datum.set_string(reinterpret_cast(&min_insert_lob_id), sizeof(ObLobId)); + max_del_lob_id_datum.set_string(reinterpret_cast(&max_del_lob_id), + sizeof(ObLobId)); + min_insert_lob_id_datum.set_string(reinterpret_cast(&min_insert_lob_id), + sizeof(ObLobId)); if (OB_FAIL(compare.init(*merge_param_->lob_meta_datum_utils_))) { LOG_WARN("fail to init compare", KR(ret)); - } else if (OB_FAIL(compare.compare(&max_del_lob_id_datum, &min_insert_lob_id_datum, cmp_ret))) { - LOG_WARN("fail to compare lob id", KR(ret), K(max_del_lob_id_datum), K(min_insert_lob_id_datum)); + } else if (OB_FAIL( + compare.compare(&max_del_lob_id_datum, &min_insert_lob_id_datum, cmp_ret))) { + LOG_WARN("fail to compare lob id", KR(ret), K(max_del_lob_id_datum), + K(min_insert_lob_id_datum)); } else if (OB_UNLIKELY(cmp_ret >= 0)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected lob id", KR(ret), K(max_del_lob_id), K(max_del_lob_id_datum), @@ -679,88 +583,58 @@ int ObDirectLoadPartitionRangeMultipleMergeTask::finish_check() */ ObDirectLoadPartitionHeapTableMergeTask::RowIterator::RowIterator() - : deserialize_datums_(nullptr), - deserialize_datum_cnt_(0), - dml_row_handler_(nullptr) + : pk_interval_(nullptr), is_inited_(false) { } ObDirectLoadPartitionHeapTableMergeTask::RowIterator::~RowIterator() {} int ObDirectLoadPartitionHeapTableMergeTask::RowIterator::init( - const ObDirectLoadMergeParam &merge_param, - const ObTabletID &tablet_id, - ObDirectLoadExternalTable *external_table, - ObTableLoadSqlStatistics *sql_statistics, - ObDirectLoadLobBuilder &lob_builder, - const ObTabletCacheInterval &pk_interval, - ObDirectLoadInsertTabletContext *insert_tablet_ctx) + ObDirectLoadExternalTable *external_table, const ObDirectLoadTableDataDesc &table_data_desc, + ObTabletCacheInterval &pk_interval) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; - LOG_WARN("ObDirectLoadPartitionHeapTableMergeTask::RowIterator init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!merge_param.is_valid() || !tablet_id.is_valid() || - nullptr == external_table || 0 == pk_interval.count() || - nullptr == insert_tablet_ctx)) { + LOG_WARN("RowIterator init twice", KR(ret), KP(this)); + } else if (OB_UNLIKELY(nullptr == external_table || !table_data_desc.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(merge_param), K(tablet_id), KP(external_table), - K(pk_interval), KP(insert_tablet_ctx)); + LOG_WARN("invalid args", KR(ret), KPC(external_table), K(table_data_desc)); } else { - // init row iterator - if (OB_FAIL(inner_init(insert_tablet_ctx, sql_statistics, lob_builder))) { - LOG_WARN("fail to inner init", KR(ret)); - } - // init scanner_ - else if (OB_FAIL(scanner_.init(merge_param.table_data_desc_.external_data_block_size_, - merge_param.table_data_desc_.compressor_type_, - external_table->get_fragments()))) { + if (OB_FAIL(scanner_.init(table_data_desc.external_data_block_size_, + table_data_desc.compressor_type_, external_table->get_fragments()))) { LOG_WARN("fail to init fragment scanner", KR(ret)); - } - // init datum_row_ - else if (OB_FAIL(insert_tablet_ctx->init_datum_row(datum_row_))) { + } else if (OB_FAIL(datum_row_.init(table_data_desc.column_count_))) { LOG_WARN("fail to init datum row", KR(ret)); } else { - deserialize_datums_ = datum_row_.storage_datums_ + merge_param.rowkey_column_num_ + - ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); - deserialize_datum_cnt_ = merge_param.store_column_count_ - merge_param.rowkey_column_num_; - pk_interval_ = pk_interval; - dml_row_handler_ = merge_param.dml_row_handler_; - tablet_id_ = tablet_id; + pk_interval_ = &pk_interval; + row_flag_.uncontain_hidden_pk_ = true; + row_flag_.has_multi_version_cols_ = false; + row_flag_.has_delete_row_ = false; + column_count_ = table_data_desc.column_count_; is_inited_ = true; } } return ret; } -int ObDirectLoadPartitionHeapTableMergeTask::RowIterator::inner_get_next_row( - ObDatumRow *&result_row) +int ObDirectLoadPartitionHeapTableMergeTask::RowIterator::get_next_row(const ObDatumRow *&datum_row) { int ret = OB_SUCCESS; - result_row = nullptr; + datum_row = nullptr; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadPartitionRangeMergeTask::RowIterator not init", KR(ret), KP(this)); + LOG_WARN("RowIterator not init", KR(ret), KP(this)); } else { const ObDirectLoadExternalRow *external_row = nullptr; - uint64_t pk_seq = OB_INVALID_ID; if (OB_FAIL(scanner_.get_next_item(external_row))) { if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("fail to get next row from merge_sort", KR(ret)); + LOG_WARN("fail to get next item", KR(ret)); } - } else if (OB_FAIL(external_row->to_datums(deserialize_datums_, deserialize_datum_cnt_))) { + } else if (OB_FAIL(external_row->to_datums(datum_row_.storage_datums_, datum_row_.count_))) { LOG_WARN("fail to transfer datum row", KR(ret)); - } else if (OB_FAIL(pk_interval_.next_value(pk_seq))) { - LOG_WARN("fail to get next pk seq", KR(ret)); } else { - // fill hide pk - datum_row_.storage_datums_[0].set_int(pk_seq); - result_row = &datum_row_; - } - if (OB_SUCC(ret)) { - if (OB_FAIL(dml_row_handler_->handle_insert_row_with_multi_version(tablet_id_, *result_row))) { - LOG_WARN("fail to handle insert row", KR(ret), KPC(result_row)); - } + datum_row = &datum_row_; } } return ret; @@ -797,37 +671,33 @@ int ObDirectLoadPartitionHeapTableMergeTask::init(ObTableLoadTableCtx *ctx, parallel_idx_ = parallel_idx; external_table_ = external_table; pk_interval_ = pk_interval; + need_handle_dml_row_ = true; is_inited_ = true; } return ret; } -int ObDirectLoadPartitionHeapTableMergeTask::construct_row_iter( - ObIAllocator &allocator, ObIStoreRowIterator *&result_row_iter) +int ObDirectLoadPartitionHeapTableMergeTask::construct_row_iters( + ObIArray &row_iters, + ObIAllocator &allocator) { int ret = OB_SUCCESS; - result_row_iter = nullptr; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadPartitionHeapTableMergeTask not init", KR(ret), KP(this)); - } else { - RowIterator *row_iter = nullptr; - if (OB_ISNULL(row_iter = OB_NEWx(RowIterator, (&allocator)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new RowIterator", KR(ret)); - } else if (OB_FAIL(row_iter->init(*merge_param_, merge_ctx_->get_tablet_id(), external_table_, - sql_statistics_, lob_builder_, pk_interval_, - insert_tablet_ctx_))) { - LOG_WARN("fail to init row iter", KR(ret)); - } else { - result_row_iter = row_iter; - } - if (OB_FAIL(ret)) { - if (nullptr != row_iter) { - row_iter->~RowIterator(); - allocator.free(row_iter); - row_iter = nullptr; - } + row_iters.reset(); + RowIterator *row_iter = nullptr; + if (OB_ISNULL(row_iter = OB_NEWx(RowIterator, &allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new RowIterator", KR(ret)); + } else if (OB_FAIL( + row_iter->init(external_table_, merge_param_->table_data_desc_, pk_interval_))) { + LOG_WARN("fail to init row iter", KR(ret)); + } else if (OB_FAIL(row_iters.push_back(row_iter))) { + LOG_WARN("fail to push back", KR(ret)); + } + if (OB_FAIL(ret)) { + if (nullptr != row_iter) { + row_iter->~RowIterator(); + allocator.free(row_iter); + row_iter = nullptr; } } return ret; @@ -838,88 +708,59 @@ int ObDirectLoadPartitionHeapTableMergeTask::construct_row_iter( */ ObDirectLoadPartitionHeapTableMultipleMergeTask::RowIterator::RowIterator() - : deserialize_datums_(nullptr), - deserialize_datum_cnt_(0), - dml_row_handler_(nullptr) + : pk_interval_(nullptr), is_inited_(false) { } ObDirectLoadPartitionHeapTableMultipleMergeTask::RowIterator::~RowIterator() {} int ObDirectLoadPartitionHeapTableMultipleMergeTask::RowIterator::init( - const ObDirectLoadMergeParam &merge_param, - const ObTabletID &tablet_id, - ObDirectLoadMultipleHeapTable *heap_table, - ObTableLoadSqlStatistics *sql_statistics, - ObDirectLoadLobBuilder &lob_builder, - const ObTabletCacheInterval &pk_interval, - ObDirectLoadInsertTabletContext *insert_tablet_ctx) + ObDirectLoadMultipleHeapTable *heap_table, const ObTabletID &tablet_id, + const ObDirectLoadTableDataDesc &table_data_desc, ObTabletCacheInterval &pk_interval) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; - LOG_WARN("ObDirectLoadPartitionHeapTableMultipleMergeTask::RowIterator init twice", KR(ret), - KP(this)); - } else if (OB_UNLIKELY(!merge_param.is_valid() || !tablet_id.is_valid() || - nullptr == heap_table || 0 == pk_interval.count() || - nullptr == insert_tablet_ctx)) { + LOG_WARN("RowIterator init twice", KR(ret), KP(this)); + } else if (OB_UNLIKELY(nullptr == heap_table || !tablet_id.is_valid() || + !table_data_desc.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(merge_param), K(tablet_id), KPC(heap_table), K(pk_interval), - KP(insert_tablet_ctx)); + LOG_WARN("invalid args", KR(ret), KPC(heap_table), K(tablet_id), K(table_data_desc)); } else { - // init row iterator - if (OB_FAIL(inner_init(insert_tablet_ctx, sql_statistics, lob_builder))) { - LOG_WARN("fail to inner init", KR(ret)); - } - // init scanner_ - else if (OB_FAIL(scanner_.init(heap_table, tablet_id, merge_param.table_data_desc_))) { + if (OB_FAIL(scanner_.init(heap_table, tablet_id, table_data_desc))) { LOG_WARN("fail to init tablet whole scanner", KR(ret)); - } - // init datum_row_ - else if (OB_FAIL(insert_tablet_ctx->init_datum_row(datum_row_))) { + } else if (OB_FAIL(datum_row_.init(table_data_desc.column_count_))) { LOG_WARN("fail to init datum row", KR(ret)); } else { - deserialize_datums_ = datum_row_.storage_datums_ + merge_param.rowkey_column_num_ + - ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); - deserialize_datum_cnt_ = merge_param.store_column_count_ - merge_param.rowkey_column_num_; - pk_interval_ = pk_interval; - dml_row_handler_ = merge_param.dml_row_handler_; - tablet_id_ = tablet_id; + pk_interval_ = &pk_interval; + row_flag_.uncontain_hidden_pk_ = true; + row_flag_.has_multi_version_cols_ = false; + row_flag_.has_delete_row_ = false; + column_count_ = table_data_desc.column_count_; is_inited_ = true; } } return ret; } -int ObDirectLoadPartitionHeapTableMultipleMergeTask::RowIterator::inner_get_next_row( - ObDatumRow *&result_row) +int ObDirectLoadPartitionHeapTableMultipleMergeTask::RowIterator::get_next_row( + const ObDatumRow *&datum_row) { int ret = OB_SUCCESS; - result_row = nullptr; + datum_row = nullptr; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadPartitionHeapTableMultipleMergeTask::RowIterator not init", KR(ret), - KP(this)); + LOG_WARN("RowIterator not init", KR(ret), KP(this)); } else { const ObDirectLoadMultipleExternalRow *external_row = nullptr; - uint64_t pk_seq = OB_INVALID_ID; if (OB_FAIL(scanner_.get_next_row(external_row))) { if (OB_UNLIKELY(OB_ITER_END != ret)) { LOG_WARN("fail to get next row", KR(ret)); } - } else if (OB_FAIL(external_row->to_datums(deserialize_datums_, deserialize_datum_cnt_))) { + } else if (OB_FAIL(external_row->to_datums(datum_row_.storage_datums_, datum_row_.count_))) { LOG_WARN("fail to transfer datum row", KR(ret)); - } else if (OB_FAIL(pk_interval_.next_value(pk_seq))) { - LOG_WARN("fail to get next pk seq", KR(ret)); } else { - // fill hide pk - datum_row_.storage_datums_[0].set_int(pk_seq); - result_row = &datum_row_; - } - if (OB_SUCC(ret)) { - if (OB_FAIL(dml_row_handler_->handle_insert_row_with_multi_version(tablet_id_, *result_row))) { - LOG_WARN("fail to handle insert row", KR(ret), KPC(result_row)); - } + datum_row = &datum_row_; } } return ret; @@ -933,21 +774,19 @@ ObDirectLoadPartitionHeapTableMultipleMergeTask::~ObDirectLoadPartitionHeapTable { } -int ObDirectLoadPartitionHeapTableMultipleMergeTask::init( - ObTableLoadTableCtx *ctx, - const ObDirectLoadMergeParam &merge_param, - ObDirectLoadTabletMergeCtx *merge_ctx, - ObDirectLoadMultipleHeapTable *heap_table, - const ObTabletCacheInterval &pk_interval, - int64_t parallel_idx) +int ObDirectLoadPartitionHeapTableMultipleMergeTask::init(ObTableLoadTableCtx *ctx, + const ObDirectLoadMergeParam &merge_param, + ObDirectLoadTabletMergeCtx *merge_ctx, + ObDirectLoadMultipleHeapTable *heap_table, + const ObTabletCacheInterval &pk_interval, + int64_t parallel_idx) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObDirectLoadPartitionHeapTableMultipleMergeTask init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(nullptr == ctx || !merge_param.is_valid() || - nullptr == merge_ctx || nullptr == heap_table || - parallel_idx < 0 || 0 == pk_interval.count())) { + } else if (OB_UNLIKELY(nullptr == ctx || !merge_param.is_valid() || nullptr == merge_ctx || + nullptr == heap_table || parallel_idx < 0 || 0 == pk_interval.count())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(merge_param), KP(merge_ctx), KPC(heap_table), K(parallel_idx), K(pk_interval)); @@ -958,37 +797,33 @@ int ObDirectLoadPartitionHeapTableMultipleMergeTask::init( parallel_idx_ = parallel_idx; heap_table_ = heap_table; pk_interval_ = pk_interval; + need_handle_dml_row_ = true; is_inited_ = true; } return ret; } -int ObDirectLoadPartitionHeapTableMultipleMergeTask::construct_row_iter( - ObIAllocator &allocator, ObIStoreRowIterator *&result_row_iter) +int ObDirectLoadPartitionHeapTableMultipleMergeTask::construct_row_iters( + ObIArray &row_iters, + ObIAllocator &allocator) { int ret = OB_SUCCESS; - result_row_iter = nullptr; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadPartitionHeapTableMultipleMergeTask not init", KR(ret), KP(this)); - } else { - RowIterator *row_iter = nullptr; - if (OB_ISNULL(row_iter = OB_NEWx(RowIterator, (&allocator)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new RowIterator", KR(ret)); - } else if (OB_FAIL(row_iter->init(*merge_param_, merge_ctx_->get_tablet_id(), heap_table_, - sql_statistics_, lob_builder_, pk_interval_, - insert_tablet_ctx_))) { - LOG_WARN("fail to init row iter", KR(ret)); - } else { - result_row_iter = row_iter; - } - if (OB_FAIL(ret)) { - if (nullptr != row_iter) { - row_iter->~RowIterator(); - allocator.free(row_iter); - row_iter = nullptr; - } + row_iters.reset(); + RowIterator *row_iter = nullptr; + if (OB_ISNULL(row_iter = OB_NEWx(RowIterator, &allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new RowIterator", KR(ret)); + } else if (OB_FAIL(row_iter->init(heap_table_, merge_ctx_->get_tablet_id(), + merge_param_->table_data_desc_, pk_interval_))) { + LOG_WARN("fail to init row iter", KR(ret)); + } else if (OB_FAIL(row_iters.push_back(row_iter))) { + LOG_WARN("fail to push back", KR(ret)); + } + if (OB_FAIL(ret)) { + if (nullptr != row_iter) { + row_iter->~RowIterator(); + allocator.free(row_iter); + row_iter = nullptr; } } return ret; @@ -999,136 +834,58 @@ int ObDirectLoadPartitionHeapTableMultipleMergeTask::construct_row_iter( */ ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::RowIterator() - : allocator_("TLD_RowIter"), - origin_iter_(nullptr), - rowkey_column_num_(0), - store_column_count_(0), - heap_table_array_(nullptr), - pos_(0), - deserialize_datums_(nullptr), - deserialize_datum_cnt_(0), - dml_row_handler_(nullptr) + : heap_table_array_(nullptr), pk_interval_(nullptr), pos_(-1), is_inited_(false) { - allocator_.set_tenant_id(MTL_ID()); } -ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::~RowIterator() -{ - if (nullptr != origin_iter_) { - origin_iter_->~ObIStoreRowIterator(); - origin_iter_ = nullptr; - } -} +ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::~RowIterator() {} int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::init( - const ObDirectLoadMergeParam &merge_param, - ObDirectLoadTabletMergeCtx *merge_ctx, - ObDirectLoadOriginTable *origin_table, - ObTableLoadSqlStatistics *sql_statistics, - ObDirectLoadLobBuilder &lob_builder, - const ObIArray *heap_table_array, - const ObTabletCacheInterval &pk_interval, - ObDirectLoadInsertTabletContext *insert_tablet_ctx) + const ObIArray *heap_table_array, const ObTabletID &tablet_id, + const ObDirectLoadTableDataDesc &table_data_desc, ObTabletCacheInterval &pk_interval) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; - LOG_WARN("ObDirectLoadPartitionHeapTableMultipleMergeTask::RowIterator init twice", KR(ret), - KP(this)); - } else if (OB_UNLIKELY(!merge_param.is_valid() || nullptr == merge_ctx || - nullptr == origin_table || nullptr == heap_table_array || - nullptr == insert_tablet_ctx)) { + LOG_WARN("RowIterator init twice", KR(ret), KP(this)); + } else if (OB_UNLIKELY(nullptr == heap_table_array || !tablet_id.is_valid() || + !table_data_desc.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(merge_param), KP(merge_ctx), KP(origin_table), - KP(heap_table_array), KP(insert_tablet_ctx)); + LOG_WARN("invalid args", KR(ret), KP(heap_table_array), K(tablet_id), K(table_data_desc)); } else { - range_.set_whole_range(); - // init row iterator - if (OB_FAIL(inner_init(insert_tablet_ctx, sql_statistics, lob_builder))) { - LOG_WARN("fail to inner init", KR(ret)); - } - // init datum_row_ - else if (OB_FAIL(insert_tablet_ctx->init_datum_row(datum_row_))) { + if (OB_FAIL(datum_row_.init(table_data_desc.column_count_))) { LOG_WARN("fail to init datum row", KR(ret)); - } else if (merge_ctx->merge_with_origin_data()) { - if (OB_FAIL(origin_table->scan(range_, allocator_, origin_iter_, false/*skip_read_lob*/))) { - LOG_WARN("fail to scan origin table", KR(ret)); - } - } - if (OB_SUCC(ret)) { - deserialize_datums_ = datum_row_.storage_datums_ + merge_param.rowkey_column_num_ + - ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); - deserialize_datum_cnt_ = merge_param.store_column_count_ - merge_param.rowkey_column_num_; - rowkey_column_num_ = merge_param.rowkey_column_num_; - store_column_count_ = merge_param.store_column_count_; - tablet_id_ = merge_ctx->get_tablet_id(); - table_data_desc_ = merge_param.table_data_desc_; + } else { + pk_interval_ = &pk_interval; heap_table_array_ = heap_table_array; - pk_interval_ = pk_interval; - dml_row_handler_ = merge_param.dml_row_handler_; + tablet_id_ = tablet_id; + table_data_desc_ = table_data_desc; + row_flag_.uncontain_hidden_pk_ = true; + row_flag_.has_multi_version_cols_ = false; + row_flag_.has_delete_row_ = false; + column_count_ = table_data_desc.column_count_; is_inited_ = true; } } return ret; } -int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::inner_get_next_row( - ObDatumRow *&result_row) +int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::get_next_row( + const ObDatumRow *&datum_row) { int ret = OB_SUCCESS; - result_row = nullptr; + datum_row = nullptr; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator not init", - KR(ret), KP(this)); + LOG_WARN("RowIterator not init", KR(ret), KP(this)); } else { - if (pos_ == 0) { - if (origin_iter_ == nullptr) { - if (OB_FAIL(switch_next_heap_table())) { - if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("fail to switch next heap table", KR(ret)); - } - } - } else { - // get row from origin table - const ObDatumRow *datum_row = nullptr; - if (OB_FAIL(origin_iter_->get_next_row(datum_row))) { - if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("fail to get next row", KR(ret)); - } else { - // switch heap table - ret = OB_SUCCESS; - if (OB_FAIL(switch_next_heap_table())) { - if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("fail to switch next heap table", KR(ret)); - } - } - } - } else if (OB_UNLIKELY(datum_row->count_ != store_column_count_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected column count", KR(ret), K(store_column_count_), KPC(datum_row)); - } else { - // copy rowkey columns - for (int64_t i = 0; i < rowkey_column_num_; ++i) { - datum_row_.storage_datums_[i] = datum_row->storage_datums_[i]; - } - // copy normal columns - for (int64_t - i = rowkey_column_num_, - j = rowkey_column_num_ + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); - i < datum_row->count_; ++i, ++j) { - datum_row_.storage_datums_[j] = datum_row->storage_datums_[i]; - } - datum_row_.row_flag_.set_flag(datum_row->row_flag_.is_delete() ? blocksstable::DF_DELETE : blocksstable::DF_INSERT, - datum_row->row_flag_.is_delete() || !insert_tablet_ctx_->is_incremental() ? DF_TYPE_NORMAL : DF_TYPE_INSERT_DELETE); - result_row = &datum_row_; - } + const ObDirectLoadMultipleExternalRow *external_row = nullptr; + if (pos_ < 0 && OB_FAIL(switch_next_heap_table())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail to switch next heap table", KR(ret)); } } - // get row from load data - while (OB_SUCC(ret) && result_row == nullptr) { - const ObDirectLoadMultipleExternalRow *external_row = nullptr; - uint64_t pk_seq = OB_INVALID_ID; + while (OB_SUCC(ret) && datum_row == nullptr) { if (OB_FAIL(scanner_.get_next_row(external_row))) { if (OB_UNLIKELY(OB_ITER_END != ret)) { LOG_WARN("fail to get next row", KR(ret)); @@ -1141,19 +898,10 @@ int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::inner } } } - } else if (OB_FAIL(external_row->to_datums(deserialize_datums_, deserialize_datum_cnt_))) { + } else if (OB_FAIL(external_row->to_datums(datum_row_.storage_datums_, datum_row_.count_))) { LOG_WARN("fail to transfer datum row", KR(ret)); - } else if (OB_FAIL(pk_interval_.next_value(pk_seq))) { - LOG_WARN("fail to get next pk seq", KR(ret)); } else { - // fill hide pk - datum_row_.storage_datums_[0].set_int(pk_seq); - result_row = &datum_row_; - } - if (OB_SUCC(ret) && nullptr != result_row) { - if (OB_FAIL(dml_row_handler_->handle_insert_row_with_multi_version(tablet_id_, *result_row))) { - LOG_WARN("fail to handle insert row", KR(ret), KPC(result_row)); - } + datum_row = &datum_row_; } } } @@ -1163,6 +911,7 @@ int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::inner int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::switch_next_heap_table() { int ret = OB_SUCCESS; + pos_ = (pos_ < 0 ? 0 : pos_ + 1); if (pos_ >= heap_table_array_->count()) { ret = OB_ITER_END; } else { @@ -1172,8 +921,6 @@ int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::RowIterator::switc new (&scanner_) ObDirectLoadMultipleHeapTableTabletWholeScanner(); if (OB_FAIL(scanner_.init(heap_table, tablet_id_, table_data_desc_))) { LOG_WARN("fail to init scanner", KR(ret)); - } else { - ++pos_; } } return ret; @@ -1191,10 +938,8 @@ ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask:: } int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::init( - ObTableLoadTableCtx *ctx, - const ObDirectLoadMergeParam &merge_param, - ObDirectLoadTabletMergeCtx *merge_ctx, - ObDirectLoadOriginTable *origin_table, + ObTableLoadTableCtx *ctx, const ObDirectLoadMergeParam &merge_param, + ObDirectLoadTabletMergeCtx *merge_ctx, ObDirectLoadOriginTable *origin_table, const ObIArray &heap_table_array, const ObTabletCacheInterval &pk_interval) { @@ -1215,31 +960,44 @@ int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::init( origin_table_ = origin_table; heap_table_array_ = &heap_table_array; pk_interval_ = pk_interval; + need_handle_dml_row_ = true; is_inited_ = true; } return ret; } -int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::construct_row_iter( - ObIAllocator &allocator, ObIStoreRowIterator *&result_row_iter) +int ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask::construct_row_iters( + ObIArray &row_iters, + ObIAllocator &allocator) { int ret = OB_SUCCESS; - result_row_iter = nullptr; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask not init", KR(ret), - KP(this)); - } else { + row_iters.reset(); + if (merge_ctx_->merge_with_origin_data()) { + ObDirectLoadIStoreRowIterator *data_iter = nullptr; + range_.set_whole_range(); + if (OB_FAIL(origin_table_->scan(range_, allocator, data_iter, false /*skip_read_lob*/))) { + LOG_WARN("fail to scan origin table", KR(ret)); + } else if (OB_FAIL(row_iters.push_back(data_iter))) { + LOG_WARN("fail to push back", KR(ret)); + } + if (OB_FAIL(ret)) { + if (nullptr != data_iter) { + data_iter->~ObDirectLoadIStoreRowIterator(); + allocator.free(data_iter); + data_iter = nullptr; + } + } + } + if (OB_SUCC(ret)) { RowIterator *row_iter = nullptr; - if (OB_ISNULL(row_iter = OB_NEWx(RowIterator, (&allocator)))) { + if (OB_ISNULL(row_iter = OB_NEWx(RowIterator, &allocator))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to new RowIterator", KR(ret)); - } else if (OB_FAIL(row_iter->init(*merge_param_, merge_ctx_, origin_table_, - sql_statistics_, lob_builder_, heap_table_array_, - pk_interval_, insert_tablet_ctx_))) { + } else if (OB_FAIL(row_iter->init(heap_table_array_, merge_ctx_->get_tablet_id(), + merge_param_->table_data_desc_, pk_interval_))) { LOG_WARN("fail to init row iter", KR(ret)); - } else { - result_row_iter = row_iter; + } else if (OB_FAIL(row_iters.push_back(row_iter))) { + LOG_WARN("fail to push back", KR(ret)); } if (OB_FAIL(ret)) { if (nullptr != row_iter) { diff --git a/src/storage/direct_load/ob_direct_load_partition_merge_task.h b/src/storage/direct_load/ob_direct_load_partition_merge_task.h index 0f8ecbe30c..48215455fe 100644 --- a/src/storage/direct_load/ob_direct_load_partition_merge_task.h +++ b/src/storage/direct_load/ob_direct_load_partition_merge_task.h @@ -13,24 +13,18 @@ #include "lib/list/ob_dlist.h" #include "share/ob_tablet_autoincrement_param.h" -#include "storage/direct_load/ob_direct_load_data_fuse.h" +#include "storage/direct_load/ob_direct_load_external_row.h" #include "storage/direct_load/ob_direct_load_external_scanner.h" #include "storage/direct_load/ob_direct_load_merge_ctx.h" #include "storage/direct_load/ob_direct_load_multiple_heap_table_scanner.h" -#include "storage/direct_load/ob_direct_load_insert_table_row_iterator.h" -#include "storage/direct_load/ob_direct_load_lob_builder.h" -#include "observer/table_load/ob_table_load_table_ctx.h" +#include "storage/direct_load/ob_direct_load_row_iterator.h" namespace oceanbase { -namespace blocksstable +namespace observer { -class ObDatumRange; -} // namespace blocksstable -namespace table -{ -class ObTableLoadSqlStatistics; -} // namespace table +class ObTableLoadTableCtx; +} // namespace observer namespace storage { class ObDirectLoadOriginTable; @@ -39,6 +33,7 @@ class ObDirectLoadMultipleSSTable; class ObDirectLoadMultipleHeapTable; class ObDirectLoadSSTableConflictCheck; class ObDirectLoadMultipleSSTableConflictCheck; +class ObDirectLoadInsertTabletContext; class ObDirectLoadPartitionMergeTask : public common::ObDLinkBase { @@ -49,24 +44,70 @@ public: int64_t get_row_count() const { return affected_rows_; } void stop(); TO_STRING_KV(KPC_(merge_param), KPC_(merge_ctx), K_(parallel_idx)); + protected: - virtual int construct_row_iter(common::ObIAllocator &allocator, - ObIStoreRowIterator *&row_iter) = 0; + virtual int construct_row_iters(ObIArray &row_iters, + ObIAllocator &allocator) = 0; virtual int finish_check() { return OB_SUCCESS; } + +private: + int fill_sstable_slice(const int64_t slice_id, + const ObIArray &row_iters); + int fill_sstable_slice_batch(const int64_t slice_id, + const ObIArray &row_iters); + protected: observer::ObTableLoadTableCtx *ctx_; const ObDirectLoadMergeParam *merge_param_; ObDirectLoadTabletMergeCtx *merge_ctx_; int64_t parallel_idx_; int64_t affected_rows_; - ObDirectLoadLobBuilder lob_builder_; - common::ObArenaAllocator allocator_; ObDirectLoadInsertTabletContext *insert_tablet_ctx_; - table::ObTableLoadSqlStatistics *sql_statistics_; + // 无主键表的hidden pk是在最后插入的时候才组装到datum_row的, 在这之前都无法调用handle_insert_row + // 需要在最后插入的时候收集 + bool need_handle_dml_row_; bool is_stop_; bool is_inited_; }; +class ObDirectLoadPartitionEmptyMergeTask : public ObDirectLoadPartitionMergeTask +{ +public: + ObDirectLoadPartitionEmptyMergeTask() = default; + virtual ~ObDirectLoadPartitionEmptyMergeTask() = default; + int init(observer::ObTableLoadTableCtx *ctx, + const ObDirectLoadMergeParam &merge_param, + ObDirectLoadTabletMergeCtx *merge_ctx); + +protected: + int construct_row_iters(ObIArray &row_iters, + ObIAllocator &allocator) override + { + return OB_SUCCESS; + } +}; + +class ObDirectLoadPartitionOriginDataMergeTask : public ObDirectLoadPartitionMergeTask +{ +public: + ObDirectLoadPartitionOriginDataMergeTask(); + virtual ~ObDirectLoadPartitionOriginDataMergeTask(); + int init(observer::ObTableLoadTableCtx *ctx, + const ObDirectLoadMergeParam &merge_param, + ObDirectLoadTabletMergeCtx *merge_ctx, + ObDirectLoadOriginTable *origin_table, + const blocksstable::ObDatumRange &range, + int64_t parallel_idx); + +protected: + int construct_row_iters(ObIArray &row_iters, + ObIAllocator &allocator) override; + +private: + ObDirectLoadOriginTable *origin_table_; + const blocksstable::ObDatumRange *range_; +}; + class ObDirectLoadPartitionRangeMergeTask : public ObDirectLoadPartitionMergeTask { public: @@ -79,36 +120,17 @@ public: const common::ObIArray &sstable_array, const blocksstable::ObDatumRange &range, int64_t parallel_idx); + protected: - int construct_row_iter(common::ObIAllocator &allocator, ObIStoreRowIterator *&row_iter) override; + int construct_row_iters(ObIArray &row_iters, + ObIAllocator &allocator) override; int finish_check() override; -private: - class RowIterator : public ObDirectLoadInsertTableRowIterator - { - public: - RowIterator(); - virtual ~RowIterator(); - int init(const ObDirectLoadMergeParam &merge_param, - ObDirectLoadTabletMergeCtx *merge_ctx, - ObDirectLoadOriginTable *origin_table, - table::ObTableLoadSqlStatistics *sql_statistics, - ObDirectLoadLobBuilder &lob_builder, - const common::ObIArray &sstable_array, - const blocksstable::ObDatumRange &range, - ObDirectLoadInsertTabletContext *insert_tablet_ctx); - int inner_get_next_row(blocksstable::ObDatumRow *&datum_row) override; - ObLobId get_max_del_lob_id() const; - private: - ObIStoreRowIterator *data_iter_; - ObDirectLoadSSTableConflictCheck *conflict_check_; - blocksstable::ObDatumRow datum_row_; - int64_t rowkey_column_num_; - }; + private: ObDirectLoadOriginTable *origin_table_; const ObIArray *sstable_array_; const blocksstable::ObDatumRange *range_; - RowIterator *row_iter_; + ObDirectLoadSSTableConflictCheck *conflict_check_; }; class ObDirectLoadPartitionRangeMultipleMergeTask : public ObDirectLoadPartitionMergeTask @@ -123,36 +145,17 @@ public: const common::ObIArray &sstable_array, const blocksstable::ObDatumRange &range, int64_t parallel_idx); + protected: - int construct_row_iter(common::ObIAllocator &allocator, ObIStoreRowIterator *&row_iter) override; + int construct_row_iters(ObIArray &row_iters, + ObIAllocator &allocator) override; int finish_check() override; -private: - class RowIterator : public ObDirectLoadInsertTableRowIterator - { - public: - RowIterator(); - virtual ~RowIterator(); - int init(const ObDirectLoadMergeParam &merge_param, - ObDirectLoadTabletMergeCtx *merge_ctx, - ObDirectLoadOriginTable *origin_table, - table::ObTableLoadSqlStatistics *sql_statistics, - ObDirectLoadLobBuilder &lob_builder, - const common::ObIArray &sstable_array, - const blocksstable::ObDatumRange &range, - ObDirectLoadInsertTabletContext *insert_tablet_ctx); - int inner_get_next_row(blocksstable::ObDatumRow *&datum_row) override; - ObLobId get_max_del_lob_id() const; - private: - ObIStoreRowIterator *data_iter_; - ObDirectLoadMultipleSSTableConflictCheck *conflict_check_; - blocksstable::ObDatumRow datum_row_; - int64_t rowkey_column_num_; - }; + private: ObDirectLoadOriginTable *origin_table_; const ObIArray *sstable_array_; const blocksstable::ObDatumRange *range_; - RowIterator *row_iter_; + ObDirectLoadMultipleSSTableConflictCheck *conflict_check_; }; class ObDirectLoadPartitionHeapTableMergeTask : public ObDirectLoadPartitionMergeTask @@ -166,31 +169,33 @@ public: ObDirectLoadExternalTable *external_table, const share::ObTabletCacheInterval &pk_interval, int64_t parallel_idx); + protected: - int construct_row_iter(common::ObIAllocator &allocator, ObIStoreRowIterator *&row_iter) override; + int construct_row_iters(ObIArray &row_iters, + ObIAllocator &allocator) override; + private: - class RowIterator : public ObDirectLoadInsertTableRowIterator + class RowIterator final : public ObDirectLoadIStoreRowIterator { public: RowIterator(); virtual ~RowIterator(); - int init(const ObDirectLoadMergeParam &merge_param, - const common::ObTabletID &tablet_id, - ObDirectLoadExternalTable *external_table, - table::ObTableLoadSqlStatistics *sql_statistics, - ObDirectLoadLobBuilder &lob_builder, - const share::ObTabletCacheInterval &pk_interval, - ObDirectLoadInsertTabletContext *insert_tablet_ctx); - int inner_get_next_row(blocksstable::ObDatumRow *&datum_row) override; + int init(ObDirectLoadExternalTable *external_table, + const ObDirectLoadTableDataDesc &table_data_desc, + share::ObTabletCacheInterval &pk_interval); + share::ObTabletCacheInterval *get_hide_pk_interval() const override { return pk_interval_; } + int get_next_row(const blocksstable::ObDatumRow *&datum_row) override; + INHERIT_TO_STRING_KV("IStoreRowIterator", ObDirectLoadIStoreRowIterator, + KPC_(pk_interval), + K_(is_inited)); + private: + share::ObTabletCacheInterval *pk_interval_; ObDirectLoadExternalSequentialScanner scanner_; blocksstable::ObDatumRow datum_row_; - blocksstable::ObStorageDatum *deserialize_datums_; - int64_t deserialize_datum_cnt_; - share::ObTabletCacheInterval pk_interval_; - ObDirectLoadDMLRowHandler *dml_row_handler_; - ObTabletID tablet_id_; + bool is_inited_; }; + private: ObDirectLoadExternalTable *external_table_; share::ObTabletCacheInterval pk_interval_; @@ -207,31 +212,34 @@ public: ObDirectLoadMultipleHeapTable *heap_table, const share::ObTabletCacheInterval &pk_interval, int64_t parallel_idx); + protected: - int construct_row_iter(common::ObIAllocator &allocator, ObIStoreRowIterator *&row_iter) override; + int construct_row_iters(ObIArray &row_iters, + ObIAllocator &allocator) override; + private: - class RowIterator : public ObDirectLoadInsertTableRowIterator + class RowIterator final : public ObDirectLoadIStoreRowIterator { public: RowIterator(); virtual ~RowIterator(); - int init(const ObDirectLoadMergeParam &merge_param, + int init(ObDirectLoadMultipleHeapTable *heap_table, const common::ObTabletID &tablet_id, - ObDirectLoadMultipleHeapTable *heap_table, - table::ObTableLoadSqlStatistics *sql_statistics, - ObDirectLoadLobBuilder &lob_builder, - const share::ObTabletCacheInterval &pk_interval, - ObDirectLoadInsertTabletContext *insert_tablet_ctx); - int inner_get_next_row(blocksstable::ObDatumRow *&datum_row) override; + const ObDirectLoadTableDataDesc &table_data_desc, + share::ObTabletCacheInterval &pk_interval); + share::ObTabletCacheInterval *get_hide_pk_interval() const override { return pk_interval_; } + int get_next_row(const blocksstable::ObDatumRow *&datum_row) override; + INHERIT_TO_STRING_KV("IStoreRowIterator", ObDirectLoadIStoreRowIterator, + KPC_(pk_interval), + K_(is_inited)); + private: + share::ObTabletCacheInterval *pk_interval_; ObDirectLoadMultipleHeapTableTabletWholeScanner scanner_; blocksstable::ObDatumRow datum_row_; - blocksstable::ObStorageDatum *deserialize_datums_; - int64_t deserialize_datum_cnt_; - share::ObTabletCacheInterval pk_interval_; - ObDirectLoadDMLRowHandler *dml_row_handler_; - ObTabletID tablet_id_; + bool is_inited_; }; + private: ObDirectLoadMultipleHeapTable *heap_table_; share::ObTabletCacheInterval pk_interval_; @@ -244,52 +252,52 @@ public: ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask(); virtual ~ObDirectLoadPartitionHeapTableMultipleAggregateMergeTask(); int init(observer::ObTableLoadTableCtx *ctx, - const ObDirectLoadMergeParam &merge_param, ObDirectLoadTabletMergeCtx *merge_ctx, + const ObDirectLoadMergeParam &merge_param, + ObDirectLoadTabletMergeCtx *merge_ctx, ObDirectLoadOriginTable *origin_table, const common::ObIArray &heap_table_array, const share::ObTabletCacheInterval &pk_interval); + protected: - int construct_row_iter(common::ObIAllocator &allocator, ObIStoreRowIterator *&row_iter) override; + int construct_row_iters(ObIArray &row_iters, + ObIAllocator &allocator) override; + private: - class RowIterator : public ObDirectLoadInsertTableRowIterator + class RowIterator final : public ObDirectLoadIStoreRowIterator { public: RowIterator(); virtual ~RowIterator(); - int init(const ObDirectLoadMergeParam &merge_param, - ObDirectLoadTabletMergeCtx *merge_ctx, - ObDirectLoadOriginTable *origin_table, - table::ObTableLoadSqlStatistics *sql_statistics, - ObDirectLoadLobBuilder &lob_builder, - const common::ObIArray *heap_table_array, - const share::ObTabletCacheInterval &pk_interval, - ObDirectLoadInsertTabletContext *insert_tablet_ctx); - int inner_get_next_row(blocksstable::ObDatumRow *&datum_row) override; + int init(const common::ObIArray *heap_table_array, + const common::ObTabletID &tablet_id, + const ObDirectLoadTableDataDesc &table_data_desc, + share::ObTabletCacheInterval &pk_interval); + share::ObTabletCacheInterval *get_hide_pk_interval() const override { return pk_interval_; } + int get_next_row(const blocksstable::ObDatumRow *&datum_row) override; + INHERIT_TO_STRING_KV("IStoreRowIterator", ObDirectLoadIStoreRowIterator, + K_(tablet_id), + KPC_(pk_interval), + K_(is_inited)); + private: int switch_next_heap_table(); + private: - // for iter origin table - common::ObArenaAllocator allocator_; - blocksstable::ObDatumRange range_; - ObIStoreRowIterator *origin_iter_; - int64_t rowkey_column_num_; - int64_t store_column_count_; - // for iter multiple heap table + const common::ObIArray *heap_table_array_; common::ObTabletID tablet_id_; ObDirectLoadTableDataDesc table_data_desc_; - const common::ObIArray *heap_table_array_; - int64_t pos_; + share::ObTabletCacheInterval *pk_interval_; ObDirectLoadMultipleHeapTableTabletWholeScanner scanner_; blocksstable::ObDatumRow datum_row_; - blocksstable::ObStorageDatum *deserialize_datums_; - int64_t deserialize_datum_cnt_; - share::ObTabletCacheInterval pk_interval_; - ObDirectLoadDMLRowHandler *dml_row_handler_; + int64_t pos_; + bool is_inited_; }; + private: ObDirectLoadOriginTable *origin_table_; const common::ObIArray *heap_table_array_; share::ObTabletCacheInterval pk_interval_; + blocksstable::ObDatumRange range_; // for origin iter }; } // namespace storage diff --git a/src/storage/direct_load/ob_direct_load_row_iterator.cpp b/src/storage/direct_load/ob_direct_load_row_iterator.cpp new file mode 100644 index 0000000000..2b33b62f1d --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_row_iterator.cpp @@ -0,0 +1,56 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX STORAGE + +#include "storage/direct_load/ob_direct_load_row_iterator.h" + +namespace oceanbase +{ +namespace storage +{ +using namespace blocksstable; +using namespace common; + +int ObDirectLoadStoreRowIteratorWrap::init(ObIStoreRowIterator *iter, + const ObDirectLoadRowFlag &row_flag, + const int64_t column_count) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObDirectLoadStoreRowIteratorWrap init twice", KR(ret), KP(this)); + } else if (OB_UNLIKELY(nullptr == iter || column_count <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(iter), K(row_flag), K(column_count)); + } else { + iter_ = iter; + row_flag_ = row_flag; + column_count_ = column_count; + is_inited_ = true; + } + return ret; +} + +int ObDirectLoadStoreRowIteratorWrap::get_next_row(const blocksstable::ObDatumRow *&row) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObDirectLoadStoreRowIteratorWrap not init", KR(ret), KP(this)); + } else { + ret = iter_->get_next_row(row); + } + return ret; +} + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_row_iterator.h b/src/storage/direct_load/ob_direct_load_row_iterator.h new file mode 100644 index 0000000000..06cd6056b0 --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_row_iterator.h @@ -0,0 +1,96 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once + +#include "storage/access/ob_store_row_iterator.h" + +namespace oceanbase +{ +namespace share +{ +class ObTabletCacheInterval; +} // namespace share +namespace storage +{ + +struct ObDirectLoadRowFlag +{ +public: + ObDirectLoadRowFlag() : flag_(0) {} + void reset() { flag_ = 0; } + OB_INLINE int64_t get_column_count(const int64_t column_count) const + { + return uncontain_hidden_pk_ ? column_count + 1 : column_count; + } + TO_STRING_KV(K_(uncontain_hidden_pk), K_(has_multi_version_cols), K_(has_delete_row)); +public: + union + { + struct + { + bool uncontain_hidden_pk_ : 1; // used for heap table + bool has_multi_version_cols_ : 1; + bool has_delete_row_ : 1; // TODO: 后续维护这个flag + int64_t reserved_ : 61; + }; + int64_t flag_; + }; +}; + +class ObDirectLoadIStoreRowIterator : public ObIStoreRowIterator +{ +public: + ObDirectLoadIStoreRowIterator() : row_flag_(), column_count_(0), is_batch_result_(false) {} + virtual ~ObDirectLoadIStoreRowIterator() = default; + virtual int get_next_batch(const IVectorPtrs *&vectors, int64_t &batch_size) + { + return OB_NOT_SUPPORTED; + } + ObDirectLoadRowFlag &get_row_flag() { return row_flag_; } + const ObDirectLoadRowFlag &get_row_flag() const { return row_flag_; } + int64_t get_column_count() const { return column_count_; } + bool is_batch_result() const { return is_batch_result_; } + virtual share::ObTabletCacheInterval *get_hide_pk_interval() const { return nullptr; } + virtual bool is_valid() const + { + return (!row_flag_.uncontain_hidden_pk_ || nullptr != get_hide_pk_interval()) && + column_count_ > 0; + } + VIRTUAL_TO_STRING_KV(K_(row_flag), + K_(column_count), + K_(is_batch_result)); +protected: + ObDirectLoadRowFlag row_flag_; + int64_t column_count_; + bool is_batch_result_; +}; + +class ObDirectLoadStoreRowIteratorWrap final : public ObDirectLoadIStoreRowIterator +{ +public: + ObDirectLoadStoreRowIteratorWrap() : iter_(nullptr), is_inited_(false) {} + virtual ~ObDirectLoadStoreRowIteratorWrap() = default; + int init(ObIStoreRowIterator *iter, + const ObDirectLoadRowFlag &row_flag, + const int64_t column_count); + int get_next_row(const blocksstable::ObDatumRow *&row) override; + INHERIT_TO_STRING_KV("IStoreRowIterator", ObDirectLoadIStoreRowIterator, + KP_(iter), + K_(is_inited)); +private: + ObIStoreRowIterator *iter_; + bool is_inited_; +}; + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_table_builder_allocator.h b/src/storage/direct_load/ob_direct_load_table_builder_allocator.h deleted file mode 100644 index c88af6622f..0000000000 --- a/src/storage/direct_load/ob_direct_load_table_builder_allocator.h +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ -#pragma once - -#include "lib/list/ob_dlist.h" -#include "share/rc/ob_tenant_base.h" -#include "storage/direct_load/ob_direct_load_i_table.h" - -namespace oceanbase -{ -namespace storage -{ - -class ObDirectLoadTableBuilderAllocator final -{ -public: - ObDirectLoadTableBuilderAllocator() - { - tid_ = get_tid_cache(); - } - ~ObDirectLoadTableBuilderAllocator() - { - assert_in_own_thread(); - OB_ASSERT(using_list_.is_empty()); - } - void reset() - { - assert_in_own_thread(); - Item *item = nullptr; - DLIST_REMOVE_ALL_NORET(item, using_list_) - { - ObIDirectLoadPartitionTableBuilder *table_builder = - (ObIDirectLoadPartitionTableBuilder *)item->buf_; - table_builder->~ObIDirectLoadPartitionTableBuilder(); - ob_free(item); - } - OB_ASSERT(using_list_.is_empty()); - } - template - T *alloc(Args &&... args) - { - assert_in_own_thread(); - T *t = nullptr; - void *buf = nullptr; - ObMemAttr attr(MTL_ID(), "TLD_TB_Alloc"); - if (OB_NOT_NULL(buf = ob_malloc(sizeof(Item) + sizeof(T), attr))) { - Item *item = new (buf) Item; - t = new (item->buf_) T(args...); - using_list_.add_last(item); - } - return t; - } - void free(ObIDirectLoadPartitionTableBuilder *table_builder) - { - assert_in_own_thread(); - if (OB_NOT_NULL(table_builder)) { - table_builder->~ObIDirectLoadPartitionTableBuilder(); - Item *item = (Item *)table_builder - 1; - using_list_.remove(item); - item->~Item(); - ob_free(item); - } - } - OB_INLINE void assert_in_own_thread() - { - const int64_t tid = get_tid_cache(); - OB_ASSERT(tid == tid_); - } - -private: - struct Item : public common::ObDLinkBase - { - char buf_[]; - }; - -private: - int64_t tid_; - ObDList using_list_; -}; - -OB_INLINE ObDirectLoadTableBuilderAllocator *get_table_builder_allocator() -{ - RLOCAL_INLINE(ObDirectLoadTableBuilderAllocator, allcator); - return &allcator; -} - -} // namespace storage -} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_table_store.cpp b/src/storage/direct_load/ob_direct_load_table_store.cpp deleted file mode 100644 index 2bcde21ee3..0000000000 --- a/src/storage/direct_load/ob_direct_load_table_store.cpp +++ /dev/null @@ -1,367 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ -#define USING_LOG_PREFIX STORAGE - -#include "storage/direct_load/ob_direct_load_table_store.h" -#include "observer/table_load/ob_table_load_stat.h" -#include "storage/direct_load/ob_direct_load_external_multi_partition_table.h" -#include "storage/direct_load/ob_direct_load_fast_heap_table_builder.h" -#include "storage/direct_load/ob_direct_load_multiple_sstable_builder.h" -#include "storage/direct_load/ob_direct_load_table_builder_allocator.h" - -namespace oceanbase -{ -namespace storage -{ -using namespace common; -using namespace blocksstable; -using namespace share; -using namespace table; - -/** - * ObDirectLoadTableStoreParam - */ - -ObDirectLoadTableStoreParam::ObDirectLoadTableStoreParam() - : datum_utils_(nullptr), - file_mgr_(nullptr), - is_multiple_mode_(false), - is_fast_heap_table_(false), - insert_table_ctx_(nullptr), - dml_row_handler_(nullptr), - extra_buf_(nullptr), - extra_buf_size_(0) -{ -} - -ObDirectLoadTableStoreParam::~ObDirectLoadTableStoreParam() -{ -} - -bool ObDirectLoadTableStoreParam::is_valid() const -{ - return table_data_desc_.is_valid() && nullptr != datum_utils_ && - nullptr != file_mgr_ && (!is_fast_heap_table_ || - (nullptr != insert_table_ctx_)) && - nullptr != dml_row_handler_; -} - -/** - * ObDirectLoadTableStoreBucket - */ - -ObDirectLoadTableStoreBucket::ObDirectLoadTableStoreBucket() - : param_(nullptr), table_builder_allocator_(nullptr), table_builder_(nullptr), is_inited_(false) -{ -} - -ObDirectLoadTableStoreBucket::~ObDirectLoadTableStoreBucket() -{ -} - -int ObDirectLoadTableStoreBucket::init(const ObDirectLoadTableStoreParam ¶m, - const ObTabletID &tablet_id) -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - LOG_WARN("ObDirectLoadTableStore init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!param.is_valid() || !tablet_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(param), K(tablet_id)); - } else { - table_builder_allocator_ = get_table_builder_allocator(); - if (param.is_multiple_mode_) { - // new external multi partition table - ObDirectLoadExternalMultiPartitionTableBuildParam external_mp_table_build_param; - external_mp_table_build_param.table_data_desc_ = param.table_data_desc_; - external_mp_table_build_param.datum_utils_ = param.datum_utils_; - external_mp_table_build_param.file_mgr_ = param.file_mgr_; - external_mp_table_build_param.extra_buf_ = param.extra_buf_; - external_mp_table_build_param.extra_buf_size_ = param.extra_buf_size_; - ObDirectLoadExternalMultiPartitionTableBuilder *external_mp_table_builder = nullptr; - if (OB_ISNULL( - external_mp_table_builder = - table_builder_allocator_->alloc())) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc ObDirectLoadExternalMultiPartitionTableBuilder", KR(ret)); - } else if (OB_FAIL(external_mp_table_builder->init(external_mp_table_build_param))) { - LOG_WARN("fail to init external multi partition table builder", KR(ret)); - } - table_builder_ = external_mp_table_builder; - } else if (param.is_fast_heap_table_) { - abort_unless(param.table_data_desc_.is_heap_table_); - // new fast heap table - ObDirectLoadFastHeapTableBuildParam fast_heap_table_build_param; - fast_heap_table_build_param.tablet_id_ = tablet_id; - fast_heap_table_build_param.table_data_desc_ = param.table_data_desc_; - fast_heap_table_build_param.insert_table_ctx_ = param.insert_table_ctx_; - fast_heap_table_build_param.dml_row_handler_ = param.dml_row_handler_; - ObDirectLoadFastHeapTableBuilder *fast_heap_table_builder = nullptr; - if (OB_ISNULL(fast_heap_table_builder = - table_builder_allocator_->alloc())) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc ObDirectLoadFastHeapTableBuilder", KR(ret)); - } else if (OB_FAIL(fast_heap_table_builder->init(fast_heap_table_build_param))) { - LOG_WARN("fail to init sstable builder", KR(ret)); - } - table_builder_ = fast_heap_table_builder; - } else { - abort_unless(!param.table_data_desc_.is_heap_table_); - // new sstable - ObDirectLoadMultipleSSTableBuildParam sstable_build_param; - sstable_build_param.tablet_id_ = tablet_id; - sstable_build_param.table_data_desc_ = param.table_data_desc_; - sstable_build_param.datum_utils_ = param.datum_utils_; - sstable_build_param.file_mgr_ = param.file_mgr_; - sstable_build_param.extra_buf_ = param.extra_buf_; - sstable_build_param.extra_buf_size_ = param.extra_buf_size_; - ObDirectLoadMultipleSSTableBuilder *sstable_builder = nullptr; - if (OB_ISNULL(sstable_builder = - table_builder_allocator_->alloc())) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc ObDirectLoadMultipleSSTableBuilder", KR(ret)); - } else if (OB_FAIL(sstable_builder->init(sstable_build_param))) { - LOG_WARN("fail to init sstable builder", KR(ret)); - } - table_builder_ = sstable_builder; - } - if (OB_SUCC(ret)) { - param_ = ¶m; - is_inited_ = true; - } else { - clean_up(); - } - } - return ret; -} - -int ObDirectLoadTableStoreBucket::append_row(const ObTabletID &tablet_id, - const ObTableLoadSequenceNo &seq_no, - const ObDatumRow &datum_row) -{ - OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, table_store_bucket_append_row); - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadTableStore not init", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!tablet_id.is_valid() || !datum_row.is_valid() || - datum_row.get_column_count() != param_->table_data_desc_.column_count_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(tablet_id), K(datum_row), KPC(param_)); - } else { - if (OB_FAIL(table_builder_->append_row(tablet_id, seq_no, datum_row))) { - LOG_WARN("fail to append row", KR(ret)); - } - } - return ret; -} - -int ObDirectLoadTableStoreBucket::close() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadTableStore not init", KR(ret), KP(this)); - } else { - if (OB_FAIL(table_builder_->close())) { - LOG_WARN("fail to close table builder", KR(ret)); - } - } - return ret; -} - -int ObDirectLoadTableStoreBucket::get_tables(ObIArray &table_array, - ObIAllocator &allocator) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadTableStore not init", KR(ret), KP(this)); - } else { - if (OB_FAIL(table_builder_->get_tables(table_array, allocator))) { - LOG_WARN("fail to get tables", KR(ret)); - } - } - return ret; -} - -void ObDirectLoadTableStoreBucket::clean_up() -{ - if (nullptr != table_builder_) { - table_builder_allocator_->free(table_builder_); - table_builder_allocator_ = nullptr; - table_builder_ = nullptr; - } -} - -/** - * ObDirectLoadTableStore - */ - -ObDirectLoadTableStore::~ObDirectLoadTableStore() -{ - for (int64_t i = 0; i < bucket_ptr_array_.count(); i++) { - if (bucket_ptr_array_.at(i) != nullptr) { - bucket_ptr_array_.at(i)->~ObDirectLoadTableStoreBucket(); - } - } -} - -int ObDirectLoadTableStore::init(const ObDirectLoadTableStoreParam ¶m) -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - LOG_WARN("ObDirectLoadTableStore init twice", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!param.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(param)); - } else { - const uint64_t tenant_id = MTL_ID(); - param_ = param; - if (OB_FAIL(tablet_index_.create(64, "TLD_TS_PartMap", "TLD_TS_PartMap", tenant_id))) { - LOG_WARN("fail to create hashmap", KR(ret)); - } else { - is_inited_ = true; - } - } - return ret; -} - -int ObDirectLoadTableStore::new_bucket(ObDirectLoadTableStoreBucket *&bucket) -{ - int ret = OB_SUCCESS; - bucket = nullptr; - if (OB_ISNULL(bucket = OB_NEWx(ObDirectLoadTableStoreBucket, (&allocator_)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObDirectLoadTableStoreBucket", K(ret)); - } else if (OB_FAIL(bucket_ptr_array_.push_back(bucket))) { - LOG_WARN("fail to push back bucket", KR(ret)); - } - return ret; -} - -int ObDirectLoadTableStore::get_bucket(const ObTabletID &tablet_id, - ObDirectLoadTableStoreBucket *&bucket) -{ - OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, table_store_get_bucket); - int ret = OB_SUCCESS; - bucket = nullptr; - if (!param_.is_multiple_mode_) { - if (OB_FAIL(tablet_index_.get_refactored(tablet_id, bucket))) { - if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) { - LOG_WARN("fail to get refactored", KR(ret), K(tablet_id)); - } else { - ret = OB_SUCCESS; - if (OB_FAIL(new_bucket(bucket))) { - LOG_WARN("fail to new bucket", KR(ret)); - } else if (OB_FAIL(bucket->init(param_, tablet_id))) { - LOG_WARN("fail to init bucket", KR(ret)); - } - if (OB_SUCC(ret)) { - if (OB_FAIL(tablet_index_.set_refactored(tablet_id, bucket))) { - LOG_WARN("fail to set refactored", KR(ret), K(tablet_id)); - } - } - } - } - } else { - if (bucket_ptr_array_.count() >= 1) { - bucket = bucket_ptr_array_.at(0); - } else { - if (OB_FAIL(new_bucket(bucket))) { - LOG_WARN("fail to new bucket", KR(ret)); - } else if (OB_FAIL(bucket->init(param_, tablet_id))) { - LOG_WARN("fail to init bucket", KR(ret)); - } - } - } - return ret; -} - -int ObDirectLoadTableStore::append_row(const ObTabletID &tablet_id, const ObTableLoadSequenceNo &seq_no, const ObDatumRow &datum_row) -{ - OB_TABLE_LOAD_STATISTICS_TIME_COST(DEBUG, table_store_append_row); - OB_TABLE_LOAD_STATISTICS_COUNTER(table_store_row_count); - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadTableStore not init", KR(ret), KP(this)); - } else if (OB_UNLIKELY(!tablet_id.is_valid() || !datum_row.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", KR(ret), K(tablet_id), K(datum_row)); - } else { - ObDirectLoadTableStoreBucket *bucket = nullptr; - if (OB_FAIL(get_bucket(tablet_id, bucket))) { - LOG_WARN("fail to get bucket", KR(ret), K(tablet_id)); - } else if (OB_FAIL(bucket->append_row(tablet_id, seq_no, datum_row))) { - LOG_WARN("fail to append row to bucket", KR(ret), K(tablet_id), K(datum_row)); - } - } - return ret; -} - -int ObDirectLoadTableStore::close() -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadTableStore not init", KR(ret), KP(this)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < bucket_ptr_array_.count(); ++i) { - if (OB_FAIL(bucket_ptr_array_.at(i)->close())) { - LOG_WARN("fail to close bucket", KR(ret), K(i)); - } - } - } - return ret; -} - -int ObDirectLoadTableStore::get_tables(ObIArray &table_array, - ObIAllocator &allocator) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObDirectLoadTableStore not init", KR(ret), KP(this)); - } else { - table_array.reset(); - ObArray bucket_table_array; - bucket_table_array.set_tenant_id(MTL_ID()); - for (int64_t i = 0; OB_SUCC(ret) && i < bucket_ptr_array_.count(); ++i) { - bucket_table_array.reset(); - if (OB_FAIL(bucket_ptr_array_.at(i)->get_tables(bucket_table_array, allocator))) { - LOG_WARN("fail to get tables from bucket", KR(ret), K(i)); - } else if (OB_UNLIKELY(bucket_table_array.empty())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected empty bucket table array", KR(ret)); - } - for (int64_t j = 0; OB_SUCC(ret) && j < bucket_table_array.count(); ++j) { - if (OB_FAIL(table_array.push_back(bucket_table_array.at(j)))) { - LOG_WARN("fail to push back table", KR(ret)); - } - } - } - } - return ret; -} - -void ObDirectLoadTableStore::clean_up() -{ - for (int64_t i = 0; i < bucket_ptr_array_.count(); ++i) { - bucket_ptr_array_.at(i)->clean_up(); - } -} - -} // namespace storage -} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_table_store.h b/src/storage/direct_load/ob_direct_load_table_store.h deleted file mode 100644 index aa91a201a7..0000000000 --- a/src/storage/direct_load/ob_direct_load_table_store.h +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ -#pragma once - -#include "share/table/ob_table_load_array.h" -#include "share/table/ob_table_load_define.h" -#include "storage/blocksstable/ob_datum_row.h" -#include "storage/direct_load/ob_direct_load_i_table.h" -#include "storage/direct_load/ob_direct_load_table_data_desc.h" - -namespace oceanbase -{ -namespace storage -{ -class ObDirectLoadTableDataDesc; -class ObDirectLoadTmpFileManager; -class ObDirectLoadTableBuilderAllocator; -class ObDirectLoadInsertTableContext; -class ObDirectLoadDMLRowHandler; - -struct ObDirectLoadTableStoreParam -{ -public: - ObDirectLoadTableStoreParam(); - ~ObDirectLoadTableStoreParam(); - bool is_valid() const; - TO_STRING_KV(K_(table_data_desc), - KP_(datum_utils), - KP_(file_mgr), - K_(is_multiple_mode), - K_(is_fast_heap_table), - KP_(insert_table_ctx), - KP_(dml_row_handler), - KP_(extra_buf), - K_(extra_buf_size)); -public: - ObDirectLoadTableDataDesc table_data_desc_; - const blocksstable::ObStorageDatumUtils *datum_utils_; - ObDirectLoadTmpFileManager *file_mgr_; - bool is_multiple_mode_; - bool is_fast_heap_table_; - ObDirectLoadInsertTableContext *insert_table_ctx_; - ObDirectLoadDMLRowHandler *dml_row_handler_; - char *extra_buf_; - int64_t extra_buf_size_; -}; - -class ObDirectLoadTableStoreBucket -{ -public: - ObDirectLoadTableStoreBucket(); - ~ObDirectLoadTableStoreBucket(); - int init(const ObDirectLoadTableStoreParam ¶m, const common::ObTabletID &tablet_id); - int append_row(const common::ObTabletID &tablet_id, const table::ObTableLoadSequenceNo &seq_no, const blocksstable::ObDatumRow &datum_row); - int close(); - int get_tables(common::ObIArray &table_array, - common::ObIAllocator &allocator); - void clean_up(); - TO_STRING_KV(KP(param_)); -private: - const ObDirectLoadTableStoreParam *param_; - ObDirectLoadTableBuilderAllocator *table_builder_allocator_; - ObIDirectLoadPartitionTableBuilder *table_builder_; - bool is_inited_; -}; - -class ObDirectLoadTableStore -{ -public: - const static constexpr int64_t MAX_BUCKET_CNT = 1024; - ObDirectLoadTableStore() : allocator_("TLD_TSBucket"), is_inited_(false) - { - allocator_.set_tenant_id(MTL_ID()); - bucket_ptr_array_.set_tenant_id(MTL_ID()); - } - ~ObDirectLoadTableStore(); - int init(const ObDirectLoadTableStoreParam ¶m); - int append_row(const common::ObTabletID &tablet_id, const table::ObTableLoadSequenceNo &seq_no, const blocksstable::ObDatumRow &datum_row); - int close(); - void clean_up(); - int get_tables(common::ObIArray &table_array, - common::ObIAllocator &allocator); -private: - int new_bucket(ObDirectLoadTableStoreBucket *&bucket); - int get_bucket(const common::ObTabletID &tablet_id, ObDirectLoadTableStoreBucket *&bucket); -private: - ObDirectLoadTableStoreParam param_; - common::ObArenaAllocator allocator_; - common::ObArray bucket_ptr_array_; - common::hash::ObHashMap tablet_index_; - bool is_inited_; -}; - -} // namespace storage -} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_vector_utils.cpp b/src/storage/direct_load/ob_direct_load_vector_utils.cpp new file mode 100644 index 0000000000..095130913b --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_vector_utils.cpp @@ -0,0 +1,989 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX STORAGE + +#include "storage/direct_load/ob_direct_load_vector_utils.h" +#include "share/ob_tablet_autoincrement_param.h" +#include "storage/blocksstable/ob_datum_row.h" +#include "storage/blocksstable/ob_storage_datum.h" + +namespace oceanbase +{ +namespace storage +{ +using namespace blocksstable; +using namespace common; +using namespace share; +using namespace sql; + +int ObDirectLoadVectorUtils::new_vector(VectorFormat format, VecValueTypeClass value_tc, + ObIAllocator &allocator, ObIVector *&vector) +{ + int ret = OB_SUCCESS; + vector = nullptr; + switch (format) { + case VEC_FIXED: { + switch (value_tc) { +#define FIXED_VECTOR_INIT_SWITCH(value_tc) \ + case value_tc: { \ + using VecType = RTVectorType; \ + static_assert(sizeof(RTVectorType) <= ObIVector::MAX_VECTOR_STRUCT_SIZE, \ + "vector size exceeds MAX_VECTOR_STRUCT_SIZE"); \ + vector = OB_NEWx(VecType, &allocator, nullptr, nullptr); \ + break; \ + } + FIXED_VECTOR_INIT_SWITCH(VEC_TC_INTEGER); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_UINTEGER); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_FLOAT); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_DOUBLE); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_FIXED_DOUBLE); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_DATETIME); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_DATE); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_TIME); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_YEAR); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_UNKNOWN); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_BIT); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_ENUM_SET); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_TIMESTAMP_TZ); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_TIMESTAMP_TINY); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_INTERVAL_YM); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_INTERVAL_DS); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT32); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT64); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT128); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT256); + FIXED_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT512); +#undef FIXED_VECTOR_INIT_SWITCH + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected fixed vector value type class", KR(ret), K(value_tc)); + break; + } + break; + } + case VEC_CONTINUOUS: { + switch (value_tc) { +#define CONTINUOUS_VECTOR_INIT_SWITCH(value_tc) \ + case value_tc: { \ + using VecType = RTVectorType; \ + static_assert(sizeof(VecType) <= ObIVector::MAX_VECTOR_STRUCT_SIZE, \ + "vector size exceeds MAX_VECTOR_STRUCT_SIZE"); \ + vector = OB_NEWx(VecType, &allocator, nullptr, nullptr, nullptr); \ + break; \ + } + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_NUMBER); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_EXTEND); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_STRING); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_ENUM_SET_INNER); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_RAW); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_ROWID); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_LOB); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_JSON); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_GEO); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_UDT); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_COLLECTION); + CONTINUOUS_VECTOR_INIT_SWITCH(VEC_TC_ROARINGBITMAP); +#undef CONTINUOUS_VECTOR_INIT_SWITCH + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected continuous vector value type class", KR(ret), K(value_tc)); + break; + } + break; + } + case VEC_DISCRETE: { + switch (value_tc) { +#define DISCRETE_VECTOR_INIT_SWITCH(value_tc) \ + case value_tc: { \ + using VecType = RTVectorType; \ + static_assert(sizeof(VecType) <= ObIVector::MAX_VECTOR_STRUCT_SIZE, \ + "vector size exceeds MAX_VECTOR_STRUCT_SIZE"); \ + vector = OB_NEWx(VecType, &allocator, nullptr, nullptr, nullptr); \ + break; \ + } + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_NUMBER); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_EXTEND); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_STRING); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_ENUM_SET_INNER); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_RAW); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_ROWID); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_LOB); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_JSON); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_GEO); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_UDT); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_COLLECTION); + DISCRETE_VECTOR_INIT_SWITCH(VEC_TC_ROARINGBITMAP); +#undef DISCRETE_VECTOR_INIT_SWITCH + default: + ret = OB_ERR_UNEXPECTED; + SQL_LOG(WARN, "unexpected vector value type class", KR(ret), K(value_tc)); + break; + } + break; + } + case VEC_UNIFORM: { + ObEvalInfo *eval_info = nullptr; + if (OB_ISNULL(eval_info = OB_NEWx(ObEvalInfo, &allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObEvalInfo", KR(ret)); + } else { + switch (value_tc) { +#define UNIFORM_VECTOR_INIT_SWITCH(value_tc) \ + case value_tc: { \ + using VecType = RTVectorType; \ + static_assert(sizeof(VecType) <= ObIVector::MAX_VECTOR_STRUCT_SIZE, \ + "vector size exceeds MAX_VECTOR_STRUCT_SIZE"); \ + vector = OB_NEWx(VecType, &allocator, nullptr, eval_info); \ + break; \ + } + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_NULL); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_INTEGER); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_UINTEGER); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_FLOAT); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_DOUBLE); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_FIXED_DOUBLE); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_DATETIME); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_DATE); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_TIME); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_YEAR); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_UNKNOWN); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_BIT); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_ENUM_SET); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_TIMESTAMP_TZ); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_TIMESTAMP_TINY); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_INTERVAL_YM); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_INTERVAL_DS); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT32); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT64); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT128); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT256); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT512); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_NUMBER); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_EXTEND); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_STRING); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_ENUM_SET_INNER); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_RAW); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_ROWID); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_LOB); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_JSON); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_GEO); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_UDT); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_COLLECTION); + UNIFORM_VECTOR_INIT_SWITCH(VEC_TC_ROARINGBITMAP); +#undef UNIFORM_VECTOR_INIT_SWITCH + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector value type class", KR(ret), K(value_tc)); + break; + } + } + break; + } + case VEC_UNIFORM_CONST: { + ObEvalInfo *eval_info = nullptr; + if (OB_ISNULL(eval_info = OB_NEWx(ObEvalInfo, &allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObEvalInfo", KR(ret)); + } else { + switch (value_tc) { +#define UNIFORM_CONST_VECTOR_INIT_SWITCH(value_tc) \ + case value_tc: { \ + using VecType = RTVectorType; \ + static_assert(sizeof(VecType) <= ObIVector::MAX_VECTOR_STRUCT_SIZE, \ + "vector size exceeds MAX_VECTOR_STRUCT_SIZE"); \ + vector = OB_NEWx(VecType, &allocator, nullptr, eval_info); \ + break; \ + } + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_NULL); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_INTEGER); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_UINTEGER); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_FLOAT); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_DOUBLE); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_FIXED_DOUBLE); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_DATETIME); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_DATE); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_TIME); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_YEAR); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_UNKNOWN); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_BIT); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_ENUM_SET); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_TIMESTAMP_TZ); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_TIMESTAMP_TINY); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_INTERVAL_YM); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_INTERVAL_DS); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT32); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT64); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT128); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT256); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_DEC_INT512); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_NUMBER); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_EXTEND); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_STRING); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_ENUM_SET_INNER); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_RAW); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_ROWID); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_LOB); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_JSON); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_GEO); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_UDT); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_COLLECTION); + UNIFORM_CONST_VECTOR_INIT_SWITCH(VEC_TC_ROARINGBITMAP); +#undef UNIFORM_CONST_VECTOR_INIT_SWITCH + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector value type class", KR(ret), K(value_tc)); + break; + } + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", KR(ret), K(format)); + break; + } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(vector)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc vector", KR(ret)); + } + return ret; +} + +int ObDirectLoadVectorUtils::prepare_vector(ObIVector *vector, const int64_t max_batch_size, + ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == vector || max_batch_size <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(vector), K(max_batch_size)); + } else { + const VectorFormat format = vector->get_format(); + switch (format) { + case VEC_FIXED: { + ObFixedLengthBase *fixed_vec = static_cast(vector); + const ObLength length = fixed_vec->get_length(); + const int64_t nulls_size = ObBitVector::memory_size(max_batch_size); + const int64_t data_size = length * max_batch_size; + ObBitVector *nulls = nullptr; + char *data = nullptr; + if (OB_ISNULL(nulls = to_bit_vector(allocator.alloc(nulls_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", KR(ret), K(nulls_size)); + } else if (OB_ISNULL(data = static_cast(allocator.alloc(data_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", KR(ret), K(data_size)); + } else { + nulls->reset(max_batch_size); + fixed_vec->set_nulls(nulls); + fixed_vec->set_data(data); + } + break; + } + case VEC_CONTINUOUS: { + ObContinuousBase *continuous_vec = static_cast(vector); + const int64_t nulls_size = ObBitVector::memory_size(max_batch_size); + const int64_t offsets_size = sizeof(uint32_t) * max_batch_size; + ObBitVector *nulls = nullptr; + uint32_t *offsets = nullptr; + if (OB_ISNULL(nulls = to_bit_vector(allocator.alloc(nulls_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", KR(ret), K(nulls_size)); + } else if (OB_ISNULL(offsets = static_cast(allocator.alloc(offsets_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", KR(ret), K(offsets_size)); + } else { + nulls->reset(max_batch_size); + continuous_vec->set_nulls(nulls); + continuous_vec->set_offsets(offsets); + } + break; + } + case VEC_DISCRETE: { + ObDiscreteBase *discrete_vec = static_cast(vector); + const int64_t nulls_size = ObBitVector::memory_size(max_batch_size); + const int64_t lens_size = sizeof(int32_t) * max_batch_size; + const int64_t ptrs_size = sizeof(char *) * max_batch_size; + ObBitVector *nulls = nullptr; + int32_t *lens = nullptr; + char **ptrs = nullptr; + if (OB_ISNULL(nulls = to_bit_vector(allocator.alloc(nulls_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", KR(ret), K(nulls_size)); + } else if (OB_ISNULL(lens = static_cast(allocator.alloc(lens_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", KR(ret), K(lens_size)); + } else if (OB_ISNULL(ptrs = static_cast(allocator.alloc(ptrs_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", KR(ret), K(ptrs_size)); + } else { + nulls->reset(max_batch_size); + discrete_vec->set_nulls(nulls); + discrete_vec->set_lens(lens); + discrete_vec->set_ptrs(ptrs); + } + break; + } + case VEC_UNIFORM: { + ObUniformBase *uniform_vec = static_cast(vector); + const int64_t datums_size = sizeof(ObDatum) * max_batch_size; + ObDatum *datums = nullptr; + if (OB_ISNULL(datums = static_cast(allocator.alloc(datums_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", KR(ret), K(datums_size)); + } else { + uniform_vec->set_datums(datums); + } + break; + } + case VEC_UNIFORM_CONST: { + ObUniformBase *uniform_vec = static_cast(vector); + const int64_t datums_size = sizeof(ObDatum); + ObDatum *datums = nullptr; + if (OB_ISNULL(datums = static_cast(allocator.alloc(datums_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", KR(ret), K(datums_size)); + } else { + uniform_vec->set_datums(datums); + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", KR(ret), K(format)); + break; + } + } + return ret; +} + +namespace +{ +inline static void shallow_copy_bitmap_null_vector_base(ObBitmapNullVectorBase *src_vec, + ObBitmapNullVectorBase *dest_vec, + const int64_t batch_size) +{ + ObBitVector *nulls = dest_vec->get_nulls(); + uint16_t flag = src_vec->get_flag(); + nulls->deep_copy(*src_vec->get_nulls(), 0, batch_size); + dest_vec->from(nulls, flag); +} + +template +inline int shallow_copy_vector_impl(SRC_VEC *src_vec, DEST_VEC *dest_vec, const int64_t batch_size); + +// VEC_CONTINUOUS -> VEC_DISCRETE +template <> +inline int shallow_copy_vector_impl(ObContinuousBase *src_vec, + ObDiscreteBase *dest_vec, + const int64_t batch_size) +{ + int ret = OB_SUCCESS; + shallow_copy_bitmap_null_vector_base(src_vec, dest_vec, batch_size); + uint32_t *offsets = src_vec->get_offsets(); + char *data = src_vec->get_data(); + ObLength *lens = dest_vec->get_lens(); + char **ptrs = dest_vec->get_ptrs(); + for (int64_t i = 0; i < batch_size; ++i) { + ptrs[i] = data + offsets[i]; + lens[i] = offsets[i + 1] - offsets[i]; + } + return ret; +} + +// VEC_DISCRETE -> VEC_DISCRETE +template <> +inline int shallow_copy_vector_impl(ObDiscreteBase *src_vec, + ObDiscreteBase *dest_vec, + const int64_t batch_size) +{ + int ret = OB_SUCCESS; + shallow_copy_bitmap_null_vector_base(src_vec, dest_vec, batch_size); + ObLength *src_lens = src_vec->get_lens(); + char **src_ptrs = src_vec->get_ptrs(); + ObLength *dest_lens = dest_vec->get_lens(); + char **dest_ptrs = dest_vec->get_ptrs(); + MEMCPY(dest_lens, src_lens, sizeof(ObLength) * batch_size); + MEMCPY(dest_ptrs, src_ptrs, sizeof(char *) * batch_size); + return ret; +} + +// VEC_UNIFORM -> VEC_DISCRETE +// VEC_UNIFORM -> VEC_UNIFORM +// VEC_UNIFORM_CONST-> VEC_DISCRETE +// VEC_UNIFORM_CONST-> VEC_UNIFORM_CONST +template +inline int shallow_copy_vector_impl(ObUniformBase *src_vec, + ObDiscreteBase *dest_vec, + const int64_t batch_size) +{ + int ret = OB_SUCCESS; + ObDatum *datums = src_vec->get_datums(); + char **ptrs = dest_vec->get_ptrs(); + ObLength *lens = dest_vec->get_lens(); + for (int64_t i = 0; i < batch_size; ++i) { + const ObDatum &datum = datums[IS_CONST ? 0 : i]; + if (datum.is_null()) { + dest_vec->set_null(i); + } else { + ptrs[i] = const_cast(datum.ptr_); + lens[i] = datum.len_; + } + } + return ret; +} +template +inline int shallow_copy_vector_impl(ObUniformBase *src_vec, + ObUniformBase *dest_vec, + const int64_t batch_size) +{ + int ret = OB_SUCCESS; + ObDatum *src_datums = src_vec->get_datums(); + ObDatum *dest_datums = dest_vec->get_datums(); + MEMCPY(dest_datums, src_datums, sizeof(ObDatum) * (IS_CONST ? 1 : batch_size)); + return ret; +} +} // namespace + +int ObDirectLoadVectorUtils::shallow_copy_vector(ObIVector *src_vec, + ObIVector *dest_vec, + const int64_t batch_size) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == src_vec || nullptr == dest_vec || batch_size <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(src_vec), KP(dest_vec), K(batch_size)); + } else { + const VectorFormat src_format = src_vec->get_format(); + const VectorFormat dest_format = dest_vec->get_format(); + switch (src_format) { + // VEC_CONTINUOUS -> VEC_DISCRETE + case VEC_CONTINUOUS: { + switch (dest_format) { + case VEC_DISCRETE: + ret = shallow_copy_vector_impl(static_cast(src_vec), static_cast(dest_vec), batch_size); + break; + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", KR(ret), K(src_format), K(dest_format)); + break; + } + break; + } + // VEC_DISCRETE -> VEC_DISCRETE + case VEC_DISCRETE: { + switch (dest_format) { + case VEC_DISCRETE: + ret = shallow_copy_vector_impl(static_cast(src_vec), static_cast(dest_vec), batch_size); + break; + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", KR(ret), K(src_format), K(dest_format)); + break; + } + break; + } + // VEC_UNIFORM -> VEC_DISCRETE + // VEC_UNIFORM -> VEC_UNIFORM + case VEC_UNIFORM: { + switch (dest_format) { + case VEC_DISCRETE: + ret = shallow_copy_vector_impl(static_cast(src_vec), static_cast(dest_vec), batch_size); + break; + case VEC_UNIFORM: + ret = shallow_copy_vector_impl(static_cast(src_vec), static_cast(dest_vec), batch_size); + break; + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", KR(ret), K(src_format), K(dest_format)); + break; + } + break; + } + // VEC_UNIFORM_CONST-> VEC_DISCRETE + // VEC_UNIFORM_CONST-> VEC_UNIFORM_CONST + case VEC_UNIFORM_CONST: { + switch (dest_format) { + case VEC_DISCRETE: + ret = shallow_copy_vector_impl(static_cast(src_vec), static_cast(dest_vec), batch_size); + break; + case VEC_UNIFORM_CONST: + ret = shallow_copy_vector_impl(static_cast(src_vec), static_cast(dest_vec), batch_size); + break; + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", KR(ret), K(src_format), K(dest_format)); + break; + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", KR(ret), K(src_format)); + break; + } + } + return ret; +} + +int ObDirectLoadVectorUtils::get_payload(ObIVector *vector, const int64_t idx, bool &is_null, + const char *&payload, ObLength &len) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == vector || idx < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(vector), K(idx)); + } else { + const VectorFormat format = vector->get_format(); + switch (format) { + case VEC_FIXED: { + ObFixedLengthBase *fixed_vec = static_cast(vector); + if (fixed_vec->is_null(idx)) { + is_null = true; + } else { + len = fixed_vec->get_length(); + payload = fixed_vec->get_data() + len * idx; + } + break; + } + case VEC_CONTINUOUS: { + ObContinuousBase *continuous_vec = static_cast(vector); + if (continuous_vec->is_null(idx)) { + is_null = true; + } else { + const uint32_t offset1 = continuous_vec->get_offsets()[idx]; + const uint32_t offset2 = continuous_vec->get_offsets()[idx + 1]; + payload = continuous_vec->get_data() + offset1; + len = (offset2 - offset1); + } + break; + } + case VEC_DISCRETE: { + ObDiscreteBase *discrete_vec = static_cast(vector); + if (discrete_vec->is_null(idx)) { + is_null = true; + } else { + payload = discrete_vec->get_ptrs()[idx]; + len = discrete_vec->get_lens()[idx]; + } + break; + } + case VEC_UNIFORM: { + ObUniformBase *uniform_vec = static_cast(vector); + ObDatum &datum = uniform_vec->get_datums()[idx]; + if (datum.is_null()) { + is_null = true; + } else { + is_null = false; + payload = datum.ptr_; + len = datum.len_; + } + break; + } + case VEC_UNIFORM_CONST: { + ObUniformBase *uniform_vec = static_cast(vector); + ObDatum &datum = uniform_vec->get_datums()[0]; + if (datum.is_null()) { + is_null = true; + } else { + is_null = false; + payload = datum.ptr_; + len = datum.len_; + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", KR(ret), K(format)); + break; + } + } + return ret; +} + +int ObDirectLoadVectorUtils::to_datum(ObIVector *vector, const int64_t idx, ObDatum &datum) +{ + int ret = OB_SUCCESS; + datum.reset(); + if (OB_UNLIKELY(nullptr == vector || idx < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(vector), K(idx)); + } else { + const VectorFormat format = vector->get_format(); + switch (format) { + case VEC_FIXED: { + ObFixedLengthBase *fixed_vec = static_cast(vector); + if (fixed_vec->is_null(idx)) { + datum.set_null(); + } else { + datum.len_ = fixed_vec->get_length(); + datum.ptr_ = fixed_vec->get_data() + datum.len_ * idx; + } + break; + } + case VEC_CONTINUOUS: { + ObContinuousBase *continuous_vec = static_cast(vector); + if (continuous_vec->is_null(idx)) { + datum.set_null(); + } else { + const uint32_t offset1 = continuous_vec->get_offsets()[idx]; + const uint32_t offset2 = continuous_vec->get_offsets()[idx + 1]; + datum.ptr_ = continuous_vec->get_data() + offset1; + datum.len_ = (offset2 - offset1); + } + break; + } + case VEC_DISCRETE: { + ObDiscreteBase *discrete_vec = static_cast(vector); + if (discrete_vec->is_null(idx)) { + datum.set_null(); + } else { + datum.len_ = discrete_vec->get_lens()[idx]; + datum.ptr_ = discrete_vec->get_ptrs()[idx]; + } + break; + } + case VEC_UNIFORM: { + ObUniformBase *uniform_vec = static_cast(vector); + datum = uniform_vec->get_datums()[idx]; + break; + } + case VEC_UNIFORM_CONST: { + ObUniformBase *uniform_vec = static_cast(vector); + datum = uniform_vec->get_datums()[0]; + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", KR(ret), K(format)); + break; + } + } + return ret; +} + +int ObDirectLoadVectorUtils::to_datums(const ObIArray &vectors, int64_t idx, + ObDatum *datums, const int64_t count) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == datums || vectors.count() != count)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(vectors), KP(datums), K(count)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < count; ++i) { + if (OB_FAIL(to_datum(vectors.at(i), idx, datums[i]))) { + LOG_WARN("fail to datum", KR(ret)); + } + } + } + return ret; +} + +int ObDirectLoadVectorUtils::to_datums(const ObIArray &vectors, int64_t idx, + ObStorageDatum *datums, const int64_t count) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(idx < 0 || nullptr == datums || vectors.count() != count)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(vectors), K(idx), KP(datums), K(count)); + } else { + bool is_null = false; + const char *payload = nullptr; + ObLength length = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < count; ++i) { + ObIVector *vec = vectors.at(i); + ObStorageDatum &datum = datums[i]; + if (OB_FAIL(to_datum(vec, idx, datum))) { + LOG_WARN("fail to datum", KR(ret)); + } + } + } + return ret; +} + +int ObDirectLoadVectorUtils::set_datum(ObIVector *vector, const int64_t idx, const ObDatum &datum) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == vector || idx < 0 || datum.is_ext())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(vector), K(idx), K(datum)); + } else { + const VectorFormat format = vector->get_format(); + switch (format) { + case VEC_FIXED: { + ObFixedLengthBase *fixed_vec = static_cast(vector); + if (datum.is_null()) { + fixed_vec->set_null(idx); + } else if (OB_UNLIKELY(fixed_vec->get_length() != datum.len_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected datum length", KR(ret), K(fixed_vec->get_length()), K(datum)); + } else { + MEMCPY(fixed_vec->get_data() + datum.len_ * idx, datum.ptr_, datum.len_); + } + break; + } + case VEC_DISCRETE: { + ObDiscreteBase *discrete_vec = static_cast(vector); + if (datum.is_null()) { + discrete_vec->set_null(idx); + } else { + discrete_vec->get_ptrs()[idx] = const_cast(datum.ptr_); + discrete_vec->get_lens()[idx] = datum.len_; + } + break; + } + case VEC_UNIFORM: { + ObUniformBase *uniform_vec = static_cast(vector); + uniform_vec->get_datums()[idx] = datum; + break; + } + case VEC_UNIFORM_CONST: { + ObUniformBase *uniform_vec = static_cast(vector); + uniform_vec->get_datums()[0] = datum; + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", KR(ret), K(format)); + break; + } + } + return ret; +} + +/** + * tablet id vector + */ + +int ObDirectLoadVectorUtils::make_const_tablet_id_vector(const ObTabletID &tablet_id, + ObIAllocator &allocator, + ObIVector *&vector) +{ + int ret = OB_SUCCESS; + vector = nullptr; + if (OB_UNLIKELY(!tablet_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), K(tablet_id)); + } else { + if (OB_FAIL(new_vector(VEC_UNIFORM_CONST, tablet_id_value_tc, allocator, vector))) { + LOG_WARN("fail to new uniform const vector", KR(ret)); + } else { + ObUniformBase *uniform_vec = static_cast(vector); + ObStorageDatum *storage_datum = nullptr; + if (OB_ISNULL(storage_datum = OB_NEWx(ObStorageDatum, &allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObStorageDatum", KR(ret)); + } else { + storage_datum->set_uint(tablet_id.id()); + uniform_vec->set_datums(storage_datum); + } + } + } + return ret; +} + +int ObDirectLoadVectorUtils::set_tablet_id(ObIVector *vector, + const int64_t batch_idx, + const ObTabletID &tablet_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == vector || batch_idx < 0 || !tablet_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(vector), K(batch_idx), K(tablet_id)); + } else { + const VectorFormat format = vector->get_format(); + switch (format) { + case VEC_FIXED: { + ObFixedLengthBase *fixed_vec = static_cast(vector); + const ObLength len = fixed_vec->get_length(); + if (OB_UNLIKELY(len != sizeof(uint64_t))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected fixed len", KR(ret), K(len), K(sizeof(uint64_t))); + } else { + reinterpret_cast(fixed_vec->get_data())[batch_idx] = tablet_id.id(); + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected tablet id vector format", KR(ret), K(format)); + break; + } + } + return ret; +} + +ObTabletID ObDirectLoadVectorUtils::get_tablet_id(ObFixedLengthBase *vec, const int64_t batch_idx) +{ + ObTabletID tablet_id; + if (OB_NOT_NULL(vec)) { + tablet_id = reinterpret_cast(vec->get_data())[batch_idx]; + } + return tablet_id; +} + +template +ObTabletID ObDirectLoadVectorUtils::get_tablet_id(ObUniformBase *vec, const int64_t batch_idx) +{ + ObTabletID tablet_id; + if (OB_NOT_NULL(vec)) { + tablet_id = vec->get_datums()[IS_CONST ? 0 : batch_idx].get_uint(); + } + return tablet_id; +} + +ObTabletID ObDirectLoadVectorUtils::get_tablet_id(ObIVector *vec, const int64_t batch_idx) +{ + ObTabletID tablet_id; + if (OB_NOT_NULL(vec)) { + const VectorFormat format = vec->get_format(); + switch (format) { + case VEC_FIXED: + tablet_id = get_tablet_id(static_cast(vec), batch_idx); + break; + case VEC_UNIFORM: + tablet_id = get_tablet_id(static_cast(vec), batch_idx); + break; + case VEC_UNIFORM_CONST: + tablet_id = get_tablet_id(static_cast(vec), batch_idx); + break; + default: + LOG_WARN_RET(OB_ERR_UNEXPECTED, "unexpected vector format", K(format)); + break; + } + } + return tablet_id; +} + +bool ObDirectLoadVectorUtils::check_all_tablet_id_is_same(ObIVector *vec, const int64_t size) +{ + bool is_same = true; + if (OB_NOT_NULL(vec)) { + const VectorFormat format = vec->get_format(); + switch (format) { + case VEC_FIXED: { + ObFixedLengthBase *fixed_vec = static_cast(vec); + const uint64_t *ids = reinterpret_cast(fixed_vec->get_data()); + for (int64_t i = 1; i < size; ++i) { + if (ids[i] != ids[0]) { + is_same = false; + break; + } + } + break; + } + case VEC_UNIFORM: { + ObUniformBase *uniform_vec = static_cast(vec); + const ObDatum *datums = uniform_vec->get_datums(); + for (int64_t i = 1; i < size; ++i) { + if (datums[i].get_uint() != datums[0].get_uint()) { + is_same = false; + break; + } + } + break; + } + case VEC_UNIFORM_CONST: + break; + default: + is_same = false; + LOG_WARN_RET(OB_ERR_UNEXPECTED, "unexpected vector format", K(format)); + break; + } + } + return is_same; +} + +/** + * hidden pk vector + */ + +int ObDirectLoadVectorUtils::batch_fill_hidden_pk(ObIVector *vector, const int64_t start, + const int64_t size, + ObTabletCacheInterval &pk_interval) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == vector || start < 0 || size < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(vector), K(start), K(size)); + } else if (size > 0) { + uint64_t start_value = OB_INVALID_ID; + if (1 == size) { + if (OB_FAIL(pk_interval.next_value(start_value))) { + LOG_WARN("fail to get next value", KR(ret), K(pk_interval)); + ret = OB_ERR_UNEXPECTED; // rewrite error code + } + } else { + ObTabletCacheInterval batch_pk; + if (OB_FAIL(pk_interval.fetch(size, batch_pk))) { + LOG_WARN("fail to fetch pk interval", KR(ret), K(pk_interval), K(size)); + ret = OB_ERR_UNEXPECTED; // rewrite error code + } else if (OB_FAIL(batch_pk.get_value(start_value))) { + LOG_WARN("fail to get value", KR(ret), K(batch_pk)); + ret = OB_ERR_UNEXPECTED; // rewrite error code + } + } + if (OB_SUCC(ret)) { + const VectorFormat format = vector->get_format(); + switch (format) { + case VEC_FIXED: { + ObFixedLengthBase *fixed_vec = static_cast(vector); + if (OB_UNLIKELY(fixed_vec->get_length() != sizeof(uint64_t))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected hidden pk vector value length", KR(ret), + K(fixed_vec->get_length())); + } else { + uint64_t *pks = reinterpret_cast(fixed_vec->get_data()); + for (int64_t i = 0; i < size; ++i) { + pks[start + i] = (start_value + i); + } + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected vector format", KR(ret), K(format)); + break; + } + } + } + return ret; +} + +/** + * multi version vector + */ + +int ObDirectLoadVectorUtils::make_const_multi_version_vector(const int64_t value, + ObIAllocator &allocator, + ObIVector *&vector) +{ + int ret = OB_SUCCESS; + vector = nullptr; + if (OB_FAIL(new_vector(VEC_UNIFORM_CONST, multi_version_value_tc, allocator, vector))) { + LOG_WARN("fail to new uniform const vector", KR(ret)); + } else { + ObUniformBase *uniform_vec = static_cast(vector); + ObStorageDatum *storage_datum = nullptr; + if (OB_ISNULL(storage_datum = OB_NEWx(ObStorageDatum, &allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObStorageDatum", KR(ret)); + } else { + storage_datum->set_int(value); + uniform_vec->set_datums(storage_datum); + } + } + return ret; +} + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/direct_load/ob_direct_load_vector_utils.h b/src/storage/direct_load/ob_direct_load_vector_utils.h new file mode 100644 index 0000000000..567328842c --- /dev/null +++ b/src/storage/direct_load/ob_direct_load_vector_utils.h @@ -0,0 +1,89 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once + +#include "common/object/ob_obj_type.h" +#include "share/vector/ob_i_vector.h" + +namespace oceanbase +{ +namespace common +{ +class ObFixedLengthBase; +class ObContinuousBase; +class ObUniformBase; +class ObDatum; +} // namespace common +namespace share +{ +class ObTabletCacheInterval; +} // namespace share +namespace blocksstable +{ +class ObStorageDatum; +class ObDatumRow; +} // namespace blocksstable +namespace sql +{ +class ObBatchRows; +} // namespace sql +namespace storage +{ +class ObDirectLoadVectorUtils +{ +public: + static int new_vector(VectorFormat format, VecValueTypeClass value_tc, ObIAllocator &allocator, + common::ObIVector *&vector); + static int prepare_vector(ObIVector *vector, const int64_t max_batch_size, + ObIAllocator &allocator); + + // 按需实现, 目前只支持以下场景: + // * VEC_CONTINUOUS, VEC_DISCRETE, VEC_UNIFORM, VEC_UNIFORM_CONST -> VEC_DISCRETE + // * VEC_UNIFORM -> VEC_UNIFORM + // * VEC_UNIFORM_CONST -> VEC_UNIFORM_CONST + static int shallow_copy_vector(ObIVector *src_vec, ObIVector *dest_vec, const int64_t batch_size); + + static int get_payload(ObIVector *vector, const int64_t idx, bool &is_null, const char *&payload, + ObLength &len); + + static int to_datum(common::ObIVector *vector, const int64_t idx, common::ObDatum &datum); + static int to_datums(const common::ObIArray &vectors, int64_t idx, + common::ObDatum *datums, const int64_t count); + static int to_datums(const common::ObIArray &vectors, int64_t idx, + blocksstable::ObStorageDatum *datums, const int64_t count); + static int set_datum(common::ObIVector *vector, const int64_t idx, const common::ObDatum &datum); + + // tablet id vector, ginore null value + static const VecValueTypeClass tablet_id_value_tc = VEC_TC_INTEGER; + static int make_const_tablet_id_vector(const ObTabletID &tablet_id, ObIAllocator &allocator, + common::ObIVector *&vector); + static int set_tablet_id(common::ObIVector *vector, const int64_t batch_idx, + const ObTabletID &tablet_id); + static ObTabletID get_tablet_id(common::ObFixedLengthBase *vector, const int64_t batch_idx); + template + static ObTabletID get_tablet_id(common::ObUniformBase *vector, const int64_t batch_idx); + static ObTabletID get_tablet_id(common::ObIVector *vector, const int64_t batch_idx); + static bool check_all_tablet_id_is_same(common::ObIVector *vector, const int64_t size); + + // hidden pk vector + static int batch_fill_hidden_pk(common::ObIVector *vector, const int64_t start, + const int64_t size, share::ObTabletCacheInterval &pk_interval); + + // multi version vector + static const VecValueTypeClass multi_version_value_tc = VEC_TC_INTEGER; + static int make_const_multi_version_vector(const int64_t value, ObIAllocator &allocator, + common::ObIVector *&vector); +}; + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/lob/ob_lob_manager.cpp b/src/storage/lob/ob_lob_manager.cpp index f96bfeddcc..9c6801d9c6 100644 --- a/src/storage/lob/ob_lob_manager.cpp +++ b/src/storage/lob/ob_lob_manager.cpp @@ -1174,7 +1174,7 @@ int ObLobManager::append(ObLobAccessParam& param, ObLobLocatorV2& lob, ObLobMeta SMART_VAR(ObLobAccessParam, read_param) { read_param.tx_desc_ = param.tx_desc_; read_param.tenant_id_ = param.src_tenant_id_; - if (OB_FAIL(build_lob_param(read_param, *param.allocator_, param.coll_type_, + if (OB_FAIL(build_lob_param(read_param, *param.get_tmp_allocator(), param.coll_type_, 0, UINT64_MAX, param.timeout_, lob))) { LOG_WARN("fail to build read param", K(ret), K(lob)); } else if (OB_FAIL(query(read_param, data))) { @@ -1188,9 +1188,6 @@ int ObLobManager::append(ObLobAccessParam& param, ObLobLocatorV2& lob, ObLobMeta ObLobData *new_lob_data = reinterpret_cast(new_lob_common->buffer_); new_lob_data->byte_size_ += data.length(); } - if (alloc_inside) { - param.allocator_->free(param.lob_common_); - } param.lob_common_ = new_lob_common; param.handle_size_ = total_size; if (OB_NOT_NULL(param.lob_locator_)) { @@ -1213,15 +1210,18 @@ int ObLobManager::append(ObLobAccessParam& param, ObLobLocatorV2& lob, ObLobMeta } else { // prepare out row ctx ObLobCtx lob_ctx = lob_ctx_; + int64_t store_chunk_size = 0; if (OB_FAIL(param.init_out_row_ctx(append_lob_len))) { LOG_WARN("init lob data out row ctx failed", K(ret)); + } else if (OB_FAIL(param.get_store_chunk_size(store_chunk_size))) { + LOG_WARN("get_store_chunk_size fail", K(ret), K(param)); } // prepare read buffer ObString read_buffer; - uint64_t read_buff_size = OB_MIN(LOB_READ_BUFFER_LEN, append_lob_len); - char *read_buff = static_cast(param.allocator_->alloc(read_buff_size)); + uint64_t read_buff_size = OB_MIN(store_chunk_size, append_lob_len); + char *read_buff = nullptr; if (OB_FAIL(ret)) { - } else if (OB_ISNULL(read_buff)) { + } else if (OB_ISNULL(read_buff = static_cast(param.get_tmp_allocator()->alloc(read_buff_size)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc read buffer failed.", K(ret), K(read_buff_size)); } else { @@ -1231,11 +1231,11 @@ int ObLobManager::append(ObLobAccessParam& param, ObLobLocatorV2& lob, ObLobMeta // prepare read full lob if (OB_SUCC(ret)) { ObLobLocatorV2* copy_locator = nullptr; - ObLobAccessParam *read_param = reinterpret_cast(param.allocator_->alloc(sizeof(ObLobAccessParam))); + ObLobAccessParam *read_param = reinterpret_cast(param.get_tmp_allocator()->alloc(sizeof(ObLobAccessParam))); if (OB_ISNULL(read_param)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc read param failed.", K(ret), K(sizeof(ObLobAccessParam))); - } else if (OB_ISNULL(copy_locator = OB_NEWx(ObLobLocatorV2, param.allocator_))) { + } else if (OB_ISNULL(copy_locator = OB_NEWx(ObLobLocatorV2, param.get_tmp_allocator()))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc ObLobLocatorV2 failed.", K(ret), K(sizeof(ObLobLocatorV2))); } else { @@ -1243,7 +1243,7 @@ int ObLobManager::append(ObLobAccessParam& param, ObLobLocatorV2& lob, ObLobMeta read_param->tx_desc_ = param.tx_desc_; read_param->tenant_id_ = param.src_tenant_id_; *copy_locator = lob; - if (OB_FAIL(build_lob_param(*read_param, *param.allocator_, param.coll_type_, + if (OB_FAIL(build_lob_param(*read_param, *param.get_tmp_allocator(), param.coll_type_, 0, UINT64_MAX, param.timeout_, *copy_locator))) { LOG_WARN("fail to build read param", K(ret), K(lob), KPC(copy_locator)); } else { diff --git a/src/storage/lob/ob_lob_meta.cpp b/src/storage/lob/ob_lob_meta.cpp index 6bee438158..8e8d842575 100644 --- a/src/storage/lob/ob_lob_meta.cpp +++ b/src/storage/lob/ob_lob_meta.cpp @@ -650,6 +650,8 @@ int ObLobMetaWriteIter::open(ObLobAccessParam ¶m, remain_buf_ = remain_buf; is_store_char_len_ = param.is_store_char_len_; allocator_ = param.get_tmp_allocator(); + seq_id_.set_allocator(allocator_); + seq_id_end_.set_allocator(allocator_); if (OB_FAIL(get_last_meta_info(param, meta_manager))) { LOG_WARN("get_last_meta_info fail", K(ret), K(param)); @@ -691,11 +693,16 @@ int ObLobMetaWriteIter::open(ObLobAccessParam ¶m, remain_buf_ = remain_buf; iter_ = iter; is_store_char_len_ = param.is_store_char_len_; - data_.assign_buffer(read_buf.ptr(), piece_block_size_); + data_.assign_buffer(read_buf.ptr(), OB_MIN(read_buf.size(), piece_block_size_)); allocator_ = param.get_tmp_allocator(); + seq_id_.set_allocator(allocator_); + seq_id_end_.set_allocator(allocator_); char *buf = nullptr; - if (OB_FAIL(get_last_meta_info(param, meta_manager))) { + if (read_buf.size() < 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("read buffer is not enough", K(ret), K(piece_block_size_), K(read_buf.length()), K(read_buf.size())); + } else if (OB_FAIL(get_last_meta_info(param, meta_manager))) { LOG_WARN("get_last_meta_info fail", K(ret), K(param)); } else if (OB_ISNULL(buf = reinterpret_cast(allocator_->alloc(piece_block_size_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; diff --git a/src/storage/lob/ob_lob_util.cpp b/src/storage/lob/ob_lob_util.cpp index 0ffe3e5108..a9aece63d2 100644 --- a/src/storage/lob/ob_lob_util.cpp +++ b/src/storage/lob/ob_lob_util.cpp @@ -208,6 +208,7 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, } int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, + ObIAllocator &lob_allocator, transaction::ObTxDesc *tx_desc, share::ObTabletCacheInterval &lob_id_geneator, const share::ObLSID ls_id, @@ -239,7 +240,18 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, LOG_WARN("fail to get lob data byte len", K(ret), K(src)); } else if (src.has_inrow_data() && lob_mngr->can_write_inrow(byte_len, lob_storage_param.inrow_threshold_)) { // do fast inrow - if (OB_FAIL(src.get_inrow_data(data))) { + if (src.is_inrow_disk_lob_locator()) { + // if is disk inrow lob, no need alloc new memory, just reset lob common header + char *buf = src.ptr_; + ObLobCommon *lob_comm = new (buf) ObLobCommon(); + if (lob_comm->buffer_ != buf + sizeof(ObLobCommon)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("lob common buffer ptr is invalid", K(ret), KPC(lob_comm)); + } else { + datum.set_lob_data(*lob_comm, src.size_); + iter.set_end(); + } + } else if (OB_FAIL(src.get_inrow_data(data))) { LOG_WARN("fail to get inrow data", K(ret), K(src)); } else { void *buf = allocator.alloc(data.length() + sizeof(ObLobCommon)); @@ -271,6 +283,7 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, lob_param.lob_id_geneator_ = &lob_id_geneator; lob_param.inrow_threshold_ = lob_storage_param.inrow_threshold_; lob_param.src_tenant_id_ = src_tenant_id; + lob_param.set_tmp_allocator(&lob_allocator); if (!src.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid src lob locator.", K(ret)); diff --git a/src/storage/lob/ob_lob_util.h b/src/storage/lob/ob_lob_util.h index 7b57914f52..e7460f6f07 100644 --- a/src/storage/lob/ob_lob_util.h +++ b/src/storage/lob/ob_lob_util.h @@ -174,8 +174,13 @@ public: const ObLobStorageParam &lob_storage_param, ObObj &obj, const int64_t timeout_ts); + + // lob_allocator is mainly used for outrow lob read and write memory allocation, + // that can be released after lob inset to avoid hold too many memory + // and res_allocator is mainly used to alloc lob result datum memory in main table // should call iter.close outter - static int insert_lob_column(ObIAllocator &allocator, + static int insert_lob_column(ObIAllocator &res_allocator, + ObIAllocator &lob_allocator, transaction::ObTxDesc *tx_desc, share::ObTabletCacheInterval &lob_id_geneator, const share::ObLSID ls_id, diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index 712f125360..bf22498fab 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -3137,21 +3137,6 @@ int ObLSTabletService::insert_rows( || OB_ISNULL(row_iter)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", K(ret), K(ctx), K(dml_param), K(column_ids), KP(row_iter)); - } else if (dml_param.is_direct_insert()) { // direct-insert mode - if (OB_FAIL(direct_insert_rows(dml_param.table_param_->get_data_table().get_table_id(), - dml_param.direct_insert_task_id_, - dml_param.ddl_task_id_, - ctx.tablet_id_, - column_ids, - row_iter, - afct_num))) { - LOG_WARN("failed to insert rows direct", KR(ret), - K(dml_param.table_param_->get_data_table().get_table_id()), - K(dml_param.direct_insert_task_id_), - K(dml_param.ddl_task_id_), - K(ctx.tablet_id_), - K(column_ids)); - } } else { ObDMLRunningCtx run_ctx(ctx, dml_param, @@ -3218,66 +3203,6 @@ int ObLSTabletService::insert_rows( return ret; } -int ObLSTabletService::direct_insert_rows( - const uint64_t table_id, - const int64_t px_task_id, - const int64_t ddl_task_id, - const ObTabletID &tablet_id, - const ObIArray &column_ids, - blocksstable::ObDatumRowIterator *row_iter, - int64_t &affected_rows) -{ - int ret = OB_SUCCESS; - ObTableLoadTableCtx *table_ctx = nullptr; - ObTableLoadUniqueKey key(table_id, ddl_task_id); - if (OB_FAIL(ObTableLoadService::get_ctx(key, table_ctx))) { - LOG_WARN("fail to get table ctx", KR(ret), K(key)); - } else { - int64_t row_count = 0; - ObDatumRow *rows = nullptr; - table::ObTableLoadTransId trans_id; - trans_id.segment_id_ = px_task_id; - trans_id.trans_gid_ = 1; - ObTableLoadStore store(table_ctx); - ObTableLoadStoreTransPXWriter writer; - if (OB_FAIL(store.init())) { - LOG_WARN("fail to init store", KR(ret)); - } else if (OB_FAIL(store.px_get_trans_writer(trans_id, writer))) { - LOG_WARN("fail to get trans writer", KR(ret), K(trans_id)); - } else if (OB_FAIL(writer.prepare_write(tablet_id, column_ids))) { - LOG_WARN("fail to prepare write", KR(ret), K(tablet_id), K(column_ids)); - } - - while (OB_SUCC(ret) && OB_SUCC(get_next_rows(row_iter, rows, row_count))) { - if (OB_UNLIKELY(row_count <= 0)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("row_count should be greater than 0", K(ret)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && (i < row_count); ++i) { - const ObDatumRow &row = rows[i]; - if (OB_FAIL(writer.write(row))) { - LOG_WARN("fail to write", KR(ret), K(i), K(row)); - } else { - ++affected_rows; - } - } - } - } - - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - } - if (OB_SUCC(ret) && OB_FAIL(writer.finish_write())) { - LOG_WARN("px wirter fail to finish write", KR(ret)); - } - } - if (OB_NOT_NULL(table_ctx)) { - ObTableLoadService::put_ctx(table_ctx); - table_ctx = nullptr; - } - return ret; -} - int ObLSTabletService::get_storage_row( const ObDatumRow &sql_row, const ObIArray &column_ids, diff --git a/src/storage/ls/ob_ls_tablet_service.h b/src/storage/ls/ob_ls_tablet_service.h index f9c5014601..baf79d8fa1 100644 --- a/src/storage/ls/ob_ls_tablet_service.h +++ b/src/storage/ls/ob_ls_tablet_service.h @@ -897,13 +897,6 @@ private: const bool data_tbl_rowkey_change); private: - int direct_insert_rows(const uint64_t table_id, - const int64_t px_task_id, - const int64_t ddl_task_id, - const common::ObTabletID &tablet_id, - const common::ObIArray &column_ids, - blocksstable::ObDatumRowIterator *row_iter, - int64_t &affected_rows); static int get_storage_row(const blocksstable::ObDatumRow &sql_row, const ObIArray &column_ids, const ObColDescIArray &column_descs, diff --git a/src/storage/tx_storage/ob_access_service.cpp b/src/storage/tx_storage/ob_access_service.cpp index c4539558eb..c292e96d44 100644 --- a/src/storage/tx_storage/ob_access_service.cpp +++ b/src/storage/tx_storage/ob_access_service.cpp @@ -748,8 +748,7 @@ int ObAccessService::check_write_allowed_( enable_table_lock = false; ret = OB_SUCCESS; } - if (!dml_param.is_direct_insert() - && OB_FAIL(check_tenant_out_of_memstore_limit_(is_out_of_mem))) { + if (OB_FAIL(check_tenant_out_of_memstore_limit_(is_out_of_mem))) { LOG_WARN("fail to check tenant out of mem limit", K(ret)); } else if (is_out_of_mem && !tablet_id.is_inner_tablet()) { ret = OB_TENANT_OUT_OF_MEM; @@ -776,7 +775,7 @@ int ObAccessService::check_write_allowed_( if (!enable_table_lock) { // do nothing - } else if (dml_param.is_direct_insert() || is_local_index_table) { + } else if (is_local_index_table) { // skip table lock } else if (OB_FAIL(get_lock_id(tablet_id, lock_id))) { LOG_WARN("get lock id failed", K(ret), K(tablet_id)); diff --git a/unittest/storage/blocksstable/cs_encoding/test_cs_encoder.cpp b/unittest/storage/blocksstable/cs_encoding/test_cs_encoder.cpp index 54bc59447c..b622fbfa55 100644 --- a/unittest/storage/blocksstable/cs_encoding/test_cs_encoder.cpp +++ b/unittest/storage/blocksstable/cs_encoding/test_cs_encoder.cpp @@ -580,7 +580,7 @@ TEST_F(TestCSEncoder, test_dict_const_ref_encoder) ASSERT_EQ(e->get_type(), ObCSColumnHeader::Type::INT_DICT); ObIntDictColumnEncoder *dict_encoder = reinterpret_cast(e); ASSERT_EQ(true, dict_encoder->dict_encoding_meta_.is_const_encoding_ref()); - ASSERT_EQ(execption_cnt, dict_encoder->const_list_header_->dict_ref_); // const value is the last one in the sorted dict + ASSERT_EQ(execption_cnt, dict_encoder->const_node_.dict_ref_); // const value is the last one in the sorted dict ASSERT_EQ(execption_cnt, dict_encoder->ref_exception_cnt_); ASSERT_EQ(2 + 2 * execption_cnt, dict_encoder->dict_encoding_meta_.ref_row_cnt_); @@ -589,7 +589,7 @@ TEST_F(TestCSEncoder, test_dict_const_ref_encoder) ObStrDictColumnEncoder *str_dict_encoder = reinterpret_cast(e); ASSERT_EQ(1, str_dict_encoder->max_ref_); ASSERT_EQ(true, str_dict_encoder->dict_encoding_meta_.is_const_encoding_ref()); - ASSERT_EQ(0, str_dict_encoder->const_list_header_->dict_ref_); + ASSERT_EQ(0, str_dict_encoder->const_node_.dict_ref_); ASSERT_EQ(execption_cnt, str_dict_encoder->ref_exception_cnt_); ASSERT_EQ(2 + 2 * execption_cnt, str_dict_encoder->dict_encoding_meta_.ref_row_cnt_); @@ -604,7 +604,7 @@ TEST_F(TestCSEncoder, test_dict_const_ref_encoder) str_dict_encoder = reinterpret_cast(e); ASSERT_EQ(true, str_dict_encoder->dict_encoding_meta_.is_const_encoding_ref()); ASSERT_EQ(2, str_dict_encoder->max_ref_); - ASSERT_EQ(2, str_dict_encoder->const_list_header_->dict_ref_); + ASSERT_EQ(2, str_dict_encoder->const_node_.dict_ref_); ASSERT_EQ(execption_cnt, str_dict_encoder->ref_exception_cnt_); ASSERT_EQ(2 + 2 * execption_cnt, str_dict_encoder->dict_encoding_meta_.ref_row_cnt_); @@ -613,7 +613,7 @@ TEST_F(TestCSEncoder, test_dict_const_ref_encoder) dict_encoder = reinterpret_cast(e); ASSERT_EQ(true, dict_encoder->dict_encoding_meta_.is_const_encoding_ref()); ASSERT_EQ(1, dict_encoder->max_ref_); - ASSERT_EQ(0, dict_encoder->const_list_header_->dict_ref_); + ASSERT_EQ(0, dict_encoder->const_node_.dict_ref_); ASSERT_EQ(execption_cnt, dict_encoder->ref_exception_cnt_); ASSERT_EQ(2 + 2 * execption_cnt, dict_encoder->dict_encoding_meta_.ref_row_cnt_);