hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

change table column to 4096 with optimization

Browse Source
This commit is contained in:
XIAO-HOU
2022-08-22 15:50:30 +08:00
committed by wangyunlai.wyl
parent ffcbed5623
commit 0771b87d44
19 changed files with 438 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
deps/oblib/src/common/row/ob_row_desc.h vendored
View File

@ -90,7 +90,7 @@ private:
};
private:
static const int64_t MAX_COLUMNS_COUNT = common::OB_ROW_MAX_COLUMNS_COUNT; // 512
static const int64_t MAX_COLUMNS_COUNT = common::OB_ROW_MAX_COLUMNS_COUNT; // 4224
static uint64_t HASH_COLLISIONS_COUNT;
// data members
typedef Desc* CellDescArray;

2
deps/oblib/src/lib/ob_define.h vendored
View File

@ -1372,7 +1372,7 @@ const int64_t OB_MAX_MYSQL_RESPONSE_PACKET_SIZE = OB_MALLOC_BIG_BLOCK_SIZE;
const int64_t MAX_FRAME_SIZE = OB_MALLOC_BIG_BLOCK_SIZE;
/// Maximum number of elements/columns a row can contain
const int64_t OB_USER_ROW_MAX_COLUMNS_COUNT = 512;
const int64_t OB_USER_ROW_MAX_COLUMNS_COUNT = 4096;
const int64_t OB_ROW_MAX_COLUMNS_COUNT =
OB_USER_ROW_MAX_COLUMNS_COUNT + 2 * OB_USER_MAX_ROWKEY_COLUMN_NUMBER; // used in ObRow
const int64_t OB_DEFAULT_COL_DEC_NUM = common::OB_ROW_MAX_COLUMNS_COUNT / 80;

4
src/sql/resolver/dml/ob_insert_resolver.h
View File

@ -119,7 +119,7 @@ protected:
{
is_oracle_tmp_table_ = is_temp_table;
}
common::hash::ObPlacementHashSet<uint64_t>& get_insert_column_ids()
common::hash::ObPlacementHashSet<uint64_t, 8209>& get_insert_column_ids()
{
return insert_column_ids_;
}
@ -132,7 +132,7 @@ private:
DISALLOW_COPY_AND_ASSIGN(ObInsertResolver);
private:
common::hash::ObPlacementHashSet<uint64_t> insert_column_ids_;
common::hash::ObPlacementHashSet<uint64_t, 8209> insert_column_ids_; // 8209 is the first prime number greater than twice 4096
int64_t row_count_;
ObSelectResolver* sub_select_resolver_;
bool autoinc_col_added_;

21
src/storage/blocksstable/ob_block_index_intermediate.cpp
View File

@ -63,6 +63,7 @@ void ObBlockIntermediateBuilder::reset()
{
intermediate_row_.reset();
header_.reset();
obj_buf_.reset();
allocator_.reset();
rowkey_column_count_ = 0;
is_inited_ = false;
@ -80,14 +81,20 @@ int ObBlockIntermediateBuilder::init(const int64_t rowkey_column_count)
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "ObBlockIntermediateBuilder init twice", K(ret));
} else if (OB_FAIL(obj_buf_.init(&allocator_))) {
STORAGE_LOG(WARN, "fail to init obj_buf_, ", K(ret));
} else {
rowkey_column_count_ = rowkey_column_count;
intermediate_row_.flag_ = common::ObActionFlag::OP_ROW_EXIST;
intermediate_row_.row_val_.cells_ = new (obj_buf_) ObObj[rowkey_column_count_ + 1];
intermediate_row_.row_val_.count_ = rowkey_column_count_ + 1;
intermediate_row_.row_val_.projector_ = nullptr;
intermediate_row_.row_val_.projector_size_ = 0;
is_inited_ = true;
if (OB_FAIL(obj_buf_.reserve(rowkey_column_count_ + 1))) {
STORAGE_LOG(WARN, "fail to reserve memory for obj_buf_, ", K(ret));
} else {
intermediate_row_.flag_ = common::ObActionFlag::OP_ROW_EXIST;
intermediate_row_.row_val_.cells_ = obj_buf_.get_buf();
intermediate_row_.row_val_.count_ = rowkey_column_count_ + 1;
intermediate_row_.row_val_.projector_ = nullptr;
intermediate_row_.row_val_.projector_size_ = 0;
is_inited_ = true;
}
}
return ret;
}
@ -146,7 +153,7 @@ int ObBlockIntermediateBuilder::set_rowkey(const ObDataStoreDesc& desc, const Ob
K(rowkey_column_count_),
K(desc),
K(ret));
} else if (OB_FAIL(row_reader_.read_compact_rowkey(desc.column_types_,
} else if (OB_FAIL(row_reader_.read_compact_rowkey(desc.column_types_.get_buf(),
desc.rowkey_column_count_,
s_rowkey.ptr(),
s_rowkey.length(),

2
src/storage/blocksstable/ob_block_index_intermediate.h
View File

@ -103,7 +103,7 @@ private:
storage::ObStoreRow intermediate_row_; // rowkey_, buf_
common::ObArenaAllocator allocator_;
ObFlatRowReader row_reader_;
char obj_buf_[common::OB_ROW_MAX_COLUMNS_COUNT * sizeof(ObObj)];
storage::ObDynamicBuffer<ObObj> obj_buf_;
int64_t rowkey_column_count_;
bool is_inited_;
};

17
src/storage/blocksstable/ob_lob_merge_writer.cpp
View File

@ -30,6 +30,7 @@ ObLobMergeWriter::ObLobMergeWriter()
data_store_desc_(NULL),
macro_start_seq_(-1),
use_old_macro_block_count_(0),
allocator_("LobMergeWriter"),
is_inited_(false)
{}
@ -39,12 +40,14 @@ ObLobMergeWriter::~ObLobMergeWriter()
void ObLobMergeWriter::reset()
{
orig_lob_macro_blocks_.reset();
buffer_.reset();
block_write_ctx_.reset();
lob_writer_.reset();
data_store_desc_ = NULL;
macro_start_seq_ = -1;
use_old_macro_block_count_ = 0;
result_row_.reset();
allocator_.reset();
is_inited_ = false;
}
@ -79,6 +82,8 @@ int ObLobMergeWriter::init(const ObMacroDataSeq& macro_start_seq, const ObDataSt
} else if (!block_write_ctx_.file_handle_.is_valid() &&
OB_FAIL(block_write_ctx_.file_handle_.assign(data_store_desc.file_handle_))) {
STORAGE_LOG(WARN, "Failed to assign file handle", K(ret));
} else if (OB_FAIL(buffer_.init(&allocator_))) {
STORAGE_LOG(WARN, "Failed to init buffer_, ", K(ret));
} else {
macro_start_seq_ = lob_data_seq.get_data_seq();
use_old_macro_block_count_ = 0;
@ -410,10 +415,14 @@ int ObLobMergeWriter::write_lob_obj(
int ObLobMergeWriter::copy_row_(const ObStoreRow& row)
{
int ret = OB_SUCCESS;
result_row_ = row;
result_row_.row_val_.cells_ = reinterpret_cast<ObObj*>(buffer_);
result_row_.row_val_.count_ = row.row_val_.count_;
MEMCPY(buffer_, row.row_val_.cells_, sizeof(ObObj) * row.row_val_.count_);
if OB_FAIL(buffer_.reserve(row.row_val_.count_)) {
STORAGE_LOG(WARN, "fail to reserve memory for buffer_, ", K(ret));
} else {
result_row_ = row;
result_row_.row_val_.cells_ = buffer_.get_buf();
result_row_.row_val_.count_ = row.row_val_.count_;
MEMCPY(buffer_.get_buf(), row.row_val_.cells_, sizeof(ObObj) * row.row_val_.count_);
}
return ret;
}

3
src/storage/blocksstable/ob_lob_merge_writer.h
View File

@ -74,9 +74,10 @@ private:
const ObDataStoreDesc* data_store_desc_;
int64_t macro_start_seq_;
int64_t use_old_macro_block_count_;
common::ObArenaAllocator allocator_;
bool is_inited_;
storage::ObStoreRow result_row_;
char buffer_[sizeof(common::ObObj) * common::OB_ROW_MAX_COLUMNS_COUNT];
storage::ObDynamicBuffer<ObObj> buffer_;
};
} // end namespace blocksstable

63
src/storage/blocksstable/ob_macro_block.cpp
View File

@ -124,6 +124,13 @@ int ObDataStoreDesc::init(const ObTableSchema& table_schema, const int64_t data_
} else {
macro_store_size_ = macro_block_size_ * DEFAULT_RESERVE_PERCENT / 100;
}
if (OB_FAIL(column_ids_.init(&allocator_))) {
STORAGE_LOG(WARN, "fail to init column_ids_, ", K(ret));
} else if (OB_FAIL(column_types_.init(&allocator_))) {
STORAGE_LOG(WARN, "fail to init column_types_, ", K(ret));
} else if (OB_FAIL(column_orders_.init(&allocator_))) {
STORAGE_LOG(WARN, "fail to init column_orders_, ", K(ret));
}
if (OB_SUCC(ret)) {
if (OB_FAIL(file_handle_.assign(file_handle))) {
STORAGE_LOG(WARN, "failed to assign file handle", K(ret), K(file_handle));
@ -199,14 +206,22 @@ int ObDataStoreDesc::init(const ObTableSchema& table_schema, const int64_t data_
}
if (OB_SUCC(ret)) {
for (int64_t i = 0; OB_SUCC(ret) && i < column_list.count(); ++i) {
if (i >= common::OB_MAX_COLUMN_NUMBER) {
ret = OB_SIZE_OVERFLOW;
STORAGE_LOG(WARN, "column type&id list overflow.", K(ret));
} else {
column_ids_[i] = column_list.at(i).col_id_;
column_types_[i] = column_list.at(i).col_type_;
column_orders_[i] = column_list.at(i).col_order_;
if (OB_FAIL(column_ids_.reserve(row_column_count_))) {
STORAGE_LOG(WARN, "fail to reserve memory for column_ids_, ", K(ret));
} else if (OB_FAIL(column_types_.reserve(row_column_count_))) {
STORAGE_LOG(WARN, "fail to reserve memory for column_types_, ", K(ret));
} else if (OB_FAIL(column_orders_.reserve(row_column_count_))) {
STORAGE_LOG(WARN, "fail to reserve memory for column_orders_, ", K(ret));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < column_list.count(); ++i) {
if (i >= row_column_count_) {
ret = OB_SIZE_OVERFLOW;
STORAGE_LOG(WARN, "column type&id list overflow.", K(ret));
} else {
column_ids_[i] = column_list.at(i).col_id_;
column_types_[i] = column_list.at(i).col_type_;
column_orders_[i] = column_list.at(i).col_order_;
}
}
}
}
@ -246,9 +261,9 @@ void ObDataStoreDesc::reset()
has_lob_column_ = false;
is_major_ = false;
MEMSET(compressor_name_, 0, OB_MAX_HEADER_COMPRESSOR_NAME_LENGTH);
MEMSET(column_ids_, 0, sizeof(column_ids_));
MEMSET(column_types_, 0, sizeof(column_types_));
MEMSET(column_orders_, 0, sizeof(column_orders_));
column_ids_.reset();
column_types_.reset();
column_orders_.reset();
need_calc_column_checksum_ = false;
need_index_tree_ = false;
store_micro_block_column_checksum_ = false;
@ -265,6 +280,7 @@ void ObDataStoreDesc::reset()
major_working_cluster_version_ = 0;
iter_complement_ = false;
is_unique_index_ = false;
allocator_.reset();
}
int ObDataStoreDesc::assign(const ObDataStoreDesc& desc)
@ -289,9 +305,26 @@ int ObDataStoreDesc::assign(const ObDataStoreDesc& desc)
has_lob_column_ = desc.has_lob_column_;
is_major_ = desc.is_major_;
MEMCPY(compressor_name_, desc.compressor_name_, OB_MAX_HEADER_COMPRESSOR_NAME_LENGTH);
MEMCPY(column_ids_, desc.column_ids_, sizeof(column_ids_));
MEMCPY(column_types_, desc.column_types_, sizeof(column_types_));
MEMCPY(column_orders_, desc.column_orders_, sizeof(column_orders_));
column_ids_.reset();
column_types_.reset();
column_orders_.reset();
if (OB_FAIL(column_ids_.init(&allocator_))) {
STORAGE_LOG(WARN, "fail to init column_ids_, ", K(ret));
} else if (OB_FAIL(column_types_.init(&allocator_))) {
STORAGE_LOG(WARN, "fail to init column_types_, ", K(ret));
} else if (OB_FAIL(column_orders_.init(&allocator_))) {
STORAGE_LOG(WARN, "fail to init column_orders_, ", K(ret));
} else if (OB_FAIL(column_ids_.reserve(desc.row_column_count_))) {
STORAGE_LOG(WARN, "fail to reserve memory for column_ids_, ", K(ret));
} else if (OB_FAIL(column_types_.reserve(desc.row_column_count_))) {
STORAGE_LOG(WARN, "fail to reserve memory for column_types_, ", K(ret));
} else if (OB_FAIL(column_orders_.reserve(desc.row_column_count_))) {
STORAGE_LOG(WARN, "fail to reserve memory for column_orders_, ", K(ret));
} else {
MEMCPY(column_ids_.get_buf(), desc.column_ids_.get_buf(), sizeof(uint64_t) * desc.row_column_count_);
MEMCPY(column_types_.get_buf(), desc.column_types_.get_buf(), sizeof(ObObjMeta) * desc.row_column_count_);
MEMCPY(column_orders_.get_buf(), desc.column_orders_.get_buf(), sizeof(ObOrderType) * desc.row_column_count_);
}
need_calc_column_checksum_ = desc.need_calc_column_checksum_;
store_micro_block_column_checksum_ = desc.store_micro_block_column_checksum_;
snapshot_version_ = desc.snapshot_version_;
@ -928,7 +961,7 @@ int ObMacroBlock::build_macro_meta(ObFullMacroBlockMeta& full_meta)
} else if (OB_FAIL(index_.get_last_rowkey(s_rowkey))) {
STORAGE_LOG(WARN, "micro block index writer fail to get last rowkey", K(ret));
} else if (OB_FAIL(row_reader_->read_compact_rowkey(
spec_->column_types_, spec_->rowkey_column_count_, s_rowkey.ptr(), s_rowkey.length(), pos, row_))) {
spec_->column_types_.get_buf(), spec_->rowkey_column_count_, s_rowkey.ptr(), s_rowkey.length(), pos, row_))) {
STORAGE_LOG(WARN, "row reader fail to read row.", K(ret));
} else {
ObMacroBlockMetaV2& mbi = const_cast<ObMacroBlockMetaV2&>(*full_meta.meta_);

10
src/storage/blocksstable/ob_macro_block.h
View File

@ -51,11 +51,12 @@ struct ObDataStoreDesc {
storage::ObSSTableMergeInfo* merge_info_;
bool has_lob_column_;
bool is_major_;
common::ObArenaAllocator allocator_;
char compressor_name_[common::OB_MAX_HEADER_COMPRESSOR_NAME_LENGTH];
uint64_t column_ids_[common::OB_MAX_COLUMN_NUMBER];
common::ObObjMeta column_types_[common::OB_MAX_COLUMN_NUMBER];
common::ObOrderType column_orders_[common::OB_MAX_COLUMN_NUMBER];
storage::ObDynamicBuffer<uint64_t> column_ids_;
storage::ObDynamicBuffer<ObObjMeta> column_types_;
storage::ObDynamicBuffer<ObOrderType> column_orders_;
bool need_calc_column_checksum_;
bool store_micro_block_column_checksum_;
int64_t snapshot_version_;
@ -75,7 +76,6 @@ struct ObDataStoreDesc {
int64_t major_working_cluster_version_;
bool iter_complement_;
bool is_unique_index_;
common::ObArenaAllocator allocator_;
ObDataStoreDesc()
{
@ -103,7 +103,7 @@ struct ObDataStoreDesc {
K_(partition_id), K_(pct_free), K_(has_lob_column), K_(is_major), K_(need_calc_column_checksum),
K_(store_micro_block_column_checksum), K_(snapshot_version), K_(need_calc_physical_checksum), K_(need_index_tree),
K_(need_prebuild_bloomfilter), K_(bloomfilter_rowkey_prefix), KP_(rowkey_helper), "column_types",
common::ObArrayWrap<common::ObObjMeta>(column_types_, row_column_count_), K_(pg_key), K_(file_handle),
common::ObArrayWrap<common::ObObjMeta>(column_types_.get_buf(), row_column_count_), K_(pg_key), K_(file_handle),
K_(need_check_order), K_(need_index_tree), K_(major_working_cluster_version), K_(iter_complement), K_(is_unique_index));
private:

40
src/storage/blocksstable/ob_macro_block_checker.cpp
View File

@ -28,6 +28,7 @@ void ObSSTableMacroBlockChecker::destroy()
{
flat_reader_.reset();
column_map_.reset();
obj_buf_.reset();
allocator_.reset();
}
@ -396,18 +397,23 @@ int ObSSTableMacroBlockChecker::check_micro_data(
STORAGE_LOG(WARN, "Invalid argument, ", K(ret), KP(micro_buf), K(micro_buf_size), K(meta));
} else if (OB_FAIL(build_column_map(meta))) {
STORAGE_LOG(WARN, "fail to build column map", K(ret));
} else if (OB_FAIL(obj_buf_.init(&allocator_))) {
STORAGE_LOG(WARN, "fail to init obj_buf_, ", K(ret));
} else {
ObIMicroBlockReader* reader = NULL;
ObRowStoreType read_out_type = MAX_ROW_STORE;
ObColumnMap* column_map_ptr = nullptr;
int64_t request_count = 0;
if (FLAT_ROW_STORE == meta.meta_->row_store_type_) {
reader = static_cast<ObIMicroBlockReader*>(&flat_reader_);
read_out_type = FLAT_ROW_STORE;
column_map_ptr = &column_map_;
request_count = column_map_.get_request_count();
} else if (SPARSE_ROW_STORE == meta.meta_->row_store_type_) {
reader = static_cast<ObIMicroBlockReader*>(&sparse_reader_);
read_out_type = SPARSE_ROW_STORE; // write row type is sparse row
column_map_ptr = nullptr; // make reader read full sparse row
request_count = meta.meta_->column_number_;
} else {
ret = OB_NOT_SUPPORTED;
STORAGE_LOG(WARN, "Unexpeceted row store type", K(ret), K(meta.meta_->row_store_type_));
@ -418,21 +424,25 @@ int ObSSTableMacroBlockChecker::check_micro_data(
STORAGE_LOG(WARN, "fail to init micro block reader ", K(ret), K(block_data), K(meta));
} else {
ObStoreRow row;
for (int64_t iter = reader->begin(); OB_SUCC(ret) && iter != reader->end(); ++iter) {
row.row_val_.cells_ = reinterpret_cast<ObObj*>(obj_buf_);
row.row_val_.count_ = OB_ROW_MAX_COLUMNS_COUNT;
row.capacity_ = OB_ROW_MAX_COLUMNS_COUNT;
if (OB_FAIL(reader->get_row(iter, row))) {
STORAGE_LOG(WARN, "fail to get row", K(ret), K(iter), K(meta));
} else if (row.row_val_.count_ != meta.meta_->column_number_) {
ret = OB_INVALID_DATA;
STORAGE_LOG(WARN, "column number not match", K(ret));
} else {
for (int64_t i = 0; i < meta.meta_->column_number_; ++i) {
if (meta.meta_->column_checksum_method_ == CCM_TYPE_AND_VALUE) {
checksum[i] += row.row_val_.cells_[i].checksum(0);
} else if (meta.meta_->column_checksum_method_ == CCM_VALUE_ONLY) {
checksum[i] += row.row_val_.cells_[i].checksum_v2(0);
if (OB_FAIL(obj_buf_.reserve(request_count))) {
STORAGE_LOG(WARN, "fail to reserve memory for obj_buf_, ", K(ret));
} else {
for (int64_t iter = reader->begin(); OB_SUCC(ret) && iter != reader->end(); ++iter) {
row.row_val_.cells_ = obj_buf_.get_buf();
row.row_val_.count_ = request_count;
row.capacity_ = request_count;
if (OB_FAIL(reader->get_row(iter, row))) {
STORAGE_LOG(WARN, "fail to get row", K(ret), K(iter), K(meta));
} else if (row.row_val_.count_ != meta.meta_->column_number_) {
ret = OB_INVALID_DATA;
STORAGE_LOG(WARN, "column number not match", K(ret));
} else {
for (int64_t i = 0; i < meta.meta_->column_number_; ++i) {
if (meta.meta_->column_checksum_method_ == CCM_TYPE_AND_VALUE) {
checksum[i] += row.row_val_.cells_[i].checksum(0);
} else if (meta.meta_->column_checksum_method_ == CCM_VALUE_ONLY) {
checksum[i] += row.row_val_.cells_[i].checksum_v2(0);
}
}
}
}

2
src/storage/blocksstable/ob_macro_block_checker.h
View File

@ -66,7 +66,7 @@ private:
common::ObArenaAllocator allocator_;
ObMacroBlockReader macro_reader_;
ObColumnMap column_map_;
char obj_buf_[common::OB_ROW_MAX_COLUMNS_COUNT * sizeof(common::ObObj)]; // for reader to get row
storage::ObDynamicBuffer<ObObj> obj_buf_;
};
} // namespace blocksstable

20
src/storage/blocksstable/ob_macro_block_writer.cpp
View File

@ -58,6 +58,8 @@ ObMacroBlockWriter::ObMacroBlockWriter()
curr_micro_column_checksum_(NULL),
allocator_("MacrBlocWriter"),
macro_reader_(),
obj_buf_(NULL),
checker_obj_buf_(NULL),
micro_rowkey_hashs_(),
rowkey_helper_(nullptr)
{
@ -107,6 +109,14 @@ void ObMacroBlockWriter::reset()
last_key_.reset();
has_lob_ = false;
lob_writer_.reset();
if (OB_NOT_NULL(obj_buf_)) {
allocator_.free(obj_buf_);
obj_buf_ = NULL;
}
if (OB_NOT_NULL(checker_obj_buf_)) {
allocator_.free(checker_obj_buf_);
checker_obj_buf_ = NULL;
}
check_flat_reader_.reset();
check_sparse_reader_.reset();
micro_rowkey_hashs_.reset();
@ -133,6 +143,12 @@ int ObMacroBlockWriter::open(ObDataStoreDesc& data_store_desc, const ObMacroData
} else if (!block_write_ctx_.file_handle_.is_valid() &&
OB_FAIL(block_write_ctx_.file_handle_.assign(data_store_desc.file_handle_))) {
STORAGE_LOG(WARN, "fail to set file handle", K(ret), K(data_store_desc.file_handle_));
} else if (OB_ISNULL(obj_buf_ = allocator_.alloc(common::OB_ROW_MAX_COLUMNS_COUNT * sizeof(common::ObObj)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "fail to alloc memory for obj_buf_, ", K(ret));
} else if (OB_ISNULL(checker_obj_buf_ = allocator_.alloc(common::OB_ROW_MAX_COLUMNS_COUNT * sizeof(common::ObObj)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "fail to alloc memory for checker_obj_buf_, ", K(ret));
} else {
macro_handle_.set_file(data_store_desc.file_handle_.get_storage_file());
// need to build index tree with leaf node
@ -676,6 +692,7 @@ int ObMacroBlockWriter::save_root_micro_block()
for (int64_t it = micro_reader.begin(); OB_SUCC(ret) && it != micro_reader.end(); ++it) {
row.row_val_.cells_ = reinterpret_cast<ObObj*>(checker_obj_buf_);
row.row_val_.count_ = OB_ROW_MAX_COLUMNS_COUNT;
row.capacity_ = OB_ROW_MAX_COLUMNS_COUNT;
if (OB_FAIL(micro_reader.get_row(it, row))) {
STORAGE_LOG(WARN, "get_row failed", K(ret), K(it), K_(*index_store_desc));
} else if (OB_FAIL(task_top_block_descs->writer_.append_row(row))) {
@ -754,6 +771,7 @@ int ObMacroBlockWriter::merge_root_micro_block()
for (int64_t it = micro_reader.begin(); OB_SUCC(ret) && it != micro_reader.end(); ++it) {
row.row_val_.cells_ = reinterpret_cast<ObObj*>(checker_obj_buf_);
row.row_val_.count_ = OB_ROW_MAX_COLUMNS_COUNT;
row.capacity_ = OB_ROW_MAX_COLUMNS_COUNT;
if (OB_FAIL(micro_reader.get_row(it, row))) {
STORAGE_LOG(WARN, "get_row failed", K(ret), K(it), K(*data_store_desc_));
} else if (OB_FAIL(update_index_tree(0 /* update from 0th level */, &row))) {
@ -1592,8 +1610,8 @@ int ObMacroBlockWriter::print_micro_block_row(ObIMicroBlockReader* micro_reader)
ObStoreRow row;
row.row_val_.cells_ = reinterpret_cast<ObObj*>(checker_obj_buf_);
row.capacity_ = OB_ROW_MAX_COLUMNS_COUNT;
row.row_val_.count_ = OB_ROW_MAX_COLUMNS_COUNT;
for (int64_t it = micro_reader->begin(); OB_SUCC(ret) && it != micro_reader->end(); ++it) {
row.row_val_.count_ = OB_ROW_MAX_COLUMNS_COUNT;
if (OB_FAIL(micro_reader->get_row(it, row))) {
STORAGE_LOG(WARN, "get_row failed", K(ret), K(it), K(*data_store_desc_));
} else { // print error row

7
src/storage/blocksstable/ob_macro_block_writer.h
View File

@ -206,10 +206,9 @@ private:
ObColumnMap column_map_;
ObColumnMap index_column_map_;
blocksstable::ObMacroBlockReader macro_reader_;
char obj_buf_[common::OB_ROW_MAX_COLUMNS_COUNT * sizeof(common::ObObj)]; // for reader to get row
char checker_obj_buf_[common::OB_ROW_MAX_COLUMNS_COUNT * sizeof(common::ObObj)]; // for calc or varify checksum, can
// NOT use same buf of data row
ObMicroBlockReader check_flat_reader_;
void* obj_buf_; // for reader to get row
void* checker_obj_buf_; // for calc or varify checksum, can not use same buf of data row
ObMicroBlockReader check_flat_reader_;
ObSparseMicroBlockReader check_sparse_reader_;
common::ObArray<uint32_t> micro_rowkey_hashs_;
storage::ObSSTableRowkeyHelper* rowkey_helper_;

257
src/storage/blocksstable/ob_micro_block_row_getter.cpp
View File

@ -145,6 +145,10 @@ int ObMicroBlockRowGetter::init(const ObTableIterParam& param, ObTableAccessCont
projector,
sstable->get_multi_version_rowkey_type()))) {
LOG_WARN("fail to init column map", K(ret));
} else if (OB_FAIL(obj_buf_.init(context.allocator_))) {
LOG_WARN("fail to init obj_buf_, ", K(ret));
} else if (OB_FAIL(full_row_obj_buf_.init(context.allocator_))) {
LOG_WARN("fail to init full_row_obj_buf_, ", K(ret));
} else if (OB_FAIL(ObIMicroBlockRowFetcher::init(param, context, sstable))) {
LOG_WARN("fail to init micro block row fecher, ", K(ret));
} else {
@ -167,72 +171,80 @@ int ObMicroBlockRowGetter::get_row(const ObStoreRowkey& rowkey, const MacroBlock
column_map_.rebuild(macro_meta, param_->need_build_column_map(column_map_.get_schema_version())))) {
LOG_WARN("fail to rebuild column map", K(ret), K(macro_meta));
} else {
row_.row_val_.cells_ = reinterpret_cast<ObObj*>(obj_buf_);
row_.row_val_.count_ = OB_ROW_MAX_COLUMNS_COUNT;
row_.capacity_ = OB_ROW_MAX_COLUMNS_COUNT;
if (!context_->enable_put_row_cache()) {
if (OB_FAIL(reader_->get_row(
if (OB_FAIL(obj_buf_.reserve(column_map_.get_request_count()))) {
STORAGE_LOG(WARN, "failed to reserve memory for obj_buf_, ", K(ret));
} else {
row_.row_val_.cells_ = obj_buf_.get_buf();
row_.row_val_.count_ = column_map_.get_request_count();
row_.capacity_ = column_map_.get_request_count();
if (OB_FAIL(reader_->get_row(
context_->pkey_.get_tenant_id(), block_data, rowkey, column_map_, macro_meta, rowkey_helper, row_))) {
if (OB_BEYOND_THE_RANGE == ret) {
if (OB_FAIL(get_not_exist_row(rowkey, row))) {
LOG_WARN("Fail to get not exist row, ", K(ret), K(rowkey));
if (OB_BEYOND_THE_RANGE == ret) {
if (OB_FAIL(get_not_exist_row(rowkey, row))) {
LOG_WARN("Fail to get not exist row, ", K(ret), K(rowkey));
}
} else {
LOG_WARN("Fail to get row, ", K(ret), K(rowkey));
}
} else {
LOG_WARN("Fail to get row, ", K(ret), K(rowkey));
}
} else {
// cast different type
if (OB_UNLIKELY(!column_map_.is_all_column_matched())) {
allocator_.reuse();
const ObColumnIndexItem* column_indexs = column_map_.get_column_indexs();
const int64_t request_count = column_map_.get_request_count();
for (int64_t i = 0; OB_SUCC(ret) && i < request_count; ++i) {
if (!column_indexs[i].is_column_type_matched_) {
if (OB_FAIL(ObIRowReader::cast_obj(
column_indexs[i].request_column_type_, allocator_, row_.row_val_.cells_[i]))) {
LOG_WARN("Fail to cast obj, ", K(ret), K(i), K(column_indexs[i]));
// cast different type
if (OB_UNLIKELY(!column_map_.is_all_column_matched())) {
allocator_.reuse();
const ObColumnIndexItem* column_indexs = column_map_.get_column_indexs();
const int64_t request_count = column_map_.get_request_count();
for (int64_t i = 0; OB_SUCC(ret) && i < request_count; ++i) {
if (!column_indexs[i].is_column_type_matched_) {
if (OB_FAIL(ObIRowReader::cast_obj(
column_indexs[i].request_column_type_, allocator_, row_.row_val_.cells_[i]))) {
LOG_WARN("Fail to cast obj, ", K(ret), K(i), K(column_indexs[i]));
}
}
}
}
}
if (OB_SUCC(ret)) {
row = &row_;
LOG_DEBUG("Success to get row, ", K(*row));
if (OB_SUCC(ret)) {
row = &row_;
LOG_DEBUG("Success to get row, ", K(*row));
}
}
}
} else {
// need read full row to put row cache
full_row_.row_val_.cells_ = reinterpret_cast<ObObj*>(full_row_obj_buf_);
full_row_.row_val_.count_ = OB_ROW_MAX_COLUMNS_COUNT;
full_row_.capacity_ = OB_ROW_MAX_COLUMNS_COUNT;
ObRowCacheValue row_cache_value;
if (OB_FAIL(reader_->get_row(
context_->pkey_.get_tenant_id(), block_data, rowkey, macro_meta, rowkey_helper, full_row_))) {
if (OB_BEYOND_THE_RANGE == ret) {
if (OB_FAIL(get_not_exist_row(rowkey, row))) {
LOG_WARN("Fail to get not exist row, ", K(ret), K(rowkey));
}
} else {
LOG_WARN("Fail to get row, ", K(ret), K(rowkey));
}
if (OB_FAIL(full_row_obj_buf_.reserve(macro_meta.meta_->column_number_))) {
STORAGE_LOG(WARN, "failed to reserve memory for full_row_obj_buf, ", K(ret));
} else {
LOG_DEBUG("Success to get row, ", K(ret), K(rowkey), K(full_row_));
if (OB_FAIL(row_cache_value.init(macro_meta, &full_row_, macro_id))) {
STORAGE_LOG(WARN, "fail to init row cache value", K(ret), K(macro_meta));
} else if (OB_FAIL(project_row(rowkey, row_cache_value, column_map_, row))) {
STORAGE_LOG(WARN, "fail to project cache row", K(ret), K(rowkey), K(row_cache_value));
} else {
// put row cache, ignore fail
ObRowCacheKey row_cache_key(param_->table_id_,
file_id,
rowkey,
sstable_->is_major_sstable() ? 0 : sstable_->get_snapshot_version(),
sstable_->get_key().table_type_);
if (OB_SUCCESS == OB_STORE_CACHE.get_row_cache().put_row(row_cache_key, row_cache_value)) {
context_->access_stat_.row_cache_put_cnt_++;
full_row_.row_val_.cells_ = full_row_obj_buf_.get_buf();
full_row_.row_val_.count_ = macro_meta.meta_->column_number_;
full_row_.capacity_ = macro_meta.meta_->column_number_;
ObRowCacheValue row_cache_value;
if (OB_FAIL(reader_->get_row(
context_->pkey_.get_tenant_id(), block_data, rowkey, macro_meta, rowkey_helper, full_row_))) {
if (OB_BEYOND_THE_RANGE == ret) {
if (OB_FAIL(get_not_exist_row(rowkey, row))) {
LOG_WARN("Fail to get not exist row, ", K(ret), K(rowkey));
}
} else {
LOG_WARN("Fail to get row, ", K(ret), K(rowkey));
}
} else {
LOG_DEBUG("Success to get row, ", K(ret), K(rowkey), K(full_row_));
if (OB_FAIL(row_cache_value.init(macro_meta, &full_row_, macro_id))) {
STORAGE_LOG(WARN, "fail to init row cache value", K(ret), K(macro_meta));
} else if (OB_FAIL(project_row(rowkey, row_cache_value, column_map_, row))) {
STORAGE_LOG(WARN, "fail to project cache row", K(ret), K(rowkey), K(row_cache_value));
} else {
// put row cache, ignore fail
ObRowCacheKey row_cache_key(param_->table_id_,
file_id,
rowkey,
sstable_->is_major_sstable() ? 0 : sstable_->get_snapshot_version(),
sstable_->get_key().table_type_);
if (OB_SUCCESS == OB_STORE_CACHE.get_row_cache().put_row(row_cache_key, row_cache_value)) {
context_->access_stat_.row_cache_put_cnt_++;
}
LOG_DEBUG("Success to get row, ", K(*row), K_(full_row), K(row_cache_value));
}
LOG_DEBUG("Success to get row, ", K(*row), K_(full_row), K(row_cache_value));
}
}
}
@ -277,18 +289,22 @@ int ObMicroBlockRowGetter::get_not_exist_row(const common::ObStoreRowkey& rowkey
} else {
const int64_t request_cnt = column_map_.get_request_count();
const int64_t rowkey_cnt = rowkey.get_obj_cnt();
row_.row_val_.cells_ = reinterpret_cast<ObObj*>(obj_buf_);
row_.flag_ = ObActionFlag::OP_ROW_DOES_NOT_EXIST;
row_.row_val_.count_ = request_cnt;
if (OB_FAIL(obj_buf_.reserve(request_cnt))) {
STORAGE_LOG(WARN, "failed to reserve memory for obj_buf_, ", K(ret));
} else {
row_.row_val_.cells_ = obj_buf_.get_buf();
row_.flag_ = ObActionFlag::OP_ROW_DOES_NOT_EXIST;
row_.row_val_.count_ = request_cnt;
for (int64_t i = 0; i < rowkey_cnt; i++) {
row_.row_val_.cells_[i] = rowkey.get_obj_ptr()[i];
}
for (int64_t i = rowkey_cnt; i < request_cnt; i++) {
row_.row_val_.cells_[i].set_nop_value();
}
for (int64_t i = 0; i < rowkey_cnt; i++) {
row_.row_val_.cells_[i] = rowkey.get_obj_ptr()[i];
}
for (int64_t i = rowkey_cnt; i < request_cnt; i++) {
row_.row_val_.cells_[i].set_nop_value();
}
row = &row_;
row = &row_;
}
}
return ret;
}
@ -319,31 +335,36 @@ int ObMicroBlockRowGetter::project_cache_row(const ObStoreRowkey& rowkey, const
const int64_t column_cnt = value.get_column_cnt();
const ObColumnIndexItem* column_indexs = column_map.get_column_indexs();
const int64_t request_count = column_map.get_request_count();
row_.row_val_.cells_ = reinterpret_cast<ObObj*>(obj_buf_);
row_.row_val_.count_ = OB_ROW_MAX_COLUMNS_COUNT;
if (OB_FAIL(obj_buf_.reserve(request_count))) {
STORAGE_LOG(WARN, "failed to reserve memory for obj_buf_, ", K(ret));
} else {
row_.row_val_.cells_ = obj_buf_.get_buf();
row_.row_val_.count_ = request_count;
row_.capacity_ = request_count;
for (int64_t i = 0; OB_SUCC(ret) && i < request_count; ++i) {
store_index = column_indexs[i].store_index_;
if (OB_INVALID_INDEX != store_index) {
if (store_index < column_cnt) {
row_.row_val_.cells_[i] = obj_array[store_index];
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected store index, ", K(ret), K(store_index), K(column_cnt));
}
} else {
row_.row_val_.cells_[i].set_nop_value();
}
}
// cast different type
if (OB_UNLIKELY(!column_map.is_all_column_matched())) {
allocator_.reuse();
for (int64_t i = 0; OB_SUCC(ret) && i < request_count; ++i) {
if (!column_indexs[i].is_column_type_matched_) {
if (OB_FAIL(
ObIRowReader::cast_obj(column_indexs[i].request_column_type_, allocator_, row_.row_val_.cells_[i]))) {
LOG_WARN("Fail to cast obj, ", K(ret), K(i), K(column_indexs[i]));
store_index = column_indexs[i].store_index_;
if (OB_INVALID_INDEX != store_index) {
if (store_index < column_cnt) {
row_.row_val_.cells_[i] = obj_array[store_index];
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected store index, ", K(ret), K(store_index), K(column_cnt));
}
} else {
row_.row_val_.cells_[i].set_nop_value();
}
}
// cast different type
if (OB_UNLIKELY(!column_map.is_all_column_matched())) {
allocator_.reuse();
for (int64_t i = 0; OB_SUCC(ret) && i < request_count; ++i) {
if (!column_indexs[i].is_column_type_matched_) {
if (OB_FAIL(
ObIRowReader::cast_obj(column_indexs[i].request_column_type_, allocator_, row_.row_val_.cells_[i]))) {
LOG_WARN("Fail to cast obj, ", K(ret), K(i), K(column_indexs[i]));
}
}
}
}
@ -374,41 +395,45 @@ int ObMicroBlockRowGetter::project_cache_sparse_row(const ObStoreRowkey& rowkey,
const ObColumnIndexItem* column_indexs = column_map.get_column_indexs();
const ColumnMap* col_id_map = column_map.get_cols_map();
const int64_t request_count = column_map.get_request_count();
row_.row_val_.cells_ = reinterpret_cast<ObObj*>(obj_buf_);
row_.capacity_ = OB_ROW_MAX_COLUMNS_COUNT;
if (OB_FAIL(obj_buf_.reserve(request_count))) {
STORAGE_LOG(WARN, "failed to reserve memory for obj_buf_, ", K(ret));
} else {
row_.row_val_.cells_ = obj_buf_.get_buf();
row_.capacity_ = request_count;
for (int64_t i = 0; OB_SUCC(ret) && i < request_count; ++i) { // set nop
row_.row_val_.cells_[i].set_nop_value();
}
int32_t col_pos = OB_INVALID_INDEX;
for (int64_t i = 0; OB_SUCC(ret) && i < column_cnt; ++i) {
if (OB_FAIL(col_id_map->get(col_id_array[i], col_pos))) {
if (OB_HASH_NOT_EXIST == ret) {
ret = OB_SUCCESS;
} else {
LOG_WARN("get column pos failed", K(ret), "column_id", col_id_array[i]);
}
} else if (OB_INVALID_INDEX != col_pos) {
if (col_pos > request_count) {
LOG_WARN("invalid column pos", K(ret), "column_id", col_id_array[i], K(col_pos));
} else {
row_.row_val_.cells_[col_pos] = obj_array[i];
}
for (int64_t i = 0; OB_SUCC(ret) && i < request_count; ++i) { // set nop
row_.row_val_.cells_[i].set_nop_value();
}
}
// cast different type
if (OB_SUCC(ret)) {
allocator_.reuse();
for (int64_t i = 0; OB_SUCC(ret) && i < request_count; ++i) {
if (!row_.row_val_.cells_[i].is_nop_value() && !row_.row_val_.cells_[i].is_null() &&
column_indexs[i].request_column_type_ != row_.row_val_.cells_[i].get_meta()) {
if (OB_FAIL(
ObIRowReader::cast_obj(column_indexs[i].request_column_type_, allocator_, row_.row_val_.cells_[i]))) {
LOG_WARN("Fail to cast obj, ", K(ret), K(i), K(column_indexs[i]));
int32_t col_pos = OB_INVALID_INDEX;
for (int64_t i = 0; OB_SUCC(ret) && i < column_cnt; ++i) {
if (OB_FAIL(col_id_map->get(col_id_array[i], col_pos))) {
if (OB_HASH_NOT_EXIST == ret) {
ret = OB_SUCCESS;
} else {
LOG_WARN("get column pos failed", K(ret), "column_id", col_id_array[i]);
}
} else if (OB_INVALID_INDEX != col_pos) {
if (col_pos > request_count) {
LOG_WARN("invalid column pos", K(ret), "column_id", col_id_array[i], K(col_pos));
} else {
row_.row_val_.cells_[col_pos] = obj_array[i];
}
}
} // end for
}
// cast different type
if (OB_SUCC(ret)) {
allocator_.reuse();
for (int64_t i = 0; OB_SUCC(ret) && i < request_count; ++i) {
if (!row_.row_val_.cells_[i].is_nop_value() && !row_.row_val_.cells_[i].is_null() &&
column_indexs[i].request_column_type_ != row_.row_val_.cells_[i].get_meta()) {
if (OB_FAIL(
ObIRowReader::cast_obj(column_indexs[i].request_column_type_, allocator_, row_.row_val_.cells_[i]))) {
LOG_WARN("Fail to cast obj, ", K(ret), K(i), K(column_indexs[i]));
}
}
} // end for
}
}
if (OB_SUCC(ret)) {

4
src/storage/blocksstable/ob_micro_block_row_getter.h
View File

@ -65,9 +65,9 @@ private:
common::ObArenaAllocator allocator_;
ObColumnMap column_map_;
storage::ObStoreRow row_;
char obj_buf_[common::OB_ROW_MAX_COLUMNS_COUNT * sizeof(common::ObObj)];
storage::ObDynamicBuffer<ObObj> obj_buf_;
storage::ObStoreRow full_row_;
char full_row_obj_buf_[common::OB_ROW_MAX_COLUMNS_COUNT * sizeof(common::ObObj)];
storage::ObDynamicBuffer<ObObj> full_row_obj_buf_;
};
} // namespace blocksstable

75
src/storage/blocksstable/ob_micro_block_row_scanner.cpp
View File

@ -469,14 +469,21 @@ int ObMicroBlockRowScanner::init(const ObTableIterParam& param, ObTableAccessCon
projector,
storage::ObMultiVersionRowkeyHelpper::MVRC_NONE /*multi version col type*/))) {
STORAGE_LOG(WARN, "Fail to init col map, ", K(ret));
} else if (OB_FAIL(obj_buf_.init(context.allocator_))) {
STORAGE_LOG(WARN, "fail to init obj_buf_, ", K(ret));
} else {
for (int64_t i = 0; i < ObIMicroBlockReader::OB_MAX_BATCH_ROW_COUNT; ++i) {
ObStoreRow& row = rows_[i];
row.row_val_.cells_ = reinterpret_cast<ObObj*>(obj_buf_) + i * OB_ROW_MAX_COLUMNS_COUNT;
row.row_val_.count_ = OB_ROW_MAX_COLUMNS_COUNT;
row.capacity_ = OB_ROW_MAX_COLUMNS_COUNT;
const int64_t request_count = column_map_.get_request_count();
if (OB_FAIL(obj_buf_.reserve(request_count * ObIMicroBlockReader::OB_MAX_BATCH_ROW_COUNT))) {
STORAGE_LOG(WARN, "fail to reserve memory for obj_buf_, ", K(ret));
} else {
for (int64_t i = 0; i < ObIMicroBlockReader::OB_MAX_BATCH_ROW_COUNT; ++i) {
ObStoreRow& row = rows_[i];
row.row_val_.cells_ = obj_buf_.get_buf() + i * request_count;
row.row_val_.count_ = request_count;
row.capacity_ = request_count;
}
is_inited_ = true;
}
is_inited_ = true;
}
return ret;
}
@ -515,7 +522,8 @@ int ObMicroBlockRowScanner::inner_get_next_row(const ObStoreRow*& row)
}
} else {
ObStoreRow& dest_row = rows_[0];
dest_row.row_val_.count_ = OB_ROW_MAX_COLUMNS_COUNT;
const int64_t request_count = column_map_.get_request_count();
dest_row.row_val_.count_ = request_count;
if (OB_FAIL(reader_->get_row(current_, dest_row))) {
STORAGE_LOG(WARN, "micro block reader fail to get row.", K(ret), K(macro_id_));
} else {
@ -570,8 +578,10 @@ int ObMicroBlockRowScanner::inner_get_next_rows(const storage::ObStoreRow*& rows
void ObMicroBlockRowScanner::reset()
{
ObIMicroBlockRowScanner::reset();
obj_buf_.reset();
}
/***************** ObMultiVersionMicroBlockRowScanner ********************/
int ObMultiVersionMicroBlockRowScanner::init(
const ObTableIterParam& param, ObTableAccessContext& context, const ObSSTable* sstable)
{
@ -605,6 +615,8 @@ int ObMultiVersionMicroBlockRowScanner::init(
STORAGE_LOG(WARN, "failed to alloc row", K(ret), K(cell_cnt_));
} else if (OB_FAIL(nop_pos_.init(*context.allocator_, cell_cnt_))) {
STORAGE_LOG(WARN, "failed to init nop_pos", K(ret), K(cell_cnt_));
} else if (OB_FAIL(tmp_row_obj_buf_.init(context.allocator_))) {
STORAGE_LOG(WARN, "failed to init tmp_row_obj_buf_, ", K(ret));
} else {
const int64_t extra_multi_version_col_cnt =
ObMultiVersionRowkeyHelpper::get_multi_version_rowkey_cnt(sstable->get_multi_version_rowkey_type());
@ -622,7 +634,6 @@ int ObMultiVersionMicroBlockRowScanner::init(
return ret;
}
/***************** ObMultiVersionMicroBlockRowScanner ********************/
int ObMultiVersionMicroBlockRowScanner::open(const MacroBlockId& macro_id, const ObFullMacroBlockMeta& macro_meta,
const ObMicroBlockData& block_data, const bool is_left_border, const bool is_right_border)
{
@ -674,6 +685,7 @@ void ObMultiVersionMicroBlockRowScanner::reset()
{
inner_reset();
ObIMicroBlockRowScanner::reset();
tmp_row_obj_buf_.reset();
}
void ObMultiVersionMicroBlockRowScanner::rescan()
@ -921,9 +933,14 @@ int ObMultiVersionMicroBlockRowScanner::inner_get_next_row_directly(
if (is_row_empty(cur_micro_row_)) {
row = &cur_micro_row_;
} else {
tmp_row_.row_val_.cells_ = reinterpret_cast<ObObj*>(tmp_row_obj_buf_);
tmp_row_.row_val_.count_ = OB_ROW_MAX_COLUMNS_COUNT;
row = &tmp_row_;
const int64_t request_count = column_map_.get_request_count();
if (OB_FAIL(tmp_row_obj_buf_.reserve(request_count))) {
STORAGE_LOG(WARN, "fail to reserve memory for tmp_row_obj_buf_, ", K(ret));
} else {
tmp_row_.row_val_.cells_ = tmp_row_obj_buf_.get_buf();
tmp_row_.row_val_.count_ = request_count;
row = &tmp_row_;
}
}
if (OB_FAIL(reader_->get_row(current_, *row))) {
STORAGE_LOG(WARN, "micro block reader fail to get block_row", K(ret), K(current_), K_(macro_id));
@ -1048,10 +1065,15 @@ int ObMultiVersionMicroBlockRowScanner::inner_inner_get_next_row(
if (is_row_empty(cur_micro_row_)) {
row = &cur_micro_row_;
} else {
tmp_row_.row_val_.cells_ = reinterpret_cast<ObObj*>(tmp_row_obj_buf_);
tmp_row_.row_val_.count_ = OB_ROW_MAX_COLUMNS_COUNT;
tmp_row_.trans_id_ptr_ = flag.is_uncommitted_row() ? &trans_id_ : NULL;
row = &tmp_row_;
const int64_t request_count = column_map_.get_request_count();
if (OB_FAIL(tmp_row_obj_buf_.reserve(request_count))) {
STORAGE_LOG(WARN, "fail to reserve memory for tmp_row_obj_buf_, ", K(ret));
} else {
tmp_row_.row_val_.cells_ = tmp_row_obj_buf_.get_buf();
tmp_row_.row_val_.count_ = request_count;
tmp_row_.trans_id_ptr_ = flag.is_uncommitted_row() ? &trans_id_ : NULL;
row = &tmp_row_;
}
}
if (OB_FAIL(reader_->get_row(current_, *row))) {
STORAGE_LOG(WARN, "micro block reader fail to get block_row", K(ret), K(current_), K_(macro_id));
@ -1384,6 +1406,10 @@ int ObMultiVersionMicroBlockMinorMergeRowScanner::init(
STORAGE_LOG(WARN, "Fail to init col map", K(ret), KPC_(param), K(multi_version_rowkey_type));
} else if (OB_FAIL(init_row_queue(out_cols.count()))) {
STORAGE_LOG(WARN, "Fail to init row queue", K(ret), K(out_cols.count()));
} else if (OB_FAIL(obj_buf_.init(context.allocator_))) {
STORAGE_LOG(WARN, "fail to init obj_buf_, ", K(ret));
} else if (OB_FAIL(col_id_buf_.init(context.allocator_))) {
STORAGE_LOG(WARN, "fail to init col_id_buf_, ", K(ret));
} else {
is_inited_ = true;
}
@ -1416,11 +1442,18 @@ int ObMultiVersionMicroBlockMinorMergeRowScanner::open(const MacroBlockId& macro
} else if (OB_FAIL(set_base_scan_param(is_left_border, is_right_border))) {
STORAGE_LOG(WARN, "failed to set base scan param", K(ret), K(is_left_border), K(is_right_border), K(macro_id));
} else {
row_.row_val_.cells_ = obj_buf_;
row_.column_ids_ = col_id_buf_;
row_.row_val_.count_ = OB_ROW_MAX_COLUMNS_COUNT;
row_.capacity_ = OB_ROW_MAX_COLUMNS_COUNT;
STORAGE_LOG(DEBUG, "set read out type", K(ret), K(context_->read_out_type_));
const int64_t request_count = column_map_.get_request_count();
if (OB_FAIL(obj_buf_.reserve(request_count))) {
STORAGE_LOG(WARN, "failed to reserve memory for obj_buf_", K(ret));
} else if (OB_FAIL(col_id_buf_.reserve(request_count))) {
STORAGE_LOG(WARN, "failed to reserve memory for col_id_buf_", K(ret));
} else {
row_.row_val_.cells_ = obj_buf_.get_buf();
row_.column_ids_ = col_id_buf_.get_buf();
row_.row_val_.count_ = request_count;
row_.capacity_ = request_count;
STORAGE_LOG(DEBUG, "set read out type", K(ret), K(context_->read_out_type_));
}
}
}
return ret;
@ -1522,6 +1555,8 @@ void ObMultiVersionMicroBlockMinorMergeRowScanner::reset()
row_.flag_ = ObActionFlag::OP_ROW_DOES_NOT_EXIST;
clear_scan_status();
ObIMicroBlockRowScanner::reset();
obj_buf_.reset();
col_id_buf_.reset();
}
void ObMultiVersionMicroBlockMinorMergeRowScanner::clear_scan_status()

8
src/storage/blocksstable/ob_micro_block_row_scanner.h
View File

@ -132,7 +132,7 @@ protected:
protected:
storage::ObStoreRow rows_[ObIMicroBlockReader::OB_MAX_BATCH_ROW_COUNT];
char obj_buf_[common::OB_ROW_MAX_COLUMNS_COUNT * sizeof(ObObj) * ObIMicroBlockReader::OB_MAX_BATCH_ROW_COUNT];
storage::ObDynamicBuffer<ObObj> obj_buf_;
};
/*
@ -227,7 +227,7 @@ private:
// TRUE: meet Last Flag of current rowkey
bool is_last_multi_version_row_;
storage::ObStoreRow tmp_row_;
char tmp_row_obj_buf_[common::OB_ROW_MAX_COLUMNS_COUNT * sizeof(ObObj)];
storage::ObDynamicBuffer<ObObj> tmp_row_obj_buf_;
int64_t trans_version_col_idx_;
int64_t sql_sequence_col_idx_;
int64_t cell_cnt_;
@ -318,8 +318,8 @@ private:
ObFixedBitSet<OB_ALL_MAX_COLUMN_ID>* bit_set_[RNPI_MAX];
storage::ObObjDeepCopy obj_copy_;
storage::ObStoreRow row_;
ObObj obj_buf_[common::OB_ROW_MAX_COLUMNS_COUNT];
uint16_t col_id_buf_[common::OB_ROW_MAX_COLUMNS_COUNT];
storage::ObDynamicBuffer<ObObj> obj_buf_;
storage::ObDynamicBuffer<uint16_t> col_id_buf_;
bool is_last_multi_version_row_;
bool is_row_queue_ready_;
bool add_sql_sequence_col_flag_; // old-version sstable should add sql_sequence col

95
src/storage/ob_i_store.h
View File

@ -1573,6 +1573,101 @@ struct ObRowStat {
TO_STRING_KV(K_(base_row_count), K_(inc_row_count), K_(merge_row_count), K_(result_row_count), K_(filt_del_count));
};
template <typename T, int64_t LOCAL_BUFFER_SIZE=64>
class ObDynamicBuffer {
public:
ObDynamicBuffer()
: data_(NULL),
allocator_(NULL),
capacity_(0),
is_inited_(false)
{}
virtual ~ObDynamicBuffer()
{
reset();
}
int init(ObIAllocator *allocator)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "init twice", K(ret));
} else if (OB_ISNULL(allocator)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid arguments", K(ret));
} else {
allocator_ = allocator;
data_ = data_buf_;
capacity_ = LOCAL_BUFFER_SIZE;
is_inited_ = true;
}
return ret;
}
// notice that we do not call the desconstructor of T
void reset()
{
if (OB_NOT_NULL(data_) && data_ != data_buf_) {
allocator_->free(data_);
}
data_ = NULL;
allocator_ = NULL;
capacity_ = 0;
is_inited_ = false;
}
inline int reserve(int64_t capacity) {
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
STORAGE_LOG(ERROR, "row buffer has not been initialized", K(ret));
} else if (capacity < 0) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid arguments", K(ret), K(capacity));
} else if (capacity > capacity_) {
int64_t new_size = capacity * sizeof(T);
void* new_data = allocator_->alloc(new_size);
if (OB_ISNULL(new_data)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "failed to alloc new_data", K(ret));
} else {
if (data_ != data_buf_) {
allocator_->free(data_);
}
data_ = new (new_data) T[capacity];
capacity_ = capacity;
}
}
return ret;
}
inline T* get_buf() const
{
return data_;
}
inline T& operator[](int64_t idx)
{
OB_ASSERT(idx >= 0 && idx < capacity_);
return data_[idx];
}
inline const T& operator[](int64_t idx) const
{
OB_ASSERT(idx >= 0 && idx < capacity_);
return data_[idx];
}
private:
T data_buf_[LOCAL_BUFFER_SIZE];
T* data_;
ObIAllocator* allocator_;
int64_t capacity_;
bool is_inited_;
};
OB_INLINE bool ObStoreRow::is_valid() const
{
bool bool_ret = true;

6
unittest/storage/ob_multi_version_sstable_test.h
View File

@ -402,6 +402,12 @@ void ObMultiVersionSSTableTest::prepare_data_store_desc(
desc_.schema_version_ = SCHEMA_VERSION;
desc_.snapshot_version_ = DATA_VERSION;
strcpy(desc_.compressor_name_, compressor_name);
desc_.column_ids_.init(&allocator_);
desc_.column_types_.init(&allocator_);
desc_.column_orders_.init(&allocator_);
desc_.column_ids_.reserve(column_cnt_);
desc_.column_types_.reserve(column_cnt_);
desc_.column_orders_.reserve(column_cnt_);
for (int i = 0; i < column_cnt_; i++) {
desc_.column_ids_[i] = i + OB_APP_MIN_COLUMN_ID;
desc_.column_types_[i] = data_iter_[0].get_column_type()[i];