hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

shrink memory use in ddl

Browse Source
This commit is contained in:
simonjoylet
2023-02-15 14:42:18 +00:00
committed by ob-robot
parent facae95a7d
commit 076558b3cc
6 changed files with 185 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

189
src/storage/ddl/ob_ddl_merge_task.cpp
View File

@ -359,7 +359,6 @@ int ObDDLTableMergeTask::process()
ObTabletDDLParam ddl_param;
ObTableHandleV2 table_handle;
bool is_data_complete = false;
bool is_scn_overlap = false;
const ObSSTable *latest_major_sstable = nullptr;
if (OB_FAIL(ObTabletDDLUtil::check_and_get_major_sstable(merge_param_.ls_id_, merge_param_.tablet_id_, latest_major_sstable))) {
LOG_WARN("check if major sstable exist failed", K(ret));
@ -377,8 +376,7 @@ int ObDDLTableMergeTask::process()
} else if (merge_param_.is_commit_ && OB_FAIL(ObTabletDDLUtil::check_data_integrity(ddl_sstable_handles,
ddl_param.start_scn_,
merge_param_.rec_scn_,
is_data_complete,
is_scn_overlap))) {
is_data_complete))) {
LOG_WARN("check ddl sstable integrity failed", K(ret), K(ddl_sstable_handles), K(ddl_param), K(merge_param_));
} else if (merge_param_.is_commit_ && !is_data_complete) {
ret = OB_EAGAIN;
@ -390,7 +388,6 @@ int ObDDLTableMergeTask::process()
} else if (OB_FAIL(ObTabletDDLUtil::compact_ddl_sstable(ddl_sstable_handles.get_tables(),
tablet_handle.get_obj()->get_index_read_info(),
ddl_param,
is_scn_overlap,
table_handle))) {
LOG_WARN("compact sstables failed", K(ret));
} else {
@ -439,18 +436,15 @@ int ObDDLTableMergeTask::process()
int ObTabletDDLUtil::check_data_integrity(const ObTablesHandleArray &ddl_sstables,
const SCN &start_scn,
const SCN &prepare_scn,
bool &is_data_complete,
bool &is_scn_overlap)
bool &is_data_complete)
{
int ret = OB_SUCCESS;
is_data_complete = false;
is_scn_overlap = false;
if (OB_UNLIKELY(!start_scn.is_valid_and_not_min() || !prepare_scn.is_valid_and_not_min() || prepare_scn <= start_scn)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(ddl_sstables.get_count()), K(start_scn), K(prepare_scn));
} else if (ddl_sstables.empty()) {
is_data_complete = false;
is_scn_overlap = false;
} else {
const ObSSTable *first_ddl_sstable = static_cast<const ObSSTable *>(ddl_sstables.get_tables().at(0));
const ObSSTable *last_ddl_sstable = static_cast<const ObSSTable *>(ddl_sstables.get_tables().at(ddl_sstables.get_count() - 1));
@ -464,7 +458,6 @@ int ObTabletDDLUtil::check_data_integrity(const ObTablesHandleArray &ddl_sstable
for (int64_t i = 1; OB_SUCC(ret) && i < ddl_sstables.get_count(); ++i) {
const ObSSTable *cur_ddl_sstable = static_cast<const ObSSTable *>(ddl_sstables.get_tables().at(i));
if (cur_ddl_sstable->get_start_scn() < last_end_scn) {
is_scn_overlap = true;
last_end_scn = SCN::max(last_end_scn, cur_ddl_sstable->get_end_scn());
} else if (cur_ddl_sstable->get_start_scn() == last_end_scn) {
last_end_scn = SCN::max(last_end_scn, cur_ddl_sstable->get_end_scn());
@ -572,53 +565,68 @@ int ObTabletDDLUtil::prepare_index_data_desc(const share::ObLSID &ls_id,
return ret;
}
int ObTabletDDLUtil::prepare_index_builder(const ObTabletDDLParam &ddl_param,
ObIAllocator &allocator,
const ObSSTableIndexBuilder::ObSpaceOptimizationMode mode,
const blocksstable::ObSSTable *first_ddl_sstable,
ObSSTableIndexBuilder *&sstable_index_builder,
ObIndexBlockRebuilder *&index_block_rebuilder)
int ObTabletDDLUtil::create_ddl_sstable(const ObTabletDDLParam &ddl_param,
const ObIArray<const ObDataMacroBlockMeta *> &meta_array,
const ObSSTable *first_ddl_sstable,
ObTableHandleV2 &table_handle)
{
int ret = OB_SUCCESS;
sstable_index_builder = nullptr;
index_block_rebuilder = nullptr;
ObDataStoreDesc data_desc;
table_handle.reset();
ObArenaAllocator arena("DdlCreateSST", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID());
void *buf = nullptr;
if (OB_FAIL(prepare_index_data_desc(ddl_param.ls_id_,
ddl_param.table_key_.get_tablet_id(),
ddl_param.table_key_.version_range_.snapshot_version_,
ddl_param.cluster_version_,
first_ddl_sstable,
data_desc))) {
LOG_WARN("prepare data desc failed", K(ret), K(ddl_param));
} else if (OB_ISNULL(buf = allocator.alloc(sizeof(ObSSTableIndexBuilder)))) {
ObSSTableIndexBuilder *sstable_index_builder = nullptr;
ObIndexBlockRebuilder *index_block_rebuilder = nullptr;
ObDataStoreDesc data_desc;
if (OB_UNLIKELY(!ddl_param.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(ddl_param));
} else if (OB_FAIL(ObTabletDDLUtil::prepare_index_data_desc(ddl_param.ls_id_,
ddl_param.table_key_.tablet_id_,
ddl_param.table_key_.version_range_.snapshot_version_,
ddl_param.cluster_version_,
first_ddl_sstable,
data_desc))) {
LOG_WARN("prepare data store desc failed", K(ret), K(ddl_param));
} else if (OB_ISNULL(buf = arena.alloc(sizeof(ObSSTableIndexBuilder)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate memory failed", K(ret));
LOG_WARN("allocate memory for sstable index builder failed", K(ret));
} else if (FALSE_IT(sstable_index_builder = new (buf) ObSSTableIndexBuilder)) {
} else if (OB_FAIL(sstable_index_builder->init(
data_desc,
nullptr, // this param is flush macro call back, nullptr is default val
mode))) {
LOG_WARN("init sstable index builder failed", K(ret));
} else if (OB_ISNULL(buf = allocator.alloc(sizeof(ObIndexBlockRebuilder)))) {
} else if (OB_FAIL(sstable_index_builder->init(data_desc,
nullptr, // macro block flush callback
ObSSTableIndexBuilder::DISABLE))) {
LOG_WARN("init sstable index builder failed", K(ret), K(data_desc));
} else if (OB_ISNULL(buf = arena.alloc(sizeof(ObIndexBlockRebuilder)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate memory failed", K(ret));
} else if (FALSE_IT(index_block_rebuilder = new (buf) ObIndexBlockRebuilder)) {
} else if (OB_FAIL(index_block_rebuilder->init(*sstable_index_builder))) {
LOG_WARN("fail to alloc index builder", K(ret));
} else if (meta_array.empty()) {
// do nothing
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < meta_array.count(); ++i) {
if (OB_FAIL(index_block_rebuilder->append_macro_row(*meta_array.at(i)))) {
LOG_WARN("append block meta failed", K(ret), K(i));
}
}
}
if (OB_FAIL(ret)) {
if (nullptr != index_block_rebuilder) {
index_block_rebuilder->~ObIndexBlockRebuilder();
allocator.free(index_block_rebuilder);
index_block_rebuilder = nullptr;
}
if (nullptr != sstable_index_builder) {
sstable_index_builder->~ObSSTableIndexBuilder();
allocator.free(sstable_index_builder);
sstable_index_builder = nullptr;
if (OB_SUCC(ret)) {
if (OB_FAIL(index_block_rebuilder->close())) {
LOG_WARN("close index block rebuilder failed", K(ret));
} else if (OB_FAIL(ObTabletDDLUtil::create_ddl_sstable(sstable_index_builder, ddl_param, nullptr/*first_ddl_sstable*/, table_handle))) {
LOG_WARN("create ddl sstable failed", K(ret), K(ddl_param));
}
}
if (nullptr != index_block_rebuilder) {
index_block_rebuilder->~ObIndexBlockRebuilder();
arena.free(index_block_rebuilder);
index_block_rebuilder = nullptr;
}
if (nullptr != sstable_index_builder) {
sstable_index_builder->~ObSSTableIndexBuilder();
arena.free(sstable_index_builder);
sstable_index_builder = nullptr;
}
return ret;
}
@ -685,7 +693,7 @@ int ObTabletDDLUtil::create_ddl_sstable(ObSSTableIndexBuilder *sstable_index_bui
param.data_checksum_ = res.data_checksum_;
param.occupy_size_ = res.occupy_size_;
param.original_size_ = res.original_size_;
param.max_merged_trans_version_ = res.max_merged_trans_version_;
param.max_merged_trans_version_ = ddl_param.snapshot_version_;
param.contain_uncommitted_row_ = res.contain_uncommitted_row_;
param.compressor_type_ = res.compressor_type_;
param.encrypt_id_ = res.encrypt_id_;
@ -715,14 +723,12 @@ int ObTabletDDLUtil::create_ddl_sstable(ObSSTableIndexBuilder *sstable_index_bui
return ret;
}
int ObTabletDDLUtil::update_ddl_table_store(ObSSTableIndexBuilder *sstable_index_builder,
const ObTabletDDLParam &ddl_param,
const ObSSTable *first_ddl_sstable,
ObTableHandleV2 &table_handle)
int ObTabletDDLUtil::update_ddl_table_store(const ObTabletDDLParam &ddl_param,
const ObTableHandleV2 &table_handle)
{
int ret = OB_SUCCESS;
if (OB_FAIL(create_ddl_sstable(sstable_index_builder, ddl_param, first_ddl_sstable, table_handle))) {
LOG_WARN("create ddl sstable failed", K(ret));
if (OB_UNLIKELY(!ddl_param.is_valid() || !table_handle.is_valid())) {
LOG_WARN("invalid argument", K(ret), K(ddl_param), K(table_handle));
} else {
ObLSService *ls_service = MTL(ObLSService *);
ObLSHandle ls_handle;
@ -759,24 +765,18 @@ int ObTabletDDLUtil::update_ddl_table_store(ObSSTableIndexBuilder *sstable_index
int ObTabletDDLUtil::compact_ddl_sstable(const ObIArray<ObITable *> &ddl_sstables,
const ObTableReadInfo &read_info,
const ObTabletDDLParam &ddl_param,
const bool is_scn_overlap,
ObTableHandleV2 &table_handle)
{
int ret = OB_SUCCESS;
table_handle.reset();
ObArenaAllocator arena;
ObSSTableIndexBuilder *sstable_index_builder = nullptr;
ObIndexBlockRebuilder *index_block_rebuilder = nullptr;
const ObSSTableIndexBuilder::ObSpaceOptimizationMode mode = ddl_param.table_key_.is_ddl_sstable()
? ObSSTableIndexBuilder::DISABLE : ObSSTableIndexBuilder::AUTO;
ObBlockMetaTree meta_tree;
ObArray<const ObDataMacroBlockMeta *> sorted_metas;
if (OB_UNLIKELY(!ddl_param.is_valid() || ddl_sstables.empty())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(ddl_param), K(ddl_sstables.count()));
} else if (OB_FAIL(prepare_index_builder(ddl_param, arena, mode,
static_cast<ObSSTable *>(ddl_sstables.at(0)),
sstable_index_builder, index_block_rebuilder))) {
LOG_WARN("prepare sstable index builder failed", K(ret));
} else if (OB_FAIL(meta_tree.init(ddl_param.ls_id_, ddl_param.table_key_, ddl_param.start_scn_, ddl_param.cluster_version_))) {
LOG_WARN("init meta tree failed", K(ret), K(ddl_param));
} else {
ObDatumRowkey last_rowkey;
SMART_VAR(ObSSTableSecMetaIterator, meta_iter) {
@ -785,6 +785,7 @@ int ObTabletDDLUtil::compact_ddl_sstable(const ObIArray<ObITable *> &ddl_sstable
ObDataMacroBlockMeta data_macro_meta;
for (int64_t i = 0; OB_SUCC(ret) && i < ddl_sstables.count(); ++i) {
const ObSSTable *cur_sstable = static_cast<const ObSSTable *>(ddl_sstables.at(i));
meta_iter.reset();
if (OB_FAIL(meta_iter.open(query_range,
ObMacroBlockMetaType::DATA_BLOCK_META,
*cur_sstable,
@ -796,30 +797,31 @@ int ObTabletDDLUtil::compact_ddl_sstable(const ObIArray<ObITable *> &ddl_sstable
if (OB_FAIL(meta_iter.get_next(data_macro_meta))) {
if (OB_ITER_END != ret) {
LOG_WARN("get data macro meta failed", K(ret));
} else {
ret = OB_SUCCESS;
break;
}
} else {
int cmp_ret = 1; // defaut 1 for last_rowkey is invalid
const ObDatumRowkey &cur_rowkey = data_macro_meta.end_key_;
if (is_scn_overlap && last_rowkey.is_valid()
&& OB_FAIL(cur_rowkey.compare(last_rowkey,
sstable_index_builder->get_index_store_desc().datum_utils_,
cmp_ret))) {
LOG_WARN("compare rowkey failed", K(ret), K(cur_rowkey), K(last_rowkey));
} else if (cmp_ret <= 0) { // cur_rowkey <= last_rowkey
// exist rowkey, skip
} else if (OB_FAIL(index_block_rebuilder->append_macro_row(data_macro_meta))) {
LOG_WARN("append macro row failed", K(ret));
}
}
}
if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
if (is_scn_overlap && data_macro_meta.end_key_.is_valid()) {
if (OB_FAIL(data_macro_meta.end_key_.deep_copy(last_rowkey, arena))) {
LOG_WARN("deep copy to last rowkey failed", K(ret));
ObDataMacroBlockMeta *copied_meta = nullptr; // copied meta will destruct in the meta tree
ObDDLMacroHandle macro_handle;
bool is_exist = false;
if (OB_FAIL(meta_tree.exist(&data_macro_meta.end_key_, is_exist))) {
LOG_WARN("check block meta exist failed", K(ret), K(data_macro_meta));
} else if (is_exist) {
// skip
} else if (OB_FAIL(macro_handle.set_block_id(data_macro_meta.get_macro_id()))) {
LOG_WARN("hold macro block failed", K(ret));
} else if (OB_FAIL(data_macro_meta.deep_copy(copied_meta, arena))) {
LOG_WARN("deep copy macro block meta failed", K(ret));
} else if (OB_FAIL(meta_tree.insert_macro_block(macro_handle, &copied_meta->end_key_, copied_meta))) {
LOG_WARN("insert meta tree failed", K(ret), K(macro_handle), KPC(copied_meta));
copied_meta->~ObDataMacroBlockMeta();
}
}
}
LOG_INFO("append meta tree finished", K(ret), K(i),
"data_macro_block_cnt_in_sstable", cur_sstable->get_meta().get_basic_meta().get_data_macro_block_count(),
K(meta_tree.get_macro_block_cnt()));
#ifdef ERRSIM
if (OB_SUCC(ret) && ddl_param.table_key_.is_major_sstable()) {
ret = OB_E(EventTable::EN_DDL_COMPACT_FAIL) OB_SUCCESS;
@ -829,35 +831,26 @@ int ObTabletDDLUtil::compact_ddl_sstable(const ObIArray<ObITable *> &ddl_sstable
}
#endif
}
if (OB_SUCC(ret)) {
meta_iter.reset();
}
}
}
}
// close
if (OB_SUCC(ret)) {
if (OB_FAIL(index_block_rebuilder->close())) {
LOG_WARN("index block rebuilder close failed", K(ret));
} else if (OB_FAIL(update_ddl_table_store(sstable_index_builder,
ddl_param,
static_cast<ObSSTable *>(ddl_sstables.at(0)),
table_handle))) {
if (OB_FAIL(meta_tree.build_sorted_rowkeys())) {
LOG_WARN("build sorted rowkey failed", K(ret));
} else if (OB_FAIL(meta_tree.get_sorted_meta_array(sorted_metas))) {
LOG_WARN("get sorted metas failed", K(ret));
} else if (OB_FAIL(create_ddl_sstable(ddl_param,
sorted_metas,
static_cast<ObSSTable *>(ddl_sstables.at(0)),
table_handle))) {
LOG_WARN("create ddl sstable failed", K(ret));
} else if (OB_FAIL(update_ddl_table_store(ddl_param, table_handle))) {
LOG_WARN("update ddl table store failed", K(ret));
} else {
LOG_INFO("compact ddl sstable success", K(ddl_param));
}
}
if (nullptr != index_block_rebuilder) {
index_block_rebuilder->~ObIndexBlockRebuilder();
arena.free(index_block_rebuilder);
index_block_rebuilder = nullptr;
}
if (nullptr != sstable_index_builder) {
sstable_index_builder->~ObSSTableIndexBuilder();
arena.free(sstable_index_builder);
sstable_index_builder = nullptr;
}
return ret;
}

20
src/storage/ddl/ob_ddl_merge_task.h
View File

@ -152,27 +152,22 @@ public:
const blocksstable::ObSSTable *first_ddl_sstable,
blocksstable::ObDataStoreDesc &data_desc);
static int prepare_index_builder(const ObTabletDDLParam &ddl_param,
ObIAllocator &allocator,
const blocksstable::ObSSTableIndexBuilder::ObSpaceOptimizationMode mode,
const blocksstable::ObSSTable *first_ddl_sstable,
blocksstable::ObSSTableIndexBuilder *&sstable_index_builder,
blocksstable::ObIndexBlockRebuilder *&index_block_rebuilder);
static int create_ddl_sstable(const ObTabletDDLParam &ddl_param,
const ObIArray<const blocksstable::ObDataMacroBlockMeta *> &meta_array,
const blocksstable::ObSSTable *first_ddl_sstable,
ObTableHandleV2 &table_handle);
static int create_ddl_sstable(blocksstable::ObSSTableIndexBuilder *sstable_index_builder,
const ObTabletDDLParam &ddl_param,
const blocksstable::ObSSTable *first_ddl_sstable,
ObTableHandleV2 &table_handle);
static int update_ddl_table_store(blocksstable::ObSSTableIndexBuilder *sstable_index_builder,
const ObTabletDDLParam &ddl_param,
const blocksstable::ObSSTable *first_ddl_sstable,
ObTableHandleV2 &table_handle);
static int update_ddl_table_store(const ObTabletDDLParam &ddl_param,
const ObTableHandleV2 &table_handle);
static int compact_ddl_sstable(const ObIArray<ObITable *> &ddl_sstables,
const ObTableReadInfo &read_info,
const ObTabletDDLParam &ddl_param,
const bool is_scn_overlap,
ObTableHandleV2 &table_handle);
static int report_ddl_checksum(const share::ObLSID &ls_id,
@ -188,8 +183,7 @@ public:
static int check_data_integrity(const ObTablesHandleArray &ddl_sstables,
const share::SCN &start_scn,
const share::SCN &prepare_scn,
bool &is_data_complete,
bool &is_scn_overlap);
bool &is_data_complete);
};
} // namespace storage

122
src/storage/ddl/ob_tablet_ddl_kv.cpp
View File

@ -114,6 +114,7 @@ int ObDDLKV::init_sstable_param(const share::ObLSID &ls_id,
sstable_param.schema_version_ = storage_schema.get_schema_version();
sstable_param.latest_row_store_type_ = storage_schema.get_row_store_type();
sstable_param.create_snapshot_version_ = table_key.get_snapshot_version();
sstable_param.max_merged_trans_version_ = table_key.get_snapshot_version();
sstable_param.ddl_scn_ = ddl_start_scn;
sstable_param.root_row_store_type_ = data_desc.row_store_type_;
sstable_param.data_index_tree_height_ = 2; // fixed tree height, because there is only one root block
@ -171,19 +172,20 @@ void ObBlockMetaTree::destroy()
}
int ObBlockMetaTree::insert_macro_block(const ObDDLMacroHandle &macro_handle,
const blocksstable::ObDatumRowkeyWrapper &rowkey,
const blocksstable::ObDatumRowkey *rowkey,
const blocksstable::ObDataMacroBlockMeta *meta)
{
int ret = OB_SUCCESS;
ObDataMacroBlockMeta *insert_meta = const_cast<ObDataMacroBlockMeta *>(meta);
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_UNLIKELY(!macro_handle.is_valid() || !rowkey.is_valid() || nullptr == meta)) {
} else if (OB_UNLIKELY(!macro_handle.is_valid() || nullptr == rowkey || nullptr == meta)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(macro_handle), K(rowkey), KP(meta));
} else if (OB_FAIL(macro_blocks_.push_back(macro_handle))) {
LOG_WARN("push back macro handle failed", K(ret), K(macro_handle));
} else if (OB_FAIL(block_tree_.insert(rowkey, meta))) {
} else if (OB_FAIL(block_tree_.insert(ObDatumRowkeyWrapper(rowkey, &data_desc_.datum_utils_), insert_meta))) {
LOG_WARN("insert block tree failed", K(ret), K(rowkey), KPC(meta));
}
return ret;
@ -212,7 +214,7 @@ int ObBlockMetaTree::build_sorted_rowkeys()
} else {
while (OB_SUCC(ret)) {
ObDatumRowkeyWrapper rowkey_wrapper;
const ObDataMacroBlockMeta *block_meta = nullptr;
ObDataMacroBlockMeta *block_meta = nullptr;
if (OB_FAIL(iter.get_next(rowkey_wrapper, block_meta))) {
if (OB_ITER_END != ret) {
LOG_WARN("get next failed", K(ret));
@ -260,6 +262,51 @@ int ObBlockMetaTree::build_sorted_rowkeys()
return ret;
}
int ObBlockMetaTree::get_sorted_meta_array(ObIArray<const ObDataMacroBlockMeta *> &meta_array) const
{
int ret = OB_SUCCESS;
meta_array.reset();
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (sorted_rowkeys_.count() != macro_blocks_.count()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sorted array not ready", K(ret), K(sorted_rowkeys_.count()), K(macro_blocks_.count()));
} else if (OB_FAIL(meta_array.reserve(sorted_rowkeys_.count()))) {
LOG_WARN("reserve meta array failed", K(ret), K(sorted_rowkeys_.count()));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < sorted_rowkeys_.count(); ++i) {
if (OB_FAIL(meta_array.push_back(sorted_rowkeys_.at(i).block_meta_))) {
LOG_WARN("push back block meta failed", K(ret), K(i));
}
}
}
return ret;
}
int ObBlockMetaTree::exist(const blocksstable::ObDatumRowkey *rowkey, bool &is_exist)
{
int ret = OB_SUCCESS;
is_exist = false;
ObDataMacroBlockMeta *dummp_meta = nullptr;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_ISNULL(rowkey)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KP(rowkey));
} else if (OB_FAIL(block_tree_.get(ObDatumRowkeyWrapper(rowkey, &data_desc_.datum_utils_), dummp_meta))) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("get value from block meta tree failed", K(ret), KPC(rowkey));
} else {
is_exist = false;
ret = OB_SUCCESS;
}
} else {
is_exist = true;
}
return ret;
}
bool ObBlockMetaTree::CompareFunctor::operator ()(const IndexItem &item,
const blocksstable::ObDatumRowkey &rowkey)
@ -387,8 +434,7 @@ int ObBlockMetaTree::get_last_rowkey(const ObDatumRowkey *&last_rowkey)
ObDDLKV::ObDDLKV()
: is_inited_(false), ls_id_(), tablet_id_(), ddl_start_scn_(SCN::min_scn()), snapshot_version_(0),
lock_(), arena_allocator_("DDL_KV"), is_freezed_(false), is_closed_(false), last_freezed_scn_(SCN::min_scn()),
min_scn_(SCN::max_scn()), max_scn_(SCN::min_scn()), freeze_scn_(SCN::max_scn()), pending_cnt_(0), cluster_version_(0),
sstable_index_builder_(nullptr), index_block_rebuilder_(nullptr), is_rebuilder_closed_(false)
min_scn_(SCN::max_scn()), max_scn_(SCN::min_scn()), freeze_scn_(SCN::max_scn()), pending_cnt_(0), cluster_version_(0)
{
}
@ -429,10 +475,7 @@ int ObDDLKV::init(const share::ObLSID &ls_id,
ddl_param.snapshot_version_ = snapshot_version;
ddl_param.cluster_version_ = cluster_version;
ObTabletCreateSSTableParam sstable_param;
if (OB_FAIL(ObTabletDDLUtil::prepare_index_builder(ddl_param, arena_allocator_, ObSSTableIndexBuilder::DISABLE,
nullptr/*first_ddl_sstable*/, sstable_index_builder_, index_block_rebuilder_))) {
LOG_WARN("prepare index builder failed", K(ret));
} else if (OB_FAIL(block_meta_tree_.init(ls_id, ddl_param.table_key_, ddl_start_scn, cluster_version))) {
if (OB_FAIL(block_meta_tree_.init(ls_id, ddl_param.table_key_, ddl_start_scn, cluster_version))) {
LOG_WARN("init mem index sstable failed", K(ret), K(ddl_param));
} else if (OB_FAIL(init_sstable_param(ls_id, ddl_param.table_key_, ddl_start_scn, sstable_param))) {
LOG_WARN("init sstable param failed", K(ret));
@ -469,17 +512,6 @@ void ObDDLKV::reset()
freeze_scn_ = SCN::max_scn();
pending_cnt_ = 0;
cluster_version_ = 0;
is_rebuilder_closed_ = false;
if (nullptr != index_block_rebuilder_) {
index_block_rebuilder_->~ObIndexBlockRebuilder();
arena_allocator_.free(index_block_rebuilder_);
index_block_rebuilder_ = nullptr;
}
if (nullptr != sstable_index_builder_) {
sstable_index_builder_->~ObSSTableIndexBuilder();
arena_allocator_.free(sstable_index_builder_);
sstable_index_builder_ = nullptr;
}
block_meta_tree_.destroy();
arena_allocator_.reset();
}
@ -544,11 +576,8 @@ int ObDDLKV::set_macro_block(const ObDDLMacroBlock &macro_block)
} else if (macro_block.scn_ > freeze_scn_) {
ret = OB_EAGAIN;
LOG_INFO("this ddl kv is freezed, retry other ddl kv", K(ret), K(ls_id_), K(tablet_id_), K(macro_block), K(freeze_scn_));
} else if (OB_FAIL(index_block_rebuilder_->get_macro_meta(macro_block.buf_, macro_block.size_, macro_block.get_block_id(), arena_allocator_, data_macro_meta))) {
} else if (OB_FAIL(ObIndexBlockRebuilder::get_macro_meta(macro_block.buf_, macro_block.size_, macro_block.get_block_id(), arena_allocator_, data_macro_meta))) {
LOG_WARN("get macro meta failed", K(ret), K(macro_block));
} else if (OB_FAIL(index_block_rebuilder_->append_macro_row(
macro_block.buf_, macro_block.size_, macro_block.get_block_id()))) {
LOG_WARN("append macro meta failed", K(ret), K(macro_block));
} else if (OB_FAIL(insert_block_meta_tree(macro_block.block_handle_, data_macro_meta))) {
LOG_WARN("insert macro block failed", K(ret), K(macro_block), KPC(data_macro_meta));
} else {
@ -563,15 +592,12 @@ int ObDDLKV::set_macro_block(const ObDDLMacroBlock &macro_block)
int ObDDLKV::insert_block_meta_tree(const ObDDLMacroHandle &macro_handle, blocksstable::ObDataMacroBlockMeta *data_macro_meta)
{
int ret = OB_SUCCESS;
if (OB_FAIL(block_meta_tree_.insert_macro_block(macro_handle,
ObDatumRowkeyWrapper(&data_macro_meta->end_key_, &sstable_index_builder_->get_index_store_desc().datum_utils_),
data_macro_meta))) {
if (OB_FAIL(block_meta_tree_.insert_macro_block(macro_handle, &data_macro_meta->end_key_, data_macro_meta))) {
LOG_WARN("insert macro block failed", K(ret), K(macro_handle), KPC(data_macro_meta));
} else {
const ObDataBlockMetaVal &meta_val = data_macro_meta->get_meta_val();
meta_.get_basic_meta().data_macro_block_count_ += 1;
meta_.get_basic_meta().data_micro_block_count_ += meta_val.micro_block_count_;
meta_.get_basic_meta().max_merged_trans_version_ = max(meta_.get_basic_meta().max_merged_trans_version_, meta_val.max_merged_trans_version_);
meta_.get_basic_meta().row_count_ += meta_val.row_count_;
meta_.get_basic_meta().data_checksum_ = ob_crc64_sse42(meta_.get_basic_meta().data_checksum_, &meta_val.data_checksum_, sizeof(meta_val.data_checksum_));
meta_.get_basic_meta().occupy_size_ += meta_val.occupy_size_;
@ -621,11 +647,14 @@ int ObDDLKV::prepare_sstable()
if (OB_EAGAIN != ret) {
LOG_WARN("wait pending failed", K(ret));
}
} else if (OB_FAIL(block_meta_tree_.build_sorted_rowkeys())) {
LOG_WARN("build sorted keys failed", K(ret), K(block_meta_tree_));
} else {
key_.scn_range_.start_scn_ = last_freezed_scn_;
key_.scn_range_.end_scn_ = freeze_scn_;
TCWLockGuard guard(lock_);
if (OB_FAIL(block_meta_tree_.build_sorted_rowkeys())) {
LOG_WARN("build sorted keys failed", K(ret), K(block_meta_tree_));
} else {
key_.scn_range_.start_scn_ = last_freezed_scn_;
key_.scn_range_.end_scn_ = freeze_scn_;
}
}
return ret;
}
@ -633,24 +662,18 @@ int ObDDLKV::prepare_sstable()
int ObDDLKV::close()
{
int ret = OB_SUCCESS;
ObArray<const ObDataMacroBlockMeta *> meta_array;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ddl kv is not init", K(ret));
} else if (is_closed_) {
// do nothing
LOG_INFO("ddl kv already closed", K(*this));
} else if (OB_FAIL(wait_pending())) {
if (OB_EAGAIN != ret) {
LOG_WARN("wait pending failed", K(ret));
}
} else if (!is_rebuilder_closed_) {
if (OB_FAIL(index_block_rebuilder_->close())) {
LOG_WARN("index block rebuilder close failed", K(ret));
} else {
is_rebuilder_closed_ = true;
}
}
if (OB_SUCC(ret) && !is_closed_) {
} else if (OB_FAIL(prepare_sstable())) {
LOG_WARN("prepare sstable failed", K(ret));
} else if (OB_FAIL(block_meta_tree_.get_sorted_meta_array(meta_array))) {
LOG_WARN("get sorted meta array failed", K(ret));
} else {
ObTableHandleV2 table_handle;
ObTabletDDLParam ddl_param;
ddl_param.tenant_id_ = MTL_ID();
@ -662,11 +685,10 @@ int ObDDLKV::close()
ddl_param.start_scn_ = ddl_start_scn_;
ddl_param.snapshot_version_ = snapshot_version_;
ddl_param.cluster_version_ = cluster_version_;
if (OB_FAIL(ObTabletDDLUtil::update_ddl_table_store(sstable_index_builder_,
ddl_param,
nullptr/*first_ddl_sstable*/,
table_handle))) {
LOG_WARN("create ddl sstable failed", K(ret));
if (OB_FAIL(ObTabletDDLUtil::create_ddl_sstable(ddl_param, meta_array, nullptr/*first_ddl_sstable*/, table_handle))) {
LOG_WARN("create ddl sstable failed", K(ret), K(ddl_param));
} else if (OB_FAIL(ObTabletDDLUtil::update_ddl_table_store(ddl_param, table_handle))) {
LOG_WARN("update ddl table store failed", K(ret), K(ddl_param), K(table_handle));
} else {
is_closed_ = true;
LOG_INFO("ddl kv closed success", K(*this));

16
src/storage/ddl/ob_tablet_ddl_kv.h
View File

@ -42,10 +42,10 @@ namespace storage
class ObBlockMetaTree
{
typedef keybtree::ObKeyBtree<blocksstable::ObDatumRowkeyWrapper, const blocksstable::ObDataMacroBlockMeta *> KeyBtree;
typedef keybtree::BtreeIterator<blocksstable::ObDatumRowkeyWrapper, const blocksstable::ObDataMacroBlockMeta *> BtreeIterator;
typedef keybtree::BtreeNodeAllocator<blocksstable::ObDatumRowkeyWrapper, const blocksstable::ObDataMacroBlockMeta *> BtreeNodeAllocator;
typedef keybtree::BtreeRawIterator<blocksstable::ObDatumRowkeyWrapper, const blocksstable::ObDataMacroBlockMeta *> BtreeRawIterator;
typedef keybtree::ObKeyBtree<blocksstable::ObDatumRowkeyWrapper, blocksstable::ObDataMacroBlockMeta *> KeyBtree;
typedef keybtree::BtreeIterator<blocksstable::ObDatumRowkeyWrapper, blocksstable::ObDataMacroBlockMeta *> BtreeIterator;
typedef keybtree::BtreeNodeAllocator<blocksstable::ObDatumRowkeyWrapper, blocksstable::ObDataMacroBlockMeta *> BtreeNodeAllocator;
typedef keybtree::BtreeRawIterator<blocksstable::ObDatumRowkeyWrapper, blocksstable::ObDataMacroBlockMeta *> BtreeRawIterator;
public:
ObBlockMetaTree();
virtual ~ObBlockMetaTree();
@ -55,7 +55,7 @@ public:
const int64_t cluster_version);
void destroy();
int insert_macro_block(const ObDDLMacroHandle &macro_handle,
const blocksstable::ObDatumRowkeyWrapper &rowkey,
const blocksstable::ObDatumRowkey *rowkey,
const blocksstable::ObDataMacroBlockMeta *meta);
int locate_range(const blocksstable::ObDatumRange &range,
const blocksstable::ObStorageDatumUtils &datum_utils,
@ -71,6 +71,8 @@ public:
int64_t get_macro_block_cnt() const { return macro_blocks_.count(); }
int get_last_rowkey(const blocksstable::ObDatumRowkey *&last_rowkey);
int build_sorted_rowkeys();
int get_sorted_meta_array(ObIArray<const blocksstable::ObDataMacroBlockMeta *> &meta_array) const;
int exist(const blocksstable::ObDatumRowkey *rowkey, bool &is_exist);
const blocksstable::ObDataStoreDesc &get_data_desc() const { return data_desc_; }
TO_STRING_KV(K(is_inited_), K(macro_blocks_.count()), K(arena_.total()), K(data_desc_), K(sorted_rowkeys_.count()));
private:
@ -136,7 +138,6 @@ public:
K_(is_freezed), K_(is_closed),
K_(last_freezed_scn), K_(min_scn), K_(max_scn), K_(freeze_scn),
K_(pending_cnt), K_(cluster_version), K_(ref_cnt),
KP_(sstable_index_builder), KP_(index_block_rebuilder), K_(is_rebuilder_closed),
K_(block_meta_tree));
private:
int insert_block_meta_tree(const ObDDLMacroHandle &macro_handle,
@ -163,9 +164,6 @@ private:
share::SCN freeze_scn_; // ddl kv refuse data larger than freeze log ts, freeze_log_ts >= max_log_ts
int64_t pending_cnt_; // the amount of kvs that are replaying
int64_t cluster_version_;
blocksstable::ObSSTableIndexBuilder *sstable_index_builder_;
blocksstable::ObIndexBlockRebuilder *index_block_rebuilder_;
bool is_rebuilder_closed_;
ObBlockMetaTree block_meta_tree_;
};

28
src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp
View File

@ -751,10 +751,6 @@ int ObTabletDDLKvMgr::create_empty_ddl_sstable(ObTableHandleV2 &table_handle)
{
int ret = OB_SUCCESS;
table_handle.reset();
ObArenaAllocator arena;
ObSSTableIndexBuilder *sstable_index_builder = nullptr;
ObIndexBlockRebuilder *index_block_rebuilder = nullptr;
const ObSSTableIndexBuilder::ObSpaceOptimizationMode mode = ObSSTableIndexBuilder::DISABLE;
ObTabletDDLParam ddl_param;
if (OB_FAIL(get_ddl_param(ddl_param))) {
LOG_WARN("get ddl param failed", K(ret));
@ -762,24 +758,12 @@ int ObTabletDDLKvMgr::create_empty_ddl_sstable(ObTableHandleV2 &table_handle)
ddl_param.table_key_.table_type_ = ObITable::DDL_DUMP_SSTABLE;
ddl_param.table_key_.scn_range_.start_scn_ = SCN::scn_dec(start_scn_);
ddl_param.table_key_.scn_range_.end_scn_ = start_scn_;
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(ObTabletDDLUtil::prepare_index_builder(ddl_param, arena, mode, nullptr/*first_ddl_sstable*/,
sstable_index_builder, index_block_rebuilder))) {
LOG_WARN("prepare sstable index builder failed", K(ret), K(ddl_param));
} else if (OB_FAIL(ObTabletDDLUtil::create_ddl_sstable(sstable_index_builder, ddl_param, nullptr/*first_ddl_sstable*/, table_handle))) {
LOG_WARN("create ddl sstable failed", K(ret), K(ddl_param));
}
if (nullptr != index_block_rebuilder) {
index_block_rebuilder->~ObIndexBlockRebuilder();
arena.free(index_block_rebuilder);
index_block_rebuilder = nullptr;
}
if (nullptr != sstable_index_builder) {
sstable_index_builder->~ObSSTableIndexBuilder();
arena.free(sstable_index_builder);
sstable_index_builder = nullptr;
ObArray<const ObDataMacroBlockMeta *> empty_meta_array;
if (OB_FAIL(ObTabletDDLUtil::create_ddl_sstable(ddl_param, empty_meta_array, nullptr/*first_ddl_sstable*/, table_handle))) {
LOG_WARN("create empty ddl sstable failed", K(ret));
} else if (OB_FAIL(ObTabletDDLUtil::update_ddl_table_store(ddl_param, table_handle))) {
LOG_WARN("update ddl table store failed", K(ret), K(ddl_param), K(table_handle));
}
}
return ret;
}

3
unittest/storage/ddl/test_ddl_kv.cpp
View File

@ -146,6 +146,7 @@ TEST_F(TestBlockMetaTree, random_keybtree)
for (int64_t i = 0; i < 10; ++i) {
ASSERT_OK(meta_tree.block_tree_.init());
meta_tree.is_inited_ = true;
ASSERT_OK(meta_tree.data_desc_.assign(data_desc_));
for (int64_t j = 0; j < 10000; ++j) {
void *buf = allocator_.alloc(sizeof(ObDatumRow));
ASSERT_TRUE(nullptr != buf);
@ -163,7 +164,7 @@ TEST_F(TestBlockMetaTree, random_keybtree)
}
ASSERT_EQ(OB_SUCCESS, meta_tree.insert_macro_block(
macro_handle,
ObDatumRowkeyWrapper(rowkey, &data_desc_.datum_utils_),
rowkey,
&meta)) << "i: " << i << ", j: " << j << "\n";
}
meta_tree.destroy();