hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix not update column_cnt on schema when upgrade

Browse Source
This commit is contained in:
yangqise7en
2023-08-17 10:58:29 +00:00
committed by ob-robot
parent 24659c98b6
commit e7b6ad0d4b
5 changed files with 59 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/storage/compaction/ob_tablet_merge_ctx.cpp
View File

@ -1060,6 +1060,7 @@ int ObTabletMergeCtx::get_storage_schema_to_merge(const ObTablesHandleArray &mer
const ObMergeType &merge_type = param_.merge_type_;
const ObStorageSchema *schema_on_tablet = nullptr;
int64_t max_column_cnt_in_memtable = 0;
int64_t max_column_cnt_on_recorder = 0;
int64_t max_schema_version_in_memtable = 0;
int64_t column_cnt_in_schema = 0;
bool use_schema_on_tablet = true; // for minor & tx_mini, use storage schema on tablet
@ -1085,8 +1086,10 @@ int ObTabletMergeCtx::get_storage_schema_to_merge(const ObTablesHandleArray &mer
}
} // end of for
if (OB_FAIL(ret)) {
if (FAILEDx(tablet_handle_.get_obj()->get_max_column_cnt_on_schema_recorder(max_column_cnt_on_recorder))) {
LOG_WARN("failed to get max column cnt on schema recorder", KR(ret));
} else if (max_column_cnt_in_memtable <= column_cnt_in_schema
&& max_column_cnt_on_recorder <= column_cnt_in_schema
&& max_schema_version_in_memtable <= schema_on_tablet->get_schema_version()) {
// do nothing
} else {
@ -1103,6 +1106,7 @@ int ObTabletMergeCtx::get_storage_schema_to_merge(const ObTablesHandleArray &mer
storage_schema = nullptr;
} else {
// only update column cnt by memtable, use schema version on tablet_schema
storage_schema->column_cnt_ = MAX(storage_schema->column_cnt_, max_column_cnt_on_recorder);
storage_schema->column_cnt_ = MAX(storage_schema->column_cnt_, max_column_cnt_in_memtable);
storage_schema->store_column_cnt_ = MAX(column_cnt_in_schema, max_column_cnt_in_memtable);
storage_schema->schema_version_ = MAX(max_schema_version_in_memtable, schema_on_tablet->get_schema_version());
@ -1119,7 +1123,7 @@ int ObTabletMergeCtx::get_storage_schema_to_merge(const ObTablesHandleArray &mer
}
schema_ctx_.schema_version_ = schema_ctx_.storage_schema_->get_schema_version();
FLOG_INFO("get storage schema to merge", K_(param), K_(schema_ctx), K(use_schema_on_tablet),
K(max_column_cnt_in_memtable), K(max_schema_version_in_memtable));
K(max_column_cnt_in_memtable), K(max_schema_version_in_memtable), K(max_column_cnt_on_recorder));
if (!use_schema_on_tablet) {
// destroy loaded schema memory after print log
ObTablet::free_storage_schema(allocator_, schema_on_tablet);

14
src/storage/ob_storage_schema_recorder.cpp
View File

@ -47,7 +47,8 @@ ObStorageSchemaRecorder::ObStorageSchemaRecorder()
allocator_(nullptr),
ls_id_(),
tablet_id_(),
table_id_(0)
table_id_(0),
max_column_cnt_(0)
{
#if defined(__x86_64__)
STATIC_ASSERT(sizeof(ObStorageSchemaRecorder) <= 128, "size of schema recorder is oversize");
@ -76,6 +77,7 @@ void ObStorageSchemaRecorder::reset()
{
if (is_inited_) {
ObIStorageClogRecorder::reset();
max_column_cnt_ = 0;
}
}
@ -143,6 +145,7 @@ int ObStorageSchemaRecorder::inner_replay_clog(
ObArenaAllocator tmp_allocator;
ObStorageSchema replay_storage_schema;
ObTabletHandle tmp_tablet_handle;
int64_t stored_col_cnt = 0;
if (OB_FAIL(replay_get_tablet_handle(ls_id_, tablet_id_, scn, tmp_tablet_handle))) {
if (OB_OBSOLETE_CLOG_NEED_SKIP == ret) {
@ -152,10 +155,13 @@ int ObStorageSchemaRecorder::inner_replay_clog(
}
} else if (OB_FAIL(replay_storage_schema.deserialize(tmp_allocator, buf, size, pos))) {
LOG_WARN("fail to deserialize table schema", K(ret), K_(tablet_id));
} else if (OB_FAIL(replay_storage_schema.get_store_column_count(stored_col_cnt, true/*full_col*/))) {
LOG_WARN("failed to get store column count from replay schema", KR(ret),K(replay_storage_schema));
} else {
// just replay schema clog, do not save storage schema
// DDL is forbidden during upgrade
LOG_INFO("success to replay schema clog", K(ret), K(replay_storage_schema));
// replay schema clog and update to ObStorageSchemaRecorder
// need get column_cnt on schema_recorder to mini merge
max_column_cnt_ = MAX(max_column_cnt_, stored_col_cnt);
FLOG_INFO("success to replay schema clog", K(ret), K(replay_storage_schema), K(stored_col_cnt), K(max_column_cnt_));
}
replay_storage_schema.reset();
tmp_tablet_handle.reset();

3
src/storage/ob_storage_schema_recorder.h
View File

@ -53,6 +53,7 @@ public:
void destroy();
void reset();
bool is_inited() const { return is_inited_; }
int64_t get_max_column_cnt() const { return max_column_cnt_; }
// follower
int replay_schema_log(const share::SCN &scn, const char *buf, const int64_t size, int64_t &pos);
@ -113,7 +114,7 @@ private:
share::ObLSID ls_id_;
ObTabletID tablet_id_;
int64_t table_id_;
int64_t max_column_cnt_;
};
} // storage

57
src/storage/tablet/ob_tablet.cpp
View File

@ -2052,16 +2052,14 @@ int ObTablet::get_max_sync_medium_scn(int64_t &max_medium_snapshot) const
{
int ret = OB_SUCCESS;
max_medium_snapshot = 0;
ObIMemtableMgr *memtable_mgr = nullptr;
ObTabletMemtableMgr *data_memtable_mgr = nullptr;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret));
} else if (tablet_meta_.tablet_id_.is_special_merge_tablet()) {
// do nothing
} else if (OB_FAIL(get_memtable_mgr(memtable_mgr))) {
} else if (OB_FAIL(get_tablet_memtable_mgr(data_memtable_mgr))) {
LOG_WARN("failed to get memtable mgr", K(ret));
} else if (FALSE_IT(data_memtable_mgr = static_cast<ObTabletMemtableMgr *>(memtable_mgr))) {
} else {
max_medium_snapshot = data_memtable_mgr->get_medium_info_recorder().get_max_saved_version();
}
@ -2072,13 +2070,11 @@ int ObTablet::get_max_sync_storage_schema_version(int64_t &max_schema_version) c
{
int ret = OB_SUCCESS;
max_schema_version = 0;
ObIMemtableMgr *memtable_mgr = nullptr;
ObTabletMemtableMgr *data_memtable_mgr = nullptr;
if (is_ls_inner_tablet()) {
// do nothing
} else if (OB_FAIL(get_memtable_mgr(memtable_mgr))) {
} else if (OB_FAIL(get_tablet_memtable_mgr(data_memtable_mgr))) {
LOG_WARN("failed to get memtable mgr", K(ret));
} else if (FALSE_IT(data_memtable_mgr = static_cast<ObTabletMemtableMgr *>(memtable_mgr))) {
} else {
max_schema_version = data_memtable_mgr->get_storage_schema_recorder().get_max_saved_version();
}
@ -2092,7 +2088,6 @@ int ObTablet::try_update_storage_schema(
const int64_t timeout_ts)
{
int ret = OB_SUCCESS;
ObIMemtableMgr *memtable_mgr = nullptr;
ObTabletMemtableMgr *data_memtable_mgr = nullptr;
if (IS_NOT_INIT) {
@ -2100,9 +2095,8 @@ int ObTablet::try_update_storage_schema(
LOG_WARN("not inited", K(ret));
} else if (tablet_meta_.tablet_id_.is_special_merge_tablet()) {
// do nothing
} else if (OB_FAIL(get_memtable_mgr(memtable_mgr))) {
} else if (OB_FAIL(get_tablet_memtable_mgr(data_memtable_mgr))) {
LOG_WARN("failed to get memtable mgr", K(ret));
} else if (FALSE_IT(data_memtable_mgr = static_cast<ObTabletMemtableMgr *>(memtable_mgr))) {
} else if (OB_FAIL(data_memtable_mgr->get_storage_schema_recorder().try_update_storage_schema(
table_id, schema_version, allocator, timeout_ts))) {
LOG_WARN("fail to record storage schema", K(ret), K(table_id), K(schema_version), K(timeout_ts));
@ -2110,6 +2104,24 @@ int ObTablet::try_update_storage_schema(
return ret;
}
int ObTablet::get_max_column_cnt_on_schema_recorder(int64_t &max_column_cnt)
{
int ret = OB_SUCCESS;
ObTabletMemtableMgr *data_memtable_mgr = nullptr;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret));
} else if (tablet_meta_.tablet_id_.is_special_merge_tablet()) {
// do nothing
} else if (OB_FAIL(get_tablet_memtable_mgr(data_memtable_mgr))) {
LOG_WARN("failed to get memtable mgr", K(ret));
} else {
max_column_cnt = data_memtable_mgr->get_storage_schema_recorder().get_max_column_cnt();
}
return ret;
}
// be careful to use this max_schem_version on storage_schema
int ObTablet::get_max_schema_version(int64_t &schema_version)
{
@ -3203,7 +3215,6 @@ int ObTablet::replay_update_storage_schema(
{
int ret = OB_SUCCESS;
int64_t new_pos = pos;
ObIMemtableMgr *memtable_mgr = nullptr;
ObTabletMemtableMgr *data_memtable_mgr = nullptr;
if (IS_NOT_INIT) {
@ -3211,9 +3222,8 @@ int ObTablet::replay_update_storage_schema(
LOG_WARN("not inited", K(ret));
} else if (tablet_meta_.tablet_id_.is_special_merge_tablet()) {
// do nothing
} else if (OB_FAIL(get_memtable_mgr(memtable_mgr))) {
} else if (OB_FAIL(get_tablet_memtable_mgr(data_memtable_mgr))) {
LOG_WARN("failed to get memtable mgr", K(ret));
} else if (FALSE_IT(data_memtable_mgr = static_cast<ObTabletMemtableMgr *>(memtable_mgr))) {
} else if (OB_FAIL(data_memtable_mgr->get_storage_schema_recorder().replay_schema_log(scn, buf, buf_size, new_pos))) {
LOG_WARN("storage schema recorder replay fail", K(ret), K(scn));
} else {
@ -3230,7 +3240,6 @@ int ObTablet::submit_medium_compaction_clog(
ObIAllocator &allocator)
{
int ret = OB_SUCCESS;
ObIMemtableMgr *memtable_mgr = nullptr;
ObTabletMemtableMgr *data_memtable_mgr = nullptr;
if (IS_NOT_INIT) {
@ -3238,9 +3247,8 @@ int ObTablet::submit_medium_compaction_clog(
LOG_WARN("not inited", K(ret));
} else if (tablet_meta_.tablet_id_.is_special_merge_tablet()) {
// do nothing
} else if (OB_FAIL(get_memtable_mgr(memtable_mgr))) {
} else if (OB_FAIL(get_tablet_memtable_mgr(data_memtable_mgr))) {
LOG_WARN("failed to get memtable mgr", K(ret));
} else if (FALSE_IT(data_memtable_mgr = static_cast<ObTabletMemtableMgr *>(memtable_mgr))) {
} else if (OB_FAIL(data_memtable_mgr->get_medium_info_recorder().submit_medium_compaction_info(
medium_info, allocator))) {
LOG_WARN("medium compaction recorder submit fail", K(ret), K(medium_info));
@ -3258,7 +3266,6 @@ int ObTablet::replay_medium_compaction_clog(
{
int ret = OB_SUCCESS;
int64_t new_pos = pos;
ObIMemtableMgr *memtable_mgr = nullptr;
ObTabletMemtableMgr *data_memtable_mgr = nullptr;
if (IS_NOT_INIT) {
@ -3268,9 +3275,8 @@ int ObTablet::replay_medium_compaction_clog(
LOG_WARN("invalid argument", K(ret), K(buf_size), K(pos));
} else if (tablet_meta_.tablet_id_.is_ls_inner_tablet()) {
// do nothing
} else if (OB_FAIL(get_memtable_mgr(memtable_mgr))) {
} else if (OB_FAIL(get_tablet_memtable_mgr(data_memtable_mgr))) {
LOG_WARN("failed to get memtable mgr", K(ret));
} else if (FALSE_IT(data_memtable_mgr = static_cast<ObTabletMemtableMgr *>(memtable_mgr))) {
} else if (OB_FAIL(data_memtable_mgr->get_medium_info_recorder().replay_medium_compaction_log(scn, buf, buf_size, new_pos))) {
LOG_WARN("medium compaction recorder replay fail", K(ret), KPC(this), K(buf_size), K(new_pos));
} else {
@ -3549,6 +3555,21 @@ int ObTablet::get_memtable_mgr(ObIMemtableMgr *&memtable_mgr) const
return ret;
}
int ObTablet::get_tablet_memtable_mgr(ObTabletMemtableMgr *&tablet_memtable_mgr) const
{
int ret = OB_SUCCESS;
tablet_memtable_mgr = nullptr;
ObIMemtableMgr *memtable_mgr = nullptr;
if (tablet_meta_.tablet_id_.is_ls_inner_tablet()) {
// do nothing
} else if (OB_FAIL(get_memtable_mgr(memtable_mgr))) {
LOG_WARN("failed to get memtable mgr", K(ret));
} else {
tablet_memtable_mgr = static_cast<ObTabletMemtableMgr *>(memtable_mgr);
}
return ret;
}
int ObTablet::get_ddl_kv_mgr(ObDDLKvMgrHandle &ddl_kv_mgr_handle, bool try_create)
{
int ret = OB_SUCCESS;

3
src/storage/tablet/ob_tablet.h
View File

@ -498,6 +498,7 @@ public:
common::ObArenaAllocator &allocator,
ObTabletFullMemoryMdsData &mds_data);
int64_t to_string(char *buf, const int64_t buf_len) const;
int get_max_column_cnt_on_schema_recorder(int64_t &max_column_cnt);
protected:// for MDS use
virtual bool check_is_inited_() const override final { return is_inited_; }
virtual const ObTabletMdsData &get_mds_data_() const override final { return mds_data_; }
@ -519,13 +520,13 @@ private:
static int inc_linked_block_ref_cnt(const ObMetaDiskAddr &head_addr, bool &inc_success);
static void dec_linked_block_ref_cnt(const ObMetaDiskAddr &head_addr);
int64_t get_try_cache_size() const;
private:
int inner_check_valid(const bool ignore_ha_status = false) const;
int get_min_medium_snapshot(int64_t &min_medium_snapshot) const;
int64_t get_self_size() const;
int get_memtable_mgr(ObIMemtableMgr *&memtable_mgr) const;
int get_tablet_memtable_mgr(ObTabletMemtableMgr *&memtable_mgr) const;
int check_schema_version(const int64_t schema_version);
int check_snapshot_readable(const int64_t snapshot_version);