[FEAT MERGE] merge transfer
Co-authored-by: wxhwang <wxhwang@126.com> Co-authored-by: godyangfight <godyangfight@gmail.com> Co-authored-by: Tyshawn <tuyunshan@gmail.com>
This commit is contained in:
xuhuleon
664
src/storage/compaction/ob_medium_compaction_func.cpp
Normal file → Executable file
664
src/storage/compaction/ob_medium_compaction_func.cpp
Normal file → Executable file
@ -25,6 +25,7 @@
|
||||
#include "lib/utility/ob_tracepoint.h"
|
||||
#include "storage/ob_partition_range_spliter.h"
|
||||
#include "storage/compaction/ob_compaction_diagnose.h"
|
||||
#include "storage/tablet/ob_tablet_medium_info_reader.h"
|
||||
|
||||
namespace oceanbase
|
||||
{
|
||||
@ -103,15 +104,28 @@ int ObMediumCompactionScheduleFunc::choose_major_snapshot(
|
||||
ObTenantFreezeInfoMgr::FreezeInfo freeze_info;
|
||||
int64_t schedule_snapshot = 0;
|
||||
const int64_t scheduler_frozen_version = MTL(ObTenantTabletScheduler*)->get_frozen_version();
|
||||
const ObMediumCompactionInfoList &medium_list = tablet.get_medium_compaction_info_list();
|
||||
ObITable *last_major = tablet.get_table_store().get_major_sstables().get_boundary_table(true/*last*/);
|
||||
ObTabletMemberWrapper<ObTabletTableStore> table_store_wrapper;
|
||||
ObSSTable *last_major = nullptr;
|
||||
int64_t last_sstable_schema_version = 0;
|
||||
bool schedule_with_newer_info = false;
|
||||
|
||||
if (OB_ISNULL(last_major)) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("major sstable is unexpected null", K(ret), KPC(last_major));
|
||||
ObMultiVersionSchemaService *schema_service = nullptr;
|
||||
if (OB_FAIL(tablet.fetch_table_store(table_store_wrapper))) {
|
||||
LOG_WARN("load medium info list fail", K(ret), K(tablet));
|
||||
} else {
|
||||
schedule_snapshot = last_major->get_snapshot_version();
|
||||
last_major = static_cast<ObSSTable *>(table_store_wrapper.get_member()->get_major_sstables().get_boundary_table(true/*last*/));
|
||||
if (OB_ISNULL(last_major)) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("major sstable is unexpected null", K(ret), KPC(last_major));
|
||||
} else if (OB_FAIL(last_major->get_frozen_schema_version(last_sstable_schema_version))) {
|
||||
LOG_WARN("failed to get frozen schema version", KR(ret), KPC(last_major));
|
||||
} else {
|
||||
schedule_snapshot = last_major->get_snapshot_version();
|
||||
}
|
||||
}
|
||||
|
||||
if (OB_SUCC(ret) && OB_ISNULL(schema_service = MTL(ObTenantSchemaService *)->get_schema_service())) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("failed to get schema service from MTL", K(ret));
|
||||
}
|
||||
while (OB_SUCC(ret)) {
|
||||
if (OB_FAIL(MTL_CALL_FREEZE_INFO_MGR(get_freeze_info_behind_snapshot_version, schedule_snapshot, freeze_info))) {
|
||||
@ -123,8 +137,12 @@ int ObMediumCompactionScheduleFunc::choose_major_snapshot(
|
||||
} else if (OB_UNLIKELY(freeze_info.schema_version <= 0)) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("schema version is invalid", K(ret), K(ls_id), K(tablet_id), K(freeze_info));
|
||||
} else if (OB_FAIL(ObMediumCompactionScheduleFunc::get_table_schema_to_merge(
|
||||
tablet, freeze_info.schema_version, allocator, medium_info))) {
|
||||
} else if (OB_UNLIKELY(freeze_info.schema_version < last_sstable_schema_version)) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_ERROR("schema version in freeze info is less than last sstable", K(ret), K(ls_id), K(tablet_id),
|
||||
K(freeze_info), K(last_sstable_schema_version));
|
||||
} else if (OB_FAIL(get_table_schema_to_merge(
|
||||
*schema_service, tablet, freeze_info.schema_version, allocator, medium_info))) {
|
||||
if (OB_TABLE_IS_DELETED == ret) {
|
||||
// do nothing, end loop
|
||||
} else if (OB_ERR_SCHEMA_HISTORY_EMPTY == ret) {
|
||||
@ -138,9 +156,10 @@ int ObMediumCompactionScheduleFunc::choose_major_snapshot(
|
||||
} else {
|
||||
LOG_WARN("failed to get table schema", K(ret), K(ls_id), K(tablet_id), K(medium_info));
|
||||
}
|
||||
} else { // success to get table schema
|
||||
}
|
||||
if (OB_SUCC(ret)) { // success to get table schema
|
||||
if (schedule_with_newer_info) {
|
||||
LOG_INFO("schedule with newer freeze info", K(ret), K(ls_id), K(tablet_id), K(freeze_info));
|
||||
FLOG_INFO("schedule with newer freeze info", K(ret), K(ls_id), K(tablet_id), K(freeze_info));
|
||||
}
|
||||
break;
|
||||
}
|
||||
@ -154,7 +173,8 @@ int ObMediumCompactionScheduleFunc::choose_major_snapshot(
|
||||
LOG_WARN("failed get result for major", K(ret), K(medium_info));
|
||||
} else {
|
||||
LOG_TRACE("choose_major_snapshot", K(ret), "ls_id", ls.get_ls_id(),
|
||||
"tablet_id", tablet.get_tablet_meta().tablet_id_, K(medium_info), K(freeze_info), K(result));
|
||||
"tablet_id", tablet.get_tablet_meta().tablet_id_, K(medium_info), K(freeze_info), K(result),
|
||||
K(last_sstable_schema_version));
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
@ -184,7 +204,7 @@ int ObMediumCompactionScheduleFunc::get_status_from_inner_table(
|
||||
return ret;
|
||||
}
|
||||
|
||||
// cal this func with PLAF LEADER ROLE && wait_check_medium_scn_ = 0
|
||||
// cal this func with PLAF LEADER ROLE && last_medium_scn_ = 0
|
||||
int ObMediumCompactionScheduleFunc::schedule_next_medium_for_leader(
|
||||
const int64_t major_snapshot,
|
||||
ObTenantTabletScheduler::ObScheduleStatistics &schedule_stat)
|
||||
@ -223,36 +243,45 @@ int ObMediumCompactionScheduleFunc::schedule_next_medium_primary_cluster(
|
||||
// check last medium type, select inner table for last major
|
||||
bool schedule_medium_flag = false;
|
||||
int64_t max_sync_medium_scn = 0;
|
||||
const bool is_major = 0 != schedule_major_snapshot;
|
||||
ObITable *last_major = nullptr;
|
||||
ObTablet *tablet = nullptr;
|
||||
const bool is_major = 0 != schedule_major_snapshot;
|
||||
ObTabletMemberWrapper<ObTabletTableStore> table_store_wrapper;
|
||||
|
||||
if (OB_UNLIKELY(!tablet_handle_.is_valid())) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("invalid tablet_handle", K(ret), K(tablet_handle_));
|
||||
} else if (FALSE_IT(tablet = tablet_handle_.get_obj())) {
|
||||
} else {
|
||||
const ObMediumCompactionInfoList &medium_list = tablet->get_medium_compaction_info_list();
|
||||
if (OB_ISNULL(last_major = tablet->get_table_store().get_major_sstables().get_boundary_table(true/*last*/))) {
|
||||
// no major, do nothing
|
||||
} else if (!medium_list.could_schedule_next_round()) { // check serialized list
|
||||
// do nothing
|
||||
} else if (OB_FAIL(tablet->get_max_sync_medium_scn(max_sync_medium_scn))) { // check info in memory
|
||||
LOG_WARN("failed to get max sync medium scn", K(ret), K(max_sync_medium_scn));
|
||||
} else if (is_major && schedule_major_snapshot > max_sync_medium_scn) {
|
||||
schedule_medium_flag = true;
|
||||
} else if (nullptr != last_major && last_major->get_snapshot_version() < max_sync_medium_scn) {
|
||||
// do nothing
|
||||
} else if (OB_FAIL(ObAdaptiveMergePolicy::get_adaptive_merge_reason(*tablet, adaptive_merge_reason))) {
|
||||
if (OB_HASH_NOT_EXIST != ret) {
|
||||
LOG_WARN("failed to get meta merge priority", K(ret), KPC(this));
|
||||
} else {
|
||||
ret = OB_SUCCESS;
|
||||
}
|
||||
} else if (adaptive_merge_reason > ObAdaptiveMergePolicy::AdaptiveMergeReason::NONE) {
|
||||
schedule_medium_flag = true;
|
||||
} else if (OB_FAIL(tablet->fetch_table_store(table_store_wrapper))) {
|
||||
LOG_WARN("fail to fetch table store", K(ret));
|
||||
} else if (OB_ISNULL(last_major =
|
||||
table_store_wrapper.get_member()->get_major_sstables().get_boundary_table(true/*last*/))) {
|
||||
// no major, do nothing
|
||||
} else if (nullptr == medium_info_list_
|
||||
&& OB_FAIL(tablet->read_medium_info_list(allocator_, medium_info_list_))) {
|
||||
LOG_WARN("failed to read medium info list", K(ret), KPC(tablet));
|
||||
} else if (OB_ISNULL(medium_info_list_)) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("medium info list is unexpected null", K(ret), KPC(this), KPC_(medium_info_list));
|
||||
} else if (!medium_info_list_->could_schedule_next_round()) { // check serialized list
|
||||
// do nothing
|
||||
} else if (OB_FAIL(tablet->get_max_sync_medium_scn(max_sync_medium_scn))) { // check info in memory
|
||||
LOG_WARN("failed to get max sync medium scn", K(ret), K(max_sync_medium_scn));
|
||||
} else if (is_major && schedule_major_snapshot > max_sync_medium_scn) {
|
||||
schedule_medium_flag = true;
|
||||
} else if (nullptr != last_major && last_major->get_snapshot_version() < max_sync_medium_scn) {
|
||||
// do nothing
|
||||
} else if (OB_FAIL(ObAdaptiveMergePolicy::get_adaptive_merge_reason(*tablet, adaptive_merge_reason))) {
|
||||
if (OB_HASH_NOT_EXIST != ret) {
|
||||
LOG_WARN("failed to get meta merge priority", K(ret), KPC(this));
|
||||
} else {
|
||||
ret = OB_SUCCESS;
|
||||
}
|
||||
LOG_TRACE("schedule next medium in primary cluster", K(ret), KPC(this), K(schedule_medium_flag),
|
||||
K(schedule_major_snapshot), K(adaptive_merge_reason), KPC(last_major), K(medium_list), K(max_sync_medium_scn));
|
||||
} else if (adaptive_merge_reason > ObAdaptiveMergePolicy::AdaptiveMergeReason::NONE) {
|
||||
schedule_medium_flag = true;
|
||||
}
|
||||
LOG_TRACE("schedule next medium in primary cluster", K(ret), KPC(this), K(schedule_medium_flag),
|
||||
K(schedule_major_snapshot), K(adaptive_merge_reason), KPC(last_major), KPC_(medium_info_list), K(max_sync_medium_scn));
|
||||
#ifdef ERRSIM
|
||||
if (OB_SUCC(ret)) {
|
||||
if (tablet->get_tablet_meta().tablet_id_.id() > ObTabletID::MIN_USER_TABLET_ID) {
|
||||
@ -267,22 +296,21 @@ int ObMediumCompactionScheduleFunc::schedule_next_medium_primary_cluster(
|
||||
}
|
||||
#endif
|
||||
|
||||
if (OB_FAIL(ret) || !schedule_medium_flag) {
|
||||
} else if (ObMediumCompactionInfo::MAJOR_COMPACTION == medium_list.get_last_compaction_type()) {
|
||||
// for normal medium, checksum error happened, wait_check_medium_scn_ will never = 0
|
||||
// for major, need select inner_table to check RS status
|
||||
if (OB_FAIL(get_status_from_inner_table(ls_.get_ls_id(), tablet->get_tablet_meta().tablet_id_, ret_info))) {
|
||||
LOG_WARN("failed to get status from inner tablet", K(ret), KPC(this));
|
||||
} else if (ret_info.could_schedule_next_round(medium_list.get_last_compaction_scn())) {
|
||||
LOG_INFO("success to check RS major checksum validation finished", K(ret), KPC(this), K(ret_info));
|
||||
ret = decide_medium_snapshot(is_major);
|
||||
} else {
|
||||
++schedule_stat.wait_rs_validate_cnt_;
|
||||
LOG_DEBUG("cannot schedule next round merge now", K(ret), K(ret_info), K(medium_list), KPC(tablet));
|
||||
}
|
||||
if (OB_FAIL(ret) || !schedule_medium_flag) {
|
||||
} else if (ObMediumCompactionInfo::MAJOR_COMPACTION == medium_info_list_->get_last_compaction_type()) {
|
||||
// for normal medium, checksum error happened, wait_check_medium_scn_ will never = 0
|
||||
// for major, need select inner_table to check RS status
|
||||
if (OB_FAIL(get_status_from_inner_table(ls_.get_ls_id(), tablet->get_tablet_meta().tablet_id_, ret_info))) {
|
||||
LOG_WARN("failed to get status from inner tablet", K(ret), KPC(this));
|
||||
} else if (ret_info.could_schedule_next_round(medium_info_list_->get_last_compaction_scn())) {
|
||||
LOG_INFO("success to check RS major checksum validation finished", K(ret), KPC(this), K(ret_info));
|
||||
ret = decide_medium_snapshot(is_major);
|
||||
} else {
|
||||
ret = decide_medium_snapshot(is_major, adaptive_merge_reason);
|
||||
++schedule_stat.wait_rs_validate_cnt_;
|
||||
LOG_DEBUG("cannot schedule next round merge now", K(ret), K(ret_info), KPC_(medium_info_list), KPC(tablet));
|
||||
}
|
||||
} else {
|
||||
ret = decide_medium_snapshot(is_major, adaptive_merge_reason);
|
||||
}
|
||||
|
||||
return ret;
|
||||
@ -296,24 +324,68 @@ int ObMediumCompactionScheduleFunc::get_max_reserved_snapshot(int64_t &max_reser
|
||||
int64_t max_merged_snapshot = 0;
|
||||
int64_t min_reserved_snapshot = INT64_MAX;
|
||||
ObTablet *tablet = nullptr;
|
||||
ObTabletMemberWrapper<ObTabletTableStore> wrapper;
|
||||
if (OB_UNLIKELY(!tablet_handle_.is_valid())) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("invalid tablet_handle", K(ret), K(tablet_handle_));
|
||||
} else if (FALSE_IT(tablet = tablet_handle_.get_obj())) {
|
||||
} else if (OB_FAIL(tablet->fetch_table_store(wrapper))) {
|
||||
LOG_WARN("fetch table store fail", K(ret), KPC(tablet));
|
||||
} else if (0 == wrapper.get_member()->get_major_sstables().count()) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("major sstable should not be empty", K(ret), KPC(tablet));
|
||||
} else if (0 == ls_.get_min_reserved_snapshot()) {
|
||||
ret = OB_NO_NEED_MERGE;
|
||||
// not sync reserved snapshot yet, should not schedule now
|
||||
} else if (FALSE_IT(max_merged_snapshot = wrapper.get_member()->get_major_sstables().get_boundary_table(true/*last*/)->get_snapshot_version())) {
|
||||
} else if (OB_FAIL(MTL(ObTenantFreezeInfoMgr*)->get_min_reserved_snapshot(
|
||||
tablet->get_tablet_meta().tablet_id_, max_merged_snapshot, min_reserved_snapshot))) {
|
||||
LOG_WARN("failed to get multi version from freeze info mgr", K(ret), K(table_id));
|
||||
} else {
|
||||
const ObTabletTableStore &table_store = tablet->get_table_store();
|
||||
if (0 == table_store.get_major_sstables().count()) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("major sstable should not be empty", K(ret), K(tablet));
|
||||
} else if (0 == ls_.get_min_reserved_snapshot()) {
|
||||
ret = OB_NO_NEED_MERGE;
|
||||
// not sync reserved snapshot yet, should not schedule now
|
||||
} else if (FALSE_IT(max_merged_snapshot = table_store.get_major_sstables().get_boundary_table(true/*last*/)->get_snapshot_version())) {
|
||||
} else if (OB_FAIL(MTL(ObTenantFreezeInfoMgr*)->get_min_reserved_snapshot(
|
||||
tablet->get_tablet_meta().tablet_id_, max_merged_snapshot, min_reserved_snapshot))) {
|
||||
LOG_WARN("failed to get multi version from freeze info mgr", K(ret), K(table_id));
|
||||
max_reserved_snapshot = MAX(ls_.get_min_reserved_snapshot(), min_reserved_snapshot);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
int ObMediumCompactionScheduleFunc::choose_new_medium_snapshot(
|
||||
const int64_t max_reserved_snapshot,
|
||||
ObMediumCompactionInfo &medium_info,
|
||||
ObGetMergeTablesResult &result)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
ObTablet *tablet = tablet_handle_.get_obj();
|
||||
int64_t snapshot_gc_ts = 0;
|
||||
if (medium_info.medium_snapshot_ == tablet->get_snapshot_version() // no uncommitted sstable
|
||||
&& weak_read_ts_ + DEFAULT_SCHEDULE_MEDIUM_INTERVAL < ObTimeUtility::current_time_ns()) {
|
||||
snapshot_gc_ts = MTL(ObTenantFreezeInfoMgr *)->get_snapshot_gc_ts();
|
||||
// data before weak_read_ts & latest storage schema on memtable is match for schedule medium
|
||||
medium_info.medium_snapshot_ = MIN(weak_read_ts_, snapshot_gc_ts);
|
||||
}
|
||||
if (medium_info.medium_snapshot_ < max_reserved_snapshot) {
|
||||
ret = OB_NO_NEED_MERGE;
|
||||
} else {
|
||||
LOG_INFO("use weak_read_ts to schedule medium", K(ret), KPC(this),
|
||||
K(medium_info), K(max_reserved_snapshot), K_(weak_read_ts), K(snapshot_gc_ts));
|
||||
}
|
||||
if (OB_SUCC(ret)) { // update schema version for cur medium scn
|
||||
ObSEArray<storage::ObITable *, MAX_MEMSTORE_CNT> memtables;
|
||||
if (OB_FAIL(tablet->get_memtables(memtables, true/*need_active*/))) {
|
||||
LOG_WARN("failed to get memtables", KR(ret), KPC(tablet));
|
||||
} else {
|
||||
max_reserved_snapshot = MAX(ls_.get_min_reserved_snapshot(), min_reserved_snapshot);
|
||||
int64_t max_schema_version_on_memtable = 0;
|
||||
int64_t unused_max_column_cnt_on_memtable = 0;
|
||||
for (int64_t idx = 0; OB_SUCC(ret) && idx < memtables.count(); ++idx) {
|
||||
memtable::ObMemtable *memtable = static_cast<memtable::ObMemtable *>(memtables.at(idx));
|
||||
if (OB_FAIL(memtable->get_schema_info(
|
||||
max_schema_version_on_memtable, unused_max_column_cnt_on_memtable))) {
|
||||
LOG_WARN("failed to get schema info from memtable", KR(ret), KPC(memtable));
|
||||
}
|
||||
}
|
||||
if (OB_SUCC(ret)) {
|
||||
result.schema_version_ = MAX(max_schema_version_on_memtable, result.schema_version_);
|
||||
LOG_INFO("chosen new medium snapshot", K(ret), KPC(this),
|
||||
K(medium_info), K(max_reserved_snapshot), K(result), K(max_schema_version_on_memtable));
|
||||
}
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
@ -349,6 +421,11 @@ int ObMediumCompactionScheduleFunc::decide_medium_snapshot(
|
||||
ObGetMergeTablesResult result;
|
||||
ObMediumCompactionInfo medium_info;
|
||||
medium_info.data_version_ = compat_version;
|
||||
if (medium_info.data_version_ < DATA_VERSION_4_2_0_0) {
|
||||
medium_info.medium_compat_version_ = ObMediumCompactionInfo::MEIDUM_COMPAT_VERSION;
|
||||
} else {
|
||||
medium_info.medium_compat_version_ = ObMediumCompactionInfo::MEIDUM_COMPAT_VERSION_V2;
|
||||
}
|
||||
|
||||
if (OB_FAIL(choose_medium_scn[is_major](ls_, *tablet, merge_reason, allocator_, medium_info, result))) {
|
||||
if (OB_NO_NEED_MERGE != ret) {
|
||||
@ -362,26 +439,20 @@ int ObMediumCompactionScheduleFunc::decide_medium_snapshot(
|
||||
LOG_WARN("failed to get multi_version_start", K(ret), KPC(this));
|
||||
} else if (medium_info.medium_snapshot_ < max_reserved_snapshot) {
|
||||
// chosen medium snapshot is far too old
|
||||
LOG_INFO("chosen medium snapshot is invalid for multi_version_start", K(ret), KPC(this),
|
||||
K(medium_info), K(max_reserved_snapshot));
|
||||
const share::SCN &weak_read_ts = ls_.get_ls_wrs_handler()->get_ls_weak_read_ts();
|
||||
if (medium_info.medium_snapshot_ == tablet->get_snapshot_version() // no uncommitted sstable
|
||||
&& weak_read_ts.get_val_for_tx() <= max_reserved_snapshot
|
||||
&& weak_read_ts.get_val_for_tx() + DEFAULT_SCHEDULE_MEDIUM_INTERVAL < ObTimeUtility::current_time_ns()) {
|
||||
const int64_t snapshot_gc_ts = MTL(ObTenantFreezeInfoMgr*)->get_snapshot_gc_ts();
|
||||
medium_info.medium_snapshot_ = MAX(max_reserved_snapshot, MIN(weak_read_ts.get_val_for_tx(), snapshot_gc_ts));
|
||||
LOG_INFO("use weak_read_ts to schedule medium", K(ret), KPC(this),
|
||||
K(medium_info), K(max_reserved_snapshot), K(weak_read_ts), K(snapshot_gc_ts));
|
||||
} else {
|
||||
ret = OB_NO_NEED_MERGE;
|
||||
if (OB_FAIL(choose_new_medium_snapshot(max_reserved_snapshot, medium_info, result))) {
|
||||
LOG_WARN("failed to choose new medium snapshot", KR(ret), K(medium_info));
|
||||
}
|
||||
}
|
||||
|
||||
if (OB_SUCC(ret) && !is_major) {
|
||||
const int64_t current_time = ObTimeUtility::current_time_ns();
|
||||
if (max_reserved_snapshot < current_time) {
|
||||
const int64_t time_interval = (current_time - max_reserved_snapshot) / 2;
|
||||
ObSSTable *table = static_cast<ObSSTable *>(tablet->get_table_store().get_major_sstables().get_boundary_table(true/*last*/));
|
||||
if (OB_ISNULL(table)) {
|
||||
ObSSTable *table = nullptr;
|
||||
ObTabletMemberWrapper<ObTabletTableStore> table_store_wrapper;
|
||||
if (OB_FAIL(tablet->fetch_table_store(table_store_wrapper))) {
|
||||
LOG_WARN("fail to fetch table store", K(ret));
|
||||
} else if (OB_ISNULL(table = static_cast<ObSSTable *>(table_store_wrapper.get_member()->get_major_sstables().get_boundary_table(true /*last*/)))) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("table is unexpected null", K(ret), KP(table));
|
||||
} else if (table->get_snapshot_version() + time_interval > medium_info.medium_snapshot_) {
|
||||
@ -396,10 +467,10 @@ int ObMediumCompactionScheduleFunc::decide_medium_snapshot(
|
||||
ret = OB_E(EventTable::EN_SCHEDULE_MEDIUM_COMPACTION) ret;
|
||||
LOG_INFO("errsim", K(ret), KPC(this));
|
||||
if (OB_FAIL(ret)) {
|
||||
const share::SCN &weak_read_ts = ls_.get_ls_wrs_handler()->get_ls_weak_read_ts();
|
||||
const int64_t snapshot_gc_ts = MTL(ObTenantFreezeInfoMgr*)->get_snapshot_gc_ts();
|
||||
medium_info.medium_snapshot_ = MAX(max_reserved_snapshot, MIN(weak_read_ts.get_val_for_tx(), snapshot_gc_ts));
|
||||
if (medium_info.medium_snapshot_ > max_sync_medium_scn) {
|
||||
medium_info.medium_snapshot_ = MIN(weak_read_ts_, snapshot_gc_ts);
|
||||
if (medium_info.medium_snapshot_ > max_sync_medium_scn
|
||||
&& medium_info.medium_snapshot_ >= max_reserved_snapshot) {
|
||||
FLOG_INFO("set schedule medium with errsim", KPC(this));
|
||||
ret = OB_SUCCESS;
|
||||
}
|
||||
@ -407,33 +478,35 @@ int ObMediumCompactionScheduleFunc::decide_medium_snapshot(
|
||||
}
|
||||
}
|
||||
#endif
|
||||
if (FAILEDx(prepare_medium_info(result, medium_info))) {
|
||||
if (OB_TABLE_IS_DELETED == ret) {
|
||||
ret = OB_SUCCESS;
|
||||
} else {
|
||||
LOG_WARN("failed to prepare medium info", K(ret), K(result), K(tablet->get_storage_schema()));
|
||||
}
|
||||
} else if (OB_FAIL(submit_medium_clog(medium_info))) {
|
||||
LOG_WARN("failed to submit medium clog and update inner table", K(ret), KPC(this));
|
||||
} else if (OB_TMP_FAIL(ls_.tablet_freeze(tablet_id, true/*is_sync*/))) {
|
||||
// need to freeze memtable with MediumCompactionInfo
|
||||
LOG_WARN("failed to freeze tablet", K(tmp_ret), KPC(this));
|
||||
if (FAILEDx(prepare_medium_info(result, medium_info))) {
|
||||
if (OB_TABLE_IS_DELETED == ret) {
|
||||
ret = OB_SUCCESS;
|
||||
} else {
|
||||
LOG_WARN("failed to prepare medium info", K(ret), K(result));
|
||||
}
|
||||
// delete tablet_id in ObLSReservedSnapshotMgr even if submit clog or update inner table failed
|
||||
if (OB_TMP_FAIL(ls_.del_dependent_medium_tablet(tablet_id))) {
|
||||
LOG_ERROR("failed to delete dependent medium tablet", K(tmp_ret), KPC(this));
|
||||
ob_abort();
|
||||
}
|
||||
ret = OB_NO_NEED_MERGE == ret ? OB_SUCCESS : ret;
|
||||
if (OB_FAIL(ret)) {
|
||||
// add schedule suspect info
|
||||
ADD_SUSPECT_INFO(MEDIUM_MERGE, ls_.get_ls_id(), tablet_id,
|
||||
"schedule medium failed",
|
||||
"compaction_scn", medium_info.medium_snapshot_,
|
||||
"schema_version", medium_info.storage_schema_.schema_version_,
|
||||
"error_no", ret);
|
||||
} else if (OB_FAIL(submit_medium_clog(medium_info))) {
|
||||
LOG_WARN("failed to submit medium clog and update inner table", K(ret), KPC(this));
|
||||
} else if (OB_TMP_FAIL(ls_.tablet_freeze(tablet_id, true/*is_sync*/))) {
|
||||
// need to freeze memtable with MediumCompactionInfo
|
||||
LOG_WARN("failed to freeze tablet", K(tmp_ret), KPC(this));
|
||||
}
|
||||
// delete tablet_id in ObLSReservedSnapshotMgr even if submit clog or update inner table failed
|
||||
if (OB_TMP_FAIL(ls_.del_dependent_medium_tablet(tablet_id))) {
|
||||
LOG_ERROR("failed to delete dependent medium tablet", K(tmp_ret), KPC(this));
|
||||
ob_abort();
|
||||
}
|
||||
ret = OB_NO_NEED_MERGE == ret ? OB_SUCCESS : ret;
|
||||
if (OB_FAIL(ret)) {
|
||||
// add schedule suspect info
|
||||
if (OB_TMP_FAIL(ADD_SUSPECT_INFO(MEDIUM_MERGE, ls_.get_ls_id(), tablet_id,
|
||||
ObSuspectInfoType::SUSPECT_SCHEDULE_MEDIUM_FAILED,
|
||||
medium_info.medium_snapshot_,
|
||||
medium_info.storage_schema_.store_column_cnt_,
|
||||
static_cast<int64_t>(ret)))) {
|
||||
LOG_WARN("failed to add suspect info", K(tmp_ret));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
@ -443,32 +516,40 @@ int ObMediumCompactionScheduleFunc::init_parallel_range(
|
||||
ObMediumCompactionInfo &medium_info)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
ObSSTableMetaHandle meta_handle;
|
||||
int64_t expected_task_count = 0;
|
||||
const int64_t tablet_size = medium_info.storage_schema_.get_tablet_size();
|
||||
const ObSSTable *first_sstable = static_cast<const ObSSTable *>(result.handle_.get_table(0));
|
||||
|
||||
ObTablet *tablet = nullptr;
|
||||
if (OB_UNLIKELY(!tablet_handle_.is_valid())) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("invalid tablet_handle", K(ret), K(tablet_handle_));
|
||||
} else if (FALSE_IT(tablet = tablet_handle_.get_obj())) {
|
||||
} else if (OB_ISNULL(first_sstable)) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("sstable is unexpected null", K(ret), K(result));
|
||||
} else if (OB_FAIL(first_sstable->get_meta(meta_handle))) {
|
||||
LOG_WARN("first sstable get meta fail", K(ret), KPC(first_sstable));
|
||||
} else {
|
||||
int64_t expected_task_count = 0;
|
||||
const int64_t tablet_size = medium_info.storage_schema_.get_tablet_size();
|
||||
const ObSSTable *first_sstable = static_cast<const ObSSTable *>(result.handle_.get_table(0));
|
||||
if (OB_ISNULL(first_sstable)) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("sstable is unexpected null", K(ret), K(tablet));
|
||||
} else {
|
||||
const int64_t macro_block_cnt = first_sstable->get_meta().get_macro_info().get_data_block_ids().count();
|
||||
int64_t inc_row_cnt = 0;
|
||||
for (int64_t i = 0; i < result.handle_.get_count(); ++i) {
|
||||
inc_row_cnt += static_cast<const ObSSTable*>(result.handle_.get_table(i))->get_meta().get_row_count();
|
||||
const int64_t macro_block_cnt = first_sstable->get_data_macro_block_count();
|
||||
int64_t inc_row_cnt = 0;
|
||||
for (int64_t i = 0; OB_SUCC(ret) && i < result.handle_.get_count(); ++i) {
|
||||
ObSSTableMetaHandle inc_handle;
|
||||
if (OB_FAIL(static_cast<const ObSSTable*>(result.handle_.get_table(i))->get_meta(inc_handle))) {
|
||||
LOG_WARN("sstable get meta fail", K(ret));
|
||||
} else {
|
||||
inc_row_cnt += inc_handle.get_sstable_meta().get_row_count();
|
||||
}
|
||||
if ((0 == macro_block_cnt && inc_row_cnt > SCHEDULE_RANGE_ROW_COUNT_THRESHOLD)
|
||||
|| (first_sstable->get_meta().get_row_count() >= SCHEDULE_RANGE_ROW_COUNT_THRESHOLD
|
||||
&& inc_row_cnt >= first_sstable->get_meta().get_row_count() * SCHEDULE_RANGE_INC_ROW_COUNT_PERCENRAGE_THRESHOLD)) {
|
||||
if (OB_FAIL(ObParallelMergeCtx::get_concurrent_cnt(tablet_size, macro_block_cnt, expected_task_count))) {
|
||||
STORAGE_LOG(WARN, "failed to get concurrent cnt", K(ret), K(tablet_size), K(expected_task_count),
|
||||
KPC(first_sstable));
|
||||
}
|
||||
}
|
||||
|
||||
if (OB_FAIL(ret)) {
|
||||
} else if ((0 == macro_block_cnt && inc_row_cnt > SCHEDULE_RANGE_ROW_COUNT_THRESHOLD)
|
||||
|| (meta_handle.get_sstable_meta().get_row_count() >= SCHEDULE_RANGE_ROW_COUNT_THRESHOLD
|
||||
&& inc_row_cnt >= meta_handle.get_sstable_meta().get_row_count() * SCHEDULE_RANGE_INC_ROW_COUNT_PERCENRAGE_THRESHOLD)) {
|
||||
if (OB_FAIL(ObParallelMergeCtx::get_concurrent_cnt(tablet_size, macro_block_cnt, expected_task_count))) {
|
||||
STORAGE_LOG(WARN, "failed to get concurrent cnt", K(ret), K(tablet_size), K(expected_task_count),
|
||||
KPC(first_sstable), K(meta_handle));
|
||||
}
|
||||
}
|
||||
|
||||
@ -491,7 +572,7 @@ int ObMediumCompactionScheduleFunc::init_parallel_range(
|
||||
} else if (OB_FAIL(range_spliter.get_split_multi_ranges(
|
||||
input_range_array,
|
||||
expected_task_count,
|
||||
tablet->get_index_read_info(),
|
||||
tablet->get_rowkey_read_info(),
|
||||
table_iter,
|
||||
allocator_,
|
||||
range_array))) {
|
||||
@ -511,22 +592,25 @@ int ObMediumCompactionScheduleFunc::get_result_for_major(
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
ObSSTable *base_table = nullptr;
|
||||
const ObTabletTableStore &table_store = tablet.get_table_store();
|
||||
ObTabletMemberWrapper<ObTabletTableStore> table_store_wrapper;
|
||||
|
||||
if (OB_UNLIKELY(!medium_info.is_valid() || !table_store.is_valid())) {
|
||||
if (OB_FAIL(tablet.fetch_table_store(table_store_wrapper))) {
|
||||
LOG_WARN("fail to fetch table store", K(ret));
|
||||
} else if (OB_UNLIKELY(!table_store_wrapper.get_member()->is_valid())) {
|
||||
ret = OB_INVALID_ARGUMENT;
|
||||
LOG_WARN("get invalid argument", K(ret), K(medium_info), K(table_store));
|
||||
} else if (OB_ISNULL(base_table = static_cast<ObSSTable*>(table_store.get_major_sstables().get_boundary_table(true/*last*/)))) {
|
||||
LOG_WARN("get invalid argument", K(ret), K(medium_info), KPC(table_store_wrapper.get_member()));
|
||||
} else if (OB_ISNULL(base_table = static_cast<ObSSTable*>(
|
||||
table_store_wrapper.get_member()->get_major_sstables().get_boundary_table(true/*last*/)))) {
|
||||
ret = OB_ENTRY_NOT_EXIST;
|
||||
LOG_WARN("major sstable not exist", K(ret), K(table_store));
|
||||
LOG_WARN("major sstable not exist", K(ret), KPC(table_store_wrapper.get_member()));
|
||||
} else if (base_table->get_snapshot_version() >= medium_info.medium_snapshot_) {
|
||||
ret = OB_NO_NEED_MERGE;
|
||||
} else if (OB_FAIL(result.handle_.add_table(base_table))) {
|
||||
} else if (OB_FAIL(result.handle_.add_sstable(base_table, table_store_wrapper.get_meta_handle()))) {
|
||||
LOG_WARN("failed to add table into iterator", K(ret), KP(base_table));
|
||||
} else {
|
||||
const ObSSTableArray &minor_tables = table_store.get_minor_sstables();
|
||||
const ObSSTableArray &minor_tables = table_store_wrapper.get_member()->get_minor_sstables();
|
||||
bool start_add_table_flag = false;
|
||||
for (int64_t i = 0; OB_SUCC(ret) && i < minor_tables.count_; ++i) {
|
||||
for (int64_t i = 0; OB_SUCC(ret) && i < minor_tables.count(); ++i) {
|
||||
if (OB_ISNULL(minor_tables[i])) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_ERROR("table must not null", K(ret), K(i), K(minor_tables));
|
||||
@ -535,7 +619,7 @@ int ObMediumCompactionScheduleFunc::get_result_for_major(
|
||||
start_add_table_flag = true;
|
||||
}
|
||||
if (OB_SUCC(ret) && start_add_table_flag) {
|
||||
if (OB_FAIL(result.handle_.add_table(minor_tables[i]))) {
|
||||
if (OB_FAIL(result.handle_.add_sstable(minor_tables[i], table_store_wrapper.get_meta_handle()))) {
|
||||
LOG_WARN("failed to add table", K(ret));
|
||||
}
|
||||
}
|
||||
@ -565,46 +649,57 @@ int ObMediumCompactionScheduleFunc::prepare_medium_info(
|
||||
ObMediumCompactionInfo &medium_info)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
const ObStorageSchema *storage_schema = nullptr;
|
||||
ObTablet *tablet = nullptr;
|
||||
ObTableStoreIterator table_iter;
|
||||
medium_info.cluster_id_ = GCONF.cluster_id;
|
||||
// get table schema
|
||||
if (OB_UNLIKELY(!tablet_handle_.is_valid())) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("invalid tablet_handle", K(ret), K(tablet_handle_));
|
||||
} else if (FALSE_IT(tablet = tablet_handle_.get_obj())) {
|
||||
} else {
|
||||
ObTableStoreIterator table_iter;
|
||||
medium_info.cluster_id_ = GCONF.cluster_id; // set cluster id
|
||||
|
||||
if (medium_info.is_medium_compaction()) {
|
||||
ObStorageSchema tmp_storage_schema;
|
||||
bool use_storage_schema_on_tablet = true;
|
||||
if (medium_info.medium_snapshot_ > tablet->get_snapshot_version()) {
|
||||
ObSEArray<ObITable*, MAX_MEMSTORE_CNT> memtables;
|
||||
if (OB_FAIL(tablet->get_memtables(memtables, true/*need_active*/))) {
|
||||
LOG_WARN("failed to get memtables", K(ret), KPC(this));
|
||||
} else if (OB_FAIL(ObMediumCompactionScheduleFunc::get_latest_storage_schema_from_memtable(
|
||||
allocator_, memtables, tmp_storage_schema))) {
|
||||
if (OB_ENTRY_NOT_EXIST == ret) {
|
||||
ret = OB_SUCCESS; // clear errno
|
||||
} else {
|
||||
LOG_WARN("failed to get storage schema from memtable", K(ret));
|
||||
}
|
||||
} else {
|
||||
use_storage_schema_on_tablet = false;
|
||||
}
|
||||
} else if (0 == result.schema_version_) { // not formal schema version
|
||||
ret = OB_NO_NEED_MERGE;
|
||||
} else if (medium_info.is_medium_compaction()) {
|
||||
const uint64_t tenant_id = MTL_ID();
|
||||
ObMultiVersionSchemaService *schema_service = nullptr;
|
||||
if (OB_ISNULL(schema_service = MTL(ObTenantSchemaService *)->get_schema_service())) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("failed to get schema service from MTL", K(ret));
|
||||
} else if (FALSE_IT(medium_info.storage_schema_.reset())) {
|
||||
} else if (OB_FAIL(get_table_schema_to_merge(*schema_service, *tablet, result.schema_version_, allocator_, medium_info))) {
|
||||
// for major compaction, storage schema is inited in choose_major_snapshot
|
||||
if (OB_TABLE_IS_DELETED != ret) {
|
||||
LOG_WARN("failed to get table schema", KR(ret), KPC(this), K(medium_info));
|
||||
}
|
||||
|
||||
if (FAILEDx(medium_info.save_storage_schema(
|
||||
allocator_,
|
||||
use_storage_schema_on_tablet ? tablet->get_storage_schema() : tmp_storage_schema))) {
|
||||
LOG_WARN("failed to save storage schema", K(ret), K(use_storage_schema_on_tablet), K(tmp_storage_schema));
|
||||
}
|
||||
}
|
||||
if (FAILEDx(init_parallel_range(result, medium_info))) {
|
||||
LOG_WARN("failed to init parallel range", K(ret), K(medium_info));
|
||||
} else if (OB_UNLIKELY(result.handle_.empty())) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("result is invalid empty", KR(ret), KPC(this), K(result));
|
||||
} else {
|
||||
LOG_INFO("success to prepare medium info", K(ret), K(medium_info));
|
||||
ObSSTable *base_table = static_cast<ObSSTable *>(result.handle_.get_table(0));
|
||||
ObSSTableMetaHandle sstable_meta_hdl;
|
||||
int64_t schema_col_cnt = 0;
|
||||
if (OB_FAIL(base_table->get_meta(sstable_meta_hdl))) {
|
||||
LOG_WARN("failed to get base table meta", KR(ret));
|
||||
} else if (OB_FAIL(medium_info.storage_schema_.get_stored_column_count_in_sstable(schema_col_cnt))) {
|
||||
LOG_WARN("failed to get store column count", KR(ret), K(medium_info));
|
||||
} else if (OB_UNLIKELY(sstable_meta_hdl.get_sstable_meta().get_column_count() > schema_col_cnt)) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_ERROR("medium column cnt is less than last major sstable", KR(ret), K(schema_col_cnt), K(sstable_meta_hdl));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (FAILEDx(init_parallel_range(result, medium_info))) {
|
||||
LOG_WARN("failed to init parallel range", K(ret), K(medium_info));
|
||||
} else if (OB_UNLIKELY(result.handle_.empty())) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("table handle in result is empty", KR(ret), K(result));
|
||||
} else {
|
||||
medium_info.last_medium_snapshot_ = result.handle_.get_table(0)->get_snapshot_version();
|
||||
}
|
||||
if (OB_SUCC(ret)) {
|
||||
LOG_INFO("success to prepare medium info", K(ret), K(medium_info));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -636,6 +731,7 @@ int ObMediumCompactionScheduleFunc::get_table_id(
|
||||
}
|
||||
|
||||
int ObMediumCompactionScheduleFunc::get_table_schema_to_merge(
|
||||
ObMultiVersionSchemaService &schema_service,
|
||||
const ObTablet &tablet,
|
||||
const int64_t schema_version,
|
||||
ObIAllocator &allocator,
|
||||
@ -645,18 +741,14 @@ int ObMediumCompactionScheduleFunc::get_table_schema_to_merge(
|
||||
const uint64_t tenant_id = MTL_ID();
|
||||
const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_;
|
||||
uint64_t table_id = OB_INVALID_ID;
|
||||
ObMultiVersionSchemaService *schema_service = nullptr;
|
||||
schema::ObSchemaGetterGuard schema_guard;
|
||||
const ObTableSchema *table_schema = nullptr;
|
||||
int64_t save_schema_version = schema_version;
|
||||
if (OB_ISNULL(schema_service = MTL(ObTenantSchemaService *)->get_schema_service())) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("failed to get schema service from MTL", K(ret));
|
||||
} else if (OB_FAIL(get_table_id(*schema_service, tablet_id, schema_version, table_id))) {
|
||||
if (OB_FAIL(get_table_id(schema_service, tablet_id, schema_version, table_id))) {
|
||||
if (OB_TABLE_IS_DELETED != ret) {
|
||||
LOG_WARN("failed to get table id", K(ret), K(tablet_id));
|
||||
}
|
||||
} else if (OB_FAIL(schema_service->retry_get_schema_guard(tenant_id,
|
||||
} else if (OB_FAIL(schema_service.retry_get_schema_guard(tenant_id,
|
||||
schema_version,
|
||||
table_id,
|
||||
schema_guard,
|
||||
@ -695,13 +787,15 @@ int ObMediumCompactionScheduleFunc::get_table_schema_to_merge(
|
||||
}
|
||||
}
|
||||
#endif
|
||||
// for old version medium info, need generate old version schema
|
||||
if (FAILEDx(medium_info.storage_schema_.init(
|
||||
allocator,
|
||||
*table_schema,
|
||||
tablet.get_tablet_meta().compat_mode_))) {
|
||||
allocator, *table_schema, tablet.get_tablet_meta().compat_mode_, false/*skip_column_info*/,
|
||||
ObMediumCompactionInfo::MEIDUM_COMPAT_VERSION_V2 == medium_info.medium_compat_version_
|
||||
? ObStorageSchema::STORAGE_SCHEMA_VERSION_V2
|
||||
: ObStorageSchema::STORAGE_SCHEMA_VERSION))) {
|
||||
LOG_WARN("failed to init storage schema", K(ret), K(schema_version));
|
||||
} else {
|
||||
FLOG_INFO("get schema to merge", K(table_id), K(schema_version), K(save_schema_version),
|
||||
LOG_TRACE("get schema to merge", K(table_id), K(schema_version), K(save_schema_version),
|
||||
K(*reinterpret_cast<const ObPrintableTableSchema*>(table_schema)));
|
||||
}
|
||||
return ret;
|
||||
@ -863,16 +957,22 @@ int ObMediumCompactionScheduleFunc::check_medium_finish(const ObLSLocality &ls_l
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
ObTablet *tablet = nullptr;
|
||||
const ObLSID &ls_id = ls_.get_ls_id();
|
||||
bool merge_finish = false;
|
||||
|
||||
if (OB_UNLIKELY(!tablet_handle_.is_valid())) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("invalid tablet_handle", K(ret), K(tablet_handle_));
|
||||
} else if (FALSE_IT(tablet = tablet_handle_.get_obj())) {
|
||||
} else if (nullptr == medium_info_list_
|
||||
&& OB_FAIL(tablet->read_medium_info_list(allocator_, medium_info_list_))) {
|
||||
LOG_WARN("failed to read medium info list", K(ret), KPC(tablet));
|
||||
} else if (OB_ISNULL(medium_info_list_)) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("medium info list is unexpected null", K(ret), KPC(this), KPC_(medium_info_list));
|
||||
} else {
|
||||
const ObLSID &ls_id = ls_.get_ls_id();
|
||||
const ObTabletID &tablet_id = tablet->get_tablet_meta().tablet_id_;
|
||||
const int64_t wait_check_medium_scn = tablet->get_medium_compaction_info_list().get_wait_check_medium_scn();
|
||||
bool merge_finish = false;
|
||||
|
||||
const int64_t wait_check_medium_scn = medium_info_list_->get_wait_check_medium_scn();
|
||||
if (0 == wait_check_medium_scn) {
|
||||
// do nothing
|
||||
} else if (OB_FAIL(check_medium_meta_table(wait_check_medium_scn, ls_id, tablet_id, ls_locality, merge_finish))) {
|
||||
@ -882,8 +982,8 @@ int ObMediumCompactionScheduleFunc::check_medium_finish(const ObLSLocality &ls_l
|
||||
} else if (OB_FAIL(check_medium_checksum_table(wait_check_medium_scn, ls_id, tablet_id))) { // check checksum
|
||||
LOG_WARN("failed to check checksum", K(ret), K(wait_check_medium_scn), KPC(this));
|
||||
} else {
|
||||
const ObMediumCompactionInfo::ObCompactionType compaction_type = tablet->get_medium_compaction_info_list().get_last_compaction_type();
|
||||
FLOG_INFO("check medium compaction info", K(ret), K(ls_id), K(tablet_id), K(compaction_type));
|
||||
const ObMediumCompactionInfo::ObCompactionType compaction_type = medium_info_list_->get_last_compaction_type();
|
||||
LOG_INFO("check medium compaction info", K(ret), K(ls_id), K(tablet_id), K(compaction_type), K(wait_check_medium_scn));
|
||||
|
||||
// clear wait_check_medium_scn on Tablet
|
||||
ObTabletHandle new_handle;
|
||||
@ -909,6 +1009,7 @@ int ObMediumCompactionScheduleFunc::schedule_tablet_medium_merge(
|
||||
const bool scheduler_called)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
ObTabletMemberWrapper<ObTabletTableStore> table_store_wrapper;
|
||||
#ifdef ERRSIM
|
||||
ret = OB_E(EventTable::EN_MEDIUM_CREATE_DAG) ret;
|
||||
if (OB_FAIL(ret)) {
|
||||
@ -916,52 +1017,59 @@ int ObMediumCompactionScheduleFunc::schedule_tablet_medium_merge(
|
||||
return ret;
|
||||
}
|
||||
#endif
|
||||
|
||||
if (MTL(ObTenantTabletScheduler *)->could_major_merge_start()) {
|
||||
const ObMediumCompactionInfoList &medium_list = tablet.get_medium_compaction_info_list();
|
||||
const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_;
|
||||
const ObLSID &ls_id = ls.get_ls_id();
|
||||
ObITable *last_major = tablet.get_table_store().get_major_sstables().get_boundary_table(true/*last*/);
|
||||
|
||||
bool schedule_flag = false;
|
||||
if (OB_ISNULL(last_major)) {
|
||||
// do nothing
|
||||
} else if (ObMediumCompactionInfo::MAJOR_COMPACTION == medium_list.get_last_compaction_type()
|
||||
&& !MTL_IS_PRIMARY_TENANT()) { // for STANDBY/RESTORE TENANT
|
||||
ObTabletCompactionScnInfo ret_info;
|
||||
// for standby/restore tenant, need select inner_table to check RS status before schedule new round
|
||||
if (!scheduler_called) { // should not visit inner table, wait for scheduler loop
|
||||
} else if (OB_FAIL(get_status_from_inner_table(ls_id, tablet_id, ret_info))) {
|
||||
LOG_WARN("failed to get status from inner tablet", K(ret), K(ls_id), K(tablet_id));
|
||||
} else if (ret_info.could_schedule_next_round(medium_list.get_last_compaction_scn())) {
|
||||
LOG_INFO("success to check RS major checksum validation finished", K(ret), K(ls_id), K(tablet_id));
|
||||
schedule_flag = true;
|
||||
}
|
||||
if (OB_FAIL(tablet.fetch_table_store(table_store_wrapper))) {
|
||||
LOG_WARN("failed to fetch table store", K(ret), K(tablet));
|
||||
} else {
|
||||
schedule_flag = true;
|
||||
}
|
||||
|
||||
if (OB_SUCC(ret) && schedule_flag) {
|
||||
int64_t major_frozen_snapshot = 0 == input_major_snapshot ? MTL(ObTenantTabletScheduler *)->get_frozen_version() : input_major_snapshot;
|
||||
ObMediumCompactionInfo::ObCompactionType compaction_type = ObMediumCompactionInfo::COMPACTION_TYPE_MAX;
|
||||
int64_t schedule_scn = 0;
|
||||
bool need_merge = false;
|
||||
|
||||
(void)tablet.get_medium_compaction_info_list().get_schedule_scn(major_frozen_snapshot, schedule_scn, compaction_type);
|
||||
|
||||
LOG_DEBUG("schedule_tablet_medium_merge", K(schedule_scn), K(major_frozen_snapshot), K(scheduler_called), K(ls_id), K(tablet_id));
|
||||
if (0 == schedule_scn
|
||||
&& 0 == tablet.get_medium_compaction_info_list().size() // serialized medium list is empty
|
||||
&& scheduler_called
|
||||
&& OB_FAIL(get_schedule_medium_from_memtable(tablet, major_frozen_snapshot, schedule_scn, compaction_type))) {
|
||||
LOG_WARN("failed to get schedule medium scn from memtables", K(ret));
|
||||
} else if (schedule_scn > 0) {
|
||||
if (OB_FAIL(check_need_merge_and_schedule(ls, tablet, schedule_scn, compaction_type))) {
|
||||
ObITable *last_major = table_store_wrapper.get_member()->get_major_sstables().get_boundary_table(true/*last*/);
|
||||
if (MTL(ObTenantTabletScheduler *)->could_major_merge_start() && OB_NOT_NULL(last_major)) {
|
||||
ObArenaAllocator temp_allocator("GetMediumInfo", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); // for load medium info
|
||||
const int64_t last_major_snapshot = last_major->get_snapshot_version();
|
||||
const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_;
|
||||
const ObLSID &ls_id = ls.get_ls_id();
|
||||
const int64_t major_frozen_snapshot = 0 == input_major_snapshot ? MTL(ObTenantTabletScheduler *)->get_frozen_version() : input_major_snapshot;
|
||||
ObMediumCompactionInfo::ObCompactionType compaction_type = ObMediumCompactionInfo::COMPACTION_TYPE_MAX;
|
||||
int64_t schedule_scn = 0;
|
||||
{
|
||||
storage::ObTabletMediumInfoReader reader(tablet);
|
||||
ObMediumCompactionInfoKey key;
|
||||
ObMediumCompactionInfo medium_info;
|
||||
if (OB_FAIL(reader.init(temp_allocator))) {
|
||||
LOG_WARN("failed to init medium info reader", K(ret));
|
||||
} else {
|
||||
// TODO(@bowen.gbw): remove this debug log later
|
||||
LOG_INFO("schedule tablet medium merge", K(ret), K(schedule_scn), K(major_frozen_snapshot), K(ls_id), K(tablet_id));
|
||||
while (OB_SUCC(ret)) {
|
||||
if (OB_FAIL(reader.get_next_medium_info(temp_allocator, key, medium_info))) {
|
||||
if (OB_ITER_END == ret) {
|
||||
ret = OB_SUCCESS;
|
||||
break;
|
||||
} else {
|
||||
LOG_WARN("failed to get medium info", K(ret));
|
||||
}
|
||||
} else if (key.get_medium_snapshot() <= last_major_snapshot) {
|
||||
// finished, this medium info could recycle
|
||||
} else {
|
||||
if (medium_info.is_medium_compaction() || medium_info.medium_snapshot_ <= major_frozen_snapshot) {
|
||||
schedule_scn = medium_info.medium_snapshot_;
|
||||
compaction_type = (ObMediumCompactionInfo::ObCompactionType)medium_info.compaction_type_;
|
||||
LOG_TRACE("set schedule scn and compaction type", K(ret), K(ls_id), K(tablet_id),
|
||||
K(schedule_scn), K(compaction_type), K(major_frozen_snapshot));
|
||||
}
|
||||
break; // break when met one undo medium info
|
||||
}
|
||||
if (OB_SUCC(ret)) {
|
||||
LOG_DEBUG("iter read medium info", K(ret), K(ls_id), K(tablet_id), K(medium_info));
|
||||
}
|
||||
medium_info.reset();
|
||||
} // end of while
|
||||
}
|
||||
}// to make reader destruct to release mds unit lock
|
||||
if (OB_FAIL(ret)) {
|
||||
} else if (schedule_scn > 0 && OB_FAIL(check_need_merge_and_schedule(ls, tablet, schedule_scn, compaction_type))) {
|
||||
LOG_WARN("failed to check medium merge", K(ret), K(ls_id), K(tablet_id), K(schedule_scn));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
@ -982,50 +1090,6 @@ int ObMediumCompactionScheduleFunc::get_palf_role(const ObLSID &ls_id, ObRole &r
|
||||
return ret;
|
||||
}
|
||||
|
||||
int ObMediumCompactionScheduleFunc::get_schedule_medium_from_memtable(
|
||||
ObTablet &tablet,
|
||||
const int64_t major_frozen_snapshot,
|
||||
int64_t &schedule_medium_scn,
|
||||
ObMediumCompactionInfo::ObCompactionType &compaction_type)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
schedule_medium_scn = 0;
|
||||
compaction_type = ObMediumCompactionInfo::COMPACTION_TYPE_MAX;
|
||||
|
||||
ObITable *last_major = tablet.get_table_store().get_major_sstables().get_boundary_table(true/*last*/);
|
||||
if (OB_NOT_NULL(last_major)) {
|
||||
ObArenaAllocator tmp_allocator;
|
||||
ObMediumCompactionInfoList tmp_medium_list;
|
||||
const int64_t last_major_snapshot = last_major->get_snapshot_version();
|
||||
ObSEArray<ObITable*, MAX_MEMSTORE_CNT> memtables;
|
||||
if (OB_FAIL(tablet.get_memtables(memtables, true/*need_active*/))) {
|
||||
LOG_WARN("failed to get memtables", K(ret), "tablet_id", tablet.get_tablet_meta().tablet_id_);
|
||||
} else if (memtables.empty()) {
|
||||
// do nothing
|
||||
} else if (OB_FAIL(get_medium_info_list_from_memtable(tmp_allocator, memtables, tmp_medium_list))) {
|
||||
LOG_WARN("failed to get medium info list from memtable", K(ret));
|
||||
} else if (!tmp_medium_list.is_empty()) {
|
||||
const ObMediumCompactionInfo *info_in_list = nullptr;
|
||||
DLIST_FOREACH_X(info, tmp_medium_list.get_list(), OB_SUCC(ret)) {
|
||||
if (OB_UNLIKELY(memtable::MultiSourceDataUnitType::MEDIUM_COMPACTION_INFO != info->type())) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("return info is invalid", K(ret), KPC(info));
|
||||
} else if (FALSE_IT(info_in_list = static_cast<const ObMediumCompactionInfo *>(info))) {
|
||||
} else if (info_in_list->medium_snapshot_ <= last_major_snapshot) {
|
||||
// finished, this medium info could recycle
|
||||
} else {
|
||||
if (info_in_list->is_medium_compaction() || info_in_list->medium_snapshot_ <= major_frozen_snapshot) {
|
||||
schedule_medium_scn = info_in_list->medium_snapshot_;
|
||||
compaction_type = (ObMediumCompactionInfo::ObCompactionType)info_in_list->compaction_type_;
|
||||
}
|
||||
break; // found one unfinish medium info, loop end
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
int ObMediumCompactionScheduleFunc::check_need_merge_and_schedule(
|
||||
ObLS &ls,
|
||||
ObTablet &tablet,
|
||||
@ -1052,7 +1116,6 @@ int ObMediumCompactionScheduleFunc::check_need_merge_and_schedule(
|
||||
need_force_freeze))) { // check merge finish
|
||||
LOG_WARN("failed to check medium merge", K(ret), K(ls_id), K(tablet_id));
|
||||
} else if (need_merge && can_merge) {
|
||||
const ObMediumCompactionInfo *medium_info = nullptr;
|
||||
if (OB_TMP_FAIL(ObTenantTabletScheduler::schedule_merge_dag(
|
||||
ls.get_ls_id(),
|
||||
tablet_id,
|
||||
@ -1074,78 +1137,5 @@ int ObMediumCompactionScheduleFunc::check_need_merge_and_schedule(
|
||||
return ret;
|
||||
}
|
||||
|
||||
int ObMediumCompactionScheduleFunc::get_latest_storage_schema_from_memtable(
|
||||
ObIAllocator &allocator,
|
||||
const ObIArray<ObITable *> &memtables,
|
||||
ObStorageSchema &storage_schema)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
ObITable *table = nullptr;
|
||||
memtable::ObMemtable * memtable = nullptr;
|
||||
bool found = false;
|
||||
for (int64_t i = memtables.count() - 1; OB_SUCC(ret) && i >= 0; --i) {
|
||||
if (OB_ISNULL(table = memtables.at(i))) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("table in tables_handle is invalid", K(ret), KPC(table));
|
||||
} else if (OB_ISNULL(memtable = dynamic_cast<memtable::ObMemtable *>(table))) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("table pointer does not point to a ObMemtable object", KPC(table));
|
||||
} else if (OB_FAIL(memtable->get_multi_source_data_unit(&storage_schema, &allocator))) {
|
||||
if (OB_ENTRY_NOT_EXIST == ret) {
|
||||
ret = OB_SUCCESS; // clear OB_ENTRY_NOT_EXIST
|
||||
} else {
|
||||
LOG_WARN("failed to get storage schema from memtable", K(ret), KPC(table));
|
||||
}
|
||||
} else {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
} // end for
|
||||
if (OB_SUCC(ret) && !found) {
|
||||
ret = OB_ENTRY_NOT_EXIST;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
// make sure only data tablet could schedule this func
|
||||
int ObMediumCompactionScheduleFunc::get_medium_info_list_from_memtable(
|
||||
ObIAllocator &allocator,
|
||||
const ObIArray<ObITable *> &memtables,
|
||||
ObMediumCompactionInfoList &medium_list)
|
||||
{
|
||||
int ret = OB_SUCCESS;
|
||||
if (OB_FAIL(medium_list.init(allocator))) {
|
||||
LOG_WARN("failed to init merge list", K(ret));
|
||||
}
|
||||
ObITable *table = nullptr;
|
||||
memtable::ObMemtable *memtable = nullptr;
|
||||
compaction::ObMediumCompactionInfo useless_medium_info;
|
||||
memtable::ObMultiSourceData::ObIMultiSourceDataUnitList dst_list;
|
||||
for (int64_t i = 0; OB_SUCC(ret) && i < memtables.count(); ++i) {
|
||||
dst_list.reset();
|
||||
if (OB_ISNULL(table = memtables.at(i))) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("table in tables_handle is invalid", K(ret), KPC(table));
|
||||
} else if (OB_ISNULL(memtable = dynamic_cast<memtable::ObMemtable *>(table))) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("table pointer does not point to a ObMemtable object", KPC(table));
|
||||
} else if (OB_FAIL(memtable->get_multi_source_data_unit_list(&useless_medium_info, dst_list, &allocator))) {
|
||||
LOG_WARN("failed to get medium info from memtable", K(ret), KPC(table));
|
||||
} else {
|
||||
ObMediumCompactionInfo *info_in_list = nullptr;
|
||||
DLIST_FOREACH_X(info, dst_list, OB_SUCC(ret)) {
|
||||
if (OB_UNLIKELY(memtable::MultiSourceDataUnitType::MEDIUM_COMPACTION_INFO != info->type())) {
|
||||
ret = OB_ERR_UNEXPECTED;
|
||||
LOG_WARN("return info is invalid", K(ret), KPC(info));
|
||||
} else if (FALSE_IT(info_in_list = static_cast<ObMediumCompactionInfo *>(info))) {
|
||||
} else if (OB_FAIL(medium_list.add_medium_compaction_info(*info_in_list))) {
|
||||
LOG_WARN("failed to add medium compaction info", K(ret), KPC(info_in_list));
|
||||
}
|
||||
}
|
||||
}
|
||||
} // end of for
|
||||
return ret;
|
||||
}
|
||||
|
||||
} //namespace compaction
|
||||
} // namespace oceanbase
|
||||
|
||||
Reference in New Issue
Block a user