diff --git a/src/rootserver/freeze/ob_major_merge_progress_checker.cpp b/src/rootserver/freeze/ob_major_merge_progress_checker.cpp index c5083cc35a..9d23fd5b03 100644 --- a/src/rootserver/freeze/ob_major_merge_progress_checker.cpp +++ b/src/rootserver/freeze/ob_major_merge_progress_checker.cpp @@ -412,8 +412,13 @@ void ObMajorMergeProgressChecker::deal_with_unfinish_table_ids( } } if (OB_FAIL(ret)) { - } else if (assgin_flag && OB_FAIL(table_ids_.assign(unfinish_table_id_array))) { - LOG_WARN("failed to assign table ids", KR(ret), K(unfinish_table_id_array)); + } else if (assgin_flag) { + int64_t retry_times = 0; + do { + if (OB_FAIL(table_ids_.assign(unfinish_table_id_array))) { + LOG_WARN("failed to assign table ids", KR(ret), K(unfinish_table_id_array)); + } + } while (OB_ALLOCATE_MEMORY_FAILED == ret && (++retry_times < ASSGIN_FAILURE_RETRY_TIMES)); } } @@ -470,7 +475,7 @@ int ObMajorMergeProgressChecker::check_index_and_rest_table() LOG_WARN("deal with rest data table", KR(ret), K_(compaction_scn)); } else if (progress_.is_merge_finished()) { LOG_INFO("progress is check finished", KR(ret), K_(progress)); - } else if (progress_.only_remain_special_table_to_verified()) { + } else if (progress_.only_remain_special_table_to_verified() || table_ids_.empty()) { bool finish_validate = false; #ifdef ERRSIM ret = OB_E(EventTable::EN_RS_CHECK_SPECIAL_TABLE) ret; @@ -485,10 +490,6 @@ int ObMajorMergeProgressChecker::check_index_and_rest_table() } else if (finish_validate) { progress_.deal_with_special_tablet(); } - } else if (table_ids_.empty()) { - // DEBUG LOG - ret = OB_ERR_UNEXPECTED; - LOG_WARN("cnt in progress is not equal to table_ids", KR(ret), K(table_ids_), K(progress_)); } (void) ckm_validator_.batch_update_report_scn(); (void) ckm_validator_.batch_write_tablet_ckm(); diff --git a/src/rootserver/freeze/ob_major_merge_progress_checker.h b/src/rootserver/freeze/ob_major_merge_progress_checker.h index b8867155fc..ffec062436 100644 --- a/src/rootserver/freeze/ob_major_merge_progress_checker.h +++ b/src/rootserver/freeze/ob_major_merge_progress_checker.h @@ -125,6 +125,7 @@ private: static const int64_t PRINT_LOG_INTERVAL = 2 * 60 * 1000 * 1000; // 2m static const int64_t DEAL_REST_TABLE_CNT_THRESHOLD = 100; static const int64_t DEAL_REST_TABLE_INTERVAL = 10 * 60 * 1000 * 1000L; // 10m + static const int64_t ASSGIN_FAILURE_RETRY_TIMES = 10; private: bool is_inited_; bool first_loop_in_cur_round_; diff --git a/src/share/compaction/ob_table_ckm_items.cpp b/src/share/compaction/ob_table_ckm_items.cpp index df915984f1..5ea3afbac7 100644 --- a/src/share/compaction/ob_table_ckm_items.cpp +++ b/src/share/compaction/ob_table_ckm_items.cpp @@ -284,8 +284,9 @@ int ObTableCkmItems::build_column_ckm_sum_array( return ret; } -#define RECORD_CKM_ERROR_INFO(tablet_array_idx) \ +#define RECORD_CKM_ERROR_INFO(tablet_array_idx, is_global_index) \ ckm_error_info.tenant_id_ = data_ckm.tenant_id_; \ + ckm_error_info.is_global_index_ = is_global_index; \ ckm_error_info.frozen_scn_ = compaction_scn; \ ckm_error_info.data_table_id_ = data_table_schema->get_table_id(); \ ckm_error_info.index_table_id_ = index_table_schema->get_table_id(); \ @@ -334,7 +335,7 @@ int ObTableCkmItems::validate_column_ckm_sum( } } if (OB_CHECKSUM_ERROR == ret) { - RECORD_CKM_ERROR_INFO(OB_INVALID_INDEX /*array_idx*/); + RECORD_CKM_ERROR_INFO(OB_INVALID_INDEX /*array_idx*/, true/*is_global_index*/); LOG_ERROR("failed to compare column checksum", KR(ret), K(ckm_error_info), K(data_ckm.ckm_items_), K(index_ckm.ckm_items_)); if (OB_TMP_FAIL(ObColumnChecksumErrorOperator::insert_column_checksum_err_info(sql_proxy, data_ckm.tenant_id_, @@ -436,7 +437,7 @@ int ObTableCkmItems::validate_tablet_column_ckm( } #endif if (OB_CHECKSUM_ERROR == ret) { - RECORD_CKM_ERROR_INFO(idx); + RECORD_CKM_ERROR_INFO(idx, false/*is_global_index*/); LOG_ERROR("failed to compare column checksum", KR(ret), K(ckm_error_info), "data_tablet", data_ckm.tablet_pairs_.at(idx), "data_row_cnt", data_replica_ckm.row_count_, K(data_replica_ckm), "index_tablet", index_ckm.tablet_pairs_.at(idx), "index_row_cnt", index_replica_ckm.row_count_, K(index_replica_ckm), diff --git a/src/share/scheduler/ob_tenant_dag_scheduler.cpp b/src/share/scheduler/ob_tenant_dag_scheduler.cpp index ed9a72a004..8596282c5d 100644 --- a/src/share/scheduler/ob_tenant_dag_scheduler.cpp +++ b/src/share/scheduler/ob_tenant_dag_scheduler.cpp @@ -2826,7 +2826,7 @@ int ObDagPrioScheduler::inner_add_dag( { \ info_list[idx].tenant_id_ = MTL_ID(); \ info_list[idx].value_type_ = value_type; \ - strcpy(info_list[idx].key_, key_str); \ + strncpy(info_list[idx].key_, key_str, MIN(common::OB_DAG_KEY_LENGTH - 1, strlen(key_str))); \ info_list[idx].value_ = value; \ (void)scheduler_infos.push_back(&info_list[idx++]); \ } diff --git a/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp b/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp index 0feffb9aab..5942f2f88a 100644 --- a/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp +++ b/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp @@ -492,7 +492,8 @@ int ObBasicTabletMergeCtx::get_merge_tables(ObGetMergeTablesResult &get_merge_ta get_merge_table_param.merge_type_ = get_merge_type(); get_merge_table_param.merge_version_ = get_merge_version(); get_merge_table_result.error_location_ = &info_collector_.error_location_; - if (OB_FAIL(ObPartitionMergePolicy::get_merge_tables[get_merge_type()]( + if (is_valid_merge_type(get_merge_type()) + && OB_FAIL(ObPartitionMergePolicy::get_merge_tables[get_merge_type()]( get_merge_table_param, *get_ls(), *get_tablet(), diff --git a/src/storage/compaction/ob_compaction_diagnose.cpp b/src/storage/compaction/ob_compaction_diagnose.cpp index 2c797d4cfc..d2c64eefa5 100644 --- a/src/storage/compaction/ob_compaction_diagnose.cpp +++ b/src/storage/compaction/ob_compaction_diagnose.cpp @@ -1638,7 +1638,7 @@ int ObCompactionDiagnoseMgr::diagnose_no_dag( int ret = OB_SUCCESS; ObScheduleSuspectInfo info; bool add_schedule_info = false; - ObSuspectInfoType suspect_type; + ObSuspectInfoType suspect_type = SUSPECT_INFO_TYPE_MAX; char tmp_str[common::OB_DIAGNOSE_INFO_LENGTH] = "\0"; if (OB_FAIL(get_suspect_and_warning_info(dag_key, merge_type, ls_id, tablet_id, info, suspect_type, tmp_str, sizeof(tmp_str)))) { @@ -1679,7 +1679,7 @@ int ObCompactionDiagnoseMgr::diagnose_no_dag( add_schedule_info = true; } - if (OB_SUCC(ret) && add_schedule_info && can_add_diagnose_info()) { + if (OB_SUCC(ret) && add_schedule_info && can_add_diagnose_info() && suspect_type < SUSPECT_INFO_TYPE_MAX) { // check tablet_type in get_diagnose_tablet_count if (suspect_tablet_count_[suspect_type] < DIAGNOSE_TABELT_MAX_COUNT) { if (OB_FAIL(ADD_DIAGNOSE_INFO_FOR_TABLET( diff --git a/src/storage/compaction/ob_compaction_diagnose.h b/src/storage/compaction/ob_compaction_diagnose.h index 4e88e3d7f0..139ee10e4a 100644 --- a/src/storage/compaction/ob_compaction_diagnose.h +++ b/src/storage/compaction/ob_compaction_diagnose.h @@ -640,17 +640,23 @@ private: return ret; \ } -#define DEFINE_COMPACITON_INFO_ADD_KV(n) \ - template \ - void ADD_COMPACTION_INFO_PARAM(char *buf, const int64_t buf_size, LOG_PARAMETER_KV##n) \ - { \ - int64_t __pos = strlen(buf); \ - int ret = OB_SUCCESS; \ - SIMPLE_TO_STRING_##n \ - if (__pos > 0) { \ - buf[__pos - 1] = ';'; \ - } \ - buf[__pos] = '\0'; \ +#define DEFINE_COMPACITON_INFO_ADD_KV(n) \ + template \ + void ADD_COMPACTION_INFO_PARAM(char *buf, const int64_t buf_size, \ + LOG_PARAMETER_KV##n) { \ + int64_t __pos = strlen(buf); \ + int ret = OB_SUCCESS; \ + SIMPLE_TO_STRING_##n \ + if (__pos < 0) { \ + __pos = 0; \ + } else if (__pos > 0) { \ + if (__pos >= buf_size) { \ + __pos = buf_size - 1; \ + } else { \ + buf[__pos - 1] = ';'; \ + } \ + } \ + buf[__pos] = '\0'; \ } #define SIMPLE_TO_STRING(n) \ diff --git a/src/storage/compaction/ob_compaction_util.h b/src/storage/compaction/ob_compaction_util.h index 9ef1cd507c..7e2b1ac9be 100644 --- a/src/storage/compaction/ob_compaction_util.h +++ b/src/storage/compaction/ob_compaction_util.h @@ -29,6 +29,8 @@ enum ObMergeType DDL_KV_MERGE = 6, // only use for ddl dag BACKFILL_TX_MERGE = 7, MDS_TABLE_MERGE = 8, + // add new merge type here + // fix merge_type_to_str & ObPartitionMergePolicy::get_merge_tables MERGE_TYPE_MAX }; diff --git a/src/storage/compaction/ob_partition_merge_policy.cpp b/src/storage/compaction/ob_partition_merge_policy.cpp index 94cd44966d..739b20687d 100644 --- a/src/storage/compaction/ob_partition_merge_policy.cpp +++ b/src/storage/compaction/ob_partition_merge_policy.cpp @@ -53,6 +53,9 @@ ObPartitionMergePolicy::GetMergeTables ObPartitionMergePolicy::get_merge_tables[ ObPartitionMergePolicy::get_mini_merge_tables, ObPartitionMergePolicy::get_medium_merge_tables, ObPartitionMergePolicy::get_medium_merge_tables, + ObPartitionMergePolicy::not_support_merge_type, + ObPartitionMergePolicy::not_support_merge_type, + ObPartitionMergePolicy::not_support_merge_type }; @@ -62,6 +65,7 @@ int ObPartitionMergePolicy::get_neighbour_freeze_info( ObTenantFreezeInfoMgr::NeighbourFreezeInfo &freeze_info, const bool is_multi_version_merge) { + STATIC_ASSERT(static_cast(MERGE_TYPE_MAX) == ARRAYSIZEOF(get_merge_tables), "get merge table func cnt is mismatch"); int ret = OB_SUCCESS; if (OB_FAIL(MTL(ObTenantFreezeInfoMgr *)->get_neighbour_major_freeze(snapshot_version, freeze_info))) { if (OB_ENTRY_NOT_EXIST == ret) { diff --git a/src/storage/compaction/ob_partition_merge_policy.h b/src/storage/compaction/ob_partition_merge_policy.h index f5fb18c3f5..a78ec91f81 100644 --- a/src/storage/compaction/ob_partition_merge_policy.h +++ b/src/storage/compaction/ob_partition_merge_policy.h @@ -67,6 +67,14 @@ public: storage::ObLS &ls, const storage::ObTablet &tablet, storage::ObGetMergeTablesResult &result); + static int not_support_merge_type( + const storage::ObGetMergeTablesParam ¶m, + storage::ObLS &ls, + const storage::ObTablet &tablet, + storage::ObGetMergeTablesResult &result) + { + return OB_NOT_SUPPORTED; + } static int check_need_medium_merge( storage::ObLS &ls, storage::ObTablet &tablet, diff --git a/src/storage/compaction/ob_tablet_merge_task.cpp b/src/storage/compaction/ob_tablet_merge_task.cpp index de6a17e86a..14f67d5a96 100644 --- a/src/storage/compaction/ob_tablet_merge_task.cpp +++ b/src/storage/compaction/ob_tablet_merge_task.cpp @@ -266,6 +266,7 @@ int64_t ObMergeParameter::to_string(char* buf, const int64_t buf_len) const ObCompactionParam::ObCompactionParam() : score_(0), occupy_size_(0), + estimate_phy_size_(0), replay_interval_(0), add_time_(0), last_end_scn_(),