From c3f0fb704b1552d7d6e59365e7dc9bcd1d4fe891 Mon Sep 17 00:00:00 2001 From: a1iive Date: Tue, 11 Jul 2023 15:11:59 +0000 Subject: [PATCH] add log in compaction diagnose --- src/share/scheduler/ob_diagnose_config.h | 8 +-- .../compaction/ob_compaction_diagnose.cpp | 61 +++++++++++-------- .../compaction/ob_tablet_merge_task.cpp | 1 + unittest/storage/test_dag_warning_history.cpp | 2 +- 4 files changed, 41 insertions(+), 31 deletions(-) diff --git a/src/share/scheduler/ob_diagnose_config.h b/src/share/scheduler/ob_diagnose_config.h index e260911278..83c5d22e9b 100755 --- a/src/share/scheduler/ob_diagnose_config.h +++ b/src/share/scheduler/ob_diagnose_config.h @@ -73,10 +73,10 @@ static constexpr ObDiagnoseInfoStruct OB_SUSPECT_INFO_TYPES[] = { static_assert(sizeof(OB_SUSPECT_INFO_TYPES) / sizeof(ObDiagnoseInfoStruct) == SUSPECT_INFO_TYPE_MAX + 1, "Not enough initializer"); static constexpr ObDiagnoseInfoStruct OB_DAG_WARNING_INFO_TYPES[] = { - {2, true, "DAG_MINI_MERGE", {"ls_id", "tablet_id"}}, - {2, true, "DAG_MINOR_MERGE", {"ls_id", "tablet_id"}}, - {2, true, "DAG_MAJOR_MERGE", {"ls_id", "tablet_id"}}, - {2, true, "DAG_TX_TABLE_MERGE", {"ls_id", "tablet_id"}}, + {3, true, "DAG_MINI_MERGE", {"ls_id", "tablet_id", "compaction_scn"}}, + {3, true, "DAG_MINOR_MERGE", {"ls_id", "tablet_id", "compaction_scn"}}, + {3, true, "DAG_MAJOR_MERGE", {"ls_id", "tablet_id", "compaction_scn"}}, + {3, true, "DAG_TX_TABLE_MERGE", {"ls_id", "tablet_id", "compaction_scn"}}, {0, false, "DAG_WRITE_CKPT", {}}, {3, false, "DAG_TYPE_MDS_TABLE_MERGE", {"ls_id", "tablet_id", "flush_scn"}}, diff --git a/src/storage/compaction/ob_compaction_diagnose.cpp b/src/storage/compaction/ob_compaction_diagnose.cpp index 20f3d4c6ea..8250439a74 100755 --- a/src/storage/compaction/ob_compaction_diagnose.cpp +++ b/src/storage/compaction/ob_compaction_diagnose.cpp @@ -624,6 +624,9 @@ int ObCompactionDiagnoseMgr::diagnose_dag( STORAGE_LOG(WARN, "failed to diagnose dag", K(ret), K(ls_id), K(tablet_id), K(progress)); } } + if (OB_HASH_NOT_EXIST == ret) { + LOG_TRACE("dag not exist", K(ret), K(dag)); + } return ret; } @@ -1162,32 +1165,35 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_medium_merge( table_store_wrapper.get_member()->get_major_sstables().get_boundary_table(true/*last*/))) { } else if (OB_FAIL(tablet.get_max_sync_medium_scn(max_sync_medium_scn))){ LOG_WARN("failed to get max sync medium scn", K(ret), K(ls_id), K(tablet_id)); - } else if (max_sync_medium_scn > last_major_sstable->get_snapshot_version()) { - if (tablet.get_snapshot_version() < max_sync_medium_scn) { // wait mini compaction or tablet freeze - if (ObTimeUtility::current_time_ns() > max_sync_medium_scn + WAIT_MEDIUM_SCHEDULE_INTERVAL * 2 - && can_add_diagnose_info() - && OB_FAIL(SET_DIAGNOSE_INFO( - info_array_[idx_++], + } else { + LOG_TRACE("diagnose tablet medium merge", K(max_sync_medium_scn)); + if (max_sync_medium_scn > last_major_sstable->get_snapshot_version()) { + if (tablet.get_snapshot_version() < max_sync_medium_scn) { // wait mini compaction or tablet freeze + if (ObTimeUtility::current_time_ns() > max_sync_medium_scn + WAIT_MEDIUM_SCHEDULE_INTERVAL * 2 + && can_add_diagnose_info() + && OB_FAIL(SET_DIAGNOSE_INFO( + info_array_[idx_++], + merge_type, + MTL_ID(), + ls_id, + tablet_id, + ObCompactionDiagnoseInfo::DIA_STATUS_NOT_SCHEDULE, + ObTimeUtility::fast_current_time(), + "max_receive_medium_scn", max_sync_medium_scn, + "max_serialized_medium_scn", max_serialized_medium_scn, + "tablet_snapshot", tablet.get_snapshot_version()))) { + LOG_WARN("failed to add diagnose info", K(ret), K(ls_id), K(tablet_id)); + } + } else if (max_sync_medium_scn != compaction_scn) { + ObTabletMajorMergeDag dag; + if (OB_FAIL(diagnose_tablet_merge( + dag, merge_type, - MTL_ID(), ls_id, tablet_id, - ObCompactionDiagnoseInfo::DIA_STATUS_NOT_SCHEDULE, - ObTimeUtility::fast_current_time(), - "max_receive_medium_scn", max_sync_medium_scn, - "max_serialized_medium_scn", max_serialized_medium_scn, - "tablet_snapshot", tablet.get_snapshot_version()))) { - LOG_WARN("failed to add diagnose info", K(ret), K(ls_id), K(tablet_id)); - } - } else if (max_sync_medium_scn != compaction_scn) { - ObTabletMajorMergeDag dag; - if (OB_FAIL(diagnose_tablet_merge( - dag, - merge_type, - ls_id, - tablet_id, - max_sync_medium_scn))) { - LOG_WARN("diagnose failed", K(ret), K(ls_id), K(tablet_id), KPC(last_major_sstable)); + max_sync_medium_scn))) { + LOG_WARN("diagnose failed", K(ret), K(ls_id), K(tablet_id), KPC(last_major_sstable)); + } } } } @@ -1218,6 +1224,7 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_major_merge( } else if (OB_FAIL(tablet.fetch_table_store(table_store_wrapper))) { LOG_WARN("fail to fetch table store", K(ret)); } else { + LOG_TRACE("diagnose tablet major merge", K(max_sync_medium_scn)); ObSSTable *latest_major_sstable = static_cast( table_store_wrapper.get_member()->get_major_sstables().get_boundary_table(true/*last*/)); int tmp_ret = OB_SUCCESS; @@ -1288,6 +1295,7 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_merge( LOG_WARN("failed to add diagnose info", K(ret), K(ls_id), K(tablet_id), K(progress)); } } + LOG_TRACE("dag exist", K(dag), K(progress)); } else if (OB_FAIL(diagnose_no_dag(dag, merge_type, ls_id, tablet_id, compaction_scn))) { LOG_WARN("failed to dagnose no dag", K(ret), K(ls_id), K(tablet_id)); } @@ -1312,6 +1320,7 @@ int ObCompactionDiagnoseMgr::get_suspect_and_warning_info( if (OB_HASH_NOT_EXIST != ret) { LOG_WARN("failed to get suspect info", K(ret), K(ls_id), K(tablet_id)); } else { // no schedule suspect info + LOG_TRACE("suspect info not exist", K(ret), K(dag)); info.info_param_ = nullptr; allocator.reuse(); char tmp_str[common::OB_DAG_WARNING_INFO_LENGTH] = "\0"; @@ -1322,12 +1331,12 @@ int ObCompactionDiagnoseMgr::get_suspect_and_warning_info( LOG_WARN("failed to get dag warning info", K(ret), K(ls_id), K(tablet_id)); } else { // no execute failure ret = OB_SUCCESS; - LOG_DEBUG("may wait for schedule", K(ret), K(ls_id), K(tablet_id)); + LOG_TRACE("may wait for schedule", K(ret), K(ls_id), K(tablet_id)); } } else if (can_add_diagnose_info()) { if (OB_FAIL(warning_info.info_param_->fill_comment(tmp_str, sizeof(tmp_str)))) { STORAGE_LOG(WARN, "failed to fill comment from info param", K(ret)); - }else if (OB_FAIL(SET_DIAGNOSE_INFO( + } else if (OB_FAIL(SET_DIAGNOSE_INFO( info_array_[idx_++], merge_type, MTL_ID(), @@ -1370,7 +1379,7 @@ int ObCompactionDiagnoseMgr::diagnose_no_dag( ret = OB_ERR_UNEXPECTED; LOG_WARN("merge version or freeze ts is invalid", K(ret), K(compaction_scn)); } else { - LOG_INFO("diagnose major", K(ret), K(ls_id), K(tablet_id), K(merge_type)); + LOG_TRACE("diagnose with suspect info", K(ret), K(ls_id), K(tablet_id), K(merge_type)); ObDiagnoseTabletCompProgress progress; ObTabletMiniMergeDag mini_dag; if (OB_FAIL(diagnose_dag(MINI_MERGE, ls_id, tablet_id, ObVersionRange::MIN_VERSION, mini_dag, progress))) { diff --git a/src/storage/compaction/ob_tablet_merge_task.cpp b/src/storage/compaction/ob_tablet_merge_task.cpp index 28cf6e03fe..118b88d709 100755 --- a/src/storage/compaction/ob_tablet_merge_task.cpp +++ b/src/storage/compaction/ob_tablet_merge_task.cpp @@ -405,6 +405,7 @@ int ObBasicTabletMergeDag::fill_info_param(compaction::ObIBasicInfoParam *&out_p if (OB_FAIL(ADD_DAG_WARN_INFO_PARAM(out_param, allocator, get_type(), ls_id_.id(), static_cast(tablet_id_.id()), + param_.merge_version_, "merge_type", merge_type))) { LOG_WARN("failed to fill info param", K(ret)); } diff --git a/unittest/storage/test_dag_warning_history.cpp b/unittest/storage/test_dag_warning_history.cpp index dd45ec247f..e7d9c2ebfb 100644 --- a/unittest/storage/test_dag_warning_history.cpp +++ b/unittest/storage/test_dag_warning_history.cpp @@ -75,7 +75,7 @@ class ObBasicDag : public ObIDag { public: ObBasicDag() : - ObIDag(ObDagType::DAG_TYPE_MINI_MERGE) + ObIDag(ObDagType::DAG_TYPE_TABLET_BACKFILL_TX) {} void init() { is_inited_ = true; } virtual int64_t hash() const {