hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Archive] Add retry and error log print when sender occur encryt/decrypt error

Browse Source
This commit is contained in:
obdev
2024-04-23 13:12:03 +00:00
committed by ob-robot
parent 28bc220a0b
commit 90f04e6d73
2 changed files with 24 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
src/logservice/archiveservice/ob_archive_sender.cpp
View File

@ -421,7 +421,7 @@ bool ObArchiveSender::in_normal_status_(const ArchiveKey &key) const
return round_mgr_->is_in_archive_status(key) || round_mgr_->is_in_suspend_status(key); return round_mgr_->is_in_archive_status(key) || round_mgr_->is_in_suspend_status(key);
} }
ERRSIM_POINT_DEF(ERRSIM_OB_BACKUP_PERMISSION_DENIED); ERRSIM_POINT_DEF(ERRSIM_OB_ARCHIVE_SENDER_ERROR);
// 仅有需要重试的任务返回错误码 // 仅有需要重试的任务返回错误码
void ObArchiveSender::handle(ObArchiveSendTask &task, TaskConsumeStatus &consume_status) void ObArchiveSender::handle(ObArchiveSendTask &task, TaskConsumeStatus &consume_status)
{ {
@ -467,8 +467,8 @@ void ObArchiveSender::handle(ObArchiveSendTask &task, TaskConsumeStatus &consume
} }
} }
if (OB_SUCC(ret) && OB_UNLIKELY(ERRSIM_OB_BACKUP_PERMISSION_DENIED)) { if (OB_SUCC(ret) && OB_UNLIKELY(ERRSIM_OB_ARCHIVE_SENDER_ERROR)) {
ret = ERRSIM_OB_BACKUP_PERMISSION_DENIED; ret = ERRSIM_OB_ARCHIVE_SENDER_ERROR;
} }
if (OB_FAIL(ret)) { if (OB_FAIL(ret)) {
@ -775,17 +775,32 @@ void ObArchiveSender::handle_archive_ret_code_(const ObLSID &id,
// skip it // skip it
} else if (OB_BACKUP_DEVICE_OUT_OF_SPACE == ret_code) { } else if (OB_BACKUP_DEVICE_OUT_OF_SPACE == ret_code) {
// ret code should report to user // ret code should report to user
if (REACH_TIME_INTERVAL(10 * 1000 * 1000L)) { if (REACH_TIME_INTERVAL(ARCHIVE_DBA_ERROR_LOG_PRINT_INTERVAL)) {
LOG_DBA_ERROR(OB_BACKUP_DEVICE_OUT_OF_SPACE, "msg", "archive device is full", "ret", ret_code, LOG_DBA_ERROR(OB_BACKUP_DEVICE_OUT_OF_SPACE, "msg", "archive device is full", "ret", ret_code,
"archive_dest_id", key.dest_id_, "archive_dest_id", key.dest_id_,
"archive_round", key.round_); "archive_round", key.round_);
} }
} else if (OB_BACKUP_PERMISSION_DENIED == ret_code) { } else if (OB_BACKUP_PERMISSION_DENIED == ret_code) {
if (REACH_TIME_INTERVAL(10 * 1000 * 1000L)) { // backup permission denied
if (REACH_TIME_INTERVAL(ARCHIVE_DBA_ERROR_LOG_PRINT_INTERVAL)) {
LOG_DBA_ERROR(OB_BACKUP_PERMISSION_DENIED, "msg", "archive dest permission denied", "ret", ret_code, LOG_DBA_ERROR(OB_BACKUP_PERMISSION_DENIED, "msg", "archive dest permission denied", "ret", ret_code,
"archive_dest_id", key.dest_id_, "archive_dest_id", key.dest_id_,
"archive_round", key.round_); "archive_round", key.round_);
} }
} else if (OB_ERR_AES_ENCRYPT == ret_code) {
// archive dest encrypt failed
if (REACH_TIME_INTERVAL(ARCHIVE_DBA_ERROR_LOG_PRINT_INTERVAL)) {
LOG_DBA_ERROR(OB_ERR_AES_ENCRYPT, "msg", "archive dest encrypt failed", "ret", ret_code,
"archive_dest_id", key.dest_id_,
"archive_round", key.round_);
}
} else if (OB_ERR_AES_DECRYPT == ret_code) {
// archive desc decrypt failed
if (REACH_TIME_INTERVAL(ARCHIVE_DBA_ERROR_LOG_PRINT_INTERVAL)) {
LOG_DBA_ERROR(OB_ERR_AES_DECRYPT, "msg", "archive dest decrypt failed", "ret", ret_code,
"archive_dest_id", key.dest_id_,
"archive_round", key.round_);
}
} else if (is_ignore_ret_code_(ret_code)) { } else if (is_ignore_ret_code_(ret_code)) {
} else { } else {
ARCHIVE_LOG(ERROR, "archive sender encounter fatal error", K(ret), K(id), K(key), K(ret_code)); ARCHIVE_LOG(ERROR, "archive sender encounter fatal error", K(ret), K(id), K(key), K(ret_code));
@ -803,7 +818,9 @@ bool ObArchiveSender::is_retry_ret_code_(const int ret_code) const
|| OB_BACKUP_DEVICE_OUT_OF_SPACE == ret_code || OB_BACKUP_DEVICE_OUT_OF_SPACE == ret_code
|| OB_BACKUP_PWRITE_OFFSET_NOT_MATCH == ret_code || OB_BACKUP_PWRITE_OFFSET_NOT_MATCH == ret_code
|| OB_IO_LIMIT == ret_code || OB_IO_LIMIT == ret_code
|| OB_BACKUP_PERMISSION_DENIED == ret_code; || OB_BACKUP_PERMISSION_DENIED == ret_code
|| OB_ERR_AES_ENCRYPT == ret_code
|| OB_ERR_AES_DECRYPT == ret_code;
} }
bool ObArchiveSender::is_ignore_ret_code_(const int ret_code) const bool ObArchiveSender::is_ignore_ret_code_(const int ret_code) const

1
src/logservice/archiveservice/ob_archive_sender.h
View File

@ -52,6 +52,7 @@ class ObArchiveSender : public share::ObThreadPool, public ObArchiveWorker
{ {
static const int64_t MAX_SEND_NUM = 10; static const int64_t MAX_SEND_NUM = 10;
static const int64_t MAX_ARCHIVE_TASK_STATUS_POP_TIMEOUT = 5 * 1000 * 1000L; static const int64_t MAX_ARCHIVE_TASK_STATUS_POP_TIMEOUT = 5 * 1000 * 1000L;
static const int64_t ARCHIVE_DBA_ERROR_LOG_PRINT_INTERVAL = 10 * 1000 * 1000L; // dba error log print interval
public: public:
ObArchiveSender(); ObArchiveSender();
virtual ~ObArchiveSender(); virtual ~ObArchiveSender();