hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] log4200

Browse Source 
Co-authored-by: zhjc1124 <zhjc1124@gmail.com>
Co-authored-by: BinChenn <binchenn.bc@gmail.com>
Co-authored-by: oceanoverflow <oceanoverflow@gmail.com>
This commit is contained in:
obdev
2023-05-06 08:15:43 +00:00
committed by ob-robot
parent f0fdf277f1
commit d6c6e05727
450 changed files with 33015 additions and 2243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

180
src/storage/backup/ob_backup_operator.cpp
View File

@ -279,6 +279,45 @@ int ObLSBackupOperator::report_ls_task_finish(const uint64_t tenant_id, const in
return ret;
}
int ObLSBackupOperator::get_all_backup_ls_id(const uint64_t tenant_id, const int64_t task_id,
common::ObIArray<share::ObLSID> &ls_array, common::ObISQLClient &sql_client)
{
int ret = OB_SUCCESS;
ObSqlString sql;
if (OB_INVALID_ID == tenant_id || task_id < 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid args", K(ret), K(task_id), K(tenant_id), K(ls_id));
} else if (OB_FAIL(construct_query_backup_sql_(tenant_id, task_id, sql))) {
LOG_WARN("failed to construct query backup sql", K(ret), K(tenant_id), K(task_id));
} else if (OB_FAIL(get_distinct_ls_id_(tenant_id, sql, ls_array, sql_client))) {
LOG_WARN("failed to get distinct ls id", K(ret), K(tenant_id), K(sql));
}
return ret;
}
int ObLSBackupOperator::get_all_archive_ls_id(const uint64_t tenant_id, const int64_t dest_id,
const share::SCN &start_scn, const share::SCN &end_scn, common::ObIArray<share::ObLSID> &ls_array,
common::ObISQLClient &sql_client)
{
int ret = OB_SUCCESS;
ObSqlString sql;
int64_t start_piece_id = 0;
int64_t end_piece_id = 0;
if (OB_INVALID_ID == tenant_id || task_id < 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid args", K(ret), K(task_id), K(tenant_id), K(ls_id));
} else if (OB_FAIL(get_start_piece_id_(tenant_id, dest_id, start_scn, sql_client, start_piece_id))) {
LOG_WARN("failed to get start piece id", K(ret), K(tenant_id), K(dest_id), K(start_scn));
} else if (OB_FAIL(get_end_piece_id_(tenant_id, dest_id, end_scn, sql_client, end_piece_id))) {
LOG_WARN("failed to get end piece id", K(ret), K(tenant_id), K(dest_id), K(end_scn));
} else if (OB_FAIL(construct_query_archive_sql_(tenant_id, dest_id, start_piece_id, end_piece_id, sql))) {
LOG_WARN("failed to construct query archive sql", K(ret), K(tenant_id), K(dest_id), K(start_piece_id), K(end_piece_id));
} else if (OB_FAIL(get_distinct_ls_id_(tenant_id, sql, ls_array, sql_client))) {
LOG_WARN("failed to get distinct ls id", K(ret), K(tenant_id), K(sql));
}
return ret;
}
int ObLSBackupOperator::report_tablet_skipped(
const uint64_t tenant_id, const ObBackupSkippedTablet &skipped_tablet, common::ObISQLClient &sql_client)
{
@ -411,5 +450,146 @@ int ObLSBackupOperator::fill_backup_skipped_tablet_(const ObBackupSkippedTablet
return ret;
}
int ObLSBackupOperator::get_distinct_ls_id_(const uint64_t tenant_id, const common::ObSqlString &sql,
common::ObIArray<share::ObLSID> &ls_array, common::ObISQLClient &sql_client)
{
int ret = OB_SUCCESS;
SMART_VAR(ObMySQLProxy::MySQLResult, res) {
common::sqlclient::ObMySQLResult *result = NULL;
if (OB_FAIL(sql_client.read(res, gen_meta_tenant_id(tenant_id), sql.ptr()))) {
LOG_WARN("failed to exec sql", K(ret), K(sql), K(tenant_id));
} else if (OB_ISNULL(result = res.get_result())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("result set from read is NULL", K(ret));
} else {/*do nothing*/}
while (OB_SUCC(ret) && OB_SUCC(result->next())) {
int64_t ls_id = 0;
EXTRACT_INT_FIELD_MYSQL(*result, "ls_id", ls_id, int64_t);
if (OB_FAIL(ls_array.push_back(ObLSID(ls_id)))) {
LOG_WARN("failed to push back ls id", K(ret), K(ls_id));
}
}
if (OB_LIKELY(OB_ITER_END == ret)) {
ret = OB_SUCCESS;
}
}
return ret;
}
int ObLSBackupOperator::get_piece_id_(const uint64_t tenant_id, const common::ObSqlString &sql,
int64_t &piece_id, common::ObISQLClient &sql_client)
{
int ret = OB_SUCCESS;
ObArray<int64_t> piece_array;
SMART_VAR(ObMySQLProxy::MySQLResult, res) {
common::sqlclient::ObMySQLResult *result = NULL;
if (OB_FAIL(sql_client.read(res, gen_meta_tenant_id(tenant_id), sql.ptr()))) {
LOG_WARN("failed to exec sql", K(ret), K(sql), K(tenant_id));
} else if (OB_ISNULL(result = res.get_result())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("result set from read is NULL", K(ret));
} else {/*do nothing*/}
while (OB_SUCC(ret) && OB_SUCC(result->next())) {
int64_t piece_id = 0;
EXTRACT_INT_FIELD_MYSQL(*result, "piece_id", piece_id, int64_t);
if (OB_FAIL(piece_array.push_back(piece_id))) {
LOG_WARN("failed to push back", K(piece_id), K(ret));
}
}
if (OB_LIKELY(OB_ITER_END == ret)) {
ret = OB_SUCCESS;
}
if (OB_SUCC(ret)) {
if (piece_array.empty()) {
ret = OB_ENTRY_NOT_EXIST;
LOG_WARN("piece array empty", K(ret));
} else if (piece_array.count() > 1) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("piece array count not correct", K(ret), K(piece_array));
} else {
piece_id = piece_array.at(0);
}
}
}
return ret;
}
// TODO(yangyi.yyy): currently, __all_ls_log_archive_progress is not cleaned,
// fix later if __all_ls_log_archive_progress is cleaned
int ObLSBackupOperator::get_start_piece_id_(const uint64_t tenant_id, const uint64_t dest_id,
const share::SCN &start_scn, common::ObISQLClient &sql_client, int64_t &start_piece_id)
{
int ret = OB_SUCCESS;
ObSqlString sql;
const char *sql_str = "SELECT piece_id "
"FROM %s "
"WHERE tenant_id = %lu AND dest_id = %ld "
"AND start_scn <= %ld ORDER BY piece_id DESC LIMIT 1";
if (OB_INVALID_ID == tenant_id || dest_id < 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid args", K(ret), K(dest_id), K(tenant_id), K(ls_id));
} else if (OB_FAIL(sql.append_fmt(sql_str, OB_ALL_LS_LOG_ARCHIVE_PROGRESS_TNAME,
tenant_id, dest_id, start_scn.get_val_for_inner_table_field()))) {
LOG_WARN("failed to append sql", K(ret), K(sql_str), K(tenant_id), K(dest_id));
} else if (OB_FAIL(get_piece_id_(tenant_id, sql, start_piece_id, sql_client))) {
LOG_WARN("failed to get piece id", K(ret), K(tenant_id), K(sql));
}
return ret;
}
int ObLSBackupOperator::get_end_piece_id_(const uint64_t tenant_id, const uint64_t dest_id,
const share::SCN &checkpoint_scn, common::ObISQLClient &sql_client, int64_t &end_piece_id)
{
int ret = OB_SUCCESS;
ObSqlString sql;
const char *sql_str = "SELECT piece_id "
"FROM %s "
"WHERE tenant_id = %lu AND dest_id = %ld "
"AND checkpoint_scn >= %ld ORDER BY piece_id ASC LIMIT 1";
if (OB_INVALID_ID == tenant_id || dest_id < 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid args", K(ret), K(dest_id), K(tenant_id), K(ls_id));
} else if (OB_FAIL(sql.append_fmt(sql_str, OB_ALL_LS_LOG_ARCHIVE_PROGRESS_TNAME,
tenant_id, dest_id, checkpoint_scn.get_val_for_inner_table_field()))) {
LOG_WARN("failed to append sql", K(ret), K(sql_str), K(tenant_id), K(dest_id));
} else if (OB_FAIL(get_piece_id_(tenant_id, sql, end_piece_id, sql_client))) {
LOG_WARN("failed to get piece id", K(ret), K(tenant_id), K(sql));
}
return ret;
}
int ObLSBackupOperator::construct_query_backup_sql_(const uint64_t tenant_id, const int64_t task_id, common::ObSqlString &sql)
{
int ret = OB_SUCCESS;
const char *sql_str = "SELECT DISTINCT ls_id "
"FROM %s "
"WHERE tenant_id = %lu and task_id = %ld";
if (OB_FAIL(sql.append_fmt(sql_str, OB_ALL_BACKUP_LS_TASK_TNAME, tenant_id, task_id))) {
LOG_WARN("failed to append sql", K(ret), K(sql_str), K(tenant_id), K(task_id));
}
return ret;
}
int ObLSBackupOperator::construct_query_archive_sql_(const uint64_t tenant_id, const int64_t dest_id,
const int64_t start_piece_id, const int64_t end_piece_id, common::ObSqlString &sql)
{
int ret = OB_SUCCESS;
const char *sql_str = "SELECT DISTINCT ls_id "
"FROM %s "
"WHERE tenant_id = %lu AND dest_id = %ld "
"AND piece_id >= %ld AND piece_id <= %ld";
if (OB_FAIL(sql.append_fmt(sql_str, OB_ALL_LS_LOG_ARCHIVE_PROGRESS_TNAME,
tenant_id, dest_id, start_piece_id, end_piece_id))) {
LOG_WARN("failed to append sql", K(ret), K(sql_str), K(tenant_id),
K(dest_id), K(start_piece_id), K(end_piece_id));
}
return ret;
}
} // namespace backup
} // namespace oceanbase

16
src/storage/backup/ob_backup_operator.h
View File

@ -46,6 +46,12 @@ public:
// __all_backup_ls_task
static int report_ls_task_finish(const uint64_t tenant_id, const int64_t task_id, const share::ObLSID &ls_id,
const int64_t turn_id, const int64_t retry_id, const int64_t result, common::ObISQLClient &sql_client);
static int get_all_backup_ls_id(const uint64_t tenant_id, const int64_t task_id,
common::ObIArray<share::ObLSID> &ls_array, common::ObISQLClient &sql_client);
static int get_all_archive_ls_id(const uint64_t tenant_id, const int64_t dest_id,
const share::SCN &start_scn, const share::SCN &end_scn, common::ObIArray<share::ObLSID> &ls_array,
common::ObISQLClient &sql_client);
// __all_backup_skipped_tablet
static int report_tablet_skipped(
const uint64_t tenant_id, const ObBackupSkippedTablet &skipped_tablet, common::ObISQLClient &sql_client);
@ -55,6 +61,16 @@ private:
static int parse_ls_task_info_results_(
sqlclient::ObMySQLResult &result, common::ObIArray<ObBackupLSTaskInfo> &task_info);
static int fill_backup_skipped_tablet_(const ObBackupSkippedTablet &skipped_tablet, share::ObDMLSqlSplicer &splicer);
static int get_distinct_ls_id_(const uint64_t tenant_id, const common::ObSqlString &sql,
common::ObIArray<share::ObLSID> &ls_array, common::ObISQLClient &sql_client);
static int get_piece_id_(const uint64_t tenant_id, const common::ObSqlString &sql, int64_t &piece_id, common::ObISQLClient &sql_client);
static int get_start_piece_id_(const uint64_t tenant_id, const uint64_t dest_id,
const share::SCN &start_scn, common::ObISQLClient &sql_client, int64_t &start_piece_id);
static int get_end_piece_id_(const uint64_t tenant_id, const uint64_t dest_id, const share::SCN &end_scn,
common::ObISQLClient &sql_client, int64_t &end_piece_id);
static int construct_query_backup_sql_(const uint64_t tenant_id, const int64_t task_id, common::ObSqlString &sql);
static int construct_query_archive_sql_(const uint64_t tenant_id, const int64_t dest_id, const int64_t start_piece_id,
const int64_t end_piece_id, common::ObSqlString &sql);
};
} // namespace backup

817
src/storage/backup/ob_backup_task.cpp
View File

File diff suppressed because it is too large Load Diff

68
src/storage/backup/ob_backup_task.h
View File

@ -32,6 +32,7 @@
#include "storage/ls/ob_ls_tablet_service.h"
#include "share/backup/ob_backup_data_store.h"
#include "storage/blocksstable/ob_logic_macro_id.h"
#include "share/backup/ob_archive_store.h"
namespace oceanbase {
namespace share
@ -395,7 +396,7 @@ public:
ObLSBackupComplementLogDag();
virtual ~ObLSBackupComplementLogDag();
int init(const ObBackupJobDesc &job_desc, const share::ObBackupDest &backup_dest, const uint64_t tenant_id,
const share::ObBackupSetDesc &backup_set_desc, const share::ObLSID &ls_id, const int64_t turn_id,
const int64_t dest_id, const share::ObBackupSetDesc &backup_set_desc, const share::ObLSID &ls_id, const int64_t turn_id,
const int64_t retry_id, const share::SCN &start_scn, const share::SCN &end_scn, const ObBackupReportCtx &report_ctx);
virtual int create_first_task() override;
virtual int fill_comment(char *buf, const int64_t buf_len) const override;
@ -410,6 +411,7 @@ private:
ObBackupJobDesc job_desc_;
share::ObBackupDest backup_dest_;
uint64_t tenant_id_;
int64_t dest_id_;
share::ObBackupSetDesc backup_set_desc_;
share::ObLSID ls_id_;
int64_t turn_id_;
@ -657,14 +659,16 @@ class ObLSBackupComplementLogTask : public share::ObITask {
struct BackupPieceFile {
BackupPieceFile();
void reset();
int set(const uint64_t tenant_id, const int64_t round_id, const int64_t piece_id, const int64_t ls_id,
const int64_t file_id);
TO_STRING_KV(K_(tenant_id), K_(round_id), K_(piece_id), K_(ls_id), K_(file_id));
uint64_t tenant_id_;
int set(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, const share::ObLSID &ls_id,
const int64_t file_id, const share::SCN &start_scn, const ObBackupPathString &path);
TO_STRING_KV(K_(dest_id), K_(round_id), K_(piece_id), K_(ls_id), K_(file_id), K_(path));
int64_t dest_id_;
int64_t round_id_;
int64_t piece_id_;
int64_t ls_id_;
share::ObLSID ls_id_;
int64_t file_id_;
share::SCN start_scn_;
ObBackupPathString path_;
};
class BackupPieceOp : public ObBaseDirEntryOperator {
@ -674,9 +678,6 @@ class ObLSBackupComplementLogTask : public share::ObITask {
int get_file_id_list(common::ObIArray<int64_t> &files) const;
TO_STRING_KV(K_(file_id_list));
private:
int parse_log_file_id_(const common::ObString &file_name, int64_t &piece_id);
private:
ObArray<int64_t> file_id_list_;
};
@ -688,43 +689,66 @@ class ObLSBackupComplementLogTask : public share::ObITask {
public:
ObLSBackupComplementLogTask();
virtual ~ObLSBackupComplementLogTask();
int init(const ObBackupJobDesc &job_desc, const share::ObBackupDest &backup_dest, const uint64_t tenant_id,
int init(const ObBackupJobDesc &job_desc, const share::ObBackupDest &backup_dest, const uint64_t tenant_id, const int64_t dest_id,
const share::ObBackupSetDesc &backup_set_desc, const share::ObLSID &ls_id, const share::SCN &start_scn,
const share::SCN &end_scn, const int64_t turn_id, const int64_t retry_id, const ObBackupReportCtx &report_ctx);
virtual int process() override;
private:
int get_newly_created_ls_in_piece_(const int64_t dest_id, const uint64_t tenant_id,
const share::SCN &start_scn, const share::SCN &end_scn, common::ObIArray<share::ObLSID> &ls_array);
int inner_process_(const int64_t archive_dest_id, const share::ObLSID &ls_id);
int get_complement_log_dir_path_(share::ObBackupPath &backup_path);
int calc_backup_file_range_(common::ObIArray<BackupPieceFile> &file_list);
int check_pieces_continue_(const common::ObIArray<share::ObTenantArchivePieceAttr> &rounds);
int get_piece_id_by_ts_(const uint64_t tenant_id, const share::SCN &scn, int64_t &piece_id);
int get_all_pieces_(const uint64_t tenant_id, const int64_t start_piece_id, const int64_t end_piece_id,
int write_format_file_();
int generate_format_desc_(share::ObBackupFormatDesc &format_desc);
int calc_backup_file_range_(const int64_t dest_id, const share::ObLSID &ls_id, common::ObIArray<BackupPieceFile> &file_list);
int get_active_round_dest_id_(const uint64_t tenant_id, int64_t &dest_id);
int get_piece_id_by_scn_(const uint64_t tenant_id, const int64_t dest_id, const share::SCN &scn, int64_t &piece_id);
int get_all_pieces_(const uint64_t tenant_id, const int64_t dest_id, const int64_t start_piece_id, const int64_t end_piece_id,
common::ObArray<share::ObTenantArchivePieceAttr> &piece_list);
int get_all_piece_file_list_(const uint64_t tenant_id,
int wait_pieces_frozen_(const common::ObArray<share::ObTenantArchivePieceAttr> &piece_list);
int wait_piece_frozen_(const share::ObTenantArchivePieceAttr &piece);
int check_piece_frozen_(const share::ObTenantArchivePieceAttr &piece, bool &is_frozen);
int get_all_piece_file_list_(const uint64_t tenant_id, const share::ObLSID &ls_id,
const common::ObIArray<share::ObTenantArchivePieceAttr> &piece_list, const share::SCN &start_scn, const share::SCN &end_scn,
common::ObIArray<BackupPieceFile> &piece_file_list);
int inner_get_piece_file_list_(
const int64_t round_id, const int64_t piece_id, common::ObIArray<BackupPieceFile> &piece_file_list);
int locate_archive_file_id_by_ts_(const uint64_t tenant_id, const int64_t round_id, const int64_t piece_id,
const share::SCN &scn, const bool is_upper_bound, int64_t &file_id);
int inner_get_piece_file_list_(const share::ObLSID &ls_id, const ObTenantArchivePieceAttr &piece_attr, common::ObIArray<BackupPieceFile> &piece_file_list);
int locate_archive_file_id_by_scn_(const ObTenantArchivePieceAttr &piece_attr, const share::ObLSID &ls_id, const SCN &scn, int64_t &file_id);
int get_file_in_between_(const int64_t start_file_id, const int64_t end_file_id, common::ObIArray<BackupPieceFile> &list);
int filter_file_id_smaller_than_(const int64_t file_id, common::ObIArray<BackupPieceFile> &list);
int filter_file_id_larger_than_(const int64_t file_id, common::ObIArray<BackupPieceFile> &list);
int get_src_backup_piece_dir_(const int64_t round_id, const int64_t piece_id, share::ObBackupPath &backup_path);
int get_src_backup_piece_dir_(const share::ObLSID &ls_id, const ObTenantArchivePieceAttr &piece_attr, share::ObBackupPath &backup_path);
int get_src_backup_file_path_(const BackupPieceFile &piece_file, share::ObBackupPath &backup_path);
int get_dst_backup_file_path_(const BackupPieceFile &piece_file, share::ObBackupPath &backup_path);
int backup_complement_log_(const common::ObIArray<BackupPieceFile> &path);
int inner_backup_complement_log_(const share::ObBackupPath &src_path, const share::ObBackupPath &dst_path);
int transfer_clog_file_(const share::ObBackupPath &src_path, const share::ObBackupPath &dst_path);
int inner_transfer_clog_file_(const share::ObBackupPath &src_path, const share::ObBackupPath &dst_path);
int inner_transfer_clog_file_(const share::ObBackupPath &src_path, const share::ObBackupPath &dst_path, int64_t &transfer_len);
int get_transfer_length_(const int64_t delta_len, int64_t &transfer_len);
int get_file_length_(const common::ObString &path, const share::ObBackupStorageInfo *storage_info, int64_t &length);
int post_rpc_result_(const int64_t result);
int get_copy_src_and_dest_(const BackupPieceFile &piece_file, share::ObBackupDest &src, share::ObBackupDest &dest);
int transform_and_copy_meta_file_(const BackupPieceFile &piece_file);
// ls_file_info
int copy_ls_file_info_(const BackupPieceFile &piece_file, const share::ObArchiveStore &src_store, const share::ObArchiveStore &dest_store);
// piece_file_info
int copy_piece_file_info_(const BackupPieceFile &piece_file, const share::ObArchiveStore &src_store, const share::ObArchiveStore &dest_store);
// single_piece_info
int copy_single_piece_info_(const BackupPieceFile &piece_file, const share::ObArchiveStore &src_store, const share::ObArchiveStore &dest_store);
// tenant_archive_piece_infos
int copy_tenant_archive_piece_infos(const BackupPieceFile &piece_file, const share::ObArchiveStore &src_store, const share::ObArchiveStore &dest_store);
// checkpoint_info
int copy_checkpoint_info(const BackupPieceFile &piece_file, const share::ObArchiveStore &src_store, const share::ObArchiveStore &dest_store);
// round_start
int copy_round_start_file(const BackupPieceFile &piece_file, const share::ObArchiveStore &src_store, const share::ObArchiveStore &dest_store);
// piece_start
int copy_piece_start_file(const BackupPieceFile &piece_file, const share::ObBackupDest &src, const share::ObBackupDest &dest);
private:
bool is_inited_;
ObBackupJobDesc job_desc_;
share::ObBackupDest backup_dest_;
uint64_t tenant_id_;
int64_t dest_id_;
share::ObBackupSetDesc backup_set_desc_;
share::ObLSID ls_id_;
share::SCN compl_start_scn_;