hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] Backup Table Names

Browse Source 
Co-authored-by: wxhwang <wxhwang@126.com>
This commit is contained in:
LoLolobster 2024-04-11 12:45:22 +00:00 committed by ob-robot
parent 3c0776dbd6
commit 41b19ba941
16 changed files with 1406 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/rootserver/CMakeLists.txt
View File

@ -20,6 +20,7 @@ ob_set_subtarget(ob_rootserver backup
backup/ob_backup_data_set_task_mgr.cpp
backup/ob_backup_proxy.cpp
backup/ob_backup_data_ls_task_mgr.cpp
backup/ob_backup_table_list_mgr.cpp
)
ob_set_subtarget(ob_rootserver common

77
src/rootserver/backup/ob_backup_data_set_task_mgr.cpp
View File

@ -36,6 +36,7 @@
#include "share/ob_tenant_info_proxy.h"
#include "observer/ob_inner_sql_connection.h"
#include "share/backup/ob_backup_server_mgr.h"
#include "rootserver/backup/ob_backup_table_list_mgr.h"
using namespace oceanbase;
using namespace omt;
@ -1088,6 +1089,41 @@ int ObBackupSetTaskMgr::backup_data_()
return ret;
}
int ObBackupSetTaskMgr::get_backup_end_scn_(share::SCN &end_scn) const
{
int ret = OB_SUCCESS;
const uint64_t tenant_id = job_attr_->tenant_id_;
const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id);
ObAllTenantInfo tenant_info;
if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id, sql_proxy_, false/*for update*/, tenant_info))) {
LOG_WARN("failed to get tenant info", K(ret), K(tenant_id));
} else if (OB_FAIL(ObBackupDataScheduler::get_backup_scn(*sql_proxy_, tenant_id, false/*is backup start*/, end_scn))) {
LOG_WARN("failed to get end scn", K(ret), K(tenant_id));
} else if (tenant_info.is_standby() && end_scn > tenant_info.get_standby_scn()) {
// For standby tenant, make sure snapshot of end_scn is readable. Otherwise, we
// can not backup table list.
int64_t abs_timeout = ObTimeUtility::current_time() + 10 * 60 * 1000 * 1000;
while (OB_SUCC(ret)) {
sleep(1); // sleep 1s
if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id, sql_proxy_, false/*for update*/, tenant_info))) {
LOG_WARN("failed to get tenant info", K(ret), K(tenant_id));
} else if (!tenant_info.is_standby()) {
ret = OB_STATE_NOT_MATCH;
LOG_WARN("tenant is not standby", K(ret), K(tenant_info));
} else if (end_scn <= tenant_info.get_standby_scn()) {
break;
} else if (ObTimeUtility::current_time() > abs_timeout) {
ret = OB_TIMEOUT;
LOG_WARN("failed to wait readable scn", K(ret), K(tenant_id));
} else if (OB_FAIL(backup_service_->check_leader())) {
LOG_WARN("failed to check leader", K(ret));
}
}
}
return ret;
}
int ObBackupSetTaskMgr::backup_data_finish_(
const ObIArray<share::ObBackupLSTaskAttr> &ls_tasks,
const ObBackupLSTaskAttr &build_index_attr)
@ -1095,7 +1131,12 @@ int ObBackupSetTaskMgr::backup_data_finish_(
int ret = OB_SUCCESS;
share::ObBackupStatus next_status;
SCN end_scn = SCN::min_scn();
if (OB_FAIL(trans_.start(sql_proxy_, meta_tenant_id_))) {
if (OB_FAIL(get_backup_end_scn_(end_scn))) {
LOG_WARN("failed to get backup end scn", K(ret), K_(job_attr));
} else if (ObBackupStatus::Status::BACKUP_DATA_MAJOR == set_task_attr_.status_.status_
&& OB_FAIL(write_table_list_(end_scn))) {
LOG_WARN("[DATA_BACKUP] fail to write table list", K(ret), "tenant_id", job_attr_->tenant_id_);
} else if (OB_FAIL(trans_.start(sql_proxy_, meta_tenant_id_))) {
LOG_WARN("fail to start trans", K(ret));
} else if (OB_FAIL(ObBackupLSTaskOperator::delete_build_index_task(trans_, build_index_attr))) {
LOG_WARN("[DATA_BACKUP]failed to delete build index task", K(ret));
@ -1105,10 +1146,7 @@ int ObBackupSetTaskMgr::backup_data_finish_(
} else if (OB_FAIL(convert_task_type_(ls_tasks))) {
LOG_WARN("[DATA_BACKUP]failed to update task type to PLUS_ARCHIVE_LOG", K(ret), K(ls_tasks));
}
if (FAILEDx(ObBackupDataScheduler::get_backup_scn(*sql_proxy_, job_attr_->tenant_id_, false/*end scn*/, end_scn))) {
LOG_WARN("[DATA_BACKUP]failed to get end ts", K(ret), "tenant_id", job_attr_->tenant_id_);
} else if (OB_FAIL(advance_status_(trans_, next_status, OB_SUCCESS, end_scn))) {
if (FAILEDx(advance_status_(trans_, next_status, OB_SUCCESS, end_scn))) {
LOG_WARN("[DATA_BACKUP]failed to update set task status to COMPLETEING", K(ret), K(set_task_attr_));
}
if (trans_.is_started()) {
@ -2041,6 +2079,35 @@ int ObBackupSetTaskMgr::write_log_format_file_()
return ret;
}
int ObBackupSetTaskMgr::write_table_list_(const share::SCN &end_scn)
{
int ret = OB_SUCCESS;
ObBackupTableListMgr table_list_mgr;
ObBackupDest backup_tenant_dest;
ObBackupDest backup_set_dest;
share::ObBackupSetDesc desc;
desc.backup_set_id_ = job_attr_->backup_set_id_;
desc.backup_type_ = job_attr_->backup_type_;
if (OB_FAIL(ObBackupStorageInfoOperator::get_backup_dest(*sql_proxy_, job_attr_->tenant_id_,
set_task_attr_.backup_path_, backup_tenant_dest))) {
LOG_WARN("fail to get backup dest", K(ret), K_(job_attr));
} else if (OB_FAIL(ObBackupPathUtil::construct_backup_set_dest(backup_tenant_dest, desc, backup_set_dest))) {
LOG_WARN("fail to construct backup set dest", K(ret), K(backup_tenant_dest), K(desc));
} else if (OB_FAIL(table_list_mgr.init(job_attr_->tenant_id_, end_scn, backup_set_dest, *backup_service_, *sql_proxy_))) {
LOG_WARN("fail to init table_list_mgr", KPC(job_attr_));
} else if (OB_FAIL(table_list_mgr.backup_table_list())) {
LOG_WARN("fail to backup table list", K(ret), KPC(job_attr_));
}
if (OB_FAIL(ret)) {
LOG_WARN("[DATA_BACKUP]fail to write table list", K(ret), K(end_scn));
} else if (OB_FAIL(backup_service_->check_leader())) {
LOG_WARN("[DATA_BACKUP]failed to check leader", K(ret));
}
return ret;
}
int ObBackupSetTaskMgr::write_backup_set_info_(
const ObBackupSetTaskAttr &set_task_attr,
ObExternBackupSetInfoDesc &backup_set_info)

2
src/rootserver/backup/ob_backup_data_set_task_mgr.h
View File

@ -113,6 +113,7 @@ private:
int do_cancel_();
int do_failed_ls_task_(ObMySQLTransaction &trans, const ObIArray<share::ObBackupLSTaskAttr> &ls_task);
int write_backup_set_placeholder_(const bool is_start);
int write_table_list_(const share::SCN &end_scn);
int write_extern_infos_();
int write_tenant_backup_set_infos_();
int write_extern_locality_info_(storage::ObExternTenantLocalityInfoDesc &locality_info);
@ -129,6 +130,7 @@ private:
int advance_status_(ObMySQLTransaction &trans, const share::ObBackupStatus &next_status, const int result = OB_SUCCESS,
const share::SCN &scn = share::SCN::min_scn(), const int64_t end_ts = 0);
int get_next_status_(const share::ObBackupStatus &cur_status, share::ObBackupStatus &next_status);
int get_backup_end_scn_(share::SCN &end_scn) const;
private:
bool is_inited_;
uint64_t meta_tenant_id_;

453
src/rootserver/backup/ob_backup_table_list_mgr.cpp Normal file
View File

@ -0,0 +1,453 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX RS
#include "ob_backup_table_list_mgr.h"
#include "storage/backup/ob_backup_data_store.h"
#include "share/backup/ob_backup_io_adapter.h"
namespace oceanbase
{
using namespace oceanbase;
using namespace storage;
using namespace common;
namespace rootserver
{
bool ObGetMaxTableListPartNoOp::is_valid() const
{
return scn_.is_valid() && max_table_list_part_no_ >= 0;
}
int ObGetMaxTableListPartNoOp::func(const dirent *entry)
{
int ret = OB_SUCCESS;
char prefix[OB_MAX_FILE_NAME_LENGTH] = { 0 };
if (OB_ISNULL(entry)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid list entry, entry is null", K(ret));
} else if (OB_ISNULL(entry->d_name) || !is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid list entry, d_name is null", K(ret));
} else if (OB_FAIL(databuff_printf(prefix, OB_MAX_FILE_NAME_LENGTH, "%s.%lu",
OB_STR_TABLE_LIST, scn_.get_val_for_inner_table_field()))) {
LOG_WARN("fail to databuff print", K(ret), K_(scn));
} else if (0 == strncmp(entry->d_name, prefix, strlen(prefix))) {
int64_t part_no = 0;
if (OB_FAIL(ObBackupPath::parse_partial_table_list_file_name(entry->d_name, scn_, part_no))) {
LOG_WARN("failed to get table list part_no", K(ret), KP(entry->d_name));
} else if (part_no > max_table_list_part_no_) {
max_table_list_part_no_ = part_no;
}
}
return ret;
}
bool ObGetMaxTableListSCNOp::is_valid() const
{
return max_scn_.is_valid();
}
int ObGetMaxTableListSCNOp::func(const dirent *entry)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(entry)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid list entry, entry is null", K(ret));
} else if (OB_ISNULL(entry->d_name) || !is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid list entry, d_name is null", K(ret));
} else if (OB_NOT_NULL(strstr(entry->d_name, OB_STR_TABLE_LIST_META_INFO))) {
share::SCN scn = share::SCN::min_scn();
if (OB_FAIL(ObBackupPath::parse_table_list_meta_file_name(entry->d_name, scn))) {
LOG_WARN("failed to get table list scn", K(ret), KP(entry->d_name));
} else if (scn > max_scn_) {
max_scn_ = scn;
}
}
return ret;
}
bool ObGetTableListPartialMetasOp::is_valid() const
{
return scn_.is_valid() && backup_set_dest_->is_valid();
}
int ObGetTableListPartialMetasOp::func(const dirent *entry)
{
int ret = OB_SUCCESS;
char prefix[OB_MAX_FILE_NAME_LENGTH] = { 0 };
if (OB_ISNULL(entry)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid list entry, entry is null", K(ret));
} else if (OB_ISNULL(entry->d_name) || !is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid list entry, d_name is null", K(ret));
} else if (OB_FAIL(databuff_printf(prefix, OB_MAX_FILE_NAME_LENGTH, "%s.%lu",
OB_STR_TABLE_LIST, scn_.get_val_for_inner_table_field()))) {
LOG_WARN("fail to databuff print", K(ret), K_(scn));
} else if (0 == strncmp(entry->d_name, prefix, strlen(prefix))) {
HEAP_VAR(ObBackupPartialTableListDesc, desc) {
ObBackupDataStore store;
char path[OB_MAX_BACKUP_PATH_LENGTH] = { 0 };
if (OB_FAIL(store.init(*backup_set_dest_))) {
LOG_WARN("fail to init backup data store", K(ret), KP_(backup_set_dest));
} else if (OB_FAIL(store.read_table_list_file(entry->d_name, desc))) {
LOG_WARN("fail to read table list file", K(ret), K(entry->d_name));
} else if (desc.count() > 0) {
ObBackupPartialTableListMeta partial_meta;
if (OB_FAIL(partial_meta.start_key_.assign(desc.items_.at(0)))
|| OB_FAIL(partial_meta.end_key_.assign(desc.items_.at(desc.count() - 1)))) {
LOG_WARN("fail to assign partial meta", K(ret));
} else if (OB_FAIL(partial_metas_.push_back(partial_meta))) {
LOG_WARN("fail to push back", K(ret), K(partial_meta));
}
}
}
}
return ret;
}
ObBackupTableListMgr::ObBackupTableListMgr()
: is_inited_(false),
tenant_id_(OB_INVALID_TENANT_ID),
snapshot_point_(SCN::min_scn()),
backup_set_dest_(),
tmp_file_(),
sql_proxy_(NULL),
backup_service_(nullptr)
{
}
int ObBackupTableListMgr::init(
const uint64_t tenant_id,
const share::SCN &snapshot_point,
const ObBackupDest &backup_set_dest,
rootserver::ObBackupDataService &backup_service,
common::ObISQLClient &sql_proxy)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("ObBackupTableListMgr init twice.", K(ret));
} else if (OB_INVALID_TENANT_ID == tenant_id
|| share::SCN::min_scn() == snapshot_point
|| !backup_set_dest.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(snapshot_point), K(backup_set_dest));
} else if (OB_FAIL(backup_set_dest_.deep_copy(backup_set_dest))) {
LOG_WARN("fail to deep copy backup set dest", K(ret) ,K(backup_set_dest));
} else {
tenant_id_ = tenant_id;
snapshot_point_ = snapshot_point;
sql_proxy_ = &sql_proxy;
backup_service_ = &backup_service;
is_inited_ = true;
}
return ret;
}
void ObBackupTableListMgr::reset()
{
int ret = OB_SUCCESS;
is_inited_ = false;
tenant_id_ = OB_INVALID_TENANT_ID;
sql_proxy_ = NULL;
snapshot_point_ = share::SCN::min_scn();
backup_set_dest_.reset();
partial_metas_.reset();
}
int ObBackupTableListMgr::backup_table_list()
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
int64_t count = 0;
bool is_meta_exist = false;
ObBackupDataStore store;
ObArray<int64_t> serialize_size_array; //serialize_size of each batch
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("backup table list mgr is not inited", K(ret));
} else if (OB_FAIL(store.init(backup_set_dest_))) {
LOG_WARN("fail to init backup store", K(ret), K_(backup_set_dest));
} else if (OB_FAIL(store.is_table_list_meta_exist(snapshot_point_, is_meta_exist))) {
LOG_WARN("fail to check is table list meta exist", K(ret), K_(snapshot_point), K_(backup_set_dest));
} else if (is_meta_exist) { // do nothing
} else if (OB_FAIL(tmp_file_.open(MTL_ID()))) {
LOG_WARN("fail to open tmp file", K(ret), K_(tenant_id));
} else if (OB_FAIL(backup_table_list_to_tmp_file_(count, serialize_size_array))) {
LOG_WARN("fail to backup table list to tmp file", K(ret), K_(snapshot_point));
} else if (OB_FAIL(backup_table_list_to_extern_device_(count, serialize_size_array))) {
LOG_WARN("fail to backup table list to external device", K(ret), K_(snapshot_point), K(count), K(serialize_size_array));
}
if (tmp_file_.is_opened() && OB_SUCCESS != (tmp_ret = tmp_file_.close())) {
LOG_WARN("fail to close tmp file", K(ret), K(tmp_ret), K_(tmp_file));
ret = OB_SUCCESS == ret ? tmp_ret : ret;
}
return ret;
}
int ObBackupTableListMgr::backup_table_list_to_tmp_file_(int64_t &count, ObIArray<int64_t> &serialize_size_array)
{
int ret = OB_SUCCESS;
int64_t offset = 0;
ObSqlString sql;
int64_t query_timeout = 30 * 60 * 1000000L; //30min
if (OB_FAIL(sql.append_fmt("SELECT /* QUERY_TIMEOUT(%ld) */ t1.database_name, t2.table_name "
"FROM %s as of snapshot %lu AS t1 "
"JOIN %s as of snapshot %lu AS t2 "
"ON t1.database_id = t2.database_id "
"WHERE t2.table_type IN (%d, %d) "
"ORDER BY t1.database_name, t2.table_name ASC",
query_timeout, OB_ALL_DATABASE_TNAME, snapshot_point_.get_val_for_inner_table_field(),
OB_ALL_TABLE_TNAME, snapshot_point_.get_val_for_inner_table_field(),
share::schema::ObTableType::USER_TABLE, share::schema::ObTableType::MATERIALIZED_VIEW))) {
LOG_WARN("failed to assign sql", K(ret), K_(snapshot_point));
} else {
HEAP_VARS_2((ObISQLClient::ReadResult, result), (ObBackupPartialTableListDesc, table_list)) {
if (OB_FAIL(sql_proxy_->read(result, tenant_id_, sql.ptr()))) {
LOG_WARN("execute sql failed", KR(ret), K(tenant_id_), K(sql));
} else if (OB_ISNULL(result.get_result())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get mysql result failed", KR(ret), K(sql));
} else {
partial_metas_.reset();
common::sqlclient::ObMySQLResult &res = *result.get_result();
while (OB_SUCC(ret) && OB_SUCC(res.next())) {
ObBackupTableListItem item;
ObString database_name;
ObString table_name;
EXTRACT_VARCHAR_FIELD_MYSQL(res, "database_name", database_name);
EXTRACT_VARCHAR_FIELD_MYSQL(res, "table_name", table_name);
if (FAILEDx(item.database_name_.assign(database_name))) {
LOG_WARN("fail to assign database name", K(ret), K(database_name));
} else if (OB_FAIL(item.table_name_.assign(table_name))) {
LOG_WARN("fail to assign table name", K(ret), K(table_name));
} else if (OB_FAIL(table_list.items_.push_back(item))) {
LOG_WARN("fail to push back", K(ret), K(item));
}
if (OB_SUCC(ret) && BATCH_SIZE == table_list.count()) {
int64_t serialize_size = 0;
ObBackupPartialTableListMeta partial_meta;
if (OB_FAIL(write_to_tmp_file_(table_list, serialize_size))) {
LOG_WARN("fail to write table list to tmp file", K(ret), K_(snapshot_point), K_(tmp_file));
} else if (OB_FAIL(serialize_size_array.push_back(serialize_size))) {
LOG_WARN("fail to push back", K(ret), K(serialize_size));
} else if (OB_FAIL(partial_metas_.push_back(partial_meta))) {
LOG_WARN("fail to push back", K(ret), K(partial_meta));
} else {
count += table_list.count();
table_list.reset();
}
}
}
if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
int64_t serialize_size = 0;
ObBackupPartialTableListMeta partial_meta;
if (0 == table_list.count()) { //skip
} else if (OB_FAIL(write_to_tmp_file_(table_list, serialize_size))) {
LOG_WARN("fail to write table list to tmp file", K(ret), K_(snapshot_point), K_(tmp_file));
} else if (OB_FAIL(serialize_size_array.push_back(serialize_size))) {
LOG_WARN("fail to push back", K(ret), K(serialize_size));
} else if (OB_FAIL(partial_metas_.push_back(partial_meta))) {
LOG_WARN("fail to push back", K(ret), K(partial_meta));
} else {
count += table_list.count();
}
} else {
if (OB_SUCC(ret)) {
ret = OB_ERR_UNEXPECTED;
}
LOG_WARN("construct results failed", KR(ret));
}
}
}
}
return ret;
}
int ObBackupTableListMgr::backup_table_list_to_extern_device_(const int64_t &count, const ObIArray<int64_t> &serialize_size_array)
{
int ret = OB_SUCCESS;
if (count < 0 || serialize_size_array.count() < 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(count), K(serialize_size_array));
} else {
int64_t max_file_part_no = 0; //file part_no starts from 1
if (OB_FAIL(get_max_complete_file_part_no_(max_file_part_no))) {
LOG_WARN("fail to get max complete file part_no", K(ret), K_(snapshot_point), K_(backup_set_dest));
} else {
int64_t read_offset = 0;
ARRAY_FOREACH_X(serialize_size_array, i, cnt, OB_SUCC(ret)) {
int64_t serialize_size = serialize_size_array.at(i);
int64_t part_no = i + 1;
if (part_no <= max_file_part_no) { // skip
read_offset += serialize_size;
LOG_INFO("table list part file already exists", K(part_no), K(max_file_part_no));
} else if (OB_FAIL(do_backup_table_list_(serialize_size, read_offset, part_no))) {
LOG_WARN("fail to do backup table list", K(ret),
K_(snapshot_point),
K(serialize_size),
K(read_offset),
K(part_no));
} else {
read_offset += serialize_size;
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(backup_service_->check_leader())) {
LOG_WARN("fail to check leader", K(ret));
} else if (OB_FAIL(write_table_list_meta_(count, BATCH_SIZE))) {
LOG_WARN("fail to write table list meta", K(ret), K_(backup_set_dest), K(count), K_(snapshot_point));
}
}
}
}
return ret;
}
int ObBackupTableListMgr::write_to_tmp_file_(const ObBackupPartialTableListDesc &table_list, int64_t &serialize_size)
{
int ret = OB_SUCCESS;
ObSelfBufferWriter buffer_writer(ObModIds::BACKUP);
int64_t last_pos = buffer_writer.pos();
const int64_t table_list_count = table_list.count();
serialize_size = table_list.get_serialize_size();
if (!table_list.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(table_list_count), K(serialize_size));
} else if (OB_FAIL(buffer_writer.write_serialize(table_list))) {
LOG_WARN("fail to write serialize", K(ret), K(table_list_count));
} else if (buffer_writer.pos() - last_pos > serialize_size) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("actual write size must not be larger than need write size",
K(ret),
"cur_pos",
buffer_writer.pos(),
K(last_pos),
K(serialize_size),
K(table_list_count));
} else if (OB_FAIL(tmp_file_.write(buffer_writer.data(), buffer_writer.pos()))) {
LOG_WARN("failed to write to tmp file", K(ret), K(buffer_writer));
} else {
buffer_writer.reuse();
}
return ret;
}
int ObBackupTableListMgr::read_from_tmp_file_(const int64_t read_size, const int64_t offset, ObBackupPartialTableListDesc &table_list)
{
int ret = OB_SUCCESS;
table_list.reset();
blocksstable::ObTmpFileIOInfo io_info;
blocksstable::ObTmpFileIOHandle handle;
io_info.fd_ = tmp_file_.get_fd();
io_info.tenant_id_ = tmp_file_.get_tenant_id();
io_info.dir_id_ = tmp_file_.get_dir();
io_info.io_desc_.set_wait_event(2);
io_info.size_ = read_size;
common::ObArenaAllocator allocator;
char *buf = NULL;
if (read_size <= 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid args", K(ret), K(read_size));
} else if (OB_ISNULL(buf = static_cast<char *>(allocator.alloc(read_size)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc memory", K(ret), K(read_size));
} else if (FALSE_IT(io_info.buf_ = buf)) {
} else if (OB_FAIL(blocksstable::ObTmpFileManager::get_instance().pread(io_info, offset, handle))) {
LOG_WARN("failed to pread from tmp file", K(ret), K(io_info), K(offset), K(read_size));
} else {
blocksstable::ObBufferReader buffer_reader(buf, read_size);
if (OB_FAIL(buffer_reader.read_serialize(table_list))) {
LOG_WARN("failed to read serialize", K(ret));
}
}
return ret;
}
int ObBackupTableListMgr::do_backup_table_list_(const int64_t read_size, const int64_t offset, const int64_t part_no)
{
int ret = OB_SUCCESS;
ObBackupDataStore store;
if (read_size <= 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid read size", K(ret), K(read_size));
} else {
HEAP_VAR(ObBackupPartialTableListDesc, table_list) {
if (OB_FAIL(store.init(backup_set_dest_))) {
LOG_WARN("fail to init backup store", K(ret), K_(backup_set_dest));
} else if (OB_FAIL(read_from_tmp_file_(read_size, offset, table_list))) {
LOG_WARN("fail to read table list from tmp file", K(read_size), K(offset), K_(tmp_file));
} else if (!table_list.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table list read from tmp file is invalid", K(ret), K(read_size), K(offset), K_(tmp_file));
} else if (OB_FAIL(store.write_single_table_list_part_file(snapshot_point_, part_no, table_list))) {
LOG_WARN("fail to write single table list file",K(ret), K_(snapshot_point), K(part_no), K_(backup_set_dest));
}
}
}
return ret;
}
int ObBackupTableListMgr::write_table_list_meta_(const int64_t total_count, const int64_t batch_size)
{
int ret = OB_SUCCESS;
ObBackupDataStore store;
DEBUG_SYNC(BEFORE_WRITE_TABLE_LIST_META_INFO);
HEAP_VAR(ObBackupTableListMetaInfoDesc, desc) {
desc.scn_ = snapshot_point_;
desc.count_ = total_count;
desc.batch_size_ = batch_size;
if (OB_FAIL(desc.partial_metas_.assign(partial_metas_))) {
LOG_WARN("fail to assign partial metas", K(ret), K_(partial_metas));
} else if (OB_FAIL(store.init(backup_set_dest_))) {
LOG_WARN("fail to init backup store", K(ret), K_(backup_set_dest));
} else if (OB_FAIL(store.write_table_list_meta_info(snapshot_point_, desc))) {
LOG_WARN("fail to write meta file", K(ret),K_(snapshot_point), K_(backup_set_dest), K(total_count));
} else {
LOG_INFO("write table list finish", K(ret),K_(snapshot_point), K_(backup_set_dest));
}
}
return ret;
}
int ObBackupTableListMgr::get_max_complete_file_part_no_(int64_t &part_no)
{
int ret = OB_SUCCESS;
part_no = 0;
share::ObBackupPath path;
ObBackupStorageInfo *storage_info;
ObBackupIoAdapter util;
ObGetMaxTableListPartNoOp max_part_no_op(snapshot_point_, part_no);
if (OB_FAIL(share::ObBackupPathUtil::get_table_list_dir_path(backup_set_dest_, path))) {
LOG_WARN("fail to get table list dir path", K(ret), K_(backup_set_dest));
} else if (OB_ISNULL(storage_info = backup_set_dest_.get_storage_info())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("fail to get storage_info", K(ret), K_(backup_set_dest));
} else if (OB_FAIL(util.list_files(path.get_obstr(), storage_info, max_part_no_op))) {
LOG_WARN("fail to get max complete file part_no", K(ret), K(path), K_(backup_set_dest));
}
return ret;
}
} //namespace rootserver
} //namespace oceanbase

125
src/rootserver/backup/ob_backup_table_list_mgr.h Normal file
View File

@ -0,0 +1,125 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_ROOTSERVER_BACKUP_OB_BACKUP_TABLE_LIST_MGR_H_
#define OCEANBASE_ROOTSERVER_BACKUP_OB_BACKUP_TABLE_LIST_MGR_H_
#include "share/backup/ob_backup_store.h"
#include "share/backup/ob_backup_path.h"
#include "storage/backup/ob_backup_tmp_file.h"
#include "rootserver/backup/ob_backup_service.h"
namespace oceanbase
{
namespace storage
{
class ObBackupPartialTableListDesc;
}
namespace rootserver
{
class ObGetMaxTableListPartNoOp : public ObBaseDirEntryOperator
{
public:
ObGetMaxTableListPartNoOp(
const share::SCN &scn,
int64_t &max_table_list_part_no)
: scn_(scn),
max_table_list_part_no_(max_table_list_part_no) {}
virtual ~ObGetMaxTableListPartNoOp() {}
bool is_valid() const;
int func(const dirent *entry);
private:
const share::SCN scn_;
int64_t &max_table_list_part_no_;
DISALLOW_COPY_AND_ASSIGN(ObGetMaxTableListPartNoOp);
};
class ObGetMaxTableListSCNOp : public ObBaseDirEntryOperator
{
public:
ObGetMaxTableListSCNOp(share::SCN &max_scn)
: max_scn_(max_scn) {}
virtual ~ObGetMaxTableListSCNOp() {}
bool is_valid() const;
int func(const dirent *entry) override;
private:
share::SCN &max_scn_;
DISALLOW_COPY_AND_ASSIGN(ObGetMaxTableListSCNOp);
};
class ObGetTableListPartialMetasOp : public ObBaseDirEntryOperator
{
public:
ObGetTableListPartialMetasOp(
const share::SCN &scn,
const ObBackupDest *backup_set_dest,
ObIArray<ObBackupPartialTableListMeta> &partial_metas)
: scn_(scn),
backup_set_dest_(backup_set_dest),
partial_metas_(partial_metas) {}
virtual ~ObGetTableListPartialMetasOp() {}
bool is_valid() const;
int func(const dirent *entry) override;
private:
const share::SCN &scn_;
const ObBackupDest *backup_set_dest_;
ObIArray<ObBackupPartialTableListMeta> &partial_metas_;
DISALLOW_COPY_AND_ASSIGN(ObGetTableListPartialMetasOp);
};
class ObBackupTableListMgr final
{
public:
ObBackupTableListMgr();
~ObBackupTableListMgr() { reset(); }
#ifdef ERRSIM
const int64_t BATCH_SIZE = GCONF.errsim_backup_table_list_batch_size;
#else
static const int64_t BATCH_SIZE = 20000;
#endif
int init(
const uint64_t tenant_id,
const share::SCN &snapshot_point,
const ObBackupDest &backup_set_dest,
rootserver::ObBackupDataService &backup_service,
common::ObISQLClient &sql_proxy);
int backup_table_list();
void reset();
private:
int backup_table_list_to_tmp_file_(int64_t &count, ObIArray<int64_t> &serialize_size_array);
int backup_table_list_to_extern_device_(const int64_t &count, const ObIArray<int64_t> &serialize_size_array);
int write_to_tmp_file_(const storage::ObBackupPartialTableListDesc &table_list, int64_t &serialize_size);
int read_from_tmp_file_(const int64_t read_size, const int64_t offset, storage::ObBackupPartialTableListDesc &table_list);
int do_backup_table_list_(const int64_t read_size, const int64_t offset, const int64_t part_no);
int write_table_list_meta_(const int64_t total_count, const int64_t batch_size);
int get_max_complete_file_part_no_(int64_t &part_no); // file part_no starts from 1
private:
bool is_inited_;
uint64_t tenant_id_;
share::SCN snapshot_point_;
ObBackupDest backup_set_dest_;
backup::ObBackupTmpFile tmp_file_;
common::ObISQLClient *sql_proxy_;
rootserver::ObBackupDataService *backup_service_;
ObSArray<ObBackupPartialTableListMeta> partial_metas_;
DISALLOW_COPY_AND_ASSIGN(ObBackupTableListMgr);
};
}// rootserver
}// oceanbase
#endif /* OCEANBASE_SHARE_BACKUP_OB_BACKUP_TABLE_LIST_MGR_H_ */

176
src/share/backup/ob_backup_path.cpp
View File

@ -482,16 +482,73 @@ int ObBackupPath::join_checkpoint_info_file(const common::ObString &file_name, c
}
return ret;
}
int ObBackupPath::join_table_list_dir()
{
int ret = OB_SUCCESS;
if (cur_pos_ <= 0) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret), K(*this));
} else if (OB_FAIL(databuff_printf(path_, sizeof(path_), cur_pos_, "/%s", OB_STR_TABLE_LIST))) {
LOG_WARN("failed to join table list dir", K(ret), K(*this));
} else if (OB_FAIL(trim_right_backslash())) {
LOG_WARN("failed to trim right backslash", K(ret));
}
return ret;
}
int ObBackupPath::join_table_list_part_file(const share::SCN &scn, const int64_t part_no)
{
int ret = OB_SUCCESS;
char file_name[OB_MAX_BACKUP_PATH_LENGTH] = { 0 };
if (cur_pos_ <= 0) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret), K(*this));
} else if (OB_FAIL(databuff_printf(file_name,
sizeof(file_name),
"%s.%lu.%ld",
OB_STR_TABLE_LIST,
scn.get_val_for_inner_table_field(),
part_no))) {
LOG_WARN("failed to join table list file", K(ret), K(part_no), K(scn), K(*this));
} else if (OB_FAIL(join(file_name, ObBackupFileSuffix::BACKUP))) {
LOG_WARN("failed to join file_name", K(ret), K(file_name));
} else if (OB_FAIL(trim_right_backslash())) {
LOG_WARN("failed to trim right backslash", K(ret));
}
return ret;
}
int ObBackupPath::join_table_list_meta_info_file(const share::SCN &scn)
{
int ret = OB_SUCCESS;
char file_name[OB_MAX_BACKUP_PATH_LENGTH] = { 0 };
if (cur_pos_ <= 0) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret), K(*this));
} else if (OB_FAIL(databuff_printf(file_name,
sizeof(file_name),
"%s.%lu",
OB_STR_TABLE_LIST_META_INFO,
scn.get_val_for_inner_table_field()))) {
LOG_WARN("failed to join table list tmp file", K(ret), K(scn), K(*this));
} else if (OB_FAIL(join(file_name, ObBackupFileSuffix::BACKUP))) {
LOG_WARN("failed to join file_name", K(ret), K(file_name));
} else if (OB_FAIL(trim_right_backslash())) {
LOG_WARN("failed to trim right backslash", K(ret));
}
return ret;
}
// param case: entry_d_name -> 'checkpoint_info.1678226622262333112.obarc', file_name -> 'checkpoint_info', type -> ARCHIVE
// result : checkpoint -> 1678226622262333112
int ObBackupPath::parse_checkpoint(const common::ObString &entry_d_name, const common::ObString &file_name, const ObBackupFileSuffix &type, uint64_t &checkpoint)
int ObBackupPath::parse_checkpoint(const char *entry_d_name, const common::ObString &file_name, const ObBackupFileSuffix &type, uint64_t &checkpoint)
{
int ret = OB_SUCCESS;
checkpoint = 0;
ObBackupPath tmp_path; //format string for sscanf
char tmp_file_name[OB_MAX_FILE_NAME_LENGTH] = { 0 };
if (entry_d_name.length() <= 0 || file_name.length() <= 0 || type > ObBackupFileSuffix::BACKUP || type < ObBackupFileSuffix::NONE) {
if (OB_ISNULL(entry_d_name) || strlen(entry_d_name) <= 0 || file_name.length() <= 0
|| type > ObBackupFileSuffix::BACKUP || type < ObBackupFileSuffix::NONE) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", K(ret), K(entry_d_name), K(file_name));
} else if (OB_FAIL(databuff_printf(tmp_file_name, sizeof(tmp_file_name), "%s.%%lu", file_name.ptr()))) {
@ -502,7 +559,7 @@ int ObBackupPath::parse_checkpoint(const common::ObString &entry_d_name, const c
LOG_WARN("failed to add backup file suffix", K(ret), K(type), K(tmp_path));
} else if (OB_FAIL(tmp_path.trim_right_backslash())) {
OB_LOG(WARN, "fail to trim_right_backslash", K(ret));
} else if (1 == sscanf(entry_d_name.ptr(), tmp_path.get_ptr(), &checkpoint)) {
} else if (1 == sscanf(entry_d_name, tmp_path.get_ptr(), &checkpoint)) {
if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) {
OB_LOG(INFO, "succeed to get checkpoint scn", K(ret), K(entry_d_name), K(checkpoint), K(tmp_path));
}
@ -513,6 +570,68 @@ int ObBackupPath::parse_checkpoint(const common::ObString &entry_d_name, const c
return ret;
}
// param case: entry_d_name -> 'table_list.1702352553000000000.1.obbak', file_name -> 'table_list', type -> BACKUP
// result : part_no -> 1
int ObBackupPath::parse_partial_table_list_file_name(const char *entry_d_name, const share::SCN &scn, int64_t &part_no)
{
int ret = OB_SUCCESS;
part_no = 0;
ObBackupPath tmp_path;
char tmp_file_name[OB_MAX_FILE_NAME_LENGTH] = { 0 };
if (OB_ISNULL(entry_d_name) || strlen(entry_d_name) <= 0 || !scn.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", K(ret), K(entry_d_name), K(scn));
} else if (OB_FAIL(databuff_printf(tmp_file_name,
sizeof(tmp_file_name),
"%s.%lu.%%ld",
OB_STR_TABLE_LIST,
scn.get_val_for_inner_table_field()))) {
LOG_WARN("failed to join tmp file name", K(ret));
} else if (OB_FAIL(tmp_path.init(tmp_file_name))) {
LOG_WARN("failed to init tmp path", K(ret), K(tmp_file_name));
} else if (OB_FAIL(tmp_path.add_backup_suffix(ObBackupFileSuffix::BACKUP))) {
LOG_WARN("failed to add backup file suffix", K(ret), K(tmp_path));
} else if (OB_FAIL(tmp_path.trim_right_backslash())) {
LOG_WARN("fail to trim_right_backslash", K(ret));
} else if (1 != sscanf(entry_d_name, tmp_path.get_ptr(), &part_no)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get part_no", K(ret), K(entry_d_name), K(part_no), K(tmp_path));
}
return ret;
}
// param case: entry_d_name -> 'table_list_meta_info.1702352553000000000.obbak', file_name -> 'table_list_meta_info', type -> BACKUP
// result : scn_val -> 1702352553000000000
int ObBackupPath::parse_table_list_meta_file_name(const char *entry_d_name, share::SCN &scn)
{
int ret = OB_SUCCESS;
uint64_t scn_val = 0;
ObBackupPath tmp_path;
char tmp_file_name[OB_MAX_FILE_NAME_LENGTH] = { 0 };
if (OB_ISNULL(entry_d_name) || strlen(entry_d_name) <= 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", K(ret), K(entry_d_name));
} else if (OB_FAIL(databuff_printf(tmp_file_name,
sizeof(tmp_file_name),
"%s.%%lu",
OB_STR_TABLE_LIST_META_INFO))) {
LOG_WARN("failed to join tmp file name", K(ret));
} else if (OB_FAIL(tmp_path.init(tmp_file_name))) {
LOG_WARN("failed to init tmp path", K(ret), K(tmp_file_name));
} else if (OB_FAIL(tmp_path.add_backup_suffix(ObBackupFileSuffix::BACKUP))) {
LOG_WARN("failed to add backup file suffix", K(ret), K(tmp_path));
} else if (OB_FAIL(tmp_path.trim_right_backslash())) {
LOG_WARN("fail to trim_right_backslash", K(ret));
} else if (1 != sscanf(entry_d_name, tmp_path.get_ptr(), &scn_val)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get scn", K(ret), K(entry_d_name), K(scn_val), K(tmp_path));
} else if (OB_FAIL(scn.convert_for_inner_table_field(scn_val))) {
LOG_WARN("fail to convert scn", K(ret), K(scn_val));
}
return ret;
}
common::ObString ObBackupPath::get_obstr() const
{
return ObString(cur_pos_, path_);
@ -1270,6 +1389,57 @@ int ObBackupPathUtil::get_ls_log_archive_prefix(const share::ObBackupDest &backu
return ret;
}
int ObBackupPathUtil::get_table_list_dir_path(const share::ObBackupDest &backup_tenant_dest,
const share::ObBackupSetDesc &desc, share::ObBackupPath &backup_path)
{
int ret = OB_SUCCESS;
if (OB_FAIL(get_ls_info_dir_path(backup_tenant_dest, desc, backup_path))) {
LOG_WARN("fail to get backup set info path", K(ret), K(backup_tenant_dest), K(desc));
} else if (OB_FAIL(backup_path.join_table_list_dir())) {
LOG_WARN("fail to join table list dir", K(ret));
}
return ret;
}
int ObBackupPathUtil::get_table_list_dir_path(const share::ObBackupDest &backup_set_dest,
share::ObBackupPath &backup_path)
{
int ret = OB_SUCCESS;
if (OB_FAIL(ObBackupPathUtil::get_ls_info_dir_path(backup_set_dest, backup_path))) {
LOG_WARN("fail to get backup set info path", K(ret), K(backup_set_dest));
} else if (OB_FAIL(backup_path.join_table_list_dir())) {
LOG_WARN("fail to join table list dir", K(ret), K(backup_set_dest), K(backup_path));
}
return ret;
}
int ObBackupPathUtil::get_table_list_meta_path(const share::ObBackupDest &backup_set_dest,
const share::SCN &scn, share::ObBackupPath &path)
{
int ret = OB_SUCCESS;
path.reset();
if (OB_FAIL(get_table_list_dir_path(backup_set_dest, path))) {
LOG_WARN("fail to get table list dir path", K(ret), K(backup_set_dest));
} else if (OB_FAIL(path.join_table_list_meta_info_file(scn))) {
LOG_WARN("fail to join table list meta file path", K(ret), K(backup_set_dest), K(path));
}
return ret;
}
int ObBackupPathUtil::get_table_list_part_file_path(const share::ObBackupDest &backup_set_dest,
const share::SCN &scn, const int64_t part_no, share::ObBackupPath &path)
{
int ret = OB_SUCCESS;
path.reset();
if (OB_FAIL(get_table_list_dir_path(backup_set_dest, path))) {
LOG_WARN("fail to get table list dir path", K(ret), K(backup_set_dest));
} else if (OB_FAIL(path.join_table_list_part_file(scn, part_no))) {
LOG_WARN("fail to join table list part file", K(ret), K(scn), K(part_no));
}
return ret;
}
int ObBackupPathUtil::construct_backup_set_dest(const share::ObBackupDest &backup_tenant_dest,
const share::ObBackupSetDesc &backup_desc, share::ObBackupDest &backup_set_dest)
{

20
src/share/backup/ob_backup_path.h
View File

@ -56,7 +56,12 @@ public:
int join_tenant_macro_range_index_file(const share::ObBackupDataType &type, const int64_t retry_id);
int join_tenant_meta_index_file(const share::ObBackupDataType &type, const int64_t retry_id, const bool is_sec_meta);
int join_checkpoint_info_file(const common::ObString &path, const uint64_t checkpoint, const ObBackupFileSuffix &type);
static int parse_checkpoint(const common::ObString &entry_d_name, const common::ObString &file_name, const ObBackupFileSuffix &type, uint64_t &checkpoint);
int join_table_list_dir();
int join_table_list_part_file(const share::SCN &scn, const int64_t part_no);
int join_table_list_meta_info_file(const share::SCN &scn);
static int parse_checkpoint(const char *entry_d_name, const common::ObString &file_name, const ObBackupFileSuffix &type, uint64_t &checkpoint);
static int parse_partial_table_list_file_name(const char *entry_d_name, const share::SCN &scn, int64_t &part_no);
static int parse_table_list_meta_file_name(const char *entry_d_name, share::SCN &scn);
int add_backup_suffix(const ObBackupFileSuffix &type);
const char *get_ptr() const { return path_; }
@ -266,6 +271,17 @@ struct ObBackupPathUtil
const int64_t incarnation, const int64_t round, const int64_t piece_id, const share::ObLSID &ls_id,
share::ObBackupPath &backup_path);
// file:///obbackup/backup_set_1_full/infos/table_list/
static int get_table_list_dir_path(const share::ObBackupDest &backup_tenant_dest,
const share::ObBackupSetDesc &desc, share::ObBackupPath &backup_path);
static int get_table_list_dir_path(const share::ObBackupDest &backup_set_dest,
share::ObBackupPath &backup_path);
// file:///obbackup/backup_set_1_full/infos/table_list/table_list_meta_info.[scn].obbak
static int get_table_list_meta_path(const share::ObBackupDest &backup_set_dest,
const share::SCN &scn, share::ObBackupPath &path);
// file:///obbackup/backup_set_1_full/infos/table_list/table_list.[scn].[part_no].obbak
static int get_table_list_part_file_path(const share::ObBackupDest &backup_set_dest,
const share::SCN &scn, const int64_t part_no, share::ObBackupPath &path);
static int construct_backup_set_dest(const share::ObBackupDest &backup_tenant_dest,
const share::ObBackupSetDesc &backup_desc, share::ObBackupDest &backup_set_dest);
static int construct_backup_complement_log_dest(const share::ObBackupDest &backup_tenant_dest,
@ -297,5 +313,3 @@ struct ObBackupPathUtilV_4_1
}//share
}//oceanbase
#endif /* SRC_SHARE_BACKUP_OB_BACKUP_INFO_H_ */

82
src/share/backup/ob_backup_struct.cpp
View File

@ -2788,7 +2788,6 @@ const char *backup_set_file_info_status_strs[] = {
"FAILED",
};
ObBackupRegion::ObBackupRegion()
: region_(),
priority_(-1)
@ -4464,3 +4463,84 @@ int ObRestoreLogPieceBriefInfo::assign(const ObRestoreLogPieceBriefInfo &that)
}
return ret;
}
OB_SERIALIZE_MEMBER(ObBackupTableListItem,
database_name_,
table_name_);
ObBackupTableListItem::ObBackupTableListItem()
: database_name_(),
table_name_()
{
}
bool ObBackupTableListItem::is_valid() const
{
return !database_name_.is_empty() && !table_name_.is_empty();
}
void ObBackupTableListItem::reset()
{
database_name_.reset();
table_name_.reset();
}
int ObBackupTableListItem::assign(const ObBackupTableListItem &o)
{
int ret = OB_SUCCESS;
if (OB_FAIL(database_name_.assign(o.database_name_))) {
LOG_WARN("fail to assign database name", K(ret), K(o.database_name_));
} else if (OB_FAIL(table_name_.assign(o.table_name_))) {
LOG_WARN("fail to assign table name", K(ret), K(o.table_name_));
}
return ret;
}
bool ObBackupTableListItem::operator==(const ObBackupTableListItem &o) const
{
return database_name_ == o.database_name_ && table_name_ == o.table_name_;
}
bool ObBackupTableListItem::operator>(const ObBackupTableListItem &o) const
{
bool b_ret = false;
if (database_name_ > o.database_name_) {
b_ret = true;
} else if (database_name_ == o.database_name_
&& table_name_ > o.table_name_) {
b_ret = true;
}
return b_ret;
}
OB_SERIALIZE_MEMBER(ObBackupPartialTableListMeta,
start_key_,
end_key_);
ObBackupPartialTableListMeta::ObBackupPartialTableListMeta()
: start_key_(),
end_key_()
{
}
bool ObBackupPartialTableListMeta::is_valid() const
{
return start_key_.is_valid() && end_key_.is_valid() && end_key_ >= start_key_;
}
void ObBackupPartialTableListMeta::reset()
{
start_key_.reset();
end_key_.reset();
}
int ObBackupPartialTableListMeta::assign(const ObBackupPartialTableListMeta &other)
{
int ret = OB_SUCCESS;
if (OB_FAIL(start_key_.assign(other.start_key_))) {
LOG_WARN("fail to assign start key", K(ret), K(other.start_key_));
} else if (OB_FAIL(end_key_.assign(other.end_key_))) {
LOG_WARN("fail to assign end key", K(ret), K(other.end_key_));
}
return ret;
}

44
src/share/backup/ob_backup_struct.h
View File

@ -419,6 +419,8 @@ const char *const OB_STR_SRC_TENANT_NAME = "src_tenant_name";
const char *const OB_STR_AUX_TENANT_NAME = "aux_tenant_name";
const char *const OB_STR_TARGET_TENANT_NAME = "target_tenant_name";
const char *const OB_STR_TARGET_TENANT_ID = "target_tenant_id";
const char *const OB_STR_TABLE_LIST = "table_list";
const char *const OB_STR_TABLE_LIST_META_INFO = "table_list_meta_info";
enum ObBackupFileType
{
@ -460,6 +462,8 @@ enum ObBackupFileType
BACKUP_TENANT_ARCHIVE_PIECE_INFOS = 35,
BACKUP_DELETED_TABLET_INFO = 36,
BACKUP_TABLET_METAS_INFO = 37,
BACKUP_TABLE_LIST_FILE = 38,
BACKUP_TABLE_LIST_META_FILE = 39,
// type <=255 is write header struct to disk directly
// type > 255 is use serialization to disk
BACKUP_MAX_DIRECT_WRITE_TYPE = 255,
@ -1737,6 +1741,46 @@ int backup_time_to_strftime(const int64_t &ts_s, char *buf, const int64_t buf_le
int backup_scn_to_time_tag(const SCN &scn, char *buf, const int64_t buf_len, int64_t &pos);
inline uint64_t trans_scn_to_second(const SCN &scn) { return scn.convert_to_ts() / 1000 / 1000; }
struct ObBackupTableListItem final
{
OB_UNIS_VERSION(1);
public:
ObBackupTableListItem();
~ObBackupTableListItem() = default;
bool is_valid() const;
void reset();
int assign(const ObBackupTableListItem &o);
bool operator==(const ObBackupTableListItem &o) const;
bool operator!=(const ObBackupTableListItem &o) const { return !(operator ==(o)); }
bool operator>(const ObBackupTableListItem &o) const;
bool operator>=(const ObBackupTableListItem &o) const { return operator > (o) || operator == (o); }
bool operator<(const ObBackupTableListItem &o) const { return !(operator >= (o)); }
bool operator<=(const ObBackupTableListItem &o) const { return !(operator > (o)); }
TO_STRING_KV(K_(database_name), K_(table_name));
common::ObFixedLengthString<OB_MAX_DATABASE_NAME_LENGTH + 1> database_name_;
common::ObFixedLengthString<OB_MAX_TABLE_NAME_LENGTH + 1> table_name_;
private:
DISALLOW_COPY_AND_ASSIGN(ObBackupTableListItem);
};
struct ObBackupPartialTableListMeta final
{
OB_UNIS_VERSION(1);
public:
ObBackupPartialTableListMeta();
~ObBackupPartialTableListMeta() = default;
bool is_valid() const;
void reset();
int assign(const ObBackupPartialTableListMeta &other);
TO_STRING_KV(K_(start_key), K_(end_key));
ObBackupTableListItem start_key_;
ObBackupTableListItem end_key_;
private:
DISALLOW_COPY_AND_ASSIGN(ObBackupPartialTableListMeta);
};
}//share
}//oceanbase

1
src/share/ob_debug_sync_point.h
View File

@ -600,6 +600,7 @@ class ObString;
ACT(HOLD_DDL_COMPLEMENT_DAG_AFTER_REPORT_FINISH,)\
ACT(BEFORE_ALTER_TABLE_EXCHANGE_PARTITION,)\
ACT(AFTER_REPORT_BACKUP_COMPL_LOG,)\
ACT(BEFORE_WRITE_TABLE_LIST_META_INFO,)\
ACT(MAX_DEBUG_SYNC_POINT,)
DECLARE_ENUM(ObDebugSyncPoint, debug_sync_point, OB_DEBUG_SYNC_POINT_DEF);

5
src/share/parameter/ob_parameter_seed.ipp
View File

@ -1151,6 +1151,11 @@ ERRSIM_DEF_INT(errsim_test_tablet_id, OB_CLUSTER_PARAMETER, "0", "[0,)",
"Range: [0,) in integer",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
ERRSIM_DEF_INT(errsim_backup_table_list_batch_size, OB_CLUSTER_PARAMETER, "20000", "[1,)",
"batch size of table list items in errsim mode"
"Range: [1,) in integer",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
#ifdef TRANS_MODULE_TEST
DEF_INT(module_test_trx_memory_errsim_percentage, OB_CLUSTER_PARAMETER, "0", "[0, 100]",
"the percentage of memory errsim. Rang:[0,100]",

20
src/sql/parser/sql_parser_mysql_mode.y
View File

@ -133,6 +133,7 @@ extern void obsql_oracle_parse_fatal_error(int32_t errcode, yyscan_t yyscanner,
%left '(' ')'
%nonassoc SQL_CACHE SQL_NO_CACHE CHARSET DATABASE_ID REPLICA_NUM/*for shift/reduce conflict between opt_query_expresion_option_list and SQL_CACHE*/
%nonassoc HIGHER_PARENS TRANSACTION SIZE AUTO SKEWONLY DEFAULT AS/*for simple_expr conflict*/
%nonassoc TENANT /*for opt_tenant conflict*/
%left '.'
%right NOT NOT2
%right BINARY COLLATE
@ -516,7 +517,7 @@ END_P SET_VAR DELIMITER
%type <node> permanent_tablespace permanent_tablespace_options permanent_tablespace_option alter_tablespace_actions alter_tablespace_action opt_force_purge
%type <node> opt_sql_throttle_for_priority opt_sql_throttle_using_cond sql_throttle_one_or_more_metrics sql_throttle_metric
%type <node> opt_copy_id opt_backup_dest opt_backup_backup_dest opt_tenant_info opt_with_active_piece get_format_unit opt_backup_tenant_list opt_backup_to opt_description policy_name opt_recovery_window opt_redundancy opt_backup_copies opt_restore_until opt_backup_key_info opt_encrypt_key
%type <node> opt_recover_tenant recover_table_list recover_table_relation_name restore_remap_list remap_relation_name table_relation_name opt_recover_remap_item_list restore_remap_item_list restore_remap_item remap_item remap_table_val
%type <node> opt_recover_tenant recover_table_list recover_table_relation_name restore_remap_list remap_relation_name table_relation_name opt_recover_remap_item_list restore_remap_item_list restore_remap_item remap_item remap_table_val opt_tenant
%type <node> new_or_old new_or_old_column_ref diagnostics_info_ref
%type <node> on_empty on_error json_on_response opt_returning_type opt_on_empty_or_error json_value_expr opt_ascii opt_truncate_clause
%type <node> ws_nweights opt_ws_as_char opt_ws_levels ws_level_flag_desc ws_level_flag_reverse ws_level_flags ws_level_list ws_level_list_item ws_level_number ws_level_range ws_level_list_or_range
@ -17774,9 +17775,22 @@ recover_table_relation_name:
;
opt_recover_tenant:
TO tenant_name
TO opt_tenant relation_name_or_string
{
$$ = $2;
(void)($2);
malloc_non_terminal_node($$, result->malloc_pool_, T_TENANT_NAME, 1, $3);
}
;
opt_tenant:
/*empty*/ %prec BASIC
{
$$ = NULL;
}
| TENANT opt_equal_mark
{
(void)($2);
$$ = NULL;
}
;

118
src/storage/backup/ob_backup_data_store.cpp
View File

@ -166,6 +166,46 @@ bool ObBackupLSMetaInfosDesc::is_valid() const
return !ls_meta_packages_.empty();
}
/*
*-----------------------------ObBackupPartialTableListDesc-----------------------
*/
OB_SERIALIZE_MEMBER(ObBackupPartialTableListDesc, items_);
bool ObBackupPartialTableListDesc::is_valid() const
{
return count() > 0;
}
void ObBackupPartialTableListDesc::reset()
{
items_.reset();
}
int64_t ObBackupPartialTableListDesc::to_string(char *buf, int64_t buf_len) const
{
int64_t pos = 0;
if (OB_ISNULL(buf) || buf_len <= 0 || !is_valid()) {
// do nothing
} else {
J_OBJ_START();
int64_t total = count();
J_KV("start_item", items_.at(0), "end_item", items_.at(count() - 1), K(total));
J_OBJ_END();
}
return pos;
}
/*
*-----------------------------ObBackupTableListMetaInfoDesc-----------------------
*/
OB_SERIALIZE_MEMBER(ObBackupTableListMetaInfoDesc, scn_, count_, batch_size_, partial_metas_);
bool ObBackupTableListMetaInfoDesc::is_valid() const
{
return scn_.is_valid() && count_ >= 0 && batch_size_ > 0;
}
int ObBackupSetFilter::get_backup_set_array(ObIArray<share::ObBackupSetDesc> &backup_set_array) const
{
@ -1253,3 +1293,81 @@ int ObBackupDataStore::read_deleted_tablet_info_v_4_1_x(
}
return ret;
}
int ObBackupDataStore::write_single_table_list_part_file(const share::SCN &scn, const int64_t part_no, const ObBackupPartialTableListDesc &table_list)
{
int ret = OB_SUCCESS;
share::ObBackupPath path;
if (!is_init()) {
ret = OB_NOT_INIT;
LOG_WARN("backup data extern mgr not init", K(ret));
} else if (!scn.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("scn is not valid", K(ret), K(scn));
} else if (OB_FAIL(ObBackupPathUtil::get_table_list_part_file_path(backup_set_dest_, scn, part_no, path))) {
LOG_WARN("fail to get table list part file path", K(ret), K_(backup_set_dest), K(scn), K(part_no));
} else if (OB_FAIL(write_single_file(path.get_obstr(), table_list))) {
LOG_WARN("fail to write single file", K(ret));
}
return ret;
}
int ObBackupDataStore::write_table_list_meta_info(const share::SCN &scn, const ObBackupTableListMetaInfoDesc &desc)
{
int ret = OB_SUCCESS;
share::ObBackupPath path;
if (!is_init()) {
ret = OB_NOT_INIT;
LOG_WARN("backup data extern mgr not init", K(ret));
} else if (!scn.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("scn is not valid", K(ret), K(scn));
} else if (!desc.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("table list meta desc is not valid", K(ret), K(desc));
} else if (OB_FAIL( ObBackupPathUtil::get_table_list_meta_path(backup_set_dest_, scn, path))) {
LOG_WARN("fail to get table list meta path", K(ret), K(scn));
} else if (OB_FAIL(write_single_file(path.get_obstr(), desc))) {
LOG_WARN("fail to write single file", K(ret), K(desc));
}
return ret;
}
int ObBackupDataStore::read_table_list_file(const char *file_name, ObBackupPartialTableListDesc &desc)
{
int ret = OB_SUCCESS;
ObBackupPath path;
char path_str[OB_MAX_BACKUP_PATH_LENGTH] = { 0 };
if (!is_init()) {
ret = OB_NOT_INIT;
LOG_WARN("backup data extern mgr not init", K(ret));
} else if (OB_FAIL(ObBackupPathUtil::get_table_list_dir_path(backup_set_dest_, path))) {
LOG_WARN("fail to get table list dir path", K(ret), K_(backup_set_dest));
} else if (OB_FAIL(databuff_printf(path_str, OB_MAX_BACKUP_PATH_LENGTH, "%s/%s", path.get_ptr(), file_name))) {
LOG_WARN("fail to databuff printf", K(ret), K(path), K(file_name));
} else if (OB_FAIL(read_single_file(path_str, desc))) {
LOG_WARN("fail to read single file", K(ret), K(path_str));
}
return ret;
}
int ObBackupDataStore::is_table_list_meta_exist(const share::SCN &scn, bool &is_exist)
{
int ret = OB_SUCCESS;
ObBackupPath full_path;
ObBackupIoAdapter util;
const ObBackupStorageInfo *storage_info = get_storage_info();
if (!is_init()) {
ret = OB_NOT_INIT;
LOG_WARN("ObBackupStore not init", K(ret));
} else if (OB_FAIL(ObBackupPathUtil::get_table_list_meta_path(backup_set_dest_, scn, full_path))) {
LOG_WARN("failed to get format file path", K(ret));
} else if (OB_FAIL(util.is_exist(full_path.get_obstr(), storage_info, is_exist))) {
LOG_WARN("failed to check format file exist.", K(ret), K(full_path));
}
return ret;
}

51
src/storage/backup/ob_backup_data_store.h
View File

@ -236,6 +236,51 @@ private:
DISALLOW_COPY_AND_ASSIGN(ObBackupLSMetaInfosDesc);
};
struct ObBackupPartialTableListDesc final : public ObExternBackupDataDesc
{
public:
static const uint8_t FILE_VERSION = 1;
OB_UNIS_VERSION(1);
public:
ObBackupPartialTableListDesc()
: ObExternBackupDataDesc(ObBackupFileType::BACKUP_TABLE_LIST_FILE, FILE_VERSION),
items_() {}
virtual ~ObBackupPartialTableListDesc() {}
bool is_valid() const override;
void reset();
int64_t count() const { return items_.count(); }
DECLARE_TO_STRING;
public:
ObSArray<ObBackupTableListItem> items_;
private:
DISALLOW_COPY_AND_ASSIGN(ObBackupPartialTableListDesc);
};
struct ObBackupTableListMetaInfoDesc final : public ObExternBackupDataDesc
{
public:
static const uint8_t FILE_VERSION = 1;
OB_UNIS_VERSION(1);
public:
ObBackupTableListMetaInfoDesc()
: ObExternBackupDataDesc(ObBackupFileType::BACKUP_TABLE_LIST_META_FILE, FILE_VERSION),
scn_(share::SCN::min_scn()),
count_(0),
batch_size_(0),
partial_metas_() {}
virtual ~ObBackupTableListMetaInfoDesc() {}
bool is_valid() const override;
TO_STRING_KV(K_(count), K_(batch_size), K_(partial_metas));
public:
share::SCN scn_;
int64_t count_;
int64_t batch_size_;
common::ObSArray<ObBackupPartialTableListMeta> partial_metas_;
private:
DISALLOW_COPY_AND_ASSIGN(ObBackupTableListMetaInfoDesc);
};
class ObBackupSetFilter : public ObBaseDirEntryOperator
{
public:
@ -316,7 +361,11 @@ public:
// 4.1 interface to get tablet to ls
int read_tablet_to_ls_info_v_4_1_x(const int64_t turn_id, const ObLSID &ls_id, ObIArray<ObTabletID> &tablet_ids);
int read_deleted_tablet_info_v_4_1_x(const ObLSID &ls_id, ObIArray<ObTabletID> &deleted_tablet_ids);
// table list
int write_single_table_list_part_file(const share::SCN &scn, const int64_t part_no, const ObBackupPartialTableListDesc &table_list);
int write_table_list_meta_info(const share::SCN &scn, const ObBackupTableListMetaInfoDesc &desc);
int read_table_list_file(const char* file_name, ObBackupPartialTableListDesc &desc);
int is_table_list_meta_exist(const share::SCN &scn, bool &is_exist);
TO_STRING_KV(K_(backup_desc));
public:

242
tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp
View File

@ -16,6 +16,7 @@
#include "storage/blocksstable/ob_data_buffer.h"
#include "../dumpsst/ob_admin_dumpsst_print_helper.h"
#include "storage/blocksstable/ob_logic_macro_id.h"
#include "rootserver/backup/ob_backup_table_list_mgr.h"
#ifdef OB_BUILD_TDE_SECURITY
#include "share/ob_master_key_getter.h"
#endif
@ -27,6 +28,7 @@ using namespace oceanbase::share;
using namespace oceanbase::common;
using namespace oceanbase::backup;
using namespace oceanbase::blocksstable;
using namespace oceanbase::rootserver;
#define HELP_FMT "\t%-30s%-12s\n"
namespace oceanbase {
@ -554,12 +556,21 @@ int ObAdminDumpBackupDataExecutor::execute(int argc, char *argv[])
if (OB_FAIL(dump_tenant_archive_path_())) {
STORAGE_LOG(WARN, "fail to dump tenant archive path");
}
} else if (OB_FAIL(check_file_exist_(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to check exist", K(ret), K_(backup_path), K_(storage_info));
} else if (OB_FAIL(get_backup_file_type_())) {
STORAGE_LOG(WARN, "failed to get backup file type", K(ret), K_(backup_path), K_(storage_info));
} else if (OB_FAIL(do_execute_())) {
STORAGE_LOG(WARN, "failed to do execute", K(ret), K_(backup_path), K_(storage_info));
} else {
bool is_table_list_dir = false;
if (OB_FAIL(check_is_table_list_dir_(is_table_list_dir))) {
STORAGE_LOG(WARN, "fail to check is table list dir", K(ret));
}
if (OB_FAIL(ret)) {
} else if (is_table_list_dir && OB_FAIL(handle_print_table_list_())) {
STORAGE_LOG(WARN, "fail to handle print table list", K(ret), K_(backup_path));
} else if (OB_FAIL(check_file_exist_(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to check exist", K(ret), K_(backup_path), K_(storage_info));
} else if (OB_FAIL(get_backup_file_type_())) {
STORAGE_LOG(WARN, "failed to get backup file type", K(ret), K_(backup_path), K_(storage_info));
} else if (OB_FAIL(do_execute_())) {
STORAGE_LOG(WARN, "failed to do execute", K(ret), K_(backup_path), K_(storage_info));
}
}
return ret;
}
@ -666,7 +677,13 @@ int ObAdminDumpBackupDataExecutor::check_dir_exist_(
if (OB_FAIL(storage_info.set(backup_path, storage_info_str))) {
STORAGE_LOG(WARN, "failed to set storage info", K(ret), K(storage_info_str));
} else if (OB_FAIL(util.is_empty_directory(backup_path, &storage_info, is_empty_dir))) {
STORAGE_LOG(WARN, "failed to check dir exist", K(ret), K(backup_path), K(storage_info));
if (OB_IO_ERROR == ret) {
is_exist = false;
ret = OB_SUCCESS;
STORAGE_LOG(INFO, "backup path may be file path rather than dir path", K(backup_path));
} else {
STORAGE_LOG(WARN, "failed to check dir exist", K(ret), K(backup_path), K(storage_info));
}
} else if (!is_empty_dir) {
is_exist = true;
}
@ -2364,7 +2381,7 @@ int ObAdminDumpBackupDataExecutor::dump_tenant_backup_set_infos_(const ObIArray<
ARRAY_FOREACH_X(backup_set_infos, i , cnt, OB_SUCC(ret)) {
const share::ObBackupSetFileDesc &backup_set_desc = backup_set_infos.at(i);
int64_t pos = 0;
char buf[OB_MAX_CHAR_LEN] = { 0 };
char buf[OB_MAX_INTEGER_DISPLAY_WIDTH] = { 0 };
char str_buf[OB_MAX_TEXT_LENGTH] = { 0 };
char min_restore_scn_str_buf[OB_MAX_TIME_STR_LENGTH] = { 0 };
if (OB_FAIL(ObTimeConverter::scn_to_str(backup_set_desc.min_restore_scn_.get_val_for_inner_table_field(),
@ -2874,5 +2891,214 @@ int ObAdminDumpBackupDataExecutor::dump_tenant_archive_path_()
return ret;
}
int ObAdminDumpBackupDataExecutor::handle_print_table_list_()
{
int ret = OB_SUCCESS;
uint64_t scn_val = 0;
if (OB_FAIL(get_max_table_list_scn(scn_val))) {
STORAGE_LOG(WARN, "fail to get max table list scn", K(ret));
} else if (scn_val > 0) {
if (OB_UNLIKELY(offset_ == 0 && length_ == 0)) {
if (OB_FAIL(print_table_list_meta_info_(scn_val))) {
STORAGE_LOG(WARN, "fail to print table list items", K(ret));
}
} else if (OB_FAIL(print_table_list_items_(scn_val))) {
STORAGE_LOG(WARN, "fail to print table list items", K(ret));
}
}
return ret;
}
int ObAdminDumpBackupDataExecutor::check_is_table_list_dir_(bool &is_table_list_dir) {
int ret = OB_SUCCESS;
bool is_exist = false;
is_table_list_dir = false;
if (OB_FAIL(check_dir_exist_(backup_path_, storage_info_, is_exist))) {
STORAGE_LOG(WARN, "fail to check dir exist", K(ret), K_(backup_path));
} else if (is_exist) {
char tmp[OB_MAX_BACKUP_PATH_LENGTH] = { 0 };
char *token = NULL;
char *saveptr = NULL;
const char *delimeter = "/";
int64_t info_len = strlen(backup_path_);
int64_t pos = 0;
if (OB_FAIL(databuff_printf(tmp, OB_MAX_BACKUP_PATH_LENGTH, pos, "%s", backup_path_))) {
STORAGE_LOG(WARN, "fail to databuff printf", K(ret), K_(backup_path));
} else {
token = strtok_r(tmp, delimeter, &saveptr);
while (token != NULL) {
char *next_token = strtok_r(NULL, delimeter, &saveptr);
if (next_token == NULL) {
break;
}
token = next_token;
}
if (OB_ISNULL(token)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), K_(backup_path));
} else if (0 == strcmp(OB_STR_TABLE_LIST, token)) {
is_table_list_dir = true;
}
}
}
return ret;
}
int ObAdminDumpBackupDataExecutor::get_max_table_list_scn(uint64_t &scn_val)
{
int ret = OB_SUCCESS;
ObBackupPath path;
ObBackupStorageInfo storage_info;
ObBackupIoAdapter util;
share::SCN scn = share::SCN::min_scn();
ObGetMaxTableListSCNOp max_scn_op(scn);
if (OB_FAIL(path.init(ObString(backup_path_)))) {
STORAGE_LOG(WARN, "fail to init backup path", K(ret), K_(backup_path));
} else if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K(ret), K_(backup_path), K_(storage_info));
} else if (OB_FAIL(util.list_files(path.get_obstr(), &storage_info, max_scn_op))) {
STORAGE_LOG(WARN, "fail to list files", K(ret), K_(backup_path));
} else {
scn_val = scn.get_val_for_inner_table_field();
STORAGE_LOG(INFO, "get max table list scn", K(scn_val));
}
return ret;
}
int ObAdminDumpBackupDataExecutor::print_table_list_meta_info_(const uint64_t scn_val)
{
int ret = OB_SUCCESS;
ObBackupTableListMetaInfoDesc desc;
if (OB_FAIL(read_table_list_meta_info_(scn_val, desc))) {
STORAGE_LOG(WARN, "fail to read table list meta info", K(ret));
} else if (OB_FAIL(dump_table_list_meta_info_(desc))) {
STORAGE_LOG(WARN, "fail to dump tenant locality info", K(ret), K(desc));
}
return ret;
}
int ObAdminDumpBackupDataExecutor::read_table_list_meta_info_(const uint64_t scn_val, ObBackupTableListMetaInfoDesc &desc)
{
int ret = OB_SUCCESS;
ObBackupPath full_path;
share::SCN scn;
if (OB_FAIL(full_path.init(ObString(backup_path_)))) {
STORAGE_LOG(WARN, "fail to init backup path", K(ret), K_(backup_path));
} else if (OB_FAIL(scn.convert_for_inner_table_field(scn_val))) {
STORAGE_LOG(WARN, "fail to convert scn", K(ret), K(scn_val));
} else if (OB_FAIL(full_path.join_table_list_meta_info_file(scn))) {
STORAGE_LOG(WARN, "fail to join table list meta info file", K(ret), K(scn));
} else if (OB_FAIL(ObAdminDumpBackupDataUtil::read_backup_info_file(full_path.get_obstr(), ObString(storage_info_), desc))) {
STORAGE_LOG(WARN, "fail to read locality info", K(ret), K(backup_path_), K(storage_info_));
}
return ret;
}
int ObAdminDumpBackupDataExecutor::print_table_list_items_(const uint64_t scn_val)
{
int ret = OB_SUCCESS;
ObBackupTableListMetaInfoDesc meta_desc;
if (OB_FAIL(read_table_list_meta_info_(scn_val, meta_desc))) {
STORAGE_LOG(WARN, "fail to read table list meta info", K(ret));
} else if (!meta_desc.is_valid()) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "backup table list meta info is not valid", K(ret), K(meta_desc));
} else {
PrintHelper::print_dump_title("table list");
if (offset_ >= meta_desc.count_) {
STORAGE_LOG(INFO, "input offset is larger than total count", K_(offset), "total_count", meta_desc.count_);
} else {
int64_t print_order = 0;
int64_t start_part_no = offset_ / meta_desc.batch_size_ + 1; // part_no starts from 1
int64_t end_part_no = (offset_ + length_) < meta_desc.count_ ?
(offset_ + length_ - 1) / meta_desc.batch_size_ + 1 : (meta_desc.count_ - 1) / meta_desc.batch_size_ + 1;
int64_t start_part_no_key = offset_ % meta_desc.batch_size_; // key starts from 0
int64_t end_part_no_key = (offset_ + length_) < meta_desc.count_ ?
(offset_ + length_ - 1) % meta_desc.batch_size_ : (meta_desc.count_ - 1) % meta_desc.batch_size_;
ObBackupPartialTableListDesc desc;
for(int64_t part_no = start_part_no; OB_SUCC(ret) && part_no <= end_part_no; part_no++) {
int64_t start_key = part_no == start_part_no ? start_part_no_key : 0;
int64_t end_key = part_no == end_part_no ? end_part_no_key : meta_desc.batch_size_ - 1;
if (OB_FAIL(read_table_list_part_file_(scn_val, part_no, desc))) {
STORAGE_LOG(WARN, "fail to read table list file", K(ret), K(scn_val), K(part_no));
} else {
for (; start_key <= end_key; start_key++) {
if (OB_FAIL(print_table_list_item_(desc, start_key, ++print_order))) {
STORAGE_LOG(WARN, "fail to print table list item", K(ret), K(desc), K(start_key));
}
}
}
}
}
}
PrintHelper::print_end_line();
return ret;
}
int ObAdminDumpBackupDataExecutor::print_table_list_item_(const ObBackupPartialTableListDesc &partial_desc,
const int64_t partial_offset, const int64_t print_order)
{
int ret = OB_SUCCESS;
char buf[OB_MAX_TEXT_LENGTH] = { 0 };
ObString escape_str;
char order_buf[OB_MAX_INTEGER_DISPLAY_WIDTH] = { 0 };
ObArenaAllocator allocator(ObModIds::BACKUP);
if (!partial_desc.is_valid() || partial_offset >= partial_desc.count()) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), K(partial_desc), K(partial_offset));
} else if (OB_FAIL(databuff_printf(order_buf, OB_MAX_INTEGER_DISPLAY_WIDTH, "%ld", print_order))) {
STORAGE_LOG(WARN, "fail to databuff print", K(ret), K(print_order));
} else if (OB_FALSE_IT(partial_desc.items_.at(partial_offset).to_string(buf, OB_MAX_TEXT_LENGTH))) {
} else if (OB_FAIL((sql::ObSQLUtils::generate_new_name_with_escape_character(
allocator,
buf,
escape_str,
true)))) {
STORAGE_LOG(WARN, "fail to generate new name with escape character", K(ret), K(buf));
} else {
PrintHelper::print_dump_line(order_buf, buf);
}
return ret;
}
int ObAdminDumpBackupDataExecutor::read_table_list_part_file_(const uint64_t scn_val, const int64_t part_no, ObBackupPartialTableListDesc &desc)
{
int ret = OB_SUCCESS;
ObBackupPath full_path;
share::SCN scn;
if (part_no <= 0) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(part_no));
} else if (OB_FAIL(scn.convert_for_inner_table_field(scn_val))) {
STORAGE_LOG(WARN, "fail to convert scn", K(ret), K(scn_val));
} else if (OB_FAIL(full_path.init(ObString(backup_path_)))) {
STORAGE_LOG(WARN, "fail to init backup path", K(ret), K_(backup_path));
} else if (OB_FAIL(full_path.join_table_list_part_file(scn, part_no))) {
STORAGE_LOG(WARN, "fail to join table list file", K(ret), K(scn_val), K(part_no));
} else if (OB_FAIL(ObAdminDumpBackupDataUtil::read_backup_info_file(full_path.get_obstr(), ObString(storage_info_), desc))) {
STORAGE_LOG(WARN, "fail to read locality info", K(ret), K(backup_path_), K(storage_info_));
}
return ret;
}
int ObAdminDumpBackupDataExecutor::dump_table_list_meta_info_(ObBackupTableListMetaInfoDesc &desc)
{
int ret = OB_SUCCESS;
PrintHelper::print_dump_title("table list");
PrintHelper::print_dump_line("count", desc.count_);
PrintHelper::print_dump_line("batch_size", desc.batch_size_);
PrintHelper::print_end_line();
return ret;
}
} // namespace tools
} // namespace oceanbase

14
tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.h
View File

@ -180,7 +180,19 @@ private:
int dump_backup_format_file_(const share::ObBackupFormatDesc &format_file);
int dump_check_exist_result_(const char *data_path, const char *storage_info_str, const bool is_exist);
int print_tablet_autoinc_seq_(const share::ObTabletAutoincSeq &autoinc_seq);
int handle_print_table_list_();
int get_max_table_list_scn(uint64_t &scn_val);
/* check if the last dir name of backup_path_ is 'table_list' or not
e.g. '/data_backup/backup_set_1_full/table_list' returns true */
int check_is_table_list_dir_(bool &is_table_list_dir);
int print_table_list_meta_info_(const uint64_t scn_val);
int read_table_list_meta_info_(const uint64_t scn_val, ObBackupTableListMetaInfoDesc &desc);
int dump_table_list_meta_info_(ObBackupTableListMetaInfoDesc &desc);
int print_table_list_items_(const uint64_t scn_val);
int print_table_list_item_(const ObBackupPartialTableListDesc &partial_desc,
const int64_t partial_offset, const int64_t print_order);
int print_table_set_is_exist_(const uint64_t scn_val);
int read_table_list_part_file_(const uint64_t scn_val, const int64_t part_no, ObBackupPartialTableListDesc &desc);
private:
int get_tenant_backup_set_infos_path_(const share::ObBackupSetDesc &backup_set_dir_name,
share::ObBackupPath &target_path);