From 72750c0340d3f284f72e4dc4c83ae21b927d5204 Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 17 Dec 2024 22:16:06 +0000 Subject: [PATCH] [CP] [FEAT] Adapt ob_admin command to assume role --- .../lib/restore/cos/ob_cos_wrapper_handle.cpp | 2 +- deps/oblib/src/lib/restore/ob_storage_info.cpp | 2 +- .../ob_admin_dump_backup_data_executor.cpp | 16 +++++++++++++++- .../io_bench/ob_admin_io_adapter_bench.cpp | 18 +++++++++++------- .../ob_admin_test_io_device_executor.cpp | 16 +++++++++++++++- tools/ob_admin/ob_admin_executor.cpp | 5 +++-- 6 files changed, 46 insertions(+), 13 deletions(-) diff --git a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.cpp b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.cpp index 1311efb3e..8ecd2ac86 100644 --- a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.cpp +++ b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper_handle.cpp @@ -159,7 +159,7 @@ int ObCosWrapperHandle::create_tmp_cos_handle( ret = OB_NOT_INIT; OB_LOG(WARN, "handle is not inited", K(ret)); } else if (OB_FAIL(qcloud_cos::ObCosWrapper::create_cos_handle(cos_mem, cos_account_, - check_md5, sts_token_.data_, &handle))) { + check_md5, sts_token_.get_data(), &handle))) { OB_LOG(WARN, "failed to create tmp cos handle", K(ret)); } else if (OB_ISNULL(handle)) { ret = OB_OBJECT_STORAGE_IO_ERROR; diff --git a/deps/oblib/src/lib/restore/ob_storage_info.cpp b/deps/oblib/src/lib/restore/ob_storage_info.cpp index aed1723e8..d61749166 100644 --- a/deps/oblib/src/lib/restore/ob_storage_info.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_info.cpp @@ -266,7 +266,7 @@ int ObObjectStorageInfo::register_cluster_version_mgr(ObClusterVersionBaseMgr *c LOG_WARN("cluster_version_mgr is null", K(ret)); } else { cluster_version_mgr_ = cluster_version_mgr; - LOG_INFO("register_cluster_version_mgr", K(ret), KP_(cluster_version_mgr)); + LOG_INFO("register cluster_version_mgr successfully", K(ret), KP_(cluster_version_mgr)); } return ret; } diff --git a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp index 8be7908e4..3e673adea 100644 --- a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp +++ b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp @@ -620,7 +620,7 @@ int ObAdminDumpBackupDataExecutor::parse_cmd_(int argc, char *argv[]) int ret = OB_SUCCESS; int opt = 0; int index = -1; - const char *opt_str = "h:d:s:o:l:qce:"; + const char *opt_str = "h:d:s:o:l:qce:i:"; struct option longopts[] = {{"help", 0, NULL, 'h'}, {"backup_path", 1, NULL, 'd'}, {"storage_info", 1, NULL, 's'}, @@ -630,6 +630,7 @@ int ObAdminDumpBackupDataExecutor::parse_cmd_(int argc, char *argv[]) {"check_exist", 0, NULL, 'c'}, {"length", 1, NULL, 'l'}, {"s3_url_encode_type", 0, NULL, 'e'}, + {"sts_credential", 0, NULL, 'i'}, {NULL, 0, NULL, 0}}; while (OB_SUCC(ret) && -1 != (opt = getopt_long(argc, argv, opt_str, longopts, &index))) { switch (opt) { @@ -677,6 +678,12 @@ int ObAdminDumpBackupDataExecutor::parse_cmd_(int argc, char *argv[]) } break; } + case 'i': { + if (OB_FAIL(set_sts_credential_key(optarg))) { + STORAGE_LOG(WARN, "failed to set sts credential", KR(ret)); + } + break; + } default: { print_usage_(); exit(1); @@ -986,6 +993,7 @@ int ObAdminDumpBackupDataExecutor::print_usage_() printf(HELP_FMT, "-l,--length", "data length"); printf(HELP_FMT, "-c,--check-exist", "check file is exist or not"); printf(HELP_FMT, "-e,--s3_url_encode_type", "set S3 protocol url encode type"); + printf(HELP_FMT, "-i, --sts_credential", "set STS credential"); printf("samples:\n"); printf(" dump meta: \n"); printf("\tob_admin dump_backup -dfile:///home/admin/backup_info \n"); @@ -999,6 +1007,12 @@ int ObAdminDumpBackupDataExecutor::print_usage_() printf("\tob_admin dump_backup -d'cos://home/admin/backup_info' " "-s'host=xxx.com&access_id=111&access_key=222®ion=333'\t" "-e'compliantRfc3986Encoding'"); + printf("\tob_admin dump_backup -d'cos://home/admin/backup_info' " + "-s'host=xxx.com&role_arn=xxx&appid=333'\n" + "-i'sts_url=xxx&sts_ak=aaa&sts_sk=bbb'"); + printf("\tob_admin dump_backup -d'cos://home/admin/backup_info' " + "-s'host=xxx.com&role_arn=xxx&external_id=xxx&appid=333'\n" + "-i'sts_url=xxx&sts_ak=aaa&sts_sk=bbb'"); return ret; } diff --git a/tools/ob_admin/io_bench/ob_admin_io_adapter_bench.cpp b/tools/ob_admin/io_bench/ob_admin_io_adapter_bench.cpp index 6c7c7b743..9eefa4c48 100644 --- a/tools/ob_admin/io_bench/ob_admin_io_adapter_bench.cpp +++ b/tools/ob_admin/io_bench/ob_admin_io_adapter_bench.cpp @@ -329,20 +329,24 @@ int ObAdminIOAdapterBenchmarkExecutor::print_usage_() printf(HELP_FMT, "-b, --clean-before-execution", "clean before execution"); printf(HELP_FMT, "-c, --clean-after-execution", "clean after execution"); printf(HELP_FMT, "-e,--s3_url_encode_type", "set S3 protocol url encode type"); + printf(HELP_FMT, "-i, --sts_credential", "set sts credential"); printf("samples:\n"); printf(" test nfs device: \n"); - printf("\tob_admin bench_io_adapter -dfile:///home/admin/backup_info \n"); + printf("\tob_admin io_adapter_benchmark -dfile:///home/admin/backup_info \n"); printf(" test object device: \n"); - printf("\tob_admin bench_io_adapter -d'oss://home/admin/backup_info' " + printf("\tob_admin io_adapter_benchmark -d'oss://home/admin/backup_info' " "-s'host=xxx.com&access_id=111&access_key=222'\n"); - printf("\tob_admin bench_io_adapter -d'cos://home/admin/backup_info' " + printf("\tob_admin io_adapter_benchmark -d'cos://home/admin/backup_info' " "-s'host=xxx.com&access_id=111&access_key=222&appid=333'\n"); - printf("\tob_admin bench_io_adapter -d's3://home/admin/backup_info' " + printf("\tob_admin io_adapter_benchmark -d's3://home/admin/backup_info' " "-s'host=xxx.com&access_id=111&access_key=222®ion=333'\t" "-e'compliantRfc3986Encoding'"); - printf("\tob_admin bench_io_adapter -d's3://home/admin/backup_info' " - "-s'host=xxx.com&role_arn=111®ion=333'\t" - "-i'sts_url=xxx&sts_ak=xxx&sts_sk=xxx'"); + printf("\tob_admin io_adapter_benchmark -d'cos://home/admin/backup_info' " + "-s'host=xxx.com&role_arn=xxx&appid=333'\n" + "-i'sts_url=xxx&sts_ak=aaa&sts_sk=bbb'"); + printf("\tob_admin io_adapter_benchmark -d'cos://home/admin/backup_info' " + "-s'host=xxx.com&role_arn=xxx&external_id=xxx&appid=333'\n" + "-i'sts_url=xxx&sts_ak=aaa&sts_sk=bbb'"); return ret; } diff --git a/tools/ob_admin/io_device/ob_admin_test_io_device_executor.cpp b/tools/ob_admin/io_device/ob_admin_test_io_device_executor.cpp index 6176a549b..03108b276 100644 --- a/tools/ob_admin/io_device/ob_admin_test_io_device_executor.cpp +++ b/tools/ob_admin/io_device/ob_admin_test_io_device_executor.cpp @@ -81,13 +81,14 @@ int ObAdminTestIODeviceExecutor::parse_cmd_(int argc, char *argv[]) int ret = OB_SUCCESS; int opt = 0; int index = -1; - const char *opt_str = "h:d:s:q:e:f:"; + const char *opt_str = "h:d:s:q:e:f:i:"; struct option longopts[] = {{"help", 0, NULL, 'h'}, {"backup_path", 1, NULL, 'd'}, {"storage_info", 1, NULL, 's'}, {"quiet", 0, NULL, 'q' }, {"s3_url_encode_type", 0, NULL, 'e'}, {"trigger_freq", 0, NULL, 'f'}, // used for internal testing only + {"sts_credential", 0, NULL, 'i'}, {NULL, 0, NULL, 0}}; while (OB_SUCC(ret) && -1 != (opt = getopt_long(argc, argv, opt_str, longopts, &index))) { switch (opt) { @@ -135,6 +136,12 @@ int ObAdminTestIODeviceExecutor::parse_cmd_(int argc, char *argv[]) } break; } + case 'i': { + if (OB_FAIL(set_sts_credential_key(optarg))) { + STORAGE_LOG(WARN, "failed to set sts credential", KR(ret)); + } + break; + } default: { print_usage_(); exit(1); @@ -790,6 +797,7 @@ int ObAdminTestIODeviceExecutor::print_usage_() printf("options:\n"); printf(HELP_FMT, "-d,--backup-file-path", "absolute backup file path with file prefix"); printf(HELP_FMT, "-s,--storage-info", "oss/cos should provide storage info"); + printf(HELP_FMT, "-i, --sts_credential", "set STS credential"); printf(HELP_FMT, "-e,--s3_url_encode_type", "set S3 protocol url encode type"); printf("samples:\n"); printf(" test nfs device: \n"); @@ -799,6 +807,12 @@ int ObAdminTestIODeviceExecutor::print_usage_() "-s'host=xxx.com&access_id=111&access_key=222'\n"); printf("\tob_admin test_io_device -d'cos://home/admin/backup_info' " "-s'host=xxx.com&access_id=111&access_key=222&appid=333'\n"); + printf("\tob_admin test_io_device -d'cos://home/admin/backup_info' " + "-s'host=xxx.com&role_arn=xxx&appid=333'\n" + "-i'sts_url=xxx&sts_ak=aaa&sts_sk=bbb'"); + printf("\tob_admin test_io_device -d'cos://home/admin/backup_info' " + "-s'host=xxx.com&role_arn=xxx&external_id=xxx&appid=333'\n" + "-i'sts_url=xxx&sts_ak=aaa&sts_sk=bbb'"); printf("\tob_admin test_io_device -d's3://home/admin/backup_info' " "-s'host=xxx.com&access_id=111&access_key=222®ion=333'\n" "-e'compliantRfc3986Encoding'"); diff --git a/tools/ob_admin/ob_admin_executor.cpp b/tools/ob_admin/ob_admin_executor.cpp index 1458d5ae3..292fbe834 100644 --- a/tools/ob_admin/ob_admin_executor.cpp +++ b/tools/ob_admin/ob_admin_executor.cpp @@ -47,7 +47,6 @@ ObAdminExecutor::ObAdminExecutor() mock_server_tenant_.set(&blocksstable::ObDecodeResourcePool::get_instance()); share::ObTenantEnv::set_tenant(&mock_server_tenant_); omt::ObTenantConfigMgr::get_instance().add_tenant_config(OB_SYS_TENANT_ID); - storage_env_.data_dir_ = data_dir_; storage_env_.sstable_dir_ = sstable_dir_; storage_env_.default_block_size_ = 2 * 1024 * 1024; @@ -219,6 +218,9 @@ int ObAdminExecutor::set_sts_credential_key(const char *sts_credential) } else { if (OB_FAIL(ObDeviceManager::get_instance().init_devices_env())) { STORAGE_LOG(WARN, "fail to init device env", KR(ret)); + } else if (OB_FAIL(ObObjectStorageInfo::register_cluster_version_mgr( + &ObClusterVersionBaseMgr::get_instance()))) { + STORAGE_LOG(WARN, "fail to register cluster version mgr", KR(ret)); } else { omt::ObTenantConfigGuard tenant_config(TENANT_CONF(OB_SYS_TENANT_ID)); if (OB_UNLIKELY(!tenant_config.is_valid())) { @@ -232,6 +234,5 @@ int ObAdminExecutor::set_sts_credential_key(const char *sts_credential) } return ret; } - } }